Fixed #109 : Mix python and concept. List comprehension

Fixed #110 : SheerkaDebugManager: add list_debug_settings
Fixed #111 : SheerkaDebugManager: Implement ListDebugLogger
Fixed #112 : SyaNodeParser: rewrite this parser
Fixed #113 : Sheerka.: Add enable_parser_caching to disable parsers caching
Fixed #114 : SyaNodeParser : Implement fast cache to resolve unrecognized tokens requests
Fixed #115 : BnfNodeParser : Implement fast cache to resolve unrecognized tokens requests
Fixed #116 : SequenceNodeParser : Implement fast cache to resolve unrecognized tokens requests
Fixed #117 : ResolveMultiplePluralAmbiguityEvaluator: Resolve Multiple plural ambiguity

sheerka_backup/admin.sb

@@ -13,12 +13,14 @@ def concept deactivate debug as set_debug(False) auto_eval True
 def concept debug on as set_debug(True) auto_eval True
 def concept debug off as set_debug(False) auto_eval True
 
-def concept activate debug on x as debug_var(x) auto_eval True
-def concept debug x as debug_var(x) auto_eval True
+def concept activate debug on x as set_debug_var(x) auto_eval True
+def concept activate debug on x id=y as set_debug_var(x, y) auto_eval True
+def concept debug x as set_debug_var(x) auto_eval True
 
-def concept debug var x as debug_var(variable=x) auto_eval True
-def concept debug variable x as debug_var(variable=x) auto_eval True
-def concept debug method x as debug_var(method=x) auto_eval True
+def concept debug var x as set_debug_var(x) auto_eval True
+def concept debug variable x as set_debug_var(variable=x) auto_eval True
+def concept debug method x as set_debug_var(method=x) auto_eval True
+def concept debug service x as set_debug_var(service=x) auto_eval True
 
 def concept deactivate debug on x as debug_var(x, enabled=False) where x auto_eval True
 

src/cache/FastCache.py

@@ -11,6 +11,7 @@ class FastCache:
         self.cache = {}
         self.lru = []
         self.default = default
+        self.calls = {}
 
     def __contains__(self, item):
         return self.has(item)
@@ -24,13 +25,16 @@ class FastCache:
 
         self.cache[key] = value
         self.lru.append(key)
+        self.calls[key] = 0
 
     def has(self, key):
         return key in self.cache
 
     def get(self, key):
         try:
-            return self.cache[key]
+            res = self.cache[key]
+            self.calls[key] += 1
+            return res
         except KeyError:
             if self.default:
                 value = self.default(key)

src/core/builtin_concepts.py

@@ -140,7 +140,10 @@ class ParserResultConcept(Concept):
 
     def __repr__(self):
         text = f"ParserResult(parser={self.parser}"
-        text += f", source='{self.source}')" if self.source else f", body='{self.value}')"
+        # text += f", source='{self.source}')" if self.source else f", body='{self.value}')"
+        from core.builtin_helpers import debug_nodes
+        value = debug_nodes(self.value) if isinstance(self.value, list) else self.value
+        text += f", source='{self.source}', '{value=}')"
         return text
 
     def __eq__(self, other):
@@ -252,6 +255,7 @@ class ListConcept(Concept):
 
 class FilteredConcept(Concept):
     ALL_ATTRIBUTES = ["filtered", "iterable", "predicate", "reason"]
+
     # To explain the reason why it's filtered, you can either
     #  provide the original list (iterable) and the predicate
     #  provide the reason (It may be a CONDITION_FAILED concept)

src/core/builtin_helpers.py

@@ -5,15 +5,11 @@ from cache.Cache import Cache
 from core.ast_helpers import ast_to_props
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF, concept_part_value
-from core.global_symbols import NotInit, NotFound, INIT_AST_PARSERS, DEFAULT_EVALUATORS
+from core.global_symbols import DEFAULT_EVALUATORS, INIT_AST_PARSERS, NotFound, NotInit
 from core.rule import Rule
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.tokenizer import Tokenizer, TokenKind
+from core.tokenizer import TokenKind, Tokenizer
 from core.utils import as_bag
-from parsers.BaseExpressionParser import compile_disjunctions, AndNode
-from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode, SourceCodeWithConceptNode, \
-    RuleNode, LexerNode
-from parsers.BaseParser import ParsingError
 
 PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
 EVAL_ONLY_STEPS = [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION, BuiltinConcepts.AFTER_EVALUATION]
@@ -226,9 +222,9 @@ def resolve_ambiguity(context, concepts):
             remaining_concepts.extend(by_complexity[complexity])
         else:
             for c in by_complexity[complexity]:
+                from core.sheerka.services.SheerkaEvaluateConcept import EvaluationHints
                 evaluated = context.sheerka.evaluate_concept(context, c,
-                                                             eval_body=False,
-                                                             validation_only=True,
+                                                             hints=EvaluationHints(eval_body=False, expression_only=True),
                                                              metadata=[ConceptParts.PRE, ConceptParts.WHERE])
                 if context.sheerka.is_success(evaluated) or evaluated.key == c.key:
                     remaining_concepts.append(c)
@@ -255,6 +251,9 @@ def get_condition_complexity(context, condition):
 
     # # count the number of conjunctions
     from parsers.LogicalOperatorParser import LogicalOperatorParser
+    from parsers.BaseExpressionParser import compile_disjunctions
+    from parsers.BaseExpressionParser import AndNode
+
     parser = LogicalOperatorParser()
     res = parser.parse(context, ParserInput(condition))
     if not res.status:
@@ -314,6 +313,9 @@ def only_parsers_results(context, return_values):
     :return:
     """
 
+    from parsers.BaseNodeParser import UnrecognizedTokensNode
+    from parsers.BaseParser import ParsingError
+
     if not isinstance(return_values, list):
         return return_values
 
@@ -479,6 +481,7 @@ def get_lexer_nodes(return_values, start, tokens):
     :return: list of list (list of concept node sequence)
     """
     from evaluators.BaseEvaluator import BaseEvaluator
+    from parsers.BaseNodeParser import ConceptNode, LexerNode, RuleNode, SourceCodeNode
 
     lexer_nodes = []
     for ret_val in return_values:
@@ -546,6 +549,7 @@ def get_lexer_nodes_using_positions(return_values, positions):
     """
 
     from evaluators.BaseEvaluator import BaseEvaluator
+    from parsers.BaseNodeParser import ConceptNode, LexerNode, RuleNode, SourceCodeNode
 
     lexer_nodes = []
     for ret_val, position in zip(return_values, positions):
@@ -615,8 +619,8 @@ def ensure_evaluated(context, concept, eval_body=True, metadata=None):
     :param metadata:
     :return:
     """
+    from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept, EvaluationHints
     if concept.get_hints().is_evaluated:
-        from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
         return SheerkaEvaluateConcept.apply_ret(concept,
                                                 eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED))
 
@@ -628,7 +632,10 @@ def ensure_evaluated(context, concept, eval_body=True, metadata=None):
                     (var_name not in concept.values() or concept.get_value(var_name) == NotInit):
                 return concept
 
-    evaluated = context.sheerka.evaluate_concept(context, concept, eval_body=eval_body, metadata=metadata)
+    evaluated = context.sheerka.evaluate_concept(context,
+                                                 concept,
+                                                 hints=EvaluationHints(eval_body=eval_body),
+                                                 metadata=metadata)
     return evaluated
 
 
@@ -663,6 +670,9 @@ def update_compiled(context, concept, errors, parsers=None):
     :param parsers: to customize the parsers to use
     :return:
     """
+
+    from parsers.BaseNodeParser import ConceptNode, SourceCodeNode, SourceCodeWithConceptNode, UnrecognizedTokensNode
+
     sheerka = context.sheerka
     parsers = parsers or PARSERS
 
@@ -676,6 +686,15 @@ def update_compiled(context, concept, errors, parsers=None):
             if isinstance(v, Concept):
                 _validate_concept(v)
 
+            elif isinstance(v, ConceptNode):
+                _validate_concept(v.concept)
+                c.get_compiled()[k] = v.concept
+
+            elif isinstance(v, SourceCodeNode):
+                if not v.return_value:
+                    raise NotImplementedError("SourceCodeNode")
+                c.get_compiled()[k] = [v.return_value]
+
             elif isinstance(v, SourceCodeWithConceptNode):
                 if v.return_value:
                     res = v.return_value
@@ -939,7 +958,8 @@ def get_possible_variables_from_concept(context, concept):
         return set()
 
     concept_name = [t.str_value for t in Tokenizer(concept.name, yield_eof=False)]
-    names = [v_value or v_name for v_name, v_value in concept.get_metadata().variables if v_name in concept_name]
+    names = [v_value.strip() or v_name for v_name, v_value in concept.get_metadata().variables if
+             v_name in concept_name]
     possible_vars = filter(lambda x: context.sheerka.is_not_a_concept_name(x), names)
     to_keep = set()
     for var in possible_vars:
@@ -961,6 +981,36 @@ def is_only_successful(sheerka, return_value):
            sheerka.isinstance(return_value.body, BuiltinConcepts.ONLY_SUCCESSFUL)
 
 
+def debug_nodes(nodes):
+    from parsers.BaseNodeParser import UnrecognizedTokensNode
+
+    res = []
+    for node in nodes:
+        if isinstance(node, UnrecognizedTokensNode):
+            res.append(node.source)
+        elif hasattr(node, "get_concept"):
+            concept = node.get_concept()
+            res.append(concept)
+        else:
+            res.append(node)
+
+    return res
+
+
+def get_new_variables_definitions(concept):
+    """
+    Return a new set of variable definition, where the default value are initialized with what was compiled
+    """
+    new_variables = []
+    for var_name, var_default_value in concept.get_metadata().variables:
+        if var_name in concept.get_metadata().parameters and hasattr(concept.get_compiled()[var_name], "source"):
+            new_variables.append((var_name, concept.get_compiled()[var_name].source))
+        else:
+            new_variables.append((var_name, var_default_value))
+
+    return new_variables
+
+
 class CreateObjectIdentifiers:
     """
     Class that creates unique identifiers for Concept or Rule objects

src/core/concept.py

@@ -44,7 +44,7 @@ def concept_part_value(c):
 class ConceptHints:
     is_evaluated: bool = False  # True is the concept is evaluated by sheerka.eval_concept()
     need_validation: bool = False  # True if the properties of the concept need to be validated
-    recognized_by: str = None  # recognized by its name, by its id or its key
+    recognized_by: str = None  # RECOGNIZED_BY_ID, RECOGNIZED_BY_NAME, RECOGNIZED_BY_KEY (from Sheerka.py)
     use_copy: bool = False  # Do not modify, make a copy first
     is_instance: bool = True  # False if we think we recognize the definition of a concept, not its usage
 

src/core/sheerka/ExecutionContext.py

@@ -6,7 +6,7 @@ from core.builtin_concepts import BuiltinConcepts, ParserResultConcept
 from core.concept import Concept, get_concept_attrs
 from core.global_symbols import EVENT_CONTEXT_DISPOSED, NO_MATCH
 from core.sheerka.services.SheerkaMemory import SheerkaMemory
-from core.utils import CONSOLE_COLORS_MAP as CCM, CONSOLE_COLUMNS
+from core.utils import CONSOLE_COLUMNS
 from sdp.sheerkaDataProvider import Event
 
 pp = pprint.PrettyPrinter(indent=2, width=CONSOLE_COLUMNS)
@@ -334,37 +334,8 @@ class ExecutionContext:
             to_str = self.return_value_to_str(r)
             self._logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
 
-    def get_debugger(self, who, method_name, new_debug_id=True):
-        return self.sheerka.get_debugger(self, who, method_name, new_debug_id)
-
-    # TODO: TO REMOVE
-    def debug(self, who, method_name, variable_name, text, is_error=False):
-        activated = self.sheerka.debug_activated_for(who)
-        if activated:
-            str_text = pp.pformat(text)
-            color = 'red' if is_error else 'green'
-            if "\n" not in str(str_text):
-                self.sheerka.debug(
-                    f"[{self._id:3}] {CCM[color]}{who}.{method_name}.{variable_name}: {CCM['reset']}{str_text}")
-            else:
-                self.sheerka.debug(f"[{self._id:3}] {CCM[color]}{who}.{method_name}.{variable_name}: {CCM['reset']}")
-                self.sheerka.debug(str_text)
-
-    # TODO: TO REMOVE
-    def debug_entering(self, who, method_name, **kwargs):
-        if self.sheerka.debug_activated_for(who):
-            str_text = pp.pformat(kwargs)
-            if "\n" not in str(str_text):
-                self.sheerka.debug(
-                    f"[{self._id:3}] {CCM['blue']}Entering {who}.{method_name} with {CCM['reset']}{str_text}")
-            else:
-                self.sheerka.debug(f"[{self._id:3}] {CCM['blue']}Entering {who}.{method_name}:{CCM['reset']}")
-                self.sheerka.debug(f"[{self._id:3}] {str_text}")
-
-    # TODO: TO REMOVE
-    def debug_log(self, who, text):
-        if self.sheerka.debug_activated_for(who):
-            self.sheerka.debug(f"[{self._id:3}] {CCM['blue']}{text}{CCM['reset']}")
+    def get_debugger(self, who, method_name, new_debug_id=True, forced_debug_id=None):
+        return self.sheerka.get_debugger(self, who, method_name, new_debug_id, forced_debug_id)
 
     def get_parent(self):
         return self._parent

src/core/sheerka/Sheerka.py

@@ -8,12 +8,12 @@ import core.utils
 from cache.Cache import Cache
 from cache.IncCache import IncCache
 from core.builtin_concepts import ErrorConcept, ReturnValueConcept, UnknownConcept
-from core.builtin_concepts_ids import BuiltinErrors, BuiltinConcepts
+from core.builtin_concepts_ids import BuiltinConcepts, BuiltinErrors
 from core.concept import Concept, ConceptParts, get_concept_attrs
-from core.global_symbols import EVENT_USER_INPUT_EVALUATED, NotInit, NotFound, ErrorObj, EVENT_ONTOLOGY_CREATED
+from core.global_symbols import EVENT_ONTOLOGY_CREATED, EVENT_USER_INPUT_EVALUATED, ErrorObj, NotFound, NotInit
 from core.profiling import profile
 from core.sheerka.ExecutionContext import ExecutionContext
-from core.sheerka.SheerkaOntologyManager import SheerkaOntologyManager, OntologyAlreadyExists
+from core.sheerka.SheerkaOntologyManager import OntologyAlreadyExists, SheerkaOntologyManager
 from core.sheerka_logger import console_handler
 from core.tokenizer import Token, TokenKind
 from printer.SheerkaPrinter import SheerkaPrinter
@@ -119,6 +119,7 @@ class Sheerka(Concept):
         self.enable_process_return_values = True
         self.enable_process_rules = True
         self.enable_commands_backup = True
+        self.enable_parser_caching = True
 
         self.methods_with_context = {"test_using_context"}  # only the names, the method is defined in sheerka_methods
         self.pipe_functions = set()
@@ -193,6 +194,7 @@ class Sheerka(Concept):
                                                        self.enable_process_return_values)
         self.enable_process_rules = kwargs.get("enable_process_rules", self.enable_process_rules)
         self.enable_commands_backup = kwargs.get("enable_commands_backup", self.enable_commands_backup)
+        self.enable_parser_caching = kwargs.get("enable_parser_caching", self.enable_parser_caching)
 
         try:
             self.during_initialisation = True
@@ -791,7 +793,7 @@ class Sheerka(Concept):
         Browse obj, looking for error
         :param context:
         :param obj:
-        :param kwargs: if defined, specialize the error
+        :param kwargs: if defined, specialize the error (example __type="PythonEvalError")
         :return:
         """
 
@@ -834,6 +836,19 @@ class Sheerka(Concept):
         errors = self.get_errors(context, obj, **kwargs)
         return len(errors) > 0
 
+    def get_error_cause(self, obj):
+        if self.isinstance(obj, BuiltinConcepts.NOT_FOR_ME):
+            res = obj.reason
+        elif self.isinstance(obj, BuiltinConcepts.ERROR):
+            res = obj.body
+        else:
+            res = None
+
+        if isinstance(res, list) and len(res) == 1:
+            return res[0]
+        else:
+            return res
+
     def get_evaluator_name(self, name):
         if self.evaluators_prefix is None:
             base_evaluator_class = core.utils.get_class("evaluators.BaseEvaluator.BaseEvaluator")

src/core/sheerka/services/SheerkaAdmin.py

@@ -3,7 +3,7 @@ import time
 from os import path
 
 from core.builtin_concepts_ids import BuiltinConcepts, BuiltinContainers
-from core.builtin_helpers import ensure_concept_or_rule, ensure_concept
+from core.builtin_helpers import ensure_concept_or_rule
 from core.concept import Concept
 from core.global_symbols import SHEERKA_BACKUP_FOLDER
 from core.sheerka.services.SheerkaExecute import SheerkaExecute
@@ -40,6 +40,7 @@ class SheerkaAdmin(BaseService):
         self.sheerka.bind_service_method(self.NAME, self.in_memory, False)
         self.sheerka.bind_service_method(self.NAME, self.admin_history, False, as_name="history")
         self.sheerka.bind_service_method(self.NAME, self.sdp, False)
+        self.sheerka.bind_service_method(self.NAME, self.set_sheerka, True)
 
     def caches_names(self):
         """
@@ -284,3 +285,13 @@ class SheerkaAdmin(BaseService):
     def admin_history(self, depth=10, start=0):
         history = self.sheerka.services[SheerkaHistoryManager.NAME].history(depth, start)
         return self.sheerka.new(BuiltinConcepts.TO_LIST, body=history)
+
+    def set_sheerka(self, context, key, value, service=None):
+        """
+        @param context:
+        @param key:
+        @param value:
+        @param service:
+        @return:
+        """
+        return self.sheerka.record_var(context, service or self.sheerka.name, key, value)

src/core/sheerka/services/SheerkaDebugManager.py

@@ -5,7 +5,7 @@ from dataclasses import dataclass
 from core.builtin_concepts import BuiltinConcepts
 from core.builtin_helpers import evaluate_expression
 from core.concept import Concept
-from core.global_symbols import NotInit, NotFound
+from core.global_symbols import NotFound, NotInit
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.sheerka.services.sheerka_service import BaseService
 from core.utils import CONSOLE_COLORS_MAP as CCM, CONSOLE_COLUMNS, PRIMITIVES_TYPES
@@ -100,13 +100,22 @@ class BaseDebugLogger:
             BaseDebugLogger.ids[hint] = 0
             return 0
 
-    def __init__(self, debug_manager, context, who, method_name, debug_id):
-        pass
+    def __init__(self, keep_track, debug_manager, context, service_name, method_name, debug_id):
+        self.debug_manager = debug_manager
+        self.service_name = service_name
+        self.method_name = method_name
+        self.context = context
+        self.debug_id = debug_id
+
+        self.keep_track = keep_track  # Does debug_manager need to keep track of this logger ?
 
     def debug_entering(self, **kwargs):
         pass
 
-    def debug_log(self, text, is_error=False):
+    def debug_leaving(self, **kwargs):
+        pass
+
+    def debug_log(self, text, is_error=False, args=None):
         pass
 
     def debug_var(self, name, value, is_error=False, hint=None):
@@ -122,7 +131,36 @@ class BaseDebugLogger:
         pass
 
     def get_enabled_vars(self):
-        pass
+        """
+        Returns the list of all enabled variables for this console debugger
+        :return:
+        """
+        return self.debug_manager.get_enabled_items("vars",
+                                                    self.context,
+                                                    self.service_name,
+                                                    self.method_name,
+                                                    self.debug_id)
+
+    def should_i_log_var(self, name, is_error=False):
+        return is_error or self.debug_manager.compute_debug_var(self.context,
+                                                                self.service_name,
+                                                                self.method_name,
+                                                                name,
+                                                                self.debug_id)
+
+    def should_i_log_rule(self, rule_id, is_error=False):
+        return is_error or self.debug_manager.compute_debug_rule(self.context,
+                                                                 self.service_name,
+                                                                 self.method_name,
+                                                                 rule_id,
+                                                                 self.debug_id)
+
+    def should_i_log_concept(self, concept_id, is_error=False):
+        return is_error or self.debug_manager.compute_debug_concept(self.context,
+                                                                    self.service_name,
+                                                                    self.method_name,
+                                                                    concept_id,
+                                                                    self.debug_id)
 
 
 class NullDebugLogger(BaseDebugLogger):
@@ -139,12 +177,7 @@ class NullDebugLogger(BaseDebugLogger):
 class ConsoleDebugLogger(BaseDebugLogger):
 
     def __init__(self, debug_manager, context, service_name, method_name, debug_id):
-        BaseDebugLogger.__init__(self, debug_manager, context, service_name, method_name, debug_id)
-        self.debug_manager = debug_manager
-        self.service_name = service_name
-        self.method_name = method_name
-        self.context = context
-        self.debug_id = debug_id
+        BaseDebugLogger.__init__(self, False, debug_manager, context, service_name, method_name, debug_id)
         self.is_highlighted = ""
 
     def is_enabled(self):
@@ -154,17 +187,6 @@ class ConsoleDebugLogger(BaseDebugLogger):
         """
         return True
 
-    def get_enabled_vars(self):
-        """
-        Returns the list of all enabled variables for this console debugger
-        :return:
-        """
-        return self.debug_manager.get_enabled_items("vars",
-                                                    self.context,
-                                                    self.service_name,
-                                                    self.method_name,
-                                                    self.debug_id)
-
     def debug_entering(self, **kwargs):
         """
         Log that we start debugging a method (for a specified service and context)
@@ -181,11 +203,31 @@ class ConsoleDebugLogger(BaseDebugLogger):
             self.debug_manager.debug(self.prefix() + str_text)
             self.debug_manager.debug(self.prefix() + str_vars)
 
-    def debug_log(self, text, is_error=False):
+    def debug_leaving(self, **kwargs):
+        """
+        Log that we start debugging a method (for a specified service and context)
+        :param kwargs:
+        :return:
+        """
+        super().debug_leaving(**kwargs)
+
+        if not kwargs:
+            return
+
+        str_text = f"{CCM['blue']}Leaving {self.service_name}.{self.method_name} with {CCM['reset']}"
+        str_vars = pp.pformat(kwargs)
+        if "\n" not in str(str_vars):
+            self.debug_manager.debug(self.prefix() + str_text + str_vars)
+        else:
+            self.debug_manager.debug(self.prefix() + str_text)
+            self.debug_manager.debug(self.prefix() + str_vars)
+
+    def debug_log(self, text, is_error=False, args=None):
         """
         Prints a debug information (not related to a specific variable, concept or rule)
         :param text:
         :param is_error:
+        :param args:
         :return:
         """
         color = 'red' if is_error else 'blue'
@@ -268,8 +310,153 @@ class ConsoleDebugLogger(BaseDebugLogger):
         return f"[{self.context.id:2}][{self.debug_id:2}] {self.is_highlighted}"
 
 
+class ListDebugLogger(BaseDebugLogger):
+    ITEM_TYPE_ENTERING = "entering"
+    ITEM_TYPE_LEAVING = "leaving"
+    ITEM_TYPE_LOG = "log"
+    ITEM_TYPE_VAR = "var"
+    ITEM_TYPE_RULE = "rule"
+    ITEM_TYPE_CONCEPT = "concept"
+
+    class DebugItem:
+        def __init__(self, item_type, text, is_error=False, args=None):
+            self.type = item_type
+            self.text = text
+            self.is_error = is_error
+            self.args = args
+
+        def __repr__(self):
+            return self.text
+
+    def __init__(self, debug_manager, context, service_name, method_name, debug_id):
+        BaseDebugLogger.__init__(self, True, debug_manager, context, service_name, method_name, debug_id)
+        self.items = []
+
+    def is_enabled(self):
+        """
+        True if the debug is activated for the current service, method and context
+        :return:
+        """
+        return True
+
+    def debug_entering(self, **kwargs):
+        """
+        Log that we start debugging a method (for a specified service and context)
+        :param kwargs:
+        :return:
+        """
+        text = f"Entering {self.service_name}.{self.method_name}"
+        self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_ENTERING, text, False, kwargs))
+
+    def debug_leaving(self, **kwargs):
+        """
+        Log that we start debugging a method (for a specified service and context)
+        :param kwargs:
+        :return:
+        """
+        super().debug_leaving(**kwargs)
+
+        if not kwargs:
+            return
+
+        text = f"Leaving {self.service_name}.{self.method_name}"
+        self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_LEAVING, text, False, kwargs))
+
+    def debug_log(self, text, is_error=False, args=None):
+        """
+        Prints a debug information (not related to a specific variable, concept or rule)
+        :param text:
+        :param is_error:
+        :param args:
+        :return:
+        """
+        self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_LOG, text, is_error, args))
+
+    def debug_var(self, name, value, is_error=False, hint=None):
+        """
+        Prints the value of a variable
+        :param name:
+        :param value:
+        :param is_error:
+        :param hint:
+        :return:
+        """
+        if not self.should_i_log_var(name, is_error):
+            return
+
+        text = name + hint if hint else name
+        self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_VAR, text, is_error, value))
+
+    def debug_rule(self, rule, results):
+        """
+        Prints debug information related to a specific rule id
+        :param rule:
+        :param results:
+        :return:
+        """
+        if not self.should_i_log_rule(rule.id):
+            return
+
+        self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_RULE, rule.id, False, results))
+
+    def debug_concept(self, concept, text=None, **kwargs):
+        """
+        Prints debug information related to a specific concept
+        :param concept:
+        :param text:
+        :param kwargs:
+        :return:
+        """
+        if not self.debug_manager.compute_debug_concept(self.context,
+                                                        self.service_name,
+                                                        self.method_name,
+                                                        concept.id,
+                                                        self.debug_id):
+            return
+
+        if not self.should_i_log_concept(concept.id):
+            return
+
+        concept_id = f"{concept.id}{text}" if text else f"{concept.id}"
+        self.items.append(ListDebugLogger.DebugItem(ListDebugLogger.ITEM_TYPE_CONCEPT, concept_id, False, kwargs))
+
+
+class TeeDebugLogger(BaseDebugLogger):
+    def __init__(self, debug_manager, context, service_name, method_name, debug_id, loggers):
+        BaseDebugLogger.__init__(self, False, debug_manager, context, service_name, method_name, debug_id)
+        self.loggers = loggers
+
+    def is_enabled(self):
+        """
+        True if the debug is activated for the current service, method and context
+        :return:
+        """
+        return True
+
+    def debug_entering(self, **kwargs):
+        for logger in self.loggers:
+            logger.debug_entering(kwargs)
+
+    def debug_log(self, text, is_error=False, args=None):
+        for logger in self.loggers:
+            logger.debug_log(text, is_error, args=None)
+
+    def debug_var(self, name, value, is_error=False, hint=None):
+        for logger in self.loggers:
+            logger.debug_var(name, value, is_error, hint)
+
+    def debug_rule(self, rule, results):
+        for logger in self.loggers:
+            logger.debug_rule(rule, results)
+
+    def debug_concept(self, concept, text=None, **kwargs):
+        for logger in self.loggers:
+            logger.debug_concept(concept, text, **kwargs)
+
+
 @dataclass
 class DebugItem:
+    debug_type: str
     item: str
     service_name: str
     method_name: str
@@ -280,6 +467,16 @@ class DebugItem:
 
     enabled: bool
 
+    def __repr__(self):
+        text = f"type={self.debug_type}"
+        text += f", setting={self.service_name or '*'}.{self.method_name or '*'}.{self.item or '*'}"
+        text += f", context_id={self.context_id}"
+        text += f", debug_id={self.debug_id}"
+        text += f", context_children={self.context_children}"
+        text += f", debug_children={self.debug_children}"
+        text += f" (enabled={self.enabled})"
+        return f"DebugItem({text})"
+
 
 class SheerkaDebugManager(BaseService):
     NAME = "Debug"
@@ -302,7 +499,10 @@ class SheerkaDebugManager(BaseService):
         self.registered_vars = []  # list of all variables that can be debugged
         self.registered_rules = []  # list of all rules that can be debugged
         self.registered_concepts = []  # list of all concept that can be debugged
+        self.debug_logger_definition = ConsoleDebugLogger  # logger to use
+        self.instantiated_loggers = {}
 
+        # variable that needs to be reset on restart
         self.state_vars = [
             "activated",
             "explicit",  # to remove ?
@@ -323,25 +523,33 @@ class SheerkaDebugManager(BaseService):
         self.sheerka.bind_service_method(self.NAME, self.set_debug, True)
         self.sheerka.bind_service_method(self.NAME, self.inspect, False)
         self.sheerka.bind_service_method(self.NAME, self.get_value, False)
-        self.sheerka.bind_service_method(self.NAME, self.get_debugger, False)
         self.sheerka.bind_service_method(self.NAME, self.reset_debug, False)
         self.sheerka.bind_service_method(self.NAME, self.set_debug_var, True)
         self.sheerka.bind_service_method(self.NAME, self.set_debug_rule, True)
         self.sheerka.bind_service_method(self.NAME, self.set_debug_concept, True)
-        self.sheerka.bind_service_method(self.NAME, self.list_debug_vars, True)
-        self.sheerka.bind_service_method(self.NAME, self.list_debug_rules, True)
-        self.sheerka.bind_service_method(self.NAME, self.list_debug_concepts, True)
+        self.sheerka.bind_service_method(self.NAME, self.list_debug_vars, False)
+        self.sheerka.bind_service_method(self.NAME, self.list_debug_rules, False)
+        self.sheerka.bind_service_method(self.NAME, self.list_debug_concepts, False)
+        self.sheerka.bind_service_method(self.NAME, self.list_debug_settings, False)
+
         self.sheerka.bind_service_method(self.NAME, self.register_debug_vars, True, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.register_debug_rules, True, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.register_debug_concepts, True, visible=False)
 
+        self.sheerka.bind_service_method(self.NAME, self.get_debugger, False, visible=False)
+        self.sheerka.bind_service_method(self.NAME, self.get_debugger_logs, False, visible=False)
+        self.sheerka.bind_service_method(self.NAME, self.set_debug_logger_definition, True, visible=False)
+
         # self.sheerka.bind_service_method(self.NAME,self.get_debug_settings, False, as_name="debug_settings")
         # register what can be registered
         from parsers.BnfNodeParser import BnfNodeParser
         from evaluators.DefConceptEvaluator import DefConceptEvaluator
         from evaluators.PythonEvaluator import PythonEvaluator
         from parsers.SyaNodeParser import SyaNodeParser
+        from parsers.SequenceNodeParser import SequenceNodeParser
         self.register_debug_vars(BnfNodeParser.NAME, "parse", "result")
+        self.register_debug_vars(BnfNodeParser.NAME, "parse", "stats")
+        self.register_debug_vars(SequenceNodeParser.NAME, "parse", "stats")
         self.register_debug_concepts(BnfNodeParser.NAME, "parse", "*")
         self.register_debug_vars(DefConceptEvaluator.NAME, "matches", "*")
         self.register_debug_vars(DefConceptEvaluator.NAME, "eval", "*")
@@ -351,7 +559,8 @@ class SheerkaDebugManager(BaseService):
         self.register_debug_vars(PythonEvaluator.NAME, "eval", "ret")
         self.register_debug_vars("Exceptions", PythonEvaluator.NAME + "-eval", "exception")
         self.register_debug_vars("Exceptions", PythonEvaluator.NAME + "-eval", "trace")
-        self.register_debug_vars(SyaNodeParser.NAME, "parse", "*")
+        self.register_debug_vars(SyaNodeParser.NAME, "parse", "#[number]")
+        self.register_debug_vars(SyaNodeParser.NAME, "parse", "stats")
         self.register_debug_vars(MultipleSuccessEvaluator.NAME, "matches", "return_values")
 
     def initialize_deferred(self, context, is_first_time):
@@ -498,14 +707,30 @@ class SheerkaDebugManager(BaseService):
         self.sheerka.record_var(context, self.NAME, "activated", self.activated)
         return self.sheerka.ret(SheerkaDebugManager.NAME, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
 
+    def set_debug_logger_definition(self, logger_definition):
+        """
+        Logger definition to use. By default it's the ConsoleDebugLogger
+        logger_definition can be a list of debug loggers
+        :param logger_definition:
+        :return:
+        """
+        self.debug_logger_definition = logger_definition
+
     def debug(self, *args, **kwargs):
         print(*args, **kwargs)
 
-    def get_debugger(self, context, who, method_name, new_debug_id=True):
+    def get_debugger(self, context, who, method_name, new_debug_id=True, forced_debug_id=None):
         if self.compute_debug(context, who, method_name):
-            debug_id = ConsoleDebugLogger.next_id(context.event.get_digest() + str(context.id)) if new_debug_id \
-                else ConsoleDebugLogger.current_id(context.event.get_digest() + str(context.id))
-            return ConsoleDebugLogger(self, context, who, method_name, debug_id)
+            hint = context.event.get_digest() + str(context.id)
+            if forced_debug_id is not None:
+                debug_id = forced_debug_id
+                BaseDebugLogger.ids[hint] = debug_id
+            elif new_debug_id:
+                debug_id = BaseDebugLogger.next_id(hint)
+            else:
+                debug_id = BaseDebugLogger.current_id(hint)
+
+            return self.get_new_debug_logger_instance(context, who, method_name, debug_id)
 
         return NullDebugLogger()
 
@@ -534,7 +759,8 @@ class SheerkaDebugManager(BaseService):
                 setting.enabled = enabled
                 break
         else:
-            items_container.append(DebugItem(item,
+            items_container.append(DebugItem(item_type,
+                                             item,
                                              service,
                                              method,
                                              context_id,
@@ -919,3 +1145,35 @@ class SheerkaDebugManager(BaseService):
             del res["self"]
 
         return res
+
+    def list_debug_settings(self):
+        settings = self.debug_vars_settings + self.debug_concepts_settings + self.debug_rules_settings
+        return self.sheerka.new(BuiltinConcepts.TO_LIST, body=settings)
+
+    def get_new_debug_logger_instance(self, context, who, method_name, debug_id):
+        if hasattr(self.debug_logger_definition, "__iter__"):
+            loggers = []
+            for logger_type in self.debug_logger_definition:
+                logger = logger_type(self, context, who, method_name, debug_id)
+                if logger.keep_track:
+                    key = (who, method_name, context.id, debug_id)
+                    self.instantiated_loggers[key] = logger
+            return TeeDebugLogger(self, context, who, method_name, debug_id, loggers)
+
+        logger = self.debug_logger_definition(self, context, who, method_name, debug_id)
+        if logger.keep_track:
+            key = (who, method_name, context.id, debug_id)
+            self.instantiated_loggers[key] = logger
+        return logger
+
+    def get_debugger_logs(self):
+        res = {}
+        for k, v in [(k, v) for k, v in self.instantiated_loggers.items() if isinstance(v, ListDebugLogger)]:
+            key = list(k)
+            if v.items and v.items[0].type == ListDebugLogger.ITEM_TYPE_ENTERING:
+                if "source" in v.items[0].args:
+                    key.append(v.items[0].args["source"])
+
+            res[tuple(key)] = v.items
+
+        return res

src/core/sheerka/services/SheerkaEvaluateConcept.py

@@ -1,15 +1,15 @@
 from dataclasses import dataclass
 
 from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import expect_one, only_successful, ensure_concept, is_only_successful, ensure_bnf
-from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, AllConceptParts, \
+from core.builtin_helpers import ensure_bnf, ensure_concept, expect_one, is_only_successful, only_successful
+from core.concept import AllConceptParts, Concept, ConceptParts, DoNotResolve, InfiniteRecursionResolved, \
     concept_part_value
-from core.global_symbols import NotInit, CURRENT_OBJ, INIT_AST_PARSERS, INIT_AST_QUESTION_PARSERS
+from core.global_symbols import CURRENT_OBJ, INIT_AST_PARSERS, INIT_AST_QUESTION_PARSERS, NotInit
 from core.rule import Rule
 from core.sheerka.services.SheerkaEvaluateRules import SheerkaEvaluateRules
 from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute
 from core.sheerka.services.SheerkaRuleManager import PythonConditionExprVisitor
-from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError, ChickenAndEggException
+from core.sheerka.services.sheerka_service import BaseService, ChickenAndEggException, FailedToCompileError
 from core.tokenizer import Tokenizer
 from core.utils import unstr_concept
 from parsers.BaseExpressionParser import TrueifyVisitor
@@ -38,6 +38,13 @@ class WhereClauseDef:
     conditions: list  # compiled trueified
 
 
+@dataclass
+class EvaluationHints:
+    eval_body: bool = None  # true if the body must be evaluated
+    eval_question: bool = None  # true if the eval_question must be set
+    expression_only: bool = None  # True if function/methods to forbid functions with side effect
+
+
 class SheerkaEvaluateConcept(BaseService):
     NAME = "EvaluateConcept"
 
@@ -48,7 +55,7 @@ class SheerkaEvaluateConcept(BaseService):
     def initialize(self):
         self.sheerka.bind_service_method(self.NAME, self.evaluate_concept, True)
         self.sheerka.bind_service_method(self.NAME, self.call_concept, True)
-        self.sheerka.bind_service_method(self.NAME, self.call_concept, False, as_name="evaluate_question")
+        self.sheerka.bind_service_method(self.NAME, self.evaluate_question, False)
         self.sheerka.bind_service_method(self.NAME, self.set_auto_eval, True)
 
     def initialize_deferred(self, context, is_first_time):
@@ -532,6 +539,8 @@ class SheerkaEvaluateConcept(BaseService):
                         value = current_concept.get_value(var[0])
                         if value != NotInit:
                             sub_context.add_to_short_term_memory(var[0], current_concept.get_value(var[0]))
+
+                # KSI 2021-08-10 It seems that a copy is not needed here, as it's the first thing done ine execute()
                 use_copy = to_resolve.copy() if isinstance(to_resolve, list) else to_resolve
                 r = self.sheerka.execute(sub_context, use_copy, CONCEPT_EVALUATION_STEPS)
 
@@ -601,22 +610,30 @@ class SheerkaEvaluateConcept(BaseService):
 
         return res
 
-    def evaluate_concept(self, context, concept: Concept, eval_body=False, validation_only=False, metadata=None):
+    def evaluate_concept(self, context, concept: Concept, hints: EvaluationHints = None, metadata=None):
         """
         Evaluation a concept
         ie : resolve its body
         :param context:
         :param concept:
-        :param eval_body:
-        :param validation_only: When set, the body is never evaluated, whatever eval_body or EVAL_BODY_REQUESTED
+        :param hints:
         :param metadata: list of metadata to evaluate ('pre', 'post'...)
         :return: value of the evaluation or error
         """
 
         failed_to_evaluate_body = False
+        hints = hints or EvaluationHints()
 
         if concept.get_hints().is_evaluated:
-            return self.apply_ret(concept, eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED))
+            return self.apply_ret(concept, hints.eval_body or context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED))
+
+        to_reset = set()
+        if isinstance(hints.eval_body, bool) and not hints.eval_body:
+            to_reset.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
+        if isinstance(hints.eval_question, bool) and not hints.eval_question:
+            to_reset.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+        if isinstance(hints.expression_only, bool) and not hints.expression_only:
+            to_reset.add(BuiltinConcepts.EXPRESSION_ONLY_REQUESTED)
 
         # if concept.get_hints().use_copy:
         #     raise Exception("Use copy")
@@ -635,17 +652,21 @@ class SheerkaEvaluateConcept(BaseService):
         #         return concept
 
         desc = f"Evaluating concept {concept}"
-        with context.push(BuiltinConcepts.EVALUATING_CONCEPT, concept, desc=desc) as sub_context:
-            sub_context.add_inputs(eval_body=eval_body)
-            if eval_body:
-                # ask for body evaluation
+        with context.push(BuiltinConcepts.EVALUATING_CONCEPT, concept, desc=desc, reset_hints=to_reset) as sub_context:
+            sub_context.add_inputs(hints=hints)
+
+            # update context with evaluate_concept parameters
+            if hints.eval_body:
                 sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
-            if validation_only:
+            if hints.eval_question:
+                sub_context.protected_hints.add(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
+
+            if hints.expression_only:
                 # Never call methods with side effect in this concept or sub concepts
                 sub_context.protected_hints.add(BuiltinConcepts.EXPRESSION_ONLY_REQUESTED)
 
-            # auto evaluate commands
+            # update context with evaluate_concept parameters
             if context.sheerka.isa(concept, context.sheerka.new(BuiltinConcepts.AUTO_EVAL)):
                 sub_context.protected_hints.add(BuiltinConcepts.EVAL_BODY_REQUESTED)
 
@@ -659,10 +680,10 @@ class SheerkaEvaluateConcept(BaseService):
             except ChickenAndEggException as ex:
                 return ex.error
 
-            # to make sure of the order, it don't use ConceptParts.get_parts()
+            # to make sure of the order, it does not use ConceptParts.get_parts()
             # variables must be evaluated first, body must be evaluated before where
             all_metadata_to_eval = metadata or self.compute_metadata_to_eval(sub_context, concept)
-            if validation_only and ConceptParts.BODY in all_metadata_to_eval:
+            if hints.expression_only and ConceptParts.BODY in all_metadata_to_eval:
                 all_metadata_to_eval.remove(ConceptParts.BODY)
 
             for metadata_to_eval in all_metadata_to_eval:
@@ -681,7 +702,7 @@ class SheerkaEvaluateConcept(BaseService):
                                 var_name,
                                 concept,
                                 True,
-                                not sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED),
+                                sub_context.in_context(BuiltinConcepts.EXPRESSION_ONLY_REQUESTED),
                                 w_clause)
                         else:
                             # Do not send the current concept for the properties
@@ -690,7 +711,7 @@ class SheerkaEvaluateConcept(BaseService):
                                                     var_name,
                                                     concept,
                                                     True,
-                                                    not sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED),
+                                                    sub_context.in_context(BuiltinConcepts.EXPRESSION_ONLY_REQUESTED),
                                                     w_clause)
 
                         if isinstance(resolved, Concept) and not sub_context.sheerka.is_success(resolved):
@@ -722,7 +743,7 @@ class SheerkaEvaluateConcept(BaseService):
                                             part_key,
                                             concept,
                                             force_concept_eval,
-                                            not sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED),
+                                            sub_context.in_context(BuiltinConcepts.EXPRESSION_ONLY_REQUESTED),
                                             None)
 
                     # 'FATAL' error is detected, let's stop
@@ -766,10 +787,25 @@ class SheerkaEvaluateConcept(BaseService):
             return self.apply_ret(concept, sub_context.in_context(BuiltinConcepts.EVAL_BODY_REQUESTED))
 
     def call_concept(self, context, concept, *args, **kwargs):
+        return self.call_concept_with_args(context,
+                                           concept,
+                                           hints=EvaluationHints(eval_body=True, eval_question=False),
+                                           *args,
+                                           **kwargs)
+
+    def evaluate_question(self, context, concept, *args, **kwargs):
+        return self.call_concept_with_args(context,
+                                           concept,
+                                           hints=EvaluationHints(eval_body=True, eval_question=True),
+                                           *args,
+                                           **kwargs)
+
+    def call_concept_with_args(self, context, concept, hints, *args, **kwargs):
         """
         call the concept using either args or kwargs (not both)
         :param context:
         :param concept:
+        :param hints:
         :param args:
         :param kwargs:
         :return:
@@ -780,9 +816,13 @@ class SheerkaEvaluateConcept(BaseService):
             concept.get_hints().is_instance = True
             concept.get_hints().is_evaluated = False  # force evaluation
 
-        # TODO : update the variables before calling the concept
+        for param_name, arg in zip(concept.get_metadata().parameters, args):
+            concept.get_compiled()[param_name] = DoNotResolve(arg)
 
-        evaluated = self.evaluate_concept(context, concept)
+        for param_name, param_value in kwargs.items():
+            concept.get_compiled()[param_name] = DoNotResolve(param_value)
+
+        evaluated = self.evaluate_concept(context, concept, hints=hints)
         if self.sheerka.has_error(context, evaluated):
             raise ConceptEvalException(evaluated)
 

src/core/sheerka/services/SheerkaExecute.py

@@ -2,9 +2,9 @@ import core.utils
 from cache.FastCache import FastCache
 from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
 from core.concept import ConceptParts
-from core.global_symbols import NotFound, NO_MATCH, EVENT_CONCEPT_CREATED, EVENT_CONCEPT_MODIFIED, EVENT_CONCEPT_DELETED
+from core.global_symbols import EVENT_CONCEPT_CREATED, EVENT_CONCEPT_DELETED, EVENT_CONCEPT_MODIFIED, NO_MATCH, NotFound
 from core.sheerka.services.sheerka_service import BaseService
-from core.tokenizer import Tokenizer, TokenKind, Token, Keywords
+from core.tokenizer import Keywords, Token, TokenKind, Tokenizer
 
 PARSE_STEPS = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
 PARSE_AND_EVAL_STEPS = PARSE_STEPS + [BuiltinConcepts.BEFORE_EVALUATION,
@@ -179,6 +179,24 @@ class ParserInput:
             return True
         return False
 
+    def clone(self):
+        clone = ParserInput(self.text)
+        clone.tokens = self.tokens
+        clone.length = self.length
+        clone.yield_oef = self.yield_oef
+        clone.start = self.start
+        clone.end = self.end
+        clone.sub_text = self.sub_text
+        clone.sub_tokens = self.sub_tokens
+        clone.from_tokens = self.from_tokens
+        clone.pos = self.pos
+        clone.token = self.token
+
+        return clone
+
+    def sub_part(self, start, end, yield_oef=None):
+        return ParserInput(self.text, self.tokens, self.length, start, end, yield_oef)
+
 
 class SheerkaExecute(BaseService):
     """
@@ -227,6 +245,7 @@ class SheerkaExecute(BaseService):
         self.sheerka.bind_service_method(self.NAME, self.parse_function, False, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.parse_python, False, visible=False)
         self.sheerka.bind_service_method(self.NAME, self.parse_expression, False, visible=False)
+        self.sheerka.bind_service_method(self.NAME, self.clear_parser_cache, True)
 
         self.reset_registered_evaluators()
         self.reset_registered_parsers()
@@ -274,6 +293,9 @@ class SheerkaExecute(BaseService):
             use_classes=True)
         self.question_parsers = [p.name for p in question_parsers]
 
+    def clear_parser_cache(self):
+        self.parsers_cache.clear()
+
     @staticmethod
     def get_grouped(evaluators, use_classes=False):
         """
@@ -442,7 +464,7 @@ class SheerkaExecute(BaseService):
         return None
 
     def add_to_parser_cache(self, parsers_key, text, return_value):
-        if parsers_key is None:
+        if not self.sheerka.enable_parser_caching or parsers_key is None:
             return
 
         key = (parsers_key, text)
@@ -517,7 +539,7 @@ class SheerkaExecute(BaseService):
                 pass
 
         # 3. Try the cache
-        if to_process and parsers_key:
+        if self.sheerka.enable_parser_caching and to_process and parsers_key:
             processed = []
             for return_value in to_process:
                 to_parse_as_str = self.get_input_as_text(return_value.body.body) \

src/core/sheerka/services/SheerkaIsAManager.py

@@ -6,7 +6,7 @@ from core.concept import Concept, DEFINITION_TYPE_BNF
 from core.global_symbols import NotFound
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.sheerka_service import BaseService
-from core.tokenizer import Tokenizer, TokenKind
+from core.tokenizer import TokenKind, Tokenizer
 from core.utils import merge_sets
 
 
@@ -66,7 +66,8 @@ class SheerkaIsAManager(BaseService):
                 return res
 
             concept.set_prop(BuiltinConcepts.ISA, new_concept_set)
-            res = self.add_concept_to_set(context, concept, concept_set)
+            # KSI 2021-08-12. Make sure to use the newly created concept to put in cache
+            res = self.add_concept_to_set(context, res.body.body, concept_set)
             return res
         else:
             concept.set_prop(BuiltinConcepts.ISA, new_concept_set)
@@ -144,15 +145,16 @@ class SheerkaIsAManager(BaseService):
             if not self.isaset(context, sub_concept):
                 return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
 
+            # is it a valid concept ?
+            sub_concept = core.builtin_helpers.ensure_evaluated(context, sub_concept)
+            if not self.sheerka.is_success(sub_concept):
+                return sub_concept
+
             # first, try to see if sub_concept has it's own group entry
             ids = self.sheerka.om.get(self.CONCEPTS_GROUPS_ENTRY, sub_concept.id)
             concepts = self._get_concepts(context, ids, True)
 
             # aggregate with en entries from its body
-            sub_concept = core.builtin_helpers.ensure_evaluated(context, sub_concept)
-            if not self.sheerka.is_success(sub_concept):
-                return sub_concept
-
             if self.isaset(context, sub_concept.body):
                 other_concepts = _get_set_elements(sub_concept.body)
                 if not self.sheerka.is_success(other_concepts):
@@ -169,6 +171,7 @@ class SheerkaIsAManager(BaseService):
         if (res := self.sheerka.om.get(self.CONCEPTS_IN_GROUPS_ENTRY, concept.id)) is not NotFound:
             return res
 
+        # get the elements that are in the set
         res = _get_set_elements(concept)
 
         # put in cache

src/core/sheerka/services/SheerkaRuleManager.py

@@ -15,7 +15,7 @@ from core.rule import Rule, ACTION_TYPE_PRINT
 from core.sheerka.Sheerka import RECOGNIZED_BY_NAME, RECOGNIZED_BY_ID
 from core.sheerka.services.sheerka_service import BaseService, FailedToCompileError, UnknownVariableError
 from core.tokenizer import Keywords, TokenKind, Token
-from core.utils import merge_dictionaries, merge_sets, get_safe_str_value
+from core.utils import merge_dicts, merge_sets, get_safe_str_value
 from evaluators.PythonEvaluator import PythonEvaluator
 from parsers.BaseExpressionParser import AndNode, ExpressionVisitor, VariableNode, ComparisonNode, FunctionNode, \
     ComparisonType, NotNode, NameExprNode
@@ -503,8 +503,11 @@ class GetConditionExprVisitor(ExpressionVisitor):
     def get_object_name(self, obj, objects=None):
         """
         object found during the parsing are not serialized
-        They are kept in a dictionary and this function returns a new name for every new object
-        :return:
+        They are kept in a dictionary.
+        This function returns a new name for every new object
+        :param obj: object for which a name is to be created
+        :param objects: already created names (it's a dictionary)
+        :return: tuple(name created, dictionary of already created names)
         """
         if objects is None:
             objects = {}
@@ -526,9 +529,9 @@ class GetConditionExprVisitor(ExpressionVisitor):
 
     def add_variable(self, target):
         """
-        Create a new variable
+        Create a new variable name of the object 'target'
         :param target:
-        :return:
+        :return: generated name
         """
         var_name = f"__x_{self.var_counter:02}__"
         self.var_counter += 1
@@ -881,7 +884,7 @@ class PythonConditionExprVisitorObj:
 
         return PythonConditionExprVisitorObj(get_source(left.text, right.text),
                                              get_source(left.source, right.source),
-                                             merge_dictionaries(left.objects, right.objects),
+                                             merge_dicts(left.objects, right.objects),
                                              merge_sets(left.variables, right.variables),
                                              merge_sets(left.not_variables, right.not_variables))
 
@@ -903,7 +906,7 @@ class PythonConditionExprVisitorObj:
 
         return PythonConditionExprVisitorObj(get_source(left.text, right.text),
                                              get_source(left.source, right.source),
-                                             merge_dictionaries(left.objects, right.objects),
+                                             merge_dicts(left.objects, right.objects),
                                              merge_sets(left.variables, right.variables),
                                              merge_sets(left.not_variables, right.not_variables))
 
@@ -929,7 +932,7 @@ class PythonConditionExprVisitorObj:
 
         return PythonConditionExprVisitorObj(text,
                                              get_source(left.source, right.source),
-                                             merge_dictionaries(left.objects, right.objects),
+                                             merge_dicts(left.objects, right.objects),
                                              merge_sets(left.variables, right.variables),
                                              merge_sets(left.not_variables, right.not_variables))
 

src/core/sheerka/services/SheerkaVariableManager.py

@@ -20,9 +20,12 @@ class Variable(ServiceObj):
     def get_key(self):
         return f"{self.who}|{self.key}"
 
-    def __str__(self):
+    def __repr__(self):
         return f"({self.who}){self.key}={self.value}"
 
+    def __str__(self):
+        return f"{self.who}.{self.key}={self.value}"
+
 
 @dataclass
 class InternalObj:
@@ -49,7 +52,8 @@ class SheerkaVariableManager(BaseService):
             self.sheerka.name: {"enable_process_return_values",
                                 "save_execution_context",
                                 "enable_process_rules",
-                                "enable_commands_backup"}
+                                "enable_commands_backup",
+                                "enable_parser_caching"}
         }
 
     def initialize(self):
@@ -79,7 +83,7 @@ class SheerkaVariableManager(BaseService):
 
     def record_var(self, context, who, key, value):
         """
-
+        Internal set
         :param context:
         :param who: entity that owns the key (acts as a namespace)
         :param key:
@@ -96,6 +100,9 @@ class SheerkaVariableManager(BaseService):
             setattr(service, key, value)
 
     def load_var(self, who, key):
+        """
+        Internal get
+        """
         variable = self.sheerka.om.get(self.VARIABLES_ENTRY, who + "|" + key)
         if variable is NotFound:
             return NotFound

src/core/simple_debug.py

@@ -1,5 +1,12 @@
+import os
+
 default_debug_name = "*default*"
 debug_activated = set()
+append_file = False
+items = []
+
+if not append_file and os.path.exists("debug.txt"):
+    os.remove("debug.txt")
 
 
 def my_debug(*args, check_started=None):
@@ -28,8 +35,10 @@ def my_debug(*args, check_started=None):
             if isinstance(arg, list):
                 for item in arg:
                     f.write(f"{item}\n")
+                    items.append(item)
             else:
                 f.write(f"{arg}\n")
+                items.append(args)
 
 
 def start_debug(debug_name=default_debug_name, msg=None):

src/core/tokenizer.py

@@ -1,3 +1,4 @@
+from contextlib import contextmanager
 from dataclasses import dataclass, field
 from enum import Enum
 
@@ -25,10 +26,10 @@ class TokenKind(Enum):
     STAR = "star"
     SLASH = "slash"
     PERCENT = "percent"
-    COMMA = "comma"
-    SEMICOLON = "semicolon"
-    COLON = "colon"
-    DOT = "dot"
+    COMMA = "comma"  # ,
+    SEMICOLON = "semicolon"  # ;
+    COLON = "colon"  # :
+    DOT = "dot"  # .
     QMARK = "qmark"
     VBAR = "vbar"
     AMPER = "amper"
@@ -95,7 +96,7 @@ class Token:
         if self.type == TokenKind.EOF:
             self._repr_value = "<EOF>"
         elif self.type == TokenKind.WHITESPACE:
-            self._repr_value = "<ws!>" if self.value == "" else "<tab>" if self.value[0] == "\t" else "<ws>"
+            self._repr_value = "<!ws>" if self.value == "" else "<tab>" if self.value[0] == "\t" else "<ws>"
         elif self.type == TokenKind.NEWLINE:
             self._repr_value = "<nl>"
         elif self.type == TokenKind.CONCEPT:
@@ -566,3 +567,18 @@ class IterParser:
             return token_after
         except StopIteration:
             return Token(TokenKind.EOF, -1, -1, -1, -1)
+
+
+def simple_token_compare(a, b):
+    return a.type == b.type and a.value == b.value
+
+
+@contextmanager
+def comparable_tokens():
+    eq = Token.__eq__
+    ne = Token.__ne__
+    setattr(Token, "__eq__", simple_token_compare)
+    setattr(Token, "__ne__", lambda a, b: not simple_token_compare(a, b))
+    yield
+    setattr(Token, "__eq__", eq)
+    setattr(Token, "__ne__", ne)

src/core/utils.py

@@ -8,10 +8,10 @@ import warnings
 from copy import deepcopy
 
 # from pyparsing import *
-from pyparsing import Literal, Word, nums, Combine, Optional, delimitedList, oneOf, alphas, Suppress
+from pyparsing import Combine, Literal, Optional, Suppress, Word, alphas, delimitedList, nums, oneOf
 
 from core.global_symbols import CustomType
-from core.tokenizer import TokenKind, Tokenizer, Token
+from core.tokenizer import Token, TokenKind, Tokenizer
 
 COLORS = {
     "black",
@@ -306,42 +306,34 @@ def dict_product(a, b):
     return res
 
 
-def merge_dictionaries(a, b):
+def merge_dicts(*items):
     """
-    Returns a new dictionary which is the merge
-    :param a: 
-    :param b: 
+    Returns a new dictionary which is the merge of all others
+    :type items: object
     :return:
     """
-    if a is None and b is None:
+    if items is None:
         return None
 
     res = {}
-    if a:
-        res.update(a)
-
-    if b:
-        res.update(b)
+    for item in [item for item in items if item]:
+        res.update(item)
 
     return res
 
 
-def merge_sets(a, b):
+def merge_sets(*items):
     """
-    Merge that can handle None
-    :param a:
-    :param b:
+    Returns a new dictionary which is the merge of all others
+    :type items: object
     :return:
     """
-    if a is None and b is None:
+    if items is None:
         return None
 
     res = set()
-    if a:
-        res.update(a)
-
-    if b:
-        res.update(b)
+    for item in [item for item in items if item]:
+        res.update(item)
 
     return res
 
@@ -612,7 +604,7 @@ def tokens_index(tokens, sub_tokens, skip=0, start_from_end=False):
             else:
                 skip -= 1
 
-    raise ValueError(f"sub tokens '{sub_tokens}' not found")
+    raise ValueError(f"sub tokens '{get_text_from_tokens(sub_tokens)}' from {sub_tokens}  not found")
 
 
 def as_bag(obj, forced_properties=None):

src/evaluators/BaseEvaluator.py

@@ -1,3 +1,4 @@
+from core.builtin_concepts_ids import BuiltinConcepts
 from core.sheerka.Sheerka import ExecutionContext
 
 
@@ -9,10 +10,6 @@ class BaseEvaluator:
     PREFIX = "evaluators."
 
     def __init__(self, name, steps, priority: int, enabled=True):
-        # self.log = get_logger(self.PREFIX + self.__class__.__name__)
-        # self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__)
-        # self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__)
-
         self.name = BaseEvaluator.get_name(name)
         self.short_name = name
         self.steps = steps
@@ -71,3 +68,8 @@ class AllReturnValuesEvaluator(BaseEvaluator):
 
     def reset(self):
         self.eaten.clear()
+
+    @staticmethod
+    def valid_parser_results(context, return_values):
+        return [ret for ret in return_values if
+                ret.status and context.sheerka.isinstance(ret.body, BuiltinConcepts.PARSER_RESULT)]

src/evaluators/DefConceptEvaluator.py

@@ -173,6 +173,7 @@ class DefConceptEvaluator(OneReturnValueEvaluator):
 
         if def_concept_node.variables != NotInit:
             certain_variables = def_concept_node.variables.copy()
+            variables_found.update(certain_variables)
             skip_variables_resolution = True
 
         # get variables and set the sources

src/evaluators/ResolveAmbiguityEvaluator.py

@@ -25,8 +25,7 @@ class ResolveAmbiguityEvaluator(AllReturnValuesEvaluator):
         # If they share the same source, that means that there are multiple results for one ParserInput
         self.sources = {}
         success = False
-        for ret in [ret for ret in return_values if
-                    ret.status and context.sheerka.isinstance(ret.body, BuiltinConcepts.PARSER_RESULT)]:
+        for ret in self.valid_parser_results(context, return_values):
             source = self.get_source(context, ret)
             concept_is_instance = self.get_has_concept_instance(ret)
 

src/evaluators/ResolveMultiplePluralAmbiguityEvaluator.py

@@ -0,0 +1,49 @@
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.sheerka.ExecutionContext import ExecutionContext
+from evaluators.BaseEvaluator import AllReturnValuesEvaluator
+from parsers.BaseNodeParser import ConceptNode
+from parsers.BaseParser import BaseParser
+from parsers.PythonParser import PythonParser
+from parsers.SequenceNodeParser import SequenceNodeParser
+
+
+class ResolveMultiplePluralAmbiguityEvaluator(AllReturnValuesEvaluator):
+    NAME = "ResolveMultiplePluralAmbiguity"
+
+    def __init__(self):
+        super().__init__(self.NAME, [BuiltinConcepts.AFTER_PARSING], 55)
+        self.python = None
+        self.sequence = None
+
+    def reset(self):
+        super().reset()
+        self.python = None
+        self.sequence = None
+
+    def matches(self, context: ExecutionContext, return_values):
+        for ret in self.valid_parser_results(context, return_values):
+            if ret.body.parser.short_name == PythonParser.NAME:
+                self.python = ret
+            elif ret.body.parser.short_name == SequenceNodeParser.NAME:
+                self.sequence = ret
+
+        if self.python and self.sequence:
+            if (len(self.sequence.body.body) == 1 and
+                    isinstance(self.sequence.body.body[0], ConceptNode) and
+                    context.sheerka.is_plural(self.sequence.body.body[0].concept)):
+                return True
+
+        return False
+
+    def eval(self, context: ExecutionContext, return_values):
+        symbol = self.sequence.body.body[0].concept.name
+        if context.sheerka.isinstance(context.sheerka.memory(context, symbol), BuiltinConcepts.NOT_FOUND):
+            return [self.copy(context, self.sequence)]
+        else:
+            return [self.copy(context, self.python)]
+
+    def copy(self, context, to_keep):
+        return context.sheerka.ret(self.name,
+                                   to_keep.status,
+                                   to_keep.value,
+                                   parents=[self.python, self.sequence])

src/evaluators/ValidateConceptEvaluator.py

@@ -1,5 +1,6 @@
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, ConceptParts
+from core.sheerka.services.SheerkaEvaluateConcept import EvaluationHints
 from evaluators.BaseEvaluator import OneReturnValueEvaluator
 from parsers.BaseNodeParser import ConceptNode
 from parsers.BaseParser import BaseParser
@@ -73,8 +74,7 @@ class ValidateConceptEvaluator(OneReturnValueEvaluator):
 
         res = sheerka.evaluate_concept(context,
                                        concept,
-                                       eval_body=False,
-                                       validation_only=True,
+                                       hints=EvaluationHints(eval_body=True, expression_only=True),
                                        metadata=["variables", ConceptParts.PRE, ConceptParts.WHERE])
 
         # either the 'pre' or the 'where' condition is not fulfilled

src/parsers/BaseCustomGrammarParser.py

@@ -323,7 +323,7 @@ class BaseCustomGrammarParser(BaseParserInputParser):
                 keywords_found.add(token.value)
                 colon_mode_activated = self.parser_input.the_token_after().type == TokenKind.COLON
                 if not self.parser_input.next_token():
-                    self.add_error(UnexpectedEofParsingError(f"While parsing keyword '{keyword.value}'."))
+                    self.add_error(UnexpectedEofParsingError(f"while parsing keyword '{keyword.value}'"))
                     break
             else:
                 buffer.append(token)

src/parsers/BaseExpressionParser.py

@@ -4,11 +4,26 @@ from typing import List, Union
 
 from core.builtin_concepts_ids import BuiltinConcepts
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.tokenizer import Token, TokenKind, Tokenizer, LexerError
-from core.utils import tokens_are_matching, get_text_from_tokens
+from core.tokenizer import LexerError, Token, TokenKind, Tokenizer
+from core.utils import get_text_from_tokens, tokens_are_matching
 from parsers.BaseNodeParser import UnrecognizedTokensNode
-from parsers.BaseParser import Node, ParsingError, BaseParser, ErrorSink, UnexpectedTokenParsingError
+from parsers.BaseParser import BaseParser, ErrorSink, Node, ParsingError, UnexpectedTokenParsingError
 
+open_parenthesis_mapping = {
+    TokenKind.LBRACKET: Token(TokenKind.LBRACKET, "[", -1, -1, -1),
+    TokenKind.LPAR: Token(TokenKind.LPAR, "(", -1, -1, -1),
+    TokenKind.LBRACE: Token(TokenKind.LBRACE, "{", -1, -1, -1),
+}
+
+end_parenthesis_mapping = {
+    TokenKind.LBRACKET: Token(TokenKind.RBRACKET, "]", -1, -1, -1),
+    TokenKind.LPAR: Token(TokenKind.RPAR, ")", -1, -1, -1),
+    TokenKind.LBRACE: Token(TokenKind.RBRACE, "}", -1, -1, -1),
+}
+
+end_parenthesis_types = {TokenKind.RBRACKET, TokenKind.RPAR, TokenKind.RBRACE}
+
+comma = Token(TokenKind.COMMA, ",", -1, -1, -1)
 
 class ComparisonType:
     EQUALS = "EQ"
@@ -26,7 +41,14 @@ class LeftPartNotFoundError(ParsingError):
     """
     When the expression starts with 'or' or 'and'
     """
-    pass
+    keyword: str
+    pos: int
+
+    def __repr__(self):
+        return f"LeftPartNotFoundError(keyword={self.keyword}, pos={self.pos})"
+
+    def __str__(self):
+        return f"There is not left part to '{self.keyword}' at position {self.pos}"
 
 
 @dataclass()
@@ -390,6 +412,67 @@ class FunctionNode(ExprNode):
         return f"{self.first} {self.parameters} {self.last}"
 
 
+@dataclass()
+class Comprehension:
+    target: ExprNode
+    iterable: ExprNode
+    if_expr: ExprNode
+
+
+class ListComprehensionNode(ExprNode):
+    def __init__(self, start, end, tokens, element: ExprNode, generators: List[Comprehension]):
+        super().__init__(start, end, tokens)
+        self.element = element
+        self.generators = generators
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, ListComprehensionNode):
+            return False
+
+        if not super().__eq__(other):
+            return False
+
+        return self.element == other.element and self.generators == other.generators
+
+    def __repr__(self):
+        msg = f"ListComprehensionNode(start={self.start}, end={self.end}, element='{self.element}', generators="
+        for comp in self.generators:
+            msg += f"['{comp.target}', {comp.iterable}, '{comp.if_expr}'], "
+        return msg + ")"
+
+
+class ListNode(ExprNode):
+    def __init__(self, start, end, tokens, first, last, items: List[ExprNode], sep=None):
+        super().__init__(start, end, tokens)
+        self.first = first
+        self.last = last
+        self.items = items
+        self.sep = sep or comma
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if not isinstance(other, ListNode):
+            return False
+
+        if not super().__eq__(other):
+            return False
+
+        return (self.first == other.first and
+                self.last == other.last and
+                self.items == other.items and
+                self.sep == other.sep)
+
+    def __repr__(self):
+        msg = f"ListNode(start={self.start}, end={self.end}, sep={self.sep}"
+        msg += f"first='{self.first}', last='{self.last}', items={self.items})"
+        return msg
+
+
 class BaseExpressionParser(BaseParser):
 
     def reset_parser_input(self, parser_input: ParserInput, error_sink):
@@ -432,12 +515,15 @@ class BaseExpressionParser(BaseParser):
         # The node is compiled in ExpressionParser.parse() or FunctionParser.parse(), depending of the requirement
         node = self.parse_input(context, parser_input, error_sink)
 
+        if not error_sink.has_error:
             token = parser_input.token
             if token and token.type != TokenKind.EOF:
                 if token.type == TokenKind.RPAR:
                     error_sink.add_error(ParenthesisMismatchError(token))
                 else:
-                error_sink.add_error(UnexpectedTokenParsingError(f"Unexpected token '{token}'", token, [TokenKind.EOF]))
+                    error_sink.add_error(UnexpectedTokenParsingError(f"Unexpected token '{token}'",
+                                                                     token,
+                                                                     [TokenKind.EOF]))
 
         if isinstance(node, ParenthesisNode):
             node = node.node
@@ -454,17 +540,16 @@ class BaseExpressionParser(BaseParser):
 
         return ret
 
-    def parse_input(self, context, parser_input: ParserInput, error_sink: list):
+    def parse_input(self, context, parser_input: ParserInput, error_sink: ErrorSink):
         raise NotImplementedError
 
     def parse_tokens_stop_condition(self, token, parser_input):
         raise NotImplementedError
 
-    def parse_tokens(self, context, parser_input, error_sink):
+    def parse_tokens(self, context, parser_input, error_sink, stop_condition, expr_parser):
         def stop():
             return token.type == TokenKind.EOF or \
-                   paren_count == 0 and (token.type == TokenKind.RPAR or
-                                         self.parse_tokens_stop_condition(token, parser_input))
+                   paren_count == 0 and (token.type == TokenKind.RPAR or stop_condition(token, parser_input))
 
         token = parser_input.token
         if token.type == TokenKind.EOF:
@@ -474,6 +559,10 @@ class BaseExpressionParser(BaseParser):
             last_paren = token
             start = parser_input.pos
             parser_input.next_token()
+            # KSI 2021-09-01. I am quite sure I need a specific parser to parse inside the parenthesis
+            # if so, add a inside_parenthesis_parser to the list of this function parameter
+            # parser_to_use = inside_parenthesis_parser or self
+            # expr = parser_to_use.parse_input(context, parser_input, error_sink)
             expr = self.parse_input(context, parser_input, error_sink)
             token = parser_input.token
             if token.type != TokenKind.RPAR:
@@ -491,10 +580,10 @@ class BaseExpressionParser(BaseParser):
         while not stop():
             last_is_whitespace = token.type == TokenKind.WHITESPACE
             end += 1
-            if token.type == TokenKind.LPAR:
+            if token.type in (TokenKind.LPAR, TokenKind.LBRACKET, TokenKind.LBRACE):
                 last_paren = token
                 paren_count += 1
-            if token.type == TokenKind.RPAR:
+            if token.type in (TokenKind.RPAR, TokenKind.RBRACKET, TokenKind.RBRACE):
                 paren_count -= 1
             parser_input.next_token(False)
             token = parser_input.token
@@ -503,15 +592,15 @@ class BaseExpressionParser(BaseParser):
             end -= 1
 
         if start == end:
-            if token.type != TokenKind.RPAR:
-                error_sink.add_error(LeftPartNotFoundError())
             return None
 
-        if paren_count != 0:
+        if paren_count > 0:
+            # Only if the count is > 0 as the left parenthesis may have occurred before the parse_tokens()
             error_sink.add_error(ParenthesisMismatchError(last_paren))
             return None
 
-        if self.expr_parser:
+        if expr_parser:
+            # to sub parse (parse with more fineness)
             new_parsing_input = ParserInput(
                 None,
                 tokens=parser_input.tokens,
@@ -520,14 +609,14 @@ class BaseExpressionParser(BaseParser):
                 end=end - 1,
                 yield_oef=False).reset()
             new_parsing_input.next_token()
-            return self.expr_parser.parse_input(context, new_parsing_input, error_sink)
+            return expr_parser.parse_input(context, new_parsing_input, error_sink)
         else:
             return NameExprNode(start, end - 1, parser_input.tokens[start:end])
 
 
 class ExpressionVisitor:
     """
-    Pyhtonic implementation of visitors for ExprNode
+    Pythonic implementation of visitors for ExprNode
     """
 
     def visit(self, expr_node):
@@ -548,6 +637,29 @@ class ExpressionVisitor:
                 self.visit(value)
 
 
+class ExpressionVisitorWithHint:
+    """
+    Pythonic implementation of visitors for ExprNode
+    """
+
+    def visit(self, expr_node, hint):
+        name = expr_node.__class__.__name__
+
+        method = 'visit_' + name
+        visitor = getattr(self, method, self.generic_visit)
+        return visitor(expr_node, hint)
+
+    def generic_visit(self, expr_node, hint):
+        """Called if no explicit visitor function exists for a node."""
+        for field, value in expr_node.__dict__.items():
+            if isinstance(value, (list, tuple)):
+                for item in value:
+                    if isinstance(item, ExprNode):
+                        self.visit(item, hint)
+            elif isinstance(value, ExprNode):
+                self.visit(value, hint)
+
+
 class TrueifyVisitor(ExpressionVisitor):
     """
     Visit an ExprNode

src/parsers/BaseNodeParser.py

@@ -1,9 +1,12 @@
 from dataclasses import dataclass
 
 import core.utils
-from core.tokenizer import TokenKind, Token
+from cache.FastCache import FastCache
+from core import builtin_helpers
+from core.sheerka.ExecutionContext import ExecutionContext
+from core.tokenizer import Token, TokenKind
 from core.var_ref import VariableRef
-from parsers.BaseParser import Node, ParsingError, BaseParserInputParser
+from parsers.BaseParser import BaseParserInputParser, Node, ParsingError
 
 DEBUG_COMPILED = True
 
@@ -411,6 +414,13 @@ class SourceCodeWithConceptNode(LexerNode):
     def get_source_to_parse(self):
         return self.python_node.source
 
+    def get_errors(self):
+        errors = []
+        for n in self.nodes:
+            if hasattr(n, "error") and n.error:
+                errors.append(n.error)
+        return errors
+
 
 class VariableNode(LexerNode):
     """
@@ -460,13 +470,52 @@ class NoMatchingTokenError(ParsingError):
     pos: int
 
 
+class UnrecognizedTokensCache:
+    def __init__(self, parsers):
+        self.parsers = parsers
+        self.cache = FastCache()
+        self.hits = 0
+        self.calls = 0
+
+    def get_lexer_nodes_from_unrecognized(self,
+                                          context: ExecutionContext,
+                                          unrecognized_tokens: UnrecognizedTokensNode):
+        to_request = unrecognized_tokens.source
+        self.calls += 1
+        if to_request in self.cache:
+            self.hits += 1
+            return self.cache.get(to_request)
+
+        res = builtin_helpers.get_lexer_nodes_from_unrecognized(context,
+                                                                unrecognized_tokens,
+                                                                self.parsers)
+        self.cache.put(to_request, res)
+        return res
+
+    @property
+    def ratio(self):
+        return self.hits / self.calls if self.calls else 0
+
+    @property
+    def calls_details(self):
+        return self.cache.calls
+
+    def to_dict(self):
+        return {
+            "calls": self.calls,
+            "hits": self.hits,
+            "ratio": self.ratio,
+            "calls_details": self.calls_details,
+        }
+
+
 class BaseNodeParser(BaseParserInputParser):
     """
     Parser that return LexerNode
     """
 
-    def __init__(self, name, priority, **kwargs):
-        super().__init__(name, priority, yield_eof=True)
+    def __init__(self, name, priority, enabled=True, **kwargs):
+        super().__init__(name, priority, yield_eof=True, enabled=enabled)
 
     def init_from_concepts(self, context, concepts, **kwargs):
         """

src/parsers/BaseParser.py

@@ -67,6 +67,9 @@ class UnexpectedTokenParsingError(ParsingError):
 class UnexpectedEofParsingError(ParsingError):
     message: str = None
 
+    def __repr__(self):
+        return f"UnexpectedEofParsingError({self.message})"
+
 
 class BaseParser:
     PREFIX = "parsers."

src/parsers/BnfDefinitionParser.py

@@ -240,31 +240,31 @@ class BnfDefinitionParser(BaseParser):
                 self.add_error(concept)
                 return None
 
-            expr = ConceptExpression(concept, rule_name=concept.name)
+            expr = ConceptExpression(concept, rule_name=concept.key)
             return self.eat_rule_name_if_needed(expr)
 
         if token.type in (TokenKind.IDENTIFIER, TokenKind.KEYWORD):
             self.next_token()
 
-            concept_name = token.str_value
+            concept_key = token.str_value
 
             # we are trying to match against a concept which is still under construction !
             # (for example of recursive bnf definition)
             if self.context.obj and hasattr(self.context.obj, "name"):
-                if concept_name == str(self.context.obj.name):
-                    return self.eat_rule_name_if_needed(ConceptExpression(concept_name))  # 2021-02-17 no rule name ?
+                if concept_key == str(self.context.obj.name):
+                    return self.eat_rule_name_if_needed(ConceptExpression(concept_key))  # 2021-02-17 no rule name ?
 
-            concept = self.context.get_concept(concept_name)
+            concept = self.context.get_concept(concept_key)
             if not self.sheerka.is_known(concept):
-                expr = VariableExpression(concept_name)
+                expr = VariableExpression(concept_key)
                 return self.eat_rule_name_if_needed(expr)
             elif hasattr(concept, "__iter__"):
                 self.add_error(
                     self.sheerka.new(BuiltinConcepts.CANNOT_RESOLVE_CONCEPT,
-                                     body=("key", concept_name)))
+                                     body=("key", concept_key)))
                 return None
             else:
-                expr = ConceptExpression(concept, rule_name=concept.name)
+                expr = ConceptExpression(concept, rule_name=concept.key)
                 return self.eat_rule_name_if_needed(expr)
 
         if token.type == TokenKind.STRING:

src/parsers/BnfNodeParser.py

@@ -16,13 +16,13 @@ import core.builtin_helpers
 import core.utils
 from cache.Cache import Cache
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import DEFINITION_TYPE_BNF, DoNotResolve, ConceptParts, Concept
+from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF, DoNotResolve
 from core.global_symbols import NotFound
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.tokenizer import Tokenizer, TokenKind, Token
+from core.tokenizer import Token, TokenKind, Tokenizer
 from core.utils import CONSOLE_COLORS_MAP as CCM
-from parsers.BaseNodeParser import BaseNodeParser, GrammarErrorNode, UnrecognizedTokensNode, ConceptNode, \
-    NoMatchingTokenError, RuleNode, SourceCodeNode, SourceCodeWithConceptNode
+from parsers.BaseNodeParser import BaseNodeParser, ConceptNode, GrammarErrorNode, NoMatchingTokenError, RuleNode, \
+    SourceCodeNode, SourceCodeWithConceptNode, UnrecognizedTokensCache, UnrecognizedTokensNode
 
 PARSERS = ["Sequence", "Sya", "Python"]
 VARIABLE_EXPR_PARSER = ["Sequence", "Sya", "Python", "Bnf"]
@@ -618,9 +618,8 @@ class VariableExpression(ParsingExpression):
             return None
 
         utn = UnrecognizedTokensNode(start, end, tokens)
-        nodes_sequences = core.builtin_helpers.get_lexer_nodes_from_unrecognized(parser_helper.parser.context,
-                                                                                 utn,
-                                                                                 VARIABLE_EXPR_PARSER)
+        nodes_sequences = parser_helper.parser.cache2.get_lexer_nodes_from_unrecognized(parser_helper.parser.context,
+                                                                                        utn)
         return nodes_sequences
 
     @staticmethod
@@ -1264,8 +1263,8 @@ class BnfConceptParserHelper:
         self.pos = -1
 
     def __repr__(self):
-        concepts = [item.concept if isinstance(item, ConceptNode) else "***" for item in self.sequence]
-        return f"BnfConceptParserHelper({concepts})"
+        nodes = core.builtin_helpers.debug_nodes(self.sequence)
+        return f"BnfConceptParserHelper({nodes})"
 
     def __eq__(self, other):
         if id(self) == id(other):
@@ -1443,9 +1442,8 @@ class BnfConceptParserHelper:
         self.unrecognized_tokens.fix_source()
 
         # try to recognize concepts
-        nodes_sequences = core.builtin_helpers.get_lexer_nodes_from_unrecognized(self.parser.context,
-                                                                                 self.unrecognized_tokens,
-                                                                                 PARSERS)
+        nodes_sequences = self.parser.cache.get_lexer_nodes_from_unrecognized(self.parser.context,
+                                                                              self.unrecognized_tokens)
 
         if nodes_sequences:
             instances = [self]
@@ -1638,6 +1636,8 @@ class BnfNodeParser(BaseNodeParser):
         else:
             self.concepts_grammars = Cache()
 
+        self.cache = UnrecognizedTokensCache(PARSERS)
+        self.cache2 = UnrecognizedTokensCache(VARIABLE_EXPR_PARSER)
         self.ignore_case = True
 
     @staticmethod
@@ -1963,7 +1963,7 @@ class BnfNodeParser(BaseNodeParser):
 
                     nodes = []
                     for c in valid_concepts:
-                        nodes.append(ConceptExpression(c, rule_name=c.name))
+                        nodes.append(ConceptExpression(c, rule_name=c.key))
 
                     resolved = self.resolve_parsing_expression(ssc,
                                                                UnOrderedChoice(*nodes),
@@ -2116,6 +2116,12 @@ class BnfNodeParser(BaseNodeParser):
 
         sequences = self.get_concepts_sequences(context)
         valid_parser_helpers = self.get_valid(sequences)
+
+        debugger = context.get_debugger(self.NAME, "parse")
+        if debugger.is_enabled:
+            debugger.debug_var("stats", self.cache.to_dict())
+            #debugger.debug_var("stats", self.cache2.to_dict())
+
         if valid_parser_helpers is None:
             return self.sheerka.ret(
                 self.name,

src/parsers/DefRuleParser.py

@@ -110,7 +110,7 @@ class DefRuleParser(BaseCustomGrammarParser):
             return None
 
         if not self.parser_input.next_token():  # eat as
-            self.add_error(UnexpectedEofParsingError("While parsing 'when'."))
+            self.add_error(UnexpectedEofParsingError("while parsing 'when'"))
             return None
 
         rule = self.parse_rule()

src/parsers/FunctionParser.py

@@ -98,7 +98,7 @@ class FunctionParser(BaseExpressionParser):
             return None
 
         if not parser_input.next_token():
-            error_sink.add_error(UnexpectedEofParsingError(f"Unexpected EOF while parsing left parenthesis"))
+            error_sink.add_error(UnexpectedEofParsingError(f"while parsing left parenthesis"))
             return None
 
         token = parser_input.token
@@ -110,7 +110,7 @@ class FunctionParser(BaseExpressionParser):
 
         start_node = NameExprNode(start, start + 1, parser_input.tokens[start:start + 2])
         if not parser_input.next_token():
-            error_sink.add_error(UnexpectedEofParsingError(f"Unexpected EOF after left parenthesis"))
+            error_sink.add_error(UnexpectedEofParsingError(f"after left parenthesis"))
             return FunctionNode(start, start + 1, [], start_node, None, None)
 
         params = self.parse_parameters(context, parser_input, error_sink)
@@ -146,7 +146,7 @@ class FunctionParser(BaseExpressionParser):
 
             token = parser_input.token
             if token.type == TokenKind.EOF:
-                error_sink.add_error(UnexpectedEofParsingError(f"Unexpected EOF while parsing parameters"))
+                error_sink.add_error(UnexpectedEofParsingError(f"while parsing parameters"))
                 return None
 
             if token.type == TokenKind.RPAR:
@@ -173,7 +173,7 @@ class FunctionParser(BaseExpressionParser):
 
         # otherwise, eat until LPAR or separator
         parser_input.seek(start_pos)
-        return self.parse_tokens(context, parser_input, error_sink)
+        return self.parse_tokens(context, parser_input, error_sink, self.parse_tokens_stop_condition, self.expr_parser)
 
     def parse_tokens_stop_condition(self, token, parser_input):
         return token.value == self.sep

src/parsers/ListComprehensionParser.py

@@ -0,0 +1,219 @@
+from dataclasses import dataclass
+
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.tokenizer import TokenKind
+from parsers.BaseExpressionParser import BaseExpressionParser, Comprehension, ListComprehensionNode, \
+    ParenthesisMismatchError, end_parenthesis_mapping, end_parenthesis_types
+from parsers.BaseParser import ErrorSink, ParsingError, UnexpectedEofParsingError
+from parsers.ListParser import ListParser
+from sheerkapython.ExprToPython import PythonExprVisitor
+
+
+class LeadingParenthesisNotFound(ParsingError):
+    pass
+
+
+@dataclass
+class FailedToParse(ParsingError):
+    part: str  # part that fails ('element', 'targets', 'generator' or 'if'
+    pos: int  # position after which the element was not found
+
+    def __repr__(self):
+        return f"Failed to find <{self.part}> after position {self.pos}"
+
+
+class ForNotFound(ParsingError):
+    pass
+
+
+class ElementNotFound(ParsingError):
+    pass
+
+
+class ListComprehensionParser(BaseExpressionParser):
+    NAME = "ListComprehension"
+
+    def __init__(self, **kwargs):
+        super().__init__(self.NAME, 55, yield_eof=True)
+        self.expr_parser = kwargs.get("expr_parser", None)
+        self.auto_compile = kwargs.get("auto_compile", True)
+        self.element_parser = ListParser()
+
+    @staticmethod
+    def stop_condition(keywords, end_parenthesis=None):
+        if end_parenthesis is None:
+            return lambda t, pi: t.type == TokenKind.IDENTIFIER and t.value in keywords
+        else:
+            return lambda t, pi: t.type == TokenKind.IDENTIFIER and t.value in keywords or t.type in end_parenthesis
+
+    def parse_input(self, context, parser_input, error_sink):
+        return self.parse_list_comprehension(context, parser_input, error_sink)
+
+    def parse_list_comprehension(self, context, parser_input, error_sink):
+        start_pos = parser_input.pos
+
+        # first define the leading parenthesis / bracket / brace
+        start_parenthesis = parser_input.token
+        if start_parenthesis.type not in end_parenthesis_mapping:
+            error_sink.add_error(LeadingParenthesisNotFound())
+            return None
+
+        end_parenthesis = end_parenthesis_mapping[start_parenthesis.type]
+
+        if not parser_input.next_token():
+            error_sink.add_error(UnexpectedEofParsingError("when start parsing"))
+            return None
+
+        element_start_pos = parser_input.pos
+
+        # search the 'for' tokens. They will be the pivot for our parsing
+        for_tokens_positions = self.get_for_positions(parser_input)
+        if not for_tokens_positions:
+            error_sink.add_error(ForNotFound())
+            return None
+
+        comprehensions = []
+        sub_error_sink = ErrorSink()
+        for start, end in reversed(for_tokens_positions):
+            sub_input = parser_input.sub_part(start, end, yield_oef=True)
+            sub_input.reset()
+            sub_input.next_token()
+            comprehension = self.parse_comprehension(context, sub_input, sub_error_sink, end_parenthesis.type)
+
+            if not comprehension:
+                element_end_pos = end
+                break
+            else:
+                comprehensions.insert(0, comprehension)
+        else:
+            element_end_pos = start - 1
+
+        if not comprehensions:
+            error_sink.sink.extend(sub_error_sink.sink)
+            return None
+
+        if element_end_pos < element_start_pos:
+            error_sink.add_error(ElementNotFound())
+            return None
+
+        sub_input = parser_input.sub_part(element_start_pos, element_end_pos, yield_oef=True)
+        sub_input.reset()
+        sub_input.next_token()
+        element_expr = self.element_parser.parse_input(context, sub_input, error_sink)
+
+        if not element_expr:
+            return None
+
+        if not parser_input.token.type == end_parenthesis.type:
+            error_sink.add_error(ParenthesisMismatchError(end_parenthesis))
+            return None
+
+        end_pos = parser_input.pos
+        parser_input.next_token()
+
+        return ListComprehensionNode(start_pos,
+                                     end_pos,
+                                     parser_input.tokens[start_pos: end_pos + 1],
+                                     element_expr,
+                                     comprehensions)
+
+    def parse_comprehension(self, context, parser_input, error_sink, end_parenthesis_type):
+        parser_input.next_token()  # eat the leading 'for'
+
+        pos = parser_input.pos
+        target_expr = self.parse_tokens(context,
+                                        parser_input,
+                                        error_sink,
+                                        self.stop_condition(["in"], [end_parenthesis_type]),
+                                        None)
+        if not target_expr:
+            error_sink.add_error(FailedToParse('target', pos))
+            return None
+
+        if not parser_input.next_token():
+            error_sink.add_error(UnexpectedEofParsingError("while parsing comprehension"))
+            return None
+
+        pos = parser_input.pos
+        generator_expr = self.parse_tokens(context,
+                                           parser_input,
+                                           error_sink,
+                                           self.stop_condition(["for", "if"], [end_parenthesis_type]),
+                                           None)
+        if not generator_expr:
+            error_sink.add_error(FailedToParse('generator', pos))
+            return None
+
+        token = parser_input.token
+        if token.value == "if":
+            if not parser_input.next_token():
+                error_sink.add_error(UnexpectedEofParsingError("while parsing comprehension"))
+                return None
+
+            pos = parser_input.pos
+            if_expr = self.parse_tokens(context,
+                                        parser_input,
+                                        error_sink,
+                                        self.stop_condition(["for"], [end_parenthesis_type]),
+                                        None)
+
+            if not if_expr:
+                error_sink.add_error(FailedToParse('if', pos))
+                return None
+        else:
+            if_expr = None
+
+        return Comprehension(target_expr, generator_expr, if_expr)
+
+    def parse_tokens_stop_condition(self, token, parser_input):
+        raise NotImplementedError()
+
+    def parse(self, context, parser_input: ParserInput):
+        ret = super().parse(context, parser_input)
+
+        if not self.auto_compile:
+            return ret
+
+        if ret is None:
+            return None
+
+        if not ret.status:
+            return ret
+
+        node = ret.body.body
+        visitor = PythonExprVisitor(context)
+
+        ret = visitor.compile(node)
+        return ret[0] if len(ret) == 1 else ret
+
+    @staticmethod
+    def get_for_positions(parser_input):
+        """
+        browse the tokens to get the starting and ending position of 'for' tokens
+        ex:
+        [ x, t for x in z if t for a in b ]
+                ^               ^
+        will return
+        :param parser_input:
+        :return: list of tuple representing where the 'for' token starts and where it ends
+        """
+        res = []
+        current_pos = None
+        nb_parenthesis = 1
+        while True:
+            if parser_input.token.value == "for":
+                if current_pos:
+                    res.append((current_pos, parser_input.pos - 1))
+                current_pos = parser_input.pos
+            elif parser_input.token.type in end_parenthesis_mapping:
+                nb_parenthesis += 1
+            elif parser_input.token.type in end_parenthesis_types:
+                nb_parenthesis -= 1
+
+            if nb_parenthesis == 0 or not parser_input.next_token():
+                break
+
+        if current_pos:
+            res.append((current_pos, parser_input.pos - 1))
+
+        return res

src/parsers/ListParser.py

@@ -0,0 +1,77 @@
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.tokenizer import TokenKind
+from parsers.BaseExpressionParser import BaseExpressionParser, ListNode, NameExprNode, ParenthesisMismatchError, \
+    comma, end_parenthesis_mapping
+from parsers.BaseParser import ErrorSink
+
+
+class ListParser(BaseExpressionParser):
+    NAME = "List"
+
+    def __init__(self, sep=None, **kwargs):
+        super().__init__(self.NAME, 50, False, yield_eof=True)
+        # KSI 2021-09-02 : The priority (50) is not set
+        self.sep = sep or comma
+        self.end_tokens = [self.sep] if sep else [self.sep]
+        self.expr_parser = kwargs.get("expr_parser", None)
+
+    def parse_input(self, context, parser_input: ParserInput, error_sink: ErrorSink):
+        start = parser_input.pos
+        token = parser_input.token
+        first = None
+        trailing_token = None
+
+        # define the opening parenthesis and set the expected closing parenthesis
+        if token.type in end_parenthesis_mapping:
+            trailing_token = end_parenthesis_mapping[token.type]
+            self.end_tokens.append(trailing_token)
+            first = NameExprNode(start, start, parser_input.tokens[start:start + 1])
+
+            parser_input.next_token()
+
+        token = parser_input.token
+        items = []
+        while token.type != TokenKind.EOF:
+            parsed = self.parse_tokens(context,
+                                       parser_input,
+                                       error_sink,
+                                       self.parse_tokens_stop_condition,
+                                       self.expr_parser)
+
+            if parsed:
+                items.append(parsed)
+
+            token = parser_input.token
+            if not parsed or error_sink.has_error or (token.type != self.sep.type or token.value != self.sep.value):
+                break
+
+            parser_input.next_token()
+
+        if error_sink.has_error:
+            return None
+
+        # make sure the trailing parenthesis is present if required
+        if trailing_token:
+            if token.type == trailing_token.type:
+                pos = parser_input.pos
+                last = NameExprNode(pos, pos, parser_input.tokens[pos:pos + 1])
+            else:
+                last = None
+                error_sink.add_error(ParenthesisMismatchError(trailing_token))
+        else:
+            last = None
+
+        end = parser_input.pos if token.type != TokenKind.EOF else parser_input.pos - 1
+        if self.parse_tokens_stop_condition(token, None):
+            parser_input.next_token()
+
+        if first is None and last is None and len(items) == 0:
+            return None
+
+        return ListNode(start, end, parser_input.tokens[start:end + 1], first, last, items, self.sep)
+
+    def parse_tokens_stop_condition(self, token, parser_input):
+        for t in self.end_tokens:
+            if t.type == token.type and t.value == token.value:
+                return True
+        return False

src/parsers/LogicalOperatorParser.py

@@ -1,60 +1,7 @@
-from core.sheerka.services.SheerkaExecute import ParserInput
-from core.sheerka.services.sheerka_service import FailedToCompileError
-from core.tokenizer import TokenKind, Tokenizer
-from core.utils import get_text_from_tokens
-from parsers.BaseExpressionParser import ParenthesisNode, OrNode, AndNode, NotNode, VariableNode, \
-    ComparisonNode, BaseExpressionParser
-from parsers.BaseParser import UnexpectedEofParsingError, ErrorSink
-from sheerkarete.common import V
-from sheerkarete.conditions import Condition, AndConditions
-
-
-class ReteConditionsEmitter:
-
-    def __init__(self, context):
-        from parsers.RelationalOperatorParser import RelationalOperatorParser
-        self.context = context
-        self.comparison_parser = RelationalOperatorParser()
-        self.var_counter = 0
-        self.variables = {}
-
-    def add_variable(self, target):
-        var_name = f"__x_{self.var_counter:02}__"
-        self.var_counter += 1
-        self.variables[target] = var_name
-        return var_name
-
-    def init_variable_if_needed(self, node, res):
-        if node.name not in self.variables:
-            var_name = self.add_variable(node.name)
-            res.append(Condition(V(var_name), "__name__", node.name))
-
-        return V(self.variables[node.name])
-
-    def get_conditions(self, expr_nodes):
-        conditions = []
-        for expr_node in expr_nodes:
-            error_sink = ErrorSink()
-            parser_input = ParserInput(None, tokens=expr_node.tokens).reset()
-            parser_input.next_token()
-
-            parsed = self.comparison_parser.parse_input(self.context, parser_input, error_sink)
-            if error_sink.has_error:
-                raise FailedToCompileError(error_sink.sink)
-
-            if isinstance(parsed, VariableNode):
-                var_name = self.init_variable_if_needed(parsed, conditions)
-                if parsed.attributes_str is not None:
-                    conditions.append(Condition(var_name, parsed.attributes_str, True))
-
-            elif isinstance(parsed, ComparisonNode):
-                if isinstance(parsed.left, VariableNode):
-                    left = self.init_variable_if_needed(parsed.left, conditions)
-                    attr = parsed.left.attributes_str or "__self__"
-                    right = eval(get_text_from_tokens(parsed.right.tokens))
-                    conditions.append(Condition(left, attr, right))
-
-        return [AndConditions(conditions)]
+from core.tokenizer import TokenKind
+from parsers.BaseExpressionParser import AndNode, BaseExpressionParser, LeftPartNotFoundError, NotNode, \
+    OrNode, ParenthesisNode
+from parsers.BaseParser import UnexpectedEofParsingError
 
 
 class LogicalOperatorParser(BaseExpressionParser):
@@ -70,9 +17,6 @@ class LogicalOperatorParser(BaseExpressionParser):
 
     def __init__(self, **kwargs):
         super().__init__(self.NAME, 50, False, yield_eof=True)
-        self.and_tokens = list(Tokenizer(" and ", yield_eof=False))
-        self.and_not_tokens = list(Tokenizer(" and not ", yield_eof=False))
-        self.not_tokens = list(Tokenizer("not ", yield_eof=False))
         self.expr_parser = kwargs.get("expr_parser", None)
 
     @staticmethod
@@ -87,16 +31,20 @@ class LogicalOperatorParser(BaseExpressionParser):
     def parse_or(self, context, parser_input, error_sink):
         start = parser_input.pos
         expr = self.parse_and(context, parser_input, error_sink)
+
         token = parser_input.token
         if token.type != TokenKind.IDENTIFIER or token.value != "or":
             return expr
 
+        if expr is None:
+            error_sink.add_error(LeftPartNotFoundError("or", start))
+
         parts = [expr]
         while token.type == TokenKind.IDENTIFIER and token.value == "or":
             parser_input.next_token()
             expr = self.parse_and(context, parser_input, error_sink)
             if expr is None:
-                error_sink.add_error(UnexpectedEofParsingError("When parsing 'or'"))
+                error_sink.add_error(UnexpectedEofParsingError("while parsing 'or'"))
                 end = parser_input.pos
                 self.clean_parenthesis_nodes(parts)
                 return OrNode(start, end, parser_input.tokens[start: end + 1], *parts)
@@ -110,16 +58,20 @@ class LogicalOperatorParser(BaseExpressionParser):
     def parse_and(self, context, parser_input, error_sink):
         start = parser_input.pos
         expr = self.parse_not(context, parser_input, error_sink)
+
         token = parser_input.token
         if token.type != TokenKind.IDENTIFIER or token.value != "and":
             return expr
 
+        if expr is None:
+            error_sink.add_error(LeftPartNotFoundError("and", start))
+
         parts = [expr]
         while token.type == TokenKind.IDENTIFIER and token.value == "and":
             parser_input.next_token()
             expr = self.parse_not(context, parser_input, error_sink)
             if expr is None:
-                error_sink.add_error(UnexpectedEofParsingError("When parsing 'and'"))
+                error_sink.add_error(UnexpectedEofParsingError("while parsing 'and'"))
                 end = parser_input.pos
                 self.clean_parenthesis_nodes(parts)
                 return AndNode(start, end, parser_input.tokens[start: end + 1], *parts)
@@ -143,7 +95,11 @@ class LogicalOperatorParser(BaseExpressionParser):
                            parser_input.tokens[start: parsed.end + 1],
                            node)
         else:
-            return self.parse_tokens(context, parser_input, error_sink)
+            return self.parse_tokens(context,
+                                     parser_input,
+                                     error_sink,
+                                     self.parse_tokens_stop_condition,
+                                     self.expr_parser)
 
     def parse_tokens_stop_condition(self, token, parser_input):
         return token.type == TokenKind.IDENTIFIER and token.value in ("and", "or") or \

src/parsers/RelationalOperatorParser.py

@@ -21,14 +21,14 @@ class RelationalOperatorParser(BaseExpressionParser):
 
     def parse_compare(self, context, parser_input, error_sink):
         start = parser_input.pos
-        left = self.parse_tokens(context, parser_input, error_sink)
+        left = self.parse_tokens(context, parser_input, error_sink, self.parse_tokens_stop_condition, self.expr_parser)
         if left is None:
             return None
 
         if (comp := self.eat_comparison(parser_input)) is None:
             return left
 
-        right = self.parse_tokens(context, parser_input, error_sink)
+        right = self.parse_tokens(context, parser_input, error_sink, self.parse_tokens_stop_condition, self.expr_parser)
 
         if comp == ComparisonType.IN and not isinstance(right, ParenthesisNode):
             t = right.tokens[0]

src/parsers/SequenceNodeParser.py

@@ -1,15 +1,15 @@
 from dataclasses import dataclass
 
-from core import builtin_helpers
 from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import update_concepts_hints
-from core.concept import DEFINITION_TYPE_BNF, Concept
+from core.builtin_helpers import debug_nodes, update_concepts_hints
+from core.concept import Concept, DEFINITION_TYPE_BNF
 from core.global_symbols import NotFound
 from core.sheerka.services.SheerkaExecute import ParserInput
-from core.tokenizer import Tokenizer, TokenKind
-from core.utils import strip_tokens, make_unique
-from parsers.BaseNodeParser import BaseNodeParser, ConceptNode, UnrecognizedTokensNode, SourceCodeNode
-from parsers.BaseParser import UnexpectedTokenParsingError, ParsingError
+from core.tokenizer import TokenKind, Tokenizer
+from core.utils import make_unique, strip_tokens
+from parsers.BaseNodeParser import BaseNodeParser, ConceptNode, SourceCodeNode, UnrecognizedTokensCache, \
+    UnrecognizedTokensNode
+from parsers.BaseParser import ParsingError, UnexpectedTokenParsingError
 from parsers.BnfNodeParser import BnfNodeParser
 from parsers.SyaNodeParser import SyaNodeParser
 
@@ -47,9 +47,10 @@ class TokensNodeFoundError(ParsingError):
 
 
 class AtomConceptParserHelper:
-    def __init__(self, context):
+    def __init__(self, parser):
 
-        self.context = context
+        self.parser = parser
+        self.context = parser.context
         self.debug = []
         self.sequence = []  # sequence of concepts already found found
         self.current_concept: ConceptNode = None  # concept being parsed
@@ -80,7 +81,7 @@ class AtomConceptParserHelper:
         return hash(len(self.sequence))
 
     def __repr__(self):
-        return f"{self.sequence}"
+        return f"{debug_nodes(self.sequence)}"
 
     def lock(self):
         self.is_locked = True
@@ -146,10 +147,7 @@ class AtomConceptParserHelper:
         self.unrecognized_tokens.fix_source()
 
         # try to recognize concepts
-        nodes_sequences = builtin_helpers.get_lexer_nodes_from_unrecognized(
-            self.context,
-            self.unrecognized_tokens,
-            PARSERS)
+        nodes_sequences = self.parser.cache.get_lexer_nodes_from_unrecognized(self.context, self.unrecognized_tokens)
 
         if nodes_sequences:
             instances = [self]
@@ -191,7 +189,7 @@ class AtomConceptParserHelper:
             self.errors.append(TokensNodeFoundError(self.expected_tokens))
 
     def clone(self):
-        clone = AtomConceptParserHelper(self.context)
+        clone = AtomConceptParserHelper(self.parser)
         clone.debug = self.debug[:]
         clone.sequence = self.sequence[:]
         clone.current_concept = self.current_concept.clone() if self.current_concept else None
@@ -224,6 +222,7 @@ class SequenceNodeParser(BaseNodeParser):
 
     def __init__(self, **kwargs):
         super().__init__(SequenceNodeParser.NAME, 50, **kwargs)
+        self.cache = UnrecognizedTokensCache(PARSERS)
 
     @staticmethod
     def _is_eligible(concept):
@@ -278,7 +277,7 @@ class SequenceNodeParser(BaseNodeParser):
                 concept_parser_helpers.extend(forked)
                 forked.clear()
 
-        concept_parser_helpers = [AtomConceptParserHelper(self.context)]
+        concept_parser_helpers = [AtomConceptParserHelper(self)]
 
         while self.parser_input.next_token(False):
             for concept_parser in concept_parser_helpers:
@@ -355,7 +354,7 @@ class SequenceNodeParser(BaseNodeParser):
         res = []
         start, end = self.get_tokens_boundaries(self.parser_input.as_tokens())
         for concept in concepts:
-            parser_helper = AtomConceptParserHelper(None)
+            parser_helper = AtomConceptParserHelper(self)
             parser_helper.sequence.append(ConceptNode(concept,
                                                       start,
                                                       end,
@@ -419,6 +418,9 @@ class SequenceNodeParser(BaseNodeParser):
                 False,
                 context.sheerka.new(BuiltinConcepts.ERROR, body=self.error_sink))
 
+        debugger = context.get_debugger(self.NAME, "parse")
+        debugger.debug_entering(source=self.parser_input.as_text())
+
         sequences = self.get_concepts_sequences()
         if by_name := self.get_by_name():
             # note that concepts by names must be appended, not prepended
@@ -427,6 +429,10 @@ class SequenceNodeParser(BaseNodeParser):
 
         parser_helpers = self.get_valid(sequences)
 
+        if debugger.is_enabled():
+            debugger.debug_var("stats", self.cache.to_dict())
+            debugger.debug_leaving(result=parser_helpers)
+
         if len(parser_helpers):
             ret = []
             for parser_helper in parser_helpers:
@@ -471,9 +477,19 @@ class SequenceNodeParser(BaseNodeParser):
         if not eligible:
             return None
 
-        return [self.sheerka.new_dynamic(c, BuiltinConcepts.PLURAL, name=token.value, props={BuiltinConcepts.PLURAL: c})
+        plural_concepts = [self.sheerka.new_dynamic(c,
+                                                    BuiltinConcepts.PLURAL,
+                                                    name=token.value,
+                                                    props={BuiltinConcepts.PLURAL: c})
                            for c in concepts]
 
+        for concept in plural_concepts:
+            underlying_concept = concept.get_prop(BuiltinConcepts.PLURAL)
+            if self.sheerka.isaset(self.context, underlying_concept):
+                concept.get_metadata().body = f"get_set_elements(c:|{underlying_concept.id}:)"
+
+        return plural_concepts
+
     @staticmethod
     def as_list(obj):
         if obj is None:

src/parsers/UnrecognizedNodeParser.py

@@ -2,9 +2,9 @@ from dataclasses import dataclass
 
 import core.utils
 from core.builtin_concepts import BuiltinConcepts
-from core.builtin_helpers import only_successful, get_lexer_nodes, update_compiled
-from parsers.BaseNodeParser import ConceptNode, UnrecognizedTokensNode, SourceCodeNode, SourceCodeWithConceptNode
-from parsers.BaseParser import BaseParser, ParsingError, BaseParserInputParser
+from core.builtin_helpers import get_lexer_nodes, only_successful, update_compiled
+from parsers.BaseNodeParser import ConceptNode, SourceCodeNode, SourceCodeWithConceptNode, UnrecognizedTokensNode
+from parsers.BaseParser import BaseParserInputParser, ParsingError
 from parsers.BnfNodeParser import BnfNodeParser
 from parsers.PythonParser import PythonParser
 from parsers.SequenceNodeParser import SequenceNodeParser

src/sheerkapython/ExprToPython.py

@@ -0,0 +1,389 @@
+from dataclasses import dataclass
+from itertools import product
+from typing import Union
+
+from core.builtin_helpers import is_only_successful, only_successful
+from core.global_symbols import INIT_AST_PARSERS, NotInit
+from core.sheerka.services.SheerkaEvaluateConcept import EvaluationHints
+from core.tokenizer import TokenKind
+from core.utils import merge_dicts, merge_sets
+from parsers.BaseExpressionParser import AndNode, ComparisonNode, ComparisonType, ExprNode, ExpressionVisitorWithHint, \
+    FunctionNode, ListComprehensionNode, \
+    ListNode, NameExprNode, NotNode, VariableNode, end_parenthesis_mapping, open_parenthesis_mapping
+from parsers.PythonParser import PythonNode
+from sheerkapython.python_wrapper import sheerka_globals
+
+
+@dataclass()
+class PythonExprVisitorObj:
+    text: Union[str, None]  # human readable
+    source: Union[str, None]  # python expression to compile
+    objects: dict  # dictionaries of object created during the visit
+    variables: set  # I intended to detect unbound symbols, but it's actually not used
+
+
+class PythonExprVisitor(ExpressionVisitorWithHint):
+    def __init__(self, context, obj_counter=0):
+        self.context = context
+        self.obj_counter = obj_counter
+        self.objects_by_id = {}
+        self.objects_by_name = {}
+        self.errors = {}
+        self.results = []
+
+    def compile(self, expr_node, hint=None):
+        hint = hint or EvaluationHints(eval_body=True)
+        visitor_objects = self.visit(expr_node, hint)
+
+        for obj in visitor_objects:
+
+            ret = self.context.sheerka.parse_python(self.context, obj.source)
+            if ret.status:
+                ret.body.body.original_source = obj.text
+                ret.body.body.objects = obj.objects
+                self.results.append(ret)
+            else:
+                self.errors[obj.text] = self.context.sheerka.get_error_cause(ret.body)
+
+        return self.results
+
+    def visit_ListComprehensionNode(self, expr_node: ListComprehensionNode, hint: EvaluationHints):
+        """
+
+        :param expr_node:
+        :param hint:
+        :return:
+        """
+        visitor_objects = []
+        source = expr_node.get_source()
+
+        not_a_question_hint = EvaluationHints(eval_body=True, eval_question=False)
+        is_a_question_hint = EvaluationHints(eval_body=True, eval_question=True)
+        product_inputs = []
+
+        # add parenthesis around the element if needed
+        # test case test_ExprToPython.test_i_can_compile_when_element_is_missing_its_parenthesis()
+        if expr_node.element.first is None and len(expr_node.element.items) > 1:
+            expr_node.element.first = NameExprNode(-1, -1, [open_parenthesis_mapping[TokenKind.LPAR]])
+            expr_node.element.last = NameExprNode(-1, -1, [end_parenthesis_mapping[TokenKind.LPAR]])
+
+        element_objs = self.visit(expr_node.element, not_a_question_hint)
+        product_inputs.append(element_objs)
+        for comp in expr_node.generators:
+            target_objs = self.visit(comp.target, not_a_question_hint)
+            iter_objs = self.visit(comp.iterable, not_a_question_hint)
+            if comp.if_expr:
+                # parse it using PythonConditionExprVisitor
+                res = self.context.sheerka.parse_expression(self.context, comp.if_expr.get_source())
+                if not res.status:
+                    self.errors[comp.if_expr.get_source()] = res.body
+                    return None
+                if_expr_objs = self.visit(res.body.body, is_a_question_hint)
+            else:
+                if_expr_objs = [None]
+
+            product_inputs.extend([target_objs, iter_objs, if_expr_objs])
+
+        for items in product(*product_inputs):
+            visitor_objects.append(self.create_list_comprehension(source, *items))
+
+        return visitor_objects
+
+    def visit_VariableNode(self, expr_node: VariableNode, hint: EvaluationHints):
+        source = expr_node.get_source()
+        return self.parse_source_code(source, hint)
+
+    def visit_NameExprNode(self, expr_node: NameExprNode, hint: EvaluationHints):
+        """
+        create visitor objects from NameExprNode
+        :param expr_node:
+        :param hint:
+        :return:
+        """
+        source = expr_node.get_source()
+        return self.parse_source_code(source, hint)
+
+    def visit_ListNode(self, expr_node: ListNode, hint: EvaluationHints):
+        visitor_objects = []
+        source = expr_node.get_source()
+
+        items_objs = []
+        for item in expr_node.items:
+            items_objs.append(self.visit(item, hint))
+
+        for items in product(*items_objs):
+            visitor_objects.append(self.create_list(source,
+                                                    expr_node.first.get_source() if expr_node.first else None,
+                                                    expr_node.last.get_source() if expr_node.last else None,
+                                                    items,
+                                                    expr_node.sep))
+        return visitor_objects
+
+    def visit_AndNode(self, expr_node: AndNode, hint: EvaluationHints):
+        """
+
+        :param expr_node:
+        :param hint:
+        :return:
+        """
+        return self.visit_or_or_and_node("and", expr_node, hint)
+
+    def visit_OrNode(self, expr_node: AndNode, hint: EvaluationHints):
+        """
+
+        :param expr_node:
+        :param hint:
+        :return:
+        """
+        return self.visit_or_or_and_node("or", expr_node, hint)
+
+    def visit_NotNode(self, expr_node: NotNode, hint: EvaluationHints):
+        """
+
+        :param expr_node:
+        :param hint:
+        :return:
+        """
+        visitor_objects = []
+
+        source = expr_node.get_source()
+        objs = self.visit(expr_node.node, hint)
+
+        for obj in objs:
+            visitor_objects.append(self.create_not(source, obj))
+
+        return visitor_objects
+
+    def visit_ComparisonNode(self, expr_node: ComparisonNode, hint: EvaluationHints):
+        """
+
+        :param expr_node:
+        :param hint:
+        :return:
+        """
+        visitor_objects = []
+        source = expr_node.get_source()
+
+        left = self.visit(expr_node.left, hint)
+        right = self.visit(expr_node.right, hint)
+
+        for left_obj, right_obj in product(left, right):
+            visitor_objects.append(self.create_comparison(source, expr_node.comp, left_obj, right_obj))
+
+        return visitor_objects
+
+    def visit_FunctionNode(self, expr_node: FunctionNode, hint: EvaluationHints):
+        visitor_objects = []
+        source = expr_node.get_source()
+
+        parameters_objects = []
+        for parameter in expr_node.parameters:
+            parameters_objects.append(self.visit(parameter.value, hint))
+
+        for parameters in product(*parameters_objects):
+            visitor_objects.append(self.create_function(source,
+                                                        expr_node.first.get_source(),
+                                                        expr_node.last.get_source(),
+                                                        parameters))
+
+        return visitor_objects
+
+    def visit_or_or_and_node(self, node_type, expr_node: ExprNode, hint: EvaluationHints):
+        """
+
+        :param node_type:
+        :param expr_node:
+        :param hint:
+        :return:
+        """
+        visitor_objects = []
+
+        source = expr_node.get_source()
+        objs = []
+        for node in expr_node.parts:
+            objs.append(self.visit(node, hint))
+
+        for objs_parts in product(*objs):
+            visitor_objects.append(self.create_and_or(node_type, source, objs_parts))
+
+        return visitor_objects
+
+    def parse_source_code(self, source, hint):
+        res = self.context.sheerka.parse_unrecognized(self.context,
+                                                      source,
+                                                      INIT_AST_PARSERS,
+                                                      filter_func=only_successful,
+                                                      is_question=hint.eval_question)
+
+        return_values = res.body.body if is_only_successful(self.context.sheerka, res) else [res]
+
+        visitor_objects = []
+        for ret_val in return_values:
+            if not ret_val.status:
+                self.errors[source] = ret_val.body
+                return
+
+            if isinstance(ret_val.body.body, list):
+                if len(ret_val.body.body) > 1:
+                    raise NotImplementedError("Too many concept found. Not handled yet !")
+                body = ret_val.body.body[0]
+            else:
+                body = ret_val.body.body
+
+            if hasattr(body, "get_concept"):
+                visitor_objects.append(self.create_call_concept(source, body.get_concept(), hint.eval_question))
+            elif hasattr(body, "get_python_node"):
+                visitor_objects.append(self.create_source_code_from_python_node(body.get_python_node()))
+            else:
+                raise NotImplementedError(f"{body=}. Not yet implemented")
+
+        return visitor_objects
+
+    @staticmethod
+    def create_source_code_from_python_node(node: PythonNode):
+        return PythonExprVisitorObj(text=node.original_source,
+                                    source=node.source,
+                                    objects=node.objects,
+                                    variables=set())
+
+    @staticmethod
+    def create_list_comprehension(text, *items):
+        objects = {}
+        variables = set()
+
+        def update_objects_and_variables(*objs):
+            for obj in [obj for obj in objs if obj]:
+                objects.update(obj.objects)
+                variables.update(obj.variables)
+
+        items = list(items)
+
+        element = items.pop(0)
+        update_objects_and_variables(element)
+        source = f"[ {element.source}"
+
+        while len(items):
+            target = items.pop(0)
+            iterable = items.pop(0)
+            if_expr = items.pop(0)
+            update_objects_and_variables(target, iterable, if_expr)
+            source += f" for {target.source} in {iterable.source}"
+            if if_expr:
+                source += f" if {if_expr.source}"
+        source += " ]"
+
+        return PythonExprVisitorObj(text=text,
+                                    source=source,
+                                    objects=objects,
+                                    variables=variables)
+
+    @staticmethod
+    def create_and_or(node_type, text, parts):
+        return PythonExprVisitorObj(text=text,
+                                    source=f" {node_type} ".join([p.source for p in parts]),
+                                    objects=merge_dicts(*[p.objects for p in parts]),
+                                    variables=merge_sets(*[p.variables for p in parts]))
+
+    @staticmethod
+    def create_comparison(text, op, left_obj, right_obj):
+        def get_source(_op, a, b):
+            if _op == ComparisonType.EQUALS and b == "sheerka":
+                return f"is_sheerka({a})"
+            else:
+                return ComparisonNode.rebuild_source(a, op, b)
+
+        return PythonExprVisitorObj(text=text,
+                                    source=get_source(op, left_obj.source, right_obj.source),
+                                    objects=merge_dicts(left_obj.objects, right_obj.objects),
+                                    variables=merge_sets(left_obj.variables, right_obj.variables))
+
+    @staticmethod
+    def create_not(text, node):
+        return PythonExprVisitorObj(text=text,
+                                    source=f"not {node.source}",
+                                    objects=node.objects,
+                                    variables=node.variables)
+
+    @staticmethod
+    def create_function(text, first, last, parameters):
+        def get_source(_first, _last, _parameters):
+            return f"{_first}{', '.join(p for p in _parameters)}{_last}"
+
+        return PythonExprVisitorObj(text=text,
+                                    source=get_source(first, last, [p.source for p in parameters]),
+                                    objects=merge_dicts(*[p.objects for p in parameters]),
+                                    variables=merge_sets(*[p.variables for p in parameters]))
+
+    @staticmethod
+    def create_list(text, first, last, items, sep):
+        def get_source(_first, _last, _items, _sep):
+            res = _first or ""
+            res += f"{_sep.value} ".join(item for item in _items)
+            if _last:
+                res += _last
+            return res
+
+        return PythonExprVisitorObj(text=text,
+                                    source=get_source(first, last, [p.source for p in items], sep),
+                                    objects=merge_dicts(*[p.objects for p in items]),
+                                    variables=merge_sets(*[p.variables for p in items]))
+
+    def get_object_name(self, obj):
+        """
+        object found during the parsing are not serialized
+        They are kept in a dictionary.
+        This function returns a new name for every new object
+        :param obj: object for which a name is to be created
+        :param objects: already created names (it's a dictionary)
+        :return: tuple(name created, dictionary of already created names)
+        """
+
+        if self.context.sheerka.is_sheerka(obj):
+            return "sheerka"
+
+        try:
+            return self.objects_by_id[id(obj)]
+        except KeyError:
+            pass
+
+        object_name = f"__o_{self.obj_counter:02}__"
+        self.obj_counter += 1
+
+        self.objects_by_id[id(obj)] = object_name
+        self.objects_by_name[object_name] = obj
+        return object_name
+
+    def create_call_concept(self, source, concept, is_question):
+        name = self.get_object_name(concept)
+        parameters = {}
+
+        for var_name, default_value in concept.get_metadata().variables:
+            if var_name not in concept.get_metadata().parameters:
+                continue
+            parameters[var_name] = default_value if default_value is not NotInit else var_name
+
+        function_to_call = "evaluate_question" if is_question else "call_concept"
+        to_compile = f"{function_to_call}({name}"
+        for p_name, p_value in parameters.items():
+            to_compile += f", {p_name}={p_value}"
+        to_compile += ")"
+
+        concept.get_hints().use_copy = True
+        concept.get_hints().is_evaluated = True
+        return PythonExprVisitorObj(source, to_compile, {name: concept}, set())
+
+    def is_a_possible_variable(self, name):
+        """
+        tells whether or not the name can be a variable
+        :param name:
+        :return:
+        """
+        if self.context.sheerka.is_a_concept_name(name):
+            return False
+
+        try:
+            eval(name, sheerka_globals)
+        except:
+            return True
+
+        return False

tests/BaseTest.py

@@ -1,12 +1,13 @@
 import ast
 from dataclasses import dataclass, field
 
-from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
+from core.builtin_concepts import BuiltinConcepts, ParserResultConcept, ReturnValueConcept
 from core.concept import Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, freeze_concept_attrs
-from core.rule import Rule, ACTION_TYPE_PRINT, ACTION_TYPE_EXEC
+from core.rule import ACTION_TYPE_EXEC, ACTION_TYPE_PRINT, Rule
 from core.sheerka.ExecutionContext import ExecutionContext
 from core.sheerka.Sheerka import Sheerka
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
+from core.sheerka.services.SheerkaDebugManager import ListDebugLogger
 from core.sheerka.services.SheerkaRuleManager import SheerkaRuleManager
 from parsers.BnfDefinitionParser import BnfDefinitionParser
 from parsers.BnfNodeParser import StrMatch
@@ -290,3 +291,10 @@ class BaseTest:
     @staticmethod
     def successful_return_values(return_values):
         return [ret_val for ret_val in return_values if ret_val.status]
+
+    @staticmethod
+    def activate_debug(context, pattern="Sya.*.*"):
+        sheerka = context.sheerka
+        sheerka.set_debug(context, True)
+        sheerka.set_debug_var(context, pattern)
+        sheerka.set_debug_logger_definition(ListDebugLogger)

tests/core/test_SheerkaDebugManager.py

@@ -73,6 +73,7 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
 
         item_container = f"debug_{item_type}_settings"
         assert getattr(service, item_container) == [DebugItem(
+            item_type,
             item,
             service_name,
             method_name,
@@ -89,7 +90,7 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
 
         service.add_or_update_debug_item(context, "vars", item="item")
         assert service.debug_vars_settings == [
-            DebugItem("item", None, None, None, False, None, False, True)
+            DebugItem("vars", "item", None, None, None, False, None, False, True)
         ]
 
     def test_i_can_update_debug_item(self):
@@ -101,8 +102,8 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
         service.add_or_update_debug_item(context, "vars", "item", "service_name", "method_name", enabled=False)
 
         assert service.debug_vars_settings == [
-            DebugItem("item", "service_name", "method_name", None, False, None, False, False),
-            DebugItem("item2", "service_name", "method_name", None, False, None, False, True),
+            DebugItem("vars", "item", "service_name", "method_name", None, False, None, False, False),
+            DebugItem("vars", "item2", "service_name", "method_name", None, False, None, False, True),
         ]
 
     @pytest.mark.parametrize("settings, expected", [
@@ -407,7 +408,7 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
 
         sheerka.set_debug_var(context, "s.m.v", "1+", 10, variable="my_var")
         assert service.debug_vars_settings == [
-            DebugItem("my_var", "s", "m", 1, True, 10, False, True)
+            DebugItem("vars", "my_var", "s", "m", 1, True, 10, False, True)
         ]
         assert service.debug_concepts_settings == []
         assert service.debug_rules_settings == []
@@ -418,7 +419,7 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
 
         sheerka.set_debug_rule(context, "s.m.v", "1+", 10, rule="my_rule")
         assert service.debug_rules_settings == [
-            DebugItem("my_rule", "s", "m", 1, True, 10, False, True)
+            DebugItem("rules", "my_rule", "s", "m", 1, True, 10, False, True)
         ]
         assert service.debug_concepts_settings == []
         assert service.debug_vars_settings == []
@@ -429,7 +430,7 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
 
         sheerka.set_debug_concept(context, "s.m.v", "1+", 10, concept="my_concept")
         assert service.debug_concepts_settings == [
-            DebugItem("my_concept", "s", "m", 1, True, 10, False, True)
+            DebugItem("concepts", "my_concept", "s", "m", 1, True, 10, False, True)
         ]
         assert service.debug_rules_settings == []
         assert service.debug_vars_settings == []
@@ -449,11 +450,11 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
 
         assert another_service.activated
         assert another_service.debug_vars_settings == [
-            DebugItem('var', 'service_name', None, None, False, None, False, True)]
+            DebugItem("vars", 'var', 'service_name', None, None, False, None, False, True)]
         assert another_service.debug_rules_settings == [
-            DebugItem('1', None, None, None, False, None, False, True)]
+            DebugItem("rules", '1', None, None, None, False, None, False, True)]
         assert another_service.debug_concepts_settings == [
-            DebugItem('1001', None, None, None, False, None, False, True)]
+            DebugItem("concepts", '1001', None, None, None, False, None, False, True)]
 
     def test_i_can_inspect_concept_all_attributes(self):
         sheerka, context, foo = self.init_concepts("foo")
@@ -766,9 +767,12 @@ class TestSheerkaDebugManager(TestUsingMemoryBasedSheerka):
 
         sheerka.pop_ontology(context)
         assert service.activated
-        assert service.debug_vars_settings == [DebugItem("v_name", "v_service", "v_method", 1, True, 1, False, True)]
-        assert service.debug_rules_settings == [DebugItem("r_name", "r_service", "r_method", 2, True, 2, False, True)]
-        assert service.debug_concepts_settings == [DebugItem("c_name", "c_serv", "c_method", 3, True, 3, False, True)]
+        assert service.debug_vars_settings == [
+            DebugItem("vars", "v_name", "v_service", "v_method", 1, True, 1, False, True)]
+        assert service.debug_rules_settings == [
+            DebugItem("rules", "r_name", "r_service", "r_method", 2, True, 2, False, True)]
+        assert service.debug_concepts_settings == [
+            DebugItem("concepts", "c_name", "c_serv", "c_method", 3, True, 3, False, True)]
 
     def test_i_can_register_debug_item(self):
         sheerka, context = self.init_concepts()

tests/core/test_SheerkaEvaluateConcept.py

@@ -4,7 +4,7 @@ from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserRes
 from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved, \
     DEFINITION_TYPE_DEF
 from core.global_symbols import NotInit, NotFound
-from core.sheerka.services.SheerkaEvaluateConcept import SheerkaEvaluateConcept
+from core.sheerka.services.SheerkaEvaluateConcept import EvaluationHints, SheerkaEvaluateConcept
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.sheerka.services.SheerkaMemory import SheerkaMemory
 from parsers.BaseParser import BaseParser
@@ -13,7 +13,7 @@ from parsers.ExpressionParser import ExpressionParser
 from parsers.PythonParser import PythonNode, PythonParser
 from parsers.SyaNodeParser import SyaNodeParser
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-from tests.evaluators.EvaluatorTestsUtils import pr_ret_val, python_ret_val
+from tests.evaluators.EvaluatorTestsUtils import exact, pr_ret_val, python_ret_val
 from tests.parsers.parsers_utils import CB, compare_with_test_object
 
 
@@ -487,7 +487,9 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
             eval_where=True,
         )
 
-        evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, True), concept, eval_body=False)
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka, False, True),
+                                             concept,
+                                             hints=EvaluationHints(eval_body=False))
 
         if expected:
             assert evaluated.key == concept.key
@@ -532,13 +534,13 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         )
 
         foo_instance = sheerka.new("foo")
-        evaluated = sheerka.evaluate_concept(context, foo_instance, eval_body=False)
+        evaluated = sheerka.evaluate_concept(context, foo_instance, hints=EvaluationHints(eval_body=False))
 
         assert ConceptParts.BODY in evaluated.get_compiled()
         assert evaluated.body == NotInit
         assert not evaluated.get_hints().is_evaluated
 
-        evaluated = sheerka.evaluate_concept(context, foo_instance, eval_body=True)  # evaluate the body this time
+        evaluated = sheerka.evaluate_concept(context, foo_instance, hints=EvaluationHints(eval_body=True))  # evaluate the body this time
         assert isinstance(evaluated.body, bool) and evaluated.body
 
     def test_i_can_apply_intermediate_where_condition_using_python(self):
@@ -811,7 +813,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
             Concept("bar", pre="print('10')"),  # print won't be executed
         )
 
-        evaluated = sheerka.evaluate_concept(context, foo, eval_body=True)
+        evaluated = sheerka.evaluate_concept(context, foo, hints=EvaluationHints(eval_body=True))
         captured = capsys.readouterr()
         assert evaluated.key == foo.key
         assert captured.out == "10\n"
@@ -828,7 +830,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
             Concept("foo", pre="in_context('foo')", body="print('10')"),
         )
 
-        evaluated = sheerka.evaluate_concept(context, concept, eval_body=True)
+        evaluated = sheerka.evaluate_concept(context, concept, hints=EvaluationHints(eval_body=True))
         assert sheerka.isinstance(evaluated, BuiltinConcepts.CONDITION_FAILED)
         assert evaluated.body == "in_context('foo')"
         assert evaluated.concept == concept
@@ -898,7 +900,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
     def test_is_evaluated_is_correctly_set(self, concept, expected):
         sheerka, context, concept = self.init_concepts(concept)
 
-        evaluated = sheerka.evaluate_concept(context, concept, eval_body=True)
+        evaluated = sheerka.evaluate_concept(context, concept, hints=EvaluationHints(eval_body=True))
 
         assert evaluated.key == concept.key
         assert concept.get_hints().is_evaluated == expected
@@ -919,24 +921,24 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         res = sheerka.evaluate_concept(context, bar)
         assert sheerka.isinstance(res, "bar")
 
-        res = sheerka.evaluate_concept(context, bar, eval_body=True)
+        res = sheerka.evaluate_concept(context, bar, hints=EvaluationHints(eval_body=True))
         assert sheerka.isinstance(res, "foo")
 
         # And the result is still the same after a second call
         assert bar.get_hints().is_evaluated
-        res = sheerka.evaluate_concept(context, bar, eval_body=True)
+        res = sheerka.evaluate_concept(context, bar, hints=EvaluationHints(eval_body=True))
         assert sheerka.isinstance(res, "foo")
 
     def test_ret_is_evaluated_only_is_body_is_requested(self):
         sheerka, context, foo, bar = self.init_concepts("foo", Concept("bar", ret="__NOT_FOUND"))
 
-        res = sheerka.evaluate_concept(context, bar, eval_body=False)
+        res = sheerka.evaluate_concept(context, bar, hints=EvaluationHints(eval_body=False))
         assert res.id == bar.id
 
     def test_i_can_eval_concept_with_rules(self):
         sheerka, context, foo = self.init_concepts(Concept("foo a", body="a.name").def_var("a", "r:|1:"))
 
-        res = sheerka.evaluate_concept(context, foo, eval_body=True)
+        res = sheerka.evaluate_concept(context, foo, hints=EvaluationHints(eval_body=True))
         assert res.body == "Print return values"
 
     def test_i_can_manage_python_concept_infinite_recursion_when_initializing_ast(self):
@@ -957,18 +959,18 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluator = SheerkaEvaluateConcept(sheerka)
 
         # 'def concept foo as foo'
-        return_values = [pr_ret_val(foo, parser="ExactConcept"), python_ret_val("foo")]
+        return_values = [pr_ret_val(foo, parser=exact), python_ret_val("foo")]
 
         res = evaluator.get_recursive_definitions(context, foo, return_values)
 
-        assert list(res) == [BaseParser.get_name("ExactConcept")]
+        assert list(r.name for r in res) == [BaseParser.get_name("ExactConcept")]
 
     def test_i_can_detect_when_no_recursive_definition(self):
         sheerka, context, foo, bar = self.init_concepts("foo", "bar")
         evaluator = SheerkaEvaluateConcept(sheerka)
 
         # 'def concept foo as bar'
-        return_values = [pr_ret_val(bar, parser="ExactConcept"), python_ret_val("foo")]
+        return_values = [pr_ret_val(bar, parser=exact), python_ret_val("foo")]
 
         res = evaluator.get_recursive_definitions(context, foo, return_values)
 
@@ -980,7 +982,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         evaluator = SheerkaEvaluateConcept(sheerka)
 
         # i dunno how to construct the return value
-        return_values = [pr_ret_val(q, parser="ExactConcept")]
+        return_values = [pr_ret_val(q, parser=exact)]
 
         res = evaluator.get_recursive_definitions(context, q, return_values)
 
@@ -1004,7 +1006,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         assert evaluated.get_compiled()["y"][0].body.body.get_hints().use_copy
 
         # get the body
-        evaluated = evaluator.evaluate_concept(context, concept, eval_body=True)
+        evaluated = evaluator.evaluate_concept(context, concept, hints=EvaluationHints(eval_body=True))
         assert evaluated.get_compiled()["x"][0].body.body.get_hints().use_copy
         assert evaluated.get_compiled()["y"][0].body.body.get_hints().use_copy
         assert not evaluated.get_value("x").get_hints().use_copy
@@ -1025,7 +1027,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         # get the body
         context.add_to_protected_hints(BuiltinConcepts.EVAL_QUESTION_REQUESTED)
-        evaluated = evaluator.evaluate_concept(context, concept, eval_body=True)
+        evaluated = evaluator.evaluate_concept(context, concept, hints=EvaluationHints(eval_body=True))
         assert evaluated.get_compiled()["x"][0].body.body.get_hints().use_copy
         assert evaluated.get_compiled()["y"][0].body.body.get_hints().use_copy
         assert not evaluated.get_value("x").get_hints().use_copy
@@ -1043,7 +1045,9 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         # Sanity check for normal behaviour
         to_evaluate1 = parsed_ret_val.body.body[0].concept.copy()
-        evaluated1 = sheerka.evaluate_concept(context, to_evaluate1, eval_body=True, validation_only=False)
+        evaluated1 = sheerka.evaluate_concept(context,
+                                              to_evaluate1,
+                                              hints=EvaluationHints(eval_body=True, expression_only=False))
 
         assert sheerka.isinstance(evaluated1, shirt)
         assert evaluated1.get_value("body_ax_is_evaluated") == True
@@ -1053,7 +1057,9 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         # check validation_only behaviour
         to_evaluate2 = parsed_ret_val.body.body[0].concept.copy()
-        evaluated2 = sheerka.evaluate_concept(context, to_evaluate2, eval_body=True, validation_only=True)
+        evaluated2 = sheerka.evaluate_concept(context,
+                                              to_evaluate2,
+                                              hints=EvaluationHints(eval_body=True, expression_only=True))
 
         assert sheerka.isinstance(evaluated2, shirt)
         assert evaluated2.get_value("body_ax_is_evaluated") == NotInit
@@ -1073,7 +1079,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         parsed_ret_val = SyaNodeParser().parse(context, ParserInput("a red shirt"))
         to_evaluate = parsed_ret_val.body.body[0].concept
 
-        evaluated = sheerka.evaluate_concept(context, to_evaluate, eval_body=False)
+        evaluated = sheerka.evaluate_concept(context, to_evaluate, hints=EvaluationHints(eval_body=False))
 
         assert sheerka.isinstance(evaluated, a_x)
         assert "x" in evaluated.get_compiled()
@@ -1085,7 +1091,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         # sanity check
         parsed_ret_val = SyaNodeParser().parse(context, ParserInput("a red shirt"))
         to_evaluate = parsed_ret_val.body.body[0].concept
-        evaluated = sheerka.evaluate_concept(context, to_evaluate, eval_body=True)
+        evaluated = sheerka.evaluate_concept(context, to_evaluate, hints=EvaluationHints(eval_body=True))
 
         assert sheerka.isinstance(evaluated, shirt)
         assert evaluated.get_value("body_ax_is_evaluated") == True
@@ -1095,7 +1101,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
     def test_concept_is_not_evaluated_when_method_access_error(self):
         sheerka, context, foo = self.init_concepts(Concept("foo", body="set_attr(self, 'prop_name', 'prop_value')"))
 
-        evaluated = sheerka.evaluate_concept(context, foo, eval_body=True, validation_only=True)
+        evaluated = sheerka.evaluate_concept(context, foo, hints=EvaluationHints(eval_body=True, expression_only=True))
 
         assert sheerka.isinstance(evaluated, foo)
         assert not foo.get_hints().is_evaluated

tests/evaluators/EvaluatorTestsUtils.py

@@ -1,14 +1,22 @@
 import ast
 
-from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts, ParserResultConcept
+from core.builtin_concepts import BuiltinConcepts, ParserResultConcept, ReturnValueConcept
 from core.concept import Concept
 from evaluators.BaseEvaluator import BaseEvaluator
-from parsers.BaseParser import BaseParser
-from parsers.PythonParser import PythonNode
+from parsers.BaseNodeParser import ConceptNode
+from parsers.ExactConceptParser import ExactConceptParser
+from parsers.PythonParser import PythonNode, PythonParser
+from parsers.SequenceNodeParser import SequenceNodeParser
+from parsers.SyaNodeParser import SyaNodeParser
 
 reduced_requested = ReturnValueConcept("Sheerka", True, Concept(name=BuiltinConcepts.REDUCE_REQUESTED,
                                                                 key=BuiltinConcepts.REDUCE_REQUESTED))
 
+sequence = SequenceNodeParser()
+sya = SyaNodeParser()
+exact = ExactConceptParser()
+python = PythonParser()
+
 
 def ret_val(value="value", who="who", status=True):
     """
@@ -21,7 +29,7 @@ def ret_val(value="value", who="who", status=True):
     return ReturnValueConcept(who, status, value)
 
 
-def p_ret_val(value="value", parser="parser", status=True):
+def p_ret_val(value="value", parser=exact, status=True):
     """
     ReturnValueConcept from parser
     :param value:
@@ -29,7 +37,7 @@ def p_ret_val(value="value", parser="parser", status=True):
     :param status:
     :return:
     """
-    return ReturnValueConcept(BaseParser.get_name(parser), status, value)
+    return ReturnValueConcept(parser.name, status, value)
 
 
 def e_ret_val(value="value", evaluator="evaluator", status=True):
@@ -43,7 +51,7 @@ def e_ret_val(value="value", evaluator="evaluator", status=True):
     return ReturnValueConcept(BaseEvaluator.PREFIX + evaluator, status, value)
 
 
-def p_ret_val_false(value="value", parser="parser"):
+def p_ret_val_false(value="value", parser=exact):
     """
     Failed ReturnValueConcept from parser
     :param value:
@@ -53,7 +61,7 @@ def p_ret_val_false(value="value", parser="parser"):
     return p_ret_val(value, parser, status=False)
 
 
-def p_ret_val_true(value="value", parser="parser"):
+def p_ret_val_true(value="value", parser=exact):
     """
     Successful ReturnValueConcept from parser
     :param value:
@@ -63,24 +71,24 @@ def p_ret_val_true(value="value", parser="parser"):
     return p_ret_val(value, parser, status=True)
 
 
-def e_ret_val_false(value="value", parser="parser"):
+def e_ret_val_false(value="value", evaluator="evaluator"):
     """
     Failed ReturnValueConcept from evaluator
     :param value:
-    :param parser:
+    :param evaluator:
     :return:
     """
-    return e_ret_val(value, parser, status=False)
+    return e_ret_val(value, evaluator, status=False)
 
 
-def e_ret_val_true(value="value", parser="parser"):
+def e_ret_val_true(value="value", evaluator="evaluator"):
     """
     Successful ReturnValueConcept from evaluator
     :param value:
-    :param parser:
+    :param evaluator:
     :return:
     """
-    return e_ret_val(value, parser, status=True)
+    return e_ret_val(value, evaluator, status=True)
 
 
 def e_ret_val_new(key, evaluator="evaluator", status=True, **kwargs):
@@ -96,7 +104,7 @@ def e_ret_val_new(key, evaluator="evaluator", status=True, **kwargs):
     return e_ret_val(body, evaluator, status)
 
 
-def pr_ret_val(value, parser="parser", source=None, status=True):
+def pr_ret_val(value, parser=exact, source=None, status=True):
     """
     ParserResult ReturnValue
     eg: ReturnValue with a ParserResult
@@ -107,7 +115,7 @@ def pr_ret_val(value, parser="parser", source=None, status=True):
     :return:
     """
     source = source or (value.name if isinstance(value, Concept) else "source")
-    parser_result = ParserResultConcept(BaseParser.get_name(parser), source=source, value=value)
+    parser_result = ParserResultConcept(parser, source=source, value=value)
     return p_ret_val(value=parser_result, parser=parser, status=status)
 
 
@@ -117,8 +125,14 @@ def python_ret_val(source):
     :param source:
     :return:
     """
-    python_node = PythonNode(source.strip(), ast.parse(source.strip(), f"<source>", 'eval'))
-    return pr_ret_val(python_node, parser="Python", source=source)
+    python_node = PythonNode(source.lstrip(), ast.parse(source.strip(), f"<source>", 'eval'))
+    return pr_ret_val(python_node, parser=python, source=source)
+
+
+def cnode_ret_val(concept, source=None, parser=sya):
+    source = source or concept.name
+    cnode = ConceptNode(concept, 0, 0, source=source)
+    return pr_ret_val([cnode], parser=parser, source=source)
 
 
 def new_concept(key, **kwargs):
@@ -129,3 +143,11 @@ def new_concept(key, **kwargs):
 
     res.get_hints().is_evaluated = True
     return res
+
+
+def new_plural(name):
+    name_stripped_s = name.lstrip("s")
+    single = Concept(name_stripped_s)
+    concept = Concept(key=name, name=name, id=f"{name}:{BuiltinConcepts.PLURAL}", is_builtin=False, is_unique=False)
+    concept.set_prop(BuiltinConcepts.PLURAL, single)
+    return concept

tests/evaluators/test_DefConceptEvaluator.py

@@ -449,6 +449,18 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
 
         assert sheerka.get_property(created_concept, BuiltinConcepts.ISA) == {sheerka.new(BuiltinConcepts.AUTO_EVAL)}
 
+    def test_i_can_eval_when_variable_are_forced(self):
+        sheerka, context = self.init_test().unpack()
+        definition = "def concept foo from [z for x in y] def_var x def_var y def_var z"
+        def_ret_val = DefConceptParser().parse(context, ParserInput(definition))
+        evaluated = DefConceptEvaluator().eval(context, def_ret_val)
+
+        assert evaluated.status
+        assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
+        created_concept = evaluated.body.body
+
+        assert created_concept.get_metadata().parameters == ["z", "x", "y"]
+
     def test_i_cannot_eval_bnf_concept_with_unknown_variable(self):
         # testing MandatoryVariable
         context = self.get_context()

tests/evaluators/test_ResolveMultiplePluralAmbiguityEvaluator.py

@@ -0,0 +1,60 @@
+import pytest
+
+from core.concept import Concept
+from evaluators.ResolveMultiplePluralAmbiguityEvaluator import ResolveMultiplePluralAmbiguityEvaluator
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.evaluators.EvaluatorTestsUtils import cnode_ret_val, exact, new_plural, pr_ret_val, python_ret_val, sequence, \
+    sya
+
+
+class TestResolveMultiplePluralAmbiguityEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("return_values, expected", [
+        ([python_ret_val("numbers"), cnode_ret_val(new_plural("numbers"), parser=sequence)], True),
+        ([python_ret_val("numbers"), cnode_ret_val(new_plural("numbers"), parser=sequence), pr_ret_val("other")], True),
+        ([python_ret_val("numbers"), cnode_ret_val(new_plural("numbers"), parser=sya)], False),
+        ([python_ret_val("numbers"), cnode_ret_val(Concept("numbers"), parser=sequence)], False),
+        ([python_ret_val("numbers"), pr_ret_val(new_plural("numbers"), parser=exact)], False),
+    ])
+    def test_i_can_match(self, return_values, expected):
+        context = self.get_context()
+        assert ResolveMultiplePluralAmbiguityEvaluator().matches(context, return_values) == expected
+
+    def test_i_can_eval_when_nothing_in_memory(self):
+        sheerka, context = self.init_test().unpack()
+
+        return_values = [
+            python_ret_val("numbers"),
+            cnode_ret_val(new_plural("numbers"), source="source", parser=sequence)
+        ]
+
+        evaluator = ResolveMultiplePluralAmbiguityEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        rets = evaluator.eval(context, return_values)
+        assert len(rets) == 1
+        ret = rets[0]
+        assert ret.who == evaluator.name
+        assert ret.status == return_values[1].status
+        assert ret.value == return_values[1].value
+        assert ret.parents == return_values
+
+    def test_i_can_eval_when_plural_in_memory(self):
+        sheerka, context = self.init_test().unpack()
+        sheerka.add_to_memory(context, "numbers", "something")
+
+        return_values = [
+            python_ret_val("numbers"),
+            cnode_ret_val(new_plural("numbers"), source="source", parser=sequence)
+        ]
+
+        evaluator = ResolveMultiplePluralAmbiguityEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        rets = evaluator.eval(context, return_values)
+        assert len(rets) == 1
+        ret = rets[0]
+        assert ret.who == evaluator.name
+        assert ret.status == return_values[0].status
+        assert ret.value == return_values[0].value
+        assert ret.parents == return_values

tests/non_reg/test_sheerka_non_reg2.py

@@ -45,13 +45,35 @@ class TestSheerkaNonRegMemory2(TestUsingMemoryBasedSheerka):
         assert res[0].status
         assert sheerka.isa(sheerka.new("one"), sheerka.new("number"))
 
-    # def test_i_can_define_plural(self):
-    #     init = [
-    #         "def concept man",
-    #         "def concept men as set_plural(man) ret man auto_eval True",
-    #     ]
-    #     sheerka = self.init_scenario(init)
-    #
-    #     res = sheerka.evaluate_user_input("men")
-    #     assert res[0].status
+    def test_i_can_get_sequence_when_evaluation_plural(self):
+        init = [
+            "def concept one",
+            "def concept two",
+            "def concept number",
+            "global_truth(set_isa(one, number))",
+            "global_truth(set_isa(two, number))",
+        ]
+        sheerka = self.init_scenario(init)
 
+        res = sheerka.evaluate_user_input("eval numbers")
+
+        assert res[0].status
+        assert set(res[0].body) == {sheerka.new("one"), sheerka.new("two")}
+
+    def test_i_can_use_list_comprehension(self):
+        init = [
+            "def concept rex",
+            "def concept rantanplan",
+            "def concept dog",
+            "def concept x is a y as set_isa(x, y)",
+        ]
+        sheerka = self.init_scenario(init)
+
+        res = sheerka.evaluate_user_input("global_truth([ x is a dog for x in [rex, rantanplan]])")
+
+        assert len(res) == 1
+        assert res[0].status
+
+        rex = sheerka.new("rex")
+        dog = sheerka.new("dog")
+        assert sheerka.isa(rex, dog)

tests/non_reg/test_sheerka_non_reg_out.py

@@ -180,3 +180,24 @@ two: (1002)two
         captured = capsys.readouterr()
         assert " : test()" in captured.out
         assert " : history()" in captured.out
+
+    def test_i_can_list_debug_settings(self, capsys):
+        init = [
+            "set_debug_var('Sya.parsers.*', 45)",
+            "set_debug_concept('c:|1015', '13+')",
+            "set_debug_rule('Out')",
+        ]
+        sheerka = self.init_scenario(init)
+        capsys.readouterr()
+
+        sheerka.enable_process_return_values = True
+        res = sheerka.evaluate_user_input(f"list_debug_settings()")
+
+        assert len(res) == 1
+        assert res[0].status
+
+        captured = capsys.readouterr()
+        assert captured.out == """DebugItem(type=vars, setting=Sya.parsers.*, context_id=45, debug_id=None, context_children=False, debug_children=False (enabled=True))
+DebugItem(type=concepts, setting=c:|1015.*.*, context_id=13, debug_id=None, context_children=True, debug_children=False (enabled=True))
+DebugItem(type=rules, setting=Out.*.*, context_id=None, debug_id=None, context_children=False, debug_children=False (enabled=True))
+"""

tests/parsers/parsers_utils.py

@@ -1,24 +1,23 @@
 import ast
 from dataclasses import dataclass
-from typing import Union, List
+from typing import List, Union
 
 from core.builtin_concepts import ReturnValueConcept
 from core.builtin_helpers import CreateObjectIdentifiers
-from core.concept import Concept, ConceptParts, DoNotResolve, AllConceptParts
+from core.concept import AllConceptParts, Concept, ConceptParts, DoNotResolve
 from core.rule import Rule
-from core.tokenizer import Tokenizer, TokenKind, Token
-from core.utils import get_text_from_tokens, tokens_index, str_concept
-from parsers.BaseExpressionParser import NameExprNode, AndNode, OrNode, NotNode, VariableNode, ComparisonNode, \
-    ComparisonType, \
-    FunctionParameter
-from parsers.BaseNodeParser import UnrecognizedTokensNode, SourceCodeNode, RuleNode, ConceptNode, \
-    SourceCodeWithConceptNode
+from core.tokenizer import Token, TokenKind, Tokenizer
+from core.utils import get_text_from_tokens, str_concept, tokens_index
+from parsers.BaseExpressionParser import AndNode, ComparisonNode, ComparisonType, Comprehension, FunctionParameter, \
+    ListComprehensionNode, ListNode, NameExprNode, \
+    NotNode, OrNode, VariableNode, comma
+from parsers.BaseNodeParser import ConceptNode, RuleNode, SourceCodeNode, SourceCodeWithConceptNode, \
+    UnrecognizedTokensNode
 from parsers.FunctionParser import FunctionNode
 from parsers.PythonParser import PythonNode
-from parsers.SyaNodeParser import SyaConceptParserHelper
 from sheerkapython.python_wrapper import sheerka_globals
 from sheerkarete.common import V
-from sheerkarete.conditions import Condition, AndConditions, NegatedCondition, NegatedConjunctiveConditions
+from sheerkarete.conditions import AndConditions, Condition, NegatedCondition, NegatedConjunctiveConditions
 
 
 @dataclass
@@ -29,104 +28,254 @@ class Obj:
     parent: object = None
 
 
-class AND:
+class ExprTestObj:
+    @staticmethod
+    def get_pos(nodes):
+        start, end = None, None
+        for n in nodes:
+            if start is None or start > n.start:
+                start = n.start
+            if end is None or end < n.end:
+                end = n.end
+        return start, end
+
+    @staticmethod
+    def get_pos_from_source(source, full_text_as_tokens):
+        if isinstance(source, tuple):
+            source, to_skip = source[0], source[1]
+        else:
+            to_skip = 0
+
+        source_as_node = list(Tokenizer(source, yield_eof=False))
+        start = tokens_index(full_text_as_tokens, source_as_node, skip=to_skip)
+        end = start + len(source_as_node) - 1
+        return start, end
+
+    @staticmethod
+    def as_tokens(source):
+        if isinstance(source, tuple):
+            source, to_skip = source
+        else:
+            source, to_skip = source, 0
+
+        return list(Tokenizer(source, yield_eof=False)), to_skip
+
+    def get_expr_node(self, full_text_as_tokens=None):
+        raise NotImplementedError
+
+    @staticmethod
+    def safe_get_expr_node(obj, full_text_as_tokens):
+        if obj is None:
+            return None
+
+        obj = EXPR(obj) if isinstance(obj, (str, tuple)) else obj
+        return obj.get_expr_node(full_text_as_tokens)
+
+
+class AND(ExprTestObj):
     """ Test class for AndNode"""
 
     def __init__(self, *parts, source=None):
         self.parts = parts
         self.source = source
 
+    def get_expr_node(self, full_text_as_tokens=None):
+        parts = [part.get_expr_node(full_text_as_tokens) for part in self.parts]
+        start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else self.get_pos(parts)
+        return AndNode(start, end, full_text_as_tokens[start: end + 1], *parts)
 
-class OR:
+
+class OR(ExprTestObj):
     """ Test class for OrNode"""
 
     def __init__(self, *parts, source=None):
         self.parts = parts
         self.source = source
 
+    def get_expr_node(self, full_text_as_tokens=None):
+        parts = [part.get_expr_node(full_text_as_tokens) for part in self.parts]
+        start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else self.get_pos(parts)
+        return OrNode(start, end, full_text_as_tokens[start: end + 1], *parts)
+
 
 @dataclass
-class NOT:
+class NOT(ExprTestObj):
     """ Test class for NotNode"""
-    expr: object
+    expr: ExprTestObj
     source: str = None
 
+    def get_expr_node(self, full_text_as_tokens=None):
+        part = self.expr.get_expr_node(full_text_as_tokens)
+        start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else (
+            part.start - 2, part.end)
+        return NotNode(start, end, full_text_as_tokens[start: end + 1], part)
+
 
 @dataclass
-class EXPR:
-    """Test class for NameNode. E stands for Expression"""
+class EXPR(ExprTestObj):
+    """Test class for NameNode"""
     source: str
 
+    def get_expr_node(self, full_text_as_tokens=None):
+        value_as_tokens, to_skip = self.as_tokens(self.source)
+        start = tokens_index(full_text_as_tokens, value_as_tokens, to_skip)
+        end = start + len(value_as_tokens) - 1
+        return NameExprNode(start, end, full_text_as_tokens[start: end + 1])
+
 
 @dataclass
-class VAR:
+class VAR(ExprTestObj):
     """Test class for VarNode"""
+
     full_name: str
     source: str = None
 
+    def get_expr_node(self, full_text_as_tokens=None):
+        value_as_tokens = list(Tokenizer(self.source or self.full_name, yield_eof=False))
+        start = tokens_index(full_text_as_tokens, value_as_tokens, 0)
+        end = start + len(value_as_tokens) - 1
+        parts = self.full_name.split(".")
+        if len(parts) == 1:
+            return VariableNode(start, end, full_text_as_tokens[start: end + 1], parts[0])
+        else:
+            return VariableNode(start, end, full_text_as_tokens[start: end + 1], parts[0], *parts[1:])
+
 
 @dataclass
-class EQ:
-    left: object
-    right: object
+class CompExprTestObj(ExprTestObj):
+    """
+    Test object for comparison ==, <=, ...
+    """
+    left: ExprTestObj
+    right: ExprTestObj
     source: str = None
 
-
-@dataclass
-class NEQ:
-    left: object
-    right: object
-    source: str = None
+    def get_expr_node(self, full_text_as_tokens=None):
+        node_type = comparison_type_mapping[type(self).__name__]
+        left_node = self.left.get_expr_node(full_text_as_tokens)
+        right_node = self.right.get_expr_node(full_text_as_tokens)
+        start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else \
+            self.get_pos([left_node, right_node])
+        return ComparisonNode(start, end, full_text_as_tokens[start: end + 1], node_type, left_node, right_node)
 
 
 @dataclass
-class GT:
-    left: object
-    right: object
-    source: str = None
+class EQ(CompExprTestObj):
+    pass
 
 
 @dataclass
-class GTE:
-    left: object
-    right: object
-    source: str = None
+class NEQ(CompExprTestObj):
+    pass
 
 
 @dataclass
-class LT:
-    left: object
-    right: object
-    source: str = None
+class GT(CompExprTestObj):
+    pass
 
 
 @dataclass
-class LTE:
-    left: object
-    right: object
-    source: str = None
+class GTE(CompExprTestObj):
+    pass
 
 
 @dataclass
-class IN:
-    left: object
-    right: object
-    source: str = None
+class LT(CompExprTestObj):
+    pass
 
 
 @dataclass
-class NIN:  # for NOT INT
-    left: object
-    right: object
-    source: str = None
+class LTE(CompExprTestObj):
+    pass
 
 
 @dataclass
-class PAREN:  # for parenthesis node
+class IN(CompExprTestObj):
+    pass
+
+
+@dataclass
+class NIN(CompExprTestObj):  # for NOT INT
+    pass
+
+
+@dataclass
+class PAREN(ExprTestObj):  # for parenthesis node
     node: object
     source: str = None
 
 
+class L_EXPR(ExprTestObj):
+    def __init__(self, first, last, *items, sep=None, source=None):
+        self.first = first
+        self.last = last
+        self.items = items
+        self.sep = sep or comma
+        self.source = source
+
+    def get_expr_node(self, full_text_as_tokens=None):
+        first = self.safe_get_expr_node(self.first, full_text_as_tokens)
+        last = self.safe_get_expr_node(self.last, full_text_as_tokens)
+
+        items = [self.safe_get_expr_node(item, full_text_as_tokens) for item in self.items]
+
+        if self.source is None:
+            source = self.first if self.first else ""
+            source += f"{self.sep.value} ".join(item.get_source() for item in items)
+            if self.last:
+                source += self.last
+        else:
+            source = self.source
+
+        start, end = self.get_pos_from_source(source, full_text_as_tokens)
+        return ListNode(start, end, full_text_as_tokens[start: end + 1], first, last, items, self.sep)
+
+
+@dataclass
+class LCC:
+    """
+    List comprehension comprehension
+    """
+    target: object
+    iterable: object
+    if_expr: object
+
+
+@dataclass
+class LC(ExprTestObj):  # for List Comprehension node
+    element: object
+    generators: list
+    source: str = None
+
+    def get_expr_node(self, full_text_as_tokens=None):
+        # first transform str into NameExprTestObj (ie EXPR)
+        if isinstance(self.element, str):
+            self.element = EXPR(self.element)
+
+        comprehensions = []
+        nodes = []
+        for comp in self.generators:
+            target = EXPR(comp[0]) if isinstance(comp[0], (str, tuple)) else comp[0]
+            iterable = EXPR(comp[1]) if isinstance(comp[1], (str, tuple)) else comp[1]
+            if_expr = EXPR(comp[2]) if isinstance(comp[2], (str, tuple)) else comp[2]
+            comprehensions.append(LCC(target, iterable, if_expr))
+        self.generators = comprehensions
+
+        # then transform into ListComprehensionNode
+        element = self.element.get_expr_node(full_text_as_tokens)
+        nodes.append(element)
+        comprehensions = []
+        for comp in self.generators:
+            target = comp.target.get_expr_node(full_text_as_tokens)
+            iterable = comp.iterable.get_expr_node(full_text_as_tokens)
+            if_expr = comp.if_expr.get_expr_node(full_text_as_tokens) if comp.if_expr else None
+            comprehensions.append(Comprehension(target, iterable, if_expr))
+            nodes.extend([target, iterable, if_expr])
+
+        start, end = self.get_pos_from_source(self.source, full_text_as_tokens) if self.source else self.get_pos(nodes)
+        return ListComprehensionNode(start, end, full_text_as_tokens[start: end + 1], element, comprehensions)
+
+
 class CC:
     """
     Concept class for test purpose
@@ -320,10 +469,13 @@ class CMV:
     Test class that only compare the key and the metadata variables
     """
 
-    def __init__(self, concept, **kwargs):
+    def __init__(self, concept, source=None, **kwargs):
         self.concept_key = concept.key if isinstance(concept, Concept) else concept
         self.concept = concept if isinstance(concept, Concept) else None
         self.variables = kwargs
+        self.source = source  # to use when the key is different from the sub str to search when filling start and stop
+        self.start = None  # for debug purpose, indicate where the concept starts
+        self.end = None  # for debug purpose, indicate where the concept ends
 
     def __eq__(self, other):
         if id(self) == id(other):
@@ -352,6 +504,21 @@ class CMV:
             txt += f", {k}='{v}'"
         return txt + ")"
 
+    def fix_pos(self, node):
+        start = node.start if hasattr(node, "start") else \
+            node[0] if isinstance(node, tuple) else None
+        end = node.end if hasattr(node, "end") else \
+            node[1] if isinstance(node, tuple) else None
+
+        if start is not None:
+            if self.start is None or start < self.start:
+                self.start = start
+
+        if end is not None:
+            if self.end is None or end > self.end:
+                self.end = end
+        return self
+
     def transform_real_obj(self, other, get_test_obj_delegate):
         if isinstance(other, CMV):
             return other
@@ -730,7 +897,7 @@ class CNC(CN):
 
             self_compile_to_use = self.compiled or compiled
 
-            compiled = get_test_obj_delegate(self_compile_to_use, compiled, get_test_obj_delegate)
+            compiled = get_test_obj_delegate(compiled, self_compile_to_use, get_test_obj_delegate)
             return CNC(other.concept,
                        other.source if self.source is not None else None,
                        other.start if self.start is not None else None,
@@ -865,7 +1032,7 @@ class RN(HelperWithPos):
         raise Exception(f"Expecting RuleNode but received {other=}")
 
 
-class FN:
+class FN(ExprTestObj):
     """
     Test class only
     It matches with FunctionNode but with less constraints
@@ -931,6 +1098,32 @@ class FN:
 
         raise Exception(f"Expecting FunctionNode but received {other=}")
 
+    def get_expr_node(self, full_text_as_tokens=None):
+        start, end = self.get_pos_from_source(self.first, full_text_as_tokens)
+        first = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
+        start, end = self.get_pos_from_source(self.last, full_text_as_tokens)
+        last = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
+        parameters = []
+        for param_value, sep in self.parameters:
+            if isinstance(param_value, str):
+                start, end = self.get_pos_from_source(param_value, full_text_as_tokens)
+                param_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
+            else:
+                param_as_expr_node = param_value.get_expr_node(full_text_as_tokens)
+
+            if sep:
+                sep_tokens = Tokenizer(sep, yield_eof=False)
+                start = param_as_expr_node.end + 1
+                end = start + len(list(sep_tokens)) - 1
+                sep_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
+            else:
+                sep_as_expr_node = None
+
+            parameters.append(FunctionParameter(param_as_expr_node, sep_as_expr_node))
+
+        start, end = first.start, last.end
+        return FunctionNode(start, end, full_text_as_tokens[start: end + 1], first, last, parameters)
+
 
 @dataclass()
 class NEGCOND:
@@ -966,94 +1159,7 @@ def get_expr_node_from_test_node(full_text, test_node):
     Returns EXPR, OR, NOT, AND object to ease the comparison with the real ExprNode
     """
     full_text_as_tokens = list(Tokenizer(full_text, yield_eof=False))
-
-    def get_pos(nodes):
-        start, end = None, None
-        for n in nodes:
-            if start is None or start > n.start:
-                start = n.start
-            if end is None or end < n.end:
-                end = n.end
-        return start, end
-
-    def get_pos_from_source(source):
-        if isinstance(source, tuple):
-            source, to_skip = source[0], source[1]
-        else:
-            to_skip = 0
-
-        source_as_node = list(Tokenizer(source, yield_eof=False))
-        start = tokens_index(full_text_as_tokens, source_as_node, skip=to_skip)
-        end = start + len(source_as_node) - 1
-        return start, end
-
-    def get_expr_node(node):
-
-        if isinstance(node, EXPR):
-            value_as_tokens = list(Tokenizer(node.source, yield_eof=False))
-            start = tokens_index(full_text_as_tokens, value_as_tokens, 0)
-            end = start + len(value_as_tokens) - 1
-            return NameExprNode(start, end, full_text_as_tokens[start: end + 1])
-
-        if isinstance(node, AND):
-            parts = [get_expr_node(part) for part in node.parts]
-            start, end = get_pos_from_source(node.source) if node.source else get_pos(parts)
-            return AndNode(start, end, full_text_as_tokens[start: end + 1], *parts)
-
-        if isinstance(node, OR):
-            parts = [get_expr_node(part) for part in node.parts]
-            start, end = get_pos_from_source(node.source) if node.source else get_pos(parts)
-            return OrNode(start, end, full_text_as_tokens[start: end + 1], *parts)
-
-        if isinstance(node, NOT):
-            part = get_expr_node(node.expr)
-            start, end = get_pos_from_source(node.source) if node.source else (part.start - 2, part.end)
-            return NotNode(start, end, full_text_as_tokens[start: end + 1], part)
-
-        if isinstance(node, VAR):
-            value_as_tokens = list(Tokenizer(node.source or node.full_name, yield_eof=False))
-            start = tokens_index(full_text_as_tokens, value_as_tokens, 0)
-            end = start + len(value_as_tokens) - 1
-            parts = node.full_name.split(".")
-            if len(parts) == 1:
-                return VariableNode(start, end, full_text_as_tokens[start: end + 1], parts[0])
-            else:
-                return VariableNode(start, end, full_text_as_tokens[start: end + 1], parts[0], *parts[1:])
-
-        if isinstance(node, (EQ, NEQ, GT, GTE, LT, LTE, IN, NIN)):
-            node_type = comparison_type_mapping[type(node).__name__]
-            left_node, right_node = get_expr_node(node.left), get_expr_node(node.right)
-            start, end = get_pos_from_source(node.source) if node.source else get_pos([left_node, right_node])
-            return ComparisonNode(start, end, full_text_as_tokens[start: end + 1],
-                                  node_type, left_node, right_node)
-
-        if isinstance(node, FN):
-            start, end = get_pos_from_source(node.first)
-            first = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
-            start, end = get_pos_from_source(node.last)
-            last = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
-            parameters = []
-            for param_value, sep in node.parameters:
-                if isinstance(param_value, str):
-                    start, end = get_pos_from_source(param_value)
-                    param_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
-                else:
-                    param_as_expr_node = get_expr_node(param_value)
-
-                if sep:
-                    sep_tokens = Tokenizer(sep, yield_eof=False)
-                    start = param_as_expr_node.end + 1
-                    end = start + len(list(sep_tokens)) - 1
-                    sep_as_expr_node = NameExprNode(start, end, full_text_as_tokens[start: end + 1])
-                else:
-                    sep_as_expr_node = None
-
-                parameters.append(FunctionParameter(param_as_expr_node, sep_as_expr_node))
-
-            start, end = first.start, last.end
-            return FunctionNode(start, end, full_text_as_tokens[start: end + 1], first, last, parameters)
-
-    return get_expr_node(test_node)
+    return test_node.get_expr_node(full_text_as_tokens)
 
 
 def _index(tokens, expr, index):
@@ -1157,7 +1263,8 @@ def get_node(
         sub_expr.end = start + length - 1
         return sub_expr
 
-    if isinstance(sub_expr, (CNC, CC, CN)):
+    if isinstance(sub_expr, (CNC, CC, CN, CMV)):
+        if sub_expr.concept is None or sub_expr.start is None or sub_expr.end is None:
             concept_node = get_node(
                 concepts_map,
                 expression_as_tokens,
@@ -1168,7 +1275,8 @@ def get_node(
             concept_found = concept_node.concept
             sub_expr.concept_key = concept_found.key
             sub_expr.concept = concept_found
-        sub_expr.fix_pos((concept_node.start, concept_node.end if hasattr(concept_node, "end") else concept_node.start))
+            sub_expr.fix_pos(
+                (concept_node.start, concept_node.end if hasattr(concept_node, "end") else concept_node.start))
         if hasattr(sub_expr, "compiled"):
             for k, v in sub_expr.compiled.items():
                 node = get_node(concepts_map, expression_as_tokens, v, sya=sya,
@@ -1210,9 +1318,9 @@ def get_node(
     concept_found = concepts_map.get(concept_key, None)
     if concept_found:
         concept_found = Concept().update_from(concept_found)  # make a copy when massively used in tests
-        if sya and len(concept_found.get_metadata().variables) > 0 and not is_bnf:
-            return SyaConceptParserHelper(concept_found, start, start + length - 1)
-        elif init_empty_body:
+        # if sya and len(concept_found.get_metadata().variables) > 0 and not is_bnf:
+        #     return SyaConceptParserHelper(concept_found, start, start + length - 1)
+        if init_empty_body:
             node = CNC(concept_found, sub_expr, start, start + length - 1, exclude_body=exclude_body)
             init_body(node, concept_found, sub_expr)
             return node
@@ -1354,8 +1462,8 @@ def get_test_obj(real_obj, test_obj, get_test_obj_delegate=None):
     """
     From a production object (Concept, ConceptNode, ....)
     Create a test object (CNC, CC ...) that can be used to validate the unit tests
-    :param test_obj:
     :param real_obj:
+    :param test_obj: test object used as a template
     :param get_test_obj_delegate:
     :return:
     """
@@ -1367,13 +1475,25 @@ def get_test_obj(real_obj, test_obj, get_test_obj_delegate=None):
     if isinstance(test_obj, dict):
         if len(test_obj) != len(real_obj):
             raise Exception(f"Not the same size ! {real_obj=}, {test_obj=}")
-
         return {k: get_test_obj(real_obj[k], v) for k, v in test_obj.items()}
 
-    if not hasattr(test_obj, "transform_real_obj"):
+    if hasattr(test_obj, "transform_real_obj"):
+        return test_obj.transform_real_obj(real_obj, get_test_obj)
+
     return real_obj
 
-    return test_obj.transform_real_obj(real_obj, get_test_obj)
+
+def prepare_nodes_comparison(concepts_map, expression, real_obj, test_obj):
+    if isinstance(real_obj, list):
+        assert len(real_obj) == len(
+            test_obj), f"The two lists do not have the same size {len(real_obj)} != {len(test_obj)}"
+        resolved_test_obj = compute_expected_array(concepts_map, expression, test_obj)
+        real_obj_as_test = [get_test_obj(r, t) for r, t in zip(real_obj, resolved_test_obj)]
+        return real_obj_as_test, resolved_test_obj
+    else:
+        resolved_test_obj = compute_expected_array(concepts_map, expression, [test_obj])[0]
+        real_obj_as_test = get_test_obj(real_obj, resolved_test_obj)
+        return real_obj_as_test, resolved_test_obj
 
 
 def compare_with_test_object(actual, expected):

tests/parsers/test_BaseCustomGrammarParser.py

@@ -124,7 +124,7 @@ func(a)
         assert parser.get_parts(["print", "when"]) is not None
         assert len(parser.error_sink) == 1
         assert isinstance(parser.error_sink[0], UnexpectedEofParsingError)
-        assert parser.error_sink[0].message == "While parsing keyword 'print'."
+        assert parser.error_sink[0].message == "while parsing keyword 'print'"
 
     def test_i_can_double_quoted_strings_are_expanded(self):
         """

tests/parsers/test_BnfNodeParser.py

@@ -4,19 +4,20 @@ import pytest
 
 import tests.parsers.parsers_utils
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, ConceptParts, DoNotResolve, DEFINITION_TYPE_BNF
+from core.concept import Concept, ConceptParts, DEFINITION_TYPE_BNF, DoNotResolve
 from core.global_symbols import NotInit
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.SheerkaExecute import ParserInput
+from core.sheerka.services.SheerkaIsAManager import SheerkaIsAManager
 from parsers.BaseNodeParser import NoMatchingTokenError
 from parsers.BnfDefinitionParser import BnfDefinitionParser
-from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
-    Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice, BnfNodeParser, RegExMatch, \
-    BnfNodeFirstTokenVisitor, Match, RegExDef, VariableExpression
+from parsers.BnfNodeParser import BnfNodeFirstTokenVisitor, BnfNodeParser, ConceptExpression, Match, NonTerminalNode, \
+    OneOrMore, Optional, OrderedChoice, RegExDef, RegExMatch, Sequence, StrMatch, TerminalNode, UnOrderedChoice, \
+    VariableExpression, ZeroOrMore
 from tests.BaseTest import BaseTest
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.evaluators.EvaluatorTestsUtils import python_ret_val
-from tests.parsers.parsers_utils import CNC, CN, UTN, CC, SCN, get_test_obj, compare_with_test_object
+from tests.parsers.parsers_utils import CC, CMV, CN, CNC, SCN, UTN, compare_with_test_object, get_test_obj
 
 cmap = {
     "one": Concept("one"),
@@ -1027,7 +1028,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         ("one tiny but beautiful shoe",
          [CNC("foo",
               "one tiny but beautiful shoe",
-              x=CC("but", source="tiny but beautiful", x="tiny", y="beautiful "))]),
+              x=CMV("but", source="tiny but beautiful", x="tiny ", y="beautiful "))]),
     ])
     def test_i_can_match_variable_in_between(self, expr, expected):
         my_map = {
@@ -1896,7 +1897,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
     def test_i_can_simplify_unordered_choices_that_refer_to_the_same_isa(self):
         my_map = {
-            "light_red": Concept("light red"),
+            "light_red": Concept("light red", key="light_red"),
             "dark_red": Concept("dark red"),
             "red colors": Concept("red colors"),
             "color": Concept("color"),
@@ -1916,6 +1917,10 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         sheerka.set_isa(global_truth_context, my_map["red colors"], my_map["color"])
         sheerka.set_isa(global_truth_context, my_map["red colors"], my_map["adjective"])
 
+        # hack to ease the tests
+        sheerka.get_by_id(my_map["light_red"].id).get_metadata().key = "light_red"
+        sheerka.om.clear(SheerkaIsAManager.CONCEPTS_IN_GROUPS_ENTRY)
+
         text = "light red table"
 
         expected = CNC("qualified table",
@@ -1953,7 +1958,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         res = parser.parse(context, ParserInput(text))
         assert not res.status
 
-
     # @pytest.mark.parametrize("parser_input, expected", [
     #     ("one", [
     #         (True, [CNC("bnf_one", source="one", one="one", body="one")]),

tests/parsers/test_DefConceptParser.py

@@ -387,8 +387,8 @@ def concept add one to a as:
         ("def concept name from def", SyntaxErrorNode([], "Empty 'from' declaration.")),
         ("def concept name from def ", SyntaxErrorNode([], "Empty 'from' declaration.")),
         ("def concept name from as True", SyntaxErrorNode([], "Empty 'from' declaration.")),
-        ("def concept name from", UnexpectedEofParsingError("While parsing keyword 'from'.")),
-        ("def concept name from ", UnexpectedEofParsingError("While parsing keyword 'from'.")),
+        ("def concept name from", UnexpectedEofParsingError("while parsing keyword 'from'")),
+        ("def concept name from ", UnexpectedEofParsingError("while parsing keyword 'from'")),
     ])
     def test_i_can_detect_empty_def_declaration(self, text, error):
         sheerka, context, parser, *concepts = self.init_parser()

tests/parsers/test_DefRuleParser.py

@@ -190,8 +190,8 @@ class TestDefRuleParser(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res.body, expected_error)
 
     @pytest.mark.parametrize("text, error_message", [
-        ("def rule rule_name as", "While parsing 'when'."),
-        ("def rule rule_name as ", "While parsing 'when'."),
+        ("def rule rule_name as", "while parsing 'when'"),
+        ("def rule rule_name as ", "while parsing 'when'"),
     ])
     def test_i_cannot_parse_when_unexpected_eof(self, text, error_message):
         sheerka, context, parser = self.init_parser()

tests/parsers/test_ListComprehensionParser.py

@@ -0,0 +1,180 @@
+import pytest
+
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.tokenizer import Token, TokenKind
+from parsers.BaseExpressionParser import ParenthesisMismatchError
+from parsers.BaseParser import UnexpectedEofParsingError, UnexpectedTokenParsingError
+from parsers.ListComprehensionParser import ElementNotFound, FailedToParse, ForNotFound, LeadingParenthesisNotFound, \
+    ListComprehensionParser
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.parsers.parsers_utils import LC, L_EXPR, get_expr_node_from_test_node
+
+
+class TestListComprehensionParser(TestUsingMemoryBasedSheerka):
+
+    def init_parser(self):
+        sheerka, context = self.init_concepts()
+        parser = ListComprehensionParser(auto_compile=False)
+        return sheerka, context, parser
+
+    @pytest.mark.parametrize("text, reason", [
+        ("foo", LeadingParenthesisNotFound()),
+        ("[]", ForNotFound()),
+        ("[ x ]", ForNotFound()),
+        ("[ x for]", FailedToParse("target", 5)),
+        ("[ x for x]", UnexpectedEofParsingError("while parsing comprehension")),
+        ("[ x for x in ]", UnexpectedEofParsingError("while parsing comprehension")),
+        ("[ x for x in lst for]", FailedToParse("target", 13)),
+        ("[", UnexpectedEofParsingError("when start parsing")),
+        ("[]", ForNotFound()),
+        ("[ for x in z ]", ElementNotFound()),
+        ("[ x for in z ]", FailedToParse("target", 6)),
+        ("[ x for x in ]", UnexpectedEofParsingError("while parsing comprehension")),
+        ("[ x for x in z if ]", UnexpectedEofParsingError("while parsing comprehension")),
+        ("[ x for x in z", ParenthesisMismatchError(Token(TokenKind.RBRACKET, "]", -1, -1, -1))),
+        ("[ x for x in z if t", ParenthesisMismatchError(Token(TokenKind.RBRACKET, "]", -1, -1, -1))),
+        ("zzz [ x for x in z if t ]", LeadingParenthesisNotFound()),
+        ("[ x for x in z )", ParenthesisMismatchError(Token(TokenKind.RBRACKET, "]", -1, -1, -1))),
+        ("[ x for x in z if t )", ParenthesisMismatchError(Token(TokenKind.RBRACKET, "]", -1, -1, -1))),
+    ])
+    def test_i_cannot_parse_when_not_for_me(self, text, reason):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput(text))
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
+        assert res.body.reason == [reason]
+
+    def test_i_cannot_parse_when_trailing_elements(self):
+        sheerka, context, parser = self.init_parser()
+
+        text = "[ x for x in z if t ] zzz"
+        res = parser.parse(context, ParserInput(text))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert len(res.body.body) == 1
+        error = res.body.body[0]
+        assert isinstance(error, UnexpectedTokenParsingError)
+
+    def test_i_can_parse_a_simple_expression(self):
+        sheerka, context, parser = self.init_parser()
+
+        expression = "[x for x in ['a', 'b'] if x == 'a']"
+        res = parser.parse(context, ParserInput(expression))
+        wrapper = res.body
+        lc_node = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+
+        expected = LC(L_EXPR(None, None, "x", source="x "), [(("x", 1), "['a', 'b']", "x == 'a'")], source=expression)
+        to_compare_to = get_expr_node_from_test_node(expression, expected)
+        assert lc_node == to_compare_to
+
+    def test_i_can_parse_when_no_if(self):
+        sheerka, context, parser = self.init_parser()
+
+        expression = "[x for x in ['a', 'b']]"
+        res = parser.parse(context, ParserInput(expression))
+        wrapper = res.body
+        lc_node = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+
+        expected = LC(L_EXPR(None, None, "x", source="x "), [(("x", 1), "['a', 'b']", None)], source=expression)
+        to_compare_to = get_expr_node_from_test_node(expression, expected)
+        assert lc_node == to_compare_to
+
+    def test_i_can_parse_when_element_is_a_tuple(self):
+        sheerka, context, parser = self.init_parser()
+
+        expression = "[(x + 1, x + 2) for x in [1, 2]]"
+        res = parser.parse(context, ParserInput(expression))
+        wrapper = res.body
+        lc_node = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+
+        elt = L_EXPR("(", ")", "x + 1", "x + 2")
+        expected = LC(elt, [(("x", 2), "[1, 2]", None)], source=expression)
+        to_compare_to = get_expr_node_from_test_node(expression, expected)
+
+        assert lc_node == to_compare_to
+
+    def test_i_can_parse_when_element_is_a_tuple_with_missing_parenthesis(self):
+        sheerka, context, parser = self.init_parser()
+
+        expression = "[x + 1, x + 2 for x in [1, 2]]"
+        res = parser.parse(context, ParserInput(expression))
+        wrapper = res.body
+        lc_node = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+
+        elt = L_EXPR(None, None, "x + 1", "x + 2", source="x + 1, x + 2 ")
+        expected = LC(elt, [(("x", 2), "[1, 2]", None)], source=expression)
+        to_compare_to = get_expr_node_from_test_node(expression, expected)
+
+        assert lc_node == to_compare_to
+
+    def test_i_can_parse_when_element_is_a_context_that_contains_for(self):
+        sheerka, context, parser = self.init_parser()
+
+        expression = "[handle x for me and for you for x in [1, 2]]"
+        res = parser.parse(context, ParserInput(expression))
+        wrapper = res.body
+        lc_node = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+
+        elt = L_EXPR(None, None, "handle x for me and for you", source="handle x for me and for you ")
+        expected = LC(elt, [(("x", 1), "[1, 2]", None)], source=expression)
+        to_compare_to = get_expr_node_from_test_node(expression, expected)
+
+        assert lc_node == to_compare_to
+
+    def test_i_can_parse_when_multiple_generators(self):
+        sheerka, context, parser = self.init_parser()
+
+        expression = "[(x, y) for x in ['a', 'b'] if x == 'a' for y in ['c', 'd'] if y == 'c']"
+        res = parser.parse(context, ParserInput(expression))
+        wrapper = res.body
+        lc_node = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+
+        elt = L_EXPR("(", ")", "x", "y")
+        expected = LC(elt,
+                      [(("x", 1), "['a', 'b']", "x == 'a'"),
+                       (("y", 1), "['c', 'd']", "y == 'c'")],
+                      source=expression)
+        to_compare_to = get_expr_node_from_test_node(expression, expected)
+
+        assert lc_node == to_compare_to
+
+    def test_i_can_parse_when_multiple_generators_when_no_if(self):
+        sheerka, context, parser = self.init_parser()
+
+        expression = "[x, y for x in ['a', 'b'] for y in ['c', 'd'] if y == 'c']"
+        res = parser.parse(context, ParserInput(expression))
+        wrapper = res.body
+        lc_node = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+
+        elt = L_EXPR(None, None, "x", "y", source="x, y ")
+        expected = LC(elt,
+                      [(("x", 1), "['a', 'b']", None),
+                       (("y", 1), "['c', 'd']", "y == 'c'")],
+                      source=expression)
+        to_compare_to = get_expr_node_from_test_node(expression, expected)
+
+        assert lc_node == to_compare_to

tests/parsers/test_ListParser.py

@@ -0,0 +1,69 @@
+import pytest
+
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.sheerka.services.SheerkaExecute import ParserInput
+from core.tokenizer import Token, TokenKind
+from parsers.BaseExpressionParser import ParenthesisMismatchError, end_parenthesis_mapping
+from parsers.BaseParser import ErrorSink
+from parsers.ListParser import ListParser
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.parsers.parsers_utils import EXPR, L_EXPR, get_expr_node_from_test_node
+
+semi_colon = Token(TokenKind.SEMICOLON, ";", -1, -1, -1)
+or_token = Token(TokenKind.IDENTIFIER, "or", -1, -1, -1)
+
+
+class TestListParser(TestUsingMemoryBasedSheerka):
+    def init_parser(self, sep=None):
+        sheerka, context = self.init_concepts()
+        parser = ListParser(sep)
+        return sheerka, context, parser
+
+    @pytest.mark.parametrize("expression, sep, expected", [
+        ("()", None, L_EXPR("(", ")")),
+        ("(x , foo y,z)", None, L_EXPR("(", ")", EXPR("x"), EXPR("foo y"), EXPR("z"), source="(x , foo y,z)")),
+        ("x , foo y,z", None, L_EXPR(None, None, EXPR("x"), EXPR("foo y"), EXPR("z"), source="x , foo y,z")),
+        ("x", None, L_EXPR(None, None, EXPR("x"))),
+        ("[x, foo y, z]", None, L_EXPR("[", "]", EXPR("x"), EXPR("foo y"), EXPR("z"))),
+        ("{x, foo y, z}", None, L_EXPR("{", "}", EXPR("x"), EXPR("foo y"), EXPR("z"))),
+        ("(x; y; z)", semi_colon, L_EXPR("(", ")", EXPR("x"), EXPR("y"), EXPR("z"), sep=semi_colon, source="(x; y; z)")),
+        ("x; y; z", semi_colon, L_EXPR(None, None, EXPR("x"), EXPR("y"), EXPR("z"), sep=semi_colon, source="x; y; z")),
+        ("x or y or z", or_token, L_EXPR(None, None, EXPR("x"), EXPR("y"), EXPR("z"), sep=or_token, source="x or y or z")),
+    ])
+    def test_i_can_parse_expression(self, expression, sep, expected):
+        sheerka, context, parser = self.init_parser(sep)
+
+        expected = get_expr_node_from_test_node(expression, expected)
+        res = parser.parse(context, ParserInput(expression))
+        wrapper = res.body
+        expressions = res.body.body
+
+        assert res.status
+        assert sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert expressions == expected
+
+    @pytest.mark.parametrize("expression, starting", [
+        ("(", TokenKind.LPAR),
+        ("(x, y", TokenKind.LPAR),
+        ("{x, y", TokenKind.LBRACE),
+        ("[x, y", TokenKind.LBRACKET),
+    ])
+    def test_i_cannot_parse_when_missing_trailing_parenthesis(self, expression, starting):
+        sheerka, context, parser = self.init_parser()
+
+        res = parser.parse(context, ParserInput(expression))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body == [ParenthesisMismatchError(end_parenthesis_mapping[starting])]
+
+    def test_none_is_return_when_empty_parser_input(self):
+        sheerka, context, parser = self.init_parser()
+
+        parser_input = ParserInput("   ").reset()
+        parser_input.next_token()
+        error_sink = ErrorSink()
+
+        parsed = parser.parse_input(context, parser_input, error_sink)
+
+        assert parsed is None

tests/parsers/test_LogicalOperatorParser.py

@@ -100,12 +100,12 @@ class TestLogicalOperatorParser(TestUsingMemoryBasedSheerka):
         assert expressions == expected
 
     @pytest.mark.parametrize("expression, expected_errors", [
-        ("one or", [UnexpectedEofParsingError("When parsing 'or'")]),
-        ("one and", [UnexpectedEofParsingError("When parsing 'and'")]),
-        ("and one", [LeftPartNotFoundError()]),
-        ("or one", [LeftPartNotFoundError()]),
-        ("or", [LeftPartNotFoundError(), UnexpectedEofParsingError("When parsing 'or'")]),
-        ("and", [LeftPartNotFoundError(), UnexpectedEofParsingError("When parsing 'and'")]),
+        ("one or", [UnexpectedEofParsingError("while parsing 'or'")]),
+        ("one and", [UnexpectedEofParsingError("while parsing 'and'")]),
+        ("and one", [LeftPartNotFoundError("and", 0)]),
+        ("or one", [LeftPartNotFoundError("or", 0)]),
+        ("or", [LeftPartNotFoundError("or", 0), UnexpectedEofParsingError("while parsing 'or'")]),
+        ("and", [LeftPartNotFoundError("and", 0), UnexpectedEofParsingError("while parsing 'and'")]),
     ])
     def test_i_can_detect_error(self, expression, expected_errors):
         sheerka, context, parser = self.init_parser()

tests/parsers/test_SequenceNodeParser.py

@@ -5,8 +5,7 @@ from core.concept import Concept, DEFINITION_TYPE_DEF
 from core.sheerka.services.SheerkaExecute import ParserInput
 from parsers.SequenceNodeParser import SequenceNodeParser
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-from tests.parsers.parsers_utils import compute_expected_array, CN, SCN, get_test_obj, compare_with_test_object, \
-    UTN
+from tests.parsers.parsers_utils import CN, SCN, UTN, compare_with_test_object, compute_expected_array, get_test_obj
 
 
 class TestSequenceNodeParser(TestUsingMemoryBasedSheerka):
@@ -463,3 +462,24 @@ class TestSequenceNodeParser(TestUsingMemoryBasedSheerka):
         assert concept_found.name == "boys"
         assert concept_found.key == "boys"
         assert concept_found.get_prop(BuiltinConcepts.PLURAL) == boy
+
+    def test_i_can_set_body_for_plurals_that_are_a_set(self):
+        concepts_map = {
+            "boy": Concept("boy"),
+            "girl": Concept("girl"),
+            "human": Concept("human"),
+        }
+
+        sheerka, context, parser = self.init_parser(concepts_map)
+        global_truth_concept = self.get_context(sheerka, global_truth=True)
+        sheerka.set_isa(global_truth_concept, concepts_map["boy"], concepts_map["human"])
+        sheerka.set_isa(global_truth_concept, concepts_map["girl"], concepts_map["human"])
+
+        res = parser.parse(context, ParserInput("humans"))
+
+        assert res.status
+        lexer_nodes = res.body.body
+        assert len(lexer_nodes) == 1
+        concept_found = lexer_nodes[0].concept
+
+        assert concept_found.get_metadata().body == "get_set_elements(c:|1003:)"

tests/sheerkapython/test_ExprToPython.py

@@ -0,0 +1,422 @@
+import pytest
+
+from core.builtin_concepts_ids import BuiltinConcepts
+from core.concept import Concept
+from core.sheerka.services.SheerkaEvaluateConcept import EvaluationHints
+from core.sheerka.services.SheerkaExecute import ParserInput
+from evaluators.PythonEvaluator import PythonEvaluator
+from parsers.BaseParser import ErrorSink
+from parsers.ExpressionParser import ExpressionParser
+from parsers.ListComprehensionParser import ListComprehensionParser
+from parsers.PythonParser import PythonNode
+from sheerkapython.ExprToPython import PythonExprVisitor
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestExprToPython(TestUsingMemoryBasedSheerka):
+
+    @staticmethod
+    def get_expr_node(context, expression, parser=None):
+        parser = parser or ExpressionParser()
+        error_sink = ErrorSink()
+        parser_input = ParserInput(expression)
+        parser.reset_parser_input(parser_input, error_sink)
+        parsed = parser.parse_input(context, parser_input, error_sink)
+
+        assert not error_sink.has_error
+
+        return parsed
+
+    @staticmethod
+    def eval(context, return_value, namespace=None):
+        evaluator = PythonEvaluator()
+        assert evaluator.matches(context, return_value)
+
+        if namespace:
+            for k, v in namespace.items():
+                context.add_to_short_term_memory(k, v)
+
+        res = evaluator.eval(context, return_value)
+        assert res.status
+        return res.body
+
+    @pytest.mark.parametrize("expression, source, objects", [
+        ("foo w", "call_concept(__o_00__, x=w)", {"__o_00__": "foo"}),
+        ("foo z + 2", "call_concept(__o_00__, x=z + 2)", {"__o_00__": "foo"}),
+        ("foo a and bar b",
+         "call_concept(__o_00__, x=a) and call_concept(__o_01__, y=b)",
+         {"__o_00__": "foo", "__o_01__": "bar"}),
+        ("foo a or bar b",
+         "call_concept(__o_00__, x=a) or call_concept(__o_01__, y=b)",
+         {"__o_00__": "foo", "__o_01__": "bar"}),
+        ("not foo w", "not call_concept(__o_00__, x=w)", {"__o_00__": "foo"}),
+        ("foo a >= bar b",
+         "call_concept(__o_00__, x=a) >= call_concept(__o_01__, y=b)",
+         {"__o_00__": "foo", "__o_01__": "bar"}),
+        ("function(foo a, bar b)",
+         "function(call_concept(__o_00__, x=a), call_concept(__o_01__, y=b))",
+         {"__o_00__": "foo", "__o_01__": "bar"})
+
+    ])
+    def test_i_can_compile_concept_when_is_question_is_false(self, expression, source, objects):
+        sheerka, context, foo, bar = self.init_test().with_concepts(
+            Concept("foo x", body="x").def_var("x"),
+            Concept("bar y", body="y").def_var("y"),
+            create_new=True
+        ).unpack()
+
+        concepts = {
+            "foo": foo,
+            "bar": bar
+        }
+
+        node = self.get_expr_node(context, expression)
+        visitor = PythonExprVisitor(context)
+
+        ret = visitor.compile(node)
+
+        assert len(ret) == 1
+        python_node = ret[0].body.body
+        assert python_node.original_source == expression
+        assert python_node.source == source
+
+        for obj_name, obj_value in objects.items():
+            assert obj_name in python_node.objects
+
+            obj = python_node.objects[obj_name]
+            if isinstance(obj, Concept):
+                assert sheerka.isinstance(obj, concepts[obj_value])
+                assert obj.get_hints().use_copy
+                assert obj.get_hints().is_evaluated
+            else:
+                assert False
+
+    @pytest.mark.parametrize("expression, source, objects", [
+        ("foo w", "evaluate_question(__o_00__, x=w)", {"__o_00__": "foo"}),
+        ("foo z + 2", "evaluate_question(__o_00__, x=z + 2)", {"__o_00__": "foo"}),
+    ])
+    def test_i_can_compile_concept_when_is_question_is_true(self, expression, source, objects):
+        sheerka, context, foo = self.init_test().with_concepts(
+            Concept("foo x", body="x", pre="is_question()").def_var("x"),
+            create_new=True
+        ).unpack()
+
+        concepts = {
+            "foo": foo
+        }
+
+        node = self.get_expr_node(context, expression)
+        visitor = PythonExprVisitor(context)
+        ret = visitor.compile(node, EvaluationHints(eval_body=True, eval_question=True))
+
+        assert len(ret) == 1
+        python_node = ret[0].body.body
+        assert python_node.original_source == expression
+        assert python_node.source == source
+
+        for obj_name, obj_value in objects.items():
+            assert obj_name in python_node.objects
+
+            obj = python_node.objects[obj_name]
+            if isinstance(obj, Concept):
+                assert sheerka.isinstance(obj, concepts[obj_value])
+                assert obj.get_hints().use_copy
+                assert obj.get_hints().is_evaluated
+            else:
+                assert False
+
+    def test_i_can_compile_simple_list_comprehension(self):
+        sheerka, context = self.init_test().unpack()
+
+        expression = "[ x for x in ['a', 'b'] if x == 'a' ]"
+        node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
+        visitor = PythonExprVisitor(context)
+
+        ret = visitor.compile(node)
+
+        assert len(ret) == 1
+        assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(ret[0].body.body, PythonNode)
+
+        python_node = ret[0].body.body
+        assert python_node.original_source == expression
+        assert python_node.source == expression
+
+        assert self.eval(context, ret[0]) == ["a"]
+
+    def test_i_can_compile_simple_list_comprehension_when_no_if(self):
+        sheerka, context = self.init_test().unpack()
+
+        expression = "[ x for x in ['a', 'b'] ]"
+        node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
+        visitor = PythonExprVisitor(context)
+
+        ret = visitor.compile(node)
+
+        assert len(ret) == 1
+        assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(ret[0].body.body, PythonNode)
+
+        python_node = ret[0].body.body
+        assert python_node.original_source == expression
+        assert python_node.source == expression
+
+        assert self.eval(context, ret[0]) == ['a', 'b']
+
+    def test_i_can_compile_list_comprehension_when_element_is_a_concept(self):
+        sheerka, context, foo = self.init_test().with_concepts(
+            Concept("foo x", body="x").def_var("x")
+        ).unpack()
+
+        expression = "[ foo w for w in ['a', 'b'] ]"
+        node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
+        visitor = PythonExprVisitor(context)
+
+        ret = visitor.compile(node)
+
+        assert len(ret) == 1
+        assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(ret[0].body.body, PythonNode)
+
+        python_node = ret[0].body.body
+        assert python_node.original_source == expression
+        assert python_node.source == "[ call_concept(__o_00__, x=w) for w in ['a', 'b'] ]"
+        assert "__o_00__" in python_node.objects
+
+        concept = python_node.objects["__o_00__"]
+        assert sheerka.isinstance(concept, foo)
+        assert concept.get_hints().use_copy
+        assert concept.get_hints().is_evaluated
+
+        assert self.eval(context, ret[0]) == ["a", "b"]
+
+    def test_i_can_compile_list_comprehension_when_concept_with_complex_parameter(self):
+        sheerka, context, foo = self.init_test().with_concepts(
+            Concept("foo x", body="x").def_var("x"),
+            create_new=True
+        ).unpack()
+
+        expression = "[ foo w + 1 for w in [1, 2] ]"
+        node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
+        visitor = PythonExprVisitor(context)
+
+        ret = visitor.compile(node)
+
+        assert len(ret) == 1
+        assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(ret[0].body.body, PythonNode)
+
+        python_node = ret[0].body.body
+        assert python_node.original_source == expression
+        assert python_node.source == "[ call_concept(__o_00__, x=w + 1) for w in [1, 2] ]"
+        assert "__o_00__" in python_node.objects
+
+        assert self.eval(context, ret[0]) == [2, 3]
+
+    def test_i_can_compile_list_comprehension_when_iter_is_a_concept(self):
+        sheerka, context, red, blue, color, foo = self.init_test().with_concepts(
+            "red",
+            "blue",
+            "color",
+            Concept("foo x", body="x").def_var("x")
+        ).unpack()
+
+        global_truth_context = self.get_context(sheerka, global_truth=True)
+        sheerka.set_isa(global_truth_context, red, color)
+        sheerka.set_isa(global_truth_context, blue, color)
+
+        expression = "[ foo x for x in colors ]"
+        node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
+        visitor = PythonExprVisitor(context)
+
+        ret = visitor.compile(node)
+
+        assert len(ret) == 1
+        assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(ret[0].body.body, PythonNode)
+
+        python_node = ret[0].body.body
+        assert python_node.original_source == expression
+        assert python_node.source == "[ call_concept(__o_00__, x=x) for x in call_concept(__o_01__) ]"
+        assert "__o_00__" in python_node.objects
+        assert "__o_01__" in python_node.objects
+
+        concept0 = python_node.objects["__o_00__"]
+        assert sheerka.isinstance(concept0, foo)
+        assert concept0.get_hints().use_copy
+        assert concept0.get_hints().is_evaluated
+
+        concept1 = python_node.objects["__o_01__"]
+        assert sheerka.isinstance(concept1, "colors")
+        assert concept1.get_hints().use_copy
+        assert concept1.get_hints().is_evaluated
+
+        assert set(self.eval(context, ret[0])) == {red, blue}
+
+    def test_i_can_compile_list_comprehension_when_if_expression_is_a_concept(self):
+        sheerka, context, red, blue, color, foo, startswith = self.init_test().with_concepts(
+            "red",
+            "blue",
+            "color",
+            Concept("foo x", body="x").def_var("x"),
+            Concept("x starts with y", body="x.name.startswith(y)", pre="is_question()").def_var("x").def_var("y")
+        ).unpack()
+        global_truth_context = self.get_context(sheerka, global_truth=True)
+        sheerka.set_isa(global_truth_context, red, color)
+        sheerka.set_isa(global_truth_context, blue, color)
+
+        expression = "[ foo x for x in colors if x starts with 'b' ]"
+
+        node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
+        visitor = PythonExprVisitor(context)
+
+        ret = visitor.compile(node)
+
+        assert len(ret) == 1
+        assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(ret[0].body.body, PythonNode)
+
+        python_node = ret[0].body.body
+        assert python_node.original_source == expression
+        assert python_node.source == "[ call_concept(__o_00__, x=x) for x in call_concept(__o_01__) if evaluate_question(__o_02__, x=x, y='b') ]"
+        assert "__o_00__" in python_node.objects
+        assert "__o_01__" in python_node.objects
+        assert "__o_02__" in python_node.objects
+        assert visitor.obj_counter == 3
+
+        assert set(self.eval(context, ret[0])) == {blue}
+
+    def test_i_can_compile_list_comprehension_when_multiple_concepts(self):
+        sheerka, context, foo1, foo2, bar1, bar2, colors1, colors2, = self.init_test().with_concepts(
+            Concept("foo x").def_var("x"),
+            Concept("foo y", ).def_var("y"),
+            Concept("bar x", pre="is_question()").def_var("x"),
+            Concept("bar y", pre="is_question()").def_var("y"),
+            Concept("colors", body="[1]"),
+            Concept("colors", body="[2]"),
+        ).unpack()
+
+        expression = "[ foo a for a in colors if bar a ]"
+
+        node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
+        visitor = PythonExprVisitor(context)
+
+        ret = visitor.compile(node)
+
+        assert len(ret) == 8
+        python_node = ret[0].body.body
+        object_to_compare = {k: v.name for k, v in python_node.objects.items()}
+        assert python_node.source == '[ call_concept(__o_00__, x=a) for a in call_concept(__o_02__) if evaluate_question(__o_04__, x=a) ]'
+        assert object_to_compare == {"__o_00__": "foo x", "__o_02__": "colors", "__o_04__": "bar x"}
+
+        # ...
+
+        python_node = ret[7].body.body
+        object_to_compare = {k: v.name for k, v in python_node.objects.items()}
+        assert python_node.source == '[ call_concept(__o_01__, y=a) for a in call_concept(__o_03__) if evaluate_question(__o_05__, y=a) ]'
+        assert object_to_compare == {"__o_01__": "foo y", "__o_03__": "colors", "__o_05__": "bar y"}
+
+    def test_i_can_compile_list_comprehension_when_missing_concept_parameter(self):
+        sheerka, context, foo = self.init_test().with_concepts(
+            Concept("foo x y", body="x").def_var("x").def_var("y")
+        ).unpack()
+
+        expression = "[ foo x k for x in ['a', 'b'] ]"
+        node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
+        visitor = PythonExprVisitor(context)
+
+        ret = visitor.compile(node)
+
+        assert len(ret) == 1
+        python_node = ret[0].body.body
+        object_to_compare = {k: v.name for k, v in python_node.objects.items()}
+        assert python_node.source == "[ call_concept(__o_00__, x=x, y=k) for x in ['a', 'b'] ]"
+        assert object_to_compare == {"__o_00__": "foo x y"}
+
+    def test_i_can_compile_simple_list_comprehension_when_multiple_for(self):
+        sheerka, context = self.init_test().unpack()
+
+        expression = "[ (x, y) for x in ['a', 'b'] if x == 'a' for y in ['c', 'd'] if y == 'c' ]"
+        node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
+        visitor = PythonExprVisitor(context)
+
+        ret = visitor.compile(node)
+
+        assert len(ret) == 1
+        assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(ret[0].body.body, PythonNode)
+
+        python_node = ret[0].body.body
+        assert python_node.original_source == expression
+        assert python_node.source == expression
+
+        assert self.eval(context, ret[0]) == [("a", "c")]
+
+    def test_i_can_compile_and_when_multiple_results(self):
+        sheerka, context, foo, foo2, bar, bar2 = self.init_test().with_concepts(
+            Concept("foo x", body="x").def_var("x"),
+            Concept("foo y", body="y").def_var("y"),
+            Concept("bar x", body="x").def_var("x"),
+            Concept("bar y", body="y").def_var("y"),
+            create_new=True
+        ).unpack()
+
+        node = self.get_expr_node(context, "foo a and bar b")
+        visitor = PythonExprVisitor(context)
+        ret = visitor.compile(node)
+
+        assert len(ret) == 4
+        python_node = ret[0].body.body
+        object_to_compare = {k: v.name for k, v in python_node.objects.items()}
+        assert python_node.source == 'call_concept(__o_00__, x=a) and call_concept(__o_02__, x=b)'
+        assert object_to_compare == {"__o_00__": "foo x", "__o_02__": "bar x"}
+
+        python_node = ret[1].body.body
+        object_to_compare = {k: v.name for k, v in python_node.objects.items()}
+        assert python_node.source == 'call_concept(__o_00__, x=a) and call_concept(__o_03__, y=b)'
+        assert object_to_compare == {"__o_00__": "foo x", "__o_03__": "bar y"}
+
+        python_node = ret[2].body.body
+        object_to_compare = {k: v.name for k, v in python_node.objects.items()}
+        assert python_node.source == 'call_concept(__o_01__, y=a) and call_concept(__o_02__, x=b)'
+        assert object_to_compare == {"__o_01__": "foo y", "__o_02__": "bar x"}
+
+        python_node = ret[3].body.body
+        object_to_compare = {k: v.name for k, v in python_node.objects.items()}
+        assert python_node.source == 'call_concept(__o_01__, y=a) and call_concept(__o_03__, y=b)'
+        assert object_to_compare == {"__o_01__": "foo y", "__o_03__": "bar y"}
+
+    def test_i_can_compile_when_element_is_missing_its_parenthesis(self):
+        sheerka, context, foo = self.init_test().with_concepts(
+            Concept("foo x", body="x").def_var("x")
+        ).unpack()
+
+        expression = "[ w, foo w for w in ['a', 'b'] ]"
+        node = self.get_expr_node(context, expression, parser=ListComprehensionParser())
+        visitor = PythonExprVisitor(context)
+
+        ret = visitor.compile(node)
+
+        assert len(ret) == 1
+        assert sheerka.isinstance(ret[0], BuiltinConcepts.RETURN_VALUE)
+        assert sheerka.isinstance(ret[0].body, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(ret[0].body.body, PythonNode)
+
+        python_node = ret[0].body.body
+        assert python_node.original_source == expression
+        assert python_node.source == "[ (w, call_concept(__o_00__, x=w)) for w in ['a', 'b'] ]"
+        assert "__o_00__" in python_node.objects
+
+        concept = python_node.objects["__o_00__"]
+        assert sheerka.isinstance(concept, foo)
+        assert concept.get_hints().use_copy
+        assert concept.get_hints().is_evaluated
+
+        assert self.eval(context, ret[0]) == [("a", "a"), ("b", "b")]