Refactored sheerka class: splitted to use sub handlers. Refactored unit tests to use classes.

main.py

@@ -2,8 +2,8 @@ import getopt
 import sys
 import logging
 
-from core.sheerka import Sheerka
 import core.utils
+from core.sheerka.Sheerka import Sheerka
 
 
 def usage():

setup.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+from __future__ import absolute_import
+from __future__ import print_function
+
+from glob import glob
+from os.path import basename
+from os.path import splitext
+
+from setuptools import find_packages
+from setuptools import setup
+
+setup(
+    name='sheerka',
+    version='0.1',
+    license='',
+    description='A human/computer communication interface',
+    long_description='',
+    author='Kodjo Sossouvi',
+    author_email='kodjo.sossouvi@gmail.com',
+    url='',
+    packages=find_packages('src'),
+    package_dir={'': 'src'},
+    py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')],
+    include_package_data=True,
+    zip_safe=False,
+    classifiers=[
+        # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers
+        'Development Status :: 5 - Production/Stable',
+        'Intended Audience :: Developers',
+        'Operating System :: Unix',
+        'Operating System :: POSIX',
+        'Operating System :: Microsoft :: Windows',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: Implementation :: CPython',
+        'Programming Language :: Python :: Implementation :: PyPy',
+        'Topic :: Utilities',
+    ], install_requires=['pytest']
+)

src/core/sheerka/ExecutionContext.py

@@ -0,0 +1,203 @@
+import logging
+import time
+
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from sdp.sheerkaDataProvider import Event
+
+DEBUG_TAB_SIZE = 4
+
+
+class ExecutionContext:
+    """
+    To keep track of the execution of a request
+    """
+
+    ids = {}
+
+    @staticmethod
+    def get_id(event_digest):
+        if event_digest in ExecutionContext.ids:
+            ExecutionContext.ids[event_digest] += 1
+        else:
+            ExecutionContext.ids[event_digest] = 0
+        return ExecutionContext.ids[event_digest]
+
+    def __init__(self,
+                 who,
+                 event: Event,
+                 sheerka,
+                 desc: str = None,
+                 **kwargs):
+
+        self._parent = None
+        self._id = ExecutionContext.get_id(event.get_digest())
+        self._tab = ""
+        self._bag = {}  # other variables
+        self._start = 0
+        self._stop = 0
+
+        self.who = who  # who is asking
+        self.event = event  # what was the (original) trigger
+        self.sheerka = sheerka  # sheerka
+        self.desc = desc  # human description of what is going on
+        self.children = []
+        self.preprocess = None
+
+        self.inputs = {}  # what was the parameters of the execution context
+        self.values = {}  # what was produced by the execution context
+
+        self.obj = kwargs.pop("obj", None)
+        self.concepts = kwargs.pop("concepts", {})
+        # update the other elements
+        for k, v in kwargs.items():
+            self._bag[k] = v
+
+    @property
+    def elapsed(self):
+        if self._start == 0:
+            return 0
+
+        return (self._stop if self._stop > 0 else time.time_ns()) - self._start
+
+    @property
+    def elapsed_str(self):
+        nano_sec = self.elapsed
+        dt = nano_sec / 1e6
+        return f"{dt} ms" if dt < 1000 else f"{dt / 1000} s"
+
+    @property
+    def id(self):
+        return self._id
+
+    def __getattr__(self, item):
+        if item in self._bag:
+            return self._bag[item]
+
+        raise AttributeError(f"'ExecutionContext' object has no attribute '{item}'")
+
+    def __enter__(self):
+        self._start = time.time_ns()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._stop = time.time_ns()
+
+    def __repr__(self):
+        msg = f"ExecutionContext(who={self.who}, id={self._id}"
+        if self.desc:
+            msg += f", desc='{self.desc}'"
+        msg += ")"
+        return msg
+
+    def add_preprocess(self, name, **kwargs):
+        preprocess = self.sheerka.new(BuiltinConcepts.EVALUATOR_PRE_PROCESS)
+        preprocess.set_prop("name", name)
+        for k, v in kwargs.items():
+            preprocess.set_prop(k, v)
+
+        if not self.preprocess:
+            self.preprocess = set()
+        self.preprocess.add(preprocess)
+        return self
+
+    def add_inputs(self, **kwargs):
+        for k, v in kwargs.items():
+            self.inputs[k] = v
+        return self
+
+    def add_values(self, **kwargs):
+        for k, v in kwargs.items():
+            self.values[k] = v
+        return self
+
+    def get_concept(self, key):
+        # search in obj
+        if isinstance(self.obj, Concept):
+            if self.obj.key == key:
+                return self.obj
+            for prop in self.obj.props:
+                if prop == key:
+                    value = self.obj.props[prop].value
+                    if isinstance(value, Concept):
+                        return value
+
+        # search in concepts
+        if self.concepts:
+            for k, c in self.concepts.items():
+                if k == key:
+                    return c
+
+        return self.sheerka.get(key)
+
+    def new_concept(self, key, **kwargs):
+        # search in obj
+        if self.obj:
+            if self.obj.key == key:
+                return self.sheerka.new_from_template(self.obj, key, **kwargs)
+            for prop in self.obj.props:
+                if prop == key:
+                    value = self.obj.props[prop].value
+                    if isinstance(value, Concept):
+                        return self.sheerka.new_from_template(value, key, **kwargs)
+                    else:
+                        return value
+
+        if self.concepts:
+            for k, c in self.concepts.items():
+                if k == key:
+                    return self.sheerka.new_from_template(c, key, **kwargs)
+
+        return self.sheerka.new(key, **kwargs)
+
+    def push(self, who=None, desc=None, **kwargs):
+        who = who or self.who
+        _kwargs = {"obj": self.obj, "concepts": self.concepts}
+        _kwargs.update(self._bag)
+        _kwargs.update(kwargs)
+        new = ExecutionContext(
+            who,
+            self.event,
+            self.sheerka,
+            desc,
+            **_kwargs,
+        )
+        new._parent = self
+        new._tab = self._tab + " " * DEBUG_TAB_SIZE
+        new.preprocess = self.preprocess
+
+        self.children.append(new)
+        return new
+
+    def log_new(self, logger):
+        logger.debug(f"[{self._id:2}]" + self._tab + str(self))
+
+    def log(self, logger, message, who=None):
+        logger.debug(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message))
+
+    def log_error(self, logger, message, who=None):
+        logger.exception(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message))
+
+    def log_result(self, logger, return_values):
+        if not logger.isEnabledFor(logging.DEBUG):
+            return
+
+        if len(return_values) == 0:
+            logger.debug(self._tab + "No return value")
+
+        for r in return_values:
+            to_str = self.return_value_to_str(r)
+            logger.debug(f"[{self._id:2}]" + self._tab + "-> " + to_str)
+
+    def to_dict(self):
+        from core.sheerka_transform import SheerkaTransform
+        st = SheerkaTransform(self.sheerka)
+        return st.to_dict(self)
+
+    @staticmethod
+    def return_value_to_str(r):
+        value = str(r.value)
+        if len(value) > 50:
+            value = value[:47] + "..."
+        to_str = f"ReturnValue(who={r.who}, status={r.status}, value={value})"
+        return to_str

src/core/sheerka/Sheerka.py

@@ -0,0 +1,600 @@
+from core.builtin_concepts import BuiltinConcepts, ErrorConcept, ReturnValueConcept, BuiltinErrors, BuiltinUnique, \
+    UnknownConcept
+from core.concept import Concept, ConceptParts, PROPERTIES_FOR_NEW
+from core.sheerka.ExecutionContext import ExecutionContext
+from core.sheerka.SheerkaCreateNewConcept import SheerkaCreateNewConcept
+from core.sheerka.SheerkaDump import SheerkaDump
+from core.sheerka.SheerkaEvaluateConcept import SheerkaEvaluateConcept
+from core.sheerka.SheerkaExecute import SheerkaExecute
+from core.sheerka.SheerkaSetsManager import SheerkaSetsManager
+from sdp.sheerkaDataProvider import SheerkaDataProvider, Event
+import core.utils
+import core.builtin_helpers
+
+from core.sheerka_logger import console_handler
+
+import logging
+
+# CONCEPT_EVALUATION_STEPS = [
+#     BuiltinConcepts.BEFORE_EVALUATION,
+#     BuiltinConcepts.EVALUATION,
+#     BuiltinConcepts.AFTER_EVALUATION]
+
+CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser"
+
+
+class Sheerka(Concept):
+    """
+    Main controller for the project
+    """
+
+    CONCEPTS_ENTRY = "All_Concepts"  # to store all the concepts
+    CONCEPTS_BY_ID_ENTRY = "Concepts_By_ID"
+    CONCEPTS_DEFINITIONS_ENTRY = "Concepts_Definitions"  # to store definitions (bnf) of concepts
+    BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts"  # sequential key for builtin concepts
+    USER_CONCEPTS_KEYS = "User_Concepts"  # sequential key for user defined concepts
+
+    def __init__(self, skip_builtins_in_db=False, debug=False, loggers=None):
+        self.init_logging(debug, loggers)
+
+        super().__init__(BuiltinConcepts.SHEERKA, True, True, BuiltinConcepts.SHEERKA)
+        self.log.debug("Starting Sheerka.")
+
+        # cache of the most used concepts
+        # Note that these are only templates
+        # They are used as a footprint for instantiation
+        # Except of source when the concept is supposed to be unique
+        # key is the key of the concept (not the name or the id)
+        self.cache_by_key = {}
+        self.cache_by_id = {}
+
+        # cache for concept definitions,
+        # Primarily used for unit test that does not have access to sdp
+        self.concepts_definition_cache = {}
+
+        #
+        # cache for concepts grammars
+        # a grammar is a resolved BNF
+        self.concepts_grammars = {}
+
+        # a concept can be instantiated
+        # ex: File is a concept, but File('foo.txt') is an instance
+        # TODO: manage contexts
+        self.instances = []
+
+        # List of the known rules by the system
+        # ex: hello => say('hello')
+        self.rules = []
+
+        self.sdp: SheerkaDataProvider = None  # SheerkaDataProvider
+        self.builtin_cache = {}  # cache for builtin concepts
+        self.parsers = {}  # cache for builtin parsers
+        self.evaluators = []  # cache for builtin evaluators
+
+        self.evaluators_prefix: str = None
+        self.parsers_prefix: str = None
+
+        self.skip_builtins_in_db = skip_builtins_in_db
+
+        self.execute_handler = SheerkaExecute(self)
+        self.create_new_concept_handler = SheerkaCreateNewConcept(self)
+        self.dump_handler = SheerkaDump(self)
+        self.sets_handler = SheerkaSetsManager(self)
+        self.evaluate_concept_handler = SheerkaEvaluateConcept(self)
+
+    def initialize(self, root_folder: str = None):
+        """
+        Starting Sheerka
+        Loads the current configuration
+        Notes that when it's the first time, it also create the needed working folders
+        :param root_folder: root configuration folder
+        :return: ReturnValue(Success or Error)
+        """
+
+        try:
+            self.sdp = SheerkaDataProvider(root_folder)
+            if self.sdp.first_time:
+                self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
+
+            event = Event("Initializing Sheerka.")
+            self.sdp.save_event(event)
+            exec_context = ExecutionContext(self.key, event, self)
+
+            self.initialize_builtin_concepts()
+            self.initialize_builtin_parsers()
+            self.initialize_builtin_evaluators()
+            self.initialize_concepts_definitions(exec_context)
+
+        except IOError as e:
+            return ReturnValueConcept(self, False, self.get(BuiltinConcepts.ERROR), e)
+
+        return ReturnValueConcept(self, True, self)
+
+    def initialize_builtin_concepts(self):
+        """
+        Initializes the builtin concepts
+        :return: None
+        """
+        self.init_log.debug("Initializing builtin concepts")
+        builtins_classes = self.get_builtins_classes_as_dict()
+
+        # this all initialization of the builtins seems to be little bit complicated
+        # why do we need to update it from DB ?
+        for key in BuiltinConcepts:
+            concept = self if key == BuiltinConcepts.SHEERKA \
+                else builtins_classes[str(key)]() if str(key) in builtins_classes \
+                else Concept(key, True, False, key)
+
+            if key in BuiltinUnique:
+                concept.metadata.is_unique = True
+                concept.metadata.is_evaluated = True
+
+            if not concept.metadata.is_unique and str(key) in builtins_classes:
+                self.builtin_cache[key] = builtins_classes[str(key)]
+
+            if not self.skip_builtins_in_db:
+                from_db = self.sdp.get_safe(self.CONCEPTS_ENTRY, concept.metadata.key)
+                if from_db is None:
+                    self.init_log.debug(f"'{concept.name}' concept is not found in db. Adding.")
+                    self.set_id_if_needed(concept, True)
+                    self.sdp.add("init", self.CONCEPTS_ENTRY, concept, use_ref=True)
+                else:
+                    self.init_log.debug(f"Found concept '{from_db}' in db. Updating.")
+                    concept.update_from(from_db)
+
+            self.add_in_cache(concept)
+
+    def initialize_builtin_parsers(self):
+        """
+        Init the parsers
+        :return:
+        """
+        core.utils.init_package_import("parsers")
+        base_class = core.utils.get_class("parsers.BaseParser.BaseParser")
+        for parser in core.utils.get_sub_classes("parsers", base_class):
+            if parser.__module__ == base_class.__module__:
+                continue
+
+            self.init_log.debug(f"Adding builtin parser '{parser.__name__}'")
+            self.parsers[core.utils.get_full_qualified_name(parser)] = parser
+
+    def initialize_builtin_evaluators(self):
+        """
+        Init the evaluators
+        :return:
+        """
+        core.utils.init_package_import("evaluators")
+        for evaluator in core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.OneReturnValueEvaluator"):
+            self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
+            self.evaluators.append(evaluator)
+
+        for evaluator in core.utils.get_sub_classes("evaluators", "evaluators.BaseEvaluator.AllReturnValuesEvaluator"):
+            self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
+            self.evaluators.append(evaluator)
+
+    def initialize_concepts_definitions(self, execution_context):
+        self.init_log.debug("Initializing concepts definitions")
+        definitions = self.sdp.get_safe(self.CONCEPTS_DEFINITIONS_ENTRY, load_origin=False)
+
+        if definitions is None:
+            self.init_log.debug("No BNF defined")
+            return
+
+        lexer_parser = self.parsers[CONCEPT_LEXER_PARSER_CLASS]()
+        ret_val = lexer_parser.initialize(execution_context, definitions)
+        if not ret_val.status:
+            self.init_log.error("Failed to initialize concepts definitions " + str(ret_val.body))
+            return
+
+        self.concepts_grammars = lexer_parser.concepts_grammars
+
+    def reset_cache(self, filter_to_use=None):
+        """
+        reset the different cache that exists
+        :param filter_to_use:
+        :return:
+        """
+        if filter_to_use is None:
+            self.cache_by_key = {}
+            self.cache_by_id = {}
+        else:
+            raise NotImplementedError()
+
+        return self
+
+    def evaluate_user_input(self, text: str, user_name="kodjo"):
+        """
+        Note to KSI: If you try to add execution context to this function,
+        You may end in an infinite loop
+        :param text:
+        :param user_name:
+        :return:
+        """
+        self.log.debug(f"Processing user input '{text}', {user_name=}.")
+        event = Event(text, user_name)
+        evt_digest = self.sdp.save_event(event)
+        self.log.debug(f"{evt_digest=}")
+
+        with ExecutionContext(self.key, event, self, f"Evaluating '{text}'") as execution_context:
+            user_input = self.ret(self.name, True, self.new(BuiltinConcepts.USER_INPUT, body=text, user_name=user_name))
+            reduce_requested = self.ret(self.name, True, self.new(BuiltinConcepts.REDUCE_REQUESTED))
+
+            steps = [
+                BuiltinConcepts.BEFORE_PARSING,
+                BuiltinConcepts.PARSING,
+                BuiltinConcepts.AFTER_PARSING,
+                BuiltinConcepts.BEFORE_EVALUATION,
+                BuiltinConcepts.EVALUATION,
+                BuiltinConcepts.AFTER_EVALUATION
+            ]
+
+            ret = self.execute(execution_context, [user_input, reduce_requested], steps)
+            execution_context.add_values(return_values=ret)
+
+            if not self.skip_builtins_in_db:
+                self.sdp.save_result(execution_context)
+        return ret
+
+    def execute(self, execution_context, return_values, execution_steps, logger=None):
+        """
+        Executes process for all initial contexts
+        :param execution_context:
+        :param return_values:
+        :param execution_steps:
+        :param logger: logger to use (if not directly called by sheerka)
+        :return:
+        """
+        return self.execute_handler.execute(execution_context, return_values, execution_steps, logger)
+
+    def set_id_if_needed(self, obj: Concept, is_builtin: bool):
+        """
+        Set the key for the concept if needed
+        For test purpose only !!!!!
+        :param obj:
+        :param is_builtin:
+        :return:
+        """
+        if obj.metadata.id is not None:
+            return
+
+        entry = self.BUILTIN_CONCEPTS_KEYS if is_builtin else self.USER_CONCEPTS_KEYS
+        obj.metadata.id = self.sdp.get_next_key(entry)
+        self.log.debug(f"Setting id '{obj.metadata.id}' to concept '{obj.metadata.name}'.")
+
+    def create_new_concept(self, context, concept: Concept, logger=None):
+        """
+        Adds a new concept to the system
+        :param context:
+        :param concept:  DefConceptNode
+        :param logger
+        :return: digest of the new concept
+        """
+
+        return self.create_new_concept_handler.create_new_concept(context, concept, logger)
+
+    def add_concept_to_set(self, context, concept, concept_set, logger=None):
+        """
+        Add an entry in sdp to tell that concept isa concept_set
+        :param context:
+        :param concept:
+        :param concept_set:
+        :param logger:
+        :return:
+        """
+        return self.sets_handler.add_concept_to_set(context, concept, concept_set, logger)
+
+    def get_set_elements(self, concept):
+        """
+        Concept is supposed to be a set
+        Returns all elements if the set
+        :param concept:
+        :return:
+        """
+
+        return self.sets_handler.get_set_elements(concept)
+
+    def evaluate_concept(self, context, concept: Concept, logger=None):
+        """
+        Evaluation a concept
+        It means that if the where clause is True, will evaluate the body
+        :param context:
+        :param concept:
+        :param logger:
+        :return: value of the evaluation or error
+        """
+        return self.evaluate_concept_handler.evaluate_concept(context, concept, logger)
+
+    def add_in_cache(self, concept: Concept):
+        """
+        Adds a concept template in cache.
+        The cache is used as a proxy before looking at sdp
+        :param concept:
+        :return:
+        """
+
+        # sanity check
+        if concept.key is None:
+            concept.init_key()
+
+        if concept.key is None:
+            raise KeyError()
+
+        self.cache_by_key[concept.key] = concept
+
+        if concept.id:
+            self.cache_by_id[concept.id] = concept
+
+        return concept
+
+    def get(self, concept_key, concept_id=None):
+        """
+        Tries to find a concept
+        What is return must be used a template for another concept.
+        You must not modify the returned concept
+        :param concept_key: key of the concept
+        :param concept_id: when multiple concepts with the same key, use the id
+        :return:
+        """
+
+        if concept_key is None:
+            return ErrorConcept("Concept key is undefined.")
+
+        if isinstance(concept_key, BuiltinConcepts):
+            concept_key = str(concept_key)
+
+        # first search in cache
+        result = self.cache_by_key[concept_key] if concept_key in self.cache_by_key else \
+            self.sdp.get_safe(self.CONCEPTS_ENTRY, concept_key)
+
+        if result and (concept_id is None or not isinstance(result, list)):
+            return result
+
+        if isinstance(result, list):
+            if concept_id:
+                for c in result:
+                    if c.id == concept_id:
+                        return c
+            else:
+                return result
+
+        metadata = [("key", concept_key), ("id", concept_id)] if concept_id else ("key", concept_key)
+        return self._get_unknown(metadata)
+
+    def get_by_id(self, concept_id):
+        if concept_id is None:
+            return ErrorConcept("Concept id is undefined.")
+
+        # first search in cache
+        result = self.cache_by_id[concept_id] if concept_id in self.cache_by_id else \
+            self.sdp.get_safe(self.CONCEPTS_BY_ID_ENTRY, concept_id)
+
+        return result or self._get_unknown(('id', concept_id))
+
+    def get_concept_definition(self):
+        if self.concepts_definition_cache:
+            return self.concepts_definition_cache
+
+        self.concepts_definition_cache = self.sdp.get_safe(
+            self.CONCEPTS_DEFINITIONS_ENTRY,
+            load_origin=False) or {}
+        return self.concepts_definition_cache
+
+    def new(self, concept_key, **kwargs):
+        """
+        Returns an instance of a new concept
+        When the concept is supposed to be unique, returns the same instance
+        :param concept_key:
+        :param kwargs:
+        :return:
+        """
+        if isinstance(concept_key, tuple):
+            concept_key, concept_id = concept_key[0], concept_key[1]
+        else:
+            concept_id = None
+
+        template = self.get(concept_key, concept_id)
+
+        # manage concept not found
+        if self.isinstance(template, BuiltinConcepts.UNKNOWN_CONCEPT) and \
+            concept_key != BuiltinConcepts.UNKNOWN_CONCEPT:
+            return template
+
+        if isinstance(template, list):
+            # if template is a list, it means that there a multiple concepts under the same key
+            concepts = [self.new_from_template(t, concept_key, **kwargs) for t in template]
+            return concepts
+        else:
+            return self.new_from_template(template, concept_key, **kwargs)
+
+    def new_from_template(self, template, key, **kwargs):
+        # manage singleton
+        if template.metadata.is_unique:
+            return template
+
+        # otherwise, create another instance
+        concept = self.builtin_cache[key]() if key in self.builtin_cache else Concept()
+        concept.update_from(template)
+
+        if len(kwargs) == 0:
+            return concept
+
+        # update the properties, values, attributes
+        # Not quite sure that this is the correct process order
+        for k, v in kwargs.items():
+            if k in concept.props:
+                concept.set_prop(k, v)
+            elif k in PROPERTIES_FOR_NEW:
+                concept.values[ConceptParts(k)] = v
+            elif hasattr(concept, k):
+                setattr(concept, k, v)
+            else:
+                return self.new(BuiltinConcepts.UNKNOWN_PROPERTY, body=k, concept=concept)
+
+        # TODO : add the concept to the list of known concepts (self.instances)
+        concept.metadata.is_evaluated = True
+        return concept
+
+    def ret(self, who: str, status: bool, value, message=None, parents=None):
+        """
+        Creates and returns a ReturnValue concept
+        :param who:
+        :param status:
+        :param value:
+        :param message:
+        :param parents:
+        :return:
+        """
+        return self.new(
+            BuiltinConcepts.RETURN_VALUE,
+            who=who,
+            status=status,
+            value=value,
+            message=message,
+            parents=parents)
+
+    def value(self, obj, reduce_simple_list=False):
+        if obj is None:
+            return None
+
+        if hasattr(obj, "get_value"):
+            return obj.get_value()
+
+        if not isinstance(obj, Concept):
+            return obj
+
+        if obj.body is None:
+            return obj
+
+        if reduce_simple_list and (isinstance(obj.body, list) or isinstance(obj.body, set)) and len(obj.body) == 1:
+            body_to_use = obj.body[0]
+        else:
+            body_to_use = obj.body
+
+        return self.value(body_to_use)
+
+    def get_values(self, objs):
+        if not (isinstance(objs, list) or
+                self.isinstance(objs, BuiltinConcepts.LIST) or
+                self.isinstance(objs, BuiltinConcepts.ENUMERATION)):
+            objs = [objs]
+
+        return (self.value(obj) for obj in objs)
+
+    def is_success(self, obj):
+        if isinstance(obj, bool):  # quick win
+            return obj
+
+        if isinstance(obj, ReturnValueConcept):
+            return obj.status
+
+        if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors:
+            return False
+
+        return obj
+
+    def is_known(self, obj):
+        if not isinstance(obj, Concept):
+            return True
+
+        return obj.key != str(BuiltinConcepts.UNKNOWN_CONCEPT)
+
+    def isinstance(self, a, b):
+        """
+        return true if the concept a is an instance of the concept b
+        :param a:
+        :param b:
+        :return:
+        """
+
+        if isinstance(a, BuiltinConcepts):  # common KSI error ;-)
+            raise SyntaxError("Remember that the first parameter of isinstance MUST be a concept")
+
+        if not isinstance(a, Concept):
+            return False
+
+        b_key = b.key if isinstance(b, Concept) else str(b)
+
+        return a.key == b_key
+
+    def isa(self, a, b):
+        return self.sets_handler.isa(a, b)
+
+    def isagroup(self, concept):
+        return self.sets_handler.isagroup(concept)
+
+    def get_evaluator_name(self, name):
+        if self.evaluators_prefix is None:
+            base_evaluator_class = core.utils.get_class("evaluators.BaseEvaluator.BaseEvaluator")
+            self.evaluators_prefix = base_evaluator_class.PREFIX
+
+        return self.evaluators_prefix + name
+
+    def get_parser_name(self, name):
+        if self.parsers_prefix is None:
+            base_parser_class = core.utils.get_class("parsers.BaseParser.BaseParser")
+            self.parsers_prefix = base_parser_class.PREFIX
+
+        return self.parsers_prefix + name
+
+    def concepts(self):
+        res = []
+        lst = self.sdp.list(self.CONCEPTS_ENTRY)
+        for item in lst:
+            if isinstance(item, list):
+                res.extend(item)
+            else:
+                res.append(item)
+
+        return sorted(res, key=lambda i: int(i.id))
+
+    def test(self):
+        return f"I have access to Sheerka !"
+
+    def test_error(self):
+        raise Exception("I can raise an error")
+
+    @staticmethod
+    def _get_unknown(metadata):
+        """
+        Returns the concept 'UnknownConcept' for a requested id or key
+        Note that I don't call the new() method to prevent cyclic call
+        :param metadata:
+        :return:
+        """
+
+        # metadata is a list of tuple that contains the known metadata for this concept
+        # ex : (key, 'not_found)
+        # or
+        # (id, invalid_id)
+        #
+        # the metadata can be a list, if several attributes where given
+        # (key, 'not_found), (id, invalid_id)
+
+        unknown_concept = UnknownConcept()
+        unknown_concept.set_metadata_value(ConceptParts.BODY, metadata)
+        for meta in (metadata if isinstance(metadata, list) else [metadata]):
+            unknown_concept.set_prop(meta[0], meta[1])
+        unknown_concept.metadata.is_evaluated = True
+        return unknown_concept
+
+    @staticmethod
+    def get_builtins_classes_as_dict():
+        res = {}
+        for c in core.utils.get_classes("core.builtin_concepts"):
+            if issubclass(c, Concept) and c != Concept:
+                res[c().metadata.key] = c
+
+        return res
+
+    @staticmethod
+    def init_logging(debug, loggers):
+        core.sheerka_logger.set_enabled(loggers)
+        if debug:
+            # log_format = "%(asctime)s %(name)s [%(levelname)s] %(message)s"
+            log_format = "%(asctime)s [%(levelname)s] %(message)s"
+            log_level = logging.DEBUG
+        else:
+            log_format = "%(message)s"
+            log_level = logging.INFO
+
+        logging.basicConfig(format=log_format, level=log_level, handlers=[console_handler])

src/core/sheerka/SheerkaCreateNewConcept.py

@@ -0,0 +1,99 @@
+from core.builtin_concepts import BuiltinConcepts, ErrorConcept
+from core.concept import Concept
+from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError
+
+CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser"
+
+
+class SheerkaCreateNewConcept:
+    """
+    Manage the creation of a new concept
+    """
+
+    def __init__(self, sheerka):
+        self.sheerka = sheerka
+        self.logger_name = self.create_new_concept.__name__
+
+    def create_new_concept(self, context, concept: Concept, logger=None):
+        """
+        Adds a new concept to the system
+        :param context:
+        :param concept:  DefConceptNode
+        :param logger
+        :return: digest of the new concept
+        """
+
+        logger = logger or self.sheerka.log
+
+        concept.init_key()
+        concepts_definitions = None
+        init_ret_value = None
+
+        # checks for duplicate concepts
+        # TODO checks if it exists in cache first
+
+        if self.sheerka.sdp.exists(self.sheerka.CONCEPTS_ENTRY, concept.key, concept.get_digest()):
+            error = SheerkaDataProviderDuplicateKeyError(self.sheerka.CONCEPTS_ENTRY + "." + concept.key, concept)
+            return self.sheerka.ret(
+                self.logger_name,
+                False,
+                self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
+                error.args[0])
+
+        # set id before saving in db
+        self.sheerka.set_id_if_needed(concept, False)
+
+        # add the BNF if known
+        if concept.bnf:
+            concepts_definitions = self.sheerka.get_concept_definition()
+            concepts_definitions[concept] = concept.bnf
+
+            # check if it's a valid BNF or whether it breaks the known rules
+            concept_lexer_parser = self.sheerka.parsers[CONCEPT_LEXER_PARSER_CLASS]()
+            with context.push(self.sheerka.name, desc=f"Initializing concept definition for {concept}") as sub_context:
+                sub_context.concepts[concept.key] = concept  # the concept is not in the real cache yet
+                sub_context.log_new(logger)
+                init_ret_value = concept_lexer_parser.initialize(sub_context, concepts_definitions)
+                sub_context.add_values(return_values=init_ret_value)
+                if not init_ret_value.status:
+                    return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
+
+        # save the new concept in sdp
+        try:
+            # TODO : needs to make these calls atomic (or at least one single call)
+            self.sheerka.sdp.add(
+                context.event.get_digest(),
+                self.sheerka.CONCEPTS_ENTRY,
+                concept,
+                use_ref=True)
+            self.sheerka.sdp.add(
+                context.event.get_digest(),
+                self.sheerka.CONCEPTS_BY_ID_ENTRY,
+                {concept.id: concept.get_digest()},
+                is_ref=True)
+            if concepts_definitions is not None:
+                self.sheerka.sdp.set(
+                    context.event.get_digest(),
+                    self.sheerka.CONCEPTS_DEFINITIONS_ENTRY,
+                    concepts_definitions,
+                    use_ref=True)
+        except SheerkaDataProviderDuplicateKeyError as error:
+            context.log_error(logger, "Failed to create a new concept.", who=self.logger_name)
+            return self.sheerka.ret(
+                self.logger_name,
+                False,
+                self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_DEFINED, body=concept),
+                error.args[0])
+
+        # Updates the caches
+        self.sheerka.cache_by_key[concept.key] = self.sheerka.sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
+        self.sheerka.cache_by_id[concept.id] = concept
+        if init_ret_value is not None and init_ret_value.status:
+            self.sheerka.concepts_grammars = init_ret_value.body
+
+        # process the return in needed
+        ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
+        return ret
+
+
+

src/core/sheerka/SheerkaDump.py

@@ -0,0 +1,44 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+
+
+class SheerkaDump:
+    def __init__(self, sheerka):
+        self.sheerka = sheerka
+
+    def dump_concepts(self):
+        lst = self.sheerkasdp.list(self.sheerkaCONCEPTS_ENTRY)
+        for item in lst:
+            if hasattr(item, "__iter__"):
+                for i in item:
+                    self.sheerkalog.info(i)
+            else:
+                self.sheerkalog.info(item)
+
+    def dump_definitions(self):
+        defs = self.sheerkasdp.get(self.sheerkaCONCEPTS_DEFINITIONS_ENTRY)
+        self.sheerkalog.info(defs)
+
+    def dump_desc(self, *concept_names):
+        first = True
+        for concept_name in concept_names:
+            if isinstance(concept_name, Concept):
+                concepts = concept_name
+            else:
+                concepts = self.sheerkaget(concept_name)
+                if self.sheerkaisinstance(concepts, BuiltinConcepts.UNKNOWN_CONCEPT):
+                    self.sheerkalog.error(f"Concept '{concept_name}' is unknown")
+                    return False
+
+            if not hasattr(concepts, "__iter__"):
+                concepts = [concepts]
+
+            for c in concepts:
+                if not first:
+                    self.sheerkalog.info("")
+                self.sheerkalog.info(f"name   : {c.name}")
+                self.sheerkalog.info(f"bnf    : {c.metadata.definition}")
+                self.sheerkalog.info(f"key    : {c.key}")
+                self.sheerkalog.info(f"body   : {c.body}")
+                self.sheerkalog.info(f"digest : {c.get_digest()}")
+                first = False

src/core/sheerka/SheerkaEvaluateConcept.py

@@ -0,0 +1,195 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept, DoNotResolve, ConceptParts
+import core.builtin_helpers
+
+CONCEPT_EVALUATION_STEPS = [
+    BuiltinConcepts.BEFORE_EVALUATION,
+    BuiltinConcepts.EVALUATION,
+    BuiltinConcepts.AFTER_EVALUATION]
+
+
+class SheerkaEvaluateConcept:
+    def __init__(self, sheerka):
+        self.sheerka = sheerka
+        self.logger_name = self.evaluate_concept.__name__
+
+    def initialize_concept_asts(self, context, concept: Concept, logger=None):
+        """
+        Updates the codes of the newly created concept
+        Basically, it runs the parsers on all parts
+        :param concept:
+        :param context:
+        :param logger:
+        :return:
+        """
+        steps = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
+        for part_key in ConceptParts:
+            if part_key in concept.compiled:
+                continue
+
+            source = getattr(concept.metadata, part_key.value)
+            if source is None or not isinstance(source, str):
+                continue
+
+            if source.strip() == "":
+                concept.compiled[part_key] = DoNotResolve(source)
+            else:
+                with context.push(desc=f"Initializing compiled for {part_key}") as sub_context:
+                    sub_context.log_new(logger)
+                    sub_context.add_inputs(source=source)
+                    to_parse = self.sheerka.ret(context.who, True,
+                                                self.sheerka.new(BuiltinConcepts.USER_INPUT, body=source))
+                    res = self.sheerka.execute(sub_context, to_parse, steps, logger)
+                    concept.compiled[part_key] = res
+                    sub_context.add_values(return_values=res)
+
+        for prop, default_value in concept.metadata.props:
+            if prop in concept.compiled:
+                continue
+
+            if default_value is None or not isinstance(default_value, str):
+                continue
+
+            if default_value.strip() == "":
+                concept.compiled[prop] = DoNotResolve(default_value)
+            else:
+                with context.push(desc=f"Initializing AST for property {prop}") as sub_context:
+                    sub_context.log_new(logger)
+                    sub_context.add_inputs(source=default_value)
+                    to_parse = self.sheerka.ret(context.who, True,
+                                                self.sheerka.new(BuiltinConcepts.USER_INPUT, body=default_value))
+                    res = self.sheerka.execute(context, to_parse, steps)
+                    concept.compiled[prop] = res
+                    sub_context.add_values(return_values=res)
+
+        # Updates the cache of concepts when possible
+        if concept.key in self.sheerka.cache_by_key:
+            entry = self.sheerka.cache_by_key[concept.key]
+            if isinstance(entry, list):
+                # TODO : manage when there are multiple entries
+                pass
+            else:
+                self.sheerka.cache_by_key[concept.key].compiled = concept.compiled
+
+    def resolve(self, context, to_resolve, current_prop, current_concept, logger):
+        if isinstance(to_resolve, DoNotResolve):
+            return to_resolve.value
+
+        desc = f"Evaluating {current_prop} (concept={current_concept})"
+        context.log(logger, desc, self.logger_name)
+        with context.push(desc=desc, obj=current_concept) as sub_context:
+            sub_context.log_new(logger)
+
+            # when it's a concept, evaluate it
+            if isinstance(to_resolve, Concept) and \
+                not context.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE):
+                evaluated = self.evaluate_concept(sub_context, to_resolve, logger)
+                sub_context.add_values(return_values=evaluated)
+                if evaluated.key == to_resolve.key:
+                    return evaluated
+                else:
+                    error = evaluated
+
+            # otherwise, execute all return values to find out what is the value
+            else:
+                r = self.sheerka.execute(sub_context, to_resolve, CONCEPT_EVALUATION_STEPS, logger)
+                one_r = core.builtin_helpers.expect_one(context, r)
+                sub_context.add_values(return_values=one_r)
+                if one_r.status:
+                    return one_r.value
+                else:
+                    error = one_r.value
+
+            return self.sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR,
+                                    body=error,
+                                    concept=current_concept,
+                                    property_name=current_prop)
+
+    def resolve_list(self, context, list_to_resolve, current_prop, current_concept, logger):
+        """When dealing with a list, there are two possibilities"""
+        # It may be a list of ReturnValueConcept to execute (always the case for metadata)
+        # or a list of single values (may be the case for properties)
+        # in this latter case, all values are to be processed one by one and a list should be returned
+        if len(list_to_resolve) == 0:
+            return []
+
+        if self.sheerka.isinstance(list_to_resolve[0], BuiltinConcepts.RETURN_VALUE):
+            return self.resolve(context, list_to_resolve, current_prop, current_concept, logger)
+
+        res = []
+        for to_resolve in list_to_resolve:
+            # sanity check
+            if self.sheerka.isinstance(to_resolve, BuiltinConcepts.RETURN_VALUE):
+                return self.sheerka.new(BuiltinConcepts.CONCEPT_EVAL_ERROR,
+                                        body="Mix between real values and return values",
+                                        concept=current_concept,
+                                        property_name=current_prop)
+
+            r = self.resolve(context, to_resolve, current_prop, current_concept, logger)
+            if self.sheerka.isinstance(r, BuiltinConcepts.CONCEPT_EVAL_ERROR):
+                return r
+            res.append(r)
+
+        return res
+
+    def evaluate_concept(self, context, concept: Concept, logger=None):
+        """
+        Evaluation a concept
+        It means that if the where clause is True, will evaluate the body
+        :param context:
+        :param concept:
+        :param logger:
+        :return: value of the evaluation or error
+        """
+
+        logger = logger or self.sheerka.log
+
+        if concept.metadata.is_evaluated:
+            return concept
+
+        # WHERE condition should already be validated by the parser.
+        # It's a mandatory condition for the concept before it can be recognized
+
+        #
+        # TODO : Validate the PRE condition
+        #
+
+        self.initialize_concept_asts(context, concept, logger)
+
+        # to make sure of the order, it don't use ConceptParts.get_parts()
+        # props must be evaluated first
+        all_metadata_to_eval = ["props", "where", "pre", "post", "body"]
+
+        for metadata_to_eval in all_metadata_to_eval:
+            if metadata_to_eval == "props":
+                for prop_name in (p for p in concept.props if p in concept.compiled):
+                    prop_ast = concept.compiled[prop_name]
+
+                    if isinstance(prop_ast, list):
+                        # Do not send the current concept for the properties
+                        resolved = self.resolve_list(context, prop_ast, prop_name, None, logger)
+                    else:
+                        # Do not send the current concept for the properties
+                        resolved = self.resolve(context, prop_ast, prop_name, None, logger)
+                    if context.sheerka.isinstance(resolved, BuiltinConcepts.CONCEPT_EVAL_ERROR):
+                        resolved.set_prop("concept", concept)  # since current concept was not sent
+                        return resolved
+                    else:
+                        concept.set_prop(prop_name, resolved)
+            else:
+                part_key = ConceptParts(metadata_to_eval)
+                if part_key in concept.compiled and concept.compiled[part_key] is not None:
+                    metadata_ast = concept.compiled[part_key]
+                    resolved = self.resolve(context, metadata_ast, part_key, concept, logger)
+                    if context.sheerka.isinstance(resolved, BuiltinConcepts.CONCEPT_EVAL_ERROR):
+                        return resolved
+                    else:
+                        concept.values[part_key] = resolved
+
+        #
+        # TODO : Validate the POST condition
+        #
+
+        concept.init_key()  # only does it if needed
+        concept.metadata.is_evaluated = True
+        return concept

src/core/sheerka/SheerkaExecute.py

@@ -0,0 +1,254 @@
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
+import core.utils
+
+
+class SheerkaExecute:
+    """
+    Manage the execution of a process flow
+    """
+
+    def __init__(self, sheerka):
+        self.sheerka = sheerka
+
+    def call_parsers(self, execution_context, return_values, logger=None):
+
+        # return_values must be a list
+        if not isinstance(return_values, list):
+            return_values = [return_values]
+
+        # first make the distinguish between what is for the parsers and what is not
+        result = []
+        to_process = []
+        for r in return_values:
+            if not r.status or not self.sheerka.isinstance(r.body, BuiltinConcepts.USER_INPUT):
+                result.append(r)
+            else:
+                to_process.append(r)
+
+        if not to_process:
+            return result
+
+        # keep track of the originals user inputs, as they need to be removed at the end
+        user_inputs = to_process[:]
+
+        # group the parsers by priorities
+        instantiated_parsers = [parser(sheerka=self.sheerka) for parser in self.sheerka.parsers.values()]
+        grouped_parsers = {}
+        for parser in [p for p in instantiated_parsers if p.enabled]:
+            if logger:
+                parser.log = logger
+            grouped_parsers.setdefault(parser.priority, []).append(parser)
+        sorted_priorities = sorted(grouped_parsers.keys(), reverse=True)
+
+        stop_processing = False
+        for priority in sorted_priorities:
+            inputs_for_this_group = to_process[:]
+
+            for parser in grouped_parsers[priority]:
+
+                return_value_success_found = False
+                for return_value in inputs_for_this_group:
+
+                    to_parse = return_value.body.body \
+                        if self.sheerka.isinstance(return_value.body, BuiltinConcepts.USER_INPUT) \
+                        else return_value.body
+
+                    # if self.sheerka.log.isEnabledFor(logging.DEBUG):
+                    #     debug_text = "'" + to_parse + "'" if isinstance(to_parse, str) \
+                    #         else "'" + BaseParser.get_text_from_tokens(to_parse) + "' as tokens"
+                    #     execution_context.log(logger or self.sheerka.log, f"Parsing {debug_text}")
+
+                    with execution_context.push(desc=f"Parsing using {parser.name}") as sub_context:
+                        sub_context.add_inputs(to_parse=to_parse)
+                        res = parser.parse(sub_context, to_parse)
+                        if res is not None:
+                            if hasattr(res, "__iter__"):
+                                for r in res:
+                                    if r is None:
+                                        continue
+                                    r.parents = [return_value]
+                                    result.append(r)
+                                    if self.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT):
+                                        to_process.append(r)
+                                    if r.status:
+                                        return_value_success_found = True
+
+                            else:
+                                res.parents = [return_value]
+                                result.append(res)
+                                if self.sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT):
+                                    to_process.append(res)
+                                if res.status:
+                                    return_value_success_found = True
+                        sub_context.add_values(return_values=res)
+
+                    if return_value_success_found:
+                        stop_processing = True
+                        break  # Stop the other return_values (but not the other parsers with the same priority)
+
+            if stop_processing:
+                break  # Do not try the other priorities if a match is found
+
+        result = core.utils.remove_list_from_list(result, user_inputs)
+        return result
+
+    def call_evaluators(self, execution_context, return_values, process_step, evaluation_context=None, logger=None):
+
+        # return_values must be a list
+        if not isinstance(return_values, list):
+            return_values = [return_values]
+
+        # Evaluation context are contexts that may modify the behaviour of the execution
+        # For example, a concept to indicate that the value is not wanted
+        # Or a concept to indicate that we want the letter form of the response
+        # But first, they need to be transformed into return values
+        if evaluation_context is None:
+            evaluation_return_values = []
+        else:
+            evaluation_return_values = [self.sheerka.ret(execution_context.who, True, c) for c in evaluation_context]
+
+        # add the current step as part as the evaluation context
+        evaluation_return_values.append(self.sheerka.ret(execution_context.who, True, self.sheerka.new(process_step)))
+
+        # the pool of return values are the mix
+        return_values.extend(evaluation_return_values)
+
+        # group the evaluators by priority and sort them
+        # The first one to be applied will be the one with the highest priority
+        grouped_evaluators = {}
+        instantiated_evaluators = [e_class() for e_class in self.sheerka.evaluators]
+
+        # pre-process evaluators if needed
+        instantiated_evaluators = self._preprocess_evaluators(execution_context, instantiated_evaluators)
+
+        for evaluator in [e for e in instantiated_evaluators if e.enabled and process_step in e.steps]:
+            if logger:
+                evaluator.log = logger
+            grouped_evaluators.setdefault(evaluator.priority, []).append(evaluator)
+
+        # order the groups by priority, the higher first
+        sorted_priorities = sorted(grouped_evaluators.keys(), reverse=True)
+
+        # process
+        iteration = 0
+        while True:
+            with execution_context.push(desc=f"iteration #{iteration}", iteration=iteration) as iteration_context:
+                simple_digest = return_values[:]
+                iteration_context.add_inputs(return_values=simple_digest)
+
+                for priority in sorted_priorities:
+
+                    original_items = return_values[:]
+                    evaluated_items = []
+                    to_delete = []
+                    for evaluator in grouped_evaluators[priority]:
+                        evaluator = self._preprocess_evaluators(execution_context, evaluator.__class__())  # fresh copy
+
+                        sub_context_desc = f"Evaluating using {evaluator.name} ({priority=})"
+                        with iteration_context.push(desc=sub_context_desc) as sub_context:
+                            sub_context.add_inputs(return_values=original_items)
+
+                            # process evaluators that work on one simple return value at the time
+                            from evaluators.BaseEvaluator import OneReturnValueEvaluator
+                            if isinstance(evaluator, OneReturnValueEvaluator):
+                                debug_result = []
+                                for item in original_items:
+                                    if evaluator.matches(sub_context, item):
+                                        result = evaluator.eval(sub_context, item)
+                                        if result is None:
+                                            debug_result.append({"input": item, "return_value": None})
+                                            continue
+
+                                        to_delete.append(item)
+                                        if isinstance(result, list):
+                                            evaluated_items.extend(result)
+                                        elif isinstance(result, ReturnValueConcept):
+                                            evaluated_items.append(result)
+                                        else:
+                                            error = self.sheerka.new(BuiltinConcepts.INVALID_RETURN_VALUE, body=result,
+                                                                     evaluator=evaluator)
+                                            result = self.sheerka.ret("sheerka.process", False, error, parents=[item])
+                                            evaluated_items.append(result)
+                                        debug_result.append({"input": item, "return_value": result})
+                                    else:
+                                        debug_result.append({"input": item, "return_value": "** No Match **"})
+                                sub_context.add_values(return_values=debug_result)
+
+                            # process evaluators that work on all return values
+                            else:
+                                if evaluator.matches(sub_context, original_items):
+                                    results = evaluator.eval(sub_context, original_items)
+                                    if results is None:
+                                        continue
+                                    if not isinstance(results, list):
+                                        results = [results]
+                                    for result in results:
+                                        evaluated_items.append(result)
+                                        to_delete.extend(result.parents)
+                                    sub_context.add_values(return_values=results)
+                                else:
+                                    sub_context.add_values(return_values="** No Match **")
+
+                    return_values = evaluated_items
+                    return_values.extend([item for item in original_items if item not in to_delete])
+
+                iteration_context.add_values(return_values=return_values[:])
+
+            # have we done something ?
+            to_compare = return_values[:]
+            if simple_digest == to_compare:
+                break
+
+            # inc the iteration and continue
+            iteration += 1
+
+        # remove all evaluation context that are not reduced
+        return_values = core.utils.remove_list_from_list(return_values, evaluation_return_values)
+        return return_values
+
+    def execute(self, execution_context, return_values, execution_steps, logger=None):
+        """
+        Executes process for all initial contexts
+        :param execution_context:
+        :param return_values:
+        :param execution_steps:
+        :param logger: logger to use (if not directly called by sheerka)
+        :return:
+        """
+
+        for step in execution_steps:
+            copy = return_values[:] if hasattr(return_values, "__iter__") else [return_values]
+            with execution_context.push(step=step, iteration=0, desc=f"{step=}", return_values=copy) as sub_context:
+                sub_context.log(logger or self.sheerka.log, f"{step=}, context='{sub_context}'")
+
+                if step == BuiltinConcepts.PARSING:
+                    return_values = self.call_parsers(sub_context, return_values, logger)
+                else:
+                    return_values = self.call_evaluators(sub_context, return_values, step, None, logger)
+
+                if copy != return_values:
+                    sub_context.log_result(logger or self.sheerka.log, return_values)
+
+                sub_context.add_values(return_values=return_values)
+
+        return return_values
+
+    def _preprocess_evaluators(self, context, evaluators):
+        if not context.preprocess:
+            return evaluators
+
+        if not hasattr(evaluators, "__iter__"):
+            single_one = True
+            evaluators = [evaluators]
+        else:
+            single_one = False
+
+        for preprocess in context.preprocess:
+            for e in evaluators:
+                if preprocess.props["name"].value == e.name:
+                    for prop, value in preprocess.props.items():
+                        if prop == "name":
+                            continue
+                        if hasattr(e, prop):
+                            setattr(e, prop, value.value)
+        return evaluators[0] if single_one else evaluators

src/core/sheerka/SheerkaSetsManager.py

@@ -0,0 +1,83 @@
+from core.builtin_concepts import BuiltinConcepts, ErrorConcept
+from core.concept import Concept
+
+GROUP_PREFIX = 'All_'
+
+
+class SheerkaSetsManager:
+    def __init__(self, sheerka):
+        self.sheerka = sheerka
+        self.logger_name = self.add_concept_to_set.__name__
+
+    def add_concept_to_set(self, context, concept, concept_set, logger=None):
+        """
+        Add an entry in sdp to tell that concept isa concept_set
+        :param context:
+        :param concept:
+        :param concept_set:
+        :param logger:
+        :return:
+        """
+        logger = logger or self.sheerka.log
+
+        context.log(logger, f"Adding concept {concept} to set {concept_set}", who=self.logger_name)
+
+        assert concept.id
+        assert concept_set.id
+
+        try:
+            ret = self.sheerka.sdp.add_unique(context.event.get_digest(), GROUP_PREFIX + concept_set.id, concept.id)
+            if ret == (None, None):  # concept already in set
+                return self.sheerka.ret(
+                    self.logger_name,
+                    False,
+                    self.sheerka.new(BuiltinConcepts.CONCEPT_ALREADY_IN_SET, body=concept, concept_set=concept_set))
+            else:
+                return self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.SUCCESS))
+        except Exception as error:
+            context.log_error(logger, "Failed to add to set.", who=self.logger_name)
+            return self.sheerka.ret(self.logger_name, False, ErrorConcept(error), error.args[0])
+
+    def get_set_elements(self, concept):
+        """
+        Concept is supposed to be a set
+        Returns all elements if the set
+        :param concept:
+        :return:
+        """
+
+        assert concept.id
+
+        ids = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept.id)
+        if ids is None:
+            return self.sheerka.new(BuiltinConcepts.NOT_A_SET, body=concept)
+
+        elements = [self.sheerka.get_by_id(element_id) for element_id in ids]
+        return elements
+
+    def isa(self, a, b):
+        """
+        return true if the concept a is a b
+        Will handle when the keyword isa will be implemented
+        :param a:
+        :param b:
+        :return:
+        """
+
+        if isinstance(a, BuiltinConcepts):  # common KSI error ;-)
+            raise SyntaxError("Remember that the first parameter of isinstance MUST be a concept")
+
+        assert isinstance(a, Concept)
+        assert isinstance(b, Concept)
+
+        # TODO, first check the 'isa' property of a
+
+        return self.sheerka.sdp.exists(GROUP_PREFIX + b.id, a.id)
+
+    def isagroup(self, concept):
+        """True if exists All_<concept_id> in sdp"""
+        if not concept.id:
+            return None
+
+        res = self.sheerka.sdp.get_safe(GROUP_PREFIX + concept.id)
+        return res is not None

src/core/sheerka_transform.py

@@ -2,7 +2,7 @@ import dataclasses
 from enum import Enum
 
 from core.concept import Concept, PROPERTIES_TO_SERIALIZE
-from core.sheerka import ExecutionContext
+from core.sheerka.Sheerka import ExecutionContext
 from core.tokenizer import Token
 from evaluators.BaseEvaluator import BaseEvaluator
 from parsers.BaseParser import BaseParser, Node

src/evaluators/BaseEvaluator.py

@@ -1,4 +1,4 @@
-from core.sheerka import ExecutionContext
+from core.sheerka.Sheerka import ExecutionContext
 from core.sheerka_logger import get_logger
 
 

src/evaluators/PythonEvaluator.py

@@ -61,9 +61,9 @@ class PythonEvaluator(OneReturnValueEvaluator):
     def get_locals(self, context, node):
         my_locals = {
             "sheerka": context.sheerka,
-            "desc": context.sheerka.dump_desc,
-            "concepts": context.sheerka.dump_concepts,
-            "definitions": context.sheerka.dump_definitions,
+            "desc": context.sheerka.dump_handler.dump_desc,
+            "concepts": context.sheerka.dump_handler.dump_concepts,
+            "definitions": context.sheerka.dump_handler.dump_definitions,
         }
         if context.obj:
             context.log(self.verbose_log,

src/parsers/BnfParser.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass
 
 import core.utils
 from core.builtin_concepts import BuiltinConcepts
-from core.sheerka import ExecutionContext
+from core.sheerka.Sheerka import ExecutionContext
 from core.tokenizer import Tokenizer, Token, TokenKind, LexerError
 from parsers.BaseParser import BaseParser, ErrorNode, UnexpectedTokenErrorNode
 from parsers.ConceptLexerParser import OrderedChoice, Sequence, Optional, ZeroOrMore, OneOrMore, ConceptExpression, StrMatch

src/parsers/DefaultParser.py

@@ -6,7 +6,7 @@ from parsers.BaseParser import BaseParser, Node, ErrorNode, NotInitializedNode
 from core.tokenizer import Tokenizer, TokenKind, Token, Keywords
 from dataclasses import dataclass, field
 from parsers.BnfParser import BnfParser
-from core.sheerka import ExecutionContext
+from core.sheerka.Sheerka import ExecutionContext
 
 
 @dataclass()

src/sdp/sheerkaSerializer.py

@@ -252,7 +252,7 @@ class ExecutionContextSerializer(BaseSerializer):
         BaseSerializer.__init__(self, "R", 1)
 
     def matches(self, obj):
-        return core.utils.get_full_qualified_name(obj) == "core.sheerka.ExecutionContext"
+        return core.utils.get_full_qualified_name(obj) == "core.sheerka.ExecutionContext.ExecutionContext"
 
     def dump(self, stream, obj, context):
         as_json = obj.to_dict()

tests/BaseTest.py

@@ -0,0 +1,58 @@
+import ast
+
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept
+from core.concept import Concept
+from core.sheerka.ExecutionContext import ExecutionContext
+from sdp.sheerkaDataProvider import Event
+
+
+class BaseTest:
+    def get_sheerka(self):
+        pass
+
+    def get_context(self, sheerka=None):
+        return ExecutionContext("test", Event(), sheerka or self.get_sheerka())
+
+    def get_default_concept(self):
+        concept = Concept(
+            name="a + b",
+            where="isinstance(a, int) and isinstance(b, int)",
+            pre="isinstance(a, int) and isinstance(b, int)",
+            post="isinstance(res, int)",
+            body="def func(x,y):\n    return x+y\nfunc(a,b)",
+            desc="specific description")
+        concept.def_prop("a", "value1")
+        concept.def_prop("b", "value2")
+
+        return concept
+
+    def dump_ast(self, node):
+        dump = ast.dump(node)
+        for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]:
+            dump = dump.replace(to_remove, "")
+        return dump
+
+    @staticmethod
+    def retval(obj, who="who", status=True):
+        """ret_val"""
+        return ReturnValueConcept.ret(who, status, obj)
+
+    @staticmethod
+    def tretval(sheerka, obj, who="who"):
+        """True ret_val + add concept in cache"""
+        if isinstance(obj, Concept):
+            obj.init_key()
+            if obj.key not in sheerka.cache_by_key:
+                sheerka.cache_by_key[obj.key] = obj
+        return sheerka.ret(who, True, obj)
+
+    @staticmethod
+    def pretval(concept, source=None, parser="parsers.name"):
+        """ParserResult ret_val"""
+        return ReturnValueConcept(
+            "some_name",
+            True,
+            ParserResultConcept(parser=parser,
+                                source=source or concept.name,
+                                value=concept,
+                                try_parsed=concept))

tests/TestUsingFileBasedSheerka.py

@@ -0,0 +1,34 @@
+import os
+import shutil
+from os import path
+
+import pytest
+from core.sheerka.Sheerka import Sheerka
+from tests.BaseTest import BaseTest
+
+
+class TestUsingFileBasedSheerka(BaseTest):
+
+    tests_root = path.abspath("../../build/tests")
+    root_folder = "init_folder"
+
+    @pytest.fixture(autouse=True)
+    def init_test(self):
+        if path.exists(self.tests_root):
+            shutil.rmtree(self.tests_root)
+
+        if not path.exists(self.tests_root):
+            os.makedirs(self.tests_root)
+        current_pwd = os.getcwd()
+        os.chdir(self.tests_root)
+
+        yield None
+
+        os.chdir(current_pwd)
+
+    def get_sheerka(self, use_dict=True, skip_builtins_in_db=True):
+        root = "mem://" if use_dict else self.root_folder
+        sheerka = Sheerka(skip_builtins_in_db=skip_builtins_in_db)
+        sheerka.initialize(root)
+
+        return sheerka

tests/TestUsingMemoryBasedSheerka.py

@@ -0,0 +1,10 @@
+from core.sheerka.Sheerka import Sheerka
+from tests.BaseTest import BaseTest
+
+
+class TestUsingMemoryBasedSheerka(BaseTest):
+
+    def get_sheerka(self, skip_builtins_in_db=True):
+        sheerka = Sheerka(skip_builtins_in_db=skip_builtins_in_db)
+        sheerka.initialize("mem://")
+        return sheerka

tests/core/test_ExecutionContext.py

@@ -2,7 +2,7 @@ import pytest
 
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
-from core.sheerka import ExecutionContext
+from core.sheerka.Sheerka import ExecutionContext
 from sdp.sheerkaDataProvider import Event
 
 

tests/core/test_SheerkaCreateNewConcept.py

@@ -0,0 +1,180 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import PROPERTIES_TO_SERIALIZE
+from sdp.sheerkaDataProvider import SheerkaDataProvider
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaCreateNewConcept(TestUsingMemoryBasedSheerka):
+    def test_i_can_add_a_concept(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        assert res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
+
+        concept_found = res.value.body
+        for prop in PROPERTIES_TO_SERIALIZE:
+            assert getattr(concept_found.metadata, prop) == getattr(concept.metadata, prop)
+
+        assert concept_found.key == "__var__0 + __var__1"
+        assert concept_found.id == "1001"
+
+        assert concept.key in sheerka.cache_by_key
+        assert concept.id in sheerka.cache_by_id
+        assert sheerka.sdp.io.exists(
+            sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_digest()))
+
+    def test_i_cannot_add_the_same_concept_twice(self):
+        """
+        Checks that duplicated concepts are managed by sheerka, not by sheerka.sdp
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+        res = sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
+        assert res.value.body == concept
+
+    def test_i_can_get_a_newly_created_concept(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        from_cache = sheerka.get(concept.key)
+        assert from_cache is not None
+        assert from_cache == concept
+
+        from_cache = sheerka.get_by_id(concept.id)
+        assert from_cache is not None
+        assert from_cache == concept
+
+    def test_i_first_look_in_local_cache(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+        sheerka.cache_by_key[concept.key].pre = "I have modified the concept in cache"
+
+        from_cache = sheerka.get(concept.key)
+        assert from_cache is not None
+        assert from_cache.key == concept.key
+        assert from_cache.pre == "I have modified the concept in cache"
+
+    def test_i_can_get_a_known_concept_when_not_in_cache(self):
+        """
+        When not in cache, uses sdp
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        sheerka.cache_by_key = {}  # reset the cache
+        loaded = sheerka.get(concept.key)
+
+        assert loaded is not None
+        assert loaded == concept
+
+        # I can also get it by its id
+        loaded = sheerka.sdp.get(sheerka.CONCEPTS_BY_ID_ENTRY, concept.id)
+        assert loaded is not None
+        assert loaded == concept
+
+    def test_i_can_get_a_concept_by_its_id(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        sheerka.cache_by_key = {}  # reset the cache
+        loaded = sheerka.get_by_id(concept.id)
+
+        assert loaded is not None
+        assert loaded == concept
+
+    def test_i_can_get_list_of_concept_when_same_key_when_no_cache(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.metadata.body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        sheerka.cache_by_key = {}  # reset the cache
+
+        result = sheerka.get(concept1.key)
+        assert len(result) == 2
+        assert result[0] == concept1
+        assert result[1] == concept2
+
+    def test_i_can_get_list_of_concept_when_same_key_when_cache(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.metadata.body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        # sheerka.cache_by_key = {}  # Do not reset the cache
+
+        result = sheerka.get(concept1.key)
+        assert len(result) == 2
+        assert result[0] == concept1
+        assert result[1] == concept2
+
+    def test_i_can_get_the_correct_concept_using_the_id_when_same_key_when_no_cache(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.metadata.body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        result = sheerka.get(concept1.key, res2.body.body.id)
+        assert result.name == "a + b"
+        assert result.metadata.body == "a+b"
+
+    def test_i_can_get_the_correct_concept_using_the_id__when_same_key_when_cache(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.metadata.body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        result = sheerka.get(concept1.key, res2.body.body.id)
+        assert result.name == "a + b"
+        assert result.metadata.body == "a+b"
+
+    def test_i_cannot_get_the_correct_concept_id_the_id_is_wrong(self):
+        sheerka = self.get_sheerka()
+        concept1 = self.get_default_concept()
+        concept2 = self.get_default_concept()
+        concept2.metadata.body = "a+b"
+
+        res1 = sheerka.create_new_concept(self.get_context(sheerka), concept1)
+        res2 = sheerka.create_new_concept(self.get_context(sheerka), concept2)
+
+        assert res1.value.body.key == res2.value.body.key  # same key
+
+        result = sheerka.get(concept1.key, "wrong id")
+        assert sheerka.isinstance(result, BuiltinConcepts.UNKNOWN_CONCEPT)

tests/core/test_SheerkaEvaluateConcept.py

@@ -0,0 +1,332 @@
+import pytest
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, ParserResultConcept
+from core.concept import Concept, simplec, DoNotResolve, ConceptParts, Property
+from parsers.PythonParser import PythonNode
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("body, expected", [
+        # (None, None),
+        # ("", ""),
+        ("1", 1),
+        ("1+1", 2),
+        ("'one'", "one"),
+        ("'one' + 'two'", "onetwo"),
+        ("True", True),
+        ("1 > 2", False),
+    ])
+    def test_i_can_evaluate_a_concept_with_simple_body(self,body, expected):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo", body=body)
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == expected
+        assert evaluated.metadata.body == body
+        assert evaluated.metadata.pre is None
+        assert evaluated.metadata.post is None
+        assert evaluated.metadata.where is None
+        assert evaluated.props == {}
+        assert evaluated.metadata.is_evaluated
+        assert len(evaluated.values) == 0 if body is None else 1
+
+    @pytest.mark.parametrize("expr, expected", [
+        ("", ""),
+        ("1", 1),
+        ("1+1", 2),
+        ("'one'", "one"),
+        ("'one' + 'two'", "onetwo"),
+        ("True", True),
+        ("1 > 2", False),
+    ])
+    def test_i_can_evaluate_the_other_metadata(self,expr, expected):
+        """
+        I only test WHERE, it's the same for the others
+        :param expr:
+        :param expected:
+        :return:
+        """
+
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo", where=expr)
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.metadata.body is None
+        assert evaluated.metadata.pre is None
+        assert evaluated.metadata.post is None
+        assert evaluated.metadata.where == expr
+        assert evaluated.get_metadata_value(ConceptParts.WHERE) == expected
+        assert evaluated.props == {}
+        assert evaluated.metadata.is_evaluated
+        assert len(evaluated.values) == 0 if expr is None else 1
+
+    @pytest.mark.parametrize("expr, expected", [
+        (None, None),
+        ("", ""),
+        ("1", 1),
+        ("1+1", 2),
+        ("'one'", "one"),
+        ("'one' + 'two'", "onetwo"),
+        ("True", True),
+        ("1 > 2", False),
+    ])
+    def test_i_can_evaluate_a_concept_with_prop(self, expr, expected):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo").def_prop("a", expr)
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.metadata.pre is None
+        assert evaluated.metadata.pre is None
+        assert evaluated.metadata.post is None
+        assert evaluated.metadata.where is None
+        assert evaluated.props == {"a": Property("a", expected)}
+        assert evaluated.metadata.is_evaluated
+
+    def test_i_can_evaluate_metadata_using_do_not_resolve(self):
+        sheerka = self.get_sheerka()
+        concept = Concept("foo")
+        concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve")
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.body == "do not resolve"
+        assert evaluated.metadata.is_evaluated
+
+    def test_i_can_evaluate_property_using_do_not_resolve(self):
+        sheerka = self.get_sheerka()
+        concept = Concept("foo").def_prop("a")
+        concept.compiled["a"] = DoNotResolve("do not resolve")
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.get_prop("a") == "do not resolve"
+        assert evaluated.metadata.is_evaluated
+
+    def test_original_value_is_overridden_when_using_do_no_resolve(self):
+        sheerka = self.get_sheerka()
+        concept = Concept("foo", body="original value").set_prop("a", "original value")
+        concept.compiled["a"] = DoNotResolve("do not resolve")
+        concept.compiled[ConceptParts.BODY] = DoNotResolve("do not resolve")
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.body == "do not resolve"
+        assert evaluated.get_prop("a") == "do not resolve"
+        assert evaluated.metadata.is_evaluated
+
+    def test_props_are_evaluated_before_body(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo", body="a+1").def_prop("a", "10").init_key()
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == 11
+
+    def test_i_can_evaluate_when_another_concept_is_referenced(self):
+        sheerka = self.get_sheerka()
+        concept_a = Concept("a")
+        sheerka.add_in_cache(concept_a)
+
+        concept = Concept("foo", body="a").init_key()
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated == simplec("foo", simplec("a", None))
+        assert id(evaluated.body) != id(concept_a)
+        assert evaluated.metadata.is_evaluated
+        assert evaluated.body.metadata.is_evaluated
+
+    def test_i_can_evaluate_when_the_referenced_concept_has_a_body(self):
+        sheerka = self.get_sheerka()
+        concept_a = Concept("a", body="1")
+        sheerka.add_in_cache(concept_a)
+
+        concept = Concept("foo", body="a")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == simplec("a", 1)
+        assert not concept_a.metadata.is_evaluated
+        assert evaluated.metadata.is_evaluated
+
+    def test_i_can_evaluate_concept_of_concept_when_the_leaf_has_a_body(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="a", body="'a'"))
+        sheerka.add_in_cache(Concept(name="b", body="a"))
+        sheerka.add_in_cache(Concept(name="c", body="b"))
+        concept_d = sheerka.add_in_cache(Concept(name="d", body="c"))
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept_d)
+
+        assert evaluated.key == concept_d.key
+        expected = simplec("c", simplec("b", simplec("a", "a")))
+        assert evaluated.body == expected
+        assert sheerka.value(evaluated) == 'a'
+        assert evaluated.metadata.is_evaluated
+
+    def test_i_can_evaluate_concept_of_concept_does_not_have_a_body(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="a"))
+        sheerka.add_in_cache(Concept(name="b", body="a"))
+        sheerka.add_in_cache(Concept(name="c", body="b"))
+        concept_d = sheerka.add_in_cache(Concept(name="d", body="c"))
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept_d)
+
+        assert evaluated.key == concept_d.key
+        expected = simplec("c", simplec("b", simplec("a", None)))
+        assert evaluated.body == expected
+        assert sheerka.value(evaluated) == Concept(name="a").init_key()
+        assert evaluated.metadata.is_evaluated
+
+    def test_i_can_evaluate_concept_when_properties_reference_others_concepts(self):
+        sheerka = self.get_sheerka()
+        concept_a = sheerka.add_in_cache(Concept(name="a").init_key())
+
+        concept = Concept("foo", body="a").def_prop("a", "a").init_key()
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        # first prop a is evaluated to concept_a
+        # then body is evaluated to prop a -> concept_a
+        assert evaluated.key == concept.key
+        assert evaluated.body == concept_a
+
+    def test_i_can_evaluate_concept_when_properties_reference_others_concepts_2(self):
+        """
+        Same test,
+        but the name of the property and the name of the concept are different
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        concept_a = sheerka.add_in_cache(Concept(name="a"))
+
+        concept = Concept("foo", body="concept_a").def_prop("concept_a", "a")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == concept_a
+
+    def test_i_can_evaluate_concept_when_properties_reference_others_concepts_with_body(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="a", body="1"))
+        sheerka.add_in_cache(Concept(name="b", body="2"))
+
+        concept = Concept("foo", body="propA + propB").def_prop("propA", "a").def_prop("propB", "b")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == 3
+
+    def test_i_can_evaluate_concept_when_properties_is_a_concept(self):
+        sheerka = self.get_sheerka()
+        concept_a = sheerka.add_in_cache(Concept(name="a", body="'a'").init_key())
+
+        concept = Concept("foo").def_prop("a")
+        concept.compiled["a"] = concept_a
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.get_prop("a") == simplec("a", "a")
+
+    def test_i_can_evaluate_when_property_asts_is_a_list(self):
+        sheerka = self.get_sheerka()
+        foo = Concept("foo", body="1")
+
+        concept = Concept("to_eval").def_prop("prop")
+        concept.compiled["prop"] = [foo, DoNotResolve("1")]
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        props = evaluated.get_prop("prop")
+        assert len(props) == 2
+        assert props[0] == simplec("foo", 1)
+        assert props[1] == "1"
+
+    def test_i_can_evaluate_when_compiled_is_set_up_with_return_value(self):
+        sheerka = self.get_sheerka()
+
+        python_node = PythonNode("1 +1 ")
+        parser_result = ParserResultConcept(parser="who", value=python_node)
+
+        concept = Concept("to_eval").def_prop("prop")
+        concept.compiled["prop"] = [ReturnValueConcept("who", True, parser_result)]
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.get_prop("prop") == 2
+
+        # also works when only one return value
+        concept = Concept("to_eval").def_prop("prop")
+        concept.compiled["prop"] = ReturnValueConcept("who", True, parser_result)
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.get_prop("prop") == 2
+
+    def test_i_can_reference_sheerka(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo", body="sheerka.test()").init_key()
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.key
+        assert evaluated.body == sheerka.test()
+
+    def test_properties_values_takes_precedence_over_the_outside_world(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="a", body="'concept_a'"))
+        sheerka.add_in_cache(Concept(name="b", body="'concept_b'"))
+
+        concept = Concept("foo", body="a")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.key == concept.key
+        assert evaluated.body == simplec("a", "concept_a")  # this test was already done
+
+        # so check this one.
+        concept = Concept("foo", body="a").def_prop("a", "'property_a'")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.key == concept.key
+        assert evaluated.body == 'property_a'
+
+        # or this one.
+        concept = Concept("foo", body="a").def_prop("a", "b")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.key == concept.key
+        assert evaluated.body == simplec(name="b", body="concept_b")
+
+    def test_properties_values_takes_precedence(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="a", body="'concept_a'"))
+        sheerka.add_in_cache(Concept(name="b", body="'concept_b'"))
+
+        concept = Concept("foo", body="a + b").def_prop("a", "'prop_a'")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.key == concept.key
+        assert evaluated.body == 'prop_aconcept_b'
+
+    def test_i_can_reference_sub_property_of_a_property(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept(name="concept_a").def_prop("subProp", "'sub_a'"))
+
+        concept = Concept("foo", body="a.props['subProp'].value").def_prop("a", "concept_a")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated == simplec(concept.key, "sub_a")
+
+    def test_i_cannot_evaluate_concept_if_property_is_in_error(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept(name="concept_a").def_prop("subProp", "undef_concept")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert sheerka.isinstance(evaluated, BuiltinConcepts.CONCEPT_EVAL_ERROR)
+
+    def test_key_is_initialized_by_evaluation(self):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        assert evaluated.key == concept.init_key().key
+

tests/core/test_SheerkaSetsManager.py

@@ -0,0 +1,124 @@
+import os
+import shutil
+from os import path
+
+import pytest
+from core.builtin_concepts import ConceptAlreadyInSet, BuiltinConcepts
+from core.concept import Concept
+
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
+
+
+class TestSheerkaSetsManager(TestUsingFileBasedSheerka):
+
+
+
+    def test_i_can_add_concept_to_set(self):
+        sheerka = self.get_sheerka(False, False)
+
+        foo = Concept("foo")
+        sheerka.set_id_if_needed(foo, False)
+
+        all_foos = Concept("all_foos")
+        sheerka.set_id_if_needed(all_foos, False)
+
+        context = self.get_context(sheerka)
+        res = sheerka.add_concept_to_set(context, foo, all_foos)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        all_entries = self.get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False)
+        assert len(all_entries) == 1
+        assert foo.id in all_entries
+
+    def test_i_can_add_several_concepts_to_set(self):
+        sheerka = self.get_sheerka(False, False)
+
+        foo1 = Concept("foo1")
+        sheerka.set_id_if_needed(foo1, False)
+
+        foo2 = Concept("foo1")
+        sheerka.set_id_if_needed(foo2, False)
+
+        all_foos = Concept("all_foos")
+        sheerka.set_id_if_needed(all_foos, False)
+
+        context = self.get_context(sheerka)
+        sheerka.add_concept_to_set(context, foo1, all_foos)
+        res = sheerka.add_concept_to_set(context, foo2, all_foos)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        all_entries = self.get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False)
+        assert len(all_entries) == 2
+        assert foo1.id in all_entries
+        assert foo2.id in all_entries
+
+    def test_i_cannot_add_the_same_concept_twice_in_a_set(self):
+        sheerka = self.get_sheerka()
+
+        foo = Concept("foo")
+        sheerka.set_id_if_needed(foo, False)
+
+        all_foos = Concept("all_foos")
+        sheerka.set_id_if_needed(all_foos, False)
+
+        context = self.get_context(sheerka)
+        sheerka.add_concept_to_set(context, foo, all_foos)
+        res = sheerka.add_concept_to_set(context, foo, all_foos)
+
+        assert not res.status
+        assert res.body == ConceptAlreadyInSet(foo, all_foos)
+
+        all_entries = sheerka.sdp.get("All_" + all_foos.id, None, False)
+        assert len(all_entries) == 1
+        assert foo.id in all_entries
+
+    def test_i_get_elements_from_a_set(self):
+        sheerka = self.get_sheerka()
+
+        one = Concept("one")
+        two = Concept("two")
+        three = Concept("three")
+        number = Concept("number")
+
+        for c in [one, two, three, number]:
+            sheerka.set_id_if_needed(c, False)
+            sheerka.add_in_cache(c)
+
+        context = self.get_context(sheerka)
+        for c in [one, two, three]:
+            sheerka.add_concept_to_set(context, c, number)
+
+        elements = sheerka.get_set_elements(number)
+
+        assert set(elements) == {one, two, three}
+
+    def test_i_cannot_get_elements_if_not_a_set(self):
+        sheerka = self.get_sheerka()
+        one = Concept("one")
+        sheerka.set_id_if_needed(one, False)
+        sheerka.add_in_cache(one)
+
+        error = sheerka.get_set_elements(one)
+
+        assert sheerka.isinstance(error, BuiltinConcepts.NOT_A_SET)
+        assert error.body == one
+
+    def test_isa_and_isa_group(self):
+        sheerka = self.get_sheerka()
+
+        group = Concept("group").init_key()
+        group.metadata.id = "1001"
+        assert not sheerka.isagroup(group)
+
+        foo = Concept("foo").init_key()
+        foo.metadata.id = "1002"
+        assert not sheerka.isa(foo, group)
+
+        context = self.get_context(sheerka)
+        sheerka.add_concept_to_set(context, foo, group)
+        assert sheerka.isagroup(group)
+        assert sheerka.isa(foo, group)

tests/core/test_ast.py

@@ -6,7 +6,7 @@ from core.ast.nodes import NodeParent, GenericNodeConcept
 import core.ast.nodes
 from core.ast.visitors import ConceptNodeVisitor, UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts
-from core.sheerka import Sheerka
+from core.sheerka.Sheerka import Sheerka
 
 
 def get_sheerka():

tests/core/test_builtin_helpers.py

@@ -0,0 +1,145 @@
+import ast
+import pytest
+import core.builtin_helpers
+
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestBuiltinHelpers(TestUsingMemoryBasedSheerka):
+
+    def test_i_can_use_expect_one_when_empty(self):
+        sheerka = self.get_sheerka()
+
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), [])
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.IS_EMPTY)
+
+    def test_i_can_use_expect_one_when_too_many_success(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", True, "value1"),
+            ReturnValueConcept("who", True, "value2"),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_SUCCESS)
+        assert res.value.body == items
+        assert res.parents == items
+
+    def test_i_can_use_expect_one_when_same_success(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", True, "value"),
+            ReturnValueConcept("who", True, "value"),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert res.status
+        assert res.value == items[0].value
+        assert res.parents == items
+
+    def test_i_can_use_expect_when_only_errors_1(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", False, sheerka.new(BuiltinConcepts.ERROR)),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert not res.status
+        assert res.value, items[0]
+        assert res.parents == items
+
+    def test_i_can_use_expect_when_only_errors_2(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", False, None),
+            ReturnValueConcept("who", False, None),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert not res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.TOO_MANY_ERRORS)
+        assert res.value.body == items
+        assert res.parents == items
+
+    def test_i_can_use_expect_one_when_one_success_1(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", True, None),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert res.status
+        assert res.body == items[0].body
+        assert res.parents == items
+
+    def test_i_can_use_expect_one_when_one_success_2(self):
+        sheerka = self.get_sheerka()
+
+        items = [
+            ReturnValueConcept("who", False, None),
+            ReturnValueConcept("who", True, None),
+            ReturnValueConcept("who", False, None),
+        ]
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), items)
+        assert res.status
+        assert res.body == items[1].body
+        assert res.parents == items
+
+    def test_i_can_use_expect_one_when_not_a_list_true(self):
+        sheerka = self.get_sheerka()
+
+        item = ReturnValueConcept("who", True, None)
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), item)
+        assert res.status
+        assert res == item
+
+    def test_i_can_use_expect_one_when_not_a_list_false(self):
+        sheerka = self.get_sheerka()
+
+        item = ReturnValueConcept("who", False, None)
+        res = core.builtin_helpers.expect_one(self.get_context(sheerka), item)
+
+        assert not res.status
+        assert res == item
+
+    @pytest.mark.parametrize("expression, vars_to_include, vars_to_exclude, expected_expr", [
+        ("a == 1", [], [], []),
+        ("a == 1", ["a"], [], ["a == 1"]),
+        ("a == 1", [], ["a"], []),
+        ("predicate(a)", [], [], []),
+        ("predicate(a)", ["a"], [], ["predicate(a)"]),
+        ("predicate(a, b)", ["a"], [], ["predicate(a, b)"]),
+        ("predicate(a, b)", ["b"], [], ["predicate(a, b)"]),
+        ("predicate(a, b)", ["a", "b"], [], ["predicate(a, b)"]),
+        ("predicate(a, b)", ["a"], ["b"], []),
+        ("a + b == 1", [], [], []),
+        ("a + b == 1", ["a"], [], ["a + b == 1"]),
+        ("a + b == 1", ["a"], ["b"], []),
+        ("a + b == 1", ["b"], [], ["a + b == 1"]),
+        ("a + b == 1", ["a", "b"], [], ["a + b == 1"]),
+        ("a == 1 and b == 2", [], [], []),
+        ("a == 1 and b == 2", ["a"], [], ["a == 1"]),
+        ("a == 1 and b == 2", ["b"], [], ["b == 2"]),
+        ("a == 1 and b == 2", ["a"], ["b"], ["a == 1"]),
+        ("a == 1 and b == 2", ["a", "b"], [], ["a == 1 and b == 2"]),
+        ("predicate(a,c) and predicate(b,c)", ["a", "b"], [], ["predicate(a,c) and predicate(b,c)"]),
+        ("not(a == 1)", [], [], []),
+        ("not(a == 1)", ["a"], [], ["not(a==1)"]),
+        ("a == 1 or b == 2", [], [], []),
+        ("a == 1 or b == 2", ["a"], [], ["a == 1"]),
+        ("a == 1 or b == 2", ["b"], [], ["b == 2"]),
+        ("a == 1 or b == 2", ["a", "b"], [], ["a == 1 or b == 2"]),
+        ("predicate(a,c) or predicate(b,c)", ["a", "b"], [], ["predicate(a,c) or predicate(b,c)"]),
+    ])
+    def test_i_can_extract_predicates(self, expression, vars_to_include, vars_to_exclude, expected_expr):
+        sheerka = self.get_sheerka()
+        expected = [ast.parse(expr, mode="eval") for expr in expected_expr]
+
+        actual = core.builtin_helpers.extract_predicates(sheerka, expression, vars_to_include, vars_to_exclude)
+        assert len(actual) == len(expected)
+        for i in range(len(actual)):
+            assert self.dump_ast(actual[i]) == self.dump_ast(expected[i])

tests/core/test_sheerka.py

@@ -0,0 +1,244 @@
+import pytest
+import os
+
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE
+from core.sheerka.Sheerka import Sheerka
+
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
+
+
+class ConceptWithGetValue(Concept):
+    def get_value(self):
+        return self.get_prop("my_prop")
+
+
+class TestSheerka(TestUsingFileBasedSheerka):
+
+    def test_root_folder_is_created_after_initialization(self):
+        return_value = Sheerka().initialize(self.root_folder)
+        assert return_value.status, "initialisation should be successful"
+        assert os.path.exists(self.root_folder), "init folder should be created"
+
+    def test_i_can_list_builtin_concepts(self):
+        sheerka = self.get_sheerka()
+        builtins = list(sheerka.get_builtins_classes_as_dict())
+
+        assert str(BuiltinConcepts.ERROR) in builtins
+        assert str(BuiltinConcepts.RETURN_VALUE) in builtins
+
+    def test_builtin_concepts_are_initialized(self):
+        sheerka = self.get_sheerka(skip_builtins_in_db=False)
+        assert len(sheerka.cache_by_key) == len(BuiltinConcepts)
+        for concept_name in BuiltinConcepts:
+            assert str(concept_name) in sheerka.cache_by_key
+            assert sheerka.sdp.get_safe(sheerka.CONCEPTS_ENTRY, str(concept_name)) is not None
+
+        for key, concept_class in sheerka.get_builtins_classes_as_dict().items():
+            assert isinstance(sheerka.cache_by_key[key], concept_class)
+
+    def test_builtin_concepts_can_be_updated(self):
+        sheerka = self.get_sheerka(False, False)
+        loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA)
+        loaded_sheerka.metadata.desc = "I have a description"
+        sheerka.sdp.modify("Test", sheerka.CONCEPTS_ENTRY, loaded_sheerka.key, loaded_sheerka)
+
+        sheerka = self.get_sheerka(False, False)
+        loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA)
+
+        assert loaded_sheerka.metadata.desc == "I have a description"
+
+    def test_i_can_get_a_builtin_concept_by_their_enum_or_the_string(self):
+        """
+        Checks that a concept can be found its name
+        even when there are variables in the name (ex 'hello + a' or 'a + b' )
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        for key in sheerka.get_builtins_classes_as_dict():
+            assert sheerka.get(key) is not None
+            assert sheerka.get(str(key)) is not None
+
+    def test_i_cannot_get_when_key_is_none(self):
+        sheerka = self.get_sheerka()
+
+        res = sheerka.get(None)
+        assert sheerka.isinstance(res, BuiltinConcepts.ERROR)
+        assert res.body == "Concept key is undefined."
+
+    def test_unknown_concept_is_return_when_the_concept_key_is_not_found(self):
+        sheerka = self.get_sheerka()
+
+        loaded = sheerka.get("key_that_does_not_exist")
+
+        assert loaded is not None
+        assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert loaded.body == ("key", "key_that_does_not_exist")
+        assert loaded.metadata.is_evaluated
+
+    def test_unknown_concept_is_return_when_the_concept_id_is_not_found(self):
+        sheerka = self.get_sheerka()
+
+        loaded = sheerka.get_by_id("id_that_does_not_exist")
+
+        assert loaded is not None
+        assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert loaded.body == ("id", "id_that_does_not_exist")
+        assert loaded.metadata.is_evaluated
+
+    def test_i_can_instantiate_a_builtin_concept_when_it_has_its_own_class(self):
+        sheerka = self.get_sheerka()
+        ret = sheerka.new(BuiltinConcepts.RETURN_VALUE, who="who", status="status", value="value", message="message")
+
+        assert isinstance(ret, ReturnValueConcept)
+        assert ret.key == str(BuiltinConcepts.RETURN_VALUE)
+        assert ret.who == "who"
+        assert ret.status == "status"
+        assert ret.value == "value"
+        assert ret.message == "message"
+
+    def test_i_can_instantiate_a_builtin_concept_when_no_specific_class(self):
+        sheerka = self.get_sheerka()
+        ret = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body="fake_concept")
+
+        assert isinstance(ret, Concept)
+        assert ret.key == str(BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert ret.body == "fake_concept"
+
+    def test_i_can_instantiate_a_concept(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new = sheerka.new(concept.key, a=10, b="value")
+
+        assert sheerka.isinstance(new, concept)
+        for prop in PROPERTIES_TO_SERIALIZE:
+            assert getattr(new.metadata, prop) == getattr(concept.metadata, prop)
+
+        assert new.props["a"].value == 10
+        assert new.props["b"].value == "value"
+
+    def test_i_can_instantiate_with_the_name_and_the_id(self):
+        sheerka = self.get_sheerka()
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo2"))
+
+        concepts = sheerka.new("foo")
+        assert len(concepts) == 2
+
+        foo1 = sheerka.new(("foo", "1001"))
+        assert foo1.metadata.body == "foo1"
+
+        foo2 = sheerka.new(("foo", "1002"))
+        assert foo2.metadata.body == "foo2"
+
+    def test_instances_are_different_when_asking_for_new(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new1 = sheerka.new(concept.key, a=10, b="value")
+        new2 = sheerka.new(concept.key, a=10, b="value")
+
+        assert new1 == new2
+        assert id(new1) != id(new2)
+
+    def test_i_get_the_same_instance_when_is_unique_is_true(self):
+        sheerka = self.get_sheerka()
+        concept = Concept(name="unique", is_unique=True)
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new1 = sheerka.new(concept.key, a=10, b="value")
+        new2 = sheerka.new(concept.key, a=10, b="value")
+
+        assert new1 == new2
+        assert id(new1) == id(new2)
+
+    def test_i_cannot_instantiate_an_unknown_concept(self):
+        sheerka = self.get_sheerka()
+
+        new = sheerka.new("fake_concept")
+
+        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert new.body == ('key', 'fake_concept')
+
+    def test_i_cannot_instantiate_with_invalid_id(self):
+        sheerka = self.get_sheerka()
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo2"))
+
+        new = sheerka.new(("foo", "invalid_id"))
+
+        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert new.body == [('key', 'foo'), ('id', 'invalid_id')]
+
+    def test_i_cannot_instantiate_with_invalid_key(self):
+        sheerka = self.get_sheerka()
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo2"))
+
+        new = sheerka.new(("invalid_key", "1001"))
+
+        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert new.body == [('key', 'invalid_key'), ('id', '1001')]
+
+    def test_concept_id_is_irrelevant_when_only_one_concept(self):
+        sheerka = self.get_sheerka()
+        sheerka.create_new_concept(self.get_context(sheerka), Concept("foo", body="foo1"))
+
+        new = sheerka.new(("foo", "invalid_id"))
+
+        assert sheerka.isinstance(new, "foo")
+        assert new.metadata.body == "foo1"
+
+    def test_i_cannot_instantiate_when_properties_are_not_recognized(self):
+        sheerka = self.get_sheerka()
+        concept = self.get_default_concept()
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new = sheerka.new(concept.key, a=10, c="value")
+
+        assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_PROPERTY)
+        assert new.property_name == "c"
+        assert sheerka.isinstance(new.concept, concept)
+
+    @pytest.mark.parametrize("concept, reduce_simple_list, expected", [
+        (None, False, None),
+        (3.14, False, 3.14),
+        ("foo", False, "foo"),
+        (True, False, True),
+        (Concept("name", body="foo"), False, "foo"),
+        (Concept("name"), False, Concept("name")),
+        (ConceptWithGetValue("name").set_prop("my_prop", "my_value"), False, "my_value"),
+        (ReturnValueConcept(value="return_value"), False, "return_value"),
+        (ReturnValueConcept(value=Concept(key=BuiltinConcepts.USER_INPUT, body="text"), status=True), False, "text"),
+        (ReturnValueConcept(value=UserInputConcept("text"), status=True), False, "text"),
+        (Concept("name", body=["foo", "bar"]), False, ["foo", "bar"]),
+        (Concept("name", body=["foo"]), True, "foo"),
+        (Concept("name", body=Concept("foo")), False, Concept("foo")),
+        (Concept("name", body=Concept("foo", body="value")), False, "value"),
+        (Concept("name", body=Concept("foo", body=Concept("bar", body="value"))), False, "value"),
+        (Concept("name", body=Concept("foo", body=ReturnValueConcept(value="return_value"))), False, "return_value"),
+    ])
+    def test_i_can_get_value(self, concept, reduce_simple_list, expected):
+        sheerka = self.get_sheerka()
+
+        # I use auto_init() instead of evaluate_concept() to be quicker
+        c = concept
+        while isinstance(c, Concept):
+            c.auto_init()
+            c = c.body
+
+        assert sheerka.value(concept, reduce_simple_list) == expected
+
+    def test_list_of_concept_is_sorted_by_id(self):
+        sheerka = self.get_sheerka(False, False)
+        concepts = sheerka.concepts()
+
+        assert concepts[0].id < concepts[-1].id
+
+    def test_builtin_error_concept_are_errors(self):
+        # only test a random one, it will be the same for the others
+        sheerka = self.get_sheerka()
+        assert not sheerka.is_success(sheerka.new(BuiltinConcepts.TOO_MANY_SUCCESS))

tests/core/test_sheerka_call_evaluators.py

@@ -0,0 +1,375 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from evaluators.BaseEvaluator import OneReturnValueEvaluator, BaseEvaluator, AllReturnValuesEvaluator
+
+from tests.BaseTest import BaseTest
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class Out:
+    debug_out = []
+
+    def out(self, method, name, context, return_value):
+        name = name[len(BaseEvaluator.PREFIX):]
+        if isinstance(return_value, list):
+            target = [str(r.body.key) for r in return_value]
+        else:
+            target = str(return_value.body.key)
+        step = str(context.step)
+        text = f"{step} [{context.iteration}] "
+        text += f"{name} - {method} - target={target}"
+        self.debug_out.append(text)
+
+    def out_all(self, method, name, context, return_values):
+        name = name[len(BaseEvaluator.PREFIX):]
+        target = [str(r.body.key) for r in return_values]
+        step = str(context.step)
+        text = f"{step} [{context.iteration}] "
+        text += f"{name} - {method} - target={target}"
+        self.debug_out.append(text)
+
+
+class OneReturnValueEvaluatorForTestingPurpose(OneReturnValueEvaluator, Out):
+    def __init__(self, name, steps, priority):
+        super().__init__(name, steps, priority)
+
+    def matches(self, context, return_value):
+        self.out("matches", self.name, context, return_value)
+        return True
+
+    def eval(self, context, return_value):
+        self.out("eval", self.name, context, return_value)
+
+
+class AllReturnValueEvaluatorForTestingPurpose(AllReturnValuesEvaluator, Out):
+    def __init__(self, name, steps, priority):
+        super().__init__(name, steps, priority)
+
+    def matches(self, context, return_values):
+        self.out("matches", self.name, context, return_values)
+        return True
+
+    def eval(self, context, return_values):
+        self.out("eval", self.name, context, return_values)
+
+
+class EvaluatorOneWithPriority(OneReturnValueEvaluatorForTestingPurpose):
+    def __init__(self, name, priority):
+        super().__init__(name, [BuiltinConcepts.EVALUATION], priority)
+
+
+class EvaluatorOneWithPriority10(EvaluatorOneWithPriority):
+    def __init__(self):
+        super().__init__("priority10", 10)
+
+
+class EvaluatorOneWithPriority15(EvaluatorOneWithPriority):
+    def __init__(self):
+        super().__init__("priority15", 15)
+
+
+class EvaluatorOneWithPriority20(EvaluatorOneWithPriority):
+    def __init__(self):
+        super().__init__("priority20", 20)
+
+
+class EvaluatorAllWithPriority(AllReturnValueEvaluatorForTestingPurpose):
+    def __init__(self, name, priority):
+        super().__init__(name, [BuiltinConcepts.EVALUATION], priority)
+
+
+class EvaluatorAllWithPriority10(EvaluatorAllWithPriority):
+    def __init__(self):
+        super().__init__("all_priority10", 10)
+
+
+class EvaluatorAllWithPriority15(EvaluatorAllWithPriority):
+    def __init__(self):
+        super().__init__("all_priority15", 15)
+
+
+class EvaluatorAllWithPriority20(EvaluatorAllWithPriority):
+    def __init__(self):
+        super().__init__("all_priority20", 20)
+
+
+class EvaluatorOneModifyFoo(EvaluatorOneWithPriority):
+    def __init__(self):
+        super().__init__("modifyFoo", 10)
+
+    def matches(self, context, return_value):
+        super().matches(context, return_value)
+        return context.sheerka.isinstance(return_value.body, "foo")
+
+    def eval(self, context, return_value):
+        super().eval(context, return_value)
+        return BaseTest.tretval(context.sheerka, Concept("bar"))
+
+
+class EvaluatorOneModifyBar(EvaluatorOneWithPriority):
+    def __init__(self):
+        super().__init__("modifyBar", 10)
+
+    def matches(self, context, return_value):
+        super().matches(context, return_value)
+        return context.sheerka.isinstance(return_value.body, "bar")
+
+    def eval(self, context, return_value):
+        super().eval(context, return_value)
+        return BaseTest.tretval(context.sheerka, Concept("baz"))
+
+
+class EvaluatorOnePreEvaluation(OneReturnValueEvaluatorForTestingPurpose):
+    def __init__(self):
+        super().__init__("preEval", [BuiltinConcepts.BEFORE_EVALUATION], 10)
+
+
+class EvaluatorOneMultiSteps(OneReturnValueEvaluatorForTestingPurpose):
+    def __init__(self):
+        super().__init__("multiStep", [BuiltinConcepts.EVALUATION, BuiltinConcepts.BEFORE_EVALUATION], 10)
+
+
+class EvaluatorAllReduceFooBar(EvaluatorAllWithPriority):
+    def __init__(self):
+        super().__init__("all_reduce_foobar", 10)
+
+    def matches(self, context, return_values):
+        super().matches(context, return_values)
+        keys = [c.body.key for c in return_values]
+        return "foo" in keys and "bar" in keys
+
+    def eval(self, context, return_values):
+        super().eval(context, return_values)
+        ret = BaseTest.tretval(context.sheerka, context.sheerka.new(BuiltinConcepts.SUCCESS))
+        ret.parents = return_values
+        return ret
+
+
+class EvaluatorAllSuppressFooEntry(EvaluatorAllWithPriority):
+
+    def __init__(self):
+        super().__init__("suppress", 100)
+
+    def matches(self, context, return_values):
+        super().matches(context, return_values)
+        return True
+
+    def eval(self, context, return_values):
+        super().eval(context, return_values)
+        foo = None
+        for ret in return_values:
+            if ret.body.name == "foo":
+                foo = ret
+
+        if foo:
+            return context.sheerka.ret(self.name, False, Concept("does not matter"), parents=[foo])
+        else:
+            return None
+
+
+class TestSheerkaExecuteEvaluators(TestUsingMemoryBasedSheerka):
+
+    def test_that_return_values_is_unchanged_when_no_evaluator(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = []
+
+        entries = self.tretval(sheerka, Concept("foo"))
+        return_values = sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert return_values == [entries]
+
+    def test_i_can_use_a_list_as_input(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = []
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+        return_values = sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert return_values == entries
+
+    def test_step_concept_is_removed_after_processing_if_not_reduced(self):
+        """
+        No evaluator
+        """
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = []
+
+        entry = self.tretval(sheerka, Concept("foo"))
+        return_values = sheerka.execute(self.get_context(sheerka), entry, [BuiltinConcepts.EVALUATION])
+
+        assert BuiltinConcepts.EVALUATION not in [r.body.key for r in return_values]
+
+    def test_step_concept_is_removed_after_processing_if_not_reduced_2(self):
+        """
+        This time the entry is modified by an evaluator,
+        nevertheless, step concept is removed
+        """
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneModifyFoo]
+
+        entry = self.tretval(sheerka, Concept("foo"))
+        return_values = sheerka.execute(self.get_context(sheerka), entry, [BuiltinConcepts.EVALUATION])
+
+        assert BuiltinConcepts.EVALUATION not in [r.body.key for r in return_values]
+
+    def test_that_higher_priority_evaluators_are_evaluated_first(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [
+            EvaluatorAllWithPriority10,
+            EvaluatorOneWithPriority20,
+            EvaluatorAllWithPriority15,
+            EvaluatorOneWithPriority10,
+            EvaluatorOneWithPriority15,
+            EvaluatorAllWithPriority20, ]
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert Out.debug_out == [
+            '__EVALUATION [0] priority20 - matches - target=foo',
+            '__EVALUATION [0] priority20 - eval - target=foo',
+            '__EVALUATION [0] priority20 - matches - target=__EVALUATION',
+            '__EVALUATION [0] priority20 - eval - target=__EVALUATION',
+            "__EVALUATION [0] all_priority20 - matches - target=['foo', '__EVALUATION']",
+            "__EVALUATION [0] all_priority20 - eval - target=['foo', '__EVALUATION']",
+            "__EVALUATION [0] all_priority15 - matches - target=['foo', '__EVALUATION']",
+            "__EVALUATION [0] all_priority15 - eval - target=['foo', '__EVALUATION']",
+            '__EVALUATION [0] priority15 - matches - target=foo',
+            '__EVALUATION [0] priority15 - eval - target=foo',
+            '__EVALUATION [0] priority15 - matches - target=__EVALUATION',
+            '__EVALUATION [0] priority15 - eval - target=__EVALUATION',
+            "__EVALUATION [0] all_priority10 - matches - target=['foo', '__EVALUATION']",
+            "__EVALUATION [0] all_priority10 - eval - target=['foo', '__EVALUATION']",
+            '__EVALUATION [0] priority10 - matches - target=foo',
+            '__EVALUATION [0] priority10 - eval - target=foo',
+            '__EVALUATION [0] priority10 - matches - target=__EVALUATION',
+            '__EVALUATION [0] priority10 - eval - target=__EVALUATION'
+        ]
+
+    def test_that_predicate_is_checked_before_evaluation_for_one_return(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneModifyFoo]
+
+        entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert Out.debug_out == [
+            '__EVALUATION [0] modifyFoo - matches - target=foo',
+            '__EVALUATION [0] modifyFoo - eval - target=foo',
+            '__EVALUATION [0] modifyFoo - matches - target=baz',
+            '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION',
+            '__EVALUATION [1] modifyFoo - matches - target=bar',
+            '__EVALUATION [1] modifyFoo - matches - target=baz',
+            '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION'
+        ]
+
+    def test_that_predicate_is_checked_before_evaluation_for_all_return(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorAllReduceFooBar]
+
+        entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("bar"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+        assert Out.debug_out == [
+            "__EVALUATION [0] all_reduce_foobar - matches - target=['foo', 'bar', '__EVALUATION']",
+            "__EVALUATION [0] all_reduce_foobar - eval - target=['foo', 'bar', '__EVALUATION']",
+            "__EVALUATION [1] all_reduce_foobar - matches - target=['__SUCCESS']"
+        ]
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+        assert Out.debug_out == [
+            "__EVALUATION [0] all_reduce_foobar - matches - target=['foo', '__EVALUATION']"
+        ]
+
+    def test_evaluation_continue_until_no_more_modification(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneModifyFoo, EvaluatorOneModifyBar]
+
+        entries = [self.tretval(sheerka, Concept("foo")), self.tretval(sheerka, Concept("baz"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+
+        assert Out.debug_out == [
+            '__EVALUATION [0] modifyFoo - matches - target=foo',
+            '__EVALUATION [0] modifyFoo - eval - target=foo',
+            '__EVALUATION [0] modifyFoo - matches - target=baz',
+            '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION',
+            '__EVALUATION [0] modifyBar - matches - target=foo',
+            '__EVALUATION [0] modifyBar - matches - target=baz',
+            '__EVALUATION [0] modifyBar - matches - target=__EVALUATION',
+            '__EVALUATION [1] modifyFoo - matches - target=bar',
+            '__EVALUATION [1] modifyFoo - matches - target=baz',
+            '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION',
+            '__EVALUATION [1] modifyBar - matches - target=bar',
+            '__EVALUATION [1] modifyBar - eval - target=bar',
+            '__EVALUATION [1] modifyBar - matches - target=baz',
+            '__EVALUATION [1] modifyBar - matches - target=__EVALUATION',
+            '__EVALUATION [2] modifyFoo - matches - target=baz',
+            '__EVALUATION [2] modifyFoo - matches - target=baz',
+            '__EVALUATION [2] modifyFoo - matches - target=__EVALUATION',
+            '__EVALUATION [2] modifyBar - matches - target=baz',
+            '__EVALUATION [2] modifyBar - matches - target=baz',
+            '__EVALUATION [2] modifyBar - matches - target=__EVALUATION'
+        ]
+
+    def test_evaluation_steps_are_respected(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneWithPriority10, EvaluatorOnePreEvaluation]
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.BEFORE_EVALUATION])
+
+        assert Out.debug_out == [
+            '__BEFORE_EVALUATION [0] preEval - matches - target=foo',
+            '__BEFORE_EVALUATION [0] preEval - eval - target=foo',
+            '__BEFORE_EVALUATION [0] preEval - matches - target=__BEFORE_EVALUATION',
+            '__BEFORE_EVALUATION [0] preEval - eval - target=__BEFORE_EVALUATION']
+
+    def test_evaluation_multi_steps_are_respected(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneMultiSteps]
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries,
+                        [BuiltinConcepts.BEFORE_EVALUATION, BuiltinConcepts.EVALUATION])
+
+        assert Out.debug_out == [
+            '__BEFORE_EVALUATION [0] multiStep - matches - target=foo',
+            '__BEFORE_EVALUATION [0] multiStep - eval - target=foo',
+            '__BEFORE_EVALUATION [0] multiStep - matches - target=__BEFORE_EVALUATION',
+            '__BEFORE_EVALUATION [0] multiStep - eval - target=__BEFORE_EVALUATION',
+            '__EVALUATION [0] multiStep - matches - target=foo',
+            '__EVALUATION [0] multiStep - eval - target=foo',
+            '__EVALUATION [0] multiStep - matches - target=__EVALUATION',
+            '__EVALUATION [0] multiStep - eval - target=__EVALUATION'
+        ]
+
+    def test_evaluators_can_be_pre_processed(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluators = [EvaluatorOneModifyFoo]
+
+        entries = [self.tretval(sheerka, Concept("foo"))]
+
+        # disable evaluator
+        context = self.get_context(sheerka)
+        context.add_preprocess(EvaluatorOneModifyFoo().name, enabled=False)  # disabled  for this exec context
+        Out.debug_out = []
+        sheerka.execute(context, entries, [BuiltinConcepts.EVALUATION])
+        assert Out.debug_out == []
+
+        # other contextes are not impacted
+        Out.debug_out = []
+        sheerka.execute(self.get_context(sheerka), entries, [BuiltinConcepts.EVALUATION])
+        assert Out.debug_out == [
+            '__EVALUATION [0] modifyFoo - matches - target=foo',
+            '__EVALUATION [0] modifyFoo - eval - target=foo',
+            '__EVALUATION [0] modifyFoo - matches - target=__EVALUATION',
+            '__EVALUATION [1] modifyFoo - matches - target=bar',
+            '__EVALUATION [1] modifyFoo - matches - target=__EVALUATION'
+        ]

tests/core/test_sheerka_call_parsers.py

@@ -0,0 +1,363 @@
+from core.builtin_concepts import ReturnValueConcept, UserInputConcept, BuiltinConcepts, ParserResultConcept
+from parsers.BaseParser import BaseParser
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def get_ret_val(text, who="who"):
+    return ReturnValueConcept(who, True, UserInputConcept(text, "user_name"))
+
+
+class BaseTestParser(BaseParser):
+    debug_out = []
+
+    def __init__(self, name, priority, status=None, parser_result=None):
+        super().__init__(name, priority)
+        self.status = status
+        self.parser_result = parser_result
+
+    @staticmethod
+    def _get_name(name):
+        return name[8:] if name.startswith("parsers.") else name
+
+    @staticmethod
+    def _get_source(text_):
+        return text_ if isinstance(text_, str) else text_.body
+
+    def _out(self, name, priority, status, source):
+        debug = f"name={name}"
+        debug += f", priority={priority}"
+        debug += f", status={status}"
+        debug += f", source={source}"
+        self.debug_out.append(debug)
+
+    def parse(self, context, text):
+        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
+        value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body)
+        parser_result = ParserResultConcept(parser=self, value=value)
+        return ReturnValueConcept(self, self.status, self.parser_result or parser_result)
+
+
+class Enabled90FalseParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled90False", 90, False)
+
+
+class Enabled80FalseParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled80False", 80, False)
+
+
+class Enabled80MultipleFalseParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled80MultipleFalse", 80, False)
+
+    def parse(self, context, text):
+        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
+        value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1"
+        value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2"
+        return [
+            ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value1)),
+            ReturnValueConcept(self, self.status, ParserResultConcept(parser=self, value=value2)),
+        ]
+
+
+class Enabled80MultipleTrueParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled80MultipleTrue", 80)
+
+    def parse(self, context, text):
+        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
+        value1 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_1"
+        value2 = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body) + "_2"
+        return [
+            ReturnValueConcept(self, True, ParserResultConcept(parser=self, value=value1)),
+            ReturnValueConcept(self, False, ParserResultConcept(parser=self, value=value2)),
+        ]
+
+
+class Enabled70FalseParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled70False", 70, False, "Not a ParserResult")
+
+
+class Enabled50TrueParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled50True", 50, True)
+
+    def parse(self, context, text):
+        source = self._get_source(text)
+        status = isinstance(text, ParserResultConcept) and source == "Enabled80False:Enabled90False:hello world"
+        self._out(self._get_name(self.name), self.priority, status, source)
+
+        value = self._get_name(self.name) + ":" + (text if isinstance(text, str) else text.body)
+        return_value = ParserResultConcept(parser=self, value=value)
+        return ReturnValueConcept(self, status, return_value)
+
+
+class Enabled50bisTrueParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled50BisTrue", 50, True)
+
+
+class Enabled50FalseParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled50False", 50, False)
+
+
+class Enabled10TrueParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Enabled10True", 10, True)
+
+
+class DisabledParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("Disabled", 90, True)
+        self.enabled = False
+
+
+class NoneParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("None", 90, True, None)
+
+    def parse(self, context, text):
+        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
+        return None
+
+
+class ListOfNoneParser(BaseTestParser):
+    def __init__(self, **kwargs):
+        super().__init__("ListOfNone", 90, True, None)
+
+    def parse(self, context, text):
+        self._out(self._get_name(self.name), self.priority, self.status, self._get_source(text))
+        return [None, None]
+
+
+class TestSheerkaExecuteParsers(TestUsingMemoryBasedSheerka):
+
+    def test_disabled_parsers_are_not_executed(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled": Enabled10TrueParser,
+            "Disabled": DisabledParser
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == ['name=Enabled10True, priority=10, status=True, source=hello world']
+
+    def test_parser_are_executed_by_priority(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled90False": Enabled90FalseParser,
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled50True": Enabled50TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled90False, priority=90, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
+            'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world',
+        ]
+
+    def test_parsing_stop_at_the_first_success(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled50bisTrue": Enabled50bisTrueParser,
+            "Enabled10True": Enabled10TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled80False, priority=80, status=False, source=hello world',
+            'name=Enabled50BisTrue, priority=50, status=True, source=hello world',
+        ]
+
+    def test_parsing_stop_at_the_first_success_2(self):
+        """
+        Same test than before, but Enabled50True takes more time to find a match
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled90False": Enabled90FalseParser,
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled50True": Enabled50TrueParser,
+            "Enabled10True": Enabled10TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled90False, priority=90, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
+            'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world',
+        ]
+
+    def test_all_parsers_of_a_given_priority_are_executed(self):
+        """
+        Make sure that all parsers with priority 50 are executed
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled90False": Enabled90FalseParser,
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled50True": Enabled50TrueParser,
+            "Enabled50bisTrue": Enabled50bisTrueParser,
+            "Enabled50False": Enabled50FalseParser,
+            "Enabled10True": Enabled10TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled90False, priority=90, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=hello world',
+            'name=Enabled80False, priority=80, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled90False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
+            'name=Enabled50True, priority=50, status=True, source=Enabled80False:Enabled90False:hello world',
+            'name=Enabled50BisTrue, priority=50, status=True, source=hello world',
+            'name=Enabled50False, priority=50, status=False, source=hello world',
+            'name=Enabled50False, priority=50, status=False, source=Enabled90False:hello world',
+            'name=Enabled50False, priority=50, status=False, source=Enabled80False:hello world',
+            'name=Enabled50False, priority=50, status=False, source=Enabled80False:Enabled90False:hello world',
+        ]
+
+    def test_a_parser_has_access_to_the_output_of_its_predecessors(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled90False": Enabled90FalseParser,
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled50True": Enabled50TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+
+        res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+        res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res]
+        assert res_as_tuple == [
+            ('Enabled90False', False, 'Enabled90False:hello world'),
+            ('Enabled80False', False, 'Enabled80False:hello world'),
+            ('Enabled80False', False, 'Enabled80False:Enabled90False:hello world'),
+            ('Enabled50True', False, 'Enabled50True:hello world'),
+            ('Enabled50True', False, 'Enabled50True:Enabled90False:hello world'),
+            ('Enabled50True', False, 'Enabled50True:Enabled80False:hello world'),
+            ('Enabled50True', True, 'Enabled50True:Enabled80False:Enabled90False:hello world'),
+        ]
+
+    def test_none_return_values_are_discarded(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "NoneParser": NoneParser,
+            "ListOfNone": ListOfNoneParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+
+        BaseTestParser.debug_out = []
+        res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+        assert res == []
+        assert BaseTestParser.debug_out == [
+            'name=None, priority=90, status=True, source=hello world',
+            'name=ListOfNone, priority=90, status=True, source=hello world'
+        ]
+
+    def test_following_priorities_can_only_see_parser_result_return_values(self):
+        """
+        Normally, lower priority parsers can see the result of the higher priority parsers
+        This is true only if the higher priority parser return a ParserResultConcept
+        :return:
+        """
+
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled80False": Enabled80FalseParser,
+            "Enabled70False": Enabled70FalseParser,
+            "Enabled50True": Enabled50TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled80False, priority=80, status=False, source=hello world',
+            'name=Enabled70False, priority=70, status=False, source=hello world',
+            'name=Enabled70False, priority=70, status=False, source=Enabled80False:hello world',
+            'name=Enabled50True, priority=50, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80False:hello world',
+        ]
+
+    def test_i_can_manage_parser_with_multiple_results(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled80MultipleFalse": Enabled80MultipleFalseParser,
+            "Enabled50True": Enabled50TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled80MultipleFalse, priority=80, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=hello world',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_1',
+            'name=Enabled50True, priority=50, status=False, source=Enabled80MultipleFalse:hello world_2',
+        ]
+
+        res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res]
+        assert res_as_tuple == [
+            ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_1'),
+            ('Enabled80MultipleFalse', False, 'Enabled80MultipleFalse:hello world_2'),
+            ('Enabled50True', False, 'Enabled50True:hello world'),
+            ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_1'),
+            ('Enabled50True', False, 'Enabled50True:Enabled80MultipleFalse:hello world_2'),
+        ]
+
+    def test_i_can_manage_parser_with_multiple_results_and_a_sucess(self):
+        sheerka = self.get_sheerka()
+        sheerka.parsers = {
+            "Enabled80MultipleTrue": Enabled80MultipleTrueParser,
+            "Enabled50True": Enabled50TrueParser,
+        }
+
+        user_input = [get_ret_val("hello world")]
+        BaseTestParser.debug_out = []
+        res = sheerka.execute(self.get_context(sheerka), user_input, [BuiltinConcepts.PARSING])
+
+        assert BaseTestParser.debug_out == [
+            'name=Enabled80MultipleTrue, priority=80, status=None, source=hello world',
+        ]
+
+        res_as_tuple = [(str(r.who)[8:], r.status, r.body.body) for r in res]
+        assert res_as_tuple == [
+            ('Enabled80MultipleTrue', True, 'Enabled80MultipleTrue:hello world_1'),
+            ('Enabled80MultipleTrue', False, 'Enabled80MultipleTrue:hello world_2'),
+        ]

tests/core/test_sheerka_transform.py

@@ -0,0 +1,318 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept, ConceptParts
+from core.sheerka.ExecutionContext import ExecutionContext
+from core.sheerka_transform import SheerkaTransform, OBJ_TYPE_KEY, SheerkaTransformType, OBJ_ID_KEY
+from sdp.sheerkaDataProvider import Event
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestSheerkaTransform(TestUsingMemoryBasedSheerka):
+    def test_i_can_transform_an_unknown_concept(self):
+        sheerka = self.get_sheerka()
+
+        foo = Concept("foo", body="body")
+        concept_with_sub = Concept("concept_with_sub", body=foo)
+
+        concept = Concept(
+            name="concept_name",
+            is_builtin=True,
+            is_unique=True,
+            key="concept_key",
+            body=concept_with_sub,
+            where=[foo, 1, "1", True, 1.0],
+            pre=foo,
+            post=None,  # will not appear
+            definition="it is a definition",
+            definition_type="def type",
+            desc="this this the desc"
+        ).def_prop("a", "10").def_prop("b", "foo").def_prop("c", "concept_with_sub")
+
+        # add values and props
+        concept.values[ConceptParts.BODY] = Concept().update_from(concept_with_sub).auto_init()
+        concept.values[ConceptParts.WHERE] = [foo, 1, "1", True, 1.0]
+        concept.values[ConceptParts.PRE] = Concept().update_from(foo).auto_init()
+        concept.values[ConceptParts.POST] = "a value for POST"
+        concept.set_prop("a", 10).set_prop("b", foo).set_prop("c", concept_with_sub)
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(concept)
+
+        assert to_dict == {
+            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+            OBJ_ID_KEY: 0,
+            'meta.name': 'concept_name',
+            'meta.key': 'concept_key',
+            'meta.is_builtin': True,
+            'meta.is_unique': True,
+            'meta.definition': 'it is a definition',
+            'meta.definition_type': 'def type',
+            'meta.desc': 'this this the desc',
+            'meta.where': [{OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+                            OBJ_ID_KEY: 1,
+                            'meta.body': 'body',
+                            'meta.name': 'foo'}, 1, '1', True, 1.0],
+            'meta.pre': {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1},
+            'meta.body': {
+                '__type__': SheerkaTransformType.Concept,
+                '__id__': 2,
+                'meta.name': 'concept_with_sub',
+                'meta.body': {
+                    '__type__': SheerkaTransformType.Reference,
+                    '__id__': 1}},
+            'meta.props': [['a', '10'], ['b', 'foo'], ['c', 'concept_with_sub']],
+            'pre': {'__type__': SheerkaTransformType.Concept,
+                    '__id__': 4,
+                    'meta.body': 'body',
+                    'meta.name': 'foo',
+                    'body': 'body'},
+            'post': "a value for POST",
+            'body': {'__type__': SheerkaTransformType.Concept,
+                     '__id__': 3,
+                     'meta.body': {'__id__': 1, '__type__': SheerkaTransformType.Reference},
+                     'meta.name': 'concept_with_sub',
+                     'body': {'__id__': 1, '__type__': SheerkaTransformType.Reference}},
+            'where': [{OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}, 1, '1', True, 1.0],
+            'props': [('a', 10),
+                      ('b', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 1}),
+                      ('c', {OBJ_TYPE_KEY: SheerkaTransformType.Reference, OBJ_ID_KEY: 2})],
+        }
+
+    def test_i_can_transform_unknown_concept_with_almost_same_value(self):
+        sheerka = self.get_sheerka()
+        concept = Concept("foo")
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(concept)
+
+        assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, 'meta.name': 'foo'}
+
+    def test_i_can_transform_known_concept_when_the_values_are_the_same(self):
+        """
+        Values are the same means that we are serializing a concept which has kept all its default values
+        There is not diff between the concept to serialize and the one which was registered with create_new_concept()
+        We serialize only the id of the concept
+        :return:
+        """
+        sheerka = self.get_sheerka()
+
+        concept = Concept(
+            name="concept_name",
+            is_builtin=True,
+            is_unique=False,
+            key="concept_key",
+            body="body definition",
+            where="where definition",
+            pre="pre definition",
+            post="post definition",
+            definition="it is a definition",
+            definition_type="def type",
+            desc="this this the desc"
+        ).def_prop("a").def_prop("b")
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new_concept = sheerka.new(concept.key)
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(new_concept)
+
+        assert to_dict == {OBJ_TYPE_KEY: SheerkaTransformType.Concept, OBJ_ID_KEY: 0, "id": "1001"}
+
+    def test_i_can_transform_known_concept_when_the_values_are_different(self):
+        """
+        Values are the different means the concept was modified.
+        It's different from the one which was registered with create_new_concept()
+        We serialize only the differences
+        :return:
+        """
+        sheerka = self.get_sheerka()
+
+        concept = Concept(
+            name="concept_name",
+            is_builtin=True,
+            is_unique=False,
+            key="concept_key",
+            body="body definition",
+            where="where definition",
+            pre="pre definition",
+            post="post definition",
+            definition="it is a definition",
+            definition_type="def type",
+            desc="this this the desc"
+        ).def_prop("a").def_prop("b")
+        sheerka.create_new_concept(self.get_context(sheerka), concept)
+
+        new_concept = sheerka.new(concept.key, body="another", a=10, pre="another pre")
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(new_concept)
+
+        assert to_dict == {
+            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+            OBJ_ID_KEY: 0,
+            "id": "1001",
+            'pre': 'another pre',
+            "body": "another",
+            'props': [('a', 10)]
+        }
+
+    def test_i_can_transform_concept_with_circular_reference(self):
+        sheerka = self.get_sheerka()
+        foo = Concept("foo")
+        bar = Concept("bar", body=foo)
+        foo.metadata.body = bar
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(foo)
+
+        assert to_dict == {
+            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+            OBJ_ID_KEY: 0,
+            'meta.name': 'foo',
+            'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+                          OBJ_ID_KEY: 1,
+                          'meta.name': 'bar',
+                          'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
+                                        OBJ_ID_KEY: 0},
+                          },
+        }
+
+    def test_i_can_transform_concept_with_circular_reference_2(self):
+        sheerka = self.get_sheerka()
+        foo = Concept("foo")
+        bar = Concept("foo", body=foo)
+        foo.metadata.body = bar
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(foo)
+
+        assert to_dict == {
+            OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+            OBJ_ID_KEY: 0,
+            'meta.name': 'foo',
+            'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Concept,
+                          OBJ_ID_KEY: 1,
+                          'meta.name': 'foo',
+                          'meta.body': {OBJ_TYPE_KEY: SheerkaTransformType.Reference,
+                                        OBJ_ID_KEY: 0},
+                          },
+        }
+
+    def test_i_can_transform_the_unknown_concept(self):
+        sheerka = self.get_sheerka(False)
+
+        unknown = sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT)
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(unknown)
+
+        assert len(to_dict) == 3
+        assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.Concept
+        assert to_dict[OBJ_ID_KEY] == 0
+        assert "id" in to_dict
+
+    def test_i_can_transform_simple_execution_context(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids = {}
+        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(context)
+
+        assert to_dict == {
+            OBJ_TYPE_KEY: SheerkaTransformType.ExecutionContext,
+            OBJ_ID_KEY: 0,
+            '_parent': None,
+            '_id': 0,
+            '_tab': '',
+            '_bag': {},
+            '_start': 0,
+            '_stop': 0,
+            'who': 'requester',
+            'event': {OBJ_TYPE_KEY: SheerkaTransformType.Event, OBJ_ID_KEY: 1, 'digest': 'xxx'},
+            'desc': 'this is the desc',
+            'children': [],
+            'preprocess': None,
+            'inputs': {},
+            'values': {},
+            'obj': None,
+            'concepts': {}
+        }
+
+    def test_i_can_transform_list(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids = {}
+        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict([context])
+
+        assert len(to_dict) == 1
+        assert isinstance(to_dict, list)
+        assert to_dict[0]["who"] == "requester"
+        assert to_dict[0]["desc"] == "this is the desc"
+
+    def test_i_can_transform_set(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids = {}
+        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict({context})
+
+        assert len(to_dict) == 1
+        assert isinstance(to_dict, list)
+        assert to_dict[0]["who"] == "requester"
+        assert to_dict[0]["desc"] == "this is the desc"
+
+    def test_i_can_transform_dict(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids = {}
+        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+        known_concept = Concept("foo", body="foo").set_prop("a", "value_of_a").init_key()
+        sheerka.create_new_concept(self.get_context(sheerka), known_concept)
+        unknown_concept = Concept("bar")
+        known = sheerka.new("foo")
+
+        bag = {
+            "context": context,
+            "known_concept": known_concept,
+            "unknown_concept": unknown_concept,
+            "True": True,
+            "Number": 1.1,
+            "String": "a string value",
+            "None": None,
+            unknown_concept: "hello",
+            known: "world"
+        }
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(bag)
+
+        assert isinstance(to_dict, dict)
+        assert to_dict['Number'] == 1.1
+        assert to_dict['String'] == 'a string value'
+        assert to_dict['True']
+        assert to_dict['None'] is None
+        assert to_dict["context"][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
+        assert to_dict["known_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
+        assert to_dict["known_concept"]["id"] == '1001'
+        assert to_dict["unknown_concept"][OBJ_TYPE_KEY] == SheerkaTransformType.Concept
+        assert to_dict["(None)bar"] == "hello"
+        assert to_dict["(1001)foo"] == "world"
+
+    def test_i_can_transform_when_circular_references(self):
+        sheerka = self.get_sheerka()
+        ExecutionContext.ids = {}
+        context = ExecutionContext("requester", Event(), sheerka, 'this is the desc')
+        context.push("another requester", "another desc")
+
+        st = SheerkaTransform(sheerka)
+        to_dict = st.to_dict(context)
+
+        assert isinstance(to_dict, dict)
+        assert to_dict[OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
+        assert len(to_dict["children"]) == 1
+        assert to_dict["children"][0][OBJ_TYPE_KEY] == SheerkaTransformType.ExecutionContext
+        assert to_dict["children"][0]['_parent'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
+        assert to_dict["children"][0]['_parent'][OBJ_ID_KEY] == 0
+        assert to_dict["children"][0]['event'][OBJ_TYPE_KEY] == SheerkaTransformType.Reference
+        assert to_dict["children"][0]['event'][OBJ_ID_KEY] == 1

tests/evaluators/test_AddConceptEvaluator.py

@@ -0,0 +1,155 @@
+import ast
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
+from core.concept import VARIABLE_PREFIX, Concept
+from core.tokenizer import Tokenizer
+from evaluators.AddConceptEvaluator import AddConceptEvaluator
+from parsers.BaseParser import BaseParser
+from parsers.ConceptLexerParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression
+from parsers.BnfParser import BnfParser
+from parsers.DefaultParser import DefConceptNode, NameNode
+from parsers.PythonParser import PythonNode, PythonParser
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
+
+    @staticmethod
+    def get_concept_part(part):
+        if isinstance(part, str):
+            node = PythonNode(part, ast.parse(part, mode="eval"))
+            return ReturnValueConcept(
+                who="parsers.Default",
+                status=True,
+                value=ParserResultConcept(
+                    source=part,
+                    parser=PythonParser(),
+                    value=node))
+
+        if isinstance(part, PythonNode):
+            return ReturnValueConcept(
+                who="parsers.Default",
+                status=True,
+                value=ParserResultConcept(
+                    source=part.source,
+                    parser=PythonParser(),
+                    value=part))
+
+        if isinstance(part, ReturnValueConcept):
+            return part
+
+    @staticmethod
+    def get_return_value(source, parsing_expression):
+        return ReturnValueConcept(
+            who="Parsers:RegexParser",
+            status=True,
+            value=ParserResultConcept(
+                source=source,
+                parser=BnfParser(),
+                value=parsing_expression
+            )
+        )
+
+    def get_def_concept(self, name, where=None, pre=None, post=None, body=None, definition=None):
+        concept = DefConceptNode([], name=NameNode(list(Tokenizer(name))))
+
+        if body:
+            concept.body = self.get_concept_part(body)
+        if where:
+            concept.where = self.get_concept_part(where)
+        if pre:
+            concept.pre = self.get_concept_part(pre)
+        if post:
+            concept.post = self.get_concept_part(post)
+        if definition:
+            concept.definition = definition
+
+        return ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=concept))
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=DefConceptNode([]))),
+            True),
+        (ReturnValueConcept(BaseParser.PREFIX + "some_name", False, ParserResultConcept(value=DefConceptNode([]))),
+         False),
+        (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, "not a ParserResultConcept"), False),
+        (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept()), False),
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert AddConceptEvaluator().matches(context, ret_val) == expected
+
+    def test_that_the_source_is_correctly_set(self):
+        context = self.get_context()
+        def_concept_return_value = self.get_def_concept(
+            name="hello a",
+            definition=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
+            where="isinstance(a, str )",
+            pre="a is not None",
+            body="print('hello' + a)")
+
+        evaluated = AddConceptEvaluator().eval(context, def_concept_return_value)
+
+        assert evaluated.status
+        assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
+
+        created_concept = evaluated.body.body
+        assert created_concept.metadata.name == "hello a"
+        assert created_concept.metadata.where == "isinstance(a, str )"
+        assert created_concept.metadata.pre == "a is not None"
+        assert created_concept.metadata.post is None
+        assert created_concept.metadata.body == "print('hello' + a)"
+        assert created_concept.metadata.definition == "hello a"
+
+    def test_that_the_new_concept_is_correctly_saved_in_db(self):
+        context = self.get_context()
+        def_concept_return_value = self.get_def_concept(
+            name="hello a",
+            definition=self.get_return_value("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
+            where="isinstance(a, str )",
+            pre="a is not None",
+            body="print('hello' + a)")
+
+        # sanity. Make sure that the concept does not already exist
+        from_db = context.sheerka.get("hello " + VARIABLE_PREFIX + "0")
+        assert context.sheerka.isinstance(from_db, BuiltinConcepts.UNKNOWN_CONCEPT)
+
+        AddConceptEvaluator().eval(context, def_concept_return_value)
+        context.sheerka.concepts_cache = {}  # reset cache
+        from_db = context.sheerka.get("hello " + VARIABLE_PREFIX + "0")
+
+        assert from_db.metadata.key == f"hello {VARIABLE_PREFIX}0"
+        assert from_db.metadata.name == "hello a"
+        assert from_db.metadata.where == "isinstance(a, str )"
+        assert from_db.metadata.pre == "a is not None"
+        assert from_db.metadata.post is None
+        assert from_db.metadata.body == "print('hello' + a)"
+        assert from_db.metadata.definition == "hello a"
+        assert len(from_db.metadata.props) == 1
+        assert from_db.metadata.props[0] == ("a", None)
+        assert "a" in from_db.props
+
+        assert from_db.compiled == {}  # ast is not saved in db
+
+    def test_i_can_get_props_from_python_node(self):
+        ret_val = self.get_concept_part("isinstance(a, str)")
+        context = self.get_context()
+
+        assert AddConceptEvaluator.get_props(context.sheerka, ret_val, ["a"]) == ["a"]
+
+    def test_i_can_get_props_from_another_concept(self):
+        concept = Concept("hello").def_prop("a").def_prop("b")
+        ret_val = ReturnValueConcept(who="some_parser",
+                                     status=True,
+                                     value=ParserResultConcept(value=concept))
+
+        assert AddConceptEvaluator.get_props(self.get_context(), ret_val, []) == ["a", "b"]
+
+    def test_i_can_get_props_from_definition(self):
+        parsing_expression = Sequence(ConceptExpression('mult'),
+                                      ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add"))))
+        ret_val = self.get_return_value("mult (('+'|'-') add)?", parsing_expression)
+
+        assert AddConceptEvaluator.get_props(self.get_context(), ret_val, []) == ["add", "mult"]

tests/evaluators/test_AddConceptInSetEvaluator.py

@@ -0,0 +1,102 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
+from core.concept import Concept
+from core.tokenizer import Tokenizer
+from evaluators.AddConceptInSetEvaluator import AddConceptInSetEvaluator
+from parsers.DefaultParser import IsaConceptNode, NameNode
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def get_ret_val(concept_name, concept_set_name):
+    n1 = NameNode(list(Tokenizer(concept_name)))
+    n2 = NameNode(list(Tokenizer(concept_set_name)))
+
+    return ReturnValueConcept("some_name", True, ParserResultConcept(value=IsaConceptNode([], n1, n2)))
+
+
+class TestAssConceptInSetEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=IsaConceptNode([]))), True),
+        (ReturnValueConcept("some_name", False, ParserResultConcept(value=IsaConceptNode([]))), False),
+        (ReturnValueConcept("some_name", True, "not a ParserResultConcept"), False),
+        (ReturnValueConcept("some_name", True, ParserResultConcept()), False),
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert AddConceptInSetEvaluator().matches(context, ret_val) == expected
+
+    def test_i_cannot_add_if_the_concept_does_not_exists(self):
+        context = self.get_context()
+
+        ret_val = get_ret_val("foo", "bar")
+        res = AddConceptInSetEvaluator().eval(context, ret_val)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert res.value.body == "foo"
+
+    def test_i_cannot_add_if_the_set_does_not_exists(self):
+        context = self.get_context()
+        foo = Concept("foo")
+        context.sheerka.set_id_if_needed(foo, False)
+        context.sheerka.add_in_cache(foo)
+
+        ret_val = get_ret_val("foo", "bar")
+        res = AddConceptInSetEvaluator().eval(context, ret_val)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert res.value.body == "bar"
+
+    def test_i_can_add_concept_to_a_set_of_concept(self):
+        context = self.get_context()
+        foo = Concept("foo")
+        context.sheerka.set_id_if_needed(foo, False)
+        context.sheerka.add_in_cache(foo)
+
+        bar = Concept("bar")
+        context.sheerka.set_id_if_needed(bar, False)
+        context.sheerka.add_in_cache(bar)
+
+        ret_val = get_ret_val("foo", "bar")
+        res = AddConceptInSetEvaluator().eval(context, ret_val)
+
+        assert res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS)
+
+    def test_i_can_add_concept_with_a_body_to_a_set_of_concept(self):
+        context = self.get_context()
+        foo = Concept("foo", body="1")
+        context.sheerka.set_id_if_needed(foo, False)
+        context.sheerka.add_in_cache(foo)
+
+        bar = Concept("bar")
+        context.sheerka.set_id_if_needed(bar, False)
+        context.sheerka.add_in_cache(bar)
+
+        ret_val = get_ret_val("foo", "bar")
+        res = AddConceptInSetEvaluator().eval(context, ret_val)
+
+        assert res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS)
+
+    def test_i_cannot_add_the_same_concept_twice(self):
+        context = self.get_context()
+        foo = Concept("foo")
+        context.sheerka.set_id_if_needed(foo, False)
+        context.sheerka.add_in_cache(foo)
+
+        bar = Concept("bar")
+        context.sheerka.set_id_if_needed(bar, False)
+        context.sheerka.add_in_cache(bar)
+
+        ret_val = get_ret_val("foo", "bar")
+        AddConceptInSetEvaluator().eval(context, ret_val)
+        res = AddConceptInSetEvaluator().eval(context, ret_val)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_IN_SET)
+        assert res.value.concept == foo
+        assert res.value.concept_set == bar

tests/evaluators/test_ConceptEvaluator.py

@@ -0,0 +1,117 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
+from core.concept import Concept, ConceptParts
+from evaluators.ConceptEvaluator import ConceptEvaluator
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=Concept())), True),
+        (ReturnValueConcept("some_name", False, ParserResultConcept(value=Concept())), False),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value="Not a concept")), False),
+        (ReturnValueConcept("some_name", True, Concept()), False),
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert ConceptEvaluator().matches(context, ret_val) == expected
+
+    def test_i_can_evaluate_concept(self):
+        context = self.get_context()
+        concept = Concept(name="foo",
+                          where="1",
+                          pre="2",
+                          post="3").def_prop("a", "4").def_prop("b", "5")
+
+        evaluator = ConceptEvaluator()
+        item = self.pretval(concept)
+        result = evaluator.eval(context, item)
+
+        assert result.who == evaluator.name
+        assert result.status
+        assert result.value.name == "foo"
+        assert result.value.get_metadata_value(ConceptParts.WHERE) == 1
+        assert result.value.get_metadata_value(ConceptParts.PRE) == 2
+        assert result.value.get_metadata_value(ConceptParts.POST) == 3
+        assert result.value.get_prop("a") == 4
+        assert result.value.get_prop("b") == 5
+        assert result.value.key == "foo"
+        assert result.parents == [item]
+
+    def test_body_is_returned_when_defined_and_requested(self):
+        context = self.get_context()
+        concept = Concept(name="foo",
+                          body="'I have a value'",
+                          where="1",
+                          pre="2",
+                          post="3").set_prop("a", "4").set_prop("b", "5")
+
+        evaluator = ConceptEvaluator(return_body=True)
+        item = self.pretval(concept)
+        result = evaluator.eval(context, item)
+
+        assert result.who == evaluator.name
+        assert result.status
+        assert result.value == "I have a value"
+        assert result.parents == [item]
+
+    def test_body_is_not_returned_if_not_requested(self):
+        context = self.get_context()
+        concept = Concept(name="foo",
+                          body="'I have a value'",
+                          where="1",
+                          pre="2",
+                          post="3").set_prop("a", "4").set_prop("b", "5")
+
+        evaluator = ConceptEvaluator(return_body=False)  # which is the default behaviour
+        item = self.pretval(concept)
+        result = evaluator.eval(context, item)
+
+        assert result.who == evaluator.name
+        assert result.status
+        assert result.value == concept
+        assert result.parents == [item]
+
+    def test_i_can_eval_if_with_the_same_name_is_defined_in_the_context(self):
+        # If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'")
+        # ConceptEvaluator will be called to resolve 'a' while we know that 'a' refers to the string 'property_a'
+
+        context = self.get_context()
+        context.obj = Concept("other").set_prop("foo", "'some_other_value'")
+        concept = Concept(name="foo")
+
+        item = self.pretval(concept)
+        result = ConceptEvaluator().eval(context, item)
+
+        assert result.status
+        assert result.value == "'some_other_value'"
+
+    def test_i_cannot_recognize_a_concept_if_one_of_the_prop_is_unknown(self):
+        context = self.get_context()
+        context.sheerka.add_in_cache(Concept(name="one").init_key())
+        concept_plus = context.sheerka.add_in_cache(Concept(name="a plus b")
+                                                    .def_prop("a", "one")
+                                                    .def_prop("b", "two").init_key())
+
+        evaluator = ConceptEvaluator()
+        item = self.pretval(concept_plus)
+        result = evaluator.eval(context, item)
+
+        assert not result.status
+        assert context.sheerka.isinstance(result.value, BuiltinConcepts.CONCEPT_EVAL_ERROR)
+        assert result.value.concept == concept_plus
+        assert result.value.property_name == "b"
+        assert context.sheerka.isinstance(result.value.error, BuiltinConcepts.TOO_MANY_ERRORS)
+
+    def test_that_error_concepts_are_transformed_into_errors_only_if_the_key_is_different(self):
+        context = self.get_context()
+
+        error_concept = context.sheerka.new(BuiltinConcepts.ERROR)
+        item = self.pretval(error_concept)
+        result = ConceptEvaluator().eval(context, item)
+
+        assert not context.sheerka.is_success(error_concept)  # it's indeed an error
+        assert result.status
+        assert result.value == error_concept

tests/evaluators/test_EvalEvaluator.py

@@ -0,0 +1,72 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from evaluators.EvalEvaluator import EvalEvaluator
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+eval_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.CONCEPT_EVAL_REQUESTED))
+
+
+def retval(obj, who="who", status=True):
+    """ret_val"""
+    return ReturnValueConcept(who, status, obj)
+
+
+class TestEvalEvaluator(TestUsingMemoryBasedSheerka):
+    def test_i_can_match_and_eval(self):
+        context = self.get_context()
+
+        to_eval1 = ReturnValueConcept("some_name", True, Concept(name="2", body="to eval").auto_init())
+        to_eval2 = ReturnValueConcept("some_name", True, Concept(name="3", body="also to eval").auto_init())
+
+        return_values = [
+            ReturnValueConcept("some_name", True, "not to eval"),
+            ReturnValueConcept("some_name", True, Concept(name="not to eval")),
+            ReturnValueConcept("some_name", False, Concept(name="1", body="'not to eval'")),
+            to_eval1,
+            to_eval2,
+            eval_requested
+        ]
+
+        evaluator = EvalEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        evaluated = evaluator.eval(context, return_values)
+        assert len(evaluated) == 2
+        assert evaluated[0].value == to_eval1.body.body
+        assert evaluated[0].parents == [to_eval1, eval_requested]
+
+        assert evaluated[1].value == to_eval2.body.body
+        assert evaluated[1].parents == [to_eval2, eval_requested]
+
+    @pytest.mark.parametrize("return_values, expected", [
+        ([retval(Concept("foo", body="bar")), eval_requested], True),
+        ([retval(Concept("status is false", body="bar"), status=False), eval_requested], True),
+        ([retval("string_value"), eval_requested], True),
+        ([retval(Concept("no body")), eval_requested], True),
+        ([retval(Concept("eval requested missing", body="bar"))], False),
+    ])
+    def test_i_cannot_match_if_eval_request_is_not_present(self, return_values, expected):
+        context = self.get_context()
+        assert EvalEvaluator().matches(context, return_values) == expected
+
+    def test_concept_eval_requested_is_reduced_when_nothing_to_reduce(self):
+        context = self.get_context()
+
+        return_values = [
+            ReturnValueConcept("some_name", True, "not to eval"),
+            ReturnValueConcept("some_name", True, Concept(name="not to eval")),
+            ReturnValueConcept("some_name", False, Concept(name="1", body="not to eval")),
+            eval_requested
+        ]
+
+        evaluator = EvalEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        evaluated = evaluator.eval(context, return_values)
+        assert evaluated == ReturnValueConcept(
+            "evaluators.Eval",
+            False,
+            context.sheerka.new(BuiltinConcepts.CONCEPT_EVAL_REQUESTED))
+        assert evaluated.parents == [eval_requested]

tests/evaluators/test_LexerNodeEvaluator.py

@@ -0,0 +1,104 @@
+import ast
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
+from core.concept import Concept, ConceptParts, DoNotResolve
+from evaluators.LexerNodeEvaluator import LexerNodeEvaluator
+from parsers.ConceptLexerParser import ConceptNode, ConceptLexerParser, StrMatch, UnrecognizedTokensNode, SourceCodeNode
+from parsers.PythonParser import PythonNode
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestLexerNodeEvaluator(TestUsingMemoryBasedSheerka):
+
+    def from_parsing(self, context, grammar, expression):
+        parser = ConceptLexerParser()
+        parser.initialize(context, grammar)
+
+        ret_val = parser.parse(context, expression)
+        assert ret_val.status
+        return ret_val
+
+    def from_fragments(self, *fragments):
+        nodes = []
+        for fragment in fragments:
+            if isinstance(fragment, str):
+                node = PythonNode(fragment, ast.parse(fragment.strip(), mode="eval"))
+                nodes.append(SourceCodeNode(node, 0, 0, [], fragment))
+            else:
+                nodes.append(ConceptNode(fragment, 0, 0, [], fragment.name))
+
+        return ReturnValueConcept("somme_name", True, ParserResultConcept(value=nodes))
+
+    def init(self, concept, grammar, text):
+        context = self.get_context()
+        if isinstance(concept, list):
+            for c in concept:
+                context.sheerka.add_in_cache(c)
+        else:
+            context.sheerka.add_in_cache(concept)
+        ret_val = self.from_parsing(context, grammar, text)
+        node = ret_val.value.value[0]
+
+        return context, node
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), True),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=ConceptNode(Concept(), 0, 0))), True),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=[SourceCodeNode(0, 0, [])])), True),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=SourceCodeNode(0, 0, []))), True),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=[UnrecognizedTokensNode(0, 0, [])])), False),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=UnrecognizedTokensNode(0, 0, []))), False),
+        (ReturnValueConcept("some_name", False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), False),
+        (ReturnValueConcept("some_name", False, ParserResultConcept(value=ConceptNode(Concept(), 0, 0))), False),
+        (ReturnValueConcept("some_name", False, ParserResultConcept(value=[SourceCodeNode(0, 0, [])])), False),
+        (ReturnValueConcept("some_name", False, ParserResultConcept(value=SourceCodeNode(0, 0, []))), False),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value="Not a concept node")), False),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=["Not a concept node"])), False),
+        (ReturnValueConcept("some_name", True, [ConceptNode(Concept(), 0, 0)]), False),
+        (ReturnValueConcept("some_name", True, ConceptNode(Concept(), 0, 0)), False),
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert LexerNodeEvaluator().matches(context, ret_val) == expected
+
+    def test_concept_is_returned_when_only_one_in_the_list(self):
+        foo = Concept("foo")
+        context = self.get_context()
+        context.sheerka.add_in_cache(foo)
+
+        ret_val = self.from_parsing(context, {foo: StrMatch("foo")}, "foo")
+
+        evaluator = LexerNodeEvaluator()
+        result = evaluator.eval(context, ret_val)
+        wrapper = result.body
+        return_value = result.body.body
+
+        assert result.who == evaluator.name
+        assert result.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert wrapper.parser == evaluator
+        assert wrapper.source == "foo"
+        assert return_value == Concept("foo").init_key()
+        assert return_value.compiled[ConceptParts.BODY] == DoNotResolve("foo")
+        assert result.parents == [ret_val]
+
+    def test_concept_python_node_is_returned_when_source_code(self):
+        context = self.get_context()
+        foo = Concept("foo")
+        ret_val = self.from_fragments(foo, " + 1")
+
+        evaluator = LexerNodeEvaluator()
+        result = evaluator.eval(context, ret_val)
+        wrapper = result.body
+        return_value = result.body.body
+
+        assert result.who == evaluator.name
+        assert result.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert wrapper.parser == evaluator
+        assert wrapper.source == "foo + 1"
+
+        assert return_value == PythonNode('foo + 1', ast.parse("__C__foo__C__ + 1", mode="eval"))
+        assert return_value.concepts == {"__C__foo__C__": foo}
+        assert result.parents == [ret_val]

tests/evaluators/test_MultipleSameSuccessEvaluator.py

@@ -0,0 +1,200 @@
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from evaluators.BaseEvaluator import BaseEvaluator
+from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
+from parsers.BaseParser import BaseParser
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestMultipleSameSuccessEvaluator(TestUsingMemoryBasedSheerka):
+
+    def test_i_can_match_and_eval(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        evaluator = MultipleSameSuccessEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        evaluated = evaluator.eval(context, return_values)
+        assert evaluated.status
+        assert evaluated.value == Concept(name="1", body="value").auto_init()  # the first concept is returned
+
+    def test_i_can_match_and_eval_when_no_body(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1").auto_init()),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        evaluator = MultipleSameSuccessEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        evaluated = evaluator.eval(context, return_values)
+        assert evaluated.status
+        assert evaluated.value == Concept(name="1").auto_init()
+
+    def test_i_can_match_and_eval_when_value_is_not_a_concept(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, "value"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, "value"),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        evaluator = MultipleSameSuccessEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        evaluated = evaluator.eval(context, return_values)
+        assert evaluated.status
+        assert evaluated.value == "value"
+
+    def test_i_can_match_and_eval_when_at_least_one_value_is_a_concept_concept_evaluator_first(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, "value"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="2", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "Concept", True, Concept(name="1", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="3", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        evaluator = MultipleSameSuccessEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        evaluated = evaluator.eval(context, return_values)
+        assert evaluated.status
+        assert evaluated.value == Concept(name="1", body="value").auto_init()  # the concept is returned, not the value
+
+    def test_i_can_match_and_eval_when_at_least_one_value_is_a_concept_python_evaluator_first(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, "value"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, Concept(name="2", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "Python", True, Concept(name="1", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, Concept(name="3", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "other", True, "value"),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        evaluator = MultipleSameSuccessEvaluator()
+        assert evaluator.matches(context, return_values)
+
+        evaluated = evaluator.eval(context, return_values)
+        assert evaluated.status
+        assert evaluated.value == Concept(name="1", body="value").auto_init()  # the concept is returned, not the value
+
+    def test_i_can_match_even_if_the_value_are_not_the_same_but_eval_will_fail(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value2").auto_init()),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        evaluator = MultipleSameSuccessEvaluator()
+        assert evaluator.matches(context, return_values)
+        assert evaluator.eval(context, return_values) is None
+
+    def test_i_can_match_even_if_the_value_are_not_the_same_but_eval_will_fail_when_no_body(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2").auto_init()),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        evaluator = MultipleSameSuccessEvaluator()
+        assert evaluator.matches(context, return_values)
+        assert evaluator.eval(context, return_values) is None
+
+    def test_i_can_match_if_no_parser(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        assert MultipleSameSuccessEvaluator().matches(context, return_values)
+
+    def test_i_cannot_match_if_not_reduced_requested(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()),
+            ReturnValueConcept("some_name", True, Concept(name="2", body="value")),
+            # ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        assert not MultipleSameSuccessEvaluator().matches(context, return_values)
+
+    def test_i_cannot_match_if_only_one_successful_evaluator(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", False, "Not relevant"),
+            # ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value")),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()),
+            ReturnValueConcept("some_name", True, Concept(name="2", body="value")),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        assert not MultipleSameSuccessEvaluator().matches(context, return_values)
+
+    def test_i_cannot_match_if_at_least_one_parser_is_successful(self):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        return_values = [
+            ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "Not relevant"),
+            ReturnValueConcept(BaseParser.PREFIX + "some_name2", True, "Not relevant"),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="1", body="value").auto_init()),
+            ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, Concept(name="2", body="value").auto_init()),
+            ReturnValueConcept("some_name", True, sheerka.new(BuiltinConcepts.REDUCE_REQUESTED))
+        ]
+
+        assert not MultipleSameSuccessEvaluator().matches(context, return_values)

tests/evaluators/test_OneErrorEvaluator.py

@@ -0,0 +1,73 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from evaluators.OneErrorEvaluator import OneErrorEvaluator
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def r(value, status=True):
+    return ReturnValueConcept(value, status, value)
+
+
+reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED))
+
+
+class TestOneErrorEvaluator(TestUsingMemoryBasedSheerka):
+    @pytest.mark.parametrize("return_values, expected", [
+        ([r("evaluators.one error", False), reduce_requested], True),
+        ([r("evaluators.one error", False), r("failed", False), r("failed", False), reduce_requested], True),
+        ([r("evaluators.error", False), r("not a parser in success"), reduce_requested], True),
+        ([r("evaluators.no reduce required", False), r("failed", False), r("failed", False)], False),
+        ([r("evaluators.no reduce required", False)], False),
+        ([r("evaluators.error", False), r("evaluators.success"), reduce_requested], False),
+        ([r("evaluators.error", False), r("parsers.success"), reduce_requested], False),
+        ([r("evaluators.success"), r("not an evaluator in error", False), reduce_requested], False),
+        ([r("evaluators.error", False), r("evaluators.another error", False), reduce_requested], False),
+    ])
+    def test_i_can_match(self, return_values, expected):
+        context = self.get_context()
+        assert OneErrorEvaluator().matches(context, return_values) == expected
+
+    def test_i_can_eval(self):
+        context = self.get_context()
+
+        return_values = [
+            r("evaluators.one error", False),
+            r("parsers.failed", False),
+            r("parsers.failed", False),
+            reduce_requested
+        ]
+
+        evaluator = OneErrorEvaluator()
+        evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert not res.status
+        assert res.body == "evaluators.one error"
+        assert len(res.parents) == 4
+
+    def test_unwanted_return_values_are_not_eaten(self):
+        context = self.get_context()
+
+        a_successful_concept = r("successful concept")
+        a_concept_in_error = r("concept in error", False)
+        return_values = [
+            r("evaluators.one error", False),
+            r("parsers.failed", False),
+            r("parsers.failed", False),
+            a_successful_concept,
+            a_concept_in_error,
+            reduce_requested
+        ]
+
+        evaluator = OneErrorEvaluator()
+        evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert not res.status
+        assert res.body == "evaluators.one error"
+        assert len(res.parents) == 4
+
+        assert a_successful_concept not in res.parents

tests/evaluators/test_OneSuccessEvaluator.py

@@ -0,0 +1,72 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from evaluators.OneSuccessEvaluator import OneSuccessEvaluator
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def r(value, status=True):
+    return ReturnValueConcept(value, status, value)
+
+
+reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED))
+
+
+class TestOneSuccessEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("return_values, expected", [
+        ([r("evaluators.one success"), reduce_requested], True),
+        ([r("evaluators.one success"), r("failed", False), r("failed", False), reduce_requested], True),
+        ([r("evaluators.no reduce required"), r("failed", False), r("failed", False)], False),
+        ([r("evaluators.no reduce required")], False),
+        ([r("evaluators.failed", False), r("failed", False), r("failed", False), reduce_requested], False),
+        ([r("evaluators.failed", False), r("not evaluator success"), reduce_requested], False),
+        ([r("evaluators.success"), r("evaluators.another success"), reduce_requested], False),
+        ([r("evaluators.one success"), r("other success"), r("failed", False), reduce_requested], True),
+        ([r("evaluators.one success"), r("parsers.success"), reduce_requested], False),
+    ])
+    def test_i_can_match(self, return_values, expected):
+        context = self.get_context()
+        assert OneSuccessEvaluator().matches(context, return_values) == expected
+
+    def test_i_can_eval(self):
+        context = self.get_context()
+
+        return_values = [
+            r("evaluators.one success"),
+            r("failed", False),
+            r("failed", False),
+            reduce_requested
+        ]
+
+        evaluator = OneSuccessEvaluator()
+        matches = evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert matches
+        assert res.status
+        assert res.body == "evaluators.one success"
+        assert len(res.parents) == 4
+
+    def test_i_do_not_eat_the_other_success(self):
+        context = self.get_context()
+
+        not_a_parser_success = r("other success")
+        return_values = [
+            r("evaluators.one success"),
+            not_a_parser_success,
+            r("failed", False),
+            reduce_requested
+        ]
+
+        evaluator = OneSuccessEvaluator()
+        matches = evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert matches
+        assert res.status
+        assert res.body == "evaluators.one success"
+        assert len(res.parents) == 3
+
+        assert not_a_parser_success not in res.parents

tests/evaluators/test_PrepareEvalEvaluator.py

@@ -0,0 +1,49 @@
+import pytest
+
+from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept, UserInputConcept
+from core.concept import Concept
+from evaluators.PrepareEvalEvaluator import PrepareEvalEvaluator
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+r = ReturnValueConcept
+
+
+class TestPrepareEvalEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (r("name", True, UserInputConcept("eval 1 + 1")), True),
+        (r("name", True, UserInputConcept("  eval 1 + 1")), True),
+        (r("name", True, UserInputConcept("eval")), False),
+        (r("name", True, UserInputConcept("1+1")), False),
+        (r("name", True, UserInputConcept("")), False),
+        (r("name", True, UserInputConcept("eval   ")), False),
+        (r("name", True, UserInputConcept([])), False),
+        (r("name", True, Concept("foo")), False),
+        (r("name", True, "not a concept"), False),
+        (r("name", False, UserInputConcept("eval 1 + 1")), False),
+        (r("name", False, UserInputConcept("  eval 1 + 1")), False),
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert PrepareEvalEvaluator().matches(context, ret_val) == expected
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (r("name", True, UserInputConcept("eval 1 + 1")), "1 + 1"),
+        (r("name", True, UserInputConcept("   eval 1 + 1")), "1 + 1"),
+    ])
+    def test_i_can_eval(self, ret_val, expected):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        prepare_evaluator = PrepareEvalEvaluator()
+        prepare_evaluator.matches(context, ret_val)
+        res = prepare_evaluator.eval(context, ret_val)
+
+        assert len(res) == 2
+        assert res[0].status
+        assert sheerka.isinstance(res[0].body, BuiltinConcepts.USER_INPUT)
+        assert res[0].body.body == expected
+
+        assert res[1].status
+        assert sheerka.isinstance(res[1].body, BuiltinConcepts.CONCEPT_EVAL_REQUESTED)

tests/evaluators/test_PythonEvaluator.py

@@ -0,0 +1,138 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
+from core.concept import Concept
+from evaluators.PythonEvaluator import PythonEvaluator
+from parsers.PythonParser import PythonNode, PythonParser
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def get_context_name(context):
+    return context.name
+
+
+class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("ret_val, expected", [
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value=PythonNode("", None))), True),
+        (ReturnValueConcept("some_name", True, ParserResultConcept(value="other thing")), False),
+        (ReturnValueConcept("some_name", False, "not relevant"), False),
+        (ReturnValueConcept("some_name", True, Concept()), False)
+    ])
+    def test_i_can_match(self, ret_val, expected):
+        context = self.get_context()
+        assert PythonEvaluator().matches(context, ret_val) == expected
+
+    @pytest.mark.parametrize("text, expected", [
+        ("1 + 1", 2),
+        ("sheerka.test()", "I have access to Sheerka !"),
+        ("a=10\na", 10),
+    ])
+    def test_i_can_eval(self, text, expected):
+        context = self.get_context()
+        parsed = PythonParser().parse(context, text)
+
+        evaluated = PythonEvaluator().eval(context, parsed)
+
+        assert evaluated.status
+        assert evaluated.value == expected
+
+    @pytest.mark.parametrize("concept", [
+        Concept("foo"),
+        Concept("foo", body="2"),
+        Concept("foo").set_prop("prop", "'a'"),
+        Concept("foo", body="bar")
+    ])
+    def test_i_cannot_eval_simple_concept(self, concept):
+        context = self.get_context()
+        context.sheerka.add_in_cache(Concept("foo"))
+
+        parsed = PythonParser().parse(context, "foo")
+        evaluated = PythonEvaluator().eval(context, parsed)
+
+        assert not evaluated.status
+        assert context.sheerka.isinstance(evaluated.value, BuiltinConcepts.NOT_FOR_ME)
+
+    def test_i_can_eval_expression_with_that_references_concepts(self):
+        """
+        I can test modules with variables
+        :return:
+        """
+        context = self.get_context()
+        context.sheerka.add_in_cache(Concept("foo", body="1"))
+
+        parsed = PythonParser().parse(context, "foo + 2")
+        evaluated = PythonEvaluator().eval(context, parsed)
+
+        assert evaluated.status
+        assert evaluated.value == 3
+
+    def test_i_can_eval_module_with_that_references_concepts(self):
+        """
+        I can test modules with variables
+        :return:
+        """
+        context = self.get_context()
+        context.sheerka.add_in_cache(Concept("foo"))
+
+        parsed = PythonParser().parse(context, "def a(b):\n    return b\na(c:foo:)")
+        evaluated = PythonEvaluator().eval(context, parsed)
+
+        assert evaluated.status
+        assert evaluated.value == Concept("foo").init_key()
+
+    def test_i_can_eval_module_with_that_references_concepts_with_body(self):
+        """
+        I can test modules with variables
+        :return:
+        """
+        context = self.get_context()
+        context.sheerka.add_in_cache(Concept("foo", body="2"))
+
+        parsed = PythonParser().parse(context, "def a(b):\n    return b\na(foo)")
+        evaluated = PythonEvaluator().eval(context, parsed)
+
+        assert evaluated.status
+        assert evaluated.value == 2
+
+    def test_i_can_eval_concept_token(self):
+        context = self.get_context()
+        context.sheerka.add_in_cache(Concept("foo", body="2"))
+
+        parsed = PythonParser().parse(context, "get_context_name(c:foo:)")
+        python_evaluator = PythonEvaluator()
+        python_evaluator.locals["get_context_name"] = get_context_name
+        evaluated = python_evaluator.eval(context, parsed)
+
+        assert evaluated.status
+        assert evaluated.value == "foo"
+
+        # sanity, does not work otherwise
+        parsed = PythonParser().parse(context, "get_context_name(foo)")
+        python_evaluator = PythonEvaluator()
+        python_evaluator.locals["get_context_name"] = get_context_name
+        evaluated = python_evaluator.eval(context, parsed)
+
+        assert not evaluated.status
+        assert evaluated.body.body.args[0] == "'int' object has no attribute 'name'"
+
+    @pytest.mark.parametrize("text, concept_key, concept_id, use_concept", [
+        ("__C__key__C__", "key", None, False),
+        ("__C__key__id__C__", "key", "id", False),
+        ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
+        ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
+    ])
+    def test_i_can_resolve_name(self, text, concept_key, concept_id, use_concept):
+        context = self.get_context()
+        assert PythonEvaluator().resolve_name(context, text) == (concept_key, concept_id, use_concept)
+
+    @pytest.mark.parametrize("text", [
+        "__C__",
+        "__C__key",
+        "__C__key____",
+        "__C____",
+        "__C__USE_CONCEPT__",
+    ])
+    def test_i_cannot_resolve_name(self, text):
+        context = self.get_context()
+        assert PythonEvaluator().resolve_name(context, text) is None

tests/evaluators/test_TooManySucessEvaluator.py

@@ -0,0 +1,105 @@
+import pytest
+
+from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from evaluators.TooManySuccessEvaluator import TooManySuccessEvaluator
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def r(name, status=True, value=None):
+    value = value or name
+    return ReturnValueConcept(name, status, value)
+
+
+reduce_requested = ReturnValueConcept("some_name", True, Concept(key=BuiltinConcepts.REDUCE_REQUESTED))
+
+
+class TestTooManySuccessEvaluator(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("return_values, expected", [
+        ([r("evaluators.success1"), r("evaluators.success2"), reduce_requested], True),
+        ([r("evaluators.success1"), r("evaluators.success2"), r("other"), reduce_requested], True),
+        ([r("evaluators.success1"), r("evaluators.success2"), r("other", False), reduce_requested], True),
+        ([r("evaluators.a", value=Concept("c1", body="1")),
+          r("evaluators.a", value=Concept("c2", body="1")),
+          r("other"),
+          reduce_requested], True),
+        ([r("evaluators.a", value=Concept("c1", body="1")),
+          r("evaluators.a", value=Concept("c2", body="1")),
+          r("parsers.other", False),
+          reduce_requested], True),
+        ([r("evaluators.a", value=Concept("c1", body="1")),
+          r("evaluators.a", value=Concept("c2", body="1")),
+          r("parsers.other"),
+          reduce_requested], False),
+        ([r("evaluators.success1"), reduce_requested], False),
+        ([reduce_requested], False),
+        ([r("evaluators.success1"), r("evaluators.success2")], False),
+    ])
+    def test_i_can_match(self, return_values, expected):
+        context = self.get_context()
+        assert TooManySuccessEvaluator().matches(context, return_values) == expected
+
+    def test_i_can_eval(self):
+        context = self.get_context()
+
+        value1 = r("evaluators.a", value=Concept("c1", body="1"))
+        value2 = r("evaluators.a", value=Concept("c2", body="2"))
+        return_values = [
+            value1,
+            value2,
+            r("other", False),
+            reduce_requested
+        ]
+
+        evaluator = TooManySuccessEvaluator()
+        matches = evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert matches
+        assert not res.status
+        assert context.sheerka.isinstance(res.body, BuiltinConcepts.TOO_MANY_SUCCESS)
+        assert res.body.body == [value1, value2]
+        assert len(res.parents) == 4
+
+    def test_i_can_eval_when_same_success(self):
+        context = self.get_context()
+
+        return_values = [
+            r("evaluators.a", value=Concept("c1", body="1").auto_init()),
+            r("evaluators.a", value=Concept("c2", body="1").auto_init()),
+            r("other", False),
+            reduce_requested
+        ]
+
+        evaluator = TooManySuccessEvaluator()
+        matches = evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert matches
+        assert res is None
+
+    def test_other_success_are_not_reduced(self):
+        context = self.get_context()
+
+        value1 = r("evaluators.a", value=Concept("c1", body="1").auto_init())
+        value2 = r("evaluators.a", value=Concept("c2", body="2").auto_init())
+        other_success = r("other")
+        return_values = [
+            value1,
+            value2,
+            other_success,
+            reduce_requested
+        ]
+
+        evaluator = TooManySuccessEvaluator()
+        matches = evaluator.matches(context, return_values)
+        res = evaluator.eval(context, return_values)
+
+        assert matches
+        assert not res.status
+        assert context.sheerka.isinstance(res.body, BuiltinConcepts.TOO_MANY_SUCCESS)
+        assert res.body.body == [value1, value2]
+        assert len(res.parents) == 3
+        assert other_success not in res.parents

tests/non_reg/test_sheerka_non_reg.py

@@ -0,0 +1,625 @@
+import pytest
+
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept, PROPERTIES_TO_SERIALIZE, Property, simplec
+from evaluators.MutipleSameSuccessEvaluator import MultipleSameSuccessEvaluator
+from parsers.ConceptLexerParser import Sequence, StrMatch, OrderedChoice, Optional, ConceptExpression
+from sdp.sheerkaDataProvider import SheerkaDataProvider
+
+from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
+
+
+class TestSheerkaNonReg(TestUsingFileBasedSheerka):
+
+    @pytest.mark.parametrize("text, expected", [
+        ("1 + 1", 2),
+        ("sheerka.test()", 'I have access to Sheerka !')
+    ])
+    def test_i_can_eval_python_expressions_with_no_variable(self, text, expected):
+        sheerka = self.get_sheerka()
+
+        res = sheerka.evaluate_user_input(text)
+
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].value == expected
+
+    def test_i_can_eval_concept_with_python_body(self):
+        sheerka = self.get_sheerka()
+        concept = Concept(name="one", body="1")
+        sheerka.add_in_cache(concept)
+
+        text = "one"
+        res = sheerka.evaluate_user_input(text)
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].value == simplec("one", 1)  # by default, the concept is returned
+
+    def test_i_can_eval_concept_with_concept_body(self):
+        sheerka = self.get_sheerka()
+        concept_one = Concept(name="one")
+        concept_un = Concept(name="un", body="one")
+        sheerka.add_in_cache(concept_one)
+        sheerka.add_in_cache(concept_un)
+
+        res = sheerka.evaluate_user_input("un")
+        return_value = res[0].value
+        assert len(res) == 1
+        assert res[0].status
+        assert return_value == simplec("un", simplec("one", None))
+
+    def test_i_can_eval_concept_with_no_body(self):
+        sheerka = self.get_sheerka()
+        concept = Concept(name="one")
+        sheerka.add_in_cache(concept)
+
+        text = "one"
+        res = sheerka.evaluate_user_input(text)
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].value == concept
+        assert id(res[0].value) != id(concept)
+
+    def test_is_unique_property_is_used_when_evaluating(self):
+        sheerka = self.get_sheerka()
+        concept = Concept(name="one", is_unique=True)
+        sheerka.add_in_cache(concept)
+
+        text = "one"
+        res = sheerka.evaluate_user_input(text)
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].value == concept
+        assert id(res[0].value) == id(concept)
+
+    def test_i_can_eval_def_concept_request(self):
+        text = """
+def concept a + b
+where isinstance(a, int) and isinstance(b, int)
+pre isinstance(a, int) and isinstance(b, int)
+post isinstance(res, int)
+as:
+    def func(x,y):
+        return x+y
+    func(a,b)
+        """
+
+        expected = self.get_default_concept()
+        expected.metadata.id = "1001"
+        expected.metadata.desc = None
+        expected.metadata.props = [("a", None), ("b", None)]
+        expected.init_key()
+
+        sheerka = self.get_sheerka()
+        res = sheerka.evaluate_user_input(text)
+
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
+
+        concept_saved = res[0].value.body
+
+        for prop in PROPERTIES_TO_SERIALIZE:
+            assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop)
+
+        assert concept_saved.key in sheerka.cache_by_key
+        assert concept_saved.id in sheerka.cache_by_id
+        assert sheerka.sdp.io.exists(
+            sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
+
+    def test_i_can_eval_def_concept_part_when_one_part_is_a_ref_of_another_concept(self):
+        """
+        In this test, we test that the properties of 'concept a xx b' (which are 'a' and 'b')
+        are correctly detected, thanks to the source code 'a plus b' in its body
+        :return:
+        """
+        sheerka = self.get_sheerka()
+
+        # concept 'a plus b' is known
+        concept_a_plus_b = Concept(name="a plus b").def_prop("a").def_prop("b")
+        sheerka.add_in_cache(concept_a_plus_b)
+
+        res = sheerka.evaluate_user_input("def concept a xx b as a plus b")
+        expected = Concept(name="a xx b", body="a plus b").def_prop("a").def_prop("b").init_key()
+        expected.metadata.id = "1001"
+
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
+
+        concept_saved = res[0].value.body
+
+        for prop in PROPERTIES_TO_SERIALIZE:
+            assert getattr(concept_saved.metadata, prop) == getattr(expected.metadata, prop)
+
+        assert concept_saved.key in sheerka.cache_by_key
+        assert concept_saved.id in sheerka.cache_by_id
+        assert sheerka.sdp.io.exists(
+            sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
+
+    def test_i_cannot_eval_the_same_def_concept_twice(self):
+        text = """
+def concept a + b
+where isinstance(a, int) and isinstance(b, int)
+pre isinstance(a, int) and isinstance(b, int)
+post isinstance(res, int)
+as:
+    def func(x,y):
+        return x+y
+    func(a,b)
+            """
+
+        sheerka = self.get_sheerka()
+        sheerka.evaluate_user_input(text)
+        res = sheerka.evaluate_user_input(text)
+
+        assert len(res) == 1
+        assert not res[0].status
+        assert sheerka.isinstance(res[0].value, BuiltinConcepts.CONCEPT_ALREADY_DEFINED)
+
+    @pytest.mark.parametrize("text", [
+        "",
+        "   ",
+        "\n",
+    ])
+    def test_i_can_eval_a_empty_input(self, text):
+        sheerka = self.get_sheerka()
+
+        res = sheerka.evaluate_user_input(text)
+
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, BuiltinConcepts.NOP)
+
+    def test_i_can_eval_concept_with_variable(self):
+        sheerka = self.get_sheerka()
+        concept_hello = Concept(name="hello a").def_prop("a")
+        concept_foo = Concept(name="foo")
+        sheerka.add_in_cache(concept_hello)
+        sheerka.add_in_cache(concept_foo)
+
+        res = sheerka.evaluate_user_input("hello foo")
+        return_value = res[0].value
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(return_value, concept_hello)
+        assert return_value.props["a"].value == concept_foo
+
+    def test_i_can_eval_concept_with_variable_and_python_as_body(self):
+        sheerka = self.get_sheerka()
+        hello_a = sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_prop("a"))
+        sheerka.add_in_cache(Concept(name="foo", body="'foo'"))
+
+        res = sheerka.evaluate_user_input("hello foo")
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, hello_a)
+        assert res[0].value.body == "hello foo"
+        assert res[0].value.metadata.is_evaluated
+        assert res[0].value.props["a"].value == simplec("foo", "foo")
+        assert res[0].value.props["a"].value.metadata.is_evaluated
+
+    def test_i_can_eval_duplicate_concepts_with_same_value(self):
+        sheerka = self.get_sheerka()
+
+        sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_prop("a"))
+        sheerka.add_in_cache(Concept(name="hello foo", body="'hello foo'"))
+        sheerka.add_in_cache(Concept(name="foo", body="'foo'"))
+
+        res = sheerka.evaluate_user_input("hello foo")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].value.body == "hello foo"
+        assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME)
+
+    def test_i_cannot_manage_duplicate_concepts_when_the_values_are_different(self):
+        sheerka = self.get_sheerka()
+
+        sheerka.add_in_cache(Concept(name="hello a", body="'hello ' + a").def_prop("a"))
+        sheerka.add_in_cache(Concept(name="hello foo", body="'hello foo'"))
+        sheerka.add_in_cache(Concept(name="foo", body="'another value'"))
+
+        res = sheerka.evaluate_user_input("hello foo")
+        assert len(res) == 1
+        assert not res[0].status
+        assert sheerka.isinstance(res[0].value, BuiltinConcepts.TOO_MANY_SUCCESS)
+
+        concepts = res[0].value.body
+        assert len(concepts) == 2
+        sorted_values = sorted(concepts, key=lambda x: x.value.body)
+        assert sorted_values[0].value.body == "hello another value"
+        assert sorted_values[1].value.body == "hello foo"
+
+    def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same(self):
+        sheerka = self.get_sheerka()
+        context = self.get_context(sheerka)
+
+        sheerka.create_new_concept(context, Concept(name="hello a", body="'hello ' + a").def_prop("a"))
+        sheerka.create_new_concept(context, Concept(name="hello b", body="'hello ' + b").def_prop("b"))
+
+        res = sheerka.evaluate_user_input("hello 'foo'")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].value.body == "hello foo"  # I don't know yet the one to choose
+        assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME)
+
+    def test_i_can_create_concepts_with_python_code_as_body(self):
+        sheerka = self.get_sheerka()
+        context = self.get_context(sheerka)
+
+        sheerka.create_new_concept(context, Concept(name="concepts", body="sheerka.concepts()"))
+        res = sheerka.evaluate_user_input("concepts")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert isinstance(res[0].value.body, list)
+
+    def test_i_can_create_concept_with_bnf_definition(self):
+        sheerka = self.get_sheerka(False, False)
+        a = Concept("a")
+        sheerka.add_in_cache(a)
+        sheerka.concepts_definition_cache = {a: OrderedChoice("one", "two")}
+
+        res = sheerka.evaluate_user_input("def concept plus from bnf a ('plus' plus)?")
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, BuiltinConcepts.NEW_CONCEPT)
+
+        saved_concept = sheerka.sdp.get_safe(sheerka.CONCEPTS_ENTRY, "plus")
+        assert saved_concept.key == "plus"
+        assert saved_concept.metadata.definition == "a ('plus' plus)?"
+        assert "a" in saved_concept.props
+        assert "plus" in saved_concept.props
+
+        saved_definitions = sheerka.sdp.get_safe(sheerka.CONCEPTS_DEFINITIONS_ENTRY)
+        expected_bnf = Sequence(
+            a, Optional(Sequence(StrMatch("plus"), ConceptExpression("plus", rule_name="plus"))))
+        assert saved_definitions[saved_concept] == expected_bnf
+
+        new_concept = res[0].value.body
+        assert new_concept.metadata.name == "plus"
+        assert new_concept.metadata.definition == "a ('plus' plus)?"
+        assert new_concept.bnf == expected_bnf
+        assert "a" in new_concept.props
+        assert "plus" in new_concept.props
+
+    def test_i_can_eval_bnf_definitions(self):
+        sheerka = self.get_sheerka()
+        concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' | 'two'")[0].body.body
+
+        res = sheerka.evaluate_user_input("one")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, concept_a)
+
+    def test_i_can_eval_bnf_definitions_with_variables(self):
+        sheerka = self.get_sheerka()
+        concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' | 'two'")[0].body.body
+        concept_b = sheerka.evaluate_user_input("def concept b from bnf a 'three'")[0].body.body
+
+        res = sheerka.evaluate_user_input("one three")
+
+        assert len(res) == 1
+        assert res[0].status
+        return_value = res[0].value
+
+        assert sheerka.isinstance(return_value, concept_b)
+        assert return_value.body == "one three"
+        assert return_value.metadata.is_evaluated
+
+        assert return_value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one").init_key())
+        assert return_value.props["a"].value.metadata.is_evaluated
+
+    def test_i_can_eval_bnf_definitions_from_separate_instances(self):
+        """
+        Same test then before,
+        but make sure that the BNF are correctly persisted and loaded
+        """
+        sheerka = self.get_sheerka(False)
+        concept_a = sheerka.evaluate_user_input("def concept a from bnf 'one' 'two'")[0].body.body
+
+        res = self.get_sheerka(False).evaluate_user_input("one two")
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, concept_a)
+
+        # add another bnf definition
+        concept_b = sheerka.evaluate_user_input("def concept b from bnf a 'three'")[0].body.body
+
+        res = self.get_sheerka(False).evaluate_user_input("one two")  # previous one still works
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, concept_a)
+
+        res = self.get_sheerka(False).evaluate_user_input("one two three")  # new one works
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].value, concept_b)
+        assert res[0].value.body == "one two three"
+        assert res[0].value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one two").init_key())
+
+    @pytest.mark.parametrize("desc, definitions", [
+        ("Simple form", [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept twenties from bnf 'twenty' (one|two)=unit as 20 + unit"
+        ]),
+        ("When twenty is a concept", [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept twenty as 20",
+            "def concept twenties from bnf twenty (one|two)=unit as twenty + unit"
+        ]),
+        ("When digit is a concept", [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept twenty as 20",
+            "def concept digit from bnf one|two",
+            "def concept twenties from bnf twenty digit as twenty + digit"
+        ]),
+        ("When using isa and concept twenty", [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept number",
+            "one isa number",
+            "two isa number",
+            "def concept twenties from bnf 'twenty' number as 20 + number"
+        ]),
+        ("When using isa and concept twenty", [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept twenty as 20",
+            "def concept number",
+            "one isa number",
+            "two isa number",
+            "def concept twenties from bnf twenty number as 20 + number"
+        ]),
+    ])
+    def test_i_can_mix_concept_with_python_to_define_numbers(self, desc, definitions):
+        sheerka = self.get_sheerka()
+
+        for definition in definitions:
+            sheerka.evaluate_user_input(definition)
+
+        res = sheerka.evaluate_user_input("twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].body, "twenties")
+        assert res[0].body.body == 21
+
+        res = sheerka.evaluate_user_input("twenty one + 1")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 22
+
+        res = sheerka.evaluate_user_input("twenty one + one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 22
+
+        res = sheerka.evaluate_user_input("twenty one + twenty two")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 43
+
+        res = sheerka.evaluate_user_input("1 + twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 22
+
+        res = sheerka.evaluate_user_input("1 + 1 + twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 23
+
+    def test_i_can_mix_bnf_and_isa(self):
+        """
+        if 'one' isa 'number, twenty number should be recognized
+        :return:
+        """
+        sheerka = self.get_sheerka()
+        sheerka.evaluate_user_input("def concept one as 1")
+        sheerka.evaluate_user_input("def concept two as 2")
+        sheerka.evaluate_user_input("def concept number")
+        sheerka.evaluate_user_input("one isa number")
+        sheerka.evaluate_user_input("two isa number")
+        sheerka.evaluate_user_input("def concept twenties from bnf 'twenty' number as 20 + number")
+
+        res = sheerka.evaluate_user_input("twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == simplec("twenties", 21)
+
+        res = sheerka.evaluate_user_input("twenty one + 1")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 22
+
+        res = sheerka.evaluate_user_input("twenty one + one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 22
+
+        res = sheerka.evaluate_user_input("twenty one + twenty two")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 43
+
+        res = sheerka.evaluate_user_input("1 + twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 22
+
+        res = sheerka.evaluate_user_input("1 + 1 + twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 23
+
+    def test_i_can_mix_concept_of_concept(self):
+        sheerka = self.get_sheerka()
+
+        definitions = [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept twenties from bnf 'twenty' (one|two)=unit as 20 + unit",
+            "def concept a plus b as a + b"
+        ]
+
+        for definition in definitions:
+            sheerka.evaluate_user_input(definition)
+
+        res = sheerka.evaluate_user_input("1 plus 2")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 3
+
+        res = sheerka.evaluate_user_input("1 plus one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 2
+
+        res = sheerka.evaluate_user_input("1 + 1 plus 1")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 3
+
+        res = sheerka.evaluate_user_input("1 plus twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 22
+
+        res = sheerka.evaluate_user_input("one plus 1")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 2
+
+        res = sheerka.evaluate_user_input("one plus two")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 3
+
+        res = sheerka.evaluate_user_input("one plus twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 22
+
+        res = sheerka.evaluate_user_input("twenty one plus 1")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 22
+
+        res = sheerka.evaluate_user_input("twenty one plus one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 22
+
+        res = sheerka.evaluate_user_input("twenty one plus twenty two")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body.body == 43
+
+    @pytest.mark.xfail
+    def test_i_can_evaluate_concept_of_concept_when_multiple_choices(self):
+        sheerka = self.get_sheerka()
+
+        definitions = [
+            "def concept little a where a",
+            "def concept blue a where a",
+            "def concept little blue a where a",
+            "def concept house"
+        ]
+
+        for definition in definitions:
+            sheerka.evaluate_user_input(definition)
+
+        ### CAUTION ####
+        # this test cannot work !!
+        # it is just to hint the result that I would like to achieve
+
+        res = sheerka.evaluate_user_input("little blue house")
+        assert len(res) == 2
+        assert res[0].status
+        assert res[0].body == "little(blue(house))"
+
+        assert res[1].status
+        assert res[1].body == "little blue(house)"
+
+    def test_i_can_say_that_a_concept_isa_another_concept(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluate_user_input("def concept foo")
+        sheerka.evaluate_user_input("def concept bar")
+
+        res = sheerka.evaluate_user_input("foo isa bar")
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].body, BuiltinConcepts.SUCCESS)
+
+    def test_eval_does_not_break_valid_result(self):
+        sheerka = self.get_sheerka()
+        sheerka.evaluate_user_input("def concept one as 1")
+        sheerka.evaluate_user_input("def concept two as 2")
+
+        res = sheerka.evaluate_user_input("one + two")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 3
+
+        res = sheerka.evaluate_user_input("eval one + two")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 3
+
+    @pytest.mark.parametrize("text", [
+        "'hello",
+        '"foo" + "string',
+        "c::",
+        "c:foo\nbar:",
+        "c:foo",
+        "def concept 'name",
+        "def concept name from bnf 'name"
+    ])
+    def test_i_can_manage_tokenizer_error(self, text):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept("foo"))
+
+        res = sheerka.evaluate_user_input(text)
+
+        assert len(res) > 1
+        for r in [r for r in res if r.who.startswith("parsers.")]:
+            assert not r.status
+
+    def test_i_can_recognize_concept_from_string(self):
+        sheerka = self.get_sheerka()
+        sheerka.add_in_cache(Concept("one", body="1"))
+
+        res = sheerka.evaluate_user_input("'one'")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == "one"
+
+        res = sheerka.evaluate_user_input("eval 'one'")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == "one"
+
+    @pytest.mark.parametrize("expression", [
+        "def concept twenties from bnf 'twenty' (one | two)=unit as 20 + unit",
+        "def concept twenties from bnf 'twenty' (one | two)=unit as twenty + unit",
+        "def concept twenties from bnf twenty (one | two)=unit as 20 + unit",
+        "def concept twenties from bnf twenty (one | two)=unit as twenty + unit",
+    ])
+    def test_i_can_evaluate_bnf_concepts(self, expression):
+        sheerka = self.get_sheerka()
+
+        sheerka.evaluate_user_input("def concept one as 1")
+        sheerka.evaluate_user_input("def concept two as 2")
+        sheerka.evaluate_user_input("def concept twenty as 20")
+        sheerka.evaluate_user_input(expression)
+        res = sheerka.evaluate_user_input("eval twenty one")
+
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == 21

tests/parsers/test_BaseParser.py

@@ -1,6 +1,6 @@
 import pytest
 
-from core.tokenizer import Tokenizer, Token, TokenKind
+from core.tokenizer import Tokenizer, TokenKind
 from parsers.BaseParser import BaseParser
 
 

tests/parsers/test_BnfParser.py

@@ -0,0 +1,176 @@
+import pytest
+
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from core.tokenizer import Tokenizer, TokenKind, LexerError
+from parsers.BaseParser import UnexpectedTokenErrorNode
+from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError
+from parsers.ConceptLexerParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \
+    ConceptLexerParser, ConceptExpression, cnode
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class ClassWithName:
+    def __init__(self, name):
+        self.name = name
+
+
+class TestBnfParser(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("expression, expected", [
+        ("'str'", StrMatch("str")),
+        ("1", StrMatch("1")),
+        ("  1", StrMatch("1")),
+        (",", StrMatch(",")),
+        ("'foo'?", Optional(StrMatch("foo"))),
+        ("'foo'*", ZeroOrMore(StrMatch("foo"))),
+        ("'foo'+", OneOrMore(StrMatch("foo"))),
+        ("1 | 2 | 3", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"))),
+        ("1|2|3", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"))),
+        ("1'|' 2 '|' 3", Sequence(StrMatch("1"), StrMatch("|"), StrMatch("2"), StrMatch("|"), StrMatch("3"))),
+        ("1 2 'foo'", Sequence(StrMatch("1"), StrMatch("2"), StrMatch("foo"))),
+        ("1 2 | 3 4+", OrderedChoice(
+            Sequence(StrMatch("1"), StrMatch("2")),
+            Sequence(StrMatch("3"), OneOrMore(StrMatch("4"))))),
+        (
+            "1 (2 | 3) 4+",
+            Sequence(StrMatch("1"), OrderedChoice(StrMatch("2"), StrMatch("3")), OneOrMore(StrMatch("4")))),
+        ("(1|2)+", OneOrMore(OrderedChoice(StrMatch("1"), StrMatch("2")))),
+        ("(1 2)+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")))),
+        ("1 *", Sequence(StrMatch("1"), StrMatch("*"))),
+        ("1 ?", Sequence(StrMatch("1"), StrMatch("?"))),
+        ("1 +", Sequence(StrMatch("1"), StrMatch("+"))),
+        ("(1|*) +", Sequence(OrderedChoice(StrMatch("1"), StrMatch("*")), StrMatch("+"))),
+        ("1, :&", Sequence(StrMatch("1"), StrMatch(","), StrMatch(":"), StrMatch("&"))),
+        ("(1 )", StrMatch("1")),
+        ("'str'=var", StrMatch("str", rule_name="var")),
+        ("'foo'?=var", Optional(StrMatch("foo"), rule_name="var")),
+        ("('foo'?)=var", Optional(StrMatch("foo"), rule_name="var")),
+        ("'foo'*=var", ZeroOrMore(StrMatch("foo"), rule_name="var")),
+        ("('foo'*)=var", ZeroOrMore(StrMatch("foo"), rule_name="var")),
+        ("'foo'+=var", OneOrMore(StrMatch("foo"), rule_name="var")),
+        ("('foo'+)=var", OneOrMore(StrMatch("foo"), rule_name="var")),
+        ("'foo'=var?", Optional(StrMatch("foo", rule_name="var"))),
+        ("('foo'=var)?", Optional(StrMatch("foo", rule_name="var"))),
+        ("'foo'=var*", ZeroOrMore(StrMatch("foo", rule_name="var"))),
+        ("('foo'=var)*", ZeroOrMore(StrMatch("foo", rule_name="var"))),
+        ("'foo'=var+", OneOrMore(StrMatch("foo", rule_name="var"))),
+        ("('foo'=var)+", OneOrMore(StrMatch("foo", rule_name="var"))),
+        ("(1 | 2 | 3)=var", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"), rule_name="var")),
+        ("(1 2)=var", Sequence(StrMatch("1"), StrMatch("2"), rule_name="var")),
+        ("(1 2)+=var", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")), rule_name="var")),
+        ("(1 2)=var+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2"), rule_name="var"))),
+    ])
+    def test_i_can_parse_regex(self, expression, expected):
+        parser = BnfParser()
+        res = parser.parse(self.get_context(), Tokenizer(expression))
+
+        assert not parser.has_error
+        assert res.status
+        assert res.value.value == expected
+        assert res.value.source == expression
+
+    @pytest.mark.parametrize("expression, expected", [
+        ("foo", Concept("foo").init_key()),
+        ("foo*", ZeroOrMore(Concept("foo").init_key())),
+        ("foo 'and' bar+", Sequence(Concept("foo").init_key(), StrMatch("and"), OneOrMore(Concept("bar").init_key()))),
+        ("foo | bar?", OrderedChoice(Concept("foo").init_key(), Optional(Concept("bar").init_key()))),
+        ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
+        ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
+    ])
+    def test_i_can_parse_regex_with_concept(self, expression, expected):
+        foo = Concept("foo")
+        bar = Concept("bar")
+        var = Concept("var")
+        context = self.get_context()
+
+        for c in (foo, bar, var):
+            context.sheerka.add_in_cache(c)
+        parser = BnfParser()
+        res = parser.parse(context, Tokenizer(expression))
+
+        assert not parser.has_error
+        assert res.status
+        assert res.value.value == expected
+        assert res.value.source == expression
+
+    def test_i_can_parse_regex_with_concept_when_the_concept_is_still_under_definition(self):
+        expression = "foo"
+        expected = ConceptExpression("foo")
+
+        context = self.get_context()
+        context.obj = ClassWithName("foo")
+
+        parser = BnfParser()
+        res = parser.parse(context, Tokenizer(expression))
+
+        assert not parser.has_error
+        assert res.status
+        assert res.value.value == expected
+        assert res.value.source == expression
+
+    @pytest.mark.parametrize("expression, error", [
+        ("1 ", UnexpectedEndOfFileError()),
+        ("1|", UnexpectedEndOfFileError()),
+        ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.RPAR])),
+        ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.IDENTIFIER])),
+        ("'name", LexerError("Missing Trailing quote", "'name", 5, 1, 6))
+    ])
+    def test_i_can_detect_errors(self, expression, error):
+        parser = BnfParser()
+        res = parser.parse(self.get_context(), Tokenizer(expression))
+        ret_value = res.value.value
+        assert parser.has_error
+        assert not res.status
+        assert ret_value[0] == error
+
+    def test_i_can_use_the_result_of_regex_parsing_to_parse_a_text(self):
+        foo = Concept(name="foo")
+        bar = Concept(name="bar")
+        context = self.get_context()
+        context.sheerka.add_in_cache(foo)
+        context.sheerka.add_in_cache(bar)
+
+        regex_parser = BnfParser()
+        foo_definition = regex_parser.parse(context, "'twenty' | 'thirty'").value.value
+        bar_definition = regex_parser.parse(context, "foo ('one' | 'two')").value.value
+
+        concepts = {bar: bar_definition, foo: foo_definition}
+        concept_parser = ConceptLexerParser()
+        concept_parser.initialize(context, concepts)
+
+        res = concept_parser.parse(context, "twenty two")
+        assert res.status
+        assert res.value.body == [cnode("bar", 0, 2, "twenty two")]
+
+        res = concept_parser.parse(context, "thirty one")
+        assert res.status
+        assert res.value.body == [cnode("bar", 0, 2, "thirty one")]
+
+        res = concept_parser.parse(context, "twenty")
+        assert res.status
+        assert res.value.body == [cnode("foo", 0, 0, "twenty")]
+
+    def test_i_cannot_parse_when_too_many_concepts(self):
+        foo1 = Concept(name="foo", body="1")
+        foo2 = Concept(name="foo", body="2")
+        context = self.get_context()
+        context.sheerka.cache_by_key["foo"] = [foo1, foo2]
+
+        regex_parser = BnfParser()
+        res = regex_parser.parse(context, "foo")
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.CANNOT_RESOLVE_CONCEPT)
+        assert res.value.body == ('key', 'foo')
+
+    def test_i_cannot_parse_when_unknown_concept(self):
+        context = self.get_context()
+
+        regex_parser = BnfParser()
+        res = regex_parser.parse(self.get_context(), "foo")
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert res.value.body == ('key', 'foo')

tests/parsers/test_ConceptsWithConceptsParser.py

@@ -0,0 +1,193 @@
+import ast
+
+import pytest
+
+from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from core.tokenizer import Token, TokenKind, Tokenizer
+from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode, SourceCodeNode
+from parsers.ConceptsWithConceptsParser import ConceptsWithConceptsParser
+from parsers.MultipleConceptsParser import MultipleConceptsParser
+from parsers.PythonParser import PythonNode
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+multiple_concepts_parser = MultipleConceptsParser()
+
+
+def ret_val(*args):
+    result = []
+    index = 0
+    source = ""
+    for item in args:
+        if isinstance(item, Concept):
+            tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)]
+            result.append(ConceptNode(item, index, index, tokens, item.name))
+            index += 1
+            source += item.name
+        elif isinstance(item, PythonNode):
+            tokens = list(Tokenizer(item.source))[:-1]  # strip trailing EOF
+            result.append(SourceCodeNode(item, index, index + len(tokens) - 1, tokens, item.source))
+            index += len(tokens)
+            source += item.source
+        else:
+            tokens = list(Tokenizer(item))[:-1]  # strip trailing EOF
+            result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens))
+            index += len(tokens)
+            source += item
+
+    return ReturnValueConcept(
+        "who",
+        False,
+        ParserResultConcept(parser=multiple_concepts_parser, value=result, source=source))
+
+
+class TestConceptsWithConceptsParser(TestUsingMemoryBasedSheerka):
+
+    def init(self, concepts, inputs):
+        context = self.get_context()
+        for concept in concepts:
+            context.sheerka.create_new_concept(context, concept)
+
+        return context, ret_val(*inputs)
+
+    def execute(self, concepts, inputs):
+        context, input_return_values = self.init(concepts, inputs)
+
+        parser = ConceptsWithConceptsParser()
+        result = parser.parse(context, input_return_values.body)
+
+        wrapper = result.body
+        return_value = result.body.body
+
+        return context, parser, result, wrapper, return_value
+
+    @pytest.mark.parametrize("text, interested", [
+        ("not parser result", False),
+        (ParserResultConcept(parser="not multiple_concepts_parser"), False),
+        (ParserResultConcept(parser=multiple_concepts_parser, value=[]), True),
+    ])
+    def test_not_interested(self, text, interested):
+        context = self.get_context()
+
+        res = ConceptsWithConceptsParser().parse(context, text)
+        if interested:
+            assert res is not None
+        else:
+            assert res is None
+
+    def test_i_can_parse_composition_of_concepts(self):
+        foo = Concept("foo")
+        bar = Concept("bar")
+        plus = Concept("a plus b").def_prop("a").def_prop("b")
+
+        context, parser, result, wrapper, return_value = self.execute([foo, bar, plus], [foo, " plus ", bar])
+
+        assert result.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert result.who == wrapper.parser.name
+        assert wrapper.source == "foo plus bar"
+        assert context.sheerka.isinstance(return_value, plus)
+
+        assert return_value.compiled["a"] == foo
+        assert return_value.compiled["b"] == bar
+
+        # sanity check, I can evaluate the result
+        evaluated = context.sheerka.evaluate_concept(context, return_value)
+        assert evaluated.key == return_value.key
+        assert evaluated.get_prop("a") == foo.init_key()
+        assert evaluated.get_prop("b") == bar.init_key()
+
+    def test_i_can_parse_when_composition_of_source_code(self):
+        plus = Concept("a plus b", body="a + b").def_prop("a").def_prop("b")
+        left = PythonNode("1+1", ast.parse("1+1", mode="eval"))
+        right = PythonNode("2+2", ast.parse("2+2", mode="eval"))
+        context, parser, result, wrapper, return_value = self.execute([plus], [left, " plus ", right])
+
+        assert result.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert result.who == wrapper.parser.name
+        assert wrapper.source == "1+1 plus 2+2"
+        assert context.sheerka.isinstance(return_value, plus)
+
+        left_parser_result = ParserResultConcept(parser=parser, source="1+1", value=left)
+        right_parser_result = ParserResultConcept(parser=parser, source="2+2", value=right)
+        assert return_value.compiled["a"] == [ReturnValueConcept(parser.name, True, left_parser_result)]
+        assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, right_parser_result)]
+
+        # sanity check, I can evaluate the result
+        evaluated = context.sheerka.evaluate_concept(context, return_value)
+        assert evaluated.key == return_value.key
+        assert evaluated.get_prop("a") == 2
+        assert evaluated.get_prop("b") == 4
+        assert evaluated.body == 6
+
+    def test_i_can_parse_when_mix_of_concept_and_code(self):
+        plus = Concept("a plus b").def_prop("a").def_prop("b")
+        code = PythonNode("1+1", ast.parse("1+1", mode="eval"))
+        foo = Concept("foo")
+        context, parser, result, wrapper, return_value = self.execute([plus, foo], [foo, " plus ", code])
+
+        assert result.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert result.who == wrapper.parser.name
+        assert wrapper.source == "foo plus 1+1"
+        assert context.sheerka.isinstance(return_value, plus)
+
+        code_parser_result = ParserResultConcept(parser=parser, source="1+1", value=code)
+        assert return_value.compiled["a"] == foo
+        assert return_value.compiled["b"] == [ReturnValueConcept(parser.name, True, code_parser_result)]
+
+        # sanity check, I can evaluate the result
+        evaluated = context.sheerka.evaluate_concept(context, return_value)
+        assert evaluated.key == return_value.key
+        assert evaluated.get_prop("a") == foo.init_key()
+        assert evaluated.get_prop("b") == 2
+
+    def test_i_can_parse_when_multiple_concepts_are_recognized(self):
+        foo = Concept("foo")
+        bar = Concept("bar")
+        plus_1 = Concept("a plus b", body="body1").def_prop("a").def_prop("b")
+        plus_2 = Concept("a plus b", body="body2").def_prop("a").def_prop("b")
+
+        context, input_return_values = self.init([foo, bar, plus_1, plus_2], [foo, " plus ", bar])
+        parser = ConceptsWithConceptsParser()
+        result = parser.parse(context, input_return_values.body)
+
+        assert len(result) == 2
+
+        res = result[0]
+        wrapper = res.value
+        return_value = res.value.value
+        assert res.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert res.who == wrapper.parser.name
+        assert wrapper.source == "foo plus bar"
+        assert context.sheerka.isinstance(return_value, plus_1)
+        assert return_value.compiled["a"] == foo
+        assert return_value.compiled["b"] == bar
+
+        res = result[1]
+        wrapper = res.value
+        return_value = res.value.value
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert res.who == wrapper.parser.name
+        assert wrapper.source == "foo plus bar"
+        assert context.sheerka.isinstance(return_value, plus_2)
+        assert return_value.compiled["a"] == foo
+        assert return_value.compiled["b"] == bar
+
+    def test_i_cannot_parse_when_unknown_concept(self):
+        foo = Concept("foo")
+        bar = Concept("bar")
+
+        context, input_return_values = self.init([foo, bar], [foo, " plus ", bar])
+        parser = ConceptsWithConceptsParser()
+        result = parser.parse(context, input_return_values.body)
+        wrapper = result.body
+        return_value = result.body.body
+
+        assert not result.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.NOT_FOR_ME)
+        assert result.who == parser.name
+        assert return_value == input_return_values.body.body

tests/parsers/test_DefaultParser.py

@@ -0,0 +1,345 @@
+import pytest
+import ast
+
+from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
+from core.concept import Concept
+from parsers.ConceptLexerParser import OrderedChoice, StrMatch
+from parsers.PythonParser import PythonParser, PythonNode
+from core.tokenizer import Keywords, Tokenizer, LexerError
+from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode, CannotHandleErrorNode, IsaConceptNode
+from parsers.DefaultParser import UnexpectedTokenErrorNode, DefConceptNode
+from parsers.BnfParser import BnfParser
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None):
+    def_concept = DefConceptNode([], name=NameNode(list(Tokenizer(name))))
+
+    if body:
+        def_concept.body = get_concept_part(body)
+    if where:
+        def_concept.where = get_concept_part(where)
+    if pre:
+        def_concept.pre = get_concept_part(pre)
+    if post:
+        def_concept.post = get_concept_part(post)
+    if definition:
+        def_concept.definition = ReturnValueConcept(
+            "parsers.Bnf",
+            True,
+            definition)
+
+    return def_concept
+
+
+def get_concept_part(part):
+    if isinstance(part, str):
+        node = PythonNode(part, ast.parse(part, mode="eval"))
+        return ReturnValueConcept(
+            who="parsers.Default",
+            status=True,
+            value=ParserResultConcept(
+                source=part,
+                parser=PythonParser(),
+                value=node))
+
+    if isinstance(part, PythonNode):
+        return ReturnValueConcept(
+            who="parsers.Default",
+            status=True,
+            value=ParserResultConcept(
+                source=part.source,
+                parser=PythonParser(),
+                value=part))
+
+    if isinstance(part, ReturnValueConcept):
+        return part
+
+
+class TestDefaultParser(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("text, expected", [
+        ("def concept hello", get_def_concept(name="hello")),
+        ("def concept hello  ", get_def_concept(name="hello")),
+        ("def concept a + b", get_def_concept(name="a + b")),
+        ("def concept a+b", get_def_concept(name="a + b")),
+        ("def concept 'a+b'+c", get_def_concept(name="'a+b' + c")),
+        ("def concept 'as if'", get_def_concept(name="'as if'")),
+        ("def concept 'as' if", get_def_concept(name="'as if'")),
+        ("def concept hello as 'hello'", get_def_concept(name="hello", body="'hello'")),
+        ("def concept hello as 1", get_def_concept(name="hello", body="1")),
+        ("def concept hello as 1 + 1", get_def_concept(name="hello", body="1 + 1")),
+    ])
+    def test_i_can_parse_def_concept(self, text, expected):
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        node = res.value.value
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+        assert node == expected
+
+    def test_i_can_parse_complex_def_concept_statement(self):
+        text = """def concept a plus b
+where a,b
+pre isinstance(a, int) and isinstance(b, float)
+post isinstance(res, int)
+as res = a + b
+    """
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        return_value = res.value
+        expected_concept = get_def_concept(
+            name="a plus b",
+            where="a,b",
+            pre="isinstance(a, int) and isinstance(b, float)",
+            post="isinstance(res, int)",
+            body=PythonNode("res = a + b", ast.parse("res = a + b", mode="exec"))
+        )
+
+        assert res.status
+        assert isinstance(return_value, ParserResultConcept)
+        assert return_value.value == expected_concept
+
+    def test_i_can_have_mutilines_declarations(self):
+        text = """
+def concept add one to a as
+def func(x):
+    return x+1
+func(a)
+        """
+
+        expected_concept = get_def_concept(
+            name="add one to a ",
+            body=PythonNode(
+                "def func(x):\n    return x+1\nfunc(a)",
+                ast.parse("def func(x):\n    return x+1\nfunc(a)", mode="exec"))
+        )
+
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        return_value = res.value
+
+        assert res.status
+        assert isinstance(return_value, ParserResultConcept)
+        assert return_value.value == expected_concept
+
+    def test_i_can_use_colon_to_use_indentation(self):
+        text = """
+def concept add one to a as:
+    def func(x):
+        return x+1
+    func(a)
+    """
+
+        expected_concept = get_def_concept(
+            name="add one to a ",
+            body=PythonNode(
+                "def func(x):\n    return x+1\nfunc(a)",
+                ast.parse("def func(x):\n    return x+1\nfunc(a)", mode="exec"))
+        )
+
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        return_value = res.value
+
+        assert res.status
+        assert isinstance(return_value, ParserResultConcept)
+        assert return_value.value == expected_concept
+
+    def test_indentation_is_mandatory_after_a_colon(self):
+        text = """
+def concept add one to a as:
+def func(x):
+    return x+1
+func(a)
+    """
+
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        return_value = res.value
+
+        assert not res.status
+        assert isinstance(return_value, ParserResultConcept)
+        assert isinstance(return_value.value[0], SyntaxErrorNode)
+        assert return_value.value[0].message == "Indentation not found."
+
+    def test_indentation_is_not_allowed_if_the_colon_is_missing(self):
+        text = """
+def concept add one to a as
+    def func(x):
+        return x+1
+    func(a)
+        """
+        context = self.get_context()
+
+        parser = DefaultParser()
+        res = parser.parse(context, text)
+        return_value = res.value
+
+        assert not res.status
+        assert context.sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS)
+
+    def test_name_is_mandatory(self):
+        text = "def concept as 'hello'"
+
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        return_value = res.value
+
+        assert not res.status
+        assert isinstance(return_value, ParserResultConcept)
+        assert isinstance(return_value.value[0], SyntaxErrorNode)
+        assert return_value.value[0].message == "Name is mandatory"
+
+    def test_concept_keyword_is_mandatory_but_the_concept_is_recognized(self):
+        text = "def hello as a where b pre c post d"
+
+        expected_concept = get_def_concept(name="hello", body="a", where="b", pre="c", post="d")
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        return_value = res.value
+
+        assert not res.status
+        assert isinstance(return_value, ParserResultConcept)
+        assert isinstance(return_value.value[0], UnexpectedTokenErrorNode)
+        assert return_value.value[0].message == "Syntax error."
+        assert return_value.value[0].expected_tokens == [Keywords.CONCEPT]
+        assert return_value.try_parsed == expected_concept
+
+    @pytest.mark.parametrize("text", [
+        "def concept hello where 1+",
+        "def concept hello pre 1+",
+        "def concept hello post 1+",
+        "def concept hello as 1+"
+    ])
+    def test_i_can_detect_error_in_declaration(self, text):
+        context = self.get_context()
+        sheerka = context.sheerka
+
+        parser = DefaultParser()
+        res = parser.parse(context, text)
+        return_value = res.value
+
+        assert not res.status
+        assert sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS)
+
+    def test_new_line_is_not_allowed_in_the_name(self):
+        text = "def concept hello \n my friend as 'hello'"
+
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+        return_value = res.value
+
+        assert not res.status
+        assert return_value.value == [SyntaxErrorNode([], "Newline are not allowed in name.")]
+
+    def test_i_can_parse_def_concept_from_regex(self):
+        context = self.get_context()
+        a_concept = Concept("a_concept")
+        context.sheerka.add_in_cache(a_concept)
+
+        text = "def concept name from bnf a_concept | 'a_string' as __definition[0]"
+        parser = DefaultParser()
+        res = parser.parse(context, text)
+        node = res.value.value
+        definition = OrderedChoice(a_concept, StrMatch("a_string"))
+        parser_result = ParserResultConcept(BnfParser(), "a_concept | 'a_string'", definition, definition)
+        expected = get_def_concept(name="name", body="__definition[0]", definition=parser_result)
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+        assert node == expected
+
+    def test_i_can_parse_def_concept_where_bnf_references_itself(self):
+        context = self.get_context()
+        a_concept = Concept("a_concept")
+        context.sheerka.add_in_cache(a_concept)
+
+        text = "def concept name from bnf 'a' + name?"
+        parser = DefaultParser()
+        parser.parse(context, text)
+
+        assert not parser.has_error
+
+    def test_i_can_detect_empty_bnf_declaration(self):
+        text = "def concept name from bnf as __definition[0]"
+
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+
+        assert not res.status
+        assert res.value.value[0] == SyntaxErrorNode([], "Empty declaration")
+
+    def test_i_can_detect_not_for_me(self):
+        text = "hello world"
+        context = self.get_context()
+        parser = DefaultParser()
+        res = parser.parse(context, text)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.NOT_FOR_ME)
+        assert isinstance(res.value.body[0], CannotHandleErrorNode)
+
+    def test_i_can_parse_is_a(self):
+        parser = DefaultParser()
+        text = "the name of my 'concept' isa the name of the set"
+        res = parser.parse(self.get_context(), text)
+        expected = IsaConceptNode([],
+                                  concept=NameNode(list(Tokenizer("the name of my 'concept'"))),
+                                  set=NameNode(list(Tokenizer("the name of the set"))))
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+        assert res.value.value == expected
+
+    @pytest.mark.parametrize("text", [
+        "concept",
+        "isa number",
+        "name isa",
+        "def",
+        "def concept_name"
+    ])
+    def test_i_cannot_parse_invalid_entries(self, text):
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+
+        assert not res.status
+        assert isinstance(res.body, ParserResultConcept)
+        assert isinstance(res.body.body[0], UnexpectedTokenErrorNode)
+
+    @pytest.mark.parametrize("text, error_msg, error_text", [
+        ("'name", "Missing Trailing quote", "'name"),
+        ("foo isa 'name", "Missing Trailing quote", "'name"),
+        ("def concept 'name", "Missing Trailing quote", "'name"),
+        ("def concept name as 'body", "Missing Trailing quote", "'body"),
+        ("def concept name from bnf 'expression", "Missing Trailing quote", "'expression"),
+        ("def concept c::", "Concept name not found", ""),
+    ])
+    def test_i_cannot_parse_when_tokenizer_fails(self, text, error_msg, error_text):
+        parser = DefaultParser()
+        res = parser.parse(self.get_context(), text)
+
+        assert not res.status
+        assert isinstance(res.body, ParserResultConcept)
+        assert isinstance(res.body.body[0], LexerError)
+        assert res.body.body[0].message == error_msg
+        assert res.body.body[0].text == error_text
+
+    def test_i_cannot_parse_bnf_definition_referencing_unknown_concept(self):
+        context = self.get_context()
+        text = "def concept name from bnf unknown"
+
+        parser = DefaultParser()
+        res = parser.parse(context, text)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert res.value.body == ("key", "unknown")

tests/parsers/test_ExactConceptParser.py

@@ -0,0 +1,138 @@
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from core.tokenizer import Tokenizer
+from parsers.ExactConceptParser import ExactConceptParser
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def metadata_prop(concept, prop_name):
+    for name, value in concept.metadata.props:
+        if name == prop_name:
+            return value
+
+    return None
+
+
+def get_concept(name, variables):
+    c = Concept(name=name)
+    if variables:
+        for v in variables:
+            c.def_prop(v)
+    c.init_key()
+    return c
+
+
+class TestExactConceptParser(TestUsingMemoryBasedSheerka):
+
+    def test_i_can_compute_combinations(self):
+        parser = ExactConceptParser()
+        res = parser.combinations(["foo", "bar", "baz"])
+
+        assert res == {('foo', 'bar', 'baz'),
+                       ('__var__0', 'bar', 'baz'),
+                       ('foo', '__var__0', 'baz'),
+                       ('foo', 'bar', '__var__0'),
+                       ('__var__0', '__var__1', 'baz'),
+                       ('__var__0', 'bar', '__var__1'),
+                       ('foo', '__var__0', '__var__1'),
+                       ('__var__0', '__var__1', '__var__2')}
+
+    def test_i_can_compute_combinations_with_duplicates(self):
+        parser = ExactConceptParser()
+        res = parser.combinations(["foo", "bar", "foo"])
+
+        assert res == {('foo', 'bar', 'foo'),
+                       ('__var__0', 'bar', '__var__0'),
+                       ('foo', '__var__0', 'foo'),
+                       ('__var__0', '__var__1', '__var__0'),
+                       ('__var__1', '__var__0', '__var__1')}
+        # TODO: the last tuple is not possible, so the algo can be improved
+
+    def test_i_can_recognize_a_simple_concept(self):
+        context = self.get_context()
+        concept = get_concept("hello world", [])
+        context.sheerka.add_in_cache(concept)
+
+        source = "hello world"
+        results = ExactConceptParser().parse(context, source)
+
+        assert len(results) == 1
+        assert results[0].status
+        assert results[0].value.value == concept
+
+    def test_i_can_recognize_concepts_defined_several_times(self):
+        context = self.get_context()
+        context.sheerka.add_in_cache(get_concept("hello world", []))
+        context.sheerka.add_in_cache(get_concept("hello a", ["a"]))
+
+        source = "hello world"
+        results = ExactConceptParser().parse(context, source)
+
+        assert len(results) == 2
+        results = sorted(results, key=lambda x: x.value.value.name)  # because of the usage of sets
+
+        assert results[0].status
+        assert results[0].value.value.name == "hello a"
+        assert metadata_prop(results[0].value.value, "a") == "world"
+
+        assert results[1].status
+        assert results[1].value.value.name == "hello world"
+
+    def test_i_can_recognize_a_concept_with_variables(self):
+        context = self.get_context()
+        concept = get_concept("a + b", ["a", "b"])
+        context.sheerka.add_in_cache(concept)
+        source = "10 + 5"
+        results = ExactConceptParser().parse(context, source)
+
+        assert len(results) == 1
+        assert results[0].status
+        concept_found = results[0].value.value
+        assert concept_found.key == concept.key
+        assert metadata_prop(concept_found, "a") == "10"
+        assert metadata_prop(concept_found, "b") == "5"
+
+    def test_i_can_recognize_a_concept_with_duplicate_variables(self):
+        context = self.get_context()
+        concept = get_concept("a + b + a", ["a", "b"])
+        context.sheerka.cache_by_key[concept.key] = concept
+        source = "10 + 5 + 10"
+        results = ExactConceptParser().parse(context, source)
+
+        assert len(results) == 1
+        assert results[0].status
+        concept_found = results[0].value.value
+        assert concept_found.key == concept.key
+        assert metadata_prop(concept_found, "a") == "10"
+        assert metadata_prop(concept_found, "b") == "5"
+
+    def test_i_can_manage_unknown_concept(self):
+        context = self.get_context()
+        source = "def concept hello world"  # this is not a concept by itself
+        res = ExactConceptParser().parse(context, source)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert res.value.body == "def concept hello world"
+
+    def test_i_can_detect_concepts_too_long(self):
+        context = self.get_context()
+        source = "a very very long concept that cannot be an unique one"
+        res = ExactConceptParser().parse(context, source)
+
+        assert not res.status
+        assert context.sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_TOO_LONG)
+        assert res.value.body == "a very very long concept that cannot be an unique one"
+
+    def test_i_can_detect_concept_from_tokens(self):
+        context = self.get_context()
+        concept = get_concept("hello world", [])
+        context.sheerka.add_in_cache(concept)
+
+        source = "hello world"
+        results = ExactConceptParser().parse(context, list(Tokenizer(source)))
+
+        assert len(results) == 1
+        assert results[0].status
+        assert results[0].value.value == concept

tests/parsers/test_MultipleConceptsParser.py

@@ -0,0 +1,215 @@
+import pytest
+
+from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
+from core.concept import Concept
+from core.tokenizer import Tokenizer, TokenKind, Token
+from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, cnode, utnode, scnode, SourceCodeNode
+from parsers.MultipleConceptsParser import MultipleConceptsParser
+from parsers.PythonParser import PythonNode
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+def get_return_value(context, grammar, expression):
+    parser = ConceptLexerParser()
+    parser.initialize(context, grammar)
+
+    ret_val = parser.parse(context, expression)
+    assert not ret_val.status
+    return ret_val
+
+
+class TestMultipleConceptsParser(TestUsingMemoryBasedSheerka):
+
+    def init(self, concepts, grammar, expression):
+        context = self.get_context()
+        for c in concepts:
+            context.sheerka.create_new_concept(context, c)
+        return_value = get_return_value(context, grammar, expression)
+
+        return context, return_value
+
+    def test_not_interested_if_not_parser_result(self):
+        context = self.get_context()
+        text = "not parser result"
+
+        res = MultipleConceptsParser().parse(context, text)
+        assert res is None
+
+    def test_not_interested_if_not_from_concept_lexer_parser(self):
+        context = self.get_context()
+        text = ParserResultConcept(parser="not concept lexer", value="some value")
+
+        res = MultipleConceptsParser().parse(context, text)
+        assert res is None
+
+    def test_i_can_parse_exact_concepts(self):
+        foo = Concept("foo", body="'foo'")
+        bar = Concept("bar", body="'bar'")
+        baz = Concept("baz", body="'baz'")
+        grammar = {}
+        context, return_value = self.init([foo, bar, baz], grammar, "bar foo    baz")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+
+        assert ret_val.status
+        assert ret_val.who == parser.name
+        assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
+        assert ret_val.value.value == [
+            ConceptNode(bar, 0, 0, source="bar"),
+            ConceptNode(foo, 2, 2, source="foo"),
+            ConceptNode(baz, 4, 4, source="baz")]
+        assert ret_val.value.source == "bar foo    baz"
+
+    def test_i_can_parse_when_ending_with_bnf(self):
+        foo = Concept("foo", body="'foo'")
+        bar = Concept("bar", body="'bar'")
+        grammar = {foo: Sequence("foo1", "foo2", "foo3")}
+        context, return_value = self.init([foo, bar], grammar, "bar foo1 foo2 foo3")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+
+        assert ret_val.status
+        assert ret_val.who == parser.name
+        assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
+        assert ret_val.value.value == [cnode("bar", 0, 0, "bar"), cnode("foo", 2, 6, "foo1 foo2 foo3")]
+        assert ret_val.value.source == "bar foo1 foo2 foo3"
+
+    def test_i_can_parse_when_starting_with_bnf(self):
+        foo = Concept("foo", body="'foo'")
+        bar = Concept("bar", body="'bar'")
+        grammar = {foo: Sequence("foo1", "foo2", "foo3")}
+        context, return_value = self.init([foo, bar], grammar, "foo1 foo2 foo3 bar")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+
+        assert ret_val.status
+        assert ret_val.who == parser.name
+        assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
+        assert ret_val.value.value == [cnode("foo", 0, 4, "foo1 foo2 foo3"), cnode("bar", 6, 6, "bar")]
+        assert ret_val.value.source == "foo1 foo2 foo3 bar"
+
+    def test_i_can_parse_when_concept_are_mixed(self):
+        foo = Concept("foo")
+        bar = Concept("bar")
+        baz = Concept("baz")
+        grammar = {foo: Sequence("foo1", "foo2", "foo3")}
+        context, return_value = self.init([foo, bar, baz], grammar, "baz foo1 foo2 foo3 bar")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+
+        assert ret_val.status
+        assert ret_val.who == parser.name
+        assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
+        assert ret_val.value.value == [
+            cnode("baz", 0, 0, "baz"),
+            cnode("foo", 2, 6, "foo1 foo2 foo3"),
+            cnode("bar", 8, 8, "bar")]
+        assert ret_val.value.source == "baz foo1 foo2 foo3 bar"
+
+    def test_i_can_parse_when_multiple_concepts_are_matching(self):
+        foo = Concept("foo")
+        bar = Concept("bar", body="bar1")
+        baz = Concept("bar", body="bar2")
+        grammar = {foo: "foo"}
+        context, return_value = self.init([foo, bar, baz], grammar, "foo bar")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+
+        assert len(ret_val) == 2
+        assert ret_val[0].status
+        assert ret_val[0].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")]
+        assert ret_val[0].value.source == "foo bar"
+        assert ret_val[0].value.value[1].concept.metadata.body == "bar1"
+
+        assert ret_val[1].status
+        assert ret_val[1].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")]
+        assert ret_val[1].value.source == "foo bar"
+        assert ret_val[1].value.value[1].concept.metadata.body == "bar2"
+
+    def test_i_can_parse_when_source_code(self):
+        foo = Concept("foo")
+        grammar = {foo: "foo"}
+        context, return_value = self.init([foo], grammar, "1 foo")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+        wrapper = ret_val.value
+        value = ret_val.value.value
+
+        assert ret_val.status
+        assert ret_val.who == parser.name
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert wrapper.source == "1 foo"
+        assert value == [
+            scnode(0, 1, "1 "),
+            cnode("foo", 2, 2, "foo")]
+
+    def test_i_cannot_parse_when_unrecognized_token(self):
+        twenty_two = Concept("twenty two")
+        one = Concept("one")
+        grammar = {twenty_two: Sequence("twenty", "two")}
+        context, return_value = self.init([twenty_two, one], grammar, "twenty two + one")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+
+        assert not ret_val.status
+        assert ret_val.who == parser.name
+        assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
+        assert ret_val.value.value == [
+            cnode("twenty two", 0, 2, "twenty two"),
+            utnode(3, 5, " + "),
+            cnode("one", 6, 6, "one")
+        ]
+        assert ret_val.value.source == "twenty two + one"
+
+    def test_i_cannot_parse_when_unknown_concepts(self):
+        twenty_two = Concept("twenty two")
+        one = Concept("one")
+        grammar = {twenty_two: Sequence("twenty", "two")}
+        context, return_value = self.init([twenty_two, one], grammar, "twenty two plus one")
+
+        parser = MultipleConceptsParser()
+        ret_val = parser.parse(context, return_value.body)
+
+        assert not ret_val.status
+        assert ret_val.who == parser.name
+        assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
+        assert ret_val.value.value == [
+            cnode("twenty two", 0, 2, "twenty two"),
+            utnode(3, 5, " plus "),
+            cnode("one", 6, 6, "one")
+        ]
+        assert ret_val.value.source == "twenty two plus one"
+
+    @pytest.mark.parametrize("text, expected_source, expected_end", [
+        ("True", "True", 0),
+        ("1 == 1", "1 == 1", 5),
+        ("1!xdf", "1", 0),
+        ("1", "1", 0),
+    ])
+    def test_i_can_get_source_code_node(self, text, expected_source, expected_end):
+        tokens = list(Tokenizer(text))[:-1]  # strip trailing EOF
+
+        start_index = 5  # a random number different of zero
+        res = MultipleConceptsParser().get_source_code_node(self.get_context(), start_index, tokens)
+
+        assert isinstance(res, SourceCodeNode)
+        assert isinstance(res.node, PythonNode)
+        assert res.source == expected_source
+        assert res.start == start_index
+        assert res.end == start_index + expected_end
+
+    def test_i_cannot_parse_null_text(self):
+        res = MultipleConceptsParser().get_source_code_node(self.get_context(), 0, [])
+        assert res is None
+
+        eof = Token(TokenKind.EOF, "", 0, 0, 0)
+        res = MultipleConceptsParser().get_source_code_node(self.get_context(), 0, [eof])
+        assert res is None

tests/parsers/test_PythonParser.py

@@ -0,0 +1,77 @@
+import ast
+import pytest
+from core.builtin_concepts import ParserResultConcept
+from core.tokenizer import Tokenizer, LexerError
+from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+
+class TestPythonParser(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("text, expected", [
+        ("1+1", PythonNode("1+1", ast.parse("1+1", mode="eval"))),
+        ("a=10", PythonNode("a=10", ast.parse("a=10", mode="exec"))),
+    ])
+    def test_i_can_parse_a_simple_expression(self, text, expected):
+        parser = PythonParser()
+        res = parser.parse(self.get_context(), text)
+
+        assert res.status
+        assert res.who == parser.name
+        assert isinstance(res.value, ParserResultConcept)
+        assert res.value.value == expected
+
+    @pytest.mark.parametrize("text, expected", [
+        ("1+1", PythonNode("1+1", ast.parse("1+1", mode="eval"))),
+        ("a=10", PythonNode("a=10", ast.parse("a=10", mode="exec"))),
+    ])
+    def test_i_can_parse_from_tokens(self, text, expected):
+        parser = PythonParser()
+        tokens = list(Tokenizer(text))
+        res = parser.parse(self.get_context(), tokens)
+
+        assert res.status
+        assert res.who == parser.name
+        assert isinstance(res.value, ParserResultConcept)
+        assert res.value.value == expected
+
+    @pytest.mark.parametrize("text", [
+        "1+",
+        "'name",
+        "foo = 'name"
+    ])
+    def test_i_can_detect_error(self, text):
+        parser = PythonParser()
+        res = parser.parse(self.get_context(), text)
+
+        assert not res.status
+        assert res.who == parser.name
+        assert isinstance(res.value, ParserResultConcept)
+        assert isinstance(res.value.value[0], PythonErrorNode)
+        assert isinstance(res.value.value[0].exception, SyntaxError)
+
+    @pytest.mark.parametrize("text, error_msg, error_text", [
+        ("c::", "Concept name not found", ""),
+        ("c:: + 1", "Concept name not found", ""),
+    ])
+    def test_i_can_detect_lexer_errors(self, text, error_msg, error_text):
+        parser = PythonParser()
+        res = parser.parse(self.get_context(), text)
+
+        assert not res.status
+        assert isinstance(res.body, ParserResultConcept)
+        assert isinstance(res.body.body[0], LexerError)
+        assert res.body.body[0].message == error_msg
+        assert res.body.body[0].text == error_text
+
+    def test_i_can_parse_a_concept(self):
+        text = "c:concept_name: + 1"
+
+        parser = PythonParser()
+        res = parser.parse(self.get_context(), text)
+
+        assert res
+        assert res.value.value == PythonNode(
+            "c:concept_name: + 1",
+            ast.parse("__C__USE_CONCEPT__concept_name__C__+1", mode="eval"))

tests/parsers/test_PythonWithConceptsParser.py

@@ -0,0 +1,138 @@
+import ast
+
+import pytest
+
+from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
+from core.concept import Concept
+from core.tokenizer import Token, TokenKind, Tokenizer
+from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode
+from parsers.MultipleConceptsParser import MultipleConceptsParser
+from parsers.PythonParser import PythonNode, PythonErrorNode
+from parsers.PythonWithConceptsParser import PythonWithConceptsParser
+
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+
+multiple_concepts_parser = MultipleConceptsParser()
+
+
+def ret_val(*args):
+    result = []
+    index = 0
+    for item in args:
+        if isinstance(item, Concept):
+            tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)]
+            result.append(ConceptNode(item, index, index, tokens, item.name))
+            index += 1
+        else:
+            tokens = list(Tokenizer(item))
+            result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens))
+            index += len(tokens)
+
+    return ReturnValueConcept("who", False, ParserResultConcept(parser=multiple_concepts_parser, value=result))
+
+
+def to_str_ast(expression):
+    return PythonNode.get_dump(ast.parse(expression, mode="eval"))
+
+
+class TestPythonWithConceptsParser(TestUsingMemoryBasedSheerka):
+
+    @pytest.mark.parametrize("text, interested", [
+        ("not parser result", False),
+        (ParserResultConcept(parser="not multiple_concepts_parser"), False),
+        (ParserResultConcept(parser=multiple_concepts_parser, value=[]), True),
+    ])
+    def test_not_interested(self, text, interested):
+        context = self.get_context()
+
+        res = PythonWithConceptsParser().parse(context, text)
+        if interested:
+            assert res is not None
+        else:
+            assert res is None
+
+    def test_i_can_parse_concepts_and_python(self):
+        context = self.get_context()
+        foo = Concept("foo")
+        input_return_value = ret_val(foo, " + 1")
+
+        parser = PythonWithConceptsParser()
+        result = parser.parse(context, input_return_value.body)
+        wrapper = result.value
+        return_value = result.value.value
+
+        assert result.status
+        assert result.who == parser.name
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert wrapper.source == "foo + 1"
+        assert isinstance(return_value, PythonNode)
+        assert return_value.source == "foo + 1"
+        assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__C__ + 1")
+        assert return_value.concepts["__C__foo__C__"] == foo
+
+    def test_i_can_parse_concepts_and_python_when_concept_is_known(self):
+        context = self.get_context()
+        foo = Concept("foo")
+        foo = context.sheerka.create_new_concept(context, foo).body.body
+        input_return_value = ret_val(foo, " + 1")
+
+        parser = PythonWithConceptsParser()
+        result = parser.parse(context, input_return_value.body)
+        wrapper = result.value
+        return_value = result.value.value
+
+        assert result.status
+        assert result.who == parser.name
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert wrapper.source == "foo + 1"
+        assert isinstance(return_value, PythonNode)
+        assert return_value.source == "foo + 1"
+        assert return_value.get_dump(return_value.ast_) == to_str_ast("__C__foo__1001__C__ + 1")
+        assert return_value.concepts["__C__foo__1001__C__"] == foo
+
+    def test_i_can_parse_when_concept_name_has_invalid_characters(self):
+        context = self.get_context()
+        foo = Concept("foo et > (,")
+        foo = context.sheerka.create_new_concept(context, foo).body.body
+        input_return_value = ret_val(foo, " + 1")
+
+        parser = PythonWithConceptsParser()
+        result = parser.parse(context, input_return_value.body)
+        return_value = result.value.value
+
+        assert result.status
+        assert return_value.concepts["__C__foo0et000000__1001__C__"] == foo
+
+    def test_python_ids_mappings_are_correct_when_concepts_with_the_same_name(self):
+        context = self.get_context()
+        foo1 = Concept("foo")
+        foo2 = Concept("foo")
+        foo3 = context.sheerka.create_new_concept(context, Concept("foo", body="foo3")).body.body
+        foo4 = context.sheerka.create_new_concept(context, Concept("foo", body="foo4")).body.body
+
+        input_return_value = ret_val(foo1, "+", foo2, "+", foo3, "+", foo4)
+
+        parser = PythonWithConceptsParser()
+        result = parser.parse(context, input_return_value.body)
+        return_value = result.value.value
+
+        assert result.status
+        assert return_value.concepts["__C__foo__C__"] == foo1
+        assert return_value.concepts["__C__foo_1__C__"] == foo2
+        assert return_value.concepts["__C__foo__1001__C__"] == foo3
+        assert return_value.concepts["__C__foo__1002__C__"] == foo4
+
+    def test_i_cannot_parse_if_syntax_error(self):
+        context = self.get_context()
+        foo = Concept("foo")
+        foo = context.sheerka.create_new_concept(context, foo).body.body
+        input_return_value = ret_val(foo, " + ")
+
+        parser = PythonWithConceptsParser()
+        result = parser.parse(context, input_return_value.body)
+        wrapper = result.value
+        return_value = result.value.value
+
+        assert not result.status
+        assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+        assert isinstance(return_value[0], PythonErrorNode)

tests/sdp/test_sheerkaDataProvider.py

@@ -12,7 +12,7 @@ import json
 from sdp.sheerkaSerializer import JsonSerializer, Serializer, PickleSerializer
 import core.utils
 
-tests_root = path.abspath("../build/tests")
+tests_root = path.abspath("../../build/tests")
 evt_digest = "3a571cb6034ef6fc8d7fe91948d0d29728eed74de02bac7968b0e9facca2c2d7"
 
 
@@ -821,17 +821,17 @@ def test_i_can_set_using_reference(root):
     entry, key = sdp.set(evt_digest, "entry", ObjWithKey(2, "foo"), use_ref=True)
 
     state = sdp.load_state(sdp.get_snapshot())
-    assert state.data == {"entry": {"2": '##REF##:95b5cbab545dded0b90b57a3d15a157b9a559fb586ee2f8d6ccbc6d2491f1268'}}
+    assert state.data == {"entry": {"2": '##REF##:43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9'}}
     assert entry == "entry"
     assert key == "2"
 
     assert sdp.io.exists(sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder,
-                                             "95b5cbab545dded0b90b57a3d15a157b9a559fb586ee2f8d6ccbc6d2491f1268"))
+                                             "43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9"))
 
     # sanity check, make sure that I can load back
     loaded = sdp.get(entry, key)
     assert loaded == ObjWithKey(2, "foo")
-    assert getattr(loaded, Serializer.ORIGIN) == "95b5cbab545dded0b90b57a3d15a157b9a559fb586ee2f8d6ccbc6d2491f1268"
+    assert getattr(loaded, Serializer.ORIGIN) == "43f07065c7bad051cdd726bdfa4de7f8d754c31486c65ddb31d6b6548dec3db9"
 
 
 @pytest.mark.parametrize("root", [
@@ -1430,7 +1430,7 @@ def test_i_can_modify_a_ref(root):
     state = sdp.load_state(sdp.get_snapshot())
     assert state.data == {"entry": {
         "key1": ObjWithKey("key1", "foo"),
-        "key2": "##REF##:d70b0247311645ed18d275337cbcf79ad186d995236cdc8ad4fcfc708085bd3d"}}
+        "key2": "##REF##:041d3cca905b51bc2c66251e73e56b836aae7b9435ee3d7eb05d44bb67ff575e"}}
     assert entry == "entry"
     assert key == "key2"
 

tests/sdp/test_sheerkaSerializer.py

@@ -37,7 +37,7 @@ def test_i_can_serialize_an_event():
 def test_i_can_serialize_an_object():
     obj = Obj("10", "value")
     serializer = Serializer()
-    serializer.register(JsonSerializer("tests.test_sheerkaSerializer.Obj"))
+    serializer.register(JsonSerializer("tests.sdp.test_sheerkaSerializer.Obj"))
     context = SerializerContext("kodjo", "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b")
 
     stream = serializer.serialize(obj, context)
@@ -51,7 +51,7 @@ def test_i_can_serialize_an_object():
 
 
 @pytest.mark.parametrize("obj, expected", [
-    (Obj("10", "value"), "tests.test_sheerkaSerializer.Obj")
+    (Obj("10", "value"), "tests.sdp.test_sheerkaSerializer.Obj")
 ])
 def test_get_full_qualified_name(obj, expected):
     assert expected == core.utils.get_full_qualified_name(obj)

tests/test_AddConceptEvaluator.py

@@ -1,194 +0,0 @@
-import ast
-
-import pytest
-
-from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
-from core.concept import VARIABLE_PREFIX, ConceptParts, Concept
-from core.sheerka import Sheerka, ExecutionContext
-from core.tokenizer import Tokenizer
-from evaluators.AddConceptEvaluator import AddConceptEvaluator
-from parsers.BaseParser import BaseParser
-from parsers.ConceptLexerParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression
-from parsers.BnfParser import BnfParser
-from parsers.DefaultParser import DefConceptNode, NameNode
-from parsers.PythonParser import PythonNode, PythonParser
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None):
-    concept = DefConceptNode([], name=NameNode(list(Tokenizer(name))))
-
-    if body:
-        concept.body = get_concept_part(body)
-    if where:
-        concept.where = get_concept_part(where)
-    if pre:
-        concept.pre = get_concept_part(pre)
-    if post:
-        concept.post = get_concept_part(post)
-    if definition:
-        concept.definition = definition
-
-    return ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=concept))
-
-
-def get_concept_part(part):
-    if isinstance(part, str):
-        node = PythonNode(part, ast.parse(part, mode="eval"))
-        return ReturnValueConcept(
-            who="parsers.Default",
-            status=True,
-            value=ParserResultConcept(
-                source=part,
-                parser=PythonParser(),
-                value=node))
-
-    if isinstance(part, PythonNode):
-        return ReturnValueConcept(
-            who="parsers.Default",
-            status=True,
-            value=ParserResultConcept(
-                source=part.source,
-                parser=PythonParser(),
-                value=part))
-
-    if isinstance(part, ReturnValueConcept):
-        return part
-
-
-def get_concept_definition(source, parsing_expression):
-    return ReturnValueConcept(
-        who="Parsers:RegexParser",
-        status=True,
-        value=ParserResultConcept(
-            source=source,
-            parser=BnfParser(),
-            value=parsing_expression
-        )
-    )
-
-
-@pytest.mark.parametrize("ret_val, expected", [
-    (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept(value=DefConceptNode([]))), True),
-    (ReturnValueConcept(BaseParser.PREFIX + "some_name", False, ParserResultConcept(value=DefConceptNode([]))), False),
-    (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, "not a ParserResultConcept"), False),
-    (ReturnValueConcept(BaseParser.PREFIX + "some_name", True, ParserResultConcept()), False),
-])
-def test_i_can_match(ret_val, expected):
-    context = get_context()
-    assert AddConceptEvaluator().matches(context, ret_val) == expected
-
-
-def test_that_the_source_is_correctly_set():
-    context = get_context()
-    def_concept_return_value = get_def_concept(
-        name="hello a",
-        definition=get_concept_definition("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
-        where="isinstance(a, str )",
-        pre="a is not None",
-        body="print('hello' + a)")
-
-    evaluated = AddConceptEvaluator().eval(context, def_concept_return_value)
-
-    assert evaluated.status
-    assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
-
-    created_concept = evaluated.body.body
-    assert created_concept.metadata.name == "hello a"
-    assert created_concept.metadata.where == "isinstance(a, str )"
-    assert created_concept.metadata.pre == "a is not None"
-    assert created_concept.metadata.post is None
-    assert created_concept.metadata.body == "print('hello' + a)"
-    assert created_concept.metadata.definition == "hello a"
-
-
-# def test_that_the_ast_is_correctly_initialized():
-#     """
-#     When I parse the definition of a concept, I evaluate the metadata (like the body)
-#     I wanted to keep in cache these evaluation for further utilisation but I have
-#     a serialization issue.
-#     So I had to comment concept.add_codes(def_concept_node.get_asts()) around line 85
-#     So this test is now irrelevant
-#     :return:
-#     """
-#     context = get_context()
-#     def_concept_return_value = get_concept(
-#         name="hello a",
-#         definition=get_concept_definition("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
-#         where="isinstance(a, str )",
-#         pre="a is not None",
-#         body="print('hello' + a)")
-#
-#     evaluated = AddConceptEvaluator().eval(context, def_concept_return_value)
-#
-#     assert evaluated.status
-#     assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
-#
-#     created_concept = evaluated.body.body
-#
-#     assert ConceptParts.WHERE in created_concept.compiled
-#     assert ConceptParts.PRE in created_concept.compiled
-#     assert ConceptParts.BODY in created_concept.compiled
-#     assert ConceptParts.POST not in created_concept.compiled
-
-
-def test_that_the_new_concept_is_correctly_saved_in_db():
-    context = get_context()
-    def_concept_return_value = get_def_concept(
-        name="hello a",
-        definition=get_concept_definition("hello a", Sequence(StrMatch("hello"), StrMatch("a"))),
-        where="isinstance(a, str )",
-        pre="a is not None",
-        body="print('hello' + a)")
-
-    # sanity. Make sure that the concept does not already exist
-    from_db = context.sheerka.get("hello " + VARIABLE_PREFIX + "0")
-    assert context.sheerka.isinstance(from_db, BuiltinConcepts.UNKNOWN_CONCEPT)
-
-    AddConceptEvaluator().eval(context, def_concept_return_value)
-    context.sheerka.concepts_cache = {}  # reset cache
-    from_db = context.sheerka.get("hello " + VARIABLE_PREFIX + "0")
-
-    assert from_db.metadata.key == f"hello {VARIABLE_PREFIX}0"
-    assert from_db.metadata.name == "hello a"
-    assert from_db.metadata.where == "isinstance(a, str )"
-    assert from_db.metadata.pre == "a is not None"
-    assert from_db.metadata.post is None
-    assert from_db.metadata.body == "print('hello' + a)"
-    assert from_db.metadata.definition == "hello a"
-    assert len(from_db.metadata.props) == 1
-    assert from_db.metadata.props[0] == ("a", None)
-    assert "a" in from_db.props
-
-    assert from_db.compiled == {}  # ast is not saved in db
-
-
-def test_i_can_get_props_from_python_node():
-    ret_val = get_concept_part("isinstance(a, str)")
-    context = get_context()
-
-    assert AddConceptEvaluator.get_props(context.sheerka, ret_val, ["a"]) == ["a"]
-
-
-def test_i_can_get_props_from_another_concept():
-    concept = Concept("hello").def_prop("a").def_prop("b")
-    ret_val = ReturnValueConcept(who="some_parser",
-                                 status=True,
-                                 value=ParserResultConcept(value=concept))
-
-    assert AddConceptEvaluator.get_props(get_context(), ret_val, []) == ["a", "b"]
-
-
-def test_i_can_get_props_from_definition():
-    parsing_expression = Sequence(ConceptExpression('mult'), ZeroOrMore(Sequence(StrMatch("+"), ConceptExpression("add"))))
-    ret_val = get_concept_definition("mult (('+'|'-') add)?", parsing_expression)
-
-    assert AddConceptEvaluator.get_props(get_context(), ret_val, []) == ["add", "mult"]
-
-

tests/test_AddConceptInSetEvaluator.py

@@ -1,112 +0,0 @@
-import pytest
-
-from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
-from core.concept import Concept
-from core.sheerka import Sheerka, ExecutionContext
-from core.tokenizer import Tokenizer
-from evaluators.AddConceptInSetEvaluator import AddConceptInSetEvaluator
-from parsers.DefaultParser import IsaConceptNode, NameNode
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def get_ret_val(concept_name, concept_set_name):
-    n1 = NameNode(list(Tokenizer(concept_name)))
-    n2 = NameNode(list(Tokenizer(concept_set_name)))
-
-    return ReturnValueConcept("some_name", True, ParserResultConcept(value=IsaConceptNode([], n1, n2)))
-
-
-@pytest.mark.parametrize("ret_val, expected", [
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value=IsaConceptNode([]))), True),
-    (ReturnValueConcept("some_name", False, ParserResultConcept(value=IsaConceptNode([]))), False),
-    (ReturnValueConcept("some_name", True, "not a ParserResultConcept"), False),
-    (ReturnValueConcept("some_name", True, ParserResultConcept()), False),
-])
-def test_i_can_match(ret_val, expected):
-    context = get_context()
-    assert AddConceptInSetEvaluator().matches(context, ret_val) == expected
-
-
-def test_i_cannot_add_if_the_concept_does_not_exists():
-    context = get_context()
-
-    ret_val = get_ret_val("foo", "bar")
-    res = AddConceptInSetEvaluator().eval(context, ret_val)
-
-    assert not res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert res.value.body == "foo"
-
-
-def test_i_cannot_add_if_the_set_does_not_exists():
-    context = get_context()
-    foo = Concept("foo")
-    context.sheerka.set_id_if_needed(foo, False)
-    context.sheerka.add_in_cache(foo)
-
-    ret_val = get_ret_val("foo", "bar")
-    res = AddConceptInSetEvaluator().eval(context, ret_val)
-
-    assert not res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert res.value.body == "bar"
-
-
-def test_i_can_add_concept_to_a_set_of_concept():
-    context = get_context()
-    foo = Concept("foo")
-    context.sheerka.set_id_if_needed(foo, False)
-    context.sheerka.add_in_cache(foo)
-
-    bar = Concept("bar")
-    context.sheerka.set_id_if_needed(bar, False)
-    context.sheerka.add_in_cache(bar)
-
-    ret_val = get_ret_val("foo", "bar")
-    res = AddConceptInSetEvaluator().eval(context, ret_val)
-
-    assert res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS)
-
-
-def test_i_can_add_concept_with_a_body_to_a_set_of_concept():
-    context = get_context()
-    foo = Concept("foo", body="1")
-    context.sheerka.set_id_if_needed(foo, False)
-    context.sheerka.add_in_cache(foo)
-
-    bar = Concept("bar")
-    context.sheerka.set_id_if_needed(bar, False)
-    context.sheerka.add_in_cache(bar)
-
-    ret_val = get_ret_val("foo", "bar")
-    res = AddConceptInSetEvaluator().eval(context, ret_val)
-
-    assert res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.SUCCESS)
-
-
-def test_i_cannot_add_the_same_concept_twice():
-    context = get_context()
-    foo = Concept("foo")
-    context.sheerka.set_id_if_needed(foo, False)
-    context.sheerka.add_in_cache(foo)
-
-    bar = Concept("bar")
-    context.sheerka.set_id_if_needed(bar, False)
-    context.sheerka.add_in_cache(bar)
-
-    ret_val = get_ret_val("foo", "bar")
-    AddConceptInSetEvaluator().eval(context, ret_val)
-    res = AddConceptInSetEvaluator().eval(context, ret_val)
-
-    assert not res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.CONCEPT_ALREADY_IN_SET)
-    assert res.value.concept == foo
-    assert res.value.concept_set == bar

tests/test_BnfParser.py

@@ -1,185 +0,0 @@
-import pytest
-
-from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept
-from core.sheerka import Sheerka, ExecutionContext
-from core.tokenizer import Tokenizer, TokenKind, LexerError
-from parsers.BaseParser import UnexpectedTokenErrorNode
-from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError
-from parsers.ConceptLexerParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \
-    ConceptLexerParser, ConceptNode, ConceptExpression, cnode
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-
-    return ExecutionContext("sheerka", Event(), sheerka)
-
-
-class ClassWithName():
-    def __init__(self, name):
-        self.name = name
-
-
-@pytest.mark.parametrize("expression, expected", [
-    ("'str'", StrMatch("str")),
-    ("1", StrMatch("1")),
-    ("  1", StrMatch("1")),
-    (",", StrMatch(",")),
-    ("'foo'?", Optional(StrMatch("foo"))),
-    ("'foo'*", ZeroOrMore(StrMatch("foo"))),
-    ("'foo'+", OneOrMore(StrMatch("foo"))),
-    ("1 | 2 | 3", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"))),
-    ("1|2|3", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"))),
-    ("1'|' 2 '|' 3", Sequence(StrMatch("1"), StrMatch("|"), StrMatch("2"), StrMatch("|"), StrMatch("3"))),
-    ("1 2 'foo'", Sequence(StrMatch("1"), StrMatch("2"), StrMatch("foo"))),
-    ("1 2 | 3 4+", OrderedChoice(
-        Sequence(StrMatch("1"), StrMatch("2")),
-        Sequence(StrMatch("3"), OneOrMore(StrMatch("4"))))),
-    ("1 (2 | 3) 4+", Sequence(StrMatch("1"), OrderedChoice(StrMatch("2"), StrMatch("3")), OneOrMore(StrMatch("4")))),
-    ("(1|2)+", OneOrMore(OrderedChoice(StrMatch("1"), StrMatch("2")))),
-    ("(1 2)+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")))),
-    ("1 *", Sequence(StrMatch("1"), StrMatch("*"))),
-    ("1 ?", Sequence(StrMatch("1"), StrMatch("?"))),
-    ("1 +", Sequence(StrMatch("1"), StrMatch("+"))),
-    ("(1|*) +", Sequence(OrderedChoice(StrMatch("1"), StrMatch("*")), StrMatch("+"))),
-    ("1, :&", Sequence(StrMatch("1"), StrMatch(","), StrMatch(":"), StrMatch("&"))),
-    ("(1 )", StrMatch("1")),
-    ("'str'=var", StrMatch("str", rule_name="var")),
-    ("'foo'?=var", Optional(StrMatch("foo"), rule_name="var")),
-    ("('foo'?)=var", Optional(StrMatch("foo"), rule_name="var")),
-    ("'foo'*=var", ZeroOrMore(StrMatch("foo"), rule_name="var")),
-    ("('foo'*)=var", ZeroOrMore(StrMatch("foo"), rule_name="var")),
-    ("'foo'+=var", OneOrMore(StrMatch("foo"), rule_name="var")),
-    ("('foo'+)=var", OneOrMore(StrMatch("foo"), rule_name="var")),
-    ("'foo'=var?", Optional(StrMatch("foo", rule_name="var"))),
-    ("('foo'=var)?", Optional(StrMatch("foo", rule_name="var"))),
-    ("'foo'=var*", ZeroOrMore(StrMatch("foo", rule_name="var"))),
-    ("('foo'=var)*", ZeroOrMore(StrMatch("foo", rule_name="var"))),
-    ("'foo'=var+", OneOrMore(StrMatch("foo", rule_name="var"))),
-    ("('foo'=var)+", OneOrMore(StrMatch("foo", rule_name="var"))),
-    ("(1 | 2 | 3)=var", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"), rule_name="var")),
-    ("(1 2)=var", Sequence(StrMatch("1"), StrMatch("2"), rule_name="var")),
-    ("(1 2)+=var", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")), rule_name="var")),
-    ("(1 2)=var+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2"), rule_name="var"))),
-])
-def test_i_can_parse_regex(expression, expected):
-    parser = BnfParser()
-    res = parser.parse(get_context(), Tokenizer(expression))
-
-    assert not parser.has_error
-    assert res.status
-    assert res.value.value == expected
-    assert res.value.source == expression
-
-
-@pytest.mark.parametrize("expression, expected", [
-    ("foo", Concept("foo").init_key()),
-    ("foo*", ZeroOrMore(Concept("foo").init_key())),
-    ("foo 'and' bar+", Sequence(Concept("foo").init_key(), StrMatch("and"), OneOrMore(Concept("bar").init_key()))),
-    ("foo | bar?", OrderedChoice(Concept("foo").init_key(), Optional(Concept("bar").init_key()))),
-    ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
-    ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
-])
-def test_i_can_parse_regex_with_concept(expression, expected):
-    foo = Concept("foo")
-    bar = Concept("bar")
-    var = Concept("var")
-    context = get_context()
-
-    for c in (foo, bar, var):
-        context.sheerka.add_in_cache(c)
-    parser = BnfParser()
-    res = parser.parse(context, Tokenizer(expression))
-
-    assert not parser.has_error
-    assert res.status
-    assert res.value.value == expected
-    assert res.value.source == expression
-
-
-def test_i_can_parse_regex_with_concept_when_the_concept_is_still_under_definition():
-    expression = "foo"
-    expected = ConceptExpression("foo")
-
-    context = get_context()
-    context.obj = ClassWithName("foo")
-
-    parser = BnfParser()
-    res = parser.parse(context, Tokenizer(expression))
-
-    assert not parser.has_error
-    assert res.status
-    assert res.value.value == expected
-    assert res.value.source == expression
-
-
-@pytest.mark.parametrize("expression, error", [
-    ("1 ", UnexpectedEndOfFileError()),
-    ("1|", UnexpectedEndOfFileError()),
-    ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.RPAR])),
-    ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.IDENTIFIER])),
-    ("'name", LexerError("Missing Trailing quote", "'name", 5, 1, 6))
-])
-def test_i_can_detect_errors(expression, error):
-    parser = BnfParser()
-    res = parser.parse(get_context(), Tokenizer(expression))
-    ret_value = res.value.value
-    assert parser.has_error
-    assert not res.status
-    assert ret_value[0] == error
-
-
-def test_i_can_use_the_result_of_regex_parsing_to_parse_a_text():
-    foo = Concept(name="foo")
-    bar = Concept(name="bar")
-    context = get_context()
-    context.sheerka.add_in_cache(foo)
-    context.sheerka.add_in_cache(bar)
-
-    regex_parser = BnfParser()
-    foo_definition = regex_parser.parse(context, "'twenty' | 'thirty'").value.value
-    bar_definition = regex_parser.parse(context, "foo ('one' | 'two')").value.value
-
-    concepts = {bar: bar_definition, foo: foo_definition}
-    concept_parser = ConceptLexerParser()
-    concept_parser.initialize(context, concepts)
-
-    res = concept_parser.parse(context, "twenty two")
-    assert res.status
-    assert res.value.body == [cnode("bar", 0, 2, "twenty two")]
-
-    res = concept_parser.parse(context, "thirty one")
-    assert res.status
-    assert res.value.body == [cnode("bar", 0, 2, "thirty one")]
-
-    res = concept_parser.parse(context, "twenty")
-    assert res.status
-    assert res.value.body == [cnode("foo", 0, 0, "twenty")]
-
-
-def test_i_cannot_parse_when_too_many_concepts():
-    foo1 = Concept(name="foo", body="1")
-    foo2 = Concept(name="foo", body="2")
-    context = get_context()
-    context.sheerka.cache_by_key["foo"] = [foo1, foo2]
-
-    regex_parser = BnfParser()
-    res = regex_parser.parse(context, "foo")
-
-    assert not res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.CANNOT_RESOLVE_CONCEPT)
-    assert res.value.body == ('key', 'foo')
-
-
-def test_i_cannot_parse_when_unknown_concept():
-    context = get_context()
-
-    regex_parser = BnfParser()
-    res = regex_parser.parse(get_context(), "foo")
-
-    assert not res.status
-    assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
-    assert res.value.body == ('key', 'foo')

tests/test_ConceptEvaluator.py

@@ -1,139 +0,0 @@
-import pytest
-
-from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
-from core.concept import Concept, ConceptParts
-from core.sheerka import Sheerka, ExecutionContext
-from evaluators.ConceptEvaluator import ConceptEvaluator
-from parsers.ExactConceptParser import ExactConceptParser
-from sdp.sheerkaDataProvider import Event
-
-
-def get_context():
-    sheerka = Sheerka(skip_builtins_in_db=True)
-    sheerka.initialize("mem://")
-    return ExecutionContext("test", Event(), sheerka)
-
-
-def get_return_value(concept, source=None):
-    return ReturnValueConcept(
-        "some_name",
-        True,
-        ParserResultConcept(parser=ExactConceptParser(),
-                            source=source or concept.name,
-                            value=concept,
-                            try_parsed=concept))
-
-
-@pytest.mark.parametrize("ret_val, expected", [
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value=Concept())), True),
-    (ReturnValueConcept("some_name", False, ParserResultConcept(value=Concept())), False),
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value="Not a concept")), False),
-    (ReturnValueConcept("some_name", True, Concept()), False),
-])
-def test_i_can_match(ret_val, expected):
-    context = get_context()
-    assert ConceptEvaluator().matches(context, ret_val) == expected
-
-
-def test_i_can_evaluate_concept():
-    context = get_context()
-    concept = Concept(name="foo",
-                      where="1",
-                      pre="2",
-                      post="3").def_prop("a", "4").def_prop("b", "5")
-
-    evaluator = ConceptEvaluator()
-    item = get_return_value(concept)
-    result = evaluator.eval(context, item)
-
-    assert result.who == evaluator.name
-    assert result.status
-    assert result.value.name == "foo"
-    assert result.value.get_metadata_value(ConceptParts.WHERE) == 1
-    assert result.value.get_metadata_value(ConceptParts.PRE) == 2
-    assert result.value.get_metadata_value(ConceptParts.POST) == 3
-    assert result.value.get_prop("a") == 4
-    assert result.value.get_prop("b") == 5
-    assert result.value.key == "foo"
-    assert result.parents == [item]
-
-
-def test_body_is_returned_when_defined_and_requested():
-    context = get_context()
-    concept = Concept(name="foo",
-                      body="'I have a value'",
-                      where="1",
-                      pre="2",
-                      post="3").set_prop("a", "4").set_prop("b", "5")
-
-    evaluator = ConceptEvaluator(return_body=True)
-    item = get_return_value(concept)
-    result = evaluator.eval(context, item)
-
-    assert result.who == evaluator.name
-    assert result.status
-    assert result.value == "I have a value"
-    assert result.parents == [item]
-
-
-def test_body_is_not_returned_if_not_requested():
-    context = get_context()
-    concept = Concept(name="foo",
-                      body="'I have a value'",
-                      where="1",
-                      pre="2",
-                      post="3").set_prop("a", "4").set_prop("b", "5")
-
-    evaluator = ConceptEvaluator(return_body=False)  # which is the default behaviour
-    item = get_return_value(concept)
-    result = evaluator.eval(context, item)
-
-    assert result.who == evaluator.name
-    assert result.status
-    assert result.value == concept
-    assert result.parents == [item]
-
-
-def test_i_can_eval_if_with_the_same_name_is_defined_in_the_context():
-    # If we evaluate Concept("foo", body="a").set_prop("a", "'property_a'")
-    # ConceptEvaluator will be called to resolve 'a' while we know that 'a' refers to the string 'property_a'
-
-    context = get_context()
-    context.obj = Concept("other").set_prop("foo", "'some_other_value'")
-    concept = Concept(name="foo")
-
-    item = get_return_value(concept)
-    result = ConceptEvaluator().eval(context, item)
-
-    assert result.status
-    assert result.value == "'some_other_value'"
-
-
-def test_i_cannot_recognize_a_concept_if_one_of_the_prop_is_unknown():
-    context = get_context()
-    context.sheerka.add_in_cache(Concept(name="one").init_key())
-    concept_plus = context.sheerka.add_in_cache(Concept(name="a plus b")
-                                                .def_prop("a", "one")
-                                                .def_prop("b", "two").init_key())
-
-    evaluator = ConceptEvaluator()
-    item = get_return_value(concept_plus)
-    result = evaluator.eval(context, item)
-
-    assert not result.status
-    assert context.sheerka.isinstance(result.value, BuiltinConcepts.CONCEPT_EVAL_ERROR)
-    assert result.value.property_name == "b"
-    assert context.sheerka.isinstance(result.value.error, BuiltinConcepts.TOO_MANY_ERRORS)
-    assert result.value.concept == concept_plus
-
-
-def test_that_error_concepts_are_transformed_into_errors_only_if_the_key_is_different():
-    context = get_context()
-
-    error_concept = context.sheerka.new(BuiltinConcepts.ERROR)
-    item = get_return_value(error_concept)
-    result = ConceptEvaluator().eval(context, item)
-
-    assert not context.sheerka.is_success(error_concept)  # it's indeed an error
-    assert result.status
-    assert result.value == error_concept