Added SyaNodeParser (finally, after one month)

src/core/builtin_concepts.py

@@ -37,6 +37,8 @@ class BuiltinConcepts(Enum):
     PARSER_RESULT = "parser result"
     TOO_MANY_SUCCESS = "too many success"  # when expecting a limited number of successful return value
     TOO_MANY_ERRORS = "too many errors"  # when expecting a limited number of successful return value
+    ONLY_SUCCESSFUL = "only successful"  # filter the result, only keep successful ones
+    MULTIPLE_ERRORS = "multiple errors"  # filter the result, only keep evaluator in error
     NOT_FOR_ME = "not for me"  # a parser recognize that the entry is not meant for it
     IS_EMPTY = "is empty"  # when a set is empty
     INVALID_RETURN_VALUE = "invalid return value"  # the return value of an evaluator is not correct
@@ -45,6 +47,7 @@ class BuiltinConcepts(Enum):
     CONCEPT_EVAL_ERROR = "concept evaluation error"  # cannot evaluate a property or metadata of a concept
     ENUMERATION = "enum"  # represents a list or a set
     LIST = "list"  # represents a list
+    FILTERED = "filtered"  # represents the result of a filtering
     CONCEPT_ALREADY_IN_SET = "concept already in set"
     EVALUATOR_PRE_PROCESS = "evaluator pre process"  # used modify / tweak behaviour of evaluators
     EVAL_BODY_REQUESTED = "eval body requested"  # to evaluate the body
@@ -91,6 +94,7 @@ BuiltinErrors = [str(e) for e in {
     BuiltinConcepts.UNKNOWN_PROPERTY,
     BuiltinConcepts.TOO_MANY_SUCCESS,
     BuiltinConcepts.TOO_MANY_ERRORS,
+    BuiltinConcepts.MULTIPLE_ERRORS,
     BuiltinConcepts.INVALID_RETURN_VALUE,
     BuiltinConcepts.CONCEPT_ALREADY_DEFINED,
     BuiltinConcepts.CONCEPT_EVAL_ERROR,
@@ -249,11 +253,12 @@ class ParserResultConcept(Concept):
     Result of a parsing
     """
 
-    def __init__(self, parser=None, source=None, value=None, try_parsed=None, validate_concept=None):
+    def __init__(self, parser=None, source=None, tokens=None, value=None, try_parsed=None):
         super().__init__(BuiltinConcepts.PARSER_RESULT, True, False, BuiltinConcepts.PARSER_RESULT)
         self.set_metadata_value(ConceptParts.BODY, value)
         self.set_prop("parser", parser)
         self.set_prop("source", source)
+        self.set_prop("tokens", tokens)
         self.set_prop("try_parsed", try_parsed)  # in case of error, what was found before the error
         self.metadata.is_evaluated = True
 
@@ -372,6 +377,14 @@ class ListConcept(Concept):
     #     return item in self.body
 
 
+class FilteredConcept(Concept):
+    def __init__(self, filtered=None, iterable=None, predicate=None):
+        super().__init__(BuiltinConcepts.FILTERED, True, False, BuiltinConcepts.FILTERED)
+        self.set_metadata_value(ConceptParts.BODY, filtered)
+        self.def_prop("iterable", iterable)
+        self.def_prop("predicate", predicate)
+
+
 class ConceptAlreadyInSet(Concept):
     def __init__(self, concept=None, concept_set=None):
         super().__init__(BuiltinConcepts.CONCEPT_ALREADY_IN_SET,
@@ -409,3 +422,17 @@ class WhereClauseFailed(Concept):
     @property
     def concept(self):
         return self.body
+
+
+class NotForMeConcept(Concept):
+    def __init__(self, source=None, reason=None):
+        super().__init__(BuiltinConcepts.NOT_FOR_ME,
+                         True,
+                         False,
+                         BuiltinConcepts.NOT_FOR_ME)
+        self.set_metadata_value(ConceptParts.BODY, source)
+        self.def_prop("reason", reason)
+        self.metadata.is_evaluated = True
+
+    def __repr__(self):
+        return f"NotForMeConcept(source={self.body}, reason={self.get_prop('reason')})"

src/core/builtin_helpers.py

@@ -6,6 +6,8 @@ from core.ast.nodes import CallNodeConcept, GenericNodeConcept
 from core.ast.visitors import UnreferencedNamesVisitor
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
+from parsers.BaseNodeParser import SourceCodeNode, ConceptNode, UnrecognizedTokensNode
+from parsers.BaseParser import BaseParser, ErrorNode
 
 
 def is_same_success(context, return_values):
@@ -132,6 +134,181 @@ def expect_one(context, return_values):
                 parents=return_values)
 
 
+def only_successful(context, return_values):
+    """
+    Removes all return values that are not successful
+    Return error when no successful return value
+    :param context:
+    :param return_values:
+    :return:
+    """
+    if not isinstance(return_values, list):
+        return return_values
+
+    sheerka = context.sheerka
+
+    if len(return_values) == 0:
+        return sheerka.ret(
+            context.who,
+            False,
+            sheerka.new(BuiltinConcepts.IS_EMPTY, body=return_values),
+            parents=return_values)
+
+    successful_results = [item for item in return_values if item.status]
+    if len(successful_results) == 0:
+        return sheerka.ret(
+            context.who,
+            False,
+            sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, body=return_values),
+            parents=return_values)
+
+    return sheerka.ret(
+        context.who,
+        True,
+        sheerka.new(BuiltinConcepts.ONLY_SUCCESSFUL, body=successful_results),
+        parents=return_values)
+
+
+def only_parsers_results(context, return_values):
+    """
+    Filters the return_values and returns when the result is a ParserResult
+    regardless of the status
+
+    So it filters errors
+    :param context:
+    :param return_values:
+    :return:
+    """
+
+    if not isinstance(return_values, list):
+        return return_values
+
+    sheerka = context.sheerka
+
+    if len(return_values) == 0:
+        return sheerka.ret(
+            context.who,
+            False,
+            sheerka.new(BuiltinConcepts.IS_EMPTY, body=return_values),
+            parents=return_values)
+
+    return_values_ok = [item for item in return_values if sheerka.isinstance(item.body, BuiltinConcepts.PARSER_RESULT)]
+
+    # hack because some parsers don't follow the NOT_FOR_ME rule
+    temp_ret_val = []
+    for ret_val in return_values_ok:
+        if isinstance(ret_val.body.body, ErrorNode):
+            continue
+        if isinstance(ret_val.body.body, list) and \
+            len(ret_val.body.body) == 1 and \
+            isinstance(ret_val.body.body[0], UnrecognizedTokensNode):
+            continue
+        temp_ret_val.append(ret_val)
+    return_values_ok = temp_ret_val
+
+    if len(return_values_ok) == 0:
+        return sheerka.ret(
+            context.who,
+            False,
+            sheerka.new(BuiltinConcepts.TOO_MANY_ERRORS, body=return_values),
+            parents=return_values)
+
+    return sheerka.ret(
+        context.who,
+        True,
+        sheerka.new(BuiltinConcepts.FILTERED,
+                    body=return_values_ok,
+                    iterable=return_values,
+                    predicate="sheerka.isinstance(item.body, BuiltinConcepts.PARSER_RESULT)"),
+        parents=return_values)
+
+
+def parse_unrecognized(context, tokens, parsers):
+    """
+    Try to recognize concepts or code from tokens using the given parsers
+    :param context:
+    :param tokens:
+    :param parsers:
+    :return:
+    """
+    steps = [BuiltinConcepts.BEFORE_PARSING, BuiltinConcepts.PARSING, BuiltinConcepts.AFTER_PARSING]
+    sheerka = context.sheerka
+
+    with context.push(desc=f"Parsing unrecognized '{tokens}'") as sub_context:
+        # disable all parsers but the following ones
+        sub_context.add_preprocess(BaseParser.PREFIX + "*", enabled=False)
+        for parser in parsers:
+            sub_context.add_preprocess(BaseParser.PREFIX + parser, enabled=True)
+
+        sub_context.add_inputs(source=tokens)
+        to_parse = sheerka.ret(
+            context.who,
+            True,
+            sheerka.new(BuiltinConcepts.USER_INPUT, body=tokens))
+        res = sheerka.execute(sub_context, to_parse, steps)
+        sub_context.add_values(return_values=res)
+
+        # discard Python response if accepted by AtomNode
+        is_concept = False
+        for r in res:
+            if r.status and r.who == "parsers.AtomNode":
+                is_concept = True
+
+        if not is_concept:
+            return res
+
+        filtered = []
+        for r in res:
+            if r.who == "parsers.Python":
+                continue
+            filtered.append(r)
+
+        return filtered
+
+
+def get_lexer_nodes(return_values, start, tokens):
+    """
+    From a parser result, return the corresponding LexerNode
+        either ConceptNode, UnrecognizedTokensNode or SourceCodeNode
+    :param return_values:
+    :param start:
+    :param tokens:
+    :return: list of list (list of concept node sequence)
+    """
+
+    lexer_nodes = []
+    for ret_val in return_values:
+        if ret_val.who == "parsers.Python":
+
+            if ret_val.body.source.strip().isalnum() and not ret_val.body.source.strip().isnumeric():
+                # Discard SourceCodeNode which seems to be a concept
+                # It may be a wrong idea, so let's see
+                continue
+
+            end = start + len(tokens) - 1
+            lexer_nodes.append([SourceCodeNode(ret_val.body.body, start, end, tokens, ret_val.body.source, ret_val)])
+
+        elif ret_val.who == "parsers.ExactConcept":
+            concepts = ret_val.body.body if hasattr(ret_val.body.body, "__iter__") else [ret_val.body.body]
+            end = start + len(tokens) - 1
+            for concept in concepts:
+                lexer_nodes.append([ConceptNode(concept, start, end, tokens, ret_val.body.source)])
+
+        elif ret_val.who in ("parsers.BnfNode", "parsers.SyaNode", "parsers.AtomNode"):
+            nodes = [node for node in ret_val.body.body]
+            for node in nodes:
+                node.start += start
+                node.end += start
+
+            # but append the whole sequence if when it's a sequence
+            lexer_nodes.append(nodes)
+
+        else:
+            raise NotImplementedError()
+
+    return lexer_nodes
+
+
 def get_names(sheerka, concept_node):
     """
     Finds all the names referenced by the concept_node

src/core/concept.py

@@ -108,11 +108,14 @@ class Concept:
 
     def __eq__(self, other):
 
+        if id(self) == id(other):
+            return True
+
         if isinstance(other, simplec):
             return self.name == other.name and self.body == other.body
 
-        if id(self) == id(other):
-            return True
+        if isinstance(other, CC):
+            return other == self
 
         if not isinstance(other, Concept):
             return False
@@ -346,6 +349,17 @@ class Concept:
         """
         return self.props[prop_name].value
 
+    def set_prop_by_index(self, index: int, value):
+        """
+        Set the value of a property (not the metadata) using the index
+        :param index: Name the property or another concept
+        :param value:
+        :return:
+        """
+        prop_name = list(self.props.keys())[index]
+        self.props[prop_name].value = value
+        return self
+
     def set_metadata_value(self, metadata: ConceptParts, value):
         """
         Set the resolved value of a metadata (not the metadata itself)
@@ -438,3 +452,68 @@ class InfiniteRecursionResolved:
 
     def get_value(self):
         return self.value
+
+
+class CC:
+    """
+    Concept class for test purpose
+    CC means concept for compiled (or concept with compiled)
+    It matches a concept if the compiles are equals
+    """
+
+    # The only properties that are testes are concept_key and compiled
+    # The other properties (concept, source, start and end)
+    # are used in tests/parsers/parsers_utils.py to help creating helper objects
+
+    def __init__(self, concept, source=None, **kwargs):
+        self.concept_key = concept.key if isinstance(concept, Concept) else concept
+        self.compiled = kwargs
+        self.concept = concept if isinstance(concept, Concept) else None
+        self.source = source  # to use when the key is different from the sub str to search when filling start and stop
+        self.start = None  # for debug purpose, indicate where the concept starts
+        self.end = None  # for debug purpose, indicate where the concept ends
+
+    def __eq__(self, other):
+        if id(self) == id(other):
+            return True
+
+        if isinstance(other, Concept):
+            if other.key != self.concept_key:
+                return False
+            return self.compiled == other.compiled
+
+        if not isinstance(other, CC):
+            return False
+
+        return self.concept_key == other.concept_key and \
+               self.compiled == other.compiled
+
+    def __hash__(self):
+        if self.concept:
+            return hash(self.concept)
+        return hash(self.concept_key)
+
+    def __repr__(self):
+        if self.concept:
+            txt = f"CC(concept='{self.concept}'"
+        else:
+            txt = f"CC(concept_key='{self.concept_key}'"
+
+        for k, v in self.compiled.items():
+            txt += f", {k}='{v}'"
+        return txt + ")"
+
+    def fix_pos(self, node):
+        start = node.start if hasattr(node, "start") else \
+            node[0] if isinstance(node, tuple) else None
+        end = node.end if hasattr(node, "end") else \
+            node[1] if isinstance(node, tuple) else None
+
+        if start is not None:
+            if self.start is None or start < self.start:
+                self.start = start
+
+        if end is not None:
+            if self.end is None or end > self.end:
+                self.end = end
+        return self

src/core/sheerka/ExecutionContext.py

@@ -43,6 +43,7 @@ class ExecutionContext:
                  desc: str = None,
                  logger=None,
                  global_hints=None,
+                 global_errors=None,
                  **kwargs):
 
         self._parent = None
@@ -61,6 +62,7 @@ class ExecutionContext:
         self.logger = logger
         self.local_hints = set()
         self.global_hints = set() if global_hints is None else global_hints
+        self.global_errors = [] if global_errors is None else global_errors
 
         self.inputs = {}  # what was the parameters of the execution context
         self.values = {}  # what was produced by the execution context
@@ -146,8 +148,8 @@ class ExecutionContext:
             preprocess.set_prop(k, v)
 
         if not self.preprocess:
-            self.preprocess = set()
-        self.preprocess.add(preprocess)
+            self.preprocess = []
+        self.preprocess.append(preprocess)
         return self
 
     def add_inputs(self, **kwargs):
@@ -212,6 +214,7 @@ class ExecutionContext:
             desc,
             logger,
             self.global_hints,
+            self.global_errors,
             **_kwargs)
         new._parent = self
         new._tab = self._tab + " " * DEBUG_TAB_SIZE
@@ -230,7 +233,8 @@ class ExecutionContext:
         if self.logger and not self.logger.disabled:
             self.logger.debug(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message))
 
-    def log_error(self, message, who=None):
+    def log_error(self, message, who=None, exc=None):
+        self.global_errors.append(exc or message)
         if self.logger and not self.logger.disabled:
             self.logger.exception(f"[{self._id:2}]" + self._tab + (f"[{who}] " if who else "") + str(message))
 

src/core/sheerka/Services/SheerkaCreateNewConcept.py

@@ -1,8 +1,10 @@
 from core.builtin_concepts import BuiltinConcepts, ErrorConcept
 from core.concept import Concept
 from sdp.sheerkaDataProvider import SheerkaDataProviderDuplicateKeyError, SheerkaDataProviderRef
+import core.utils
 
-CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser"
+BNF_NODE_PARSER_CLASS = "parsers.BnfNodeParser.BnfNodeParser"
+BASE_NODE_PARSER_CLASS = "parsers.BaseNodeParser.BaseNodeParser"
 
 
 class SheerkaCreateNewConcept:
@@ -13,6 +15,7 @@ class SheerkaCreateNewConcept:
     def __init__(self, sheerka):
         self.sheerka = sheerka
         self.logger_name = self.create_new_concept.__name__
+        self.base_lexer_parser = core.utils.get_class(BASE_NODE_PARSER_CLASS)("BaseNodeParser", 0)
 
     def create_new_concept(self, context, concept: Concept):
         """
@@ -25,7 +28,7 @@ class SheerkaCreateNewConcept:
 
         concept.init_key()
         concepts_definitions = None
-        init_ret_value = None
+        init_bnf_ret_value = None
 
         sdp = self.sheerka.sdp
 
@@ -49,13 +52,19 @@ class SheerkaCreateNewConcept:
             concepts_definitions[concept] = concept.bnf
 
             # check if it's a valid BNF or whether it breaks the known rules
-            concept_lexer_parser = self.sheerka.parsers[CONCEPT_LEXER_PARSER_CLASS]()
+            bnf_lexer_parser = self.sheerka.parsers[BNF_NODE_PARSER_CLASS]()
             with context.push(self.sheerka.name, desc=f"Initializing concept definition for {concept}") as sub_context:
                 sub_context.concepts[concept.key] = concept  # the concept is not in the real cache yet
-                init_ret_value = concept_lexer_parser.initialize(sub_context, concepts_definitions)
-                sub_context.add_values(return_values=init_ret_value)
-                if not init_ret_value.status:
-                    return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_ret_value.value))
+                init_bnf_ret_value = bnf_lexer_parser.initialize(sub_context, concepts_definitions)
+                sub_context.add_values(return_values=init_bnf_ret_value)
+                if not init_bnf_ret_value.status:
+                    return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_bnf_ret_value.value))
+
+        # update concept definition by key
+        init_sya_ret_value = self.base_lexer_parser.initialize(context, [concept], use_sheerka=True)
+        if not init_sya_ret_value.status:
+            return self.sheerka.ret(self.logger_name, False, ErrorConcept(init_sya_ret_value.value))
+        concepts_by_first_keyword = init_sya_ret_value.body
 
         concept.freeze_definition_hash()
 
@@ -97,9 +106,15 @@ class SheerkaCreateNewConcept:
                 sdp.set(
                     context.event.get_digest(),
                     self.sheerka.CONCEPTS_DEFINITIONS_ENTRY,
-                    concept_lexer_parser.encode_grammar(init_ret_value.body),
+                    bnf_lexer_parser.encode_grammar(init_bnf_ret_value.body),
                     use_ref=True)
                 self.sheerka.concepts_definitions_cache = None  # invalidate cache
+
+            # update the concepts by first keyword
+            sdp.set(context.event.get_digest(),
+                    self.sheerka.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
+                    concepts_by_first_keyword)
+
         except SheerkaDataProviderDuplicateKeyError as error:
             context.log_error("Failed to create a new concept.", who=self.logger_name)
             return self.sheerka.ret(
@@ -109,13 +124,13 @@ class SheerkaCreateNewConcept:
                 error.args[0])
 
         # Updates the caches
-
         self.sheerka.cache_by_key[concept.key] = sdp.get_safe(self.sheerka.CONCEPTS_ENTRY, concept.key)
         self.sheerka.cache_by_name[concept.name] = sdp.get_safe(self.sheerka.CONCEPTS_BY_NAME_ENTRY, concept.name)
         self.sheerka.cache_by_id[concept.id] = concept
-        if init_ret_value is not None and init_ret_value.status:
-            self.sheerka.concepts_grammars = init_ret_value.body
+        if init_bnf_ret_value is not None and init_bnf_ret_value.status:
+            self.sheerka.concepts_grammars = init_bnf_ret_value.body
+        self.sheerka.concepts_by_first_keyword = concepts_by_first_keyword
 
-        # process the return in needed
+        # process the return if needed
         ret = self.sheerka.ret(self.logger_name, True, self.sheerka.new(BuiltinConcepts.NEW_CONCEPT, body=concept))
         return ret

src/core/sheerka/Services/SheerkaEvaluateConcept.py

@@ -1,6 +1,6 @@
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, DoNotResolve, ConceptParts, InfiniteRecursionResolved
-from core.builtin_helpers import add_to_ret_val, remove_from_ret_val, expect_one
+from core.builtin_helpers import expect_one
 
 CONCEPT_EVALUATION_STEPS = [
     BuiltinConcepts.BEFORE_EVALUATION,

src/core/sheerka/Services/SheerkaExecute.py

@@ -33,6 +33,8 @@ class SheerkaExecute:
 
         # group the parsers by priorities
         instantiated_parsers = [parser(sheerka=self.sheerka) for parser in self.sheerka.parsers.values()]
+        instantiated_parsers = self.preprocess(execution_context, instantiated_parsers)
+
         grouped_parsers = {}
         for parser in [p for p in instantiated_parsers if p.enabled]:
             grouped_parsers.setdefault(parser.priority, []).append(parser)
@@ -44,7 +46,6 @@ class SheerkaExecute:
 
             for parser in grouped_parsers[priority]:
 
-                return_value_success_found = False
                 for return_value in inputs_for_this_group:
 
                     to_parse = return_value.body.body \
@@ -67,22 +68,23 @@ class SheerkaExecute:
                                     r.parents = [return_value]
                                     result.append(r)
                                     if self.sheerka.isinstance(r.body, BuiltinConcepts.PARSER_RESULT):
+                                        # if a ParserResultConcept is returned, it will be used by the parsers
+                                        # of the following groups
                                         to_process.append(r)
                                     if r.status:
-                                        return_value_success_found = True
+                                        stop_processing = True
 
                             else:
                                 res.parents = [return_value]
                                 result.append(res)
                                 if self.sheerka.isinstance(res.body, BuiltinConcepts.PARSER_RESULT):
+                                    # if a ParserResultConcept is returned, it will be used by the parsers
+                                    # of the following groups
                                     to_process.append(res)
                                 if res.status:
-                                    return_value_success_found = True
+                                    stop_processing = True
                         sub_context.add_values(return_values=res)
 
-                    if return_value_success_found:
-                        stop_processing = True
-                        break  # Stop the other return_values (but not the other parsers with the same priority)
 
             if stop_processing:
                 break  # Do not try the other priorities if a match is found
@@ -102,7 +104,7 @@ class SheerkaExecute:
         instantiated_evaluators = [e_class() for e_class in self.sheerka.evaluators]
 
         # pre-process evaluators if needed
-        instantiated_evaluators = self._preprocess_evaluators(execution_context, instantiated_evaluators)
+        instantiated_evaluators = self.preprocess(execution_context, instantiated_evaluators)
 
         for evaluator in [e for e in instantiated_evaluators if e.enabled and process_step in e.steps]:
             grouped_evaluators.setdefault(evaluator.priority, []).append(evaluator)
@@ -123,7 +125,7 @@ class SheerkaExecute:
                     evaluated_items = []
                     to_delete = []
                     for evaluator in grouped_evaluators[priority]:
-                        evaluator = self._preprocess_evaluators(execution_context, evaluator.__class__())  # fresh copy
+                        evaluator = self.preprocess(execution_context, evaluator.__class__())  # fresh copy
 
                         sub_context_desc = f"Evaluating using {evaluator.name} ({priority=})"
                         with iteration_context.push(desc=sub_context_desc, logger=evaluator.verbose_log) as sub_context:
@@ -215,22 +217,29 @@ class SheerkaExecute:
 
         return return_values
 
-    def _preprocess_evaluators(self, context, evaluators):
+    def preprocess(self, context, parsers_or_evaluators):
         if not context.preprocess:
-            return evaluators
+            return parsers_or_evaluators
 
-        if not hasattr(evaluators, "__iter__"):
+        if not hasattr(parsers_or_evaluators, "__iter__"):
             single_one = True
-            evaluators = [evaluators]
+            parsers_or_evaluators = [parsers_or_evaluators]
         else:
             single_one = False
 
         for preprocess in context.preprocess:
-            for e in evaluators:
-                if preprocess.props["name"].value == e.name:
+            for e in parsers_or_evaluators:
+                if self.matches(e.name, preprocess.get_prop("name")):
                     for prop, value in preprocess.props.items():
                         if prop == "name":
                             continue
                         if hasattr(e, prop):
                             setattr(e, prop, value.value)
-        return evaluators[0] if single_one else evaluators
+        return parsers_or_evaluators[0] if single_one else parsers_or_evaluators
+
+    @staticmethod
+    def matches(parser_or_evaluator_name, preprocessor_name):
+        if preprocessor_name.endswith("*"):
+            return parser_or_evaluator_name.startswith(preprocessor_name[:-1])
+        else:
+            return parser_or_evaluator_name == preprocessor_name

src/core/sheerka/Sheerka.py

@@ -17,12 +17,7 @@ from core.sheerka_logger import console_handler
 
 import logging
 
-# CONCEPT_EVALUATION_STEPS = [
-#     BuiltinConcepts.BEFORE_EVALUATION,
-#     BuiltinConcepts.EVALUATION,
-#     BuiltinConcepts.AFTER_EVALUATION]
-
-CONCEPT_LEXER_PARSER_CLASS = "parsers.ConceptLexerParser.ConceptLexerParser"
+CONCEPT_LEXER_PARSER_CLASS = "parsers.BnfNodeParser.BnfNodeParser"
 BNF_PARSER_CLASS = "parsers.BnfParser.BnfParser"
 CONCEPTS_FILE = "_concepts.txt"
 
@@ -37,6 +32,9 @@ class Sheerka(Concept):
     CONCEPTS_BY_NAME_ENTRY = "Concepts_By_Name"
     CONCEPTS_BY_HASH_ENTRY = "Concepts_By_Hash"  # store hash of concepts definitions (not values)
     CONCEPTS_DEFINITIONS_ENTRY = "Concepts_Definitions"  # to store definitions (bnf) of concepts
+    CONCEPTS_BY_FIRST_KEYWORD_ENTRY = "Concepts_By_First_Keyword"
+    CONCEPTS_SYA_DEFINITION_ENTRY = "Concepts_Sya_Definitions"
+
     BUILTIN_CONCEPTS_KEYS = "Builtins_Concepts"  # sequential key for builtin concepts
     USER_CONCEPTS_KEYS = "User_Concepts"  # sequential key for user defined concepts
 
@@ -65,6 +63,10 @@ class Sheerka(Concept):
         # a grammar is a resolved BNF
         self.concepts_grammars = {}
 
+        # cache for SYA concepts
+        self.concepts_by_first_keyword = {}
+        self.sya_definitions = {}
+
         # a concept can be instantiated
         # ex: File is a concept, but File('foo.txt') is an instance
         # TODO: manage contexts
@@ -119,7 +121,8 @@ class Sheerka(Concept):
                 self.initialize_builtin_concepts()
                 self.initialize_builtin_parsers()
                 self.initialize_builtin_evaluators()
-                self.initialize_concepts_definitions(exec_context)
+                self.initialize_bnf_parsing(exec_context)
+                self.initialize_sya_parsing()
                 res = ReturnValueConcept(self, True, self)
 
                 exec_context.add_values(return_values=res)
@@ -174,12 +177,25 @@ class Sheerka(Concept):
         """
         core.utils.init_package_import("parsers")
         base_class = core.utils.get_class("parsers.BaseParser.BaseParser")
+        modules_to_skip = ["parsers.BaseNodeParser"]
+
+        temp_result = {}
         for parser in core.utils.get_sub_classes("parsers", base_class):
             if parser.__module__ == base_class.__module__:
                 continue
 
-            self.init_log.debug(f"Adding builtin parser '{parser.__name__}'")
-            self.parsers[core.utils.get_full_qualified_name(parser)] = parser
+            if parser.__module__ in modules_to_skip:
+                continue
+
+            qualified_name = core.utils.get_full_qualified_name(parser)
+            self.init_log.debug(f"Adding builtin parser '{qualified_name}'")
+            temp_result[qualified_name] = parser
+
+        # Now we sort the parser by name.
+        #  It's not important for the logic of their usage as they have their priority anyway,
+        #  We do that for the unit tests. They are to complicated to write otherwise
+        for name in sorted(temp_result.keys()):
+            self.parsers[name] = temp_result[name]
 
     def initialize_builtin_evaluators(self):
         """
@@ -195,7 +211,7 @@ class Sheerka(Concept):
             self.init_log.debug(f"Adding builtin evaluator '{evaluator.__name__}'")
             self.evaluators.append(evaluator)
 
-    def initialize_concepts_definitions(self, execution_context):
+    def initialize_bnf_parsing(self, execution_context):
         self.init_log.debug("Initializing concepts grammars.")
         definitions = self.get_concepts_definitions(execution_context)
 
@@ -211,6 +227,25 @@ class Sheerka(Concept):
 
         self.concepts_grammars = lexer_parser.concepts_grammars
 
+    def initialize_sya_parsing(self):
+        self.init_log.debug("Initializing sya definitions.")
+
+        self.concepts_by_first_keyword = self.sdp.get_safe(
+            self.CONCEPTS_BY_FIRST_KEYWORD_ENTRY,
+            load_origin=False) or {}
+
+        self.sya_definitions = self.sdp.get_safe(
+            self.CONCEPTS_SYA_DEFINITION_ENTRY,
+            load_origin=False) or {}
+
+    def reset(self):
+        self.reset_cache()
+        self.concepts_by_first_keyword = {}
+        self.concepts_grammars = {}
+        self.sya_definitions = {}
+        self.sdp.reset()
+        self.sdp.set_key(self.USER_CONCEPTS_KEYS, 1000)
+
     def reset_cache(self, filter_to_use=None):
         """
         reset the different cache that exists
@@ -220,6 +255,7 @@ class Sheerka(Concept):
         if filter_to_use is None:
             self.cache_by_key = {}
             self.cache_by_id = {}
+            self.cache_by_name = {}
         else:
             raise NotImplementedError()
 
@@ -324,6 +360,38 @@ class Sheerka(Concept):
         """
         return self.sets_handler.set_isa(context, concept, concept_set)
 
+    def set_sya_def(self, context, list_of_def):
+        """
+        Set the precedence and/or the associativity of a concept
+        :param context:
+        :param list_of_def list of tuple(concept_id, precedence (int), SyaAssociativity)
+        :return:
+        """
+
+        # validate the entries
+        for concept_id, precedence, associativity in list_of_def:
+            if concept_id == BuiltinConcepts.UNKNOWN_CONCEPT:
+                return self.ret(self.name,
+                                False,
+                                self.new(BuiltinConcepts.ERROR, body=f"Concept {concept_id} is not known"))
+
+        # update the definitions
+        for concept_id, precedence, associativity in list_of_def:
+            if precedence is None and associativity is None:
+                try:
+                    del self.sya_definitions[concept_id]
+                except KeyError:
+                    pass
+            else:
+                self.sya_definitions[concept_id] = (precedence, associativity.value)
+
+        # then save
+        self.sdp.set(context.event.get_digest(),
+                     self.CONCEPTS_SYA_DEFINITION_ENTRY,
+                     self.sya_definitions)
+
+        return self.ret(self.name, True, self.new(BuiltinConcepts.SUCCESS))
+
     def get_set_elements(self, context, concept):
         """
         Concept is supposed to be a set
@@ -571,6 +639,22 @@ class Sheerka(Concept):
 
         return self.value(body_to_use)
 
+    def get_error(self, obj):
+        if isinstance(obj, Concept) and obj.metadata.is_builtin and obj.key in BuiltinErrors:
+            return obj
+
+        if isinstance(obj, list):
+            return  obj
+
+        if self.isinstance(obj, BuiltinConcepts.RETURN_VALUE):
+            if obj.status:
+                return None
+
+            if self.isinstance(obj.body, BuiltinConcepts.PARSER_RESULT):
+                return self.get_error(obj.body.body)
+
+        return NotImplementedError()
+
     def get_values(self, objs):
         if not (isinstance(objs, list) or
                 self.isinstance(objs, BuiltinConcepts.LIST) or

src/core/utils.py

@@ -163,7 +163,7 @@ def remove_list_from_list(lst, to_remove):
 def product(a, b):
     """
     Kind of cartesian product between lists a and b
-    knowing that a is also a list
+    knowing that a is also a list : a is a list of list !!!
 
     So it's a cartesian product between a list of list and a list
     """
@@ -176,7 +176,12 @@ def product(a, b):
     res = []
     for item_b in b:
         for item_a in a:
-            items = item_a + [item_b]
+            #items = item_a + [item_b]
+            items = item_a[:]
+            if hasattr(item_b, "__iter__"):
+                items.extend(item_b)
+            else:
+                items.append(item_b)
             res.append(items)
 
     return res
@@ -276,6 +281,7 @@ def str_concept(t):
     >>> assert str_concept((None, "id")) == "c:|id:"
     >>> assert str_concept(("key", None)) == "c:key:"
     >>> assert str_concept((None, None)) == ""
+    >>> assert str_concept(Concept(key="foo", id="bar")) == "c:foo|bar:"
     :param t:
     :return:
     """
@@ -297,6 +303,12 @@ def unstr_concept(concept_repr):
     """
     if concept_repr is like :c:key:id:
     return the key and the id
+    >>> assert unstr_concept("c:key:") == "key"
+    >>> assert unstr_concept("c:key|id:") == ("key", "id")
+    >>> assert unstr_concept("c:|id:") == ("None", "id")
+    >>> assert unstr_concept("c:key|:") == ("key", "None")
+    >>> # Otherwise, return (None,None)
+
     :param concept_repr:
     :return:
     """
@@ -371,3 +383,25 @@ def decode_concept(text):
         return key, id_, use_concept
 
     return None, None, None
+
+
+def tokens_index(tokens, sub_tokens, skip=0):
+    """
+    Index of the sub tokens in tokens
+    :param tokens: tokens
+    :param sub_tokens: sub tokens to search
+    :param skip: number of found to skip
+    :return:
+    """
+    expected = [token.value for token in sub_tokens if token.type != TokenKind.EOF]
+    for i in range(0, len(tokens) - len(expected) + 1):
+        for j in range(len(expected)):
+            if tokens[i + j].value != expected[j]:
+                break
+        else:
+            if skip == 0:
+                return i
+            else:
+                skip -= 1
+
+    raise ValueError(f"sub tokens '{sub_tokens}' not found")