#####################################################################################################
# This implementation of the parser is highly inspired by the arpeggio project (https://github.com/textX/Arpeggio)
# I don't directly use the project, but it helped me figure out
# what to do.
#       Dejanović I., Milosavljević G., Vaderna R.:
#       Arpeggio: A flexible PEG parser for Python,
#       Knowledge-Based Systems, 2016, 95, 71 - 74, doi:10.1016/j.knosys.2015.12.004
#####################################################################################################
from collections import defaultdict
from dataclasses import dataclass

import core.utils
from cache.Cache import Cache
from core import builtin_helpers
from core.builtin_concepts import BuiltinConcepts
from core.concept import Concept, DEFINITION_TYPE_BNF, DoNotResolve, ConceptParts
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer, Token, TokenKind
from parsers.BaseNodeParser import BaseNodeParser, LexerNode, UnrecognizedTokensNode, ConceptNode, GrammarErrorNode
from parsers.BaseParser import ErrorNode

PARSERS = ["AtomNode", "SyaNode", "Python"]


@dataclass
class ConceptParsingError(ErrorNode):
    concept: Concept


class NonTerminalNode(LexerNode):
    """
    Returned by the BnfNodeParser
    """

    def __init__(self, parsing_expression, start, end, tokens, children=None):
        super().__init__(start, end, tokens)
        self.parsing_expression = parsing_expression
        self.children = children

    def __repr__(self):
        name = "Node:" + (self.parsing_expression.rule_name or self.parsing_expression.__class__.__name__)
        if len(self.children) > 0:
            sub_names = "(" + ",".join([repr(child) for child in self.children]) + ")"
        else:
            sub_names = ""
        return name + sub_names

    def __eq__(self, other):
        if not isinstance(other, NonTerminalNode):
            return False

        return self.parsing_expression == other.parsing_expression and \
               self.start == other.start and \
               self.end == other.end and \
               self.children == other.children

    def __hash__(self):
        return hash((self.parsing_expression, self.start, self.end, self.children))


class TerminalNode(LexerNode):
    """
    Returned by the BnfNodeParser
    """

    def __init__(self, parsing_expression, start, end, value):
        super().__init__(start, end, source=value)
        self.parsing_expression = parsing_expression
        self.value = value

    def __repr__(self):
        name = "Node:" + (self.parsing_expression.rule_name or "")
        return name + f"'{self.value}'"

    def __eq__(self, other):
        if not isinstance(other, TerminalNode):
            return False

        return self.parsing_expression == other.parsing_expression and \
               self.start == other.start and \
               self.end == other.end and \
               self.value == other.value

    def __hash__(self):
        return hash((self.parsing_expression, self.start, self.end, self.value))


class ParsingExpression:
    def __init__(self, *args, **kwargs):
        self.elements = args

        nodes = kwargs.get('nodes', [])
        if not hasattr(nodes, '__iter__'):
            nodes = [nodes]
        self.nodes = nodes

        self.rule_name = kwargs.get('rule_name', '')

    def __eq__(self, other):
        if not isinstance(other, ParsingExpression):
            return False

        return self.rule_name == other.rule_name and self.elements == other.elements

    def __hash__(self):
        return hash((self.rule_name, self.elements))

    def parse(self, parser):
        return self._parse(parser)

    def add_rule_name_if_needed(self, text):
        return text + "=" + self.rule_name if self.rule_name else text


class ConceptExpression(ParsingExpression):
    """
    Will match a concept
    It used only for rule definition

    When the grammar is created, it is replaced by the actual concept
    """

    def __init__(self, concept, rule_name=""):
        super().__init__(rule_name=rule_name)
        self.concept = concept

    def __repr__(self):
        return self.add_rule_name_if_needed(f"{self.concept}")

    def __eq__(self, other):
        if not super().__eq__(other):
            return False

        if not isinstance(other, ConceptExpression):
            return False

        if isinstance(self.concept, Concept):
            return self.concept.name == other.concept.name

        # when it's only the name of the concept
        return self.concept == other.concept

    def __hash__(self):
        return hash((self.concept, self.rule_name))

    def _parse(self, parser_helper):
        node = self.nodes[0].parse(parser_helper)
        if node is None:
            return None
        return NonTerminalNode(self,
                               node.start,
                               node.end,
                               parser_helper.parser.parser_input.tokens[node.start: node.end + 1],
                               [node])


class Sequence(ParsingExpression):
    """
    Will match sequence of parser expressions in exact order they are defined.
    """

    def _parse(self, parser_helper):
        init_pos = parser_helper.pos
        end_pos = parser_helper.pos

        children = []
        for e in self.nodes:
            node = e.parse(parser_helper)
            if node is None:
                return None
            else:
                if node.end != -1:  # because returns -1 when no match
                    children.append(node)
                    end_pos = node.end

        return NonTerminalNode(self,
                               init_pos,
                               end_pos,
                               parser_helper.parser.parser_input.tokens[init_pos: end_pos + 1],
                               children)

    def __repr__(self):
        to_str = ", ".join(repr(n) for n in self.elements)
        return self.add_rule_name_if_needed(f"({to_str})")


class OrderedChoice(ParsingExpression):
    """
    Will match the first one among multiple
    It will stop at the first match (so the order of definition is important)
    """

    def _parse(self, parser_helper):
        init_pos = parser_helper.pos

        for e in self.nodes:
            node = e.parse(parser_helper)
            if node:
                return NonTerminalNode(self,
                                       init_pos,
                                       node.end,
                                       parser_helper.parser.parser_input.tokens[init_pos: node.end + 1],
                                       [node])

            parser_helper.seek(init_pos)  # backtrack

        return None

    def __repr__(self):
        to_str = "| ".join(repr(n) for n in self.elements)
        return self.add_rule_name_if_needed(f"({to_str})")


class LongestChoice(ParsingExpression):
    """
    Will match the longest one among multiple
    All elements will be tested, so the order is not important
    The behaviour when multiple candidate is found is not defined yet
    """

    def _parse(self, parser_helper):
        init_pos = parser_helper.pos
        longest_node = None
        end_pos = -1

        for e in self.nodes:
            node = e.parse(parser_helper)
            if node:
                if longest_node is None or node.end > longest_node.end:
                    longest_node = node
                    end_pos = parser_helper.pos

            parser_helper.seek(init_pos)  # backtrack

        if longest_node is None:
            return None

        parser_helper.seek(end_pos)
        return NonTerminalNode(self,
                               init_pos,
                               longest_node.end,
                               parser_helper.parser.parser_input.tokens[init_pos: longest_node.end + 1],
                               [longest_node])

    def __repr__(self):
        to_str = "# ".join(repr(n) for n in self.elements)
        return self.add_rule_name_if_needed(f"({to_str})")


class Optional(ParsingExpression):
    """
    Will match or not the elements
    if many matches, will choose longest one
    If you need order, use Optional(OrderedChoice)
    """

    def _parse(self, parser_helper):
        init_pos = parser_helper.pos
        selected_node = NonTerminalNode(self, parser_helper.pos, -1, [], [])  # means that nothing is found

        for e in self.nodes:
            node = e.parse(parser_helper)
            if node:
                if node.end > selected_node.end:
                    selected_node = NonTerminalNode(
                        self,
                        node.start,
                        node.end,
                        parser_helper.parser.parser_input.tokens[node.start: node.end + 1],
                        [node])

            parser_helper.seek(init_pos)  # backtrack

        if selected_node.end != -1:
            parser_helper.seek(selected_node.end)
            parser_helper.next_token()  # eat the tokens found

        return selected_node

    def __repr__(self):
        if len(self.elements) == 1:
            return f"{self.elements[0]}?"
        else:
            to_str = ", ".join(repr(n) for n in self.elements)
        return self.add_rule_name_if_needed(f"({to_str})?")


class Repetition(ParsingExpression):
    """
    Base class for all repetition-like parser expressions (?,*,+)
    Args:
        eolterm(bool): Flag that indicates that end of line should
            terminate repetition match.
    """

    def __init__(self, *elements, **kwargs):
        super(Repetition, self).__init__(*elements, **kwargs)
        self.sep = kwargs.get('sep', None)


class ZeroOrMore(Repetition):
    """
    ZeroOrMore will try to match parser expression specified zero or more
    times. It will never fail.
    """

    def _parse(self, parser_helper):
        init_pos = parser_helper.pos
        end_pos = -1
        children = []

        while True:
            current_pos = parser_helper.pos

            # maybe eat the separator if needed
            if self.sep and children:
                sep_result = self.sep.parse(parser_helper)
                if sep_result is None:
                    parser_helper.seek(current_pos)
                    break

            # eat the ZeroOrMore
            node = self.nodes[0].parse(parser_helper)
            if node is None:
                parser_helper.seek(current_pos)
                break
            else:
                if node.end != -1:  # because returns -1 when no match
                    children.append(node)
                    end_pos = node.end

        if len(children) == 0:
            return NonTerminalNode(self, init_pos, -1, [], [])

        return NonTerminalNode(self, init_pos, end_pos, parser_helper.parser.parser_input.tokens[init_pos: end_pos + 1],
                               children)

    def __repr__(self):
        to_str = ", ".join(repr(n) for n in self.elements)
        return self.add_rule_name_if_needed(f"({to_str})*")


class OneOrMore(Repetition):
    """
    OneOrMore will try to match parser expression specified one or more times.
    """

    def _parse(self, parser_helper):
        init_pos = parser_helper.pos
        end_pos = -1
        children = []

        while True:
            current_pos = parser_helper.pos

            # maybe eat the separator if needed
            if self.sep and children:
                sep_result = self.sep.parse(parser_helper)
                if sep_result is None:
                    parser_helper.seek(current_pos)
                    break

            # eat the ZeroOrMore
            node = self.nodes[0].parse(parser_helper)
            if node is None:
                parser_helper.seek(current_pos)
                break
            else:
                if node.end != -1:  # because returns -1 when no match
                    children.append(node)
                    end_pos = node.end

        if len(children) == 0:  # if nothing is found, it's an error
            return None

        return NonTerminalNode(self,
                               init_pos,
                               end_pos,
                               parser_helper.parser.parser_input.tokens[init_pos: end_pos + 1],
                               children)

    def __repr__(self):
        to_str = ", ".join(repr(n) for n in self.elements)
        return self.add_rule_name_if_needed(f"({to_str})+")


class UnorderedGroup(Repetition):
    """
    Will try to match all of the parsing expression in any order.
    """

    def _parse(self, parser):
        raise NotImplementedError()

    # def __repr__(self):
    #     to_str = ", ".join(repr(n) for n in self.elements)
    #     return f"({to_str})#"


class Match(ParsingExpression):
    """
    Base class for all classes that will try to match something from the input.
    """

    def __init__(self, rule_name, root=False):
        super(Match, self).__init__(rule_name=rule_name, root=root)

    def parse(self, parser):
        result = self._parse(parser)
        return result


class StrMatch(Match):
    """
    Matches a literal
    """

    def __init__(self, to_match, rule_name="", ignore_case=True, skip_whitespace=True):
        super(Match, self).__init__(rule_name=rule_name)
        self.to_match = to_match
        self.ignore_case = ignore_case
        self.skip_white_space = skip_whitespace

    def __repr__(self):
        text = self.to_match
        if not self.ignore_case:
            text += "#!ic"
        if not self.skip_white_space:
            text += "#!sw"
        return self.add_rule_name_if_needed(f"'{text}'")

    def __eq__(self, other):
        if not super().__eq__(other):
            return False

        if not isinstance(other, StrMatch):
            return False

        return self.to_match == other.to_match and \
               self.ignore_case == other.ignore_case and \
               self.skip_white_space == other.skip_white_space

    def _parse(self, parser_helper):
        token = parser_helper.get_token()
        m = token.str_value.lower() == self.to_match.lower() if self.ignore_case \
            else token.strip_quote == self.to_match

        if m:
            node = TerminalNode(self, parser_helper.pos, parser_helper.pos, token.str_value)
            parser_helper.next_token(self.skip_white_space)
            return node

        return None


# class RegExMatch(Match):
#     '''
#     This Match class will perform input matching based on Regular Expressions.
#
#     Args:
#         to_match (regex string): A regular expression string to match.
#             It will be used to create regular expression using re.compile.
#         ignore_case(bool): If case insensitive match is needed.
#             Default is None to support propagation from global parser setting.
#         multiline(bool): allow regex to works on multiple lines
#             (re.DOTALL flag). Default is None to support propagation from
#             global parser setting.
#         str_repr(str): A string that is used to represent this regex.
#         re_flags: flags parameter for re.compile if neither ignore_case
#             or multiple are set.
#
#     '''
#     def __init__(self, to_match, rule_name='', root=False, ignore_case=None,
#                  multiline=None, str_repr=None, re_flags=re.MULTILINE):
#         super(RegExMatch, self).__init__(rule_name, root)
#         self.to_match_regex = to_match
#         self.ignore_case = ignore_case
#         self.multiline = multiline
#         self.explicit_flags = re_flags
#
#         self.to_match = str_repr if str_repr is not None else to_match
#
#     def compile(self):
#         flags = self.explicit_flags
#         if self.multiline is True:
#             flags |= re.DOTALL
#         if self.multiline is False and flags & re.DOTALL:
#             flags -= re.DOTALL
#         if self.ignore_case is True:
#             flags |= re.IGNORECASE
#         if self.ignore_case is False and flags & re.IGNORECASE:
#             flags -= re.IGNORECASE
#         self.regex = re.compile(self.to_match_regex, flags)
#
#     def __str__(self):
#         return self.to_match
#
#     def __unicode__(self):
#         return self.__str__()
#
#     def _parse(self, parser):
#         c_pos = parser.position
#         m = self.regex.match(parser.input, c_pos)
#         if m:
#             matched = m.group()
#             if parser.debug:
#                 parser.dprint(
#                     "++ Match '%s' at %d => '%s'" %
#                     (matched, c_pos, parser.context(len(matched))))
#             parser.position += len(matched)
#             if matched:
#                 return Terminal(self, c_pos, matched, extra_info=m)
#         else:
#             if parser.debug:
#                 parser.dprint("-- NoMatch at {}".format(c_pos))
#             parser._nm_raise(self, c_pos, parser)


class ParsingExpressionVisitor:
    """
    visit ParsingExpression
    """

    STOP = "##_Stop_##"

    def visit(self, parsing_expression):
        name = parsing_expression.__class__.__name__

        method = 'visit_' + name
        visitor = getattr(self, method, self.generic_visit)
        return visitor(parsing_expression)

    def generic_visit(self, parsing_expression):
        if hasattr(self, "visit_all"):
            self.visit_all(parsing_expression)

        for node in parsing_expression.elements:
            if isinstance(node, Concept):
                res = self.visit(ConceptExpression(node.key or node.name))
            elif isinstance(node, str):
                res = self.visit(StrMatch(node))
            else:
                res = self.visit(node)

            if res == self.STOP:
                return


class BnfNodeFirstTokenVisitor(ParsingExpressionVisitor):
    def __init__(self, sheerka):
        self.sheerka = sheerka
        self.first_tokens = None

    def add_first_token(self, first_token):
        if not self.first_tokens:
            self.first_tokens = [first_token]
        else:
            self.first_tokens.append(first_token)

    def visit_ConceptExpression(self, pe):
        concept = self.sheerka.get_by_key(pe.concept) if isinstance(pe.concept, str) else pe.concept
        if self.sheerka.is_known(concept):
            self.add_first_token(core.utils.str_concept(concept, drop_name=True))
        return self.STOP

    def visit_StrMatch(self, pe):
        if not pe.to_match:
            return
        self.add_first_token(pe.to_match)
        return self.STOP

    def visit_OrderedChoice(self, parsing_expression):
        for node in parsing_expression.elements:
            self.visit(node)
        return self.STOP


class BnfConceptParserHelper:
    def __init__(self, parser):
        self.parser = parser
        self.debug = []
        self.errors = []
        self.sequence = []

        self.unrecognized_tokens = UnrecognizedTokensNode(-1, -1, [])
        self.has_unrecognized = False
        self.bnf_parsed = False

        self.forked = []

        self.token = None
        self.pos = -1

    def __repr__(self):
        return f"BnfConceptParserHelper({self.sequence})"

    def __eq__(self, other):
        if id(self) == id(other):
            return True

        if not isinstance(other, BnfConceptParserHelper):
            return False

        return self.sequence == other.sequence and self.errors == other.errors

    def __hash__(self):
        return len(self.sequence) + len(self.errors)

    def get_token(self) -> Token:
        return self.token

    def next_token(self, skip_whitespace=True):
        if self.token and self.token.type == TokenKind.EOF:
            return False

        self.pos += 1
        self.token = self.parser.parser_input.tokens[self.pos]

        if skip_whitespace:
            while self.token.type == TokenKind.WHITESPACE or self.token.type == TokenKind.NEWLINE:
                self.pos += 1
                self.token = self.parser.parser_input.tokens[self.pos]

        return self.token.type != TokenKind.EOF

    def seek(self, pos):
        self.pos = pos
        self.token = self.parser.parser_input.tokens[self.pos]

    def has_error(self):
        return len(self.errors) > 0

    def is_locked(self):
        return self.parser.parser_input.pos <= self.pos or self.has_error()

    def eat_concept(self, concept, token):
        if self.is_locked():
            return

        self.debug.append(concept)
        self.manage_unrecognized()
        for forked in self.forked:
            # manage the fact that some clone may have been forked
            forked.eat_concept(concept, token)

        # init
        parsing_expression = self.parser.get_parsing_expression(self.parser.context, concept)
        if not isinstance(parsing_expression, ParsingExpression):
            self.debug.append(concept)
            error_msg = f"Failed to parse concept '{concept}'"
            if parsing_expression is not None:
                error_msg += f". Reason: '{parsing_expression}'"
            self.errors.append(GrammarErrorNode(error_msg))
            return

        self.pos = self.parser.parser_input.pos
        self.token = self.parser.parser_input.tokens[self.pos]

        # parse
        node = parsing_expression.parse(self)
        if node is not None and node.end != -1:
            self.sequence.append(self.create_concept_node(concept, node))
            self.pos = node.end
            self.bnf_parsed = True
        else:
            self.debug.append(("Rewind", token))
            self.unrecognized_tokens.add_token(token, self.parser.parser_input.pos)
            self.pos = self.parser.parser_input.pos  # reset position

    def eat_unrecognized(self, token):
        if self.is_locked():
            return

        self.debug.append(token)
        self.unrecognized_tokens.add_token(token, self.parser.parser_input.pos)

    def manage_unrecognized(self):
        if self.unrecognized_tokens.is_empty():
            return

        # do not put empty UnrecognizedToken in out
        if self.unrecognized_tokens.is_whitespace():
            self.unrecognized_tokens.reset()
            return

        self.unrecognized_tokens.fix_source()

        # try to recognize concepts
        nodes_sequences = builtin_helpers.get_lexer_nodes_from_unrecognized(
            self.parser.context,
            self.unrecognized_tokens,
            PARSERS)

        if nodes_sequences:
            instances = [self]
            for i in range(len(nodes_sequences) - 1):
                clone = self.clone()
                instances.append(clone)
                self.forked.append(clone)

            for instance, node_sequence in zip(instances, nodes_sequences):
                for node in node_sequence:
                    instance.sequence.append(node)
                    if isinstance(node, UnrecognizedTokensNode) or \
                        hasattr(node, "unrecognized_tokens") and node.unrecognized_tokens:
                        instance.has_unrecognized = True
                instance.unrecognized_tokens = UnrecognizedTokensNode(-1, -1, [])

        else:
            self.sequence.append(self.unrecognized_tokens)
            self.has_unrecognized = True

        # create another instance
        self.unrecognized_tokens = UnrecognizedTokensNode(-1, -1, [])

    def clone(self):
        clone = BnfConceptParserHelper(self.parser)
        clone.debug = self.debug[:]
        self.errors = self.errors[:]
        clone.sequence = self.sequence[:]
        clone.pos = self.pos
        clone.unrecognized_tokens = self.unrecognized_tokens.clone()
        return clone

    def finalize(self):
        if self.bnf_parsed > 0:
            self.manage_unrecognized()
            for forked in self.forked:
                # manage that some clones may have been forked
                forked.finalize()

    def create_concept_node(self, template, underlying):
        sheerka = self.parser.context.sheerka
        key = (template.key, template.id) if template.id else template.key
        concept = sheerka.new(key)
        concept = self.finalize_concept(sheerka, concept, underlying)
        concept_node = ConceptNode(
            concept,
            underlying.start,
            underlying.end,
            self.parser.parser_input.tokens[underlying.start: underlying.end + 1],
            None,
            underlying)
        return concept_node

    def finalize_concept(self, sheerka, concept, underlying, init_empty_body=True):
        """
        Updates the properties of the concept
        Goes in recursion if the property is a concept
        """

        # this cache is to make sure that we return the same concept for the same ConceptExpression
        _underlying_value_cache = {}

        def _add_prop(_concept, prop_name, value):
            """
            Adds a new entry,
            makes a list if the property already exists
            """
            if prop_name not in _concept.compiled or _concept.compiled[prop_name] is None:
                # new entry
                _concept.compiled[prop_name] = value
            else:
                # make a list if there was a value
                previous_value = _concept.compiled[prop_name]
                if isinstance(previous_value, list):
                    previous_value.append(value)
                else:
                    new_value = [previous_value, value]
                    _concept.compiled[prop_name] = new_value

        def _look_for_concept_match(_underlying):
            """
            At some point, there is either an StrMatch or a ConceptMatch,
            that allowed the recognition.
            Look for the ConceptMatch, with recursion if needed
            """
            if isinstance(_underlying.parsing_expression, ConceptExpression):
                return _underlying

            if not isinstance(_underlying, NonTerminalNode):
                return None

            if len(_underlying.children) != 1:
                return None

            return _look_for_concept_match(_underlying.children[0])

        def _get_underlying_value(_underlying):
            concept_match_node = _look_for_concept_match(_underlying)
            if concept_match_node:
                # the value is a concept
                if id(concept_match_node) in _underlying_value_cache:
                    result = _underlying_value_cache[id(concept_match_node)]
                else:
                    ref_tpl = concept_match_node.parsing_expression.concept
                    new = sheerka.new_from_template(ref_tpl, ref_tpl.key)
                    result = self.finalize_concept(sheerka, new, concept_match_node.children[0], init_empty_body)
                    _underlying_value_cache[id(concept_match_node)] = result
            else:
                # the value is a string
                result = DoNotResolve(_underlying.source)

            return result

        def _process_rule_name(_concept, _underlying):
            if _underlying.parsing_expression.rule_name:
                value = _get_underlying_value(_underlying)
                _add_prop(_concept, _underlying.parsing_expression.rule_name, value)
                _concept.metadata.need_validation = True

            elif isinstance(_underlying, NonTerminalNode):
                for child in _underlying.children:
                    _process_rule_name(_concept, child)

        if init_empty_body and concept.metadata.body is None:
            value = _get_underlying_value(underlying)
            concept.compiled[ConceptParts.BODY] = value
            if underlying.parsing_expression.rule_name:
                _add_prop(concept, underlying.parsing_expression.rule_name, value)
                # KSI : Why don't we set concept.metadata.need_validation to True ?

        if isinstance(underlying, NonTerminalNode):
            for node in underlying.children:
                _process_rule_name(concept, node)

        return concept


@dataclass
class UnderConstruction:
    concept_id: str


@dataclass()
class ToUpdate:
    parent_id: int
    parsing_expression: ParsingExpression

    def __hash__(self):
        return hash(self.parent_id)


class BnfNodeParser(BaseNodeParser):
    def __init__(self, **kwargs):
        super().__init__("BnfNode", 50, **kwargs)

        if 'sheerka' in kwargs:
            sheerka = kwargs.get("sheerka")
            self.concepts_grammars = sheerka.concepts_grammars
        else:
            self.concepts_grammars = Cache()

        self.ignore_case = True

    @staticmethod
    def _is_eligible(concept):
        """
        Predicate that select concepts that must handled by AtomNodeParser
        :param concept:
        :return:
        """
        return concept.metadata.definition_type == DEFINITION_TYPE_BNF

    @staticmethod
    def get_valid(parsers_helpers):
        valid_parser_helpers = []
        for parser_helper in parsers_helpers:
            if not parser_helper.bnf_parsed or parser_helper.has_error():
                continue

            if parser_helper in valid_parser_helpers:
                continue

            valid_parser_helpers.append(parser_helper)

        return valid_parser_helpers

    @staticmethod
    def get_expression_from_concept_name(name):
        """
        Create the parsing expression from the name
        This function differs from BNFParser.parse() as it does not try to resolve identifiers into concepts
        >>> assert get_expression_from_concept_name('one hundred') == Sequence(StrMatch("one"), StrMatch("hundred"))
        while BNFParser.parse("one hundred") will look for concept 'one' and concept 'hundred'
        :param name:
        :return:
        """
        if name is None or name.strip() == "":
            return []

        res = []
        tokens = Tokenizer(name, yield_eof=False)
        for token in tokens:
            if token.type == TokenKind.WHITESPACE:
                continue
            elif token.type == TokenKind.STRING:
                sub_tokens = list(Tokenizer(token.strip_quote, yield_eof=False))
                for sub_token in sub_tokens[:-1]:
                    res.append(StrMatch(sub_token.str_value, skip_whitespace=False))
                res.append(StrMatch(sub_tokens[-1].str_value))
            else:
                res.append(StrMatch(token.str_value))

        return res[0] if len(res) == 1 else Sequence(*res)

    def get_concepts_sequences(self):
        """
        Main method that parses the tokens and extract the concepts
        :return:
        """

        def _add_forked_to_concept_parser_helpers():
            # check that if some new InfixToPostfix are created
            for parser in concept_parser_helpers:
                if len(parser.forked) > 0:
                    forked.extend(parser.forked)
                    parser.forked.clear()
            if len(forked) > 0:
                concept_parser_helpers.extend(forked)
                forked.clear()

        def _get_longest(parser_helpers):
            # when there is a match with several concepts
            # on keep the ones that eat the more tokens
            by_end_pos = defaultdict(list)
            for helper in parser_helpers:
                by_end_pos[helper.pos].append(helper)

            return by_end_pos[max(by_end_pos)]

        forked = []

        concept_parser_helpers = [BnfConceptParserHelper(self)]

        while self.parser_input.next_token(False):

            token = self.parser_input.token

            try:
                concepts = self.get_concepts(token, self._is_eligible, strip_quotes=False)

                if not concepts:
                    for concept_parser in concept_parser_helpers:
                        concept_parser.eat_unrecognized(token)
                    continue

                if len(concepts) == 1:
                    for concept_parser in concept_parser_helpers:
                        concept_parser.eat_concept(concepts[0], token)
                    continue

                # make the cartesian product
                temp_res = []
                for concept_parser in concept_parser_helpers:
                    if concept_parser.is_locked():
                        # It means that it already eat the token
                        # so simply add it, do not clone
                        temp_res.append(concept_parser)
                        continue

                    for concept in concepts:
                        clone = concept_parser.clone()
                        temp_res.append(clone)
                        clone.eat_concept(concept, token)

                # only keep the longest
                concept_parser_helpers = _get_longest(temp_res)

            finally:
                _add_forked_to_concept_parser_helpers()

        # make sure that remaining items in stack are moved to out
        for concept_parser in concept_parser_helpers:
            concept_parser.finalize()
            _add_forked_to_concept_parser_helpers()

        return concept_parser_helpers

    def check_for_infinite_recursion(self, parsing_expression, already_found, only_first=False):

        if isinstance(parsing_expression, ConceptExpression):
            if parsing_expression.concept.id in already_found:
                return True
            already_found.add(parsing_expression.concept.id)
            return self.check_for_infinite_recursion(parsing_expression.nodes[0], already_found, only_first)

        if isinstance(parsing_expression, Sequence):
            # for sequence, we need to check all nodes
            if only_first:
                nodes = [] if len(parsing_expression.nodes) == 0 else [parsing_expression.nodes[0]]
            else:
                nodes = parsing_expression.nodes
            for node in nodes:
                already_found_for_current_node = already_found.copy()
                if self.check_for_infinite_recursion(node, already_found_for_current_node, False):
                    already_found.update(already_found_for_current_node)
                    return True
            return False

        if isinstance(parsing_expression, OrderedChoice):
            # for ordered choice, if there is at least one node that does not resolved to a recursion
            # we are safe
            for node in parsing_expression.nodes:
                already_found_for_current_node = already_found.copy()
                if self.check_for_infinite_recursion(node, already_found, True):
                    already_found.update(already_found_for_current_node)
                    return True
                else:
                    return False
            return False

        if isinstance(parsing_expression, LongestChoice):
            for node in parsing_expression.nodes:
                already_found_for_current_node = already_found.copy()
                if self.check_for_infinite_recursion(node, already_found_for_current_node, True):
                    already_found.update(already_found_for_current_node)
                    return True
            return False

        if isinstance(parsing_expression, UnderConstruction):
            if parsing_expression.concept_id in already_found:
                return True
            already_found.add(parsing_expression.concept_id)

        return False

    def get_parsing_expression(self, context, concept):
        """
        Compute the parsing expression for a given concept
        :param context:
        :param concept:
        :return:
        """
        if concept.id in self.concepts_grammars:
            return self.concepts_grammars.get(concept.id)

        # internal cache of already computed parsing expression to use during the recursion
        grammar = {}

        # concept that are not totally resolved, because they reference parsing expression under construction
        to_update = set()  # the key is the instance id of the parsing expression

        # during the parsing of concept, we will resolve other concepts
        # keep the track of the concepts that can safely be added to self.concept_grammars
        to_keep = {concept.id}

        desc = f"Get parsing expression for concept {concept}"
        with context.push(BuiltinConcepts.INIT_BNF, concept,
                          who=self.name,
                          obj=concept,
                          root_concept=concept,
                          desc=desc) as sub_context:
            # get the parsing expression
            ret = self.resolve_concept_parsing_expression(sub_context, concept, grammar, to_update, to_keep)

            # check and update parsing expression that are still under construction
            # Note that we only update the concept that will update concepts_grammars
            # because pe.node may be large
            for item in to_update:
                if item.parent_id in to_keep:
                    pe = item.parsing_expression
                    for i, node in enumerate(pe.nodes):
                        if isinstance(node, UnderConstruction):
                            pe.nodes[i] = grammar.get(node.concept_id)

            # check for infinite recursion.
            # We are adding a new concept. Does it create an infinite recursion ?
            concepts_in_recursion = set()
            if self.check_for_infinite_recursion(ret, concepts_in_recursion):
                cycle = context.sheerka.new(BuiltinConcepts.CHICKEN_AND_EGG, body=concepts_in_recursion)
                for concept_id in concepts_in_recursion:
                    grammar[concept_id] = cycle

            # update, in case of infinite circular recursion
            ret = grammar[concept.id]

            # finally, update concept grammar
            for k, v in grammar.items():
                if k in to_keep:
                    self.concepts_grammars.put(k, v)

                # not quite sure that it is a good idea.
                # Why do we want to corrupt previous valid entries ?
                if context.sheerka.isinstance(v, BuiltinConcepts.CHICKEN_AND_EGG):
                    self.concepts_grammars.put(k, v)

            sub_context.add_values(return_values=ret)

        return ret

    def resolve_concept_parsing_expression(self, context, concept, grammar, to_update, to_keep):
        if concept.id in self.concepts_grammars:  # validated entry
            return self.concepts_grammars.get(concept.id)

        if concept.id in grammar:  # under construction entry
            return grammar.get(concept.id)

        desc = f"Resolve concept parsing expression for '{concept}'"
        with context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as sub_context:
            if not concept.bnf:  # to save a function call. Not sure it worth it.
                BaseNodeParser.ensure_bnf(sub_context, concept, self.name)

            grammar[concept.id] = UnderConstruction(concept.id)
            sheerka = context.sheerka

            if concept.metadata.definition_type == DEFINITION_TYPE_BNF:
                expression = concept.bnf
                desc = f"Bnf concept detected. Resolving parsing expression '{expression}'"
                with sub_context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as ssc:
                    ssc.add_inputs(expression=expression)
                    resolved = self.resolve_parsing_expression(ssc, expression, grammar, to_update, to_keep)
                    ssc.add_values(return_values=resolved)

            elif sheerka.isaset(context, concept):
                desc = f"Concept is a group. Resolving parsing expression using 'isa'"
                with sub_context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as ssc:
                    ssc.add_inputs(concept=concept)
                    concepts_in_group = self.sheerka.get_set_elements(ssc, concept)

                    valid_concepts = []
                    for c in concepts_in_group:
                        if c.id == context.root_concept.id:
                            continue

                        c_pe = self.resolve_concept_parsing_expression(context, c, grammar, to_update, to_keep)
                        if self.check_for_infinite_recursion(c_pe, {concept.id}, True):
                            continue

                        valid_concepts.append(c)

                    nodes = [ConceptExpression(c, rule_name=c.name) for c in valid_concepts]
                    resolved = self.resolve_parsing_expression(ssc,
                                                               LongestChoice(*nodes),
                                                               grammar,
                                                               to_update,
                                                               to_keep)
                    ssc.add_values(concepts_in_group=concepts_in_group)
                    ssc.add_values(return_values=resolved)

            else:
                desc = f"Concept is a simple concept."
                with sub_context.push(BuiltinConcepts.INIT_BNF, concept, who=self.name, obj=concept, desc=desc) as ssc:
                    to_keep.add(concept.id)
                    expression = self.get_expression_from_concept_name(concept.name)
                    resolved = self.resolve_parsing_expression(ssc, expression, grammar, to_update, to_keep)

            grammar[concept.id] = resolved

            if self.has_error:
                sub_context.add_values(errors=self.error_sink)
                return None

            sub_context.add_values(return_values=resolved)
            return resolved

    def resolve_parsing_expression(self, context, expression, grammar, to_update, to_keep):

        if isinstance(expression, str):
            ret = StrMatch(expression, ignore_case=self.ignore_case)

        elif not isinstance(expression, ParsingExpression):
            return expression  # escalate the error

        elif isinstance(expression, ConceptExpression):
            concept = self.get_concept(context, expression.concept)
            expression.concept = concept

            if not self.sheerka.is_known(concept):
                unknown_concept = self.sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=concept)
                return self.add_error(unknown_concept)

            pe = self.resolve_concept_parsing_expression(context, concept, grammar, to_update, to_keep)

            if not isinstance(pe, (ParsingExpression, UnderConstruction)):
                return pe  # an error is detected, escalate it
            #
            # if isinstance(pe, UnderConstruction) and expression.concept.id == pe.concept_id:
            #     return pe  # we are looking for ourself, just return it

            if isinstance(pe, UnderConstruction):
                to_update.add(ToUpdate(context.obj.id, expression))

            expression.nodes = [pe]
            expression.rule_name = expression.rule_name or concept.name
            ret = expression

        elif isinstance(expression, StrMatch):
            ret = expression
            if ret.ignore_case is None:
                ret.ignore_case = self.ignore_case

        elif isinstance(expression, Sequence) or \
            isinstance(expression, OrderedChoice) or \
            isinstance(expression, LongestChoice) or \
            isinstance(expression, ZeroOrMore) or \
            isinstance(expression, OneOrMore) or \
            isinstance(expression, Optional):
            ret = expression
            ret.nodes = []
            for e in ret.elements:
                pe = self.resolve_parsing_expression(context, e, grammar, to_update, to_keep)
                if not isinstance(pe, (ParsingExpression, UnderConstruction)):
                    return pe  # an error is detected, escalate it
                if isinstance(pe, UnderConstruction):
                    to_update.add(ToUpdate(context.obj.id, ret))
                ret.nodes.append(pe)

        else:
            ret = self.add_error(GrammarErrorNode(f"Unrecognized grammar element '{expression}'."), False)

        # Translate separator expression.
        if isinstance(ret, Repetition) and expression.sep:
            expression.sep = self.resolve_parsing_expression(context,
                                                             expression.sep,
                                                             grammar,
                                                             to_update,
                                                             to_keep)

        return ret

    def get_concept(self, context, concept):
        if isinstance(concept, Concept):
            return concept

        if concept in context.concepts:
            return context.concepts[concept]
        return self.sheerka.get_by_key(concept)

    def parse(self, context, parser_input: ParserInput):
        """
        parser_input can be string, but text can also be an list of tokens
        :param context:
        :param parser_input:
        :return:
        """

        if not isinstance(parser_input, ParserInput):
            return None

        context.log(f"Parsing '{parser_input}' with BnfNode", self.name)
        sheerka = context.sheerka

        if parser_input.is_empty():
            return sheerka.ret(self.name,
                               False,
                               sheerka.new(BuiltinConcepts.NOT_FOR_ME,
                                           body=parser_input.as_text(),
                                           reason=BuiltinConcepts.IS_EMPTY))

        if not self.reset_parser(context, parser_input):
            return self.sheerka.ret(
                self.name,
                False,
                context.sheerka.new(BuiltinConcepts.ERROR, body=self.error_sink))

        sequences = self.get_concepts_sequences()
        valid_parser_helpers = self.get_valid(sequences)
        if valid_parser_helpers is None:
            # token error
            return self.sheerka.ret(
                self.name,
                False,
                context.sheerka.new(BuiltinConcepts.ERROR, body=self.error_sink))

        if len(valid_parser_helpers) == 0:
            return self.sheerka.ret(
                self.name,
                False,
                context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text()))

        ret = []
        for parser_helper in valid_parser_helpers:
            ret.append(
                self.sheerka.ret(
                    self.name,
                    not parser_helper.has_unrecognized,
                    self.sheerka.new(
                        BuiltinConcepts.PARSER_RESULT,
                        parser=self,
                        source=parser_input.as_text(),
                        body=parser_helper.sequence,
                        try_parsed=parser_helper.sequence)))

        if len(ret) == 1:
            self.log_result(context, parser_input.as_text(), ret[0])
            return ret[0]
        else:
            self.log_multiple_results(context, parser_input.as_text(), ret)
            return ret