import logging from dataclasses import dataclass from typing import Union from core.builtin_concepts import BuiltinConcepts, ParserResultConcept from core.concept import Concept from core.error import ErrorObj from core.sheerka.ExecutionContext import ExecutionContext from core.sheerka.services.SheerkaExecute import ParserInput from core.sheerka_logger import get_logger from core.tokenizer import TokenKind, Token, Tokenizer, LexerError # # keep a cache for the parser input # pi_cache = Cache(default=lambda key: ParserInput(key), max_size=20) # # # def get_parser_input(text, tokens=None, length=None): # """ # Returns new or existing parser input # :param text: # :param tokens: # :param length: # :return: # """ # if tokens is None or pi_cache.has(text): # return pi_cache.get(text) # pi = ParserInput(text, tokens, length) # pi_cache.put(text, pi) # return pi @dataclass() class Node: pass @dataclass() class NopNode(Node): pass def __repr__(self): return "nop" class NotInitializedNode(Node): pass def __repr__(self): return "**N/A**" @dataclass() class ErrorNode(Node, ErrorObj): pass @dataclass() class UnexpectedTokenErrorNode(ErrorNode): message: str token: Union[Token, str] expected_tokens: list def __eq__(self, other): if id(other) == id(self): return True if not isinstance(other, UnexpectedTokenErrorNode): return False if self.message != other.message: return False to_compare = self.token.repr_value if isinstance(other.token, str) else self.token if to_compare != other.token: return False return self.expected_tokens == other.expected_tokens def __hash__(self): return hash((self.message, self.token, self.expected_tokens)) @dataclass() class UnexpectedEofNode(ErrorNode): message: str class BaseParser: PREFIX = "parsers." def __init__(self, name, priority: int, enabled=True, yield_eof=False): # self.log = get_logger("parsers." + self.__class__.__name__) # self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__) # self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__) self.name = BaseParser.get_name(name) self.short_name = name self.priority = priority self.enabled = enabled self.error_sink = [] self.context: ExecutionContext = None self.sheerka = None self.parser_input: ParserInput = None self.yield_eof = yield_eof def __eq__(self, other): if not isinstance(other, self.__class__): return False return self.name == other.name def __hash__(self): return hash(self.name) def __repr__(self): return self.name def reset_parser(self, context, parser_input: ParserInput): self.context = context self.sheerka = context.sheerka self.parser_input = parser_input self.error_sink.clear() try: self.parser_input.reset(self.yield_eof) except LexerError as e: self.add_error(e, False) return False return True def parse(self, context, parser_input): pass def add_error(self, error, next_token=True): self.error_sink.append(error) if next_token: self.parser_input.next_token() return error @property def has_error(self): return len(self.error_sink) > 0 def log_result(self, context, source, ret): pass # if not self.log.isEnabledFor(logging.DEBUG): # return # # if ret.status: # value = context.return_value_to_str(ret) # context.log(f"Recognized '{source}' as {value}", self.name) # else: # context.log(f"Failed to recognize '{source}'", self.name) def log_multiple_results(self, context, source, list_of_ret): pass # if not self.log.isEnabledFor(logging.DEBUG): # return # # context.log(f"Recognized '{source}' as multiple concepts", self.name) # for r in list_of_ret: # value = context.return_value_to_str(r) # context.log(f" Recognized '{value}'", self.name) def get_return_value_body(self, sheerka, source, parsed, try_parse): """ All parsers must return their result in a standard way :param sheerka: :param source: :param parsed: :param try_parse: :return: """ if len(self.error_sink) == 1 and isinstance(self.error_sink[0], Concept): return self.error_sink[0] if self.has_error: if parsed is None: return sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=source, reason=self.error_sink) else: return sheerka.new(BuiltinConcepts.ERROR, body=self.error_sink) return sheerka.new(BuiltinConcepts.PARSER_RESULT, parser=self, source=source, body=parsed, try_parsed=try_parse) @staticmethod def get_input_as_lexer_nodes(parser_input, expected_parser=None): """ Extract the lexer node from the parser_input :param parser_input: :param expected_parser: returns the nodes if the parent parser is the expected one :return: """ if not isinstance(parser_input, ParserResultConcept): return None if expected_parser and parser_input.parser != expected_parser: return None if len(parser_input.value) == 0: return None for node in parser_input.value: from parsers.BaseNodeParser import LexerNode if not isinstance(node, LexerNode): return None return parser_input.value @staticmethod def manage_eof(lst, strip_eof): if strip_eof: if len(lst) and lst[-1].type == TokenKind.EOF: lst.pop() return lst if len(lst) == 0 or not lst[-1].type == TokenKind.EOF: lst.append(Token(TokenKind.EOF, "", -1, -1, -1)) return lst # @staticmethod # def get_text_from_tokens(tokens, custom_switcher=None, tracker=None): # """ # Create the source code, from the list of token # :param tokens: list of tokens # :param custom_switcher: to override the behaviour (the return value) of some token # :param tracker: keep track of the original token value when custom switched # :return: # """ # if tokens is None: # return "" # res = "" # # if not hasattr(tokens, "__iter__"): # tokens = [tokens] # # switcher = { # # TokenKind.CONCEPT: lambda t: core.utils.str_concept(t.value), # } # # if custom_switcher: # switcher.update(custom_switcher) # # for token in tokens: # value = switcher.get(token.type, lambda t: t.str_value)(token) # res += value # if tracker is not None and token.type in custom_switcher: # tracker[value] = token.value # return res @staticmethod def get_tokens_boundaries(tokens): """ Returns the first and the last valid index of the tokens a valid index is a token that is not a whitespace nor and EOF :param tokens: :return: """ if tokens is None: return None if len(tokens) == 0: return 0, 0 if tokens[0].type == TokenKind.EOF: return 0, 0 start = 1 if tokens[0].type == TokenKind.WHITESPACE else 0 end = len(tokens) - 1 while tokens[end].type in (TokenKind.WHITESPACE, TokenKind.EOF): end -= 1 return start, end @staticmethod def merge_concepts(list_a, b): if not b: return list_a list_b = b if isinstance(b, list) else [b] if not list_a: return list_b by_ids = {c.id for c in list_b} for c in list_b: if c.id in by_ids: # and c.metadata.is_evaluated == by_ids[c.id].metadata.is_evaluated: continue list_a.append(c) by_ids.add(c.id) return list_a @staticmethod def get_name(name): return BaseParser.PREFIX + name class BaseTokenizerIterParser(BaseParser): def __init__(self, name, priority, parse_word=False, none_on_eof=True): super().__init__(name, priority) self.lexer_iter = None self._current = None self.context: ExecutionContext = None self.text = None self.sheerka = None self.parse_word = parse_word self.none_on_eof = none_on_eof def reset_parser(self, context, text): self.context = context self.sheerka = context.sheerka self.text = text self.lexer_iter = iter(Tokenizer(text, self.parse_word)) self._current = None self.next_token() def add_error(self, error, next_token=True): self.error_sink.append(error) if next_token: self.next_token() return error def get_token(self) -> Token: return self._current def next_token(self, skip_whitespace=True): try: self._current = next(self.lexer_iter) if self.none_on_eof and self._current.type == TokenKind.EOF: self._current = None return False if skip_whitespace: while self._current.type == TokenKind.WHITESPACE or self._current.type == TokenKind.NEWLINE: self._current = next(self.lexer_iter) except StopIteration: self._current = None return False return True class BaseSplitIterParser(BaseParser): def __init__(self, name, priority, none_on_eof=False): super().__init__(name, priority) self._current = None self.context: ExecutionContext = None self.text = None self.sheerka = None self.iter_split = None self.split_and_eat_tokens = (" ", "\n", "\t") self.split_and_keep_tokens = ("=", ")", "(", ",") self.split_tokens = self.split_and_eat_tokens + self.split_and_keep_tokens self.none_on_eof = none_on_eof # current token is set to None when EOF is hit def parse_word(self, c, index, line, column): end = self.split_tokens escaped = False buffer = "" while escaped or c not in end: if not escaped and c == "\\": escaped = True elif not escaped and c in ("'", '"'): end = [c] else: buffer += c escaped = False index, column = index + 1, column + 1 if index == len(self.text): break c = self.text[index] if c == "\n": line += 1 column = 0 if c not in self.split_and_keep_tokens: # 'not in' instead of 'in' to when c is a quote index, column = index + 1, column + 1 return buffer, index, line, column def split(self): index = 0 line = 1 column = 1 while index < len(self.text): c = self.text[index] if c == "=": if index + 1 < len(self.text) and self.text[index + 1] == "=": yield Token(TokenKind.EQUALSEQUALS, "==", index, line, column) index, column = index + 2, column + 2 else: yield Token(TokenKind.EQUALS, "=", index, line, column) index, column = index + 1, column + 1 elif c == ")": yield Token(TokenKind.RPAR, ")", index, line, column) index, column = index + 1, column + 1 elif c == "(": yield Token(TokenKind.LPAR, "(", index, line, column) index, column = index + 1, column + 1 elif c == ",": yield Token(TokenKind.COMMA, ",", index, line, column) index, column = index + 1, column + 1 else: buffer, end_index, end_line, end_column = self.parse_word(c, index, line, column) if buffer: yield Token(TokenKind.WORD, buffer, index, line, column) index, line, column = end_index, end_line, end_column yield Token(TokenKind.EOF, "", index, line, column) def reset_parser(self, context, text): self.context = context self.sheerka = context.sheerka if context else None self.text = text self._current = None self.iter_split = iter(self.split()) def add_error(self, error, next_token=True): self.error_sink.append(error) if next_token: self.next_token() return error def get_token(self) -> Token: return self._current def next_token(self): try: self._current = next(self.iter_split) if self._current.type == TokenKind.EOF: if self.none_on_eof: self._current = None return False except StopIteration: self._current = None return False return True