kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored sheerka execution flow + Enhanced log management

Browse Source
This commit is contained in:
Kodjo Sossouvi
2019-12-19 21:02:20 +01:00
parent 8dbe2e1b20
commit 5c95d918ad
32 changed files with 942 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files
parsers/BaseParser.py
+26 -8
View File
@@ -1,5 +1,7 @@
from dataclasses import dataclass
from core.tokenizer import TokenKind, Keywords
from core.sheerka_logger import get_logger
import logging
@dataclass()
@@ -34,9 +36,13 @@ class UnexpectedTokenErrorNode(ErrorNode):
class BaseParser:
PREFIX = "Parsers:"
PREFIX = "parsers."
def __init__(self, name):
self.log = get_logger("parsers." + self.__class__.__name__)
self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__)
self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__)
self.name = self.PREFIX + name
self.has_error = False
self.error_sink = []
@@ -52,6 +58,25 @@ class BaseParser:
def parse(self, context, text):
pass
def log_result(self, context, source, ret):
if not self.log.isEnabledFor(logging.DEBUG):
return
if ret.status:
value = context.return_value_to_str(ret)
context.log(self.log, f"Recognized '{source}' as {value}", self.name)
else:
context.log(self.log, f"Failed to recognize '{source}'", self.name)
def log_multiple_results(self, context, source, list_of_ret):
if not self.log.isEnabledFor(logging.DEBUG):
return
context.log(self.log, f"Recognized '{source}' as multiple concepts", self.name)
for r in list_of_ret:
value = context.return_value_to_str(r)
context.log(self.log, f" Recognized '{value}'", self.name)
@staticmethod
def get_text_from_tokens(tokens):
if tokens is None:
@@ -65,10 +90,3 @@ class BaseParser:
value = Keywords(token.value).value if token.type == TokenKind.KEYWORD else token.value
res += value
return res
@staticmethod
def log_result(log, text, ret):
if ret.status:
log.debug(f"Recognized '{text}' as {ret.value}")
else:
log.debug(f"Failed to recognize '{text}'")
parsers/BnfParser.py
+1 -1
View File
@@ -30,7 +30,7 @@ class BnfParser:
def __init__(self):
self.has_error = False
self.error_sink = []
self.name = BaseParser.PREFIX + "RegexParser"
self.name = BaseParser.PREFIX + "Bnf"
self.lexer_iter = None
self._current = None
parsers/ConceptLexerParser.py
+14 -7
View File
@@ -13,9 +13,6 @@ from core.concept import Concept
from core.tokenizer import TokenKind, Tokenizer, Token
from parsers.BaseParser import BaseParser, Node, ErrorNode
import core.utils
import logging
log = logging.getLogger(__name__)
def flatten(iterable):
@@ -588,8 +585,8 @@ class ConceptLexerParser(BaseParser):
return self.sheerka.ret(self.name, True, self.concepts_grammars)
def get_concept(self, concept_name):
if concept_name in self.context.concepts_cache:
return self.context.concepts_cache[concept_name]
if concept_name in self.context.concepts:
return self.context.concepts[concept_name]
return self.sheerka.get(concept_name)
def get_model(self, concept_def, concepts_to_resolve):
@@ -668,6 +665,9 @@ class ConceptLexerParser(BaseParser):
if self.sheerka.isinstance(e, BuiltinConcepts.UNKNOWN_CONCEPT):
continue
if e not in self.concepts_grammars:
continue
to_resolve = self.concepts_grammars[e]
if _is_infinite_recursion(e, to_resolve):
removed_concepts.append(e)
@@ -730,7 +730,7 @@ class ConceptLexerParser(BaseParser):
# manage when nothing is recognized (or other error)
if self.has_error:
return self.sheerka.ret(
ret = self.sheerka.ret(
self.name,
False,
self.sheerka.new(
@@ -739,6 +739,8 @@ class ConceptLexerParser(BaseParser):
source=text,
body=self.error_sink,
try_parsed=concepts_found[0] if len(concepts_found) == 1 else concepts_found))
self.log_result(context, text, ret)
return ret
# else
# returns as many ReturnValue than choices found
@@ -755,7 +757,12 @@ class ConceptLexerParser(BaseParser):
body=choice,
try_parsed=choice)))
return ret[0] if len(ret) == 1 else ret
if len(ret) == 1:
self.log_result(context, text, ret[0])
return ret[0]
else:
self.log_multiple_results(context, text, ret)
return ret
@staticmethod
def get_bests(results):
parsers/DefaultParser.py
+29 -21
View File
@@ -5,11 +5,8 @@ import core.utils
from parsers.BaseParser import BaseParser, Node, ErrorNode, NotInitializedNode
from core.tokenizer import Tokenizer, TokenKind, Token, Keywords
from dataclasses import dataclass, field
import logging
from parsers.BnfParser import BnfParser
log = logging.getLogger(__name__)
from core.sheerka import ExecutionContext
@dataclass()
@@ -207,10 +204,10 @@ class DefaultParser(BaseParser):
"""
def __init__(self, **kwargs):
BaseParser.__init__(self, "DefaultParser")
BaseParser.__init__(self, "Default")
self.lexer_iter = None
self._current = None
self.context = None
self.context: ExecutionContext = None
self.text = None
self.sheerka = None
@@ -287,11 +284,12 @@ class DefaultParser(BaseParser):
def parse(self, context, text):
# default parser can only manage string text
if not isinstance(text, str):
log.debug(f"Failed to recognize '{text}'")
return context.sheerka.ret(
ret = context.sheerka.ret(
self.name,
False,
context.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=text))
self.log_result(context, text, ret)
return ret
self.reset_parser(context, text)
tree = self.parse_statement()
@@ -299,23 +297,29 @@ class DefaultParser(BaseParser):
# If a error is found it must be sent to error_sink
# tree must contain what was recognized
ret = self.sheerka.ret(
self.name,
not self.has_error,
self.sheerka.new(
if self.has_error and isinstance(self.error_sink[0], CannotHandleErrorNode):
body = self.sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=self.error_sink)
else:
body = self.sheerka.new(
BuiltinConcepts.PARSER_RESULT,
parser=self,
source=text,
body=self.error_sink if self.has_error else tree,
try_parsed=tree))
try_parsed=tree)
self.log_result(log, text, ret)
ret = self.sheerka.ret(
self.name,
not self.has_error,
body)
self.log_result(context, text, ret)
return ret
def parse_statement(self):
token = self.get_token()
if token.value == Keywords.DEF:
self.next_token()
self.context.log(self.verbose_log, "Keyword DEF found.", self.name)
return self.parse_def_concept(token)
else:
return self.add_error(CannotHandleErrorNode([], self.text))
@@ -326,7 +330,6 @@ class DefaultParser(BaseParser):
"""
# init
log.debug("It may be a definition of a concept")
keywords_tokens = [def_token]
concept_found = DefConceptNode(keywords_tokens)
@@ -354,7 +357,6 @@ class DefaultParser(BaseParser):
concept_found.post = asts_found_by_parts[Keywords.POST]
concept_found.body = asts_found_by_parts[Keywords.AS]
log.debug(f"Found DefConcept node '{concept_found}'")
return concept_found
def regroup_tokens_by_parts(self, keywords_tokens):
@@ -412,7 +414,8 @@ class DefaultParser(BaseParser):
if TokenKind.NEWLINE in [t.type for t in name_tokens]:
self.add_error(SyntaxErrorNode(tokens_found_by_parts[Keywords.CONCEPT], "Newline are not allowed in name."))
return NameNode(name_tokens[name_first_token_index:]) # skip the first token
name_node = NameNode(name_tokens[name_first_token_index:]) # skip the first token
return name_node
def get_concept_definition(self, tokens_found_by_parts):
if tokens_found_by_parts[Keywords.FROM] is None:
@@ -448,8 +451,6 @@ class DefaultParser(BaseParser):
if keyword == Keywords.CONCEPT or keyword == Keywords.FROM:
continue # already done
log.debug("Processing part '" + keyword.name + "'")
tokens = tokens_found_by_parts[keyword]
if tokens is None:
continue # nothing to do
@@ -464,8 +465,15 @@ class DefaultParser(BaseParser):
continue
# ask the other parsers if they recognize the tokens
new_context = self.context.push(self.name)
parsing_result = core.builtin_helpers.expect_one(new_context, self.sheerka.parse(new_context, tokens))
new_context = self.context.push(self.name, desc=f"Parsing {keyword}")
new_context.log_new(self.verbose_log)
to_parse = self.sheerka.ret(
new_context.who,
True,
self.sheerka.new(BuiltinConcepts.USER_INPUT, body=tokens))
steps = [BuiltinConcepts.PARSING]
parsed = self.sheerka.execute(new_context, to_parse, steps, self.verbose_log)
parsing_result = core.builtin_helpers.expect_one(new_context, parsed)
if not parsing_result.status:
self.add_error(parsing_result.value)
continue
parsers/EmptyStringParser.py
+6 -5
View File
@@ -11,7 +11,7 @@ class EmptyStringParser(BaseParser):
"""
def __init__(self, **kwargs):
BaseParser.__init__(self, "NullParser")
BaseParser.__init__(self, "EmptyString")
def parse(self, context, text):
sheerka = context.sheerka
@@ -19,12 +19,13 @@ class EmptyStringParser(BaseParser):
if isinstance(text, str) and text.strip() == "" or \
isinstance(text, list) and text == [] or \
text is None:
log.debug(f"Recognized '{text}' as BuiltinConcepts.NOP.")
return sheerka.ret(self.name, True, sheerka.new(
ret = sheerka.ret(self.name, True, sheerka.new(
BuiltinConcepts.PARSER_RESULT,
parser=self,
source="",
body=sheerka.new(BuiltinConcepts.NOP)))
else:
ret = sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.NOT_FOR_ME))
log.debug(f"Failed to recognize '{text}'")
return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.NOT_FOR_ME))
self.log_result(context, text, ret)
return ret
parsers/ExactConceptParser.py
+9 -4
View File
@@ -15,7 +15,7 @@ class ExactConceptParser(BaseParser):
MAX_WORDS_SIZE = 10
def __init__(self, **kwargs):
BaseParser.__init__(self, "ConceptParser")
BaseParser.__init__(self, "ExactConcept")
def parse(self, context, text):
"""
@@ -56,14 +56,19 @@ class ExactConceptParser(BaseParser):
source=text if isinstance(text, str) else self.get_text_from_tokens(text),
body=concept,
try_parsed=concept)))
log.debug(f"Recognized '{text}' as '{concept}'")
recognized = True
if recognized:
if len(res) == 1:
self.log_result(context, text, res[0])
else:
self.log_multiple_results(context, text, res)
return res
return res
log.debug(f"Failed to recognize {words}")
return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=text))
ret = sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body=text))
self.log_result(context, text, ret)
return ret
@staticmethod
def get_words(text):
parsers/PythonParser.py
+3 -5
View File
@@ -13,7 +13,7 @@ class PythonErrorNode(ErrorNode):
exception: Exception
# def __post_init__(self):
# log.debug("-> PythonErrorNode: " + str(self.exception))
# self.log.debug("-> PythonErrorNode: " + str(self.exception))
@dataclass()
@@ -58,7 +58,7 @@ class PythonParser(BaseParser):
def __init__(self, **kwargs):
BaseParser.__init__(self, "PythonParser")
BaseParser.__init__(self, "Python")
self.source = kwargs.get("source", "<undef>")
def parse(self, context, text):
@@ -87,7 +87,7 @@ class PythonParser(BaseParser):
body=self.error_sink if self.has_error else PythonNode(text, tree),
try_parsed=None))
self.log_result(log, text, ret)
self.log_result(context, text, ret)
return ret
def try_parse_expression(self, text):
@@ -110,8 +110,6 @@ class PythonGetNamesVisitor(ast.NodeVisitor):
def __init__(self):
self.names = set()
log.debug("Searching for names.")
def visit_Name(self, node):
log.debug(f"Found name : {node.id}")
self.names.add(node.id)