kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

I can manage infinite recursion when building concept

Browse Source
This commit is contained in:
Kodjo Sossouvi
2020-12-10 21:08:10 +01:00
parent 4b6e1dd55b
commit 657c7536f7
28 changed files with 816 additions and 446 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/parsers/BaseNodeParser.py
-8
View File
@@ -824,14 +824,6 @@ class BaseNodeParser(BaseParser):
else:
self.concepts_by_first_keyword = None
# self.token = None
# self.pos = -1
# self.tokens = None
#
# self.context: ExecutionContext = None
# self.text = None
# self.sheerka = None
def init_from_concepts(self, context, concepts, **kwargs):
"""
Initialize the parser with a list of concepts
src/parsers/BaseParser.py
+6 -1
View File
@@ -94,7 +94,8 @@ class BaseParser:
# self.init_log = get_logger("init." + self.PREFIX + self.__class__.__name__)
# self.verbose_log = get_logger("verbose." + self.PREFIX + self.__class__.__name__)
self.name = self.PREFIX + name
self.name = BaseParser.get_name(name)
self.short_name = name
self.priority = priority
self.enabled = enabled
@@ -298,6 +299,10 @@ class BaseParser:
return list_a
@staticmethod
def get_name(name):
return BaseParser.PREFIX + name
class BaseTokenizerIterParser(BaseParser):
src/parsers/BnfDefinitionParser.py
+3 -1
View File
@@ -48,9 +48,11 @@ class BnfDefinitionParser(BaseParser):
return True
def reset_parser(self, context, text):
self.error_sink.clear()
self.context = context
self.sheerka = context.sheerka
self.source = ""
self.lexer_iter = iter(Tokenizer(text.strip())) if isinstance(text, str) else iter(text)
self._current = None
self.after_current = None
src/parsers/ExactConceptParser.py
+6 -7
View File
@@ -1,10 +1,8 @@
import logging
import core.builtin_helpers
from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
from core.concept import VARIABLE_PREFIX
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import TokenKind, LexerError
from core.tokenizer import TokenKind
from core.utils import str_concept
from parsers.BaseParser import BaseParser
@@ -31,12 +29,13 @@ class ExactConceptParser(BaseParser):
context.log(f"Parsing '{parser_input}'", self.name)
sheerka = context.sheerka
try:
if self.reset_parser(context, parser_input):
parser_input.reset()
words = self.get_words(parser_input)
except LexerError as e:
context.log(f"Error found in tokenizer {e}", self.name)
return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR, body=e))
else:
error = self.error_sink[0]
context.log(f"Error found in tokenizer {error}", self.name)
return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR, body=error))
if len(words) > (self.max_word_size or self.MAX_WORDS_SIZE):
context.log(f"Max words reached. Stopping.", self.name)
src/parsers/PythonParser.py
+10 -13
View File
@@ -5,7 +5,7 @@ from dataclasses import dataclass
import core.utils
from core.builtin_concepts import BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import LexerError, TokenKind
from core.tokenizer import TokenKind
from parsers.BaseParser import BaseParser, Node, ErrorNode
log = logging.getLogger(__name__)
@@ -101,9 +101,11 @@ class PythonParser(BaseParser):
Parse Python scripts
"""
NAME = "Python"
def __init__(self, **kwargs):
BaseParser.__init__(self, "Python", 50)
BaseParser.__init__(self, PythonParser.NAME, 50)
self.source = kwargs.get("source", "<undef>")
def parse(self, context, parser_input: ParserInput):
@@ -119,9 +121,7 @@ class PythonParser(BaseParser):
TokenKind.RULE: lambda t: core.utils.encode_concept(t.value, "R")
}
try:
parser_input.reset()
if self.reset_parser(context, parser_input):
source_code = parser_input.as_text(python_switcher, tracker)
source_code = source_code.strip()
@@ -134,14 +134,11 @@ class PythonParser(BaseParser):
error_node = PythonErrorNode(parser_input.as_text(), error)
self.error_sink.append(error_node)
except LexerError as e:
self.error_sink.append(e)
# Python parser will refuse input that directly refers to a concept
if isinstance(tree, ast.Expression) and isinstance(tree.body, ast.Name):
if tree.body.id in tracker or context.sheerka.fast_resolve(tree.body.id, return_new=False) is not None:
context.log("It's a simple concept. Not for me.", self.name)
self.error_sink.append(ConceptDetected(tree.body.id))
# Python parser will refuse input that directly refers to a concept
if isinstance(tree, ast.Expression) and isinstance(tree.body, ast.Name):
if tree.body.id in tracker or context.sheerka.fast_resolve(tree.body.id, return_new=False) is not None:
context.log("It's a simple concept. Not for me.", self.name)
self.error_sink.append(ConceptDetected(tree.body.id))
if self.has_error:
ret = sheerka.ret(
src/parsers/PythonWithConceptsParser.py
+1
View File
@@ -34,6 +34,7 @@ class PythonWithConceptsParser(BaseParser):
yield node
def parse(self, context, parser_input):
self.error_sink.clear()
nodes = self.get_input_as_lexer_nodes(parser_input, unrecognized_nodes_parser)
return self.parse_nodes(context, nodes)
src/parsers/RuleParser.py
+38 -40
View File
@@ -1,7 +1,7 @@
from core.builtin_concepts import BuiltinConcepts
from core.rule import Rule, ACTION_TYPE_DEFERRED
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import LexerError, TokenKind
from core.tokenizer import TokenKind
from parsers.BaseParser import BaseParser, ErrorNode, UnexpectedTokenErrorNode
@@ -40,49 +40,47 @@ class RuleParser(BaseParser):
False,
sheerka.new(BuiltinConcepts.IS_EMPTY))
try:
parser_input.reset()
if not self.reset_parser(context, parser_input):
error = self.error_sink[0]
context.log(f"Error found in tokenizer {error}", self.name)
return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR, body=error))
parser_input.next_token()
if parser_input.token.type != TokenKind.RULE:
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text()))
parser_input.next_token()
if parser_input.token.type != TokenKind.RULE:
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text()))
token = parser_input.token
token = parser_input.token
if parser_input.next_token():
reason = UnexpectedTokenErrorNode("Only one rule supported",
parser_input.token,
[TokenKind.EOF])
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text(), reason=reason))
if parser_input.next_token():
reason = UnexpectedTokenErrorNode("Only one rule supported",
parser_input.token,
[TokenKind.EOF])
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.NOT_FOR_ME, body=parser_input.as_text(), reason=reason))
if token.value[1] is None:
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.NOT_IMPLEMENTED))
if token.value[1] is None:
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.NOT_IMPLEMENTED))
if token.value[1].isdigit():
rule = sheerka.get_rule_by_id(token.value[1])
else:
rule = Rule().set_id(token.value[1])
rule.metadata.action_type = ACTION_TYPE_DEFERRED
if token.value[1].isdigit():
rule = sheerka.get_rule_by_id(token.value[1])
else:
rule = Rule().set_id(token.value[1])
rule.metadata.action_type = ACTION_TYPE_DEFERRED
if sheerka.isinstance(rule, BuiltinConcepts.UNKNOWN_RULE):
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.ERROR,
body=[RuleNotFound(token.value)]))
body = sheerka.new(BuiltinConcepts.PARSER_RESULT,
parser=self,
source=parser_input.as_text(),
body=[rule],
try_parsed=[rule])
if sheerka.isinstance(rule, BuiltinConcepts.UNKNOWN_RULE):
return sheerka.ret(self.name,
False,
sheerka.new(BuiltinConcepts.ERROR,
body=[RuleNotFound(token.value)]))
body = sheerka.new(BuiltinConcepts.PARSER_RESULT,
parser=self,
source=parser_input.as_text(),
body=[rule],
try_parsed=[rule])
return sheerka.ret(self.name, True, body)
except LexerError as e:
context.log(f"Error found in tokenizer {e}", self.name)
return sheerka.ret(self.name, False, sheerka.new(BuiltinConcepts.ERROR, body=e))
return sheerka.ret(self.name, True, body)