kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Working on #48 : Refactored RelationalOperatorParser.py

Browse Source
This commit is contained in:
Kodjo Sossouvi
2021-03-11 11:23:00 +01:00
parent 9c4991923e
commit aa42bcb2ec
3 changed files with 124 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/parsers/LogicalOperatorParser.py
+16 -13
View File
@@ -40,21 +40,24 @@ class ReteConditionsEmitter:
def get_conditions(self, expr_nodes):
conditions = []
for expr_node in expr_nodes:
parsed_ret = self.comparison_parser.parse(self.context, expr_node.tokens)
if not parsed_ret.status:
raise FailedToCompileError(parsed_ret.body)
tree = parsed_ret.body.body
error_sink = ErrorSink()
parser_input = ParserInput(None, tokens=expr_node.tokens).reset()
parser_input.next_token()
if isinstance(tree, VariableNode):
var_name = self.init_variable_if_needed(tree, conditions)
if tree.attributes_str is not None:
conditions.append(Condition(var_name, tree.attributes_str, True))
parsed = self.comparison_parser.parse_input(self.context, parser_input, error_sink)
if error_sink.has_error:
raise FailedToCompileError(error_sink.sink)
elif isinstance(tree, ComparisonNode):
if isinstance(tree.left, VariableNode):
left = self.init_variable_if_needed(tree.left, conditions)
attr = tree.left.attributes_str or "__self__"
right = eval(get_text_from_tokens(tree.right.tokens))
if isinstance(parsed, VariableNode):
var_name = self.init_variable_if_needed(parsed, conditions)
if parsed.attributes_str is not None:
conditions.append(Condition(var_name, parsed.attributes_str, True))
elif isinstance(parsed, ComparisonNode):
if isinstance(parsed.left, VariableNode):
left = self.init_variable_if_needed(parsed.left, conditions)
attr = parsed.left.attributes_str or "__self__"
right = eval(get_text_from_tokens(parsed.right.tokens))
conditions.append(Condition(left, attr, right))
return [AndConditions(conditions)]
src/parsers/RelationalOperatorParser.py
+101 -76
View File
@@ -1,15 +1,13 @@
from typing import Union, List
from core.builtin_concepts_ids import BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute
from core.tokenizer import TokenKind, Token
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import TokenKind
from core.utils import get_text_from_tokens
from parsers.BaseParser import UnexpectedTokenParsingError, BaseExprParser, BaseParserInputParser
from parsers.BaseParser import UnexpectedTokenParsingError, BaseExprParser, ErrorSink
from parsers.expressions import ComparisonNode, ParenthesisMismatchError, NameExprNode, ComparisonType, VariableNode, \
ParenthesisNode, LeftPartNotFoundError
class RelationalOperatorParser(BaseParserInputParser):
class RelationalOperatorParser(BaseExprParser):
"""
Parses xxx (== | > | < | >= | <= | != | in | not in) yyy
Nothing else
@@ -19,18 +17,16 @@ class RelationalOperatorParser(BaseParserInputParser):
def __init__(self, **kwargs):
super().__init__(self.NAME, 60, False, yield_eof=True)
self.expr_parser = kwargs.get("expr_parser", None)
def parse(self, context, parser_input: Union[ParserInput, List[Token]]):
def parse(self, context, parser_input: ParserInput):
"""
:param context:
:param parser_input:
:return:
"""
if isinstance(parser_input, list):
parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(None, tokens=parser_input)
elif not isinstance(parser_input, ParserInput):
if not isinstance(parser_input, ParserInput):
return None
context.log(f"Parsing '{parser_input}' with {self.NAME}Parser", self.name)
@@ -41,150 +37,179 @@ class RelationalOperatorParser(BaseParserInputParser):
False,
sheerka.new(BuiltinConcepts.IS_EMPTY))
if not self.reset_parser(context, parser_input):
return self.sheerka.ret(
error_sink = ErrorSink()
if not self.reset_parser_input(parser_input, error_sink):
return context.sheerka.ret(
self.name,
False,
context.sheerka.new(BuiltinConcepts.ERROR, body=self.error_sink))
context.sheerka.new(BuiltinConcepts.ERROR, body=error_sink.sink))
self.parser_input.next_token()
node = self.parse_input(context, parser_input, error_sink)
token = parser_input.token
if token and token.type != TokenKind.EOF:
if token.type == TokenKind.RPAR:
error_sink.add_error(ParenthesisMismatchError(token))
else:
error_sink.add_error(UnexpectedTokenParsingError(f"Unexpected token '{token}'", token, [TokenKind.EOF]))
node = self.parse_input()
if isinstance(node, ParenthesisNode):
node = node.node
value = self.get_return_value_body(context.sheerka, self.parser_input.as_text(), node, node, self.error_sink)
value = self.get_return_value_body(context.sheerka,
parser_input.as_text(),
node,
node,
error_sink.sink)
ret = self.sheerka.ret(
self.name,
not self.has_error,
ret = context.sheerka.ret(self.name,
not error_sink.has_error,
value)
return ret
def parse_input(self):
return self.parse_compare()
def parse_input(self, context, parser_input, error_sink):
return self.parse_compare(context, parser_input, error_sink)
def parse_compare(self):
start = self.parser_input.pos
left = self.parse_names()
def parse_compare(self, context, parser_input, error_sink):
start = parser_input.pos
left = self.parse_names(context, parser_input, error_sink)
if left is None:
return None
if (comp := self.eat_comparison()) is None:
if (comp := self.eat_comparison(parser_input)) is None:
return left
right = self.parse_names()
right = self.parse_names(context, parser_input, error_sink)
if comp == ComparisonType.IN and not isinstance(right, ParenthesisNode):
t = right.tokens[0]
error_sink.add_error(UnexpectedTokenParsingError(f"Expected parenthesis", t, [TokenKind.LPAR]))
if isinstance(right, ParenthesisNode):
right = right.node
end = right.end if right else self.parser_input.pos
return ComparisonNode(start, end, self.parser_input.tokens[start: end + 1], comp, left, right)
end = right.end if right else parser_input.pos
return ComparisonNode(start, end, parser_input.tokens[start: end + 1], comp, left, right)
def parse_names(self):
def parse_names(self, context, parser_input, error_sink):
def stop():
return token.type == TokenKind.EOF or \
paren_count == 0 and token.type == TokenKind.RPAR or \
self.eat_comparison(False)
self.eat_comparison(parser_input, False)
token = self.parser_input.token
token = parser_input.token
if token.type == TokenKind.EOF:
return None
if token.type == TokenKind.LPAR:
start = self.parser_input.pos
self.parser_input.next_token()
expr = self.parse_compare()
token = self.parser_input.token
last_paren = token
start = parser_input.pos
parser_input.next_token()
expr = self.parse_compare(context, parser_input, error_sink)
token = parser_input.token
if token.type != TokenKind.RPAR:
self.error_sink.append(
UnexpectedTokenParsingError(f"Unexpected token '{token}'", token, [TokenKind.RPAR]))
error_sink.add_error(ParenthesisMismatchError(last_paren))
return expr
end = self.parser_input.pos
self.parser_input.next_token()
end = parser_input.pos
parser_input.next_token()
return ParenthesisNode(start, end, None, expr)
buffer = []
paren_count = 0
last_lparen = None
last_rparen = None
start = self.parser_input.pos
last_left_paren = None
last_right_paren = None
start = parser_input.pos
end = parser_input.pos
last_is_whitespace = False
while not stop():
buffer.append(token)
last_is_whitespace = token.type == TokenKind.WHITESPACE
end += 1
if token.type == TokenKind.LPAR:
last_lparen = token
last_left_paren = token
paren_count += 1
if token.type == TokenKind.RPAR:
last_rparen = token
last_right_paren = token
paren_count -= 1
self.parser_input.next_token(False)
token = self.parser_input.token
parser_input.next_token(False)
token = parser_input.token
if len(buffer) == 0:
if last_is_whitespace:
end -= 1
if start == end:
if token.type != TokenKind.RPAR:
self.error_sink.append(LeftPartNotFoundError())
error_sink.add_error(LeftPartNotFoundError())
return None
if paren_count > 0:
self.error_sink.append(ParenthesisMismatchError(last_lparen))
error_sink.add_error(ParenthesisMismatchError(last_left_paren))
return None
if paren_count < 0:
self.error_sink.append(ParenthesisMismatchError(last_rparen))
error_sink.add_error(ParenthesisMismatchError(last_right_paren))
return None
if buffer[-1].type == TokenKind.WHITESPACE:
buffer.pop()
if self.expr_parser:
new_parsing_input = ParserInput(
None,
tokens=parser_input.tokens,
length=parser_input.length,
start=start,
end=end - 1,
yield_oef=False).reset()
new_parsing_input.next_token()
return self.expr_parser.parse_input(context, new_parsing_input, error_sink)
else:
return self.try_to_recognize(NameExprNode(start, end - 1, parser_input.tokens[start:end]))
end = start + len(buffer) - 1
return self.try_to_recognize(NameExprNode(start, end, buffer))
def eat_comparison(self, eat=True):
token = self.parser_input.token
@staticmethod
def eat_comparison(parser_input, eat=True):
token = parser_input.token
if token.type == TokenKind.EQUALSEQUALS:
if eat:
self.parser_input.next_token()
parser_input.next_token()
return ComparisonType.EQUALS
if token.type == TokenKind.LESS:
if self.parser_input.the_token_after(False).type == TokenKind.EQUALS:
if parser_input.the_token_after(False).type == TokenKind.EQUALS:
if eat:
self.parser_input.next_token()
self.parser_input.next_token()
parser_input.next_token()
parser_input.next_token()
return ComparisonType.LESS_THAN_OR_EQUALS
else:
if eat:
self.parser_input.next_token()
parser_input.next_token()
return ComparisonType.LESS_THAN
if token.type == TokenKind.GREATER:
if self.parser_input.the_token_after(False).type == TokenKind.EQUALS:
if parser_input.the_token_after(False).type == TokenKind.EQUALS:
if eat:
self.parser_input.next_token()
self.parser_input.next_token()
parser_input.next_token()
parser_input.next_token()
return ComparisonType.GREATER_THAN_OR_EQUALS
else:
if eat:
self.parser_input.next_token()
parser_input.next_token()
return ComparisonType.GREATER_THAN
if token.type == TokenKind.IDENTIFIER and token.value == "not":
if self.parser_input.the_token_after(True).value == "in":
if parser_input.the_token_after(True).value == "in":
if eat:
self.parser_input.next_token()
self.parser_input.next_token()
parser_input.next_token()
parser_input.next_token()
return ComparisonType.NOT_IN
if token.type == TokenKind.IDENTIFIER and token.value == "in":
if eat:
self.parser_input.next_token()
parser_input.next_token()
return ComparisonType.IN
if token.type == TokenKind.EMARK and self.parser_input.the_token_after(False).type == TokenKind.EQUALS:
if token.type == TokenKind.EMARK and parser_input.the_token_after(False).type == TokenKind.EQUALS:
if eat:
self.parser_input.next_token()
self.parser_input.next_token()
parser_input.next_token()
parser_input.next_token()
return ComparisonType.NOT_EQUAlS
return None
tests/parsers/test_RelationalOperatorParser.py
+6 -22
View File
@@ -2,13 +2,12 @@ import pytest
from core.builtin_concepts_ids import BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import TokenKind, Tokenizer
from core.tokenizer import TokenKind
from parsers.BaseParser import UnexpectedTokenParsingError
from parsers.RelationalOperatorParser import RelationalOperatorParser
from parsers.expressions import ParenthesisMismatchError
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import get_expr_node_from_test_node, VAR, EXPR, EQ, NEQ, GT, GTE, LT, LTE, IN, NIN, \
PAREN
from tests.parsers.parsers_utils import get_expr_node_from_test_node, VAR, EXPR, EQ, NEQ, GT, GTE, LT, LTE, IN, NIN
class TestRelationalOperatorParser(TestUsingMemoryBasedSheerka):
@@ -41,11 +40,11 @@ class TestRelationalOperatorParser(TestUsingMemoryBasedSheerka):
("var_name.attr not in (a, b)", NIN(VAR("var_name.attr"), EXPR("a, b"))),
("var1.attr1 == var2.attr2", EQ(VAR("var1.attr1"), VAR("var2.attr2"))),
("var1.attr1 == (var2.attr2)", EQ(VAR("var1.attr1"), VAR("var2.attr2"))),
("var_name.attr in (a.b, b.c)", IN(VAR("var_name.attr"), PAREN(EXPR("a.b, b.c"), source="(a.b, b.c)"))),
# ("var_name.attr in (a.b, b.c)", IN(VAR("var_name.attr"), PAREN(EXPR("a.b, b.c"), source="(a.b, b.c)"))),
("not a var identifier", EXPR("not a var identifier")),
("func()", EXPR("func()")),
("func(a, not an identifier, x >5)", EXPR("func(a, not an identifier, x >5)")),
#("func(a, not an identifier, x >5)", EXPR("func(a, not an identifier, x >5)")),
("(var_name.attr != var_name2.attr2)", NEQ(VAR("var_name.attr"), VAR("var_name2.attr2")))
])
def test_i_can_parse_simple_expressions(self, expression, expected):
@@ -65,11 +64,11 @@ class TestRelationalOperatorParser(TestUsingMemoryBasedSheerka):
("(", BuiltinConcepts.NOT_FOR_ME, TokenKind.LPAR, 0),
(")", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 0),
("something (", BuiltinConcepts.NOT_FOR_ME, TokenKind.LPAR, 10),
("something )", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 10),
# ("something )", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 10),
("something == (", BuiltinConcepts.ERROR, TokenKind.LPAR, 13),
("something == )", BuiltinConcepts.ERROR, TokenKind.RPAR, 13),
("something (==", BuiltinConcepts.NOT_FOR_ME, TokenKind.LPAR, 10),
("something )==", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 10),
# ("something )==", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 10),
])
def test_i_can_detect_unbalanced_parenthesis(self, expression, expected_error, parenthesis_type, index):
sheerka, context, parser = self.init_parser()
@@ -96,18 +95,3 @@ class TestRelationalOperatorParser(TestUsingMemoryBasedSheerka):
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert isinstance(res.body.body[0], UnexpectedTokenParsingError)
def test_i_can_parse_tokens_rather_than_parser_input(self):
sheerka, context, parser = self.init_parser()
expression = "var1.attr1 == var2.attr2"
expected = EQ(VAR("var1.attr1"), VAR("var2.attr2"))
expected = get_expr_node_from_test_node(expression, expected)
res = parser.parse(context, list(Tokenizer(expression)))
parser_result = res.body
parsed_expr = parser_result.body
assert res.status
assert res.who == parser.name
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert parsed_expr == expected