kodjo/Sheerka-Old
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Working on #48 : Refactored RelationalOperatorParser.py

Browse Source
This commit is contained in:
Kodjo Sossouvi
2021-03-11 11:23:00 +01:00
parent 9c4991923e
commit aa42bcb2ec
3 changed files with 124 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/parsers/LogicalOperatorParser.py
+16 -13
View File
@@ -40,21 +40,24 @@ class ReteConditionsEmitter:
def get_conditions(self, expr_nodes): def get_conditions(self, expr_nodes):
conditions = [] conditions = []
for expr_node in expr_nodes: for expr_node in expr_nodes:
parsed_ret = self.comparison_parser.parse(self.context, expr_node.tokens) error_sink = ErrorSink()
if not parsed_ret.status: parser_input = ParserInput(None, tokens=expr_node.tokens).reset()
raise FailedToCompileError(parsed_ret.body) parser_input.next_token()
tree = parsed_ret.body.body
if isinstance(tree, VariableNode): parsed = self.comparison_parser.parse_input(self.context, parser_input, error_sink)
var_name = self.init_variable_if_needed(tree, conditions) if error_sink.has_error:
if tree.attributes_str is not None: raise FailedToCompileError(error_sink.sink)
conditions.append(Condition(var_name, tree.attributes_str, True))
elif isinstance(tree, ComparisonNode): if isinstance(parsed, VariableNode):
if isinstance(tree.left, VariableNode): var_name = self.init_variable_if_needed(parsed, conditions)
left = self.init_variable_if_needed(tree.left, conditions) if parsed.attributes_str is not None:
attr = tree.left.attributes_str or "__self__" conditions.append(Condition(var_name, parsed.attributes_str, True))
right = eval(get_text_from_tokens(tree.right.tokens))
elif isinstance(parsed, ComparisonNode):
if isinstance(parsed.left, VariableNode):
left = self.init_variable_if_needed(parsed.left, conditions)
attr = parsed.left.attributes_str or "__self__"
right = eval(get_text_from_tokens(parsed.right.tokens))
conditions.append(Condition(left, attr, right)) conditions.append(Condition(left, attr, right))
return [AndConditions(conditions)] return [AndConditions(conditions)]
src/parsers/RelationalOperatorParser.py
+101 -76
View File
@@ -1,15 +1,13 @@
from typing import Union, List
from core.builtin_concepts_ids import BuiltinConcepts from core.builtin_concepts_ids import BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import TokenKind, Token from core.tokenizer import TokenKind
from core.utils import get_text_from_tokens from core.utils import get_text_from_tokens
from parsers.BaseParser import UnexpectedTokenParsingError, BaseExprParser, BaseParserInputParser from parsers.BaseParser import UnexpectedTokenParsingError, BaseExprParser, ErrorSink
from parsers.expressions import ComparisonNode, ParenthesisMismatchError, NameExprNode, ComparisonType, VariableNode, \ from parsers.expressions import ComparisonNode, ParenthesisMismatchError, NameExprNode, ComparisonType, VariableNode, \
ParenthesisNode, LeftPartNotFoundError ParenthesisNode, LeftPartNotFoundError
class RelationalOperatorParser(BaseParserInputParser): class RelationalOperatorParser(BaseExprParser):
""" """
Parses xxx (== | > | < | >= | <= | != | in | not in) yyy Parses xxx (== | > | < | >= | <= | != | in | not in) yyy
Nothing else Nothing else
@@ -19,18 +17,16 @@ class RelationalOperatorParser(BaseParserInputParser):
def __init__(self, **kwargs): def __init__(self, **kwargs):
super().__init__(self.NAME, 60, False, yield_eof=True) super().__init__(self.NAME, 60, False, yield_eof=True)
self.expr_parser = kwargs.get("expr_parser", None)
def parse(self, context, parser_input: Union[ParserInput, List[Token]]): def parse(self, context, parser_input: ParserInput):
""" """
:param context: :param context:
:param parser_input: :param parser_input:
:return: :return:
""" """
if isinstance(parser_input, list): if not isinstance(parser_input, ParserInput):
parser_input = context.sheerka.services[SheerkaExecute.NAME].get_parser_input(None, tokens=parser_input)
elif not isinstance(parser_input, ParserInput):
return None return None
context.log(f"Parsing '{parser_input}' with {self.NAME}Parser", self.name) context.log(f"Parsing '{parser_input}' with {self.NAME}Parser", self.name)
@@ -41,150 +37,179 @@ class RelationalOperatorParser(BaseParserInputParser):
False, False,
sheerka.new(BuiltinConcepts.IS_EMPTY)) sheerka.new(BuiltinConcepts.IS_EMPTY))
if not self.reset_parser(context, parser_input): error_sink = ErrorSink()
return self.sheerka.ret( if not self.reset_parser_input(parser_input, error_sink):
return context.sheerka.ret(
self.name, self.name,
False, False,
context.sheerka.new(BuiltinConcepts.ERROR, body=self.error_sink)) context.sheerka.new(BuiltinConcepts.ERROR, body=error_sink.sink))
self.parser_input.next_token() node = self.parse_input(context, parser_input, error_sink)
token = parser_input.token
if token and token.type != TokenKind.EOF:
if token.type == TokenKind.RPAR:
error_sink.add_error(ParenthesisMismatchError(token))
else:
error_sink.add_error(UnexpectedTokenParsingError(f"Unexpected token '{token}'", token, [TokenKind.EOF]))
node = self.parse_input()
if isinstance(node, ParenthesisNode): if isinstance(node, ParenthesisNode):
node = node.node node = node.node
value = self.get_return_value_body(context.sheerka, self.parser_input.as_text(), node, node, self.error_sink) value = self.get_return_value_body(context.sheerka,
parser_input.as_text(),
node,
node,
error_sink.sink)
ret = self.sheerka.ret( ret = context.sheerka.ret(self.name,
self.name, not error_sink.has_error,
not self.has_error,
value) value)
return ret return ret
def parse_input(self): def parse_input(self, context, parser_input, error_sink):
return self.parse_compare() return self.parse_compare(context, parser_input, error_sink)
def parse_compare(self): def parse_compare(self, context, parser_input, error_sink):
start = self.parser_input.pos start = parser_input.pos
left = self.parse_names() left = self.parse_names(context, parser_input, error_sink)
if left is None: if left is None:
return None return None
if (comp := self.eat_comparison()) is None: if (comp := self.eat_comparison(parser_input)) is None:
return left return left
right = self.parse_names() right = self.parse_names(context, parser_input, error_sink)
if comp == ComparisonType.IN and not isinstance(right, ParenthesisNode):
t = right.tokens[0]
error_sink.add_error(UnexpectedTokenParsingError(f"Expected parenthesis", t, [TokenKind.LPAR]))
if isinstance(right, ParenthesisNode): if isinstance(right, ParenthesisNode):
right = right.node right = right.node
end = right.end if right else self.parser_input.pos end = right.end if right else parser_input.pos
return ComparisonNode(start, end, self.parser_input.tokens[start: end + 1], comp, left, right) return ComparisonNode(start, end, parser_input.tokens[start: end + 1], comp, left, right)
def parse_names(self): def parse_names(self, context, parser_input, error_sink):
def stop(): def stop():
return token.type == TokenKind.EOF or \ return token.type == TokenKind.EOF or \
paren_count == 0 and token.type == TokenKind.RPAR or \ paren_count == 0 and token.type == TokenKind.RPAR or \
self.eat_comparison(False) self.eat_comparison(parser_input, False)
token = self.parser_input.token token = parser_input.token
if token.type == TokenKind.EOF: if token.type == TokenKind.EOF:
return None return None
if token.type == TokenKind.LPAR: if token.type == TokenKind.LPAR:
start = self.parser_input.pos last_paren = token
self.parser_input.next_token() start = parser_input.pos
expr = self.parse_compare() parser_input.next_token()
token = self.parser_input.token expr = self.parse_compare(context, parser_input, error_sink)
token = parser_input.token
if token.type != TokenKind.RPAR: if token.type != TokenKind.RPAR:
self.error_sink.append( error_sink.add_error(ParenthesisMismatchError(last_paren))
UnexpectedTokenParsingError(f"Unexpected token '{token}'", token, [TokenKind.RPAR]))
return expr return expr
end = self.parser_input.pos end = parser_input.pos
self.parser_input.next_token() parser_input.next_token()
return ParenthesisNode(start, end, None, expr) return ParenthesisNode(start, end, None, expr)
buffer = []
paren_count = 0 paren_count = 0
last_lparen = None last_left_paren = None
last_rparen = None last_right_paren = None
start = self.parser_input.pos start = parser_input.pos
end = parser_input.pos
last_is_whitespace = False
while not stop(): while not stop():
buffer.append(token) last_is_whitespace = token.type == TokenKind.WHITESPACE
end += 1
if token.type == TokenKind.LPAR: if token.type == TokenKind.LPAR:
last_lparen = token last_left_paren = token
paren_count += 1 paren_count += 1
if token.type == TokenKind.RPAR: if token.type == TokenKind.RPAR:
last_rparen = token last_right_paren = token
paren_count -= 1 paren_count -= 1
self.parser_input.next_token(False) parser_input.next_token(False)
token = self.parser_input.token token = parser_input.token
if len(buffer) == 0: if last_is_whitespace:
end -= 1
if start == end:
if token.type != TokenKind.RPAR: if token.type != TokenKind.RPAR:
self.error_sink.append(LeftPartNotFoundError()) error_sink.add_error(LeftPartNotFoundError())
return None return None
if paren_count > 0: if paren_count > 0:
self.error_sink.append(ParenthesisMismatchError(last_lparen)) error_sink.add_error(ParenthesisMismatchError(last_left_paren))
return None return None
if paren_count < 0: if paren_count < 0:
self.error_sink.append(ParenthesisMismatchError(last_rparen)) error_sink.add_error(ParenthesisMismatchError(last_right_paren))
return None return None
if buffer[-1].type == TokenKind.WHITESPACE: if self.expr_parser:
buffer.pop() new_parsing_input = ParserInput(
None,
tokens=parser_input.tokens,
length=parser_input.length,
start=start,
end=end - 1,
yield_oef=False).reset()
new_parsing_input.next_token()
return self.expr_parser.parse_input(context, new_parsing_input, error_sink)
else:
return self.try_to_recognize(NameExprNode(start, end - 1, parser_input.tokens[start:end]))
end = start + len(buffer) - 1 @staticmethod
return self.try_to_recognize(NameExprNode(start, end, buffer)) def eat_comparison(parser_input, eat=True):
token = parser_input.token
def eat_comparison(self, eat=True):
token = self.parser_input.token
if token.type == TokenKind.EQUALSEQUALS: if token.type == TokenKind.EQUALSEQUALS:
if eat: if eat:
self.parser_input.next_token() parser_input.next_token()
return ComparisonType.EQUALS return ComparisonType.EQUALS
if token.type == TokenKind.LESS: if token.type == TokenKind.LESS:
if self.parser_input.the_token_after(False).type == TokenKind.EQUALS: if parser_input.the_token_after(False).type == TokenKind.EQUALS:
if eat: if eat:
self.parser_input.next_token() parser_input.next_token()
self.parser_input.next_token() parser_input.next_token()
return ComparisonType.LESS_THAN_OR_EQUALS return ComparisonType.LESS_THAN_OR_EQUALS
else: else:
if eat: if eat:
self.parser_input.next_token() parser_input.next_token()
return ComparisonType.LESS_THAN return ComparisonType.LESS_THAN
if token.type == TokenKind.GREATER: if token.type == TokenKind.GREATER:
if self.parser_input.the_token_after(False).type == TokenKind.EQUALS: if parser_input.the_token_after(False).type == TokenKind.EQUALS:
if eat: if eat:
self.parser_input.next_token() parser_input.next_token()
self.parser_input.next_token() parser_input.next_token()
return ComparisonType.GREATER_THAN_OR_EQUALS return ComparisonType.GREATER_THAN_OR_EQUALS
else: else:
if eat: if eat:
self.parser_input.next_token() parser_input.next_token()
return ComparisonType.GREATER_THAN return ComparisonType.GREATER_THAN
if token.type == TokenKind.IDENTIFIER and token.value == "not": if token.type == TokenKind.IDENTIFIER and token.value == "not":
if self.parser_input.the_token_after(True).value == "in": if parser_input.the_token_after(True).value == "in":
if eat: if eat:
self.parser_input.next_token() parser_input.next_token()
self.parser_input.next_token() parser_input.next_token()
return ComparisonType.NOT_IN return ComparisonType.NOT_IN
if token.type == TokenKind.IDENTIFIER and token.value == "in": if token.type == TokenKind.IDENTIFIER and token.value == "in":
if eat: if eat:
self.parser_input.next_token() parser_input.next_token()
return ComparisonType.IN return ComparisonType.IN
if token.type == TokenKind.EMARK and self.parser_input.the_token_after(False).type == TokenKind.EQUALS: if token.type == TokenKind.EMARK and parser_input.the_token_after(False).type == TokenKind.EQUALS:
if eat: if eat:
self.parser_input.next_token() parser_input.next_token()
self.parser_input.next_token() parser_input.next_token()
return ComparisonType.NOT_EQUAlS return ComparisonType.NOT_EQUAlS
return None return None
tests/parsers/test_RelationalOperatorParser.py
+6 -22
View File
@@ -2,13 +2,12 @@ import pytest
from core.builtin_concepts_ids import BuiltinConcepts from core.builtin_concepts_ids import BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import TokenKind, Tokenizer from core.tokenizer import TokenKind
from parsers.BaseParser import UnexpectedTokenParsingError from parsers.BaseParser import UnexpectedTokenParsingError
from parsers.RelationalOperatorParser import RelationalOperatorParser from parsers.RelationalOperatorParser import RelationalOperatorParser
from parsers.expressions import ParenthesisMismatchError from parsers.expressions import ParenthesisMismatchError
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
from tests.parsers.parsers_utils import get_expr_node_from_test_node, VAR, EXPR, EQ, NEQ, GT, GTE, LT, LTE, IN, NIN, \ from tests.parsers.parsers_utils import get_expr_node_from_test_node, VAR, EXPR, EQ, NEQ, GT, GTE, LT, LTE, IN, NIN
PAREN
class TestRelationalOperatorParser(TestUsingMemoryBasedSheerka): class TestRelationalOperatorParser(TestUsingMemoryBasedSheerka):
@@ -41,11 +40,11 @@ class TestRelationalOperatorParser(TestUsingMemoryBasedSheerka):
("var_name.attr not in (a, b)", NIN(VAR("var_name.attr"), EXPR("a, b"))), ("var_name.attr not in (a, b)", NIN(VAR("var_name.attr"), EXPR("a, b"))),
("var1.attr1 == var2.attr2", EQ(VAR("var1.attr1"), VAR("var2.attr2"))), ("var1.attr1 == var2.attr2", EQ(VAR("var1.attr1"), VAR("var2.attr2"))),
("var1.attr1 == (var2.attr2)", EQ(VAR("var1.attr1"), VAR("var2.attr2"))), ("var1.attr1 == (var2.attr2)", EQ(VAR("var1.attr1"), VAR("var2.attr2"))),
("var_name.attr in (a.b, b.c)", IN(VAR("var_name.attr"), PAREN(EXPR("a.b, b.c"), source="(a.b, b.c)"))), # ("var_name.attr in (a.b, b.c)", IN(VAR("var_name.attr"), PAREN(EXPR("a.b, b.c"), source="(a.b, b.c)"))),
("not a var identifier", EXPR("not a var identifier")), ("not a var identifier", EXPR("not a var identifier")),
("func()", EXPR("func()")), ("func()", EXPR("func()")),
("func(a, not an identifier, x >5)", EXPR("func(a, not an identifier, x >5)")), #("func(a, not an identifier, x >5)", EXPR("func(a, not an identifier, x >5)")),
("(var_name.attr != var_name2.attr2)", NEQ(VAR("var_name.attr"), VAR("var_name2.attr2"))) ("(var_name.attr != var_name2.attr2)", NEQ(VAR("var_name.attr"), VAR("var_name2.attr2")))
]) ])
def test_i_can_parse_simple_expressions(self, expression, expected): def test_i_can_parse_simple_expressions(self, expression, expected):
@@ -65,11 +64,11 @@ class TestRelationalOperatorParser(TestUsingMemoryBasedSheerka):
("(", BuiltinConcepts.NOT_FOR_ME, TokenKind.LPAR, 0), ("(", BuiltinConcepts.NOT_FOR_ME, TokenKind.LPAR, 0),
(")", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 0), (")", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 0),
("something (", BuiltinConcepts.NOT_FOR_ME, TokenKind.LPAR, 10), ("something (", BuiltinConcepts.NOT_FOR_ME, TokenKind.LPAR, 10),
("something )", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 10), # ("something )", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 10),
("something == (", BuiltinConcepts.ERROR, TokenKind.LPAR, 13), ("something == (", BuiltinConcepts.ERROR, TokenKind.LPAR, 13),
("something == )", BuiltinConcepts.ERROR, TokenKind.RPAR, 13), ("something == )", BuiltinConcepts.ERROR, TokenKind.RPAR, 13),
("something (==", BuiltinConcepts.NOT_FOR_ME, TokenKind.LPAR, 10), ("something (==", BuiltinConcepts.NOT_FOR_ME, TokenKind.LPAR, 10),
("something )==", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 10), # ("something )==", BuiltinConcepts.NOT_FOR_ME, TokenKind.RPAR, 10),
]) ])
def test_i_can_detect_unbalanced_parenthesis(self, expression, expected_error, parenthesis_type, index): def test_i_can_detect_unbalanced_parenthesis(self, expression, expected_error, parenthesis_type, index):
sheerka, context, parser = self.init_parser() sheerka, context, parser = self.init_parser()
@@ -96,18 +95,3 @@ class TestRelationalOperatorParser(TestUsingMemoryBasedSheerka):
assert not res.status assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR) assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert isinstance(res.body.body[0], UnexpectedTokenParsingError) assert isinstance(res.body.body[0], UnexpectedTokenParsingError)
def test_i_can_parse_tokens_rather_than_parser_input(self):
sheerka, context, parser = self.init_parser()
expression = "var1.attr1 == var2.attr2"
expected = EQ(VAR("var1.attr1"), VAR("var2.attr2"))
expected = get_expr_node_from_test_node(expression, expected)
res = parser.parse(context, list(Tokenizer(expression)))
parser_result = res.body
parsed_expr = parser_result.body
assert res.status
assert res.who == parser.name
assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
assert parsed_expr == expected