From 07f0d3670de0e808204ffa14b74a9930f141c08f Mon Sep 17 00:00:00 2001 From: Kodjo Sossouvi Date: Tue, 9 Mar 2021 09:33:50 +0100 Subject: [PATCH] Working on #48 : Working --- src/parsers/LogicalOperatorParser.py | 10 +++-- ...nParser.py => RelationalOperatorParser.py} | 38 ++++++++++++++++--- src/parsers/expressions.py | 10 ++++- tests/parsers/parsers_utils.py | 31 ++++++++++----- tests/parsers/test_LogicalOperatorParser.py | 3 ++ ...er.py => test_RelationalOperatorParser.py} | 28 +++++++++++--- 6 files changed, 94 insertions(+), 26 deletions(-) rename src/parsers/{ComparisonParser.py => RelationalOperatorParser.py} (82%) rename tests/parsers/{test_ComparisonParser.py => test_RelationalOperatorParser.py} (83%) diff --git a/src/parsers/LogicalOperatorParser.py b/src/parsers/LogicalOperatorParser.py index d1d6e6e..1ae22b5 100644 --- a/src/parsers/LogicalOperatorParser.py +++ b/src/parsers/LogicalOperatorParser.py @@ -18,9 +18,9 @@ from sheerkarete.conditions import Condition, AndConditions class ReteConditionsEmitter: def __init__(self, context): - from parsers.ComparisonParser import ComparisonParser + from parsers.RelationalOperatorParser import RelationalOperatorParser self.context = context - self.comparison_parser = ComparisonParser() + self.comparison_parser = RelationalOperatorParser() self.var_counter = 0 self.variables = {} @@ -69,8 +69,10 @@ class LogicalOperatorParser(BaseParser): Or to help to understand why a python expression returns True or False """ + NAME = "LogicalOperator" + def __init__(self, **kwargs): - super().__init__("Expression", 50, False, yield_eof=True) + super().__init__(self.NAME, 50, False, yield_eof=True) self.and_tokens = list(Tokenizer(" and ", yield_eof=False)) self.and_not_tokens = list(Tokenizer(" and not ", yield_eof=False)) self.not_tokens = list(Tokenizer("not ", yield_eof=False)) @@ -111,6 +113,8 @@ class LogicalOperatorParser(BaseParser): if token and token.type != TokenKind.EOF: self.add_error(UnexpectedTokenParsingError(f"Unexpected token '{token}'", token, [])) + if isinstance(tree, ParenthesisNode): + tree = tree.node value = self.get_return_value_body(context.sheerka, self.parser_input.as_text(), tree, tree) ret = self.sheerka.ret( diff --git a/src/parsers/ComparisonParser.py b/src/parsers/RelationalOperatorParser.py similarity index 82% rename from src/parsers/ComparisonParser.py rename to src/parsers/RelationalOperatorParser.py index 5d4b08b..d8519f2 100644 --- a/src/parsers/ComparisonParser.py +++ b/src/parsers/RelationalOperatorParser.py @@ -4,11 +4,12 @@ from core.builtin_concepts_ids import BuiltinConcepts from core.sheerka.services.SheerkaExecute import ParserInput, SheerkaExecute from core.tokenizer import TokenKind, Token from core.utils import get_text_from_tokens -from parsers.BaseParser import BaseParser -from parsers.expressions import ComparisonNode, ParenthesisMismatchError, NameExprNode, ComparisonType, VariableNode +from parsers.BaseParser import BaseParser, UnexpectedTokenParsingError +from parsers.expressions import ComparisonNode, ParenthesisMismatchError, NameExprNode, ComparisonType, VariableNode, \ + ParenthesisNode, LeftPartNotFoundError -class ComparisonParser(BaseParser): +class RelationalOperatorParser(BaseParser): """ Parses xxx (== | > | < | >= | <= | != | in | not in) yyy Nothing else @@ -47,6 +48,8 @@ class ComparisonParser(BaseParser): self.parser_input.next_token() node = self.parse_compare() + if isinstance(node, ParenthesisNode): + node = node.node value = self.get_return_value_body(context.sheerka, self.parser_input.as_text(), node, node) @@ -67,21 +70,42 @@ class ComparisonParser(BaseParser): return left right = self.parse_names() + if isinstance(right, ParenthesisNode): + right = right.node + end = right.end if right else self.parser_input.pos return ComparisonNode(start, end, self.parser_input.tokens[start: end + 1], comp, left, right) def parse_names(self): + def stop(): + return token.type == TokenKind.EOF or \ + paren_count == 0 and token.type == TokenKind.RPAR or \ + self.eat_comparison(False) + token = self.parser_input.token if token.type == TokenKind.EOF: return None + if token.type == TokenKind.LPAR: + start = self.parser_input.pos + self.parser_input.next_token() + expr = self.parse_compare() + token = self.parser_input.token + if token.type != TokenKind.RPAR: + self.error_sink.append( + UnexpectedTokenParsingError(f"Unexpected token '{token}'", token, [TokenKind.RPAR])) + return expr + end = self.parser_input.pos + self.parser_input.next_token() + return ParenthesisNode(start, end, None, expr) + buffer = [] paren_count = 0 last_lparen = None last_rparen = None start = self.parser_input.pos - while (paren_count > 0 or not self.eat_comparison(False)) and token.type != TokenKind.EOF: + while not stop(): buffer.append(token) if token.type == TokenKind.LPAR: last_lparen = token @@ -92,8 +116,10 @@ class ComparisonParser(BaseParser): self.parser_input.next_token(False) token = self.parser_input.token - if paren_count != 0: - pass + if len(buffer) == 0: + if token.type != TokenKind.RPAR: + self.error_sink.append(LeftPartNotFoundError()) + return None if paren_count > 0: self.error_sink.append(ParenthesisMismatchError(last_lparen)) diff --git a/src/parsers/expressions.py b/src/parsers/expressions.py index f0e5225..d2c5be6 100644 --- a/src/parsers/expressions.py +++ b/src/parsers/expressions.py @@ -224,14 +224,20 @@ class ParenthesisNode(ExprNode): if self.start != other.start or self.end != other.end: return False - if other.tokens is not None and other.tokens != self.tokens: - return False + # if other.tokens is not None and other.tokens != self.tokens: + # return False return self.node == other.node def __hash__(self): return hash((self.start, self.end, self.node)) + def __repr__(self): + return f"ParenthesisNode(start={self.start}, end={self.end}, node={self.node!r})" + + def __str__(self): + return f"({self.node})" + class VariableNode(ExprNode): def __init__(self, start, end, tokens, name, *attributes): diff --git a/tests/parsers/parsers_utils.py b/tests/parsers/parsers_utils.py index f441bae..0d01124 100644 --- a/tests/parsers/parsers_utils.py +++ b/tests/parsers/parsers_utils.py @@ -13,7 +13,8 @@ from parsers.BaseNodeParser import UnrecognizedTokensNode, SourceCodeNode, RuleN from parsers.FunctionParser import FunctionNode from parsers.PythonParser import PythonNode from parsers.SyaNodeParser import SyaConceptParserHelper -from parsers.expressions import NameExprNode, AndNode, OrNode, NotNode, VariableNode, ComparisonNode, ComparisonType +from parsers.expressions import NameExprNode, AndNode, OrNode, NotNode, VariableNode, ComparisonNode, ComparisonType, \ + ParenthesisNode from sheerkarete.common import V from sheerkarete.conditions import Condition, AndConditions @@ -66,56 +67,62 @@ class VAR: class EQ: left: object right: object - source = None + source: str = None @dataclass class NEQ: left: object right: object - source = None + source: str = None @dataclass class GT: left: object right: object - source = None + source: str = None @dataclass class GTE: left: object right: object - source = None + source: str = None @dataclass class LT: left: object right: object - source = None + source: str = None @dataclass class LTE: left: object right: object - source = None + source: str = None @dataclass class IN: left: object right: object - source = None + source: str = None @dataclass class NIN: # for NOT INT left: object right: object - source = None + source: str = None + + +@dataclass +class PAREN: # for parenthesis node + node: object + source: str = None class CC: @@ -1010,6 +1017,12 @@ def get_expr_node_from_test_node(full_text, test_node): return ComparisonNode(start, end, full_text_as_tokens[start: end + 1], node_type, left_node, right_node) + if isinstance(node, PAREN): + value_as_tokens = list(Tokenizer(node.source, yield_eof=False)) + start = tokens_index(full_text_as_tokens, value_as_tokens, 0) + end = start + len(value_as_tokens) - 1 + return ParenthesisNode(start, end, value_as_tokens, get_expr_node(node.node)) + return get_expr_node(test_node) diff --git a/tests/parsers/test_LogicalOperatorParser.py b/tests/parsers/test_LogicalOperatorParser.py index 0800380..2810915 100644 --- a/tests/parsers/test_LogicalOperatorParser.py +++ b/tests/parsers/test_LogicalOperatorParser.py @@ -49,6 +49,9 @@ class TestLogicalOperatorParser(TestUsingMemoryBasedSheerka): ("not ((a and b) or (c and d))", NOT(OR(AND(EXPR("a"), EXPR("b")), AND(EXPR("c"), EXPR("d")), source="(a and b) or (c and d)"), source="not ((a and b) or (c and d))")), + ("(one and two)", AND(EXPR("one"), EXPR("two"))), + ("(one or two)", OR(EXPR("one"), EXPR("two"))), + ("(not one)", NOT(EXPR("one"))), ]) def test_i_can_parse_expression(self, expression, expected): sheerka, context, parser = self.init_parser() diff --git a/tests/parsers/test_ComparisonParser.py b/tests/parsers/test_RelationalOperatorParser.py similarity index 83% rename from tests/parsers/test_ComparisonParser.py rename to tests/parsers/test_RelationalOperatorParser.py index d8bfcec..6e63b5e 100644 --- a/tests/parsers/test_ComparisonParser.py +++ b/tests/parsers/test_RelationalOperatorParser.py @@ -3,16 +3,17 @@ import pytest from core.builtin_concepts_ids import BuiltinConcepts from core.sheerka.services.SheerkaExecute import ParserInput from core.tokenizer import TokenKind, Tokenizer -from parsers.ComparisonParser import ComparisonParser +from parsers.RelationalOperatorParser import RelationalOperatorParser from parsers.expressions import ParenthesisMismatchError from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka -from tests.parsers.parsers_utils import get_expr_node_from_test_node, VAR, EXPR, EQ, NEQ, GT, GTE, LT, LTE, IN, NIN +from tests.parsers.parsers_utils import get_expr_node_from_test_node, VAR, EXPR, EQ, NEQ, GT, GTE, LT, LTE, IN, NIN, \ + PAREN -class TestComparisonParser(TestUsingMemoryBasedSheerka): +class TestRelationalOperatorParser(TestUsingMemoryBasedSheerka): def init_parser(self): sheerka, context = self.init_concepts() - parser = ComparisonParser() + parser = RelationalOperatorParser() return sheerka, context, parser def test_i_can_detect_empty_expression(self): @@ -35,13 +36,16 @@ class TestComparisonParser(TestUsingMemoryBasedSheerka): ("var_name.attr >= 10", GTE(VAR("var_name.attr"), EXPR("10"))), ("var_name.attr < 10", LT(VAR("var_name.attr"), EXPR("10"))), ("var_name.attr <= 10", LTE(VAR("var_name.attr"), EXPR("10"))), - ("var_name.attr in (a, b)", IN(VAR("var_name.attr"), EXPR("(a, b)"))), - ("var_name.attr not in (a, b)", NIN(VAR("var_name.attr"), EXPR("(a, b)"))), + ("var_name.attr in (a, b)", IN(VAR("var_name.attr"), EXPR("a, b"))), + ("var_name.attr not in (a, b)", NIN(VAR("var_name.attr"), EXPR("a, b"))), ("var1.attr1 == var2.attr2", EQ(VAR("var1.attr1"), VAR("var2.attr2"))), + ("var1.attr1 == (var2.attr2)", EQ(VAR("var1.attr1"), VAR("var2.attr2"))), + ("var_name.attr in (a.b, b.c)", IN(VAR("var_name.attr"), PAREN(EXPR("a.b, b.c"), source="(a.b, b.c)"))), ("not a var identifier", EXPR("not a var identifier")), ("func()", EXPR("func()")), ("func(a, not an identifier, x >5)", EXPR("func(a, not an identifier, x >5)")), + ("(var_name.attr != var_name2.attr2)", NEQ(VAR("var_name.attr"), VAR("var_name2.attr2"))) ]) def test_i_can_parse_simple_expressions(self, expression, expected): sheerka, context, parser = self.init_parser() @@ -82,6 +86,18 @@ class TestComparisonParser(TestUsingMemoryBasedSheerka): assert res.body.body[0].token.type == parenthesis_type assert res.body.body[0].token.index == index + def test_i_can_detect_syntax_error(self): + sheerka, context, parser = self.init_parser() + + expression = "var in a" + res = parser.parse(context, ParserInput(expression)) + + assert not res.status + assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR) + assert isinstance(res.body.body[0], UnexpectedTokenError) + + + def test_i_can_parse_tokens_rather_than_parser_input(self): sheerka, context, parser = self.init_parser() expression = "var1.attr1 == var2.attr2"