Implemented some enhancement requests

This commit is contained in:
2020-12-14 10:30:10 +01:00
parent 657c7536f7
commit e3c2adb533
46 changed files with 352 additions and 1286 deletions
@@ -2,7 +2,7 @@ import pytest
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Keywords, Tokenizer, TokenKind
from parsers.BaseCustomGrammarParser import BaseCustomGrammarParser, SyntaxErrorNode, KeywordNotFound
from parsers.BaseParser import UnexpectedEofNode, UnexpectedTokenErrorNode
from parsers.BaseParser import UnexpectedEofParsingError, UnexpectedTokenParsingError
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -83,9 +83,9 @@ func(a)
sheerka, context, parser = self.init_parser("when xxx print yyy")
assert parser.get_parts(["when", "print"], Keywords.PRINT) is None
assert parser.error_sink == [UnexpectedTokenErrorNode(f"'print' keyword not found.",
assert parser.error_sink == [UnexpectedTokenParsingError(f"'print' keyword not found.",
"when",
[Keywords.PRINT])]
[Keywords.PRINT])]
def test_i_can_detect_when_a_keyword_appears_several_times(self):
sheerka, context, parser = self.init_parser("print hello when True print True")
@@ -106,7 +106,7 @@ func(a)
assert parser.get_parts(["print", "when"]) is not None
assert len(parser.error_sink) == 1
assert isinstance(parser.error_sink[0], UnexpectedEofNode)
assert isinstance(parser.error_sink[0], UnexpectedEofParsingError)
assert parser.error_sink[0].message == "While parsing keyword 'print'."
def test_i_can_double_quoted_strings_are_expanded(self):
@@ -178,7 +178,7 @@ print xxx"""
sheerka, context, parser = self.init_parser(text)
assert parser.get_parts(["when"])
assert parser.error_sink == [UnexpectedTokenErrorNode("Indentation not found.", "x", [TokenKind.WHITESPACE])]
assert parser.error_sink == [UnexpectedTokenParsingError("Indentation not found.", "x", [TokenKind.WHITESPACE])]
@pytest.mark.parametrize("text", [
"",
@@ -197,7 +197,7 @@ print xxx"""
sheerka, context, parser = self.init_parser("")
assert parser.get_body(list(Tokenizer("not a newline", yield_eof=False))) is None
assert parser.error_sink == [UnexpectedTokenErrorNode("New line not found.", "not", [TokenKind.NEWLINE])]
assert parser.error_sink == [UnexpectedTokenParsingError("New line not found.", "not", [TokenKind.NEWLINE])]
@pytest.mark.parametrize("text", [
"\nx x",
@@ -207,14 +207,14 @@ print xxx"""
sheerka, context, parser = self.init_parser("")
assert parser.get_body(list(Tokenizer(text, yield_eof=False))) is None
assert parser.error_sink == [UnexpectedTokenErrorNode("Indentation not found.", "x", [TokenKind.WHITESPACE])]
assert parser.error_sink == [UnexpectedTokenParsingError("Indentation not found.", "x", [TokenKind.WHITESPACE])]
def test_i_can_detect_missing_tab_when_get_body(self):
text = "\n\txxx\n\tyyy\nzzz"
sheerka, context, parser = self.init_parser("")
assert parser.get_body(list(Tokenizer(text, yield_eof=False))) is None
assert parser.error_sink == [UnexpectedTokenErrorNode("Indentation not found.", "zzz", [TokenKind.WHITESPACE])]
assert parser.error_sink == [UnexpectedTokenParsingError("Indentation not found.", "zzz", [TokenKind.WHITESPACE])]
def test_i_can_detect_invalid_indentation_when_get_body(self):
sheerka, context, parser = self.init_parser("")
+2 -46
View File
@@ -1,50 +1,7 @@
import pytest
from core.tokenizer import Tokenizer, TokenKind, Token
from parsers.BaseParser import BaseParser, BaseSplitIterParser
@pytest.mark.parametrize("text, expected", [
("", ["<eof>"]),
("one two -f --file", ["one", "two", "-f", "--file", "<eof>"]),
("one 'two three'", ["one", "two three", "<eof>"]),
('one "two three"', ["one", "two three", "<eof>"]),
('one\\ two three"', ["one two", "three", "<eof>"]),
("one 'two\\' three'", ["one", "two' three", "<eof>"]),
("one\\\\two three", ["one\\two", "three", "<eof>"]),
("one\ntwo three", ["one", "two", "three", "<eof>"]),
("one \n two three", ["one", "two", "three", "<eof>"]),
("'one \n two' three", ["one \n two", "three", "<eof>"]),
("a=b", ["a", "=", "b", "<eof>"]),
("a = b", ["a", "=", "b", "<eof>"]),
("a==b", ["a", "==", "b", "<eof>"]),
("a == b", ["a", "==", "b", "<eof>"]),
])
def test_i_can_split_using_base_split_iterparser_class(text, expected):
parser = BaseSplitIterParser("BaseSplitIterParser", 0)
parser.reset_parser(None, text)
res = [t.value for t in parser.split()]
assert res == expected
def test_i_can_test_split_iter_parser_indexes():
parser = BaseSplitIterParser("BaseSplitIterParser", 0)
text = "one two \n three = ==(),"
parser.reset_parser(None, text)
res = []
while parser.next_token():
res.append(parser.get_token())
assert res[0] == Token(TokenKind.WORD, "one", 0, 1, 1)
assert res[1] == Token(TokenKind.WORD, "two", 4, 1, 5)
assert res[2] == Token(TokenKind.WORD, "three", 10, 2, 2)
assert res[3] == Token(TokenKind.EQUALS, "=", 16, 2, 8)
assert res[4] == Token(TokenKind.EQUALSEQUALS, "==", 18, 2, 10)
assert res[5] == Token(TokenKind.LPAR, "(", 20, 2, 12)
assert res[6] == Token(TokenKind.RPAR, ")", 21, 2, 13)
assert res[7] == Token(TokenKind.COMMA, ",", 22, 2, 14)
from core.tokenizer import Tokenizer
from parsers.BaseParser import BaseParser
@pytest.mark.parametrize("tokens, expected", [
@@ -61,4 +18,3 @@ def test_i_can_test_split_iter_parser_indexes():
])
def test_i_can_get_tokens_boundaries(tokens, expected):
assert BaseParser.get_tokens_boundaries(tokens) == expected
+6 -6
View File
@@ -4,8 +4,8 @@ from core.concept import Concept, DEFINITION_TYPE_BNF
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer, TokenKind, LexerError
from parsers.BaseNodeParser import cnode
from parsers.BaseParser import UnexpectedTokenErrorNode
from parsers.BnfDefinitionParser import BnfDefinitionParser, UnexpectedEndOfFileError
from parsers.BaseParser import UnexpectedTokenParsingError, UnexpectedEofParsingError
from parsers.BnfDefinitionParser import BnfDefinitionParser
from parsers.BnfNodeParser import BnfNodeParser
from parsers.BnfNodeParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, \
OneOrMore, ConceptExpression
@@ -147,10 +147,10 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
assert res.value.source == expression
@pytest.mark.parametrize("expression, error", [
("1 ", UnexpectedEndOfFileError()),
("1|", UnexpectedEndOfFileError()),
("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", eof_token, [TokenKind.RPAR])),
("1=", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", eof_token, [TokenKind.IDENTIFIER])),
("1 ", UnexpectedEofParsingError()),
("1|", UnexpectedEofParsingError()),
("(1|)", UnexpectedTokenParsingError("Unexpected token 'Token(<EOF>)'", eof_token, [TokenKind.RPAR])),
("1=", UnexpectedTokenParsingError("Unexpected token 'Token(<EOF>)'", eof_token, [TokenKind.IDENTIFIER])),
])
def test_i_can_detect_errors(self, expression, error):
sheerka, context, parser = self.init_parser()
+9 -9
View File
@@ -7,11 +7,11 @@ from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept, CV
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Keywords, Tokenizer, LexerError
from parsers.BaseNodeParser import SCWC
from parsers.BaseParser import NotInitializedNode, UnexpectedEofNode
from parsers.BaseParser import NotInitializedNode, UnexpectedEofParsingError
from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence
from parsers.BnfDefinitionParser import BnfDefinitionParser
from parsers.DefConceptParser import DefConceptParser, NameNode, SyntaxErrorNode
from parsers.DefConceptParser import UnexpectedTokenErrorNode, DefConceptNode
from parsers.DefConceptParser import UnexpectedTokenParsingError, DefConceptNode
from parsers.FunctionParser import FunctionParser
from parsers.PythonParser import PythonParser, PythonNode
@@ -119,9 +119,9 @@ class TestDefConceptParser(TestUsingMemoryBasedSheerka):
return sheerka, context, parser, *updated
@pytest.mark.parametrize("text, error", [
("concept", UnexpectedTokenErrorNode("'def' keyword not found.", "concept", [Keywords.DEF])),
("hello word", UnexpectedTokenErrorNode("'def' keyword not found.", "hello", [Keywords.DEF])),
("def hello", UnexpectedTokenErrorNode("'concept' keyword not found.", "hello", [Keywords.CONCEPT])),
("concept", UnexpectedTokenParsingError("'def' keyword not found.", "concept", [Keywords.DEF])),
("hello word", UnexpectedTokenParsingError("'def' keyword not found.", "hello", [Keywords.DEF])),
("def hello", UnexpectedTokenParsingError("'concept' keyword not found.", "hello", [Keywords.CONCEPT])),
])
def test_i_can_detect_not_for_me(self, text, error):
sheerka, context, parser, *concepts = self.init_parser()
@@ -196,7 +196,7 @@ class TestDefConceptParser(TestUsingMemoryBasedSheerka):
assert not res.status
assert sheerka.isinstance(return_value, BuiltinConcepts.NOT_FOR_ME)
assert isinstance(return_value.reason[0], UnexpectedTokenErrorNode)
assert isinstance(return_value.reason[0], UnexpectedTokenParsingError)
assert return_value.reason[0].message == "'concept' keyword not found."
assert return_value.reason[0].expected_tokens == [Keywords.CONCEPT]
assert return_value.reason[0].token.value == "hello"
@@ -381,8 +381,8 @@ def concept add one to a as:
("def concept name from def", SyntaxErrorNode([], "Empty 'from' declaration.")),
("def concept name from def ", SyntaxErrorNode([], "Empty 'from' declaration.")),
("def concept name from as True", SyntaxErrorNode([], "Empty 'from' declaration.")),
("def concept name from", UnexpectedEofNode("While parsing keyword 'from'.")),
("def concept name from ", UnexpectedEofNode("While parsing keyword 'from'.")),
("def concept name from", UnexpectedEofParsingError("While parsing keyword 'from'.")),
("def concept name from ", UnexpectedEofParsingError("While parsing keyword 'from'.")),
])
def test_i_can_detect_empty_def_declaration(self, text, error):
sheerka, context, parser, *concepts = self.init_parser()
@@ -474,7 +474,7 @@ from give me the date !
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
assert isinstance(res.body.reason[0], UnexpectedTokenErrorNode)
assert isinstance(res.body.reason[0], UnexpectedTokenParsingError)
@pytest.mark.parametrize("text, error_msg, error_text", [
("'name", "Missing Trailing quote", "'name"),
@@ -5,7 +5,7 @@ from core.sheerka.services.SheerkaExecute import ParserInput
from core.sheerka.services.SheerkaRuleManager import FormatAstRawText, RulePredicate, FormatAstVariable
from core.tokenizer import Keywords, Tokenizer
from parsers.BaseCustomGrammarParser import KeywordNotFound
from parsers.FormatRuleParser import FormatRuleParser, FormatRuleNode
from parsers.DefFormatRuleParser import DefFormatRuleParser, FormatRuleNode
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -17,7 +17,7 @@ cmap = {
}
class TestFormatRuleParser(TestUsingMemoryBasedSheerka):
class TestDefFormatRuleParser(TestUsingMemoryBasedSheerka):
sheerka = None
@classmethod
@@ -29,10 +29,10 @@ class TestFormatRuleParser(TestUsingMemoryBasedSheerka):
if concepts_map is not None:
sheerka, context, *concepts = self.init_concepts(*concepts_map.values(), create_new=True)
else:
sheerka = TestFormatRuleParser.sheerka
sheerka = TestDefFormatRuleParser.sheerka
context = self.get_context(sheerka)
parser = FormatRuleParser()
parser = DefFormatRuleParser()
return sheerka, context, parser
def test_i_can_detect_empty_expression(self):
+9 -9
View File
@@ -5,7 +5,7 @@ from core.builtin_concepts import BuiltinConcepts, ReturnValueConcept
from core.concept import Concept
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import Tokenizer, TokenKind
from parsers.BaseParser import UnexpectedEofNode, UnexpectedTokenErrorNode
from parsers.BaseParser import UnexpectedEofParsingError, UnexpectedTokenParsingError
from parsers.ExpressionParser import PropertyEqualsNode, PropertyEqualsSequenceNode, PropertyContainsNode, AndNode, \
OrNode, NotNode, LambdaNode, IsaNode, NameExprNode, ExpressionParser, LeftPartNotFoundError, TrueifyVisitor
@@ -54,12 +54,12 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
assert expressions == expected
@pytest.mark.parametrize("expression, expected_errors", [
("one or", [UnexpectedEofNode("When parsing 'or'")]),
("one and", [UnexpectedEofNode("When parsing 'and'")]),
("one or", [UnexpectedEofParsingError("When parsing 'or'")]),
("one and", [UnexpectedEofParsingError("When parsing 'and'")]),
("and one", [LeftPartNotFoundError()]),
("or one", [LeftPartNotFoundError()]),
("or", [LeftPartNotFoundError(), UnexpectedEofNode("When parsing 'or'")]),
("and", [LeftPartNotFoundError(), UnexpectedEofNode("When parsing 'and'")]),
("or", [LeftPartNotFoundError(), UnexpectedEofParsingError("When parsing 'or'")]),
("and", [LeftPartNotFoundError(), UnexpectedEofParsingError("When parsing 'and'")]),
])
def test_i_can_detect_error(self, expression, expected_errors):
sheerka, context, parser = self.init_parser()
@@ -75,28 +75,28 @@ class TestExpressionParser(TestUsingMemoryBasedSheerka):
res = parser.parse(context, ParserInput("("))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
assert isinstance(res.body.reason[0], UnexpectedTokenErrorNode)
assert isinstance(res.body.reason[0], UnexpectedTokenParsingError)
assert res.body.reason[0].token.type == TokenKind.EOF
assert res.body.reason[0].expected_tokens == [TokenKind.RPAR]
res = parser.parse(context, ParserInput(")"))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
assert isinstance(res.body.reason[0], UnexpectedTokenErrorNode)
assert isinstance(res.body.reason[0], UnexpectedTokenParsingError)
assert res.body.reason[0].token.type == TokenKind.RPAR
assert res.body.reason[0].expected_tokens == []
res = parser.parse(context, ParserInput("one and two)"))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert isinstance(res.body.body[0], UnexpectedTokenErrorNode)
assert isinstance(res.body.body[0], UnexpectedTokenParsingError)
assert res.body.body[0].token.type == TokenKind.RPAR
assert res.body.body[0].expected_tokens == []
res = parser.parse(context, ParserInput("one and two)"))
assert not res.status
assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
assert isinstance(res.body.body[0], UnexpectedTokenErrorNode)
assert isinstance(res.body.body[0], UnexpectedTokenParsingError)
assert res.body.body[0].token.type == TokenKind.RPAR
assert res.body.body[0].expected_tokens == []
+2 -2
View File
@@ -5,7 +5,7 @@ import pytest
from core.builtin_concepts import ParserResultConcept, NotForMeConcept, BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput
from core.tokenizer import LexerError, TokenKind
from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode, ConceptDetected
from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode, ConceptDetectedError
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -118,4 +118,4 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
assert not res.status
assert sheerka.isinstance(res.value, BuiltinConcepts.NOT_FOR_ME)
assert res.value.reason == [ConceptDetected(expected_id)]
assert res.value.reason == [ConceptDetectedError(expected_id)]
+2 -2
View File
@@ -1,7 +1,7 @@
import pytest
from core.builtin_concepts import BuiltinConcepts
from core.sheerka.services.SheerkaExecute import ParserInput
from parsers.RuleParser import RuleParser, RuleNotFound
from parsers.RuleParser import RuleParser, RuleNotFoundError
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -59,7 +59,7 @@ class TestRuleParser(TestUsingMemoryBasedSheerka):
assert not res.status
assert sheerka.isinstance(error, BuiltinConcepts.ERROR)
assert errors_causes == [RuleNotFound("999999")]
assert errors_causes == [RuleNotFoundError("999999")]
def test_i_can_parse_rule(self):
sheerka, context, parser = self.init_parser()
+5 -5
View File
@@ -9,7 +9,7 @@ from parsers.BaseNodeParser import utnode, cnode, short_cnode, UnrecognizedToken
SCWC, CNC, UTN, SCN, CN
from parsers.PythonParser import PythonNode
from parsers.SyaNodeParser import SyaNodeParser, SyaConceptParserHelper, SyaAssociativity, \
NoneAssociativeSequenceErrorNode, TooManyParametersFound, InFixToPostFix, ParenthesisMismatchErrorNode
NoneAssociativeSequenceError, TooManyParametersFoundError, InFixToPostFix, ParenthesisMismatchError
import tests.parsers.parsers_utils
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -758,12 +758,12 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == 1
assert res[0].errors == [ParenthesisMismatchErrorNode(expected)]
assert res[0].errors == [ParenthesisMismatchError(expected)]
def test_i_can_detect_parenthesis_mismatch_error_special_case(self):
sheerka, context, parser = self.init_parser()
expression = "one ? function( : two"
expected = [ParenthesisMismatchErrorNode(("(", 5)), ParenthesisMismatchErrorNode(("(", 5))]
expected = [ParenthesisMismatchError(("(", 5)), ParenthesisMismatchError(("(", 5))]
res = parser.infix_to_postfix(context, ParserInput(expression))
assert len(res) == 1
@@ -780,7 +780,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
assert len(res) == 1
assert len(res[0].errors) == 1
error = res[0].errors[0]
assert isinstance(error, TooManyParametersFound)
assert isinstance(error, TooManyParametersFoundError)
assert error.concept == cmap[expected[0]]
assert error.token.value == expected[1]
@@ -897,7 +897,7 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
res = parser.infix_to_postfix(context, ParserInput("one less than two less than three"))
assert len(res) == 1
assert res[0].errors == [NoneAssociativeSequenceErrorNode(concepts_map["less than"], 2, 8)]
assert res[0].errors == [NoneAssociativeSequenceError(concepts_map["less than"], 2, 8)]
def test_i_can_post_fix_bnf_definition(self):
"""