Hardened DefaultParser
This commit is contained in:
@@ -77,3 +77,19 @@ def test_i_can_parse_twice():
|
||||
while p2.next_token():
|
||||
p1.next_token()
|
||||
assert p1.token == p2.token
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, skip_whitespace, expected", [
|
||||
("first second", True, "second"),
|
||||
("first second", False, "<ws>"),
|
||||
("first", True, "<EOF>"),
|
||||
("first", False, "<EOF>"),
|
||||
("first ", True, "<EOF>"),
|
||||
("first ", False, "<ws>"),
|
||||
("first:", True, ":"),
|
||||
("first:", False, ":"),
|
||||
])
|
||||
def test_i_can_get_the_token_after(text, skip_whitespace, expected):
|
||||
parser_input = ParserInput(text).reset()
|
||||
parser_input.next_token()
|
||||
assert parser_input.the_token_after(skip_whitespace).repr_value == expected
|
||||
|
||||
@@ -6,8 +6,8 @@ from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnVa
|
||||
from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept, CV
|
||||
from core.sheerka.services.SheerkaExecute import ParserInput
|
||||
from core.tokenizer import Keywords, Tokenizer, LexerError
|
||||
from parsers.BaseNodeParser import SCN, SCWC
|
||||
from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch
|
||||
from parsers.BaseNodeParser import SCWC
|
||||
from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence
|
||||
from parsers.BnfParser import BnfParser
|
||||
from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode, CannotHandleErrorNode
|
||||
from parsers.DefaultParser import UnexpectedTokenErrorNode, DefConceptNode
|
||||
@@ -15,7 +15,7 @@ from parsers.FunctionParser import FunctionParser
|
||||
from parsers.PythonParser import PythonParser, PythonNode
|
||||
|
||||
from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
|
||||
from tests.parsers.parsers_utils import get_node, compute_expected_array
|
||||
from tests.parsers.parsers_utils import compute_expected_array
|
||||
|
||||
|
||||
def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None, bnf_def=None, ret=None):
|
||||
@@ -164,7 +164,7 @@ ret a if isinstance(a, Concept) else self
|
||||
assert isinstance(return_value, ParserResultConcept)
|
||||
assert return_value.value == expected_concept
|
||||
|
||||
def test_i_can_have_mutilines_declarations(self):
|
||||
def test_i_can_parse_mutilines_declarations(self):
|
||||
text = """
|
||||
def concept add one to a as
|
||||
def func(x):
|
||||
@@ -207,14 +207,16 @@ def concept add one to a as:
|
||||
assert isinstance(return_value, ParserResultConcept)
|
||||
assert return_value.value == expected_concept
|
||||
|
||||
def test_indentation_is_mandatory_after_a_colon(self):
|
||||
text = """
|
||||
def concept add one to a as:
|
||||
def func(x):
|
||||
return x+1
|
||||
func(a)
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize("text", [
|
||||
"def concept foo as:\npass",
|
||||
"def concept foo where:\npass",
|
||||
"def concept foo pre:\npass",
|
||||
"def concept foo post:\npass",
|
||||
"def concept foo from:\nanother definition",
|
||||
"def concept foo from def:\nanother definition",
|
||||
"def concept foo from bnf:\n'another' 'definition'",
|
||||
])
|
||||
def test_indentation_is_mandatory_after_a_colon(self, text):
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, ParserInput(text))
|
||||
return_value = res.value
|
||||
@@ -224,19 +226,76 @@ func(a)
|
||||
assert isinstance(return_value.body[0], SyntaxErrorNode)
|
||||
assert return_value.body[0].message == "Indentation not found."
|
||||
|
||||
def test_indentation_is_not_allowed_if_the_colon_is_missing(self):
|
||||
text = """
|
||||
def concept add one to a as
|
||||
def func(x):
|
||||
return x+1
|
||||
func(a)
|
||||
"""
|
||||
@pytest.mark.parametrize("text", [
|
||||
"def concept plus from:\n\ta plus b",
|
||||
"def concept plus from def:\n\ta plus b",
|
||||
|
||||
# space before the colon
|
||||
"def concept plus from :\n\ta plus b",
|
||||
"def concept plus from def :\n\ta plus b",
|
||||
|
||||
# space after the colon
|
||||
"def concept plus from: \n\ta plus b",
|
||||
"def concept plus from def: \n\ta plus b",
|
||||
])
|
||||
def test_i_can_use_colon_and_definition_together(self, text):
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, ParserInput(text))
|
||||
return_value = res.value
|
||||
defined_concept = res.body.body
|
||||
defined_concept_tokens = [t.repr_value for t in defined_concept.definition.tokens]
|
||||
|
||||
assert not res.status
|
||||
assert context.sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS)
|
||||
assert res.status
|
||||
assert defined_concept.definition_type == DEFINITION_TYPE_DEF
|
||||
assert defined_concept_tokens == [t.repr_value for t in Tokenizer("a plus b", yield_eof=False)]
|
||||
|
||||
@pytest.mark.parametrize("text", [
|
||||
"def concept plus from bnf:\n\t'a' 'plus' 'b'",
|
||||
"def concept plus from bnf :\n\t'a' 'plus' 'b'",
|
||||
"def concept plus from bnf: \n\t'a' 'plus' 'b'",
|
||||
])
|
||||
def test_i_can_use_colon_and_bnf_definition_together(self, text):
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, ParserInput(text))
|
||||
defined_concept = res.body.body
|
||||
|
||||
assert res.status
|
||||
assert defined_concept.definition.status
|
||||
assert defined_concept.definition.body.body == Sequence(StrMatch("a"), StrMatch("plus"), StrMatch("b"))
|
||||
|
||||
def test_i_can_use_colon_to_protect_keyword(self):
|
||||
text = """
|
||||
def concept today as:
|
||||
from datetime import date
|
||||
today = date.today()
|
||||
from:
|
||||
give me the date !
|
||||
"""
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, ParserInput(text))
|
||||
defined_concept = res.body.body
|
||||
defined_concept_tokens = [t.repr_value for t in defined_concept.definition.tokens]
|
||||
|
||||
assert res.status
|
||||
assert defined_concept.definition_type == DEFINITION_TYPE_DEF
|
||||
assert defined_concept_tokens == [t.repr_value for t in Tokenizer("give me the date !", yield_eof=False)]
|
||||
assert defined_concept.body.status
|
||||
|
||||
def test_i_can_use_colon_to_protect_keyword_2(self):
|
||||
text = """
|
||||
def concept today as:
|
||||
from datetime import date
|
||||
today = date.today()
|
||||
from give me the date !
|
||||
"""
|
||||
sheerka, context, parser = self.init_parser()
|
||||
res = parser.parse(context, ParserInput(text))
|
||||
defined_concept = res.body.body
|
||||
defined_concept_tokens = [t.repr_value for t in defined_concept.definition.tokens]
|
||||
|
||||
assert res.status
|
||||
assert defined_concept.definition_type == DEFINITION_TYPE_DEF
|
||||
assert defined_concept_tokens == [t.repr_value for t in Tokenizer("give me the date !", yield_eof=False)]
|
||||
assert defined_concept.body.status
|
||||
|
||||
def test_name_is_mandatory(self):
|
||||
text = "def concept as 'hello'"
|
||||
@@ -277,7 +336,19 @@ def concept add one to a as
|
||||
assert not res.status
|
||||
assert sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS)
|
||||
|
||||
def test_new_line_is_not_allowed_in_the_name(self):
|
||||
@pytest.mark.parametrize("text", [
|
||||
"def concept hello\nmy friend",
|
||||
"def concept hello \nmy friend",
|
||||
"def concept hello\n my friend",
|
||||
"def concept hello \n my friend",
|
||||
"def concept hello from hello\nmy friend",
|
||||
"def concept hello from def hello\nmy friend",
|
||||
"def concept hello from bnf hello\nmy friend",
|
||||
"def concept hello from:\n\thello\nmy friend",
|
||||
"def concept hello from def:\n\thello\nmy friend",
|
||||
"def concept hello from bnf:\n\thello\nmy friend",
|
||||
])
|
||||
def test_new_line_is_not_allowed_in_the_name(self, text):
|
||||
text = "def concept hello \n my friend as 'hello'"
|
||||
|
||||
sheerka, context, parser = self.init_parser()
|
||||
|
||||
Reference in New Issue
Block a user