Hardened DefaultParser

tests/parsers/test_DefaultParser.py

@@ -6,8 +6,8 @@ from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnVa
 from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept, CV
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Keywords, Tokenizer, LexerError
-from parsers.BaseNodeParser import SCN, SCWC
-from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch
+from parsers.BaseNodeParser import SCWC
+from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence
 from parsers.BnfParser import BnfParser
 from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode, CannotHandleErrorNode
 from parsers.DefaultParser import UnexpectedTokenErrorNode, DefConceptNode
@@ -15,7 +15,7 @@ from parsers.FunctionParser import FunctionParser
 from parsers.PythonParser import PythonParser, PythonNode
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
-from tests.parsers.parsers_utils import get_node, compute_expected_array
+from tests.parsers.parsers_utils import compute_expected_array
 
 
 def get_def_concept(name, where=None, pre=None, post=None, body=None, definition=None, bnf_def=None, ret=None):
@@ -164,7 +164,7 @@ ret a if isinstance(a, Concept) else self
         assert isinstance(return_value, ParserResultConcept)
         assert return_value.value == expected_concept
 
-    def test_i_can_have_mutilines_declarations(self):
+    def test_i_can_parse_mutilines_declarations(self):
         text = """
 def concept add one to a as
 def func(x):
@@ -207,14 +207,16 @@ def concept add one to a as:
         assert isinstance(return_value, ParserResultConcept)
         assert return_value.value == expected_concept
 
-    def test_indentation_is_mandatory_after_a_colon(self):
-        text = """
-def concept add one to a as:
-def func(x):
-    return x+1
-func(a)
-    """
-
+    @pytest.mark.parametrize("text", [
+        "def concept foo as:\npass",
+        "def concept foo where:\npass",
+        "def concept foo pre:\npass",
+        "def concept foo post:\npass",
+        "def concept foo from:\nanother definition",
+        "def concept foo from def:\nanother definition",
+        "def concept foo from bnf:\n'another' 'definition'",
+    ])
+    def test_indentation_is_mandatory_after_a_colon(self, text):
         sheerka, context, parser = self.init_parser()
         res = parser.parse(context, ParserInput(text))
         return_value = res.value
@@ -224,19 +226,76 @@ func(a)
         assert isinstance(return_value.body[0], SyntaxErrorNode)
         assert return_value.body[0].message == "Indentation not found."
 
-    def test_indentation_is_not_allowed_if_the_colon_is_missing(self):
-        text = """
-def concept add one to a as
-    def func(x):
-        return x+1
-    func(a)
-        """
+    @pytest.mark.parametrize("text", [
+        "def concept plus from:\n\ta plus b",
+        "def concept plus from def:\n\ta plus b",
+
+        # space before the colon
+        "def concept plus from :\n\ta plus b",
+        "def concept plus from def :\n\ta plus b",
+
+        # space after the colon
+        "def concept plus from: \n\ta plus b",
+        "def concept plus from def: \n\ta plus b",
+    ])
+    def test_i_can_use_colon_and_definition_together(self, text):
         sheerka, context, parser = self.init_parser()
         res = parser.parse(context, ParserInput(text))
-        return_value = res.value
+        defined_concept = res.body.body
+        defined_concept_tokens = [t.repr_value for t in defined_concept.definition.tokens]
 
-        assert not res.status
-        assert context.sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS)
+        assert res.status
+        assert defined_concept.definition_type == DEFINITION_TYPE_DEF
+        assert defined_concept_tokens == [t.repr_value for t in Tokenizer("a plus b", yield_eof=False)]
+
+    @pytest.mark.parametrize("text", [
+        "def concept plus from bnf:\n\t'a' 'plus' 'b'",
+        "def concept plus from bnf :\n\t'a' 'plus' 'b'",
+        "def concept plus from bnf: \n\t'a' 'plus' 'b'",
+    ])
+    def test_i_can_use_colon_and_bnf_definition_together(self, text):
+        sheerka, context, parser = self.init_parser()
+        res = parser.parse(context, ParserInput(text))
+        defined_concept = res.body.body
+
+        assert res.status
+        assert defined_concept.definition.status
+        assert defined_concept.definition.body.body == Sequence(StrMatch("a"), StrMatch("plus"), StrMatch("b"))
+
+    def test_i_can_use_colon_to_protect_keyword(self):
+        text = """
+def concept today as:
+    from datetime import date
+    today = date.today()
+from:
+    give me the date !
+"""
+        sheerka, context, parser = self.init_parser()
+        res = parser.parse(context, ParserInput(text))
+        defined_concept = res.body.body
+        defined_concept_tokens = [t.repr_value for t in defined_concept.definition.tokens]
+
+        assert res.status
+        assert defined_concept.definition_type == DEFINITION_TYPE_DEF
+        assert defined_concept_tokens == [t.repr_value for t in Tokenizer("give me the date !", yield_eof=False)]
+        assert defined_concept.body.status
+
+    def test_i_can_use_colon_to_protect_keyword_2(self):
+        text = """
+def concept today as:
+    from datetime import date
+    today = date.today()
+from give me the date !
+"""
+        sheerka, context, parser = self.init_parser()
+        res = parser.parse(context, ParserInput(text))
+        defined_concept = res.body.body
+        defined_concept_tokens = [t.repr_value for t in defined_concept.definition.tokens]
+
+        assert res.status
+        assert defined_concept.definition_type == DEFINITION_TYPE_DEF
+        assert defined_concept_tokens == [t.repr_value for t in Tokenizer("give me the date !", yield_eof=False)]
+        assert defined_concept.body.status
 
     def test_name_is_mandatory(self):
         text = "def concept as 'hello'"
@@ -277,7 +336,19 @@ def concept add one to a as
         assert not res.status
         assert sheerka.isinstance(return_value, BuiltinConcepts.TOO_MANY_ERRORS)
 
-    def test_new_line_is_not_allowed_in_the_name(self):
+    @pytest.mark.parametrize("text", [
+        "def concept hello\nmy friend",
+        "def concept hello \nmy friend",
+        "def concept hello\n my friend",
+        "def concept hello \n my friend",
+        "def concept hello from hello\nmy friend",
+        "def concept hello from def hello\nmy friend",
+        "def concept hello from bnf hello\nmy friend",
+        "def concept hello from:\n\thello\nmy friend",
+        "def concept hello from def:\n\thello\nmy friend",
+        "def concept hello from bnf:\n\thello\nmy friend",
+    ])
+    def test_new_line_is_not_allowed_in_the_name(self, text):
         text = "def concept hello \n my friend as 'hello'"
 
         sheerka, context, parser = self.init_parser()