Added ZeroAndMore and OneAndMore to BNF. BNF expressions can now be captured

tests/test_ConceptLexerParser.py

@@ -3,7 +3,7 @@ from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
 from core.sheerka import Sheerka, ExecutionContext
 from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, StrMatch, OrderedChoice, Optional, \
-    ParsingExpressionVisitor, TerminalNode, NonTerminalNode, LexerNode, ConceptMatch
+    ParsingExpressionVisitor, TerminalNode, NonTerminalNode, LexerNode, ConceptMatch, ZeroOrMore, OneOrMore
 
 
 class ConceptVisitor(ParsingExpressionVisitor):
@@ -352,6 +352,22 @@ def test_i_can_parse_sequence_with_optional_in_between():
     assert res.value.body == [(foo, 0, 2, "one three")]
 
 
+def test_i_cannot_parse_wrong_input_with_optional():
+    context = get_context()
+    foo = Concept(name="foo")
+
+    concepts = {foo: Optional("one")}
+    parser = ConceptLexerParser()
+    parser.initialize(context, concepts)
+
+    res = parser.parse(context, "two")
+    assert not res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.PARSER_RESULT)
+    assert res.value.try_parsed == []
+    assert context.sheerka.isinstance(res.value.body[0], BuiltinConcepts.UNKNOWN_CONCEPT)
+    assert res.value.body[0].body == "two"
+
+
 def test_i_can_use_reference():
     # when there are multiple matches for the same input
     # Do I need to create a choice concept ?
@@ -469,6 +485,172 @@ def test_i_can_detect_duplicates_when_reference():
     assert res[1].value.body == [(foo, 0, 0, "twenty")]
 
 
+def test_i_can_parse_zero_or_more():
+    context = get_context()
+    foo = Concept(name="foo")
+
+    concepts = {foo: ZeroOrMore("one")}
+    parser = ConceptLexerParser()
+    parser.initialize(context, concepts)
+
+    res = parser.parse(context, "one one")
+    assert res.status
+    assert res.value.value == [ConceptNode(foo, 0, 2, source="one one",
+                                           underlying=u(concepts[foo], 0, 2, [
+                                               u("one", 0, 0),
+                                               u("one", 2, 2)]))]
+
+
+def test_i_can_parse_sequence_and_zero_or_more():
+    context = get_context()
+    foo = Concept(name="foo")
+
+    concepts = {foo: Sequence(ZeroOrMore("one"), "two")}
+    parser = ConceptLexerParser()
+    parser.initialize(context, concepts)
+
+    res = parser.parse(context, "one one two")
+    assert res.status
+    assert res.value.value == [ConceptNode(foo, 0, 4, source="one one two",
+                                           underlying=u(concepts[foo], 0, 4, [
+                                               u(ZeroOrMore("one"), 0, 2, [
+                                                   u("one", 0, 0),
+                                                   u("one", 2, 2)]),
+                                               u("two", 4, 4)]))]
+
+    res = parser.parse(context, "two")
+    assert res.status
+    assert res.value.value == [ConceptNode(foo, 0, 0, source="two",
+                                           underlying=u(concepts[foo], 0, 0, [u("two", 0, 0)]))]
+
+
+def test_i_cannot_parse_zero_and_more_when_wrong_entry():
+    context = get_context()
+    foo = Concept(name="foo")
+
+    concepts = {foo: ZeroOrMore("one")}
+    parser = ConceptLexerParser()
+    parser.initialize(context, concepts)
+
+    res = parser.parse(context, "one two")
+    assert not res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.PARSER_RESULT)
+    assert res.value.try_parsed == [
+        ConceptNode(foo, 0, 0, source="one", underlying=u(ZeroOrMore("one"), 0, 0, [u("one", 0, 0)]))]
+    assert context.sheerka.isinstance(res.value.body[0], BuiltinConcepts.UNKNOWN_CONCEPT)
+    assert res.value.body[0].body == "two"
+
+    res = parser.parse(context, "two")
+    assert not res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.PARSER_RESULT)
+    assert res.value.try_parsed == []
+    assert context.sheerka.isinstance(res.value.body[0], BuiltinConcepts.UNKNOWN_CONCEPT)
+    assert res.value.body[0].body == "two"
+
+
+def test_i_can_parse_zero_and_more_with_separator():
+    context = get_context()
+    foo = Concept(name="foo")
+
+    concepts = {foo: ZeroOrMore("one", sep=",")}
+    parser = ConceptLexerParser()
+    parser.initialize(context, concepts)
+
+    res = parser.parse(context, "one, one , one")
+    assert res.status
+    assert res.value.value == [ConceptNode(foo, 0, 7, source="one, one , one",
+                                           underlying=u(concepts[foo], 0, 7, [
+                                               u("one", 0, 0),
+                                               u("one", 3, 3),
+                                               u("one", 7, 7)]))]
+
+
+def test_that_zero_and_more_is_greedy():
+    context = get_context()
+    foo = Concept(name="foo")
+    bar = Concept(name="bar")
+
+    concepts = {foo: ZeroOrMore("one"), bar: "one"}
+    parser = ConceptLexerParser()
+    parser.initialize(context, concepts)
+
+    res = parser.parse(context, "one one one")
+    assert res.status
+    assert res.value.value == [(foo, 0, 4, "one one one")]
+
+
+def test_i_can_parse_one_and_more():
+    context = get_context()
+    foo = Concept(name="foo")
+
+    concepts = {foo: OneOrMore("one")}
+    parser = ConceptLexerParser()
+    parser.initialize(context, concepts)
+
+    res = parser.parse(context, "one one")
+    assert res.status
+    assert res.value.value == [ConceptNode(foo, 0, 2, source="one one",
+                                           underlying=u(concepts[foo], 0, 2, [
+                                               u("one", 0, 0),
+                                               u("one", 2, 2)]))]
+
+
+def test_i_can_parse_sequence_and_one_or_more():
+    context = get_context()
+    foo = Concept(name="foo")
+
+    concepts = {foo: Sequence(OneOrMore("one"), "two")}
+    parser = ConceptLexerParser()
+    parser.initialize(context, concepts)
+
+    res = parser.parse(context, "one one two")
+    assert res.status
+    assert res.value.value == [ConceptNode(foo, 0, 4, source="one one two",
+                                           underlying=u(concepts[foo], 0, 4, [
+                                               u(ZeroOrMore("one"), 0, 2, [
+                                                   u("one", 0, 0),
+                                                   u("one", 2, 2)]),
+                                               u("two", 4, 4)]))]
+
+    res = parser.parse(context, "two")
+    assert not res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.PARSER_RESULT)
+    assert res.value.try_parsed == []
+    assert context.sheerka.isinstance(res.value.body[0], BuiltinConcepts.UNKNOWN_CONCEPT)
+    assert res.value.body[0].body == "two"
+
+
+def test_i_can_parse_one_and_more_with_separator():
+    context = get_context()
+    foo = Concept(name="foo")
+
+    concepts = {foo: OneOrMore("one", sep=",")}
+    parser = ConceptLexerParser()
+    parser.initialize(context, concepts)
+
+    res = parser.parse(context, "one, one , one")
+    assert res.status
+    assert res.value.value == [ConceptNode(foo, 0, 7, source="one, one , one",
+                                           underlying=u(concepts[foo], 0, 7, [
+                                               u("one", 0, 0),
+                                               u("one", 3, 3),
+                                               u("one", 7, 7)]))]
+
+
+def test_that_one_and_more_is_greedy():
+    context = get_context()
+    foo = Concept(name="foo")
+    bar = Concept(name="bar")
+
+    concepts = {foo: OneOrMore("one"), bar: "one"}
+    parser = ConceptLexerParser()
+    parser.initialize(context, concepts)
+
+    res = parser.parse(context, "one one one")
+    assert res.status
+    assert res.value.value == [(foo, 0, 4, "one one one")]
+
+
 def test_i_can_detect_infinite_recursion():
     foo = Concept(name="foo")
     bar = Concept(name="bar")
@@ -552,6 +734,16 @@ def test_i_can_detect_indirect_infinite_recursion_with_optional():
     pass
 
 
+def test_i_can_detect_indirect_infinite_recursion_with_zero_and_more():
+    # TODO infinite recursion with optional
+    pass
+
+
+def test_i_can_detect_indirect_infinite_recursion_with_one_and_more():
+    # TODO infinite recursion with optional
+    pass
+
+
 def test_i_can_visit_parsing_expression():
     mult = Concept(name="mult")
     add = Concept(name="add")