Added simple form of concept composition

tests/test_MultipleConceptsParser.py

@@ -1,8 +1,12 @@
+import pytest
+
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
 from core.concept import Concept
 from core.sheerka import Sheerka, ExecutionContext
-from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, UnrecognizedTokensNode
+from core.tokenizer import Tokenizer, TokenKind, Token
+from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, cnode, utnode, scnode, SourceCodeNode
 from parsers.MultipleConceptsParser import MultipleConceptsParser
+from parsers.PythonParser import PythonNode
 from sdp.sheerkaDataProvider import Event
 
 
@@ -78,7 +82,7 @@ def test_i_can_parse_when_ending_with_bnf():
     assert ret_val.status
     assert ret_val.who == parser.name
     assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
-    assert ret_val.value.value == [("bar", 0, 0, "bar"), ("foo", 2, 6, "foo1 foo2 foo3")]
+    assert ret_val.value.value == [cnode("bar", 0, 0, "bar"), cnode("foo", 2, 6, "foo1 foo2 foo3")]
     assert ret_val.value.source == "bar foo1 foo2 foo3"
 
 
@@ -94,7 +98,7 @@ def test_i_can_parse_when_starting_with_bnf():
     assert ret_val.status
     assert ret_val.who == parser.name
     assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
-    assert ret_val.value.value == [("foo", 0, 4, "foo1 foo2 foo3"), ("bar", 6, 6, "bar")]
+    assert ret_val.value.value == [cnode("foo", 0, 4, "foo1 foo2 foo3"), cnode("bar", 6, 6, "bar")]
     assert ret_val.value.source == "foo1 foo2 foo3 bar"
 
 
@@ -112,13 +116,13 @@ def test_i_can_parse_when_concept_are_mixed():
     assert ret_val.who == parser.name
     assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
     assert ret_val.value.value == [
-        ("baz", 0, 0, "baz"),
-        ("foo", 2, 6, "foo1 foo2 foo3"),
-        ("bar", 8, 8, "bar")]
+        cnode("baz", 0, 0, "baz"),
+        cnode("foo", 2, 6, "foo1 foo2 foo3"),
+        cnode("bar", 8, 8, "bar")]
     assert ret_val.value.source == "baz foo1 foo2 foo3 bar"
 
 
-def test_i_can_parse_when_multiple_concept_are_matching():
+def test_i_can_parse_when_multiple_concepts_are_matching():
     foo = Concept("foo")
     bar = Concept("bar", body="bar1")
     baz = Concept("bar", body="bar2")
@@ -130,16 +134,35 @@ def test_i_can_parse_when_multiple_concept_are_matching():
 
     assert len(ret_val) == 2
     assert ret_val[0].status
-    assert ret_val[0].value.value == [("foo", 0, 0, "foo"), ("bar", 2, 2, "bar")]
+    assert ret_val[0].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")]
     assert ret_val[0].value.source == "foo bar"
     assert ret_val[0].value.value[1].concept.body == "bar1"
 
     assert ret_val[1].status
-    assert ret_val[1].value.value == [("foo", 0, 0, "foo"), ("bar", 2, 2, "bar")]
+    assert ret_val[1].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")]
     assert ret_val[1].value.source == "foo bar"
     assert ret_val[1].value.value[1].concept.body == "bar2"
 
 
+def test_i_can_parse_when_source_code():
+    foo = Concept("foo")
+    grammar = {foo: "foo"}
+    context, return_value = init([foo], grammar, "1 foo")
+
+    parser = MultipleConceptsParser()
+    ret_val = parser.parse(context, return_value.body)
+    wrapper = ret_val.value
+    value = ret_val.value.value
+
+    assert ret_val.status
+    assert ret_val.who == parser.name
+    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+    assert wrapper.source == "1 foo"
+    assert value == [
+        scnode(0, 1, "1 "),
+        cnode("foo", 2, 2, "foo")]
+
+
 def test_i_cannot_parse_when_unrecognized_token():
     twenty_two = Concept("twenty two")
     one = Concept("one")
@@ -153,8 +176,56 @@ def test_i_cannot_parse_when_unrecognized_token():
     assert ret_val.who == parser.name
     assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
     assert ret_val.value.value == [
-        ("twenty two", 0, 2, "twenty two"),
-        (3, 5, " + "),
-        ("one", 6, 6, "one")
+        cnode("twenty two", 0, 2, "twenty two"),
+        utnode(3, 5, " + "),
+        cnode("one", 6, 6, "one")
     ]
     assert ret_val.value.source == "twenty two + one"
+
+
+def test_i_cannot_parse_when_unknown_concepts():
+    twenty_two = Concept("twenty two")
+    one = Concept("one")
+    grammar = {twenty_two: Sequence("twenty", "two")}
+    context, return_value = init([twenty_two, one], grammar, "twenty two plus one")
+
+    parser = MultipleConceptsParser()
+    ret_val = parser.parse(context, return_value.body)
+
+    assert not ret_val.status
+    assert ret_val.who == parser.name
+    assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
+    assert ret_val.value.value == [
+        cnode("twenty two", 0, 2, "twenty two"),
+        utnode(3, 5, " plus "),
+        cnode("one", 6, 6, "one")
+    ]
+    assert ret_val.value.source == "twenty two plus one"
+
+
+@pytest.mark.parametrize("text, expected_source, expected_end", [
+    ("True", "True", 0),
+    ("1 == 1", "1 == 1", 5),
+    ("1!xdf", "1", 0),
+    ("1", "1", 0),
+])
+def test_i_can_get_source_code_node(text, expected_source, expected_end):
+    tokens = list(Tokenizer(text))[:-1]  # strip trailing EOF
+
+    start_index = 5  # a random number different of zero
+    res = MultipleConceptsParser().get_source_code_node(get_context(), start_index, tokens)
+
+    assert isinstance(res, SourceCodeNode)
+    assert isinstance(res.node, PythonNode)
+    assert res.source == expected_source
+    assert res.start == start_index
+    assert res.end == start_index + expected_end
+
+
+def test_i_cannot_parse_null_text():
+    res = MultipleConceptsParser().get_source_code_node(get_context(), 0, [])
+    assert res is None
+
+    eof = Token(TokenKind.EOF, "", 0, 0, 0)
+    res = MultipleConceptsParser().get_source_code_node(get_context(), 0, [eof])
+    assert res is None