Added simple form of concept composition

tests/test_BnfParser.py

@@ -6,7 +6,7 @@ from core.tokenizer import Tokenizer, TokenKind, LexerError
 from parsers.BaseParser import UnexpectedTokenErrorNode
 from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError
 from parsers.ConceptLexerParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \
-    ConceptLexerParser, ConceptNode, ConceptMatch
+    ConceptLexerParser, ConceptNode, ConceptMatch, cnode
 from sdp.sheerkaDataProvider import Event
 
 
@@ -108,12 +108,12 @@ def test_i_can_use_the_result_of_regex_parsing_to_parse_a_text():
 
     res = concept_parser.parse(context, "twenty two")
     assert res.status
-    assert res.value.body == [("bar", 0, 2, "twenty two")]
+    assert res.value.body == [cnode("bar", 0, 2, "twenty two")]
 
     res = concept_parser.parse(context, "thirty one")
     assert res.status
-    assert res.value.body == [("bar", 0, 2, "thirty one")]
+    assert res.value.body == [cnode("bar", 0, 2, "thirty one")]
 
     res = concept_parser.parse(context, "twenty")
     assert res.status
-    assert res.value.body == [("foo", 0, 0, "twenty")]
+    assert res.value.body == [cnode("foo", 0, 0, "twenty")]

tests/test_ConceptComposerEvaluator.py

@@ -1,128 +0,0 @@
-# import pytest
-#
-# from core.builtin_concepts import ReturnValueConcept, ParserResultConcept
-# from core.concept import Concept
-# from core.sheerka import Sheerka, ExecutionContext
-# from evaluators.BaseEvaluator import BaseEvaluator
-# from evaluators.ConceptComposerEvaluator import ConceptComposerEvaluator
-# from parsers.BaseParser import BaseParser
-# from parsers.ConceptLexerParser import ConceptNode, ConceptLexerParser, Sequence
-# from sdp.sheerkaDataProvider import Event
-#
-# concept_lexer_name = ConceptLexerParser().name
-#
-#
-# def get_context():
-#     sheerka = Sheerka(skip_builtins_in_db=True)
-#     sheerka.initialize("mem://")
-#     return ExecutionContext("test", Event(), sheerka)
-#
-#
-# def get_return_values(context, grammar, expression):
-#     parser = ConceptLexerParser()
-#     parser.initialize(context, grammar)
-#
-#     ret_val = parser.parse(context, expression)
-#     assert not ret_val.status
-#     return [ret_val]
-#
-#
-# def init(concepts, grammar, expression):
-#     context = get_context()
-#     for c in concepts:
-#         context.sheerka.add_in_cache(c)
-#     return_values = get_return_values(context, grammar, expression)
-#
-#     return context, return_values
-#
-#
-# @pytest.mark.parametrize("return_values, expected", [
-#     ([
-#          ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "in error"),
-#          ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])),
-#          ReturnValueConcept("not a parser", True, "some value"),
-#      ], True),
-#     ([
-#          ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])),
-#      ], True),
-#     ([
-#          ReturnValueConcept(BaseParser.PREFIX + "some_name", True, "not in error"),
-#          ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])),
-#      ], False),
-#     ([
-#          ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "in error"),
-#          ReturnValueConcept(concept_lexer_name, True, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])),
-#      ], False),
-#     ([
-#          ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "in error"),
-#          ReturnValueConcept(concept_lexer_name, False, "some value"),
-#      ], False),
-#     ([
-#          ReturnValueConcept(BaseParser.PREFIX + "some_name", False, "in error"),
-#          ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=["not a concept"])),
-#      ], False),
-#     ([
-#          ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", False, "evaluator in error"),
-#          ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])),
-#          ReturnValueConcept("not a parser", True, "some value"),
-#      ], False),
-#     ([
-#          ReturnValueConcept(BaseEvaluator.PREFIX + "some_name", True, "evaluator"),
-#          ReturnValueConcept(concept_lexer_name, False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])),
-#          ReturnValueConcept("not a parser", True, "some value"),
-#      ], False),
-# ])
-# def test_i_can_match(return_values, expected):
-#     context = get_context()
-#     assert ConceptComposerEvaluator().matches(context, return_values) == expected
-#
-#
-# def test_i_can_eval_simple_concepts():
-#     foo = Concept("foo", body="'foo'")
-#     bar = Concept("bar", body="'bar'")
-#     grammar = {}
-#     context, return_values = init([foo, bar], grammar, "bar foo")
-#
-#     composer = ConceptComposerEvaluator()
-#     assert composer.matches(context, return_values)
-#
-#     ret_val = composer.eval(context, return_values)
-#     assert ret_val.status
-#     assert ret_val.who == composer.name
-#     assert ret_val.value == [Concept("bar", body="bar").init_key(), Concept("foo", body="foo").init_key()]
-#     assert ret_val.value[0].metadata.is_evaluated
-#     assert ret_val.value[1].metadata.is_evaluated
-#     assert ret_val.parents == [return_values[0]]
-#
-#
-# def test_i_can_eval_simple_concepts_when_some_are_bnf():
-#     foo = Concept("foo", body="'foo'")
-#     bar = Concept("bar", body="'bar'")
-#     grammar = {foo: "foo"}
-#     context, return_values = init([foo, bar], grammar, "bar foo")
-#
-#     composer = ConceptComposerEvaluator()
-#     assert composer.matches(context, return_values)
-#
-#     ret_val = composer.eval(context, return_values)
-#     assert ret_val.status
-#     assert ret_val.who == composer.name
-#     assert ret_val.value == [Concept("bar", body="bar").init_key(), Concept("foo", body="foo").init_key()]
-#     assert ret_val.value[0].metadata.is_evaluated
-#     assert ret_val.value[1].metadata.is_evaluated
-#     assert ret_val.parents == [return_values[0]]
-#
-#
-# def test_i_can_eval_simple_concept_and_text():
-#     foo = Concept("foo", body="'foo'")
-#     grammar = {}
-#     context, return_values = init([foo], grammar, "'bar' foo")
-#
-#     composer = ConceptComposerEvaluator()
-#     assert composer.matches(context, return_values)
-#
-#     ret_val = composer.eval(context, return_values)
-#     assert ret_val.status
-#     assert ret_val.who == composer.name
-#     assert ret_val.value == "bar foo"
-#     assert ret_val.parents == [return_values[0]]

tests/test_ConceptLexerParser.py

@@ -6,7 +6,7 @@ from core.sheerka import Sheerka, ExecutionContext
 from core.tokenizer import Tokenizer, TokenKind, Token
 from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, StrMatch, OrderedChoice, Optional, \
     ParsingExpressionVisitor, TerminalNode, NonTerminalNode, LexerNode, ConceptMatch, ZeroOrMore, OneOrMore, \
-    UnrecognizedTokensNode
+    UnrecognizedTokensNode, cnode, short_cnode
 from sdp.sheerkaDataProvider import Event
 
 
@@ -163,7 +163,7 @@ def test_i_always_choose_the_longest_match():
 
     assert res.status
     assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert return_value == [("foo", 0, 4, "one two three")]
+    assert return_value == [cnode("foo", 0, 4, "one two three")]
 
 
 def test_i_can_match_several_sequences():
@@ -176,8 +176,8 @@ def test_i_can_match_several_sequences():
     assert res.status
     assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
     assert return_value == [
-        ("foo", 0, 4, "one two three"),
-        ("bar", 6, 8, "one two"),
+        cnode("foo", 0, 4, "one two three"),
+        cnode("bar", 6, 8, "one two"),
     ]
 
 
@@ -189,13 +189,13 @@ def test_i_can_match_ordered_choice():
     res1 = parser.parse(context, "one")
     assert res1.status
     assert context.sheerka.isinstance(res1.value, BuiltinConcepts.PARSER_RESULT)
-    assert res1.value.body == [("foo", 0, 0, "one")]
+    assert res1.value.body == [cnode("foo", 0, 0, "one")]
     assert res1.value.body[0].underlying == u(grammar[foo], 0, 0, [u("one", 0, 0)])
 
     res2 = parser.parse(context, "two")
     assert res2.status
     assert context.sheerka.isinstance(res2.value, BuiltinConcepts.PARSER_RESULT)
-    assert res2.value.body == [("foo", 0, 0, "two")]
+    assert res2.value.body == [cnode("foo", 0, 0, "two")]
     assert res2.value.body[0].underlying == u(grammar[foo], 0, 0, [u("two", 0, 0)])
 
     res3 = parser.parse(context, "three")
@@ -259,11 +259,11 @@ def test_i_can_mix_ordered_choices_and_sequences():
 
     res = parser.parse(context, "twenty thirty")
     assert res.status
-    assert res.value.value == [("foo", 0, 2, "twenty thirty")]
+    assert res.value.value == [cnode("foo", 0, 2, "twenty thirty")]
 
     res = parser.parse(context, "one")
     assert res.status
-    assert res.value.value == [("foo", 0, 0, "one")]
+    assert res.value.value == [cnode("foo", 0, 0, "one")]
 
 
 def test_i_cannot_parse_empty_optional():
@@ -319,11 +319,11 @@ def test_i_can_parse_sequence_ending_with_optional():
 
     res = parser.parse(context, "one two three")
     assert res.status
-    assert res.value.body == [("foo", 0, 4, "one two three")]
+    assert res.value.body == [cnode("foo", 0, 4, "one two three")]
 
     res = parser.parse(context, "one two")
     assert res.status
-    assert res.value.body == [("foo", 0, 2, "one two")]
+    assert res.value.body == [cnode("foo", 0, 2, "one two")]
 
 
 def test_i_can_parse_sequence_with_optional_in_between():
@@ -335,11 +335,11 @@ def test_i_can_parse_sequence_with_optional_in_between():
 
     res = parser.parse(context, "one two three")
     assert res.status
-    assert res.value.body == [("foo", 0, 4, "one two three")]
+    assert res.value.body == [cnode("foo", 0, 4, "one two three")]
 
     res = parser.parse(context, "one three")
     assert res.status
-    assert res.value.body == [("foo", 0, 2, "one three")]
+    assert res.value.body == [cnode("foo", 0, 2, "one three")]
 
 
 def test_i_cannot_parse_wrong_input_with_optional():
@@ -370,13 +370,13 @@ def test_i_can_use_reference():
 
     assert res[0].status
     assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
-    assert res[0].value.body == [("foo", 0, 2, "one two")]
+    assert res[0].value.body == [cnode("foo", 0, 2, "one two")]
     concept_found_1 = res[0].value.body[0].concept
     assert cbody(concept_found_1) == DoNotResolve("one two")
 
     assert res[1].status
     assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT)
-    assert res[1].value.body == [("bar", 0, 2, "one two")]
+    assert res[1].value.body == [cnode("bar", 0, 2, "one two")]
     concept_found_2 = res[1].value.body[0].concept
     # the body and the prop['foo'] are the same concept 'foo'
     assert cbody(concept_found_2) == get_expected(foo, "one two")
@@ -400,13 +400,13 @@ def test_i_can_use_a_reference_with_a_body():
 
     assert res[0].status
     assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
-    assert res[0].value.body == [("foo", 0, 2, "one two")]
+    assert res[0].value.body == [cnode("foo", 0, 2, "one two")]
     concept_found_1 = res[0].value.body[0].concept
     assert concept_found_1.body == "'foo'"
 
     assert res[1].status
     assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT)
-    assert res[1].value.body == [("bar", 0, 2, "one two")]
+    assert res[1].value.body == [cnode("bar", 0, 2, "one two")]
     concept_found_2 = res[1].value.body[0].concept
     # the body and the prop['foo'] are the same concept 'foo'
     assert cbody(concept_found_2) == foo
@@ -430,20 +430,20 @@ def test_i_can_use_context_reference_with_multiple_levels():
 
     assert res[0].status
     assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
-    assert res[0].value.body == [("foo", 0, 2, "one two")]
+    assert res[0].value.body == [cnode("foo", 0, 2, "one two")]
     concept_found_1 = res[0].value.body[0].concept
     assert cbody(concept_found_1) == DoNotResolve("one two")
 
     assert res[1].status
     assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT)
-    assert res[1].value.body == [("bar", 0, 2, "one two")]
+    assert res[1].value.body == [cnode("bar", 0, 2, "one two")]
     concept_found_2 = res[1].value.body[0].concept
     assert cbody(concept_found_2) == get_expected(foo, "one two")
     assert id(cprop(concept_found_2, "foo")) == id(cbody(concept_found_2))
 
     assert res[2].status
     assert context.sheerka.isinstance(res[2].value, BuiltinConcepts.PARSER_RESULT)
-    assert res[2].value.body == [("baz", 0, 2, "one two")]
+    assert res[2].value.body == [cnode("baz", 0, 2, "one two")]
     concept_found_3 = res[2].value.body[0].concept
     expected_foo = get_expected(foo, "one two")
     assert cbody(concept_found_3) == get_expected(bar, expected_foo)
@@ -465,8 +465,8 @@ def test_order_is_not_important_when_using_references():
 
     res = parser.parse(context, "one two")
     assert len(res) == 2
-    assert res[0].value.body == [("bar", 0, 2, "one two")]
-    assert res[1].value.body == [("foo", 0, 2, "one two")]
+    assert res[0].value.body == [cnode("bar", 0, 2, "one two")]
+    assert res[1].value.body == [cnode("foo", 0, 2, "one two")]
 
 
 def test_i_can_parse_when_reference():
@@ -477,21 +477,21 @@ def test_i_can_parse_when_reference():
 
     res = parser.parse(context, "twenty two")
     assert res.status
-    assert res.value.body == [("bar", 0, 2, "twenty two")]
+    assert res.value.body == [cnode("bar", 0, 2, "twenty two")]
     concept_found = res.value.body[0].concept
     assert cbody(concept_found) == DoNotResolve("twenty two")
     assert cprop(concept_found, "foo") == get_expected(foo, "twenty")
 
     res = parser.parse(context, "thirty one")
     assert res.status
-    assert res.value.body == [("bar", 0, 2, "thirty one")]
+    assert res.value.body == [cnode("bar", 0, 2, "thirty one")]
     concept_found = res.value.body[0].concept
     assert cbody(concept_found) == DoNotResolve("thirty one")
     assert cprop(concept_found, "foo") == get_expected(foo, "thirty")
 
     res = parser.parse(context, "twenty")
     assert res.status
-    assert res.value.body == [("foo", 0, 0, "twenty")]
+    assert res.value.body == [cnode("foo", 0, 0, "twenty")]
     concept_found = res.value.body[0].concept
     assert cbody(concept_found) == DoNotResolve("twenty")
 
@@ -504,14 +504,14 @@ def test_i_can_parse_when_reference_has_a_body():
 
     res = parser.parse(context, "twenty two")
     assert res.status
-    assert res.value.body == [("bar", 0, 2, "twenty two")]
+    assert res.value.body == [cnode("bar", 0, 2, "twenty two")]
     concept_found = res.value.body[0].concept
     assert cbody(concept_found) == DoNotResolve("twenty two")
     assert cprop(concept_found, "foo") == foo
 
     res = parser.parse(context, "twenty")
     assert res.status
-    assert res.value.body == [("foo", 0, 0, "twenty")]
+    assert res.value.body == [cnode("foo", 0, 0, "twenty")]
     concept_found = res.value.body[0].concept
     assert concept_found.body == "'one'"
 
@@ -529,14 +529,14 @@ def test_i_can_parse_multiple_results():
     assert len(res) == 2
     assert res[0].status
     assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
-    assert res[0].value.body == [("bar", 0, 2, "one two")]
+    assert res[0].value.body == [cnode("bar", 0, 2, "one two")]
     concept_found_0 = res[0].value.body[0].concept
     assert cbody(concept_found_0) == DoNotResolve("one two")
     assert len(concept_found_0.props) == 0
 
     assert res[1].status
     assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT)
-    assert res[1].value.body == [("foo", 0, 2, "one two")]
+    assert res[1].value.body == [cnode("foo", 0, 2, "one two")]
     concept_found_1 = res[1].value.body[0].concept
     assert cbody(concept_found_1) == DoNotResolve("one two")
     assert len(concept_found_1.props) == 0
@@ -555,19 +555,19 @@ def test_i_can_parse_multiple_results_times_two():
     assert len(res) == 4
     assert res[0].status
     assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
-    assert res[0].value.body == [("bar", "one two"), ("bar", "one two")]
+    assert res[0].value.body == [short_cnode("bar", "one two"), short_cnode("bar", "one two")]
 
     assert res[1].status
     assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT)
-    assert res[1].value.body == [("foo", "one two"), ("bar", "one two")]
+    assert res[1].value.body == [short_cnode("foo", "one two"), short_cnode("bar", "one two")]
 
     assert res[2].status
     assert context.sheerka.isinstance(res[2].value, BuiltinConcepts.PARSER_RESULT)
-    assert res[2].value.body == [("bar", "one two"), ("foo", "one two")]
+    assert res[2].value.body == [short_cnode("bar", "one two"), short_cnode("foo", "one two")]
 
     assert res[3].status
     assert context.sheerka.isinstance(res[3].value, BuiltinConcepts.PARSER_RESULT)
-    assert res[3].value.body == [("foo", "one two"), ("foo", "one two")]
+    assert res[3].value.body == [short_cnode("foo", "one two"), short_cnode("foo", "one two")]
 
 
 def test_i_can_parse_multiple_results_when_reference():
@@ -589,11 +589,11 @@ def test_i_can_parse_multiple_results_when_reference():
     assert len(res) == 2
     assert res[0].status
     assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
-    assert res[0].value.body == [("bar", 0, 0, "twenty")]
+    assert res[0].value.body == [cnode("bar", 0, 0, "twenty")]
 
     assert res[1].status
     assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT)
-    assert res[1].value.body == [("foo", 0, 0, "twenty")]
+    assert res[1].value.body == [cnode("foo", 0, 0, "twenty")]
 
 
 def test_i_can_parse_concept_reference_that_is_not_in_grammar():
@@ -608,14 +608,14 @@ def test_i_can_parse_concept_reference_that_is_not_in_grammar():
 
     res = parser.parse(context, "twenty two")
     assert res.status
-    assert res.value.body == [("foo", 0, 2, "twenty two")]
+    assert res.value.body == [cnode("foo", 0, 2, "twenty two")]
     concept_found = res.value.body[0].concept
     assert cbody(concept_found) == DoNotResolve("twenty two")
     assert cprop(concept_found, "two") == get_expected(two, "two")
 
     res = parser.parse(context, "twenty one")
     assert res.status
-    assert res.value.body == [("foo", 0, 2, "twenty one")]
+    assert res.value.body == [cnode("foo", 0, 2, "twenty one")]
 
 
 def test_i_can_parse_zero_or_more():
@@ -625,7 +625,7 @@ def test_i_can_parse_zero_or_more():
     context, res, wrapper, return_value = execute([foo], grammar, "one one")
 
     assert res.status
-    assert return_value == [("foo", 0, 2, "one one")]
+    assert return_value == [cnode("foo", 0, 2, "one one")]
     assert return_value[0].underlying == u(grammar[foo], 0, 2, [u("one", 0, 0), u("one", 2, 2)])
 
     concept_found = return_value[0].concept
@@ -639,11 +639,11 @@ def test_i_can_parse_sequence_and_zero_or_more():
 
     res = parser.parse(context, "one one two")
     assert res.status
-    assert res.value.value == [("foo", 0, 4, "one one two")]
+    assert res.value.value == [cnode("foo", 0, 4, "one one two")]
 
     res = parser.parse(context, "two")
     assert res.status
-    assert res.value.value == [("foo", 0, 0, "two")]
+    assert res.value.value == [cnode("foo", 0, 0, "two")]
 
 
 def test_i_cannot_parse_zero_and_more_when_wrong_entry():
@@ -657,7 +657,7 @@ def test_i_cannot_parse_zero_and_more_when_wrong_entry():
     res = parser.parse(context, "one two")
     assert not res.status
     assert res.value.value == [
-        ("foo", 0, 0, "one"),
+        cnode("foo", 0, 0, "one"),
         UnrecognizedTokensNode(1, 2, [t(" "), t("two")])
     ]
 
@@ -675,7 +675,7 @@ def test_i_can_parse_zero_and_more_with_separator():
     context, res, wrapper, return_value = execute([foo], grammar, "one, one , one")
 
     assert res.status
-    assert return_value == [("foo", 0, 7, "one, one , one")]
+    assert return_value == [cnode("foo", 0, 7, "one, one , one")]
 
 
 def test_that_zero_and_more_is_greedy():
@@ -686,7 +686,7 @@ def test_that_zero_and_more_is_greedy():
     context, res, wrapper, return_value = execute([foo], grammar, "one one one")
 
     assert res.status
-    assert return_value == [("foo", 0, 4, "one one one")]
+    assert return_value == [cnode("foo", 0, 4, "one one one")]
 
 
 def test_i_can_parse_one_and_more():
@@ -696,7 +696,7 @@ def test_i_can_parse_one_and_more():
     context, res, wrapper, return_value = execute([foo], grammar, "one one")
 
     assert res.status
-    assert return_value == [("foo", 0, 2, "one one")]
+    assert return_value == [cnode("foo", 0, 2, "one one")]
     assert return_value[0].underlying == u(grammar[foo], 0, 2, [
         u("one", 0, 0),
         u("one", 2, 2)])
@@ -709,7 +709,7 @@ def test_i_can_parse_sequence_and_one_or_more():
 
     res = parser.parse(context, "one one two")
     assert res.status
-    assert res.value.value == [("foo", 0, 4, "one one two")]
+    assert res.value.value == [cnode("foo", 0, 4, "one one two")]
 
     res = parser.parse(context, "two")
     assert not res.status
@@ -725,7 +725,7 @@ def test_i_can_parse_one_and_more_with_separator():
     context, res, wrapper, return_value = execute([foo], grammar, "one, one , one")
 
     assert res.status
-    assert return_value == [("foo", 0, 7, "one, one , one")]
+    assert return_value == [cnode("foo", 0, 7, "one, one , one")]
     assert return_value[0].underlying == u(grammar[foo], 0, 7, [
         u("one", 0, 0),
         u("one", 3, 3),
@@ -740,7 +740,7 @@ def test_that_one_and_more_is_greedy():
     context, res, wrapper, return_value = execute([foo], grammar, "one one one")
 
     assert res.status
-    assert return_value == [("foo", 0, 4, "one one one")]
+    assert return_value == [cnode("foo", 0, 4, "one one one")]
 
 
 def test_i_can_detect_infinite_recursion():
@@ -785,9 +785,9 @@ def test_i_can_detect_indirect_infinite_recursion_with_ordered_choice():
     res = parser.parse(context, "foo")
     assert len(res) == 2
     assert res[0].status
-    assert res[0].value.body == [("bar", 0, 0, "foo")]
+    assert res[0].value.body == [cnode("bar", 0, 0, "foo")]
     assert res[1].status
-    assert res[1].value.body == [("foo", 0, 0, "foo")]
+    assert res[1].value.body == [cnode("foo", 0, 0, "foo")]
 
 
 def test_i_can_detect_indirect_infinite_recursion_with_sequence():
@@ -912,7 +912,7 @@ def test_i_cannot_parse_when_wrong_sequence():
     assert not res.status
     assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
     assert return_value == [
-        ("foo", "one two three"),
+        short_cnode("foo", "one two three"),
         UnrecognizedTokensNode(5, 6, [t(" "), t("one")])
     ]
 
@@ -945,14 +945,14 @@ def test_i_cannot_parse_multiple_results_when_unknown_tokens_at_the_end():
     assert not res[0].status
     assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
     assert res[0].value.body == [
-        ("bar", 0, 2, "one two"),
+        cnode("bar", 0, 2, "one two"),
         UnrecognizedTokensNode(3, 6, [t(" "), t("four"), t(" "), t("five")])
     ]
 
     assert not res[1].status
     assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT)
     assert res[1].value.body == [
-        ("foo", 0, 2, "one two"),
+        cnode("foo", 0, 2, "one two"),
         UnrecognizedTokensNode(3, 6, [t(" "), t("four"), t(" "), t("five")])
     ]
 
@@ -973,14 +973,14 @@ def test_i_cannot_parse_multiple_results_when_beginning_by_unknown_tokens():
     assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
     assert res[0].value.body == [
         UnrecognizedTokensNode(0, 3, [t("four"), t(" "), t("five"), t(" ")]),
-        ("bar", 4, 6, "one two"),
+        cnode("bar", 4, 6, "one two"),
     ]
 
     assert not res[1].status
     assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT)
     assert res[1].value.body == [
         UnrecognizedTokensNode(0, 3, [t("four"), t(" "), t("five"), t(" ")]),
-        ("foo", 4, 6, "one two"),
+        cnode("foo", 4, 6, "one two"),
     ]
 
 
@@ -999,7 +999,7 @@ def test_i_cannot_parse_multiple_results_when_surrounded_by_unknown_tokens():
     assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
     assert res[0].value.body == [
         UnrecognizedTokensNode(0, 3, [t("four"), t(" "), t("five"), t(" ")]),
-        ("bar", 4, 6, "one two"),
+        cnode("bar", 4, 6, "one two"),
         UnrecognizedTokensNode(7, 10, [t(" "), t("six"), t(" "), t("seven")]),
     ]
 
@@ -1007,7 +1007,7 @@ def test_i_cannot_parse_multiple_results_when_surrounded_by_unknown_tokens():
     assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT)
     assert res[1].value.body == [
         UnrecognizedTokensNode(0, 3, [t("four"), t(" "), t("five"), t(" ")]),
-        ("foo", 4, 6, "one two"),
+        cnode("foo", 4, 6, "one two"),
         UnrecognizedTokensNode(7, 10, [t(" "), t("six"), t(" "), t("seven")]),
     ]
 
@@ -1029,17 +1029,17 @@ def test_i_cannot_parse_multiple_results_when_unknown_tokens_in_the_middle():
     assert not res[0].status
     assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
     assert res[0].value.body == [
-        ("bar", 0, 2, "one two"),
+        cnode("bar", 0, 2, "one two"),
         UnrecognizedTokensNode(3, 7, [t(" "), t("four"), t(" "), t("five"), t(" ")]),
-        ("baz", 8, 8, "six"),
+        cnode("baz", 8, 8, "six"),
     ]
 
     assert not res[1].status
     assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT)
     assert res[1].value.body == [
-        ("foo", 0, 2, "one two"),
+        cnode("foo", 0, 2, "one two"),
         UnrecognizedTokensNode(3, 7, [t(" "), t("four"), t(" "), t("five"), t(" ")]),
-        ("baz", 8, 8, "six"),
+        cnode("baz", 8, 8, "six"),
     ]
 
 
@@ -1052,7 +1052,7 @@ def test_i_can_get_the_inner_concept_when_possible():
 
     assert res.status
     assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert return_value == [("foo", 0, 0, "one")]
+    assert return_value == [cnode("foo", 0, 0, "one")]
     concept_found = return_value[0].concept
     assert cbody(concept_found) == get_expected(one, "one")
     assert id(cprop(concept_found, "one")) == id(cbody(concept_found))
@@ -1069,7 +1069,7 @@ def test_i_can_get_the_inner_concept_when_possible_with_rule_name():
 
     assert res.status
     assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert return_value == [("foo", 0, 0, "one")]
+    assert return_value == [cnode("foo", 0, 0, "one")]
     concept_found = return_value[0].concept
     assert cbody(concept_found) == get_expected(one, "one")
     assert id(cprop(concept_found, "one")) == id(cbody(concept_found))
@@ -1086,7 +1086,7 @@ def test_i_get_multiple_props_when_zero_or_more():
     context, res, wrapper, return_value = execute([foo, one], grammar, "one one one")
     assert res.status
     assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert return_value == [("foo", 0, 4, "one one one")]
+    assert return_value == [cnode("foo", 0, 4, "one one one")]
     concept_found = return_value[0].concept
     assert cbody(concept_found) == DoNotResolve("one one one")
     assert len(concept_found.cached_asts["one"]) == 3
@@ -1106,7 +1106,7 @@ def test_i_get_multiple_props_when_zero_or_more_and_different_values():
     context, res, wrapper, return_value = execute([foo, one], grammar, "one ok un ok uno ok")
     assert res.status
     assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
-    assert return_value == [("foo", "one ok un ok uno ok")]
+    assert return_value == [short_cnode("foo", "one ok un ok uno ok")]
     concept_found = return_value[0].concept
     assert cprop(concept_found, "one")[0] == get_expected(one, "one")
     assert cprop(concept_found, "one")[1] == get_expected(one, "un")

tests/test_ConceptsWithConceptsParser.py

@@ -0,0 +1,204 @@
+import ast
+
+import pytest
+
+from core.builtin_concepts import ParserResultConcept, ReturnValueConcept, BuiltinConcepts
+from core.concept import Concept
+from core.sheerka import Sheerka, ExecutionContext
+from core.tokenizer import Token, TokenKind, Tokenizer
+from parsers.ConceptLexerParser import ConceptNode, UnrecognizedTokensNode, SourceCodeNode
+from parsers.ConceptsWithConceptsParser import ConceptsWithConceptsParser
+from parsers.MultipleConceptsParser import MultipleConceptsParser
+from parsers.PythonParser import PythonNode
+from sdp.sheerkaDataProvider import Event
+
+multiple_concepts_parser = MultipleConceptsParser()
+
+
+def get_context():
+    sheerka = Sheerka(skip_builtins_in_db=True)
+    sheerka.initialize("mem://")
+    return ExecutionContext("test", Event(), sheerka)
+
+
+def get_ret_from(*args):
+    result = []
+    index = 0
+    source = ""
+    for item in args:
+        if isinstance(item, Concept):
+            tokens = [Token(TokenKind.IDENTIFIER, item.name, 0, 0, 0)]
+            result.append(ConceptNode(item, index, index, tokens, item.name))
+            index += 1
+            source += item.name
+        elif isinstance(item, PythonNode):
+            tokens = list(Tokenizer(item.source))[:-1]  # strip trailing EOF
+            result.append(SourceCodeNode(item, index, index + len(tokens) - 1, tokens, item.source))
+            index += len(tokens)
+            source += item.source
+        else:
+            tokens = list(Tokenizer(item))[:-1]  # strip trailing EOF
+            result.append(UnrecognizedTokensNode(index, index + len(tokens) - 1, tokens))
+            index += len(tokens)
+            source += item
+
+    return ReturnValueConcept(
+        "who",
+        False,
+        ParserResultConcept(parser=multiple_concepts_parser, value=result, source=source))
+
+
+def init(concepts, inputs):
+    context = get_context()
+    for concept in concepts:
+        context.sheerka.create_new_concept(context, concept)
+
+    return context, get_ret_from(*inputs)
+
+
+def execute(concepts, inputs):
+    context, input_return_values = init(concepts, inputs)
+
+    parser = ConceptsWithConceptsParser()
+    result = parser.parse(context, input_return_values.body)
+
+    wrapper = result.body
+    return_value = result.body.body
+
+    return context, parser, result, wrapper, return_value
+
+
+@pytest.mark.parametrize("text, interested", [
+    ("not parser result", False),
+    (ParserResultConcept(parser="not multiple_concepts_parser"), False),
+    (ParserResultConcept(parser=multiple_concepts_parser, value=[]), True),
+])
+def test_not_interested(text, interested):
+    context = get_context()
+
+    res = ConceptsWithConceptsParser().parse(context, text)
+    if interested:
+        assert res is not None
+    else:
+        assert res is None
+
+
+def test_i_can_parse_composition_of_concepts():
+    foo = Concept("foo")
+    bar = Concept("bar")
+    plus = Concept("a plus b").set_prop("a").set_prop("b")
+
+    context, parser, result, wrapper, return_value = execute([foo, bar, plus], [foo, " plus ", bar])
+
+    assert result.status
+    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+    assert result.who == wrapper.parser.name
+    assert wrapper.source == "foo plus bar"
+    assert context.sheerka.isinstance(return_value, plus)
+
+    assert return_value.cached_asts["a"] == foo
+    assert return_value.cached_asts["b"] == bar
+
+    # sanity check, I can evaluate the result
+    evaluated = context.sheerka.evaluate_concept(context, return_value)
+    assert evaluated.key == return_value.key
+    assert evaluated.get_prop("a") == foo.init_key()
+    assert evaluated.get_prop("b") == bar.init_key()
+
+
+def test_i_can_parse_when_composition_of_source_code():
+    plus = Concept("a plus b", body="a + b").set_prop("a").set_prop("b")
+    left = PythonNode("1+1", ast.parse("1+1", mode="eval"))
+    right = PythonNode("2+2", ast.parse("2+2", mode="eval"))
+    context, parser, result, wrapper, return_value = execute([plus], [left, " plus ", right])
+
+    assert result.status
+    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+    assert result.who == wrapper.parser.name
+    assert wrapper.source == "1+1 plus 2+2"
+    assert context.sheerka.isinstance(return_value, plus)
+
+    left_parser_result = ParserResultConcept(parser=parser, source="1+1", value=left)
+    right_parser_result = ParserResultConcept(parser=parser, source="2+2", value=right)
+    assert return_value.cached_asts["a"] == [ReturnValueConcept(parser.name, True, left_parser_result)]
+    assert return_value.cached_asts["b"] == [ReturnValueConcept(parser.name, True, right_parser_result)]
+
+    # sanity check, I can evaluate the result
+    evaluated = context.sheerka.evaluate_concept(context, return_value)
+    assert evaluated.key == return_value.key
+    assert evaluated.get_prop("a") == 2
+    assert evaluated.get_prop("b") == 4
+    assert evaluated.body == 6
+
+
+def test_i_can_parse_when_mix_of_concept_and_code():
+    plus = Concept("a plus b").set_prop("a").set_prop("b")
+    code = PythonNode("1+1", ast.parse("1+1", mode="eval"))
+    foo = Concept("foo")
+    context, parser, result, wrapper, return_value = execute([plus, foo], [foo, " plus ", code])
+
+    assert result.status
+    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+    assert result.who == wrapper.parser.name
+    assert wrapper.source == "foo plus 1+1"
+    assert context.sheerka.isinstance(return_value, plus)
+
+    code_parser_result = ParserResultConcept(parser=parser, source="1+1", value=code)
+    assert return_value.cached_asts["a"] == foo
+    assert return_value.cached_asts["b"] == [ReturnValueConcept(parser.name, True, code_parser_result)]
+
+    # sanity check, I can evaluate the result
+    evaluated = context.sheerka.evaluate_concept(context, return_value)
+    assert evaluated.key == return_value.key
+    assert evaluated.get_prop("a") == foo.init_key()
+    assert evaluated.get_prop("b") == 2
+
+
+def test_i_can_parse_when_multiple_concepts_are_recognized():
+    foo = Concept("foo")
+    bar = Concept("bar")
+    plus_1 = Concept("a plus b", body="body1").set_prop("a").set_prop("b")
+    plus_2 = Concept("a plus b", body="body2").set_prop("a").set_prop("b")
+
+    context, input_return_values = init([foo, bar, plus_1, plus_2], [foo, " plus ", bar])
+    parser = ConceptsWithConceptsParser()
+    result = parser.parse(context, input_return_values.body)
+
+    assert len(result) == 2
+
+    res = result[0]
+    wrapper = res.value
+    return_value = res.value.value
+    assert res.status
+    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+    assert res.who == wrapper.parser.name
+    assert wrapper.source == "foo plus bar"
+    assert context.sheerka.isinstance(return_value, plus_1)
+    assert return_value.cached_asts["a"] == foo
+    assert return_value.cached_asts["b"] == bar
+
+    res = result[1]
+    wrapper = res.value
+    return_value = res.value.value
+    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+    assert res.who == wrapper.parser.name
+    assert wrapper.source == "foo plus bar"
+    assert context.sheerka.isinstance(return_value, plus_2)
+    assert return_value.cached_asts["a"] == foo
+    assert return_value.cached_asts["b"] == bar
+
+
+def test_i_cannot_parse_when_unknown_concept():
+    foo = Concept("foo")
+    bar = Concept("bar")
+
+    context, input_return_values = init([foo, bar], [foo, " plus ", bar])
+    parser = ConceptsWithConceptsParser()
+    result = parser.parse(context, input_return_values.body)
+    wrapper = result.body
+    return_value = result.body.body
+
+    assert not result.status
+    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.NOT_FOR_ME)
+    assert result.who == parser.name
+    assert return_value == input_return_values.body.body

tests/test_LexerNodeEvaluator.py

@@ -1,11 +1,13 @@
+import ast
+
 import pytest
 
 from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
 from core.concept import Concept, ConceptParts, DoNotResolve
 from core.sheerka import Sheerka, ExecutionContext
-from evaluators.ConceptNodeEvaluator import ConceptNodeEvaluator
-from parsers.ConceptLexerParser import ConceptNode, ConceptLexerParser, Sequence, TerminalNode, \
-    StrMatch, Optional, OrderedChoice, ZeroOrMore, UnrecognizedTokensNode, ConceptMatch
+from evaluators.LexerNodeEvaluator import LexerNodeEvaluator
+from parsers.ConceptLexerParser import ConceptNode, ConceptLexerParser, StrMatch, UnrecognizedTokensNode, SourceCodeNode
+from parsers.PythonParser import PythonNode
 from sdp.sheerkaDataProvider import Event
 
 
@@ -24,6 +26,18 @@ def from_parsing(context, grammar, expression):
     return ret_val
 
 
+def from_fragments(*fragments):
+    nodes = []
+    for fragment in fragments:
+        if isinstance(fragment, str):
+            node = PythonNode(fragment, ast.parse(fragment.strip(), mode="eval"))
+            nodes.append(SourceCodeNode(node, 0, 0, [], fragment))
+        else:
+            nodes.append(ConceptNode(fragment, 0, 0, [], fragment.name))
+
+    return ReturnValueConcept("somme_name", True, ParserResultConcept(value=nodes))
+
+
 def init(concept, grammar, text):
     context = get_context()
     if isinstance(concept, list):
@@ -40,12 +54,14 @@ def init(concept, grammar, text):
 @pytest.mark.parametrize("ret_val, expected", [
     (ReturnValueConcept("some_name", True, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), True),
     (ReturnValueConcept("some_name", True, ParserResultConcept(value=ConceptNode(Concept(), 0, 0))), True),
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value=[UnrecognizedTokensNode(0, 0, [])])), True),
-    (ReturnValueConcept("some_name", True, ParserResultConcept(value=UnrecognizedTokensNode(0, 0, []))), True),
+    (ReturnValueConcept("some_name", True, ParserResultConcept(value=[SourceCodeNode(0, 0, [])])), True),
+    (ReturnValueConcept("some_name", True, ParserResultConcept(value=SourceCodeNode(0, 0, []))), True),
+    (ReturnValueConcept("some_name", True, ParserResultConcept(value=[UnrecognizedTokensNode(0, 0, [])])), False),
+    (ReturnValueConcept("some_name", True, ParserResultConcept(value=UnrecognizedTokensNode(0, 0, []))), False),
     (ReturnValueConcept("some_name", False, ParserResultConcept(value=[ConceptNode(Concept(), 0, 0)])), False),
     (ReturnValueConcept("some_name", False, ParserResultConcept(value=ConceptNode(Concept(), 0, 0))), False),
-    (ReturnValueConcept("some_name", False, ParserResultConcept(value=[UnrecognizedTokensNode(0, 0, [])])), False),
-    (ReturnValueConcept("some_name", False, ParserResultConcept(value=UnrecognizedTokensNode(0, 0, []))), False),
+    (ReturnValueConcept("some_name", False, ParserResultConcept(value=[SourceCodeNode(0, 0, [])])), False),
+    (ReturnValueConcept("some_name", False, ParserResultConcept(value=SourceCodeNode(0, 0, []))), False),
     (ReturnValueConcept("some_name", True, ParserResultConcept(value="Not a concept node")), False),
     (ReturnValueConcept("some_name", True, ParserResultConcept(value=["Not a concept node"])), False),
     (ReturnValueConcept("some_name", True, [ConceptNode(Concept(), 0, 0)]), False),
@@ -53,7 +69,7 @@ def init(concept, grammar, text):
 ])
 def test_i_can_match(ret_val, expected):
     context = get_context()
-    assert ConceptNodeEvaluator().matches(context, ret_val) == expected
+    assert LexerNodeEvaluator().matches(context, ret_val) == expected
 
 
 def test_concept_is_returned_when_only_one_in_the_list():
@@ -61,9 +77,9 @@ def test_concept_is_returned_when_only_one_in_the_list():
     context = get_context()
     context.sheerka.add_in_cache(foo)
 
-    evaluator = ConceptNodeEvaluator()
     ret_val = from_parsing(context, {foo: StrMatch("foo")}, "foo")
 
+    evaluator = LexerNodeEvaluator()
     result = evaluator.eval(context, ret_val)
     wrapper = result.body
     return_value = result.body.body
@@ -77,3 +93,23 @@ def test_concept_is_returned_when_only_one_in_the_list():
     assert return_value.cached_asts[ConceptParts.BODY] == DoNotResolve("foo")
     assert result.parents == [ret_val]
 
+
+def test_concept_python_node_is_returned_when_source_code():
+    context = get_context()
+    foo = Concept("foo")
+    ret_val = from_fragments(foo, " + 1")
+
+    evaluator = LexerNodeEvaluator()
+    result = evaluator.eval(context, ret_val)
+    wrapper = result.body
+    return_value = result.body.body
+
+    assert result.who == evaluator.name
+    assert result.status
+    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+    assert wrapper.parser == evaluator
+    assert wrapper.source == "foo + 1"
+
+    assert return_value == PythonNode('foo + 1', ast.parse("__C__foo__C__ + 1", mode="eval"))
+    assert return_value.concepts == {"__C__foo__C__": foo}
+    assert result.parents == [ret_val]

tests/test_MultipleConceptsParser.py

@@ -1,8 +1,12 @@
+import pytest
+
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts
 from core.concept import Concept
 from core.sheerka import Sheerka, ExecutionContext
-from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, UnrecognizedTokensNode
+from core.tokenizer import Tokenizer, TokenKind, Token
+from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, cnode, utnode, scnode, SourceCodeNode
 from parsers.MultipleConceptsParser import MultipleConceptsParser
+from parsers.PythonParser import PythonNode
 from sdp.sheerkaDataProvider import Event
 
 
@@ -78,7 +82,7 @@ def test_i_can_parse_when_ending_with_bnf():
     assert ret_val.status
     assert ret_val.who == parser.name
     assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
-    assert ret_val.value.value == [("bar", 0, 0, "bar"), ("foo", 2, 6, "foo1 foo2 foo3")]
+    assert ret_val.value.value == [cnode("bar", 0, 0, "bar"), cnode("foo", 2, 6, "foo1 foo2 foo3")]
     assert ret_val.value.source == "bar foo1 foo2 foo3"
 
 
@@ -94,7 +98,7 @@ def test_i_can_parse_when_starting_with_bnf():
     assert ret_val.status
     assert ret_val.who == parser.name
     assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
-    assert ret_val.value.value == [("foo", 0, 4, "foo1 foo2 foo3"), ("bar", 6, 6, "bar")]
+    assert ret_val.value.value == [cnode("foo", 0, 4, "foo1 foo2 foo3"), cnode("bar", 6, 6, "bar")]
     assert ret_val.value.source == "foo1 foo2 foo3 bar"
 
 
@@ -112,13 +116,13 @@ def test_i_can_parse_when_concept_are_mixed():
     assert ret_val.who == parser.name
     assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
     assert ret_val.value.value == [
-        ("baz", 0, 0, "baz"),
-        ("foo", 2, 6, "foo1 foo2 foo3"),
-        ("bar", 8, 8, "bar")]
+        cnode("baz", 0, 0, "baz"),
+        cnode("foo", 2, 6, "foo1 foo2 foo3"),
+        cnode("bar", 8, 8, "bar")]
     assert ret_val.value.source == "baz foo1 foo2 foo3 bar"
 
 
-def test_i_can_parse_when_multiple_concept_are_matching():
+def test_i_can_parse_when_multiple_concepts_are_matching():
     foo = Concept("foo")
     bar = Concept("bar", body="bar1")
     baz = Concept("bar", body="bar2")
@@ -130,16 +134,35 @@ def test_i_can_parse_when_multiple_concept_are_matching():
 
     assert len(ret_val) == 2
     assert ret_val[0].status
-    assert ret_val[0].value.value == [("foo", 0, 0, "foo"), ("bar", 2, 2, "bar")]
+    assert ret_val[0].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")]
     assert ret_val[0].value.source == "foo bar"
     assert ret_val[0].value.value[1].concept.body == "bar1"
 
     assert ret_val[1].status
-    assert ret_val[1].value.value == [("foo", 0, 0, "foo"), ("bar", 2, 2, "bar")]
+    assert ret_val[1].value.value == [cnode("foo", 0, 0, "foo"), cnode("bar", 2, 2, "bar")]
     assert ret_val[1].value.source == "foo bar"
     assert ret_val[1].value.value[1].concept.body == "bar2"
 
 
+def test_i_can_parse_when_source_code():
+    foo = Concept("foo")
+    grammar = {foo: "foo"}
+    context, return_value = init([foo], grammar, "1 foo")
+
+    parser = MultipleConceptsParser()
+    ret_val = parser.parse(context, return_value.body)
+    wrapper = ret_val.value
+    value = ret_val.value.value
+
+    assert ret_val.status
+    assert ret_val.who == parser.name
+    assert context.sheerka.isinstance(wrapper, BuiltinConcepts.PARSER_RESULT)
+    assert wrapper.source == "1 foo"
+    assert value == [
+        scnode(0, 1, "1 "),
+        cnode("foo", 2, 2, "foo")]
+
+
 def test_i_cannot_parse_when_unrecognized_token():
     twenty_two = Concept("twenty two")
     one = Concept("one")
@@ -153,8 +176,56 @@ def test_i_cannot_parse_when_unrecognized_token():
     assert ret_val.who == parser.name
     assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
     assert ret_val.value.value == [
-        ("twenty two", 0, 2, "twenty two"),
-        (3, 5, " + "),
-        ("one", 6, 6, "one")
+        cnode("twenty two", 0, 2, "twenty two"),
+        utnode(3, 5, " + "),
+        cnode("one", 6, 6, "one")
     ]
     assert ret_val.value.source == "twenty two + one"
+
+
+def test_i_cannot_parse_when_unknown_concepts():
+    twenty_two = Concept("twenty two")
+    one = Concept("one")
+    grammar = {twenty_two: Sequence("twenty", "two")}
+    context, return_value = init([twenty_two, one], grammar, "twenty two plus one")
+
+    parser = MultipleConceptsParser()
+    ret_val = parser.parse(context, return_value.body)
+
+    assert not ret_val.status
+    assert ret_val.who == parser.name
+    assert context.sheerka.isinstance(ret_val.value, BuiltinConcepts.PARSER_RESULT)
+    assert ret_val.value.value == [
+        cnode("twenty two", 0, 2, "twenty two"),
+        utnode(3, 5, " plus "),
+        cnode("one", 6, 6, "one")
+    ]
+    assert ret_val.value.source == "twenty two plus one"
+
+
+@pytest.mark.parametrize("text, expected_source, expected_end", [
+    ("True", "True", 0),
+    ("1 == 1", "1 == 1", 5),
+    ("1!xdf", "1", 0),
+    ("1", "1", 0),
+])
+def test_i_can_get_source_code_node(text, expected_source, expected_end):
+    tokens = list(Tokenizer(text))[:-1]  # strip trailing EOF
+
+    start_index = 5  # a random number different of zero
+    res = MultipleConceptsParser().get_source_code_node(get_context(), start_index, tokens)
+
+    assert isinstance(res, SourceCodeNode)
+    assert isinstance(res.node, PythonNode)
+    assert res.source == expected_source
+    assert res.start == start_index
+    assert res.end == start_index + expected_end
+
+
+def test_i_cannot_parse_null_text():
+    res = MultipleConceptsParser().get_source_code_node(get_context(), 0, [])
+    assert res is None
+
+    eof = Token(TokenKind.EOF, "", 0, 0, 0)
+    res = MultipleConceptsParser().get_source_code_node(get_context(), 0, [eof])
+    assert res is None

tests/test_PythonWithConceptsParser.py

@@ -41,17 +41,19 @@ def to_str_ast(expression):
     return PythonNode.get_dump(ast.parse(expression, mode="eval"))
 
 
-@pytest.mark.parametrize("text", [
-    "not parser result",
-    ParserResultConcept(value="not a list"),
-    ParserResultConcept(value=[]),
-    ParserResultConcept(value=["not a Node"]),
+@pytest.mark.parametrize("text, interested", [
+    ("not parser result", False),
+    (ParserResultConcept(parser="not multiple_concepts_parser"), False),
+    (ParserResultConcept(parser=multiple_concepts_parser, value=[]), True),
 ])
-def test_not_interested(text):
+def test_not_interested(text, interested):
     context = get_context()
 
     res = PythonWithConceptsParser().parse(context, text)
-    assert res is None
+    if interested:
+        assert res is not None
+    else:
+        assert res is None
 
 
 def test_i_can_parse_concepts_and_python():

tests/test_sheerka_non_reg.py

@@ -401,18 +401,6 @@ def test_i_can_eval_bnf_definitions_from_separate_instances():
     assert res[0].value.props["a"] == Property("a", sheerka.new(concept_a.key, body="one two").init_key())
 
 
-def test_i_can_eval_a_mix_with_bnf_and_python():
-    sheerka = get_sheerka()
-
-    sheerka.evaluate_user_input("def concept one as 1")
-    sheerka.evaluate_user_input("def concept twenties from bnf 'twenty' one as 20 + one")
-
-    res = sheerka.evaluate_user_input("twenty one + 1")
-    assert len(res) == 1
-    assert res[0].status
-    assert res[0].body == 22
-
-
 @pytest.mark.parametrize("desc, definitions", [
     ("Simple form", [
         "def concept one as 1",
@@ -450,16 +438,116 @@ def test_i_can_mix_concept_with_python_to_define_numbers(desc, definitions):
     assert res[0].status
     assert res[0].body == 22
 
+    res = sheerka.evaluate_user_input("twenty one + one")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body == 22
+
     res = sheerka.evaluate_user_input("twenty one + twenty two")
     assert len(res) == 1
     assert res[0].status
     assert res[0].body == 43
 
-    res = sheerka.evaluate_user_input("twenty one + one")
+    res = sheerka.evaluate_user_input("1 + twenty one")
     assert len(res) == 1
     assert res[0].status
     assert res[0].body == 22
 
+    # res = sheerka.evaluate_user_input("1 + 1 + twenty one")
+    # assert len(res) == 1
+    # assert res[0].status
+    # assert res[0].body == 23
+
+
+def test_i_can_mix_concept_of_concept():
+    sheerka = get_sheerka()
+
+    definitions = [
+        "def concept one as 1",
+        "def concept two as 2",
+        "def concept twenties from bnf 'twenty' (one|two)=unit as 20 + unit",
+        "def concept a plus b as a + b"
+    ]
+
+    for definition in definitions:
+        sheerka.evaluate_user_input(definition)
+
+    # res = sheerka.evaluate_user_input("1 plus 2")
+    # assert len(res) == 1
+    # assert res[0].status
+    # assert res[0].body.body == 3
+    #
+    # res = sheerka.evaluate_user_input("1 plus one")
+    # assert len(res) == 1
+    # assert res[0].status
+    # assert res[0].body.body == 2
+    
+    # res = sheerka.evaluate_user_input("1 + 1 plus 1")
+    # assert len(res) == 1
+    # assert res[0].status
+    # assert res[0].body.body == 3
+
+    res = sheerka.evaluate_user_input("1 plus twenty one")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body.body == 22
+
+    res = sheerka.evaluate_user_input("one plus 1")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body.body == 2
+
+    res = sheerka.evaluate_user_input("one plus two")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body.body == 3
+
+    res = sheerka.evaluate_user_input("one plus twenty one")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body.body == 22
+
+    res = sheerka.evaluate_user_input("twenty one plus 1")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body.body == 22
+
+    res = sheerka.evaluate_user_input("twenty one plus one")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body.body == 22
+
+    res = sheerka.evaluate_user_input("twenty one plus twenty two")
+    assert len(res) == 1
+    assert res[0].status
+    assert res[0].body.body == 43
+
+
+# def test_i_can_evaluate_concept_of_concept_when_multiple_choices():
+#     sheerka = get_sheerka()
+#
+#     definitions = [
+#         "def concept little a where a",
+#         "def concept blue a where a",
+#         "def concept little blue a where a",
+#         "def concept house"
+#     ]
+#
+#     for definition in definitions:
+#         sheerka.evaluate_user_input(definition)
+#
+#     ### CAUTION ####
+#     # this test cannot work !!
+#     # it is just to hint the result that I would like to achieve
+#
+#     res = sheerka.evaluate_user_input("little blue house")
+#     assert len(res) == 2
+#     assert res[0].status
+#     assert res[0].body == "little(blue(house))"
+#
+#     assert res[1].status
+#     assert res[1].body == "little blue(house)"
+
 
 def test_i_can_say_that_a_concept_isa_another_concept():
     sheerka = get_sheerka()

tests/test_sheerka_transform.py

@@ -213,6 +213,7 @@ def test_i_can_transform_simple_execution_context():
         'desc': 'this is the desc',
         'children': [],
         'preprocess': None,
+        'inputs': {},
         'values': {},
         'obj': None,
         'concepts': {}

tests/test_tokenizer.py

@@ -4,6 +4,7 @@ from core.tokenizer import Tokenizer, Token, TokenKind, LexerError, Keywords
 
 def test_i_can_tokenize():
     source = "+*-/{}[]()    ,;:.?\n\n\r\r\r\nidentifier_0\t  \t10.15 10 'string\n' \"another string\"=|&<>c:name:"
+    source += "$£€!_identifier°~_^\\`#"
     tokens = list(Tokenizer(source))
     assert tokens[0] == Token(TokenKind.PLUS, "+", 0, 1, 1)
     assert tokens[1] == Token(TokenKind.STAR, "*", 1, 1, 2)
@@ -40,8 +41,20 @@ def test_i_can_tokenize():
     assert tokens[32] == Token(TokenKind.LESS, '<', 79, 6, 21)
     assert tokens[33] == Token(TokenKind.GREATER, '>', 80, 6, 22)
     assert tokens[34] == Token(TokenKind.CONCEPT, 'name', 81, 6, 23)
+    assert tokens[35] == Token(TokenKind.DOLLAR, '$', 88, 6, 30)
+    assert tokens[36] == Token(TokenKind.STERLING, '£', 89, 6, 31)
+    assert tokens[37] == Token(TokenKind.EURO, '€', 90, 6, 32)
+    assert tokens[38] == Token(TokenKind.EMARK, '!', 91, 6, 33)
+    assert tokens[39] == Token(TokenKind.IDENTIFIER, '_identifier', 92, 6, 34)
+    assert tokens[40] == Token(TokenKind.DEGREE, '°', 103, 6, 45)
+    assert tokens[41] == Token(TokenKind.TILDE, '~', 104, 6, 46)
+    assert tokens[42] == Token(TokenKind.UNDERSCORE, '_', 105, 6, 47)
+    assert tokens[43] == Token(TokenKind.CARAT, '^', 106, 6, 48)
+    assert tokens[44] == Token(TokenKind.BACK_SLASH, '\\', 107, 6, 49)
+    assert tokens[45] == Token(TokenKind.BACK_QUOTE, '`', 108, 6, 50)
+    assert tokens[46] == Token(TokenKind.HASH, '#', 109, 6, 51)
 
-    assert tokens[35] == Token(TokenKind.EOF, '', 88, 6, 30)
+    assert tokens[47] == Token(TokenKind.EOF, '', 110, 6, 52)
 
 
 @pytest.mark.parametrize("text, expected", [