First steps of ConceptLexer. Need to update DefaultParser before continuing

tests/test_ConceptEvaluator.py

@@ -8,7 +8,7 @@ from parsers.BaseParser import BaseParser
 
 
 def get_context():
-    sheerka = Sheerka()
+    sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
     return ExecutionContext("test", "xxx", sheerka)
 

tests/test_ConceptLexerParser.py

@@ -0,0 +1,560 @@
+import pytest
+from core.builtin_concepts import BuiltinConcepts
+from core.concept import Concept
+from core.sheerka import Sheerka, ExecutionContext
+from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, StrMatch, OrderedChoice, Optional, \
+    CrossRef
+
+
+@pytest.mark.parametrize("match, text", [
+    ("foo", "foo"),
+    ("'foo'", "'foo'"),
+    ("1", "1"),
+    ("3.14", "3.14"),
+    ("+", "+"),
+    (StrMatch("foo"), "foo"),
+    (StrMatch("'foo'"), "'foo'"),
+    (StrMatch("1"), "1"),
+    (StrMatch("3.14"), "3.14"),
+    (StrMatch("+"), "+"),
+])
+def test_i_can_match_simple_tokens(match, text):
+    context = get_context()
+    foo = Concept(name="foo")
+    concepts = {foo: text}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, text)
+
+    assert res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.PARSER_RESULT)
+    assert res.value.value == [ConceptNode(foo, 0, 0, source=text)]
+
+
+def test_i_can_match_multiple_concepts_in_one_input():
+    context = get_context()
+    one = Concept(name="one")
+    two = Concept(name="two")
+    concepts = {one: "one", two: "two"}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "one two one")
+
+    assert res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.PARSER_RESULT)
+    assert res.value.value == [
+        ConceptNode(one, 0, 0, source="one"),
+        ConceptNode(two, 2, 2, source="two"),
+        ConceptNode(one, 4, 4, source="one"),
+    ]
+
+
+def test_i_cannot_match_an_unknown_input():
+    context = get_context()
+    parser = ConceptLexerParser()  # no grammar registered
+
+    res = parser.parse(context, "foo")
+
+    assert not res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.PARSER_RESULT)
+    assert context.sheerka.isinstance(res.value.body[0], BuiltinConcepts.UNKNOWN_CONCEPT)
+    assert res.value.body[0].body == "foo"
+
+
+def test_i_cannot_match_when_part_of_the_input_is_unknown():
+    context = get_context()
+    one = Concept(name="one")
+    two = Concept(name="two")
+    concepts = {one: "one", two: "two"}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "one two three")
+    assert not res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.PARSER_RESULT)
+    assert res.value.try_parsed == [
+        ConceptNode(one, 0, 0, source="one"),
+        ConceptNode(two, 2, 2, source="two")]  # these two were recognized
+    assert context.sheerka.isinstance(res.value.body[0], BuiltinConcepts.UNKNOWN_CONCEPT)
+    assert res.value.body[0].body == "three"
+
+
+def test_i_can_match_sequence():
+    context = get_context()
+    foo = Concept(name="foo")
+    concepts = {foo: Sequence("one", "two", "three")}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "one two three")
+
+    assert res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.PARSER_RESULT)
+    assert res.value.value == [ConceptNode(foo, 0, 4, source="one two three")]
+
+
+def test_wrong_sequence_is_not_matched():
+    context = get_context()
+    foo = Concept(name="foo")
+    concepts = {foo: Sequence("one", "two", "three")}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "one two three one")
+
+    assert not res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.PARSER_RESULT)
+    assert res.value.try_parsed == [ConceptNode(foo, 0, 4, source="one two three")]
+    assert context.sheerka.isinstance(res.value.body[0], BuiltinConcepts.UNKNOWN_CONCEPT)
+    assert res.value.body[0].body == "one"
+
+
+def test_i_cannot_match_sequence_if_end_of_file():
+    context = get_context()
+    foo = Concept(name="foo")
+    concepts = {foo: Sequence("one", "two", "three")}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "one two")
+    assert not res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.PARSER_RESULT)
+    assert res.value.try_parsed == []
+    assert context.sheerka.isinstance(res.value.body[0], BuiltinConcepts.UNKNOWN_CONCEPT)
+    assert res.value.body[0].body == "one"
+
+
+def test_i_always_choose_the_longest_match():
+    context = get_context()
+    foo = Concept(name="foo")
+    bar = Concept(name="bar")
+    concepts = {bar: Sequence("one", "two"), foo: Sequence("one", "two", "three")}
+
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "one two three")
+
+    assert res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.PARSER_RESULT)
+    assert res.value.value == [ConceptNode(foo, 0, 4, source="one two three")]
+
+
+def test_i_can_match_several_sequences():
+    context = get_context()
+    foo = Concept(name="foo")
+    bar = Concept(name="bar")
+    concepts = {bar: Sequence("one", "two"), foo: Sequence("one", "two", "three")}
+
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "one two three one two")
+
+    assert res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.PARSER_RESULT)
+    assert res.value.value == [
+        ConceptNode(foo, 0, 4, source="one two three"),
+        ConceptNode(bar, 6, 8, source="one two"),
+    ]
+
+
+def test_i_can_match_ordered_choice():
+    context = get_context()
+    foo = Concept(name="foo")
+    concepts = {foo: OrderedChoice("one", "two")}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res1 = parser.parse(context, "one")
+    assert res1.status
+    assert context.sheerka.isinstance(res1.value, BuiltinConcepts.PARSER_RESULT)
+    assert res1.value.body == [ConceptNode(foo, 0, 0, source="one")]
+
+    res2 = parser.parse(context, "two")
+    assert res2.status
+    assert context.sheerka.isinstance(res2.value, BuiltinConcepts.PARSER_RESULT)
+    assert res2.value.body == [ConceptNode(foo, 0, 0, source="two")]
+
+    res3 = parser.parse(context, "three")
+    assert not res3.status
+    assert context.sheerka.isinstance(res3.value.body[0], BuiltinConcepts.UNKNOWN_CONCEPT)
+    assert res3.value.body[0].body == "three"
+
+
+def test_i_cannot_match_ordered_choice_with_empty_alternative():
+    context = get_context()
+    foo = Concept(name="foo")
+    concepts = {foo: Sequence(OrderedChoice("one", ""), "two")}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "ok")  # because token[0] is not "one" and not "" (it is 'two')
+    assert not res.status
+
+
+def test_i_can_mix_sequences_and_ordered_choices():
+    context = get_context()
+    foo = Concept(name="foo")
+
+    concepts = {foo: Sequence(OrderedChoice("twenty", "thirty"), "one", "ok")}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res1 = parser.parse(context, "twenty one ok")
+    assert res1.status
+    assert context.sheerka.isinstance(res1.value, BuiltinConcepts.PARSER_RESULT)
+    assert res1.value.body == [ConceptNode(foo, 0, 4, source="twenty one ok")]
+
+    res2 = parser.parse(context, "thirty one ok")
+    assert res2.status
+    assert context.sheerka.isinstance(res2.value, BuiltinConcepts.PARSER_RESULT)
+    assert res2.value.body == [ConceptNode(foo, 0, 4, source="thirty one ok")]
+
+    res3 = parser.parse(context, "twenty one")
+    assert not res3.status
+    assert res3.value.body[0].body == "twenty"
+    assert res3.value.try_parsed == []
+
+
+def test_i_can_mix_ordered_choices_and_sequences():
+    context = get_context()
+    foo = Concept(name="foo")
+
+    concepts = {foo: OrderedChoice(Sequence("twenty", "thirty"), "one")}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "twenty thirty")
+    assert res.status
+
+    res = parser.parse(context, "one")
+    assert res.status
+
+
+def test_i_cannot_parse_empty_optional():
+    context = get_context()
+    foo = Concept(name="foo")
+
+    concepts = {foo: Optional("one")}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "")
+    assert not res.status
+    assert context.sheerka.isinstance(res.value, BuiltinConcepts.IS_EMPTY)
+
+
+def test_i_can_parse_optional():
+    context = get_context()
+    foo = Concept(name="foo")
+
+    concepts = {foo: Optional("one")}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "one")
+    assert res.status
+    assert res.value.value == [ConceptNode(foo, 0, 0, source="one")]
+
+
+def test_i_can_parse_sequence_starting_with_optional():
+    context = get_context()
+    foo = Concept(name="foo")
+
+    concepts = {foo: Sequence(Optional("twenty"), "one")}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "twenty one")
+    assert res.status
+    assert res.value.body == [ConceptNode(foo, 0, 2, source="twenty one")]
+
+    res = parser.parse(context, "one")
+    assert res.status
+    assert res.value.body == [ConceptNode(foo, 0, 0, source="one")]
+
+
+def test_i_can_parse_sequence_ending_with_optional():
+    context = get_context()
+    foo = Concept(name="foo")
+
+    concepts = {foo: Sequence("one", "two", Optional("three"))}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "one two three")
+    assert res.status
+    assert res.value.body == [ConceptNode(foo, 0, 4, source="one two three")]
+
+    res = parser.parse(context, "one two")
+    assert res.status
+    assert res.value.body == [ConceptNode(foo, 0, 2, source="one two")]
+
+
+def test_i_can_parse_sequence_with_optional_in_between():
+    context = get_context()
+    foo = Concept(name="foo")
+
+    concepts = {foo: Sequence("one", Optional("two"), "three")}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "one two three")
+    assert res.status
+    assert res.value.body == [ConceptNode(foo, 0, 4, source="one two three")]
+
+    res = parser.parse(context, "one three")
+    assert res.status
+    assert res.value.body == [ConceptNode(foo, 0, 2, source="one three")]
+
+
+def test_i_can_use_reference():
+    # The problem here is when there are multiple match for the same input
+    # The parsing result is a list of all concepts found
+    # So it's already a list that represents a sequence, not a choice
+    # So I need to create a choice concept
+    # create the return value for every possible graph
+    # --> The latter seems to be the best as we don't defer the resolution of the problem to someone else
+    context = get_context()
+    foo = Concept(name="foo")
+    bar = Concept(name="bar")
+
+    concepts = {foo: Sequence("one", "two"), bar: foo}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "one two")
+    assert len(res) == 2
+
+    assert res[0].status
+    assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
+    assert res[0].value.body == [ConceptNode(foo, 0, 2, source="one two")]
+
+    assert res[1].status
+    assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT)
+    assert res[1].value.body == [ConceptNode(bar, 0, 2, source="one two")]
+
+
+def test_i_can_use_context_reference_with_multiple_levels():
+    """
+    Same than previous one, but with reference of reference
+    :return:
+    """
+    context = get_context()
+    foo = Concept(name="foo")
+    bar = Concept(name="bar")
+    baz = Concept(name="baz")
+
+    concepts = {foo: Sequence("one", "two"), bar: foo, baz: bar}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "one two")
+    assert len(res) == 3
+
+    assert res[0].status
+    assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
+    assert res[0].value.body == [ConceptNode(foo, 0, 2, source="one two")]
+
+    assert res[1].status
+    assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT)
+    assert res[1].value.body == [ConceptNode(bar, 0, 2, source="one two")]
+
+    assert res[2].status
+    assert context.sheerka.isinstance(res[2].value, BuiltinConcepts.PARSER_RESULT)
+    assert res[2].value.body == [ConceptNode(baz, 0, 2, source="one two")]
+
+
+def test_order_is_not_important_when_using_references():
+    context = get_context()
+    foo = Concept(name="foo")
+    bar = Concept(name="bar")
+
+    concepts = {bar: foo, foo: Sequence("one", "two")}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "one two")
+    assert len(res) == 2
+    assert res[0].value.body == [ConceptNode(bar, 0, 2, source="one two")]
+    assert res[1].value.body == [ConceptNode(foo, 0, 2, source="one two")]
+
+
+def test_i_can_parse_when_reference():
+    context = get_context()
+    foo = Concept(name="foo")
+    bar = Concept(name="bar")
+
+    concepts = {bar: Sequence(foo, OrderedChoice("one", "two")), foo: OrderedChoice("twenty", "thirty")}
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "twenty two")
+    assert res.status
+    assert res.value.body == [ConceptNode(bar, 0, 2, source="twenty two")]
+
+    res = parser.parse(context, "thirty one")
+    assert res.status
+    assert res.value.body == [ConceptNode(bar, 0, 2, source="thirty one")]
+
+    res = parser.parse(context, "twenty")
+    assert res.status
+    assert res.value.body == [ConceptNode(foo, 0, 0, source="twenty")]
+
+
+def test_i_can_detect_duplicates_when_reference():
+    context = get_context()
+    foo = Concept(name="foo")
+    bar = Concept(name="bar")
+
+    concepts = {
+        bar: Sequence(foo, Optional(OrderedChoice("one", "two"))),
+        foo: OrderedChoice("twenty", "thirty")
+    }
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    res = parser.parse(context, "twenty")
+    assert len(res) == 2
+    assert res[0].status
+    assert context.sheerka.isinstance(res[0].value, BuiltinConcepts.PARSER_RESULT)
+    assert res[0].value.body == [ConceptNode(bar, 0, 0, source="twenty")]
+
+    assert res[1].status
+    assert context.sheerka.isinstance(res[1].value, BuiltinConcepts.PARSER_RESULT)
+    assert res[1].value.body == [ConceptNode(foo, 0, 0, source="twenty")]
+
+
+def test_i_can_detect_infinite_recursion():
+    foo = Concept(name="foo")
+    bar = Concept(name="bar")
+
+    concepts = {
+        bar: foo,
+        foo: bar
+    }
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    assert bar not in parser.concepts_dict
+    assert foo not in parser.concepts_dict
+
+
+def test_i_can_detect_indirect_infinite_recursion_with_ordered_choice():
+    foo = Concept(name="foo")
+    bar = Concept(name="bar")
+
+    concepts = {
+        bar: foo,
+        foo: OrderedChoice(bar, "foo")
+    }
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    assert foo not in parser.concepts_dict  # removed because of the infinite recursion
+    assert bar not in parser.concepts_dict  # removed because of the infinite recursion
+
+    # the other way around is possible
+    context = get_context()
+    concepts = {
+        bar: foo,
+        foo: OrderedChoice("foo", bar)
+    }
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+    assert foo in parser.concepts_dict
+    assert bar in parser.concepts_dict
+
+    res = parser.parse(context, "foo")
+    assert len(res) == 2
+    assert res[0].status
+    assert res[0].value.body == [ConceptNode(bar, 0, 0, source="foo")]
+    assert res[1].status
+    assert res[1].value.body == [ConceptNode(foo, 0, 0, source="foo")]
+
+
+def test_i_can_detect_indirect_infinite_recursion_with_sequence():
+    foo = Concept(name="foo")
+    bar = Concept(name="bar")
+
+    concepts = {
+        bar: foo,
+        foo: Sequence("one", bar, "two")
+    }
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    assert foo not in parser.concepts_dict  # removed because of the infinite recursion
+    assert bar not in parser.concepts_dict  # removed because of the infinite recursion
+
+
+def test_i_can_detect_indirect_infinite_recursion_with_sequence_or_ordered_choice():
+    foo = Concept(name="foo")
+    bar = Concept(name="bar")
+
+    concepts = {
+        bar: foo,
+        foo: Sequence("one", OrderedChoice(bar, "other"), "two")
+    }
+    parser = ConceptLexerParser()
+    parser.initialize(concepts)
+
+    assert foo not in parser.concepts_dict  # removed because of the infinite recursion
+    assert bar not in parser.concepts_dict  # removed because of the infinite recursion
+
+
+def test_i_can_detect_indirect_infinite_recursion_with_optional():
+    # TODO infinite recursion with optional
+    pass
+
+#
+# def test_i_can_parse_basic_arithmetic_operations_and_resolve_properties():
+#     context = get_context()
+#     add = Concept(name="add")
+#     mult = Concept(name="mult")
+#     atom = Concept(name="atom")
+#
+#     concepts = {
+#         add: Sequence(mult, Optional(Sequence(OrderedChoice('+', '-', rule_name="sign"), add))),
+#         mult: Sequence(atom, Optional(Sequence(OrderedChoice('*', '/'), mult))),
+#         atom: OrderedChoice(OrderedChoice('1', '2', '3'), Sequence('(', add, ')')),
+#     }
+#
+#     parser = ConceptLexerParser()
+#     parser.register(concepts)
+#
+#     # res = parser.parse(context, "1")
+#     # assert len(res) == 3  # add, mult, atom
+#     #
+#     # res = parser.parse(context, "1 * 2")
+#     # assert len(res) == 2  # add and mult
+#     #
+#     # res = parser.parse(context, "1 + 2")
+#     # assert res.status
+#     # assert res.value.value == [ConceptNode(add, 0, 4, source="1 + 2")]
+#
+#     res = parser.parse(context, "1 * 2 + 3")
+#     assert res.status
+#     assert res.value.value == [ConceptNode(add, 0, 4, source="1 + 2 + 3")]
+
+
+def test_i_can_register_concepts_with_the_same_name():
+    # TODO : concepts are registered by name,
+    #  what when two concepts have the same name ?
+    pass
+
+
+def test_i_can_parse_very_very_long_input():
+    # TODO: In the current implementation, all the tokens are loaded in memory
+    # It's clearly not the good approach
+    pass
+
+
+def get_context():
+    sheerka = Sheerka(skip_builtins_in_db=True)
+    sheerka.initialize("mem://")
+
+    return ExecutionContext("sheerka", "xxxx", sheerka)

tests/test_DefaultParser.py

@@ -67,7 +67,7 @@ def get_concept(name, where=None, pre=None, post=None, body=None):
 
 
 def get_context():
-    sheerka = Sheerka()
+    sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
     return ExecutionContext("test", "xxx", sheerka)
 

tests/test_ExactConceptParser.py

@@ -125,7 +125,7 @@ def test_i_can_detect_concept_from_tokens():
 
 
 def get_context():
-    sheerka = Sheerka()
+    sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
 
     return ExecutionContext("sheerka", "xxxx", sheerka)

tests/test_MultipleSameSuccessEvaluator.py

@@ -7,7 +7,7 @@ from parsers.BaseParser import BaseParser
 
 
 def get_context():
-    sheerka = Sheerka()
+    sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
     return ExecutionContext("test", "xxx", sheerka)
 

tests/test_PyhtonEvaluator.py

@@ -8,7 +8,7 @@ from parsers.PythonParser import PythonNode, PythonParser
 
 
 def get_context():
-    sheerka = Sheerka()
+    sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
     return ExecutionContext("test", "xxx", sheerka)
 

tests/test_PythonParser.py

@@ -9,7 +9,7 @@ from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode
 
 
 def get_context():
-    sheerka = Sheerka()
+    sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
     return ExecutionContext("test", "xxx", sheerka)
 

tests/test_ast.py

@@ -8,7 +8,7 @@ from core.sheerka import Sheerka
 
 
 def get_sheerka():
-    sheerka = Sheerka()
+    sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
 
     return sheerka
@@ -34,7 +34,7 @@ def my_function(a,b):
     return a
 """
     tree = ast.parse(source)
-    tree_as_concept = core.ast.nodes.transform(tree)
+    tree_as_concept = core.ast.nodes.python_to_concept(tree)
     sheerka = get_sheerka()
 
     assert tree_as_concept.node_type == "Module"
@@ -87,7 +87,7 @@ def my_function(a,b):
     """
 
     node = ast.parse(source)
-    concept_node = core.ast.nodes.transform(node)
+    concept_node = core.ast.nodes.python_to_concept(node)
 
     visitor = TestNameVisitor()
     visitor.visit(concept_node)
@@ -115,7 +115,7 @@ my_function(x,y)
     sheerka = get_sheerka()
 
     node = ast.parse(source)
-    concept_node = core.ast.nodes.transform(node)
+    concept_node = core.ast.nodes.python_to_concept(node)
 
     visitor = UnreferencedNamesVisitor(sheerka)
     visitor.visit(concept_node)
@@ -129,3 +129,28 @@ my_function(x,y)
 def test_i_can_compare_NodeParent_with_tuple():
     node_parent = NodeParent(GenericNodeConcept("For", None), "target")
     assert node_parent == ("For", "target")
+
+
+def test_i_can_transform_back():
+    source = """
+def my_function(a,b):
+    for i in range(b):
+        a = a + b
+    return a
+
+
+my_function(x, y)
+    """
+
+    node = ast.parse(source)
+    concept_node = core.ast.nodes.python_to_concept(node)
+
+    transformed_back = core.ast.nodes.concept_to_python(concept_node)
+    assert dump_ast(transformed_back) == dump_ast(node)
+
+
+def dump_ast(node):
+    dump = ast.dump(node)
+    for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]:
+        dump = dump.replace(to_remove, "")
+    return dump

tests/test_builtin_helpers.py

@@ -1,3 +1,7 @@
+import ast
+
+import pytest
+
 from core.builtin_concepts import ReturnValueConcept, BuiltinConcepts
 from core.sheerka import Sheerka, ExecutionContext
 import core.builtin_helpers
@@ -110,8 +114,47 @@ def test_i_can_use_expect_one_when_not_a_list_false():
     assert res == item
 
 
+@pytest.mark.parametrize("expression, vars_to_include, vars_to_exclude, expected_expr", [
+    ("a == 1", [], [], []),
+    ("a == 1", ["a"], [], ["a == 1"]),
+    ("a == 1", [], ["a"], []),
+    ("predicate(a)", [], [], []),
+    ("predicate(a)", ["a"], [], ["predicate(a)"]),
+    ("predicate(a, b)", ["a"], [], ["predicate(a, b)"]),
+    ("predicate(a, b)", ["b"], [], ["predicate(a, b)"]),
+    ("predicate(a, b)", ["a", "b"], [], ["predicate(a, b)"]),
+    ("predicate(a, b)", ["a"], ["b"], []),
+    ("a + b == 1", [], [], []),
+    ("a + b == 1", ["a"], [], ["a + b == 1"]),
+    ("a + b == 1", ["a"], ["b"], []),
+    ("a + b == 1", ["b"], [], ["a + b == 1"]),
+    ("a + b == 1", ["a", "b"], [], ["a + b == 1"]),
+    ("a == 1 and b == 2", [], [], []),
+    ("a == 1 and b == 2", ["a"], [], ["a == 1"]),
+    ("a == 1 and b == 2", ["b"], [], ["b == 2"]),
+    ("a == 1 and b == 2", ["a"], ["b"], ["a == 1"]),
+    ("a == 1 and b == 2", ["a", "b"], [], ["a == 1 and b == 2"]),
+    ("predicate(a,c) and predicate(b,c)", ["a", "b"], [], ["predicate(a,c) and predicate(b,c)"]),
+    ("not(a == 1)", [], [], []),
+    ("not(a == 1)", ["a"], [], ["not(a==1)"]),
+    ("a == 1 or b == 2", [], [], []),
+    ("a == 1 or b == 2", ["a"], [], ["a == 1"]),
+    ("a == 1 or b == 2", ["b"], [], ["b == 2"]),
+    ("a == 1 or b == 2", ["a", "b"], [], ["a == 1 or b == 2"]),
+    ("predicate(a,c) or predicate(b,c)", ["a", "b"], [], ["predicate(a,c) or predicate(b,c)"]),
+])
+def test_i_can_extract_predicates(expression, vars_to_include, vars_to_exclude, expected_expr):
+    sheerka = get_sheerka()
+    expected = [ast.parse(expr, mode="eval") for expr in expected_expr]
+
+    actual = core.builtin_helpers.extract_predicates(sheerka, expression, vars_to_include, vars_to_exclude)
+    assert len(actual) == len(expected)
+    for i in range(len(actual)):
+        assert dump_ast(actual[i]) == dump_ast(expected[i])
+
+
 def get_sheerka():
-    sheerka = Sheerka()
+    sheerka = Sheerka(skip_builtins_in_db=True)
     sheerka.initialize("mem://")
 
     return sheerka
@@ -119,3 +162,10 @@ def get_sheerka():
 
 def get_context(sheerka):
     return ExecutionContext("test", "xxx", sheerka)
+
+
+def dump_ast(node):
+    dump = ast.dump(node)
+    for to_remove in [", ctx=Load()", ", kind=None", ", type_ignores=[]"]:
+        dump = dump.replace(to_remove, "")
+    return dump

tests/test_sheerka.py

@@ -50,7 +50,7 @@ def test_i_can_list_builtin_concepts():
 
 
 def test_builtin_concepts_are_initialized():
-    sheerka = get_sheerka()
+    sheerka = get_sheerka(skip_builtins_in_db=False)
     assert len(sheerka.concepts_cache) == len(BuiltinConcepts)
     for concept_name in BuiltinConcepts:
         assert str(concept_name) in sheerka.concepts_cache
@@ -61,7 +61,7 @@ def test_builtin_concepts_are_initialized():
 
 
 def test_builtin_concepts_can_be_updated():
-    sheerka = get_sheerka(root_folder)
+    sheerka = get_sheerka(root_folder, skip_builtins_in_db=False)
     loaded_sheerka = sheerka.get(BuiltinConcepts.SHEERKA)
     loaded_sheerka.desc = "I have a description"
     sheerka.sdp.modify("Test", sheerka.CONCEPTS_ENTRY, loaded_sheerka.key, loaded_sheerka)
@@ -89,7 +89,8 @@ def test_i_can_add_a_concept():
     assert concept_found.id == "1001"
 
     assert concept.key in sheerka.concepts_cache
-    assert sheerka.sdp.io.exists(sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_digest()))
+    assert sheerka.sdp.io.exists(
+        sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_found.get_digest()))
 
 
 def test_i_cannot_add_the_same_concept_twice():
@@ -414,7 +415,8 @@ as:
         assert getattr(concept_saved, prop) == getattr(expected, prop)
 
     assert concept_saved.key in sheerka.concepts_cache
-    assert sheerka.sdp.io.exists(sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
+    assert sheerka.sdp.io.exists(
+        sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
 
 
 def test_i_can_eval_def_concept_part_when_one_part_is_a_ref_of_another_concept():
@@ -443,7 +445,8 @@ def test_i_can_eval_def_concept_part_when_one_part_is_a_ref_of_another_concept()
         assert getattr(concept_saved, prop) == getattr(expected, prop)
 
     assert concept_saved.key in sheerka.concepts_cache
-    assert sheerka.sdp.io.exists(sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
+    assert sheerka.sdp.io.exists(
+        sheerka.sdp.io.get_obj_path(SheerkaDataProvider.ObjectsFolder, concept_saved.get_digest()))
 
 
 def test_i_cannot_eval_the_same_def_concept_twice():
@@ -551,12 +554,24 @@ def test_i_can_manage_concepts_with_the_same_key_when_values_are_the_same():
     res = sheerka.eval("hello 'foo'")
     assert len(res) == 1
     assert res[0].status
-    assert res[0].value, "hello foo"
+    assert res[0].value ==  "hello foo"
     assert res[0].who == sheerka.get_evaluator_name(MultipleSameSuccessEvaluator.NAME)
 
 
-def get_sheerka(root="mem://"):
-    sheerka = Sheerka()
+def test_i_can_create_concepts_on_python_codes():
+    sheerka = get_sheerka()
+    context = get_context(sheerka)
+
+    sheerka.create_new_concept(context, Concept(name="concepts", body="sheerka.concepts()"))
+    res = sheerka.eval("concepts")
+
+    assert len(res) == 1
+    assert res[0].status
+    assert isinstance(res[0].value, list)
+
+
+def get_sheerka(root="mem://", skip_builtins_in_db=True):
+    sheerka = Sheerka(skip_builtins_in_db)
     sheerka.initialize(root)
 
     return sheerka

tests/test_utils.py

@@ -51,8 +51,28 @@ def test_i_can_get_sub_classes():
     default_parser = core.utils.get_class("parsers.DefaultParser.DefaultParser")
     exact_concept_parser = core.utils.get_class("parsers.ExactConceptParser.ExactConceptParser")
     python_parser = core.utils.get_class("parsers.PythonParser.PythonParser")
+    concept_lexer_parser = core.utils.get_class("parsers.ConceptLexerParser.ConceptLexerParser")
 
     assert base_parser not in sub_classes
     assert default_parser in sub_classes
     assert exact_concept_parser in sub_classes
     assert python_parser in sub_classes
+    assert concept_lexer_parser in sub_classes
+
+
+@pytest.mark.parametrize("a,b, expected", [
+    ([], [], []),
+    ([], ['a'], ['a']),
+    ([[]], ['a'], [['a']]),
+    (['a'], [], ['a']),
+    ([['a']], [], [['a']]),
+
+    ([['a']], ['b'], [['a', 'b']]),
+    ([['a'], ['b']], ['c'], [['a', 'c'], ['b', 'c']]),
+    ([['a1', 'a2'], ['b1', 'b2', 'b3']], ['c'], [['a1', 'a2', 'c'], ['b1', 'b2', 'b3', 'c']]),
+    ([[]], ['a', 'b'], [['a'], ['b']]),
+    ([['a'], ['b']], ['c', 'd', 'e'], [['a', 'c'], ['b', 'c'], ['a', 'd'], ['b', 'd'], ['a', 'e'], ['b', 'e']]),
+])
+def test_i_can_product(a, b, expected):
+    res = core.utils.product(a, b)
+    assert res == expected