Added basic implentation for where

tests/_concepts.txt

@@ -1,15 +0,0 @@
-def concept one as 1
-def concept two as 2
-def concept three as 3
-def concept four as 4
-def concept five as 5
-def concept one as 1
-def concept two as 2
-def concept three as 3
-def concept four as 4
-def concept five as 5
-def concept one as 1
-def concept two as 2
-def concept three as 3
-def concept four as 4
-def concept five as 5

tests/core/_concepts.txt

@@ -8,3 +8,13 @@ def concept two as 2
 def concept three as 3
 def concept four as 4
 def concept five as 5
+def concept one as 1
+def concept two as 2
+def concept three as 3
+def concept four as 4
+def concept five as 5
+def concept one as 1
+def concept two as 2
+def concept three as 3
+def concept four as 4
+def concept five as 5

tests/core/test_SheerkaEvaluateConcept.py

@@ -18,7 +18,7 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         ("True", True),
         ("1 > 2", False),
     ])
-    def test_i_can_evaluate_a_concept_with_simple_body(self,body, expected):
+    def test_i_can_evaluate_a_concept_with_simple_body(self, body, expected):
         sheerka = self.get_sheerka()
 
         concept = Concept("foo", body=body)
@@ -43,9 +43,9 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
         ("True", True),
         ("1 > 2", False),
     ])
-    def test_i_can_evaluate_the_other_metadata(self,expr, expected):
+    def test_i_can_evaluate_the_other_metadata(self, expr, expected):
         """
-        I only test WHERE, it's the same for the others
+        I only test PRE, it's the same for the others
         :param expr:
         :param expected:
         :return:
@@ -53,15 +53,15 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         sheerka = self.get_sheerka()
 
-        concept = Concept("foo", where=expr)
+        concept = Concept("foo", pre=expr)
         evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
 
         assert evaluated.key == concept.key
         assert evaluated.metadata.body is None
-        assert evaluated.metadata.pre is None
+        assert evaluated.metadata.pre == expr
         assert evaluated.metadata.post is None
-        assert evaluated.metadata.where == expr
-        assert evaluated.get_metadata_value(ConceptParts.WHERE) == expected
+        assert evaluated.metadata.where is None
+        assert evaluated.get_metadata_value(ConceptParts.PRE) == expected
         assert evaluated.props == {}
         assert evaluated.metadata.is_evaluated
         assert len(evaluated.values) == 0 if expr is None else 1
@@ -330,3 +330,42 @@ class TestSheerkaEvaluateConcept(TestUsingMemoryBasedSheerka):
 
         assert evaluated.key == concept.init_key().key
 
+    @pytest.mark.parametrize("where_clause, expected", [
+        ("True", True),
+        ("False", False),
+        ("self < 10", False),
+        ("self < 11", True),
+        ("a < 20", False),
+        ("a > 19", True),
+        ("a + self > 20", True),
+    ])
+    def test_i_can_evaluate_simple_where(self, where_clause, expected):
+        sheerka = self.get_sheerka()
+
+        concept = Concept("foo", body="10", where=where_clause).def_prop("a", "20")
+        sheerka.add_in_cache(concept)
+
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+
+        if expected:
+            assert evaluated.key == concept.key
+        else:
+            assert sheerka.isinstance(evaluated, BuiltinConcepts.WHERE_CLAUSE_FAILED)
+            assert evaluated.body == concept
+
+    def test_i_can_evaluate_where_when_using_other_concept(self):
+        sheerka = self.get_sheerka()
+
+        foo_true = Concept("foo_true", body="True").init_key()
+        foo_false = Concept("foo_false", body="False").init_key()
+        sheerka.add_in_cache(foo_false)
+        sheerka.add_in_cache(foo_true)
+
+        concept = Concept("foo", where="foo_true").init_key()
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert evaluated.key == concept.key
+
+        concept = Concept("foo", where="foo_false")
+        evaluated = sheerka.evaluate_concept(self.get_context(sheerka), concept)
+        assert sheerka.isinstance(evaluated, BuiltinConcepts.WHERE_CLAUSE_FAILED)
+        assert evaluated.body == concept

tests/core/test_SheerkaSetsManager.py

@@ -1,8 +1,3 @@
-import os
-import shutil
-from os import path
-
-import pytest
 from core.builtin_concepts import ConceptAlreadyInSet, BuiltinConcepts
 from core.concept import Concept
 
@@ -11,42 +6,36 @@ from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
 
 class TestSheerkaSetsManager(TestUsingFileBasedSheerka):
 
-
-
-    def test_i_can_add_concept_to_set(self):
-        sheerka = self.get_sheerka(False, False)
-
-        foo = Concept("foo")
-        sheerka.set_id_if_needed(foo, False)
-
-        all_foos = Concept("all_foos")
-        sheerka.set_id_if_needed(all_foos, False)
+    def init(self, use_dict, *concepts):
+        sheerka = self.get_sheerka(use_dict, True)
+        for c in concepts:
+            sheerka.set_id_if_needed(c, False)
+            sheerka.add_in_cache(c)
 
         context = self.get_context(sheerka)
+        return sheerka, context
+
+    def test_i_can_add_concept_to_set(self):
+        foo = Concept("foo")
+        all_foos = Concept("all_foos")
+        sheerka, context = self.init(False, foo, all_foos)
+
         res = sheerka.add_concept_to_set(context, foo, all_foos)
 
         assert res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
 
-        all_entries = self.get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False)
+        all_entries = self.get_sheerka(False, True).sdp.get("All_" + all_foos.id, None, False)
         assert len(all_entries) == 1
         assert foo.id in all_entries
 
     def test_i_can_add_several_concepts_to_set(self):
-        sheerka = self.get_sheerka(False, False)
-
         foo1 = Concept("foo1")
-        sheerka.set_id_if_needed(foo1, False)
-
-        foo2 = Concept("foo1")
-        sheerka.set_id_if_needed(foo2, False)
-
+        foo2 = Concept("foo2")
         all_foos = Concept("all_foos")
-        sheerka.set_id_if_needed(all_foos, False)
+        sheerka, context = self.init(False, foo1, foo2, all_foos)
 
-        context = self.get_context(sheerka)
-        sheerka.add_concept_to_set(context, foo1, all_foos)
-        res = sheerka.add_concept_to_set(context, foo2, all_foos)
+        res = sheerka.sets_handler.add_concepts_to_set(context, (foo1, foo2), all_foos)
 
         assert res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
@@ -56,16 +45,30 @@ class TestSheerkaSetsManager(TestUsingFileBasedSheerka):
         assert foo1.id in all_entries
         assert foo2.id in all_entries
 
+        # I can add another elements
+        foo3 = Concept("foo3")
+        foo4 = Concept("foo4")
+        for c in [foo3, foo4]:
+            sheerka.set_id_if_needed(c, False)
+            sheerka.add_in_cache(c)
+
+        res = sheerka.sets_handler.add_concepts_to_set(context, (foo3, foo4), all_foos)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        all_entries = self.get_sheerka(False, False).sdp.get("All_" + all_foos.id, None, False)
+        assert len(all_entries) == 4
+        assert foo1.id in all_entries
+        assert foo2.id in all_entries
+        assert foo3.id in all_entries
+        assert foo4.id in all_entries
+
     def test_i_cannot_add_the_same_concept_twice_in_a_set(self):
-        sheerka = self.get_sheerka()
-
         foo = Concept("foo")
-        sheerka.set_id_if_needed(foo, False)
-
         all_foos = Concept("all_foos")
-        sheerka.set_id_if_needed(all_foos, False)
+        sheerka, context = self.init(True, foo, all_foos)
 
-        context = self.get_context(sheerka)
         sheerka.add_concept_to_set(context, foo, all_foos)
         res = sheerka.add_concept_to_set(context, foo, all_foos)
 
@@ -77,18 +80,12 @@ class TestSheerkaSetsManager(TestUsingFileBasedSheerka):
         assert foo.id in all_entries
 
     def test_i_get_elements_from_a_set(self):
-        sheerka = self.get_sheerka()
-
         one = Concept("one")
         two = Concept("two")
         three = Concept("three")
         number = Concept("number")
+        sheerka, context = self.init(True, one, two, three, number)
 
-        for c in [one, two, three, number]:
-            sheerka.set_id_if_needed(c, False)
-            sheerka.add_in_cache(c)
-
-        context = self.get_context(sheerka)
         for c in [one, two, three]:
             sheerka.add_concept_to_set(context, c, number)
 
@@ -97,10 +94,8 @@ class TestSheerkaSetsManager(TestUsingFileBasedSheerka):
         assert set(elements) == {one, two, three}
 
     def test_i_cannot_get_elements_if_not_a_set(self):
-        sheerka = self.get_sheerka()
         one = Concept("one")
-        sheerka.set_id_if_needed(one, False)
-        sheerka.add_in_cache(one)
+        sheerka, context = self.init(True, one)
 
         error = sheerka.get_set_elements(one)
 
@@ -108,17 +103,17 @@ class TestSheerkaSetsManager(TestUsingFileBasedSheerka):
         assert error.body == one
 
     def test_isa_and_isa_group(self):
-        sheerka = self.get_sheerka()
+        group = Concept("group")
+        foo = Concept("foo")
+        sheerka, context = self.init(True, group, foo)
 
-        group = Concept("group").init_key()
-        group.metadata.id = "1001"
         assert not sheerka.isaset(group)
-
-        foo = Concept("foo").init_key()
-        foo.metadata.id = "1002"
         assert not sheerka.isa(foo, group)
 
         context = self.get_context(sheerka)
         sheerka.add_concept_to_set(context, foo, group)
         assert sheerka.isaset(group)
         assert sheerka.isa(foo, group)
+
+    def test_i_can_a_multiples_concepts(self):
+        pass

tests/core/test_tokenizer.py

@@ -40,7 +40,7 @@ def test_i_can_tokenize():
     assert tokens[31] == Token(TokenKind.AMPER, '&', 78, 6, 20)
     assert tokens[32] == Token(TokenKind.LESS, '<', 79, 6, 21)
     assert tokens[33] == Token(TokenKind.GREATER, '>', 80, 6, 22)
-    assert tokens[34] == Token(TokenKind.CONCEPT, 'name', 81, 6, 23)
+    assert tokens[34] == Token(TokenKind.CONCEPT, ('name', None), 81, 6, 23)
     assert tokens[35] == Token(TokenKind.DOLLAR, '$', 88, 6, 30)
     assert tokens[36] == Token(TokenKind.STERLING, '£', 89, 6, 31)
     assert tokens[37] == Token(TokenKind.EURO, '€', 90, 6, 32)
@@ -79,8 +79,8 @@ def test_i_can_tokenize_identifiers(text, expected):
     ('"string', "Missing Trailing quote", '"string', 7, 1, 8),
     ('"a" + "string', "Missing Trailing quote", '"string', 13, 1, 14),
     ('"a"\n\n"string', "Missing Trailing quote", '"string', 12, 3, 8),
-    ("c::", "Concept name not found", "", 2, 1, 3),
-    ("c:foo\nbar:", "New line is forbidden in concept name", "foo", 5, 1, 6),
+    ("c::", "Concept identifiers not found", "", 2, 1, 3),
+    ("c:foo\nbar:", "New line in concept name", "foo", 5, 1, 6),
     ("c:foo", "Missing ending colon", "foo", 5, 1, 6)
 ])
 def test_i_can_detect_tokenizer_errors(text, message, error_text, index, line, column):
@@ -139,3 +139,17 @@ def test_i_can_recognize_keywords(text, expected):
     tokens = list(Tokenizer(text))
     assert tokens[0].type == TokenKind.KEYWORD
     assert tokens[0].value == expected
+
+
+@pytest.mark.parametrize("text, expected", [
+    ("c:key:", ("key", None)),
+    ("c:key|id:", ("key", "id")),
+    ("c:key|:", ("key", None)),
+    ("c:|id:", (None, "id")),
+    ("c:125:", ("125", None)),
+])
+def test_i_can_parse_concept_token(text, expected):
+    tokens = list(Tokenizer(text))
+
+    assert tokens[0].type == TokenKind.CONCEPT
+    assert tokens[0].value == expected

tests/core/test_utils.py

@@ -1,10 +1,25 @@
 import core.utils
 import pytest
-from core.concept import ConceptParts
+from core.concept import ConceptParts, Concept
 
 from core.tokenizer import Token, TokenKind
 
 
+def get_tokens(lst):
+    res = []
+    for e in lst:
+        if e == " ":
+            res.append(Token(TokenKind.WHITESPACE, " ", 0, 0, 0))
+        elif e == "\n":
+            res.append(Token(TokenKind.NEWLINE, "\n", 0, 0, 0))
+        elif e == "<EOF>":
+            res.append(Token(TokenKind.EOF, "\n", 0, 0, 0))
+        else:
+            res.append(Token(TokenKind.IDENTIFIER, e, 0, 0, 0))
+
+    return res
+
+
 @pytest.mark.parametrize("lst, as_string", [
     (None, "",),
     ([], ""),
@@ -136,18 +151,33 @@ def test_i_can_escape():
     (10, None, None),
     ("", None, None),
     ("xxx", None, None),
-    (":c:", None, None),
-    (":c:key", None, None),
-    (":c:key:", "key", None),
-    (":c:key:id", None, None),
-    (":c:key:id:", "key", "id"),
+    ("c:", None, None),
+    ("c:key", None, None),
+    ("c:key:", "key", None),
+    ("c:key|id", None, None),
+    ("c:key|id:", "key", "id"),
+    ("c:|id:", None, "id"),
+    ("c:key|:", "key", None),
 ])
-def test_i_can_decode_concept_repr(text, expected_key, expected_id):
-    k, i = core.utils.decode_concept(text)
+def test_i_can_unstr_concept(text, expected_key, expected_id):
+    k, i = core.utils.unstr_concept(text)
     assert k == expected_key
     assert i == expected_id
 
 
+def test_i_can_str_concept():
+    assert core.utils.str_concept(("key", "id")) == "c:key|id:"
+    assert core.utils.str_concept((None, "id")) == "c:|id:"
+    assert core.utils.str_concept(("key", None)) == "c:key:"
+    assert core.utils.str_concept((None, None)) == ""
+
+    concept = Concept("foo").init_key()
+    assert core.utils.str_concept(concept) == "c:foo:"
+
+    concept.metadata.id = "1001"
+    assert core.utils.str_concept(concept) == "c:foo|1001:"
+
+
 @pytest.mark.parametrize("text, expected", [
     (None, None),
     (10, None),
@@ -162,16 +192,22 @@ def test_i_can_decode_enum(text, expected):
     assert actual == expected
 
 
-def get_tokens(lst):
-    res = []
-    for e in lst:
-        if e == " ":
-            res.append(Token(TokenKind.WHITESPACE, " ", 0, 0, 0))
-        elif e == "\n":
-            res.append(Token(TokenKind.NEWLINE, "\n", 0, 0, 0))
-        elif e == "<EOF>":
-            res.append(Token(TokenKind.EOF, "\n", 0, 0, 0))
-        else:
-            res.append(Token(TokenKind.IDENTIFIER, e, 0, 0, 0))
+def test_encode_concept_key_id():
+    assert core.utils.encode_concept(("key", "id")) == "__C__KEY_key__ID_id__C__"
+    assert core.utils.encode_concept((None, "id")) == "__C__KEY_00None00__ID_id__C__"
+    assert core.utils.encode_concept(("key", None)) == "__C__KEY_key__ID_00None00__C__"
+    assert core.utils.encode_concept(("key", "id"), True) == "__C__USE_CONCEPT__KEY_key__ID_id__C__"
+    assert core.utils.encode_concept(("k + y", "id")) == "__C__KEY_k000y__ID_id__C__"
 
-    return res
+    concept = Concept("foo").init_key()
+    assert core.utils.encode_concept(concept) == "__C__KEY_foo__ID_00None00__C__"
+
+    concept.metadata.id = "1001"
+    assert core.utils.encode_concept(concept) == "__C__KEY_foo__ID_1001__C__"
+
+
+def test_decode_concept_key_id():
+    assert core.utils.decode_concept("__C__KEY_key__ID_id__C__") == ("key", "id", False)
+    assert core.utils.decode_concept("__C__KEY_00None00__ID_id__C__") == (None, "id", False)
+    assert core.utils.decode_concept("__C__KEY_key__ID_00None00__C__") == ("key", None, False)
+    assert core.utils.decode_concept("__C__USE_CONCEPT__KEY_key__ID_id__C__") == ("key", "id", True)

tests/evaluators/test_ConceptEvaluator.py

@@ -21,7 +21,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
     def test_i_can_evaluate_concept(self):
         context = self.get_context()
         concept = Concept(name="foo",
-                          where="1",
+                          where="True",
                           pre="2",
                           post="3").def_prop("a", "4").def_prop("b", "5")
 
@@ -32,7 +32,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
         assert result.who == evaluator.name
         assert result.status
         assert result.value.name == "foo"
-        assert result.value.get_metadata_value(ConceptParts.WHERE) == 1
+        assert result.value.get_metadata_value(ConceptParts.WHERE) == True
         assert result.value.get_metadata_value(ConceptParts.PRE) == 2
         assert result.value.get_metadata_value(ConceptParts.POST) == 3
         assert result.value.get_prop("a") == 4
@@ -44,7 +44,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         concept = Concept(name="foo",
                           body="'I have a value'",
-                          where="1",
+                          where="True",
                           pre="2",
                           post="3").set_prop("a", "4").set_prop("b", "5")
 
@@ -61,7 +61,7 @@ class TestAddConceptEvaluator(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         concept = Concept(name="foo",
                           body="'I have a value'",
-                          where="1",
+                          where="True",
                           pre="2",
                           post="3").set_prop("a", "4").set_prop("b", "5")
 

tests/evaluators/test_PythonEvaluator.py

@@ -116,23 +116,23 @@ class TestPythonEvaluator(TestUsingMemoryBasedSheerka):
         assert not evaluated.status
         assert evaluated.body.body.args[0] == "'int' object has no attribute 'name'"
 
-    @pytest.mark.parametrize("text, concept_key, concept_id, use_concept", [
-        ("__C__key__C__", "key", None, False),
-        ("__C__key__id__C__", "key", "id", False),
-        ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
-        ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
-    ])
-    def test_i_can_resolve_name(self, text, concept_key, concept_id, use_concept):
-        context = self.get_context()
-        assert PythonEvaluator().resolve_name(context, text) == (concept_key, concept_id, use_concept)
-
-    @pytest.mark.parametrize("text", [
-        "__C__",
-        "__C__key",
-        "__C__key____",
-        "__C____",
-        "__C__USE_CONCEPT__",
-    ])
-    def test_i_cannot_resolve_name(self, text):
-        context = self.get_context()
-        assert PythonEvaluator().resolve_name(context, text) is None
+    # @pytest.mark.parametrize("text, concept_key, concept_id, use_concept", [
+    #     ("__C__key__C__", "key", None, False),
+    #     ("__C__key__id__C__", "key", "id", False),
+    #     ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
+    #     ("__C__USE_CONCEPT__key__id__C__", "key", "id", True),
+    # ])
+    # def test_i_can_resolve_name(self, text, concept_key, concept_id, use_concept):
+    #     context = self.get_context()
+    #     assert PythonEvaluator().resolve_name(context, text) == (concept_key, concept_id, use_concept)
+    #
+    # @pytest.mark.parametrize("text", [
+    #     "__C__",
+    #     "__C__key",
+    #     "__C__key____",
+    #     "__C____",
+    #     "__C__USE_CONCEPT__",
+    # ])
+    # def test_i_cannot_resolve_name(self, text):
+    #     context = self.get_context()
+    #     assert PythonEvaluator().resolve_name(context, text) is None

tests/non_reg/test_sheerka_non_reg.py

@@ -273,8 +273,9 @@ as:
 
         saved_definitions = sheerka.sdp.get_safe(sheerka.CONCEPTS_DEFINITIONS_ENTRY)
         expected_bnf = Sequence(
-            a, Optional(Sequence(StrMatch("plus"), ConceptExpression("plus", rule_name="plus"))))
-        assert saved_definitions[saved_concept] == expected_bnf
+            ConceptExpression(a, rule_name="a"),
+            Optional(Sequence(StrMatch("plus"), ConceptExpression(saved_concept, rule_name="plus"))))
+        assert saved_definitions["c:plus|1001:"] == "(c:a:=a ('plus' c:plus|1001:=plus)?)"
 
         new_concept = res[0].value.body
         assert new_concept.metadata.name == "plus"
@@ -456,6 +457,26 @@ as:
         assert res[0].status
         assert res[0].body == 23
 
+    def test_i_can_mix_bnf_and_isa_when_concept_other_case(self):
+        sheerka = self.get_sheerka()
+
+        init = [
+            "def concept one as 1",
+            "def concept twenty as 20",
+            "def concept number",
+            "one isa number",
+            "twenty isa number",
+            "def concept twenties from bnf twenty number as twenty + number"
+        ]
+
+        for exp in init:
+            sheerka.evaluate_user_input(exp)
+
+        res = sheerka.evaluate_user_input("twenty one")
+        assert len(res) == 1
+        assert res[0].status
+        assert res[0].body == simplec("twenties", 21)
+
     def test_i_can_mix_concept_of_concept(self):
         sheerka = self.get_sheerka()
 
@@ -623,3 +644,30 @@ as:
         assert len(res) == 1
         assert res[0].status
         assert res[0].body == 21
+
+    def test_i_can_use_where_in_bnf(self):
+        sheerka = self.get_sheerka()
+
+        init = [
+            "def concept one as 1",
+            "def concept two as 2",
+            "def concept three as 3",
+            "def concept twenty as 20",
+            "def concept number",
+            "one isa number",
+            "two isa number",
+            "three isa number",
+            "def concept twenties from bnf twenty number where number <= 2 as twenty + number"
+        ]
+
+        for exp in init:
+            sheerka.evaluate_user_input(exp)
+
+        res = sheerka.evaluate_user_input("eval twenty one")
+        assert len(res) == 1 and res[0].status and res[0].body == 21
+
+        res = sheerka.evaluate_user_input("eval twenty two")
+        assert len(res) == 1 and res[0].status and res[0].body == 22
+
+        res = sheerka.evaluate_user_input("eval twenty three")
+        assert len(res) > 1

tests/parsers/test_BaseParser.py

@@ -18,7 +18,7 @@ def test_i_can_get_text_from_tokens(text, expected_text):
 
 
 @pytest.mark.parametrize("text, custom, expected_text", [
-    ("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value}"}, "execute(__C__concept_name)")
+    ("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value[0]}"}, "execute(__C__concept_name)")
 ])
 def test_i_can_get_text_from_tokens_with_custom_switcher(text, custom, expected_text):
     tokens = list(Tokenizer(text))

tests/parsers/test_BnfParser.py

@@ -2,7 +2,7 @@ import pytest
 
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
-from core.tokenizer import Tokenizer, TokenKind, LexerError
+from core.tokenizer import Tokenizer, TokenKind, LexerError, Token
 from parsers.BaseParser import UnexpectedTokenErrorNode
 from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError
 from parsers.ConceptLexerParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \
@@ -16,6 +16,14 @@ class ClassWithName:
         self.name = name
 
 
+def c(name):
+    concept = Concept(name).init_key()
+    return ConceptExpression(concept, rule_name=name)
+
+
+eof_token = Token(TokenKind.EOF, "", 0, 0, 0)
+
+
 class TestBnfParser(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("expression, expected", [
@@ -33,9 +41,10 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         ("1 2 | 3 4+", OrderedChoice(
             Sequence(StrMatch("1"), StrMatch("2")),
             Sequence(StrMatch("3"), OneOrMore(StrMatch("4"))))),
-        (
-            "1 (2 | 3) 4+",
-            Sequence(StrMatch("1"), OrderedChoice(StrMatch("2"), StrMatch("3")), OneOrMore(StrMatch("4")))),
+        ("1 (2 | 3) 4+", Sequence(
+            StrMatch("1"),
+            OrderedChoice(StrMatch("2"), StrMatch("3")),
+            OneOrMore(StrMatch("4")))),
         ("(1|2)+", OneOrMore(OrderedChoice(StrMatch("1"), StrMatch("2")))),
         ("(1 2)+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")))),
         ("1 *", Sequence(StrMatch("1"), StrMatch("*"))),
@@ -61,6 +70,13 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         ("(1 2)=var", Sequence(StrMatch("1"), StrMatch("2"), rule_name="var")),
         ("(1 2)+=var", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")), rule_name="var")),
         ("(1 2)=var+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2"), rule_name="var"))),
+        ("(1=a 2=b)=c", Sequence(StrMatch("1", rule_name="a"), StrMatch("2", rule_name="b"), rule_name="c")),
+        ("(1*=a)", ZeroOrMore(StrMatch("1"), rule_name="a")),
+        ("'a'* 'b'+", Sequence(ZeroOrMore(StrMatch("a")), OneOrMore(StrMatch("b")))),
+        ("('a'* 'b'+)", Sequence(ZeroOrMore(StrMatch("a")), OneOrMore(StrMatch("b")))),
+        ("('a'*=x 'b'+=y)=z", Sequence(
+            ZeroOrMore(StrMatch("a"), rule_name="x"),
+            OneOrMore(StrMatch("b"), rule_name="y"), rule_name="z")),
     ])
     def test_i_can_parse_regex(self, expression, expected):
         parser = BnfParser()
@@ -72,12 +88,12 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         assert res.value.source == expression
 
     @pytest.mark.parametrize("expression, expected", [
-        ("foo", Concept("foo").init_key()),
-        ("foo*", ZeroOrMore(Concept("foo").init_key())),
-        ("foo 'and' bar+", Sequence(Concept("foo").init_key(), StrMatch("and"), OneOrMore(Concept("bar").init_key()))),
-        ("foo | bar?", OrderedChoice(Concept("foo").init_key(), Optional(Concept("bar").init_key()))),
-        ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
-        ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
+        ("foo", c("foo")),
+        ("foo*", ZeroOrMore(c("foo"))),
+        ("foo 'and' bar+", Sequence(c("foo"), StrMatch("and"), OneOrMore(c("bar")))),
+        ("foo | bar?", OrderedChoice(c("foo"), Optional(c("bar")))),
+        ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), c("var"))),
+        ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), c("var"))),
     ])
     def test_i_can_parse_regex_with_concept(self, expression, expected):
         foo = Concept("foo")
@@ -113,8 +129,8 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
     @pytest.mark.parametrize("expression, error", [
         ("1 ", UnexpectedEndOfFileError()),
         ("1|", UnexpectedEndOfFileError()),
-        ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.RPAR])),
-        ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.IDENTIFIER])),
+        ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", eof_token, [TokenKind.RPAR])),
+        ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", eof_token, [TokenKind.IDENTIFIER])),
         ("'name", LexerError("Missing Trailing quote", "'name", 5, 1, 6))
     ])
     def test_i_can_detect_errors(self, expression, error):

tests/parsers/test_ConceptLexerParser.py

@@ -1,10 +1,13 @@
+from ast import Str
+
 import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, ConceptParts, DoNotResolve
 from core.tokenizer import Tokenizer, TokenKind, Token
+from parsers.BnfParser import BnfParser
 from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, StrMatch, OrderedChoice, Optional, \
     ParsingExpressionVisitor, TerminalNode, NonTerminalNode, ZeroOrMore, OneOrMore, \
-    UnrecognizedTokensNode, cnode, short_cnode
+    UnrecognizedTokensNode, cnode, short_cnode, ConceptExpression, ConceptGroupExpression
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -75,6 +78,7 @@ class TestConceptLexerParser(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         for c in concepts:
             context.sheerka.add_in_cache(c)
+            context.sheerka.set_id_if_needed(c, False)
 
         parser = ConceptLexerParser()
         parser.initialize(context, grammar)
@@ -586,6 +590,32 @@ class TestConceptLexerParser(TestUsingMemoryBasedSheerka):
         assert res.status
         assert res.value.body == [cnode("foo", 0, 2, "twenty one")]
 
+    def test_i_can_initialize_when_cyclic_reference(self):
+        foo = Concept(name="foo")
+        grammar = {foo: Optional("one", ConceptExpression("foo"))}
+        context, parser = self.init([foo], grammar)
+
+        assert parser.concepts_grammars[foo] == Optional("one", ConceptExpression(foo, rule_name="foo"))
+
+    def test_i_cannot_initialize_when_cyclic_reference_when_concept_is_under_construction_and_not_known(self):
+        foo = Concept(name="foo").init_key()
+        grammar = {foo: Optional("one", ConceptExpression("foo"))}
+
+        context = self.get_context()
+        parser = ConceptLexerParser()
+        parser.initialize(context, grammar)
+        assert parser.concepts_grammars[foo] == Optional("one", ConceptExpression("foo", rule_name="foo"))
+
+    def test_i_can_initialize_when_cyclic_reference_when_concept_is_under_construction_and_known(self):
+        foo = Concept(name="foo").init_key()
+        grammar = {foo: Optional("one", ConceptExpression("foo"))}
+
+        context = self.get_context()
+        context.concepts["foo"] = foo
+        parser = ConceptLexerParser()
+        parser.initialize(context, grammar)
+        assert parser.concepts_grammars[foo] == Optional("one", ConceptExpression(foo, rule_name="foo"))
+
     def test_i_can_parse_concept_reference_that_is_group(self):
         """
         if one is  number, then number is a 'group'
@@ -1092,6 +1122,109 @@ class TestConceptLexerParser(TestUsingMemoryBasedSheerka):
         assert cprop(concept_found, "seq")[1] == DoNotResolve("un ok")
         assert cprop(concept_found, "seq")[2] == DoNotResolve("uno ok")
 
+    @pytest.mark.parametrize("rule, expected", [
+        (StrMatch("string"), "'string'"),
+        (StrMatch("string", rule_name="rule_name"), "'string'=rule_name"),
+        (Sequence(StrMatch("foo"), StrMatch("bar")), "('foo' 'bar')"),
+        (Sequence(StrMatch("foo"), StrMatch("bar"), rule_name="rule_name"), "('foo' 'bar')=rule_name"),
+        (OrderedChoice(StrMatch("foo"), StrMatch("bar")), "('foo'|'bar')"),
+        (OrderedChoice(StrMatch("foo"), StrMatch("bar"), rule_name="rule_name"), "('foo'|'bar')=rule_name"),
+        (Optional(StrMatch("foo")), "'foo'?"),
+        (Optional(StrMatch("foo"), rule_name="rule_name"), "'foo'?=rule_name"),
+        (ZeroOrMore(StrMatch("foo")), "'foo'*"),
+        (ZeroOrMore(StrMatch("foo"), rule_name="rule_name"), "'foo'*=rule_name"),
+        (OneOrMore(StrMatch("foo")), "'foo'+"),
+        (OneOrMore(StrMatch("foo"), rule_name="rule_name"), "'foo'+=rule_name"),
+        (Sequence(
+            Optional(StrMatch("foo"), rule_name="a"),
+            ZeroOrMore(StrMatch("bar"), rule_name="b"),
+            OneOrMore(StrMatch("baz"), rule_name="c"),
+            rule_name="d"), "('foo'?=a 'bar'*=b 'baz'+=c)=d"),
+        (OrderedChoice(
+            Optional(StrMatch("foo"), rule_name="a"),
+            ZeroOrMore(StrMatch("bar"), rule_name="b"),
+            OneOrMore(StrMatch("baz"), rule_name="c"),
+            rule_name="d"), "('foo'?=a|'bar'*=b|'baz'+=c)=d"),
+        (Sequence(
+            OrderedChoice(StrMatch("foo"), StrMatch("bar"), rule_name="a"),
+            OrderedChoice(StrMatch("x"), StrMatch("y"), rule_name="b"),
+            rule_name="c"), "(('foo'|'bar')=a ('x'|'y')=b)=c")
+    ])
+    def test_i_can_encode_grammar(self, rule, expected):
+        foo = Concept(name="foo")
+        grammar = {foo: rule}
+        context, parser = self.init([foo], grammar)
+
+        encoded = parser.encode_grammar(parser.concepts_grammars)
+        assert encoded["c:foo|1001:"] == expected
+
+        bnf_parser = BnfParser()
+        parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"])
+        assert parse_res.status
+        assert parse_res.value.value == rule
+
+    def test_i_can_encode_grammar_when_concept_simple(self):
+        foo = Concept(name="foo")
+        bar = Concept(name="bar")
+        grammar = {foo: ConceptExpression(bar)}
+        context, parser = self.init([foo, bar], grammar)
+
+        encoded = parser.encode_grammar(parser.concepts_grammars)
+        assert encoded["c:foo|1001:"] == "c:bar|1002:=bar"
+
+        bnf_parser = BnfParser()
+        parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"])
+        assert parse_res.status
+        assert parse_res.value.value == grammar[foo]
+
+    def test_i_can_encode_grammar_when_concepts(self):
+        foo = Concept(name="foo")
+        bar = Concept(name="bar")
+        baz = Concept(name="baz")
+        grammar = {foo: Sequence(
+            StrMatch("a"),
+            OrderedChoice(ConceptExpression(bar),
+                          OneOrMore(ConceptExpression(baz)), rule_name="oc"), rule_name="s")}
+        context, parser = self.init([foo, bar, baz], grammar)
+
+        encoded = parser.encode_grammar(parser.concepts_grammars)
+        assert encoded["c:foo|1001:"] == "('a' (c:bar|1002:=bar|c:baz|1003:=baz+)=oc)=s"
+
+        bnf_parser = BnfParser()
+        parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"])
+        assert parse_res.status
+        assert parse_res.value.value == grammar[foo]
+
+    def test_i_can_encode_grammar_when_set_concepts(self):
+        foo = Concept(name="foo")
+        bar = Concept(name="bar")
+        baz = Concept(name="baz")
+        grammar = {foo: Sequence(
+            StrMatch("a"),
+            OrderedChoice(bar,
+                          OneOrMore(ConceptExpression(baz)), rule_name="oc"), rule_name="s")}
+        context = self.get_context()
+        for c in [foo, bar, baz]:
+            context.sheerka.add_in_cache(c)
+            context.sheerka.set_id_if_needed(c, False)
+        context.sheerka.add_concept_to_set(context, baz, bar)
+
+        parser = ConceptLexerParser()
+        parser.initialize(context, grammar)
+
+        encoded = parser.encode_grammar(parser.concepts_grammars)
+        assert encoded["c:foo|1001:"] == "('a' (c:bar|1002:=bar|c:baz|1003:=baz+)=oc)=s"
+
+        bnf_parser = BnfParser()
+        parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"])
+        assert parse_res.status
+
+        expected = Sequence(
+            StrMatch("a"),
+            OrderedChoice(ConceptGroupExpression(bar, rule_name="bar"),
+                          OneOrMore(ConceptExpression(baz, rule_name="baz")), rule_name="oc"), rule_name="s")
+        assert parse_res.value.value == expected
+
     #
     # def test_i_can_parse_basic_arithmetic_operations_and_resolve_properties(self):
     #     context = self.get_context()

tests/parsers/test_DefaultParser.py

@@ -3,7 +3,7 @@ import ast
 
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
 from core.concept import Concept
-from parsers.ConceptLexerParser import OrderedChoice, StrMatch
+from parsers.ConceptLexerParser import OrderedChoice, StrMatch, ConceptExpression
 from parsers.PythonParser import PythonParser, PythonNode
 from core.tokenizer import Keywords, Tokenizer, LexerError
 from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode, CannotHandleErrorNode, IsaConceptNode
@@ -246,7 +246,7 @@ def concept add one to a as
         parser = DefaultParser()
         res = parser.parse(context, text)
         node = res.value.value
-        definition = OrderedChoice(a_concept, StrMatch("a_string"))
+        definition = OrderedChoice(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("a_string"))
         parser_result = ParserResultConcept(BnfParser(), "a_concept | 'a_string'", definition, definition)
         expected = get_def_concept(name="name", body="__definition[0]", definition=parser_result)
 
@@ -321,7 +321,7 @@ def concept add one to a as
         ("def concept 'name", "Missing Trailing quote", "'name"),
         ("def concept name as 'body", "Missing Trailing quote", "'body"),
         ("def concept name from bnf 'expression", "Missing Trailing quote", "'expression"),
-        ("def concept c::", "Concept name not found", ""),
+        ("def concept c::", "Concept identifiers not found", ""),
     ])
     def test_i_cannot_parse_when_tokenizer_fails(self, text, error_msg, error_text):
         parser = DefaultParser()

tests/parsers/test_PythonParser.py

@@ -3,6 +3,7 @@ import pytest
 from core.builtin_concepts import ParserResultConcept
 from core.tokenizer import Tokenizer, LexerError
 from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode
+import core.utils
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -52,8 +53,8 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
         assert isinstance(res.value.value[0].exception, SyntaxError)
 
     @pytest.mark.parametrize("text, error_msg, error_text", [
-        ("c::", "Concept name not found", ""),
-        ("c:: + 1", "Concept name not found", ""),
+        ("c::", "Concept identifiers not found", ""),
+        ("c:: + 1", "Concept identifiers not found", ""),
     ])
     def test_i_can_detect_lexer_errors(self, text, error_msg, error_text):
         parser = PythonParser()
@@ -66,12 +67,12 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
         assert res.body.body[0].text == error_text
 
     def test_i_can_parse_a_concept(self):
-        text = "c:concept_name: + 1"
+        text = "c:name|key: + 1"
 
         parser = PythonParser()
         res = parser.parse(self.get_context(), text)
 
         assert res
         assert res.value.value == PythonNode(
-            "c:concept_name: + 1",
-            ast.parse("__C__USE_CONCEPT__concept_name__C__+1", mode="eval"))
+            "c:name|key: + 1",
+            ast.parse(core.utils.encode_concept(("name", "key"), True) + "+1", mode="eval"))

tests/sheerkapickle/test_SheerkaPickler.py

@@ -127,7 +127,7 @@ class TestSheerkaPickler(TestUsingFileBasedSheerka):
         sheerka.add_in_cache(concept)
         obj = {concept: "a"}
         flatten = SheerkaPickler(sheerka).flatten(obj)
-        assert flatten == {':c:foo:1001:': 'a'}
+        assert flatten == {'c:foo|1001:': 'a'}
 
         decoded = SheerkaUnpickler(sheerka).restore(flatten)
         assert decoded == obj