Fixed #30 : Add variable support in BNF concept definition

Fixed #31 : Add regex support in BNF Concept
Fixed #33 : Do not memorize object during restore

tests/core/test_SheerkaConceptManager.py

@@ -8,7 +8,8 @@ from core.concept import PROPERTIES_TO_SERIALIZE, Concept, DEFINITION_TYPE_DEF,
 from core.global_symbols import NotInit, NotFound
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager, NoModificationFound, ForbiddenAttribute, \
     UnknownAttribute, CannotRemoveMeta, ValueNotFound, ConceptIsReferenced, NoFirstTokenError
-from parsers.BnfNodeParser import Sequence, StrMatch, ConceptExpression, OrderedChoice, Optional, ZeroOrMore, OneOrMore
+from parsers.BnfNodeParser import Sequence, StrMatch, ConceptExpression, OrderedChoice, Optional, ZeroOrMore, OneOrMore, \
+    RegExDef, RegExMatch
 from tests.TestUsingFileBasedSheerka import TestUsingFileBasedSheerka
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -60,6 +61,50 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_HASH_ENTRY, concept.get_definition_hash())
         assert sheerka.om.current_sdp().exists(service.CONCEPTS_BY_FIRST_KEYWORD_ENTRY, "+")
 
+    def test_i_can_create_a_bnf_concept_that_starts_with_a_regex(self):
+        sheerka = self.get_sheerka(cache_only=False)
+        context = self.get_context(sheerka)
+        service = sheerka.services[SheerkaConceptManager.NAME]
+        foo = self.bnf_concept("foo", RegExMatch("[a-z]+"))
+        bar = self.bnf_concept("bar", RegExMatch("[0-9]+"))
+
+        res = sheerka.create_new_concept(context, foo)
+
+        assert res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
+
+        # I can get by the first regex
+        assert sheerka.om.get(service.CONCEPTS_BY_REGEX_ENTRY, RegExDef("[a-z]+").serialize()) == [foo.id]
+        assert len(service.compiled_concepts_by_regex) == 1
+
+        # I can commit
+        sheerka.om.commit(context)
+
+        # I can load from DB
+        entry = sheerka.om.current_sdp().get(service.CONCEPTS_BY_REGEX_ENTRY)
+        assert entry == {RegExDef("[a-z]+").serialize(): [foo.id]}
+
+        # I can create another concept
+        res = sheerka.create_new_concept(context, bar)
+
+        assert res.status
+        assert sheerka.isinstance(res.value, BuiltinConcepts.NEW_CONCEPT)
+
+        # I can get by the first regex
+        assert sheerka.om.get(service.CONCEPTS_BY_REGEX_ENTRY, RegExDef("[0-9]+").serialize()) == [bar.id]
+        assert sheerka.om.get(service.CONCEPTS_BY_REGEX_ENTRY, RegExDef("[a-z]+").serialize()) == [foo.id]
+        assert len(service.compiled_concepts_by_regex) == 2
+
+        # I can commit
+        sheerka.om.commit(context)
+
+        # I can load from DB
+        entry = sheerka.om.current_sdp().get(service.CONCEPTS_BY_REGEX_ENTRY)
+        assert entry == {
+            RegExDef("[a-z]+").serialize(): [foo.id],
+            RegExDef("[0-9]+").serialize(): [bar.id]
+        }
+
     def test_i_cannot_create_a_bnf_concept_that_references_a_concept_that_cannot_be_resolved(self):
         sheerka, context, one_1, one_1_0 = self.init_concepts(Concept("one", body="1"), Concept("one", body="1.0"))
         twenty_one = Concept("twenty one", definition="'twenty' one", definition_type=DEFINITION_TYPE_BNF)
@@ -361,17 +406,16 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
 
         # sdp is updated
         sheerka.om.commit(context)
-        from_sdp = sheerka.om.current_sdp().get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id)
+        sdp = sheerka.om.current_sdp()
+        from_sdp = sdp.get(service.CONCEPTS_BY_ID_ENTRY, new_concept.id)
         assert from_sdp.get_metadata().body == "metadata value"
         assert from_sdp.get_metadata().variables == [("var_name", "default value")]
         assert from_sdp.get_prop(BuiltinConcepts.ISA) == {bar}
 
-        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_NAME_ENTRY,
-                                            new_concept.name).get_metadata().body == "metadata value"
-        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_KEY_ENTRY,
-                                            new_concept.key).get_metadata().body == "metadata value"
-        assert sheerka.om.current_sdp().get(service.CONCEPTS_BY_HASH_ENTRY,
-                                            new_concept.get_definition_hash()).get_metadata().body == "metadata value"
+        assert sdp.get(service.CONCEPTS_BY_NAME_ENTRY, new_concept.name).get_metadata().body == "metadata value"
+        assert sdp.get(service.CONCEPTS_BY_KEY_ENTRY, new_concept.key).get_metadata().body == "metadata value"
+        assert sdp.get(service.CONCEPTS_BY_HASH_ENTRY,
+                       new_concept.get_definition_hash()).get_metadata().body == "metadata value"
 
     def test_caches_are_update_when_i_modify_the_name(self):
         sheerka, context, foo = self.init_concepts("foo", cache_only=False)
@@ -496,6 +540,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
             Concept("baz", definition="foo"),
             create_new=True).unpack()
 
+        # sanity check
         assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
             "foo": ["1001"],
             "bar": ["1002"],
@@ -514,6 +559,71 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
             'bar': ['1002', '1001', '1003']}
 
+    def test_i_can_modify_bnf_definition_from_first_token_to_first_regex(self):
+        sheerka, context, foo, = self.init_test().with_concepts(
+            Concept("foo", definition="'hello'|'hola'"), create_new=True).unpack()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        # sanity
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "hello": ["1001"],
+            "hola": ["1001"]}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
+        assert len(service.compiled_concepts_by_regex) == 0
+
+        to_add = {"meta": {"definition": "r'[a-z]+'"}}
+        res = sheerka.modify_concept(context, foo, to_add)
+
+        assert res.status
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {
+            RegExDef("[a-z]+").serialize(): ["1001"]
+        }
+        assert len(service.compiled_concepts_by_regex) == 1
+
+    def test_i_can_modify_bnf_definition_from_first_regex_to_first_token(self):
+        sheerka, context, foo, = self.init_test().with_concepts(
+            Concept("foo", definition="r'[a-z]+'"), create_new=True).unpack()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        # sanity
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {
+            RegExDef("[a-z]+").serialize(): ["1001"]
+        }
+        assert len(service.compiled_concepts_by_regex) == 1
+
+        to_add = {"meta": {"definition": "'hello'|'hola'"}}
+        res = sheerka.modify_concept(context, foo, to_add)
+
+        assert res.status
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "hello": ["1001"],
+            "hola": ["1001"]}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
+        assert len(service.compiled_concepts_by_regex) == 0
+
+    def test_i_can_modify_when_multiple_bnf_definitions_are_already_defined(self):
+        sheerka, context, foo, bar, baz = self.init_test().with_concepts(
+            Concept("foo", definition="r'[a-z]+'"),
+            Concept("bar", definition="r'[0-1]+'"),
+            Concept("baz", definition="'one'|'twox'"), create_new=True).unpack()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+        # it does not matter than baz is a bnf
+
+        to_add = {"meta": {"definition": "'one'|'two'"}}
+        res = sheerka.modify_concept(context, baz, to_add)
+
+        assert res.status
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "one": ["1003"],
+            "two": ["1003"]}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {
+            RegExDef("[a-z]+").serialize(): ["1001"],
+            RegExDef("[0-1]+").serialize(): ["1002"],
+        }
+        assert len(service.compiled_concepts_by_regex) == 2
+
     def test_references_are_updated_after_concept_modification(self):
         sheerka, context, one, twenty_one = self.init_test().with_concepts(
             "onz",
@@ -602,7 +712,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
         assert res.body.body == NoModificationFound(foo, {"name": "foo", "body": "a body"})
 
-    def test_i_cannot_remove_meta_attributes(self):
+    def test_i_cannot_modify_and_remove_meta_attributes(self):
         sheerka, context, foo = self.init_concepts(Concept("foo"))
 
         res = sheerka.modify_concept(context, foo, to_remove={"meta": {"any_value": "foo"}})
@@ -611,7 +721,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
         assert res.body.body == CannotRemoveMeta({"any_value": "foo"})
 
-    def test_i_cannot_remove_props_that_does_not_exists(self):
+    def test_i_cannot_modify_and_remove_props_that_does_not_exists(self):
         sheerka, context, foo = self.init_concepts(Concept("foo"))
 
         res = sheerka.modify_concept(context, foo, to_remove={"props": {"any_value": "foo"}})
@@ -620,7 +730,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
         assert res.body.body == UnknownAttribute("any_value")
 
-    def test_i_cannot_remove_props_value_that_does_not_exists(self):
+    def test_i_cannot_modify_and_remove_props_value_that_does_not_exists(self):
         # Need to returns an error, otherwise, we will save a concept that is not modified
         sheerka, context, foo = self.init_concepts(Concept("foo", props={"a": {"value"}}))
 
@@ -630,7 +740,7 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
         assert res.body.body == ValueNotFound("a", "dummy")
 
-    def test_i_cannot_remove_variable_that_does_not_exists(self):
+    def test_i_cannot_modify_and_remove_variable_that_does_not_exists(self):
         sheerka, context, foo = self.init_concepts(Concept("foo").def_var("a"))
 
         res = sheerka.modify_concept(context, foo, to_remove={"variables": ["b"]})
@@ -649,6 +759,30 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert not res.status
         assert sheerka.isinstance(res.body, BuiltinConcepts.UNKNOWN_CONCEPT)
 
+    def test_i_cannot_modify_with_an_invalid_regex_expression(self):
+        sheerka, context, foo, = self.init_test().with_concepts(
+            Concept("foo", definition="'hello'|'hola'"), create_new=True).unpack()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        # sanity
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "hello": ["1001"],
+            "hola": ["1001"]}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
+        assert len(service.compiled_concepts_by_regex) == 0
+
+        to_add = {"meta": {"definition": "r'[a-z+'"}}  # invalid regex definition
+        res = sheerka.modify_concept(context, foo, to_add)
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.ERROR)
+        assert res.body.body.msg == 'unterminated character set'
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {
+            "hello": ["1001"],
+            "hola": ["1001"]}
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
+        assert len(service.compiled_concepts_by_regex) == 0
+
     def test_i_can_get_and_set_attribute(self):
         sheerka, context = self.init_concepts()
         foo = Concept("foo")
@@ -683,6 +817,8 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.get_by_name(one.name) == one
         assert sheerka.get_by_key(one.key) == one
         assert sheerka.get_by_hash(one.get_definition_hash()) == one
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) != {}
+        assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) != {}
 
         res = sheerka.remove_concept(context, one)
 
@@ -694,6 +830,35 @@ class TestSheerkaConceptManager(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(sheerka.get_by_key(one.key), BuiltinConcepts.UNKNOWN_CONCEPT)
         assert sheerka.isinstance(sheerka.get_by_hash(one.get_definition_hash()), BuiltinConcepts.UNKNOWN_CONCEPT)
 
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
+        assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
+
+    def test_i_can_remove_a_first_regex_concept(self):
+        sheerka, context, one = self.init_test().with_concepts(
+            Concept("one", definition="r'[a-z]+'"),
+            create_new=True).unpack()
+        service = sheerka.services[SheerkaConceptManager.NAME]
+
+        # sanity check
+        assert sheerka.get_by_id(one.id) == one
+        assert sheerka.get_by_name(one.name) == one
+        assert sheerka.get_by_key(one.key) == one
+        assert sheerka.get_by_hash(one.get_definition_hash()) == one
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) != {}
+        assert len(service.compiled_concepts_by_regex) != 0
+
+        res = sheerka.remove_concept(context, one)
+
+        assert res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.SUCCESS)
+
+        assert sheerka.isinstance(sheerka.get_by_id(one.id), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(sheerka.get_by_name(one.name), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(sheerka.get_by_key(one.key), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.isinstance(sheerka.get_by_hash(one.get_definition_hash()), BuiltinConcepts.UNKNOWN_CONCEPT)
+        assert sheerka.om.copy(SheerkaConceptManager.CONCEPTS_BY_REGEX_ENTRY) == {}
+        assert len(service.compiled_concepts_by_regex) == 0
+
     def test_i_cannot_remove_a_concept_that_does_not_exist(self):
         sheerka, context = self.init_concepts()
         one = Concept("one", id="1001")

tests/core/test_sheerka.py

@@ -73,7 +73,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
 
         assert loaded is not None
         assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
-        assert loaded.body == ("key", "key_that_does_not_exist")
+        assert loaded.body == {"key": "key_that_does_not_exist"}
         assert loaded.get_metadata().is_evaluated
 
     def test_i_cannot_get_when_id_is_not_found(self):
@@ -83,7 +83,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
 
         assert loaded is not None
         assert sheerka.isinstance(loaded, BuiltinConcepts.UNKNOWN_CONCEPT)
-        assert loaded.body == ("id", "id_that_does_not_exist")
+        assert loaded.body == {"id": "id_that_does_not_exist"}
         assert loaded.get_metadata().is_evaluated
 
     def test_i_can_instantiate_a_builtin_concept_when_it_has_its_own_class(self):
@@ -200,7 +200,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         new = sheerka.new("fake_concept")
 
         assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
-        assert new.body == ('key', 'fake_concept')
+        assert new.body == {'key': 'fake_concept'}
 
     def test_i_cannot_instantiate_with_invalid_id(self):
         sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo", body="foo1"),
@@ -210,7 +210,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         new = sheerka.new(("foo", "invalid_id"))
 
         assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
-        assert new.body == [('key', 'foo'), ('id', 'invalid_id')]
+        assert new.body == {'key': 'foo', 'id': 'invalid_id'}
 
     def test_i_cannot_instantiate_with_invalid_key(self):
         sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo", body="foo1"),
@@ -220,7 +220,7 @@ class TestSheerkaUsingMemoryBasedSheerka(TestUsingMemoryBasedSheerka):
         new = sheerka.new(("invalid_key", "1001"))
 
         assert sheerka.isinstance(new, BuiltinConcepts.UNKNOWN_CONCEPT)
-        assert new.body == [('key', 'invalid_key'), ('id', '1001')]
+        assert new.body == {'key': 'invalid_key', 'id': '1001'}
 
     def test_concept_id_is_irrelevant_when_only_one_concept(self):
         sheerka, context, *concepts = self.init_test().with_concepts(Concept("foo", body="foo1"),

tests/core/test_tokenizer.py

@@ -1,4 +1,5 @@
 import pytest
+
 from core.tokenizer import Tokenizer, Token, TokenKind, LexerError
 
 
@@ -172,6 +173,7 @@ def test_i_can_parse_concept_token(text, expected):
     assert tokens[0].type == TokenKind.CONCEPT
     assert tokens[0].value == expected
 
+
 @pytest.mark.parametrize("text, expected", [
     ("r:key:", ("key", None)),
     ("r:key|id:", ("key", "id")),
@@ -197,3 +199,6 @@ def test_i_can_parse_regex_token(text, expected):
 
     assert tokens[0].type == TokenKind.REGEX
     assert tokens[0].value == expected
+    assert tokens[0].str_value == "r" + expected
+    assert tokens[0].repr_value == "r" + expected
+    assert tokens[0].strip_quote == expected[1:-1]

tests/evaluators/EvaluatorTestsUtils.py

@@ -58,7 +58,7 @@ def pr_ret_val(value, parser="parser", source=None):
 
 
 def python_ret_val(source):
-    python_node = PythonNode(source, ast.parse(source, f"<source>", 'eval'))
+    python_node = PythonNode(source.strip(), ast.parse(source.strip(), f"<source>", 'eval'))
     return pr_ret_val(python_node, parser="Python", source=source)
 
 

tests/evaluators/test_DefConceptEvaluator.py

@@ -4,12 +4,13 @@ import pytest
 
 from core.builtin_concepts import ReturnValueConcept, ParserResultConcept, BuiltinConcepts
 from core.concept import VARIABLE_PREFIX, Concept, DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF
+from core.sheerka.services.SheerkaConceptManager import NoFirstTokenError
 from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Tokenizer
 from evaluators.DefConceptEvaluator import DefConceptEvaluator
 from parsers.BaseParser import BaseParser
 from parsers.BnfDefinitionParser import BnfDefinitionParser
-from parsers.BnfNodeParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression
+from parsers.BnfNodeParser import Sequence, StrMatch, ZeroOrMore, ConceptExpression, VariableExpression
 from parsers.DefConceptParser import DefConceptNode, NameNode, DefConceptParser
 from parsers.PythonParser import PythonNode, PythonParser
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
@@ -277,3 +278,55 @@ class TestDefConceptEvaluator(TestUsingMemoryBasedSheerka):
         assert evaluated.status
         assert evaluated.body.body.key == "foo2 __var__0"
         assert evaluated.body.body.get_metadata().variables == [("x", None)]
+
+    def test_i_can_eval_when_bnf_concept_with_regex(self):
+        context = self.get_context()
+        def_ret_val = DefConceptParser().parse(context, ParserInput("def concept hello a from bnf r'[a-z]+'=a 'hello'"))
+        evaluated = DefConceptEvaluator().eval(context, def_ret_val)
+
+        assert evaluated.status
+        assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
+
+        created_concept = evaluated.body.body
+        assert created_concept.get_metadata().name == "hello a"
+        assert created_concept.get_metadata().key == "hello __var__0"
+        assert created_concept.get_metadata().definition == "r'[a-z]+'=a 'hello'"
+        assert created_concept.get_metadata().definition_type == "bnf"
+
+    def test_i_can_eval_when_bnf_concept_with_variable(self):
+        context = self.get_context()
+        def_ret_val = DefConceptParser().parse(context, ParserInput("def concept hello x from bnf 'hello' x"))
+        evaluated = DefConceptEvaluator().eval(context, def_ret_val)
+
+        assert evaluated.status
+        assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.NEW_CONCEPT)
+
+        created_concept = evaluated.body.body
+        assert created_concept.get_metadata().name == "hello x"
+        assert created_concept.get_metadata().key == "hello __var__0"
+        assert created_concept.get_metadata().definition == "'hello' x"
+        assert created_concept.get_metadata().definition_type == "bnf"
+        assert created_concept.get_metadata().variables == [("x", None)]
+        assert created_concept._bnf == Sequence(StrMatch("hello"), VariableExpression("x"))
+
+    def test_i_cannot_eval_bnf_concept_with_unknown_variable(self):
+        context = self.get_context()
+        def_ret_val = DefConceptParser().parse(context, ParserInput("def concept name from bnf unknown foo"))
+        evaluated = DefConceptEvaluator().eval(context, def_ret_val)
+
+        assert not evaluated.status
+        assert context.sheerka.isinstance(evaluated.body, BuiltinConcepts.ERROR)
+        unknown_concepts = [
+            context.sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body={"name": "foo"}),
+            context.sheerka.new(BuiltinConcepts.UNKNOWN_CONCEPT, body={"name": "unknown"}),
+        ]
+        assert evaluated.body.body == unknown_concepts
+
+    def test_i_cannot_eval_bnf_concept_with_only_variable(self):
+        sheerka, context = self.init_test().unpack()
+        def_ret_val = DefConceptParser().parse(context, ParserInput("def concept foo x from bnf x"))
+        evaluated = DefConceptEvaluator().eval(context, def_ret_val)
+
+        assert not evaluated.status
+        assert sheerka.isinstance(evaluated.body, BuiltinConcepts.ERROR)
+        assert isinstance(evaluated.body.body, NoFirstTokenError)

tests/non_reg/test_sheerka_non_reg.py

@@ -1237,6 +1237,18 @@ as:
         assert len(l) > 0
         sheerka.save_execution_context = False
 
+    def test_i_can_define_a_regex_concept_and_parse_it(self):
+        sheerka, context = self.init_test().unpack()
+
+        res = sheerka.evaluate_user_input("def concept binary from bnf r'[01]+'")
+        assert len(res) == 1
+        assert res[0].status
+
+        res = sheerka.evaluate_user_input("01001")
+        assert len(res) == 1
+        assert res[0].status
+        assert sheerka.isinstance(res[0].body, "binary")
+
 
 class TestSheerkaNonRegFile(TestUsingFileBasedSheerka):
     def test_i_can_def_several_concepts(self):

tests/parsers/parsers_utils.py

@@ -1,6 +1,7 @@
 import ast
 from dataclasses import dataclass
 
+from core.builtin_concepts import ReturnValueConcept
 from core.builtin_helpers import CreateObjectIdentifiers
 from core.concept import CC, Concept, ConceptParts, DoNotResolve, CIO, CMV
 from core.tokenizer import Tokenizer, TokenKind, Token
@@ -256,13 +257,17 @@ def get_node(
     if sub_expr == "')'":
         return ")"
 
+    if isinstance(sub_expr, ReturnValueConcept):
+        return sub_expr
+
     if isinstance(sub_expr, (scnode, utnode, DoNotResolve)):
         return sub_expr
 
     if isinstance(sub_expr, CIO):
         sub_expr.set_concept(concepts_map[sub_expr.concept_name])
-        if sub_expr.source:
-            node = get_node(concepts_map, expression_as_tokens, sub_expr.source, sya=sya)
+        source = sub_expr.source or sub_expr.concept_name
+        if source:
+            node = get_node(concepts_map, expression_as_tokens, source, sya=sya)
             sub_expr.start = node.start
             sub_expr.end = node.end
         return sub_expr
@@ -366,7 +371,7 @@ def get_node(
             return CN(concept_found, start, start + length - 1, source=sub_expr)
     else:
         # else an UnrecognizedTokensNode
-        return utnode(start, start + length - 1, sub_expr)
+        return UTN(sub_expr, start, start + length - 1)
 
 
 def init_body(item, concept, value):
@@ -482,3 +487,29 @@ def get_rete_conditions(*conditions_as_string):
 
         res.append(Condition(identifier, attribute, value))
     return AndConditions(res)
+
+
+def get_test_obj(test_obj, real_obj, to_compare_delegate=None):
+    """
+    From a production object (Concept, ConceptNode, ....)
+    Create a test object (CNC, CC ...) that can be used to validate the unit tests
+    :param test_obj:
+    :param real_obj:
+    :param to_compare_delegate:
+    :return:
+    """
+    if isinstance(test_obj, list):
+        if len(test_obj) != len(real_obj):
+            raise Exception(f"Not the same size ! {test_obj=}, {real_obj=}")
+        return [get_test_obj(t, r) for t, r in zip(test_obj, real_obj)]
+
+    if isinstance(test_obj, dict):
+        if len(test_obj) != len(real_obj):
+            raise Exception(f"Not the same size ! {test_obj=}, {real_obj=}")
+
+        return {k: get_test_obj(v, real_obj[k]) for k, v in test_obj.items()}
+
+    if not hasattr(test_obj, "to_compare"):
+        return real_obj
+
+    return test_obj.to_compare(real_obj, get_test_obj)

tests/parsers/test_BnfNodeParser.py

@@ -1,3 +1,5 @@
+import re
+
 import pytest
 
 import tests.parsers.parsers_utils
@@ -6,12 +8,14 @@ from core.concept import Concept, ConceptParts, DoNotResolve, CC, DEFINITION_TYP
 from core.global_symbols import NotInit
 from core.sheerka.services.SheerkaConceptManager import SheerkaConceptManager
 from core.sheerka.services.SheerkaExecute import ParserInput
-from parsers.BaseNodeParser import CNC, UTN, CN
+from parsers.BaseNodeParser import CNC, UTN, CN, NoMatchingTokenError, SCN
 from parsers.BnfDefinitionParser import BnfDefinitionParser
 from parsers.BnfNodeParser import StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
-    Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice, BnfNodeParser
+    Optional, ZeroOrMore, OneOrMore, ConceptExpression, UnOrderedChoice, BnfNodeParser, RegExMatch, \
+    BnfNodeFirstTokenVisitor, Match, RegExDef, VariableExpression
 from tests.BaseTest import BaseTest
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.evaluators.EvaluatorTestsUtils import python_ret_val
 
 cmap = {
     "one": Concept("one"),
@@ -42,9 +46,14 @@ cmap = {
     "three_four": Concept("three_four", definition="three | four").def_var("three").def_var("four"),
     "t2": Concept("t2", definition="'twenty' three_four=unit").def_var("unit").def_var("three").def_var("four"),
 
+    # bnf with variable
+    "one thing": Concept("one x", definition="one x").def_var("x"),
+    "x shoe": Concept("x shoe", definition="x 'shoe'").def_var("x"),
+
     # testing keywords
     "def_only": Concept("def"),
     "def number": Concept("def number", definition="def (one|two)=number"),
+
     # sequence of keywords using bnf definition
     # "def_concept_bnf": Concept("def_concept_bnf", definition="'def' 'concept'"),
     # "def concept_bnf number": Concept("def number", definition="def_concept_bnf (one|two)=number"),
@@ -68,8 +77,8 @@ def u(parsing_expression, start, end, children=None):
     if isinstance(parsing_expression, str):
         parsing_expression = StrMatch(parsing_expression)
 
-    if isinstance(parsing_expression, StrMatch):
-        return TerminalNode(parsing_expression, start, end, parsing_expression.to_match)
+    if isinstance(parsing_expression, Match):
+        return TerminalNode(parsing_expression, start, end, parsing_expression.to_match, parsing_expression.to_match)
 
     return NonTerminalNode(parsing_expression, start, end, [], children)
 
@@ -105,7 +114,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         sheerka.set_isa(context, cmap["one hundred"], cmap["number"])
         sheerka.set_isa(context, cmap["hundreds"], cmap["number"])
 
-        # Pay attention. 'twenties (t1 and t2) are not set as number
+        # Pay attention. 'twenties (t1 and t2) are not set as 'number'
 
         thirties = cls.update_bnf(context, Concept("thirties",
                                                    definition="thirty number",
@@ -158,7 +167,10 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
             for i, pair in enumerate(my_concepts_map):
                 my_concepts_map[pair] = updated[i]
 
-        parser = BnfNodeParser(sheerka=sheerka) if init_from_sheerka else BnfNodeParser()
+        if init_from_sheerka:
+            parser = BnfNodeParser(sheerka=sheerka)
+        else:
+            parser = BnfNodeParser().init_from_concepts(context, my_concepts_map.values())
         return sheerka, context, parser
 
     def validate_get_concepts_sequences(self, my_map, text, expected, multiple_result=False, post_init_concepts=None):
@@ -198,7 +210,9 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
 
         assert len(bnf_parsers_helpers) == len(expected_array)
         for parser_helper, expected_sequence in zip(bnf_parsers_helpers, expected_array):
-            assert parser_helper.sequence == expected_sequence
+            to_compare = tests.parsers.parsers_utils.get_test_obj(expected_sequence, parser_helper.sequence)
+            # assert parser_helper.sequence == expected_sequence
+            assert to_compare == expected_sequence
 
         if len(bnf_parsers_helpers) == 1:
             return bnf_parsers_helpers[0].sequence
@@ -221,7 +235,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         (StrMatch("3.14"), "3.14"),
         (StrMatch("+"), "+"),
     ])
-    def test_i_can_match_simple_bnf(self, expr, text):
+    def test_i_can_match_str_bnf(self, expr, text):
         my_map = {
             text: self.bnf_concept("foo", expr)
         }
@@ -229,6 +243,57 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         sequence = self.validate_get_concepts_sequences(my_map, text, [text])
         assert sequence[0].underlying == u(expr, 0, 0)
 
+    @pytest.mark.parametrize("expr, text, end", [
+        (RegExMatch("bar"), "bar", 0),
+        (RegExMatch("[a-z]+"), "xyz", 0),
+        (RegExMatch("[a-z=]+"), "uvt=xyz=abc", 4),
+    ])
+    def test_i_can_match_regex_bnf(self, expr, text, end):
+        my_map = {
+            text: self.bnf_concept("foo", expr)
+        }
+
+        sequence = self.validate_get_concepts_sequences(my_map, text, [text])
+        assert sequence[0].underlying.start == 0
+        assert sequence[0].underlying.end == end
+        assert sequence[0].underlying.parsing_expression == expr
+
+    @pytest.mark.parametrize("expr, text, end", [
+        (Sequence(StrMatch("foo"), RegExMatch("bar")), "foo bar", 2),
+        (Sequence(StrMatch("foo"), RegExMatch("[a-z]+")), "foo xyz", 2),
+        (Sequence(StrMatch("foo"), RegExMatch("[a-z=]+")), "foo uvt=xyz=abc", 6),
+    ])
+    def test_i_can_match_sequence_str_regex(self, expr, text, end):
+        my_map = {
+            text: self.bnf_concept("foo", expr)
+        }
+
+        sequence = self.validate_get_concepts_sequences(my_map, text, [text])
+        assert sequence[0].underlying == u(expr, 0, end, sequence[0].underlying.children)
+
+    @pytest.mark.parametrize("expr, text, end", [
+        (Sequence(RegExMatch("bar"), StrMatch("foo")), "bar foo", 2),
+        (Sequence(RegExMatch("[a-z]+"), StrMatch("foo")), "xyz foo", 2),
+        (Sequence(RegExMatch("[a-z=]+"), StrMatch("foo")), "uvt=xyz=abc foo", 6),
+    ])
+    def test_i_can_match_sequence_regex_str(self, expr, text, end):
+        my_map = {
+            text: self.bnf_concept("foo", expr)
+        }
+
+        sequence = self.validate_get_concepts_sequences(my_map, text, [text])
+        assert sequence[0].underlying == u(expr, 0, end, sequence[0].underlying.children)
+
+    def test_i_can_match_sequence_str_regex_str(self):
+        text = "foo uvt=xyz=abc baz"
+        expr = Sequence(StrMatch("foo"), RegExMatch("[a-z=]+"), StrMatch("baz"))
+        my_map = {
+            text: self.bnf_concept("foo", expr)
+        }
+
+        sequence = self.validate_get_concepts_sequences(my_map, text, [text])
+        assert sequence[0].underlying == u(expr, 0, 8, sequence[0].underlying.children)
+
     def test_i_can_match_multiple_concepts_in_one_input(self):
         my_map = {
             "one": self.bnf_concept("one"),
@@ -356,8 +421,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         self.validate_get_concepts_sequences(my_map, text, expected)
 
     @pytest.mark.parametrize("text, expected", [
-        # ("twenty one", [CNC("foo", source="twenty one")]),
-        # ("twenty three", []),  # three does not exist
+        ("twenty one", [CNC("foo", source="twenty one")]),
+        ("twenty three", []),  # three does not exist
         ("twenty four", []),  # four exists but should not be seen
     ])
     def test_i_can_mix_sequence_and_ordered_2(self, text, expected):
@@ -388,7 +453,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         ("three", []),
 
     ])
-    def test_i_can_parse_unordered_choice(self, text, expected):
+    def test_i_can_match_unordered_choice(self, text, expected):
         my_map = {
             "foo": self.bnf_concept("foo", UnOrderedChoice(
                 StrMatch("one"),
@@ -402,7 +467,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         ("", []),
         ("two", []),
     ])
-    def test_i_can_parse_optional(self, text, expected):
+    def test_i_can_match_optional(self, text, expected):
         my_map = {
             "foo": self.bnf_concept("foo", Optional(StrMatch("one")))
         }
@@ -413,7 +478,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         ("twenty one", [CNC("foo", source="twenty one")]),
         ("one", [CNC("foo", source="one")]),
     ])
-    def test_i_can_parse_sequence_starting_with_optional(self, text, expected):
+    def test_i_can_match_sequence_starting_with_optional(self, text, expected):
         my_map = {
             "foo": self.bnf_concept("foo",
                                     Sequence(
@@ -427,7 +492,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         ("one two three", [CNC("foo", source="one two three")]),
         ("one two", [CNC("foo", source="one two")]),
     ])
-    def test_i_can_parse_sequence_ending_with_optional(self, text, expected):
+    def test_i_can_match_sequence_ending_with_optional(self, text, expected):
         my_map = {
             "foo": self.bnf_concept("foo",
                                     Sequence(
@@ -442,7 +507,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         ("one two three", [CNC("foo", source="one two three")]),
         ("one three", [CNC("foo", source="one three")]),
     ])
-    def test_i_can_parse_sequence_with_optional_in_between(self, text, expected):
+    def test_i_can_match_sequence_with_optional_in_between(self, text, expected):
         my_map = {
             "foo": self.bnf_concept("foo",
                                     Sequence(
@@ -459,7 +524,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         ("one", [CNC("foo", source="one")]),
         ("one one", [CNC("foo", source="one one")]),
     ])
-    def test_i_can_parse_zero_or_more(self, text, expected):
+    def test_i_can_match_zero_or_more(self, text, expected):
         my_map = {
             "foo": self.bnf_concept("foo", ZeroOrMore(StrMatch("one")))
         }
@@ -471,7 +536,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         ("one two", [CNC("foo", source="one two")]),
         ("one one two", [CNC("foo", source="one one two")]),
     ])
-    def test_i_can_parse_sequence_and_zero_or_more(self, text, expected):
+    def test_i_can_match_sequence_and_zero_or_more(self, text, expected):
         my_map = {
             "foo": self.bnf_concept("foo",
                                     Sequence(
@@ -485,7 +550,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
     @pytest.mark.parametrize("text, expected", [
         ("one, one , one", [CNC("foo", source="one, one , one")]),
     ])
-    def test_i_can_parse_zero_or_more_with_separator(self, text, expected):
+    def test_i_can_match_zero_or_more_with_separator(self, text, expected):
         my_map = {
             "foo": self.bnf_concept("foo", ZeroOrMore(StrMatch("one"), sep=","))
         }
@@ -508,7 +573,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         ("one", [CNC("foo", source="one")]),
         ("one one one", [CNC("foo", source="one one one")]),
     ])
-    def test_i_can_parse_one_or_more(self, text, expected):
+    def test_i_can_match_one_or_more(self, text, expected):
         my_map = {
             "foo": self.bnf_concept("foo", OneOrMore(StrMatch("one"))),
         }
@@ -520,7 +585,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         ("one two", [CNC("foo", source="one two")]),
         ("one one two", [CNC("foo", source="one one two")]),
     ])
-    def test_i_can_parse_sequence_one_and_or_more(self, text, expected):
+    def test_i_can_match_sequence_one_and_or_more(self, text, expected):
         my_map = {
             "foo": self.bnf_concept("foo",
                                     Sequence(
@@ -534,7 +599,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
     @pytest.mark.parametrize("text, expected", [
         ("one, one , one", [CNC("foo", source="one, one , one")]),
     ])
-    def test_i_can_parse_one_or_more_with_separator(self, text, expected):
+    def test_i_can_match_one_or_more_with_separator(self, text, expected):
         my_map = {
             "foo": self.bnf_concept("foo", OneOrMore(StrMatch("one"), sep=","))
         }
@@ -763,7 +828,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         assert concept_bar.get_compiled()["foo"][1].get_compiled() == {ConceptParts.BODY: DoNotResolve("two")}
         assert concept_bar.get_compiled()["foo"][2].get_compiled() == {ConceptParts.BODY: DoNotResolve("three")}
 
-    def test_i_can_parse_concept_reference_that_is_not_in_grammar(self):
+    def test_i_can_match_concept_reference_that_is_not_in_grammar(self):
         my_map = {
             "one": Concept("one"),
             "two": Concept("two"),
@@ -817,6 +882,234 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         assert concept_foo.get_compiled() == {'number': CC(my_map["number"], body=my_map["one"], one=my_map["one"]),
                                               ConceptParts.BODY: DoNotResolve(value='twenty one')}
 
+    @pytest.mark.parametrize("expr, expected", [
+        ("one 'car'", [CNC("foo", source="one 'car'", x=python_ret_val("'car'"))]),  # python
+        ("one bar", [CNC("foo", source="one bar", x=CC("bar"))]),  # simple concept
+        ("one super car", [CNC("foo", source="one super car", x=CC("super car"))]),  # long concept
+        ("one shoe", [CNC("foo", source="one shoe", x=CC("thing", source="shoe", body=DoNotResolve("shoe")))]),  # bnf
+    ])
+    def test_i_can_match_variable_when_ending_with_one_variable(self, expr, expected):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(StrMatch("one"), VariableExpression("x"))),
+            "bar": Concept("bar"),
+            "baz": Concept("baz"),
+            "thing": Concept("thing", definition="'shoe'|'skirt'"),
+            "super car": Concept("super car"),
+            "plus": Concept("x plus y").def_var("x").def_var("y"),
+        }
+
+        self.validate_get_concepts_sequences(my_map, expr, expected)
+
+    def test_i_can_match_variable_when_ending_with_one_variable_and_sya(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(StrMatch("one"), VariableExpression("x"))),
+            "bar": Concept("bar"),
+            "baz": Concept("baz"),
+            "plus": Concept("x plus y").def_var("x").def_var("y"),
+        }
+
+        expr = "one bar plus baz"
+        expected = [
+            [CNC("foo", source="one bar", x=CC("bar")), UTN(" plus "), CN("baz")],
+            [CNC("foo", source="one bar plus baz", x=CC("plus", source="bar plus baz", x="bar", y="baz"))],
+        ]
+
+        self.validate_get_concepts_sequences(my_map, expr, expected, multiple_result=True)
+
+    def test_i_can_match_variable_when_ending_with_one_variable_and_multiple_results(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(StrMatch("one"), VariableExpression("x"))),
+            "pretty big": Concept("pretty big", body="'pretty big'"),
+            "pbig": Concept("pretty big"),
+        }
+
+        expr = "one pretty big"
+        expected = [
+            [CNC("foo", source="one pretty big", x=CC("pretty big"))],
+            [CNC("foo", source="one pretty big", x=CC("pbig", source="pretty big"))]
+        ]
+        self.validate_get_concepts_sequences(my_map, expr, expected, multiple_result=True)
+
+    def test_i_can_match_variable_when_ending_with_multiple_variables_and_multiple_results(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(StrMatch("one"), VariableExpression("x"), VariableExpression("y"))),
+            "pretty": Concept("pretty", body="pretty"),
+            "pretty2": Concept("pretty"),
+            "big": Concept("big", body="big"),
+        }
+
+        expr = "one pretty big"
+        expected = [
+            [CNC("foo", source="one pretty big", x=CC("pretty"), y=CC("big"))],
+            [CNC("foo", source="one pretty big", x=CC("pretty2", source="pretty"), y=CC("big"))]
+        ]
+        self.validate_get_concepts_sequences(my_map, expr, expected, multiple_result=True)
+
+    @pytest.mark.parametrize("expr, expected", [
+        ("'my' shoe", [CNC("foo", source="'my' shoe", x=python_ret_val("'my' "))]),  # python
+        ("one shoe", [CNC("foo", source="one shoe", x=CC("one"))]),  # concept
+        ("my little shoe", [CNC("foo", source="my little shoe", x=CC("my little"))]),  # long concept
+        ("black shoe", [CNC("foo", source="black shoe", x=CC("color", source="black", body=DoNotResolve('black')))]),
+    ])
+    def test_i_can_match_variable_when_starting_with_one_variable(self, expr, expected):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(VariableExpression("x"), StrMatch("shoe"))),
+            "one": Concept("one"),
+            "my little": Concept("my little"),
+            "color": Concept("color", definition="'blue'|'black'"),
+            "and": Concept("x and y").def_var("x").def_var("y"),
+        }
+
+        self.validate_get_concepts_sequences(my_map, expr, expected)
+
+    def test_i_can_match_variable_when_starting_with_one_variable_and_sya(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(VariableExpression("x"), StrMatch("shoe"))),
+            "tiny": Concept("tiny"),
+            "beautiful": Concept("beautiful"),
+            "but": Concept("x but y").def_var("x").def_var("y"),
+        }
+        expr = "tiny but beautiful shoe"
+        expected_res = [
+            CNC("foo",
+                source="tiny but beautiful shoe",
+                x=CC("but", source="tiny but beautiful", x="tiny", y="beautiful"))]
+        unwanted_res = [CN("tiny"), UTN(" but "), CNC("foo", source="beautiful shoe", x=CC("beautiful"))]
+        self.validate_get_concepts_sequences(my_map, expr, [unwanted_res, expected_res], multiple_result=True)
+
+    def test_i_can_match_variable_when_starting_with_multiple_variables(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(VariableExpression("x"),
+                                                    VariableExpression("y"),
+                                                    VariableExpression("z"),
+                                                    StrMatch("shoe"))),
+            "one": Concept("one"),
+            "two": Concept("two"),
+            "plus": Concept("x plus y").def_var("x").def_var("y"),
+        }
+
+        text = "one 'one' one plus two shoe"
+
+        unwanted_res = [CN("one"), SCN(" 'one' "), ("one", 1), UTN(" plus "), CN("two")]
+        expected_res = [CNC("foo",
+                            source="one 'one' one plus two shoe",
+                            x=CC("one"),
+                            y=python_ret_val(" 'one' "),
+                            z=CC("plus", source="one plus two", x="one", y="two"))]
+        expected = [unwanted_res, expected_res]
+
+        self.validate_get_concepts_sequences(my_map, text, expected, multiple_result=True)
+
+    def test_i_can_match_variable_when_starting_with_one_variable_and_longer_str(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(VariableExpression("x"),
+                                                    StrMatch("foo"),
+                                                    StrMatch("bar"),
+                                                    StrMatch("baz"))),
+            "one": Concept("one")
+        }
+        text = "one foo bar baz"
+        expected = [CNC("foo", source="one foo bar baz", x=CC("one"))]
+        self.validate_get_concepts_sequences(my_map, text, expected)
+
+    @pytest.mark.parametrize("expr, expected", [
+        ("one 'pretty' shoe", [CNC("foo", source="one 'pretty' shoe", x=python_ret_val("'pretty' "))]),  # python
+        ("one little shoe", [CNC("foo", source="one little shoe", x=CC("little"))]),  # concept
+        ("one very big shoe", [CNC("foo", source="one very big shoe", x=CC("very big"))]),  # long concept
+        ("one black shoe",
+         [CNC("foo", source="one black shoe", x=CC("color", source="black", body=DoNotResolve('black')))]),
+        ("one tiny but beautiful shoe",
+         [CNC("foo",
+              source="one tiny but beautiful shoe",
+              x=CC("but", source="tiny but beautiful", x="tiny", y="beautiful "))]),
+    ])
+    def test_i_can_match_variable_in_between(self, expr, expected):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(StrMatch("one"), VariableExpression("x"), StrMatch("shoe"))),
+            "little": Concept("little"),
+            "very big": Concept("very big"),
+            "color": Concept("color", definition="'blue'|'black'"),
+            "but": Concept("x but y").def_var("x").def_var("y"),
+        }
+
+        self.validate_get_concepts_sequences(my_map, expr, expected)
+
+    def test_i_can_match_variable_when_multiple_results_in_between(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(StrMatch("one"), VariableExpression("x"), StrMatch("shoe"))),
+            "pretty big": Concept("pretty big", body="'pretty big'"),
+            "pbig": Concept("pretty big"),
+        }
+
+        expr = "one pretty big shoe"
+        expected = [
+            [CNC("foo", source="one pretty big shoe", x=CC("pretty big"))],
+            [CNC("foo", source="one pretty big shoe", x=CC("pbig", source="pretty big"))]
+        ]
+        self.validate_get_concepts_sequences(my_map, expr, expected, multiple_result=True)
+
+    def test_i_can_match_regex_and_variable(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(RegExMatch("[a-z]+"),
+                                                    VariableExpression("x"))),
+            "shoe": Concept("shoe")
+        }
+        text = "onyx shoe"
+        expected = [CNC("foo", source="onyx shoe", x=CC("shoe"))]
+        self.validate_get_concepts_sequences(my_map, text, expected)
+
+    def test_i_can_match_variable_and_regex(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(VariableExpression("x"),
+                                                    RegExMatch("[a-z]+"))),
+            "one": Concept("one")
+        }
+        text = "one onyx"
+        expected = [CNC("foo", source="one onyx", x=CC("one"))]
+        self.validate_get_concepts_sequences(my_map, text, expected)
+
+    def test_i_can_reuse_the_same_variable(self):
+        # in this test, the variable appears several times, but only once in concept.compiled
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(VariableExpression("x"),
+                                                    StrMatch("equals"),
+                                                    VariableExpression("x"))),
+            "one": Concept("one"),
+            "two": Concept("two"),
+        }
+        sheerka, context, *updated = self.init_concepts(*my_map.values())
+        parser = BnfNodeParser()
+        parser.init_from_concepts(context, updated)
+
+        # same variable appears only once in the compiled variables
+        text = "one equals one"
+        expected = [CNC("foo", source="one equals one", x=CC("one"))]
+        expected_sequence = compute_expected_array(my_map, text, expected)
+
+        parser.reset_parser(context, ParserInput(text))
+        bnf_parsers_helpers = parser.get_concepts_sequences(context)
+        to_compare = tests.parsers.parsers_utils.get_test_obj(expected_sequence, bnf_parsers_helpers[0].sequence)
+        assert to_compare == expected
+
+    def test_i_cannot_match_variable_when_variables_discrepancy(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(VariableExpression("x"),
+                                                    StrMatch("equals"),
+                                                    VariableExpression("x"))),
+            "one": Concept("one"),
+            "one_1": Concept("one", body="1"),
+            "two": Concept("two"),
+            "two_2": Concept("two", body="2"),
+        }
+        sheerka, context, *updated = self.init_concepts(*my_map.values())
+        parser = BnfNodeParser()
+        parser.init_from_concepts(context, updated)
+
+        text = "one equals two"
+        parser.reset_parser(context, ParserInput(text))
+        bnf_parsers_helpers = parser.get_concepts_sequences(context)
+        assert bnf_parsers_helpers[0].sequence == []
+
     @pytest.mark.parametrize("bar_expr, expected", [
         (ConceptExpression("foo"), {}),
         (OrderedChoice(ConceptExpression("foo"), StrMatch("one")), {'one': ['1002']}),
@@ -833,7 +1126,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         parser.sheerka = sheerka
 
         # every obvious cyclic recursion are removed from concept_by_first_keyword dict
-        parser.init_from_concepts(context, my_map.values())
         assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == expected
 
         # get_parsing_expression() also returns CHICKEN_AND_EGG
@@ -858,7 +1150,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         parser.sheerka = sheerka
 
         # every obvious cyclic recursion are removed from concept_by_first_keyword dict
-        parser.init_from_concepts(context, my_map.values())
         assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
 
         parsing_expression = parser.get_parsing_expression(context, my_map["foo"])
@@ -884,7 +1175,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         parser.sheerka = sheerka
 
         # every obvious cyclic recursion are removed from concept_by_first_keyword dict
-        parser.init_from_concepts(context, my_map.values())
         assert sheerka.om.copy(SheerkaConceptManager.RESOLVED_CONCEPTS_BY_FIRST_KEYWORD_ENTRY) == {}
 
         parsing_expression = parser.get_parsing_expression(context, my_map["foo"])
@@ -908,8 +1198,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
             "foo": self.bnf_concept("foo", expr),
         }
 
-        sheerka, context, parser = self.init_parser(my_map, singleton=True)
-        parser.init_from_concepts(context, my_map.values())
+        sheerka, context, parser = self.init_parser(my_map)
         parser.context = context
         parser.sheerka = sheerka
 
@@ -923,7 +1212,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
             "number": Concept("number"),
             "twenties": self.bnf_concept("twenties", Sequence(ConceptExpression("twenty"), ConceptExpression("number")))
         }
-        sheerka, context, parser = self.init_parser(my_map, singleton=True)
+        sheerka, context, parser = self.init_parser(my_map)
         parser.context = context
         parser.sheerka = sheerka
         sheerka.set_isa(context, sheerka.new("one"), my_map["number"])
@@ -1025,8 +1314,8 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
             ConceptExpression(my_map["one"], rule_name="one"))
 
     @pytest.mark.parametrize("expr, text, expected", [
-        # (ZeroOrMore(StrMatch("one"), sep=","), "one,", [CNC("foo", source="one"), UTN(",")]),
-        # (StrMatch("one"), "one two", [CNC("foo", source="one"), UTN(" two")]),
+        (ZeroOrMore(StrMatch("one"), sep=","), "one,", [CNC("foo", source="one"), UTN(",")]),
+        (StrMatch("one"), "one two", [CNC("foo", source="one"), UTN(" two")]),
         (StrMatch("one"), "two one", [UTN("two "), CNC("foo", source="one")]),
     ])
     def test_i_can_recognize_unknown_concepts(self, expr, text, expected):
@@ -1053,7 +1342,6 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
             "three": self.bnf_concept("three")
         }
         sheerka, context, parser = self.init_parser(my_map, singleton=True)
-        parser.init_from_concepts(context, my_map.values())
 
         parser.reset_parser(context, ParserInput("one three"))
         sequences = parser.get_concepts_sequences(context)
@@ -1067,6 +1355,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         ("bar", True, [CNC("foo or bar", source="bar", bar="bar", body="bar")]),
         ("one plus two", True, [CNC("plus", source="one plus two", one="one", two="two")]),
         ("twenty one", True, [CNC("t1", source="twenty one", unit="one")]),
+        ("one 'car'", True, [CNC("one thing", source="one 'car'", x=python_ret_val("'car'"), one="one")])
     ])
     def test_i_can_parse_simple_expressions(self, parser_input, expected_status, expected):
         sheerka, context, parser = self.init_parser(init_from_sheerka=True)
@@ -1359,8 +1648,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
             "expr": Concept("expr", definition="term ('+' term)*"),
         }
 
-        sheerka, context, parser = self.init_parser(my_map, singleton=True)
-        parser.init_from_concepts(context, my_map.values())
+        sheerka, context, parser = self.init_parser(my_map)
 
         text = "1 + 2 * 3"
 
@@ -1396,8 +1684,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
                 Sequence(ConceptExpression("term"), StrMatch("+"), ConceptExpression("expr")),
                 ConceptExpression("term"))),
         }
-        sheerka, context, parser = self.init_parser(my_map, singleton=True)
-        parser.init_from_concepts(context, my_map.values())
+        sheerka, context, parser = self.init_parser(my_map)
 
         text = "1 + 2 * 3"
 
@@ -1437,8 +1724,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
                                                     OrderedChoice(StrMatch("bar"), ConceptExpression("foo")))),
         }
 
-        sheerka, context, parser = self.init_parser(my_map, singleton=True)
-        parser.init_from_concepts(context, my_map.values())
+        sheerka, context, parser = self.init_parser(my_map)
 
         assert parser.parse(context, ParserInput("foo bar")).status
         assert parser.parse(context, ParserInput("foo foo foo bar")).status
@@ -1475,6 +1761,128 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         assert res.status
         assert res.value.value == compute_expected_array(cmap, text, [CN("thirties", source=text)])
 
+    def test_i_do_not_eat_unwanted_tokens_at_the_beginning_when_concept_with_variable(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(VariableExpression("x"), StrMatch("shoe"))),
+            "one": Concept("one"),
+            "two": Concept("two"),
+        }
+        sheerka, context, parser = self.init_parser(my_map)
+
+        text = "two one shoe"
+        res = parser.parse(context, ParserInput(text))
+        assert res.status
+        assert res.value.value == compute_expected_array(my_map, text, [
+            CN("two"),
+            CNC("foo", source="one shoe", x=CC("one"))])
+
+    def test_i_do_not_eat_unwanted_tokens_at_the_end_when_concept_with_variable(self):
+        my_map = {
+            "foo": self.bnf_concept("foo", Sequence(StrMatch("one"), VariableExpression("x"))),
+            "bar": Concept("bar"),
+            "baz": Concept("baz"),
+        }
+        sheerka, context, parser = self.init_parser(my_map)
+
+        text = "one bar baz"
+        res = parser.parse(context, ParserInput(text))
+        assert res.status
+        assert res.value.value == compute_expected_array(my_map, text, [
+            CNC("foo", source="one bar", x=CC("bar")),
+            CN("baz")])
+
+    @pytest.mark.parametrize("parsing_expression, expected", [
+        (RegExMatch("a"), [RegExDef("a")]),
+        (OrderedChoice(StrMatch("first"), RegExMatch("a|b")), ["first", RegExDef("a|b")]),
+        (OrderedChoice(RegExMatch("a|b"), StrMatch("first")), [RegExDef("a|b"), "first"]),
+        (Sequence(StrMatch("a"), RegExMatch("a|b")), ["a"]),
+        (Sequence(RegExMatch("a|b"), StrMatch("a")), [RegExDef("a|b")]),
+        (OneOrMore(StrMatch("a"), RegExMatch("a|b")), ["a"]),
+        (OneOrMore(RegExMatch("a|b"), StrMatch("a")), [RegExDef("a|b")]),
+        (ZeroOrMore(StrMatch("a"), RegExMatch("a|b")), ["a"]),
+        (ZeroOrMore(RegExMatch("a|b"), StrMatch("a")), [RegExDef("a|b")]),
+    ])
+    def test_i_can_get_first_item(self, parsing_expression, expected):
+        sheerka = self.get_sheerka()
+
+        visitor = BnfNodeFirstTokenVisitor(sheerka)
+
+        visitor.visit(parsing_expression)
+        assert visitor.first_tokens == expected
+
+    def test_i_cannot_parse_regex_when_no_next_matching_token_cannot_be_found(self):
+        sheerka, context, foo = self.init_test().with_concepts(Concept("foo", definition="r'abcd'"),
+                                                               create_new=True).unpack()
+
+        parser = BnfNodeParser(sheerka=sheerka)
+        res = parser.parse(context, ParserInput("abcdef"))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
+        assert res.body.reason == [NoMatchingTokenError(4)]
+
+    @pytest.mark.parametrize("text", [
+        "one",
+        " one",
+        "one ",
+        " one "
+    ])
+    def test_i_cannot_parse_empty_variable(self, text):
+        sheerka, context, parser = self.init_parser(init_from_sheerka=True)
+
+        res = parser.parse(context, ParserInput("one"))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
+
+    @pytest.mark.parametrize("bnf, text", [
+        (Sequence(VariableExpression("x"), StrMatch("foo")), "one foo"),
+        (Sequence(StrMatch("foo"), VariableExpression("x")), "foo one"),
+        (Sequence(StrMatch("foo"), VariableExpression("x"), StrMatch("bar")), "foo one bar"),
+    ])
+    def test_i_cannot_parse_variable_when_unrecognized_nodes(self, bnf, text):
+        sheerka, context, foo = self.init_test().with_concepts(
+            self.bnf_concept("foo", Sequence(VariableExpression("x"), StrMatch("shoe")))
+        ).unpack()
+        parser = BnfNodeParser()
+        parser.init_from_concepts(context, [foo])
+
+        res = parser.parse(context, ParserInput(text))
+
+        assert not res.status
+        assert sheerka.isinstance(res.body, BuiltinConcepts.NOT_FOR_ME)
+
+    @pytest.mark.parametrize("to_match, ignore_case, multiline, explicit_flags", [
+        ("xxy", None, None, re.MULTILINE),
+        ("xxy", True, True, re.MULTILINE),
+        ("xxy", False, False, re.MULTILINE),
+    ])
+    def test_i_can_serialize_reg_ex_def(self, to_match, ignore_case, multiline, explicit_flags):
+        r = RegExDef(to_match, ignore_case, multiline, explicit_flags)
+        serialized = r.serialize()
+
+        r2 = RegExDef().deserialize(serialized)
+
+        assert r == r2
+
+    def test_i_can_resolve_parsing_expression_for_variable_concept(self):
+        sheerka, context, parser = self.init_parser(init_from_sheerka=True)
+
+        expression = Sequence(VariableExpression("x"), StrMatch("x"))
+        resolved = parser.resolve_parsing_expression(context, expression, {}, set(), set())
+
+        assert isinstance(resolved.nodes[0], VariableExpression)
+        assert resolved.nodes[0].nodes[0] == resolved.nodes[1]
+
+    def test_i_can_resolve_parsing_expression_when_ending_with_variable_concept(self):
+        sheerka, context, parser = self.init_parser(init_from_sheerka=True)
+
+        expression = Sequence(StrMatch("x"), VariableExpression("x"))
+        resolved = parser.resolve_parsing_expression(context, expression, {}, set(), set())
+
+        assert isinstance(resolved.nodes[1], VariableExpression)
+        assert resolved.nodes[0].nodes == []
+
     # @pytest.mark.parametrize("parser_input, expected", [
     #     ("one", [
     #         (True, [CNC("bnf_one", source="one", one="one", body="one")]),

tests/parsers/test_BnfParser.py

@@ -1,4 +1,5 @@
 import pytest
+
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, DEFINITION_TYPE_BNF
 from core.sheerka.services.SheerkaExecute import ParserInput
@@ -6,10 +7,9 @@ from core.tokenizer import Tokenizer, TokenKind, LexerError
 from parsers.BaseNodeParser import cnode
 from parsers.BaseParser import UnexpectedTokenParsingError, UnexpectedEofParsingError
 from parsers.BnfDefinitionParser import BnfDefinitionParser
-from parsers.BnfNodeParser import BnfNodeParser
+from parsers.BnfNodeParser import BnfNodeParser, RegExMatch, VariableExpression
 from parsers.BnfNodeParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, \
     OneOrMore, ConceptExpression
-
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
 
@@ -49,6 +49,7 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         ("1", StrMatch("1")),
         ("  1", StrMatch("1")),
         (",", StrMatch(",")),
+        ("r'str'", RegExMatch("str")),
         ("'foo'?", Optional(StrMatch("foo"))),
         ("'foo'*", ZeroOrMore(StrMatch("foo"))),
         ("'foo'+", OneOrMore(StrMatch("foo"))),
@@ -84,6 +85,19 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         ("('foo'=var)*", ZeroOrMore(StrMatch("foo", rule_name="var"))),
         ("'foo'=var+", OneOrMore(StrMatch("foo", rule_name="var"))),
         ("('foo'=var)+", OneOrMore(StrMatch("foo", rule_name="var"))),
+        ("r'str'=var", RegExMatch("str", rule_name="var")),
+        ("r'foo'?=var", Optional(RegExMatch("foo"), rule_name="var")),
+        ("(r'foo'?)=var", Optional(RegExMatch("foo"), rule_name="var")),
+        ("r'foo'*=var", ZeroOrMore(RegExMatch("foo"), rule_name="var")),
+        ("(r'foo'*)=var", ZeroOrMore(RegExMatch("foo"), rule_name="var")),
+        ("r'foo'+=var", OneOrMore(RegExMatch("foo"), rule_name="var")),
+        ("(r'foo'+)=var", OneOrMore(RegExMatch("foo"), rule_name="var")),
+        ("r'foo'=var?", Optional(RegExMatch("foo", rule_name="var"))),
+        ("(r'foo'=var)?", Optional(RegExMatch("foo", rule_name="var"))),
+        ("r'foo'=var*", ZeroOrMore(RegExMatch("foo", rule_name="var"))),
+        ("(r'foo'=var)*", ZeroOrMore(RegExMatch("foo", rule_name="var"))),
+        ("r'foo'=var+", OneOrMore(RegExMatch("foo", rule_name="var"))),
+        ("(r'foo'=var)+", OneOrMore(RegExMatch("foo", rule_name="var"))),
         ("(1 | 2 | 3)=var", OrderedChoice(StrMatch("1"), StrMatch("2"), StrMatch("3"), rule_name="var")),
         ("(1 2)=var", Sequence(StrMatch("1"), StrMatch("2"), rule_name="var")),
         ("(1 2)+=var", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")), rule_name="var")),
@@ -118,6 +132,8 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         ("foo=f", c("foo", "f")),
         ("foo=f 'constant'", Sequence(c("foo", "f"), StrMatch("constant"))),
         ("def 'concept'", Sequence(c("def"), StrMatch("concept"))),
+        ("c:foo:", c("foo")),
+        ("c:|1001:", c("foo")),
     ])
     def test_i_can_parse_regex_with_concept(self, expression, expected):
         sheerka, context, parser, *concepts = self.init_parser("foo", "bar", "var", "def")
@@ -131,6 +147,29 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         assert res.value.value == expected
         assert res.value.source == expression
 
+    @pytest.mark.parametrize("expression, expected", [
+        ("x", VariableExpression("x")),
+        ("x bar", Sequence(VariableExpression("x"), c("bar"))),
+        ("bar x", Sequence(c("bar"), VariableExpression("x"))),
+        ("x 'and' bar", Sequence(VariableExpression("x"), StrMatch("and"), c("bar"))),
+        ("x | bar", OrderedChoice(VariableExpression("x"), c("bar"))),
+        ("x*", ZeroOrMore(VariableExpression("x"))),
+        ("x+", OneOrMore(VariableExpression("x"))),
+        ("'str' = x", Sequence(StrMatch("str"), StrMatch("="), VariableExpression("x"))),
+        ("'str''='x", Sequence(StrMatch("str"), StrMatch("="), VariableExpression("x"))),
+        ("foo=x", VariableExpression("x")),
+    ])
+    def test_i_can_parse_regex_with_variable(self, expression, expected):
+        # A variable is an identifier that cannot be resolved to a concept
+        sheerka, context, regex_parser, bar = self.init_parser("bar")
+        update_concepts_ids(sheerka, expected)
+
+        res = regex_parser.parse(self.get_context(), expression)
+
+        assert res.status
+        assert res.value.value == expected
+        assert res.value.source == expression
+
     @pytest.mark.parametrize("expression, expected", [
         ("foo", ConceptExpression("foo")),
         ("foo=f", ConceptExpression("foo", rule_name="f")),
@@ -208,13 +247,18 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         assert context.sheerka.isinstance(res.value, BuiltinConcepts.CANNOT_RESOLVE_CONCEPT)
         assert res.value.body == ('key', 'foo')
 
-    def test_i_cannot_parse_when_unknown_concept(self):
+    @pytest.mark.parametrize("text, expected", [
+        ("c:foo:", {'key': 'foo'}),
+        ("c:|1001:", {'id': '1001'}),
+        ("c:foo|1001:", {'key': 'foo', 'id': '1001'}),
+    ])
+    def test_i_cannot_parse_when_unknown_concept(self, text, expected):
         sheerka, context, regex_parser = self.init_parser()
-        res = regex_parser.parse(self.get_context(), "foo")
+        res = regex_parser.parse(self.get_context(), text)
 
         assert not res.status
         assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
-        assert res.value.body == ('key', 'foo')
+        assert res.value.body == expected
 
     def test_concept_expression_are_correctly_created_when_isa_concept_is_detected(self):
         sheerka, context, parser, one, two, number, twenties = self.init_parser(

tests/parsers/test_DefConceptParser.py

@@ -2,6 +2,7 @@ import ast
 from dataclasses import dataclass
 
 import pytest
+
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
 from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept, CV
 from core.global_symbols import NotInit
@@ -9,13 +10,13 @@ from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Keywords, Tokenizer, LexerError
 from parsers.BaseNodeParser import SCWC
 from parsers.BaseParser import UnexpectedEofParsingError
-from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence
 from parsers.BnfDefinitionParser import BnfDefinitionParser
+from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence, RegExMatch, OneOrMore, \
+    VariableExpression
 from parsers.DefConceptParser import DefConceptParser, NameNode, SyntaxErrorNode
 from parsers.DefConceptParser import UnexpectedTokenParsingError, DefConceptNode
 from parsers.FunctionParser import FunctionParser
 from parsers.PythonParser import PythonParser, PythonNode
-
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.parsers.parsers_utils import compute_expected_array
 
@@ -332,7 +333,7 @@ def concept add one to a as:
         "def concept name from bnf ",
         "def concept name from bnf as True",
     ])
-    def test_i_cannot_parse_empty_bnf_definition(self, text):
+    def test_i_cannot_parse_empty_bnf_definition_when_no_definition(self, text):
         sheerka, context, parser, *concepts = self.init_parser()
         res = parser.parse(context, ParserInput(text))
         error = res.body
@@ -347,7 +348,8 @@ def concept add one to a as:
 
         node = res.value.value
         definition = OrderedChoice(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("a_string"))
-        parser_result = ParserResultConcept(BnfDefinitionParser(), "a_concept | 'a_string'", None, definition, definition)
+        parser_result = ParserResultConcept(BnfDefinitionParser(), "a_concept | 'a_string'", None, definition,
+                                            definition)
         expected = get_def_concept(name="name", body="__definition[0]", bnf_def=parser_result)
 
         assert res.status
@@ -356,6 +358,22 @@ def concept add one to a as:
         assert isinstance(res.value, ParserResultConcept)
         assert node == expected
 
+    def test_i_can_parse_def_concept_from_bnf_when_using_concept_token(self):
+        text = "def concept name from bnf c:a_concept: 'xxx'"
+        sheerka, context, parser, a_concept = self.init_parser("a_concept")
+        res = parser.parse(context, ParserInput(text))
+
+        node = res.value.value
+        definition = Sequence(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("xxx"))
+        parser_result = ParserResultConcept(BnfDefinitionParser(), "c:a_concept: 'xxx'", None, definition, definition)
+        expected = get_def_concept(name="name", bnf_def=parser_result)
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+        assert node == expected
+
     def test_i_can_parse_def_concept_where_bnf_references_itself(self):
         text = "def concept name from bnf 'a' + name?"
         sheerka, context, parser, a_concept = self.init_parser("a_concept")
@@ -495,15 +513,6 @@ from give me the date !
         assert res.body.body[0].message == error_msg
         assert res.body.body[0].text == error_text
 
-    def test_i_cannot_parse_bnf_definition_referencing_unknown_concept(self):
-        text = "def concept name from bnf unknown"
-        sheerka, context, parser, *concepts = self.init_parser()
-        res = parser.parse(context, ParserInput(text))
-
-        assert not res.status
-        assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
-        assert res.value.body == ("key", "unknown")
-
     def test_i_cannot_parse_bnf_definition_referencing_multiple_concepts_sharing_the_same_name(self):
         text = "def concept twenty one from bnf 'twenty' one"
         sheerka, context, parser, *concepts = self.init_parser(Concept("one", body="1"), Concept("one", body="1.0"))
@@ -557,5 +566,78 @@ from give me the date !
         assert isinstance(res.value, ParserResultConcept)
         assert node == expected
 
+    def test_i_can_parse_bnf_concept_with_regex(self):
+        sheerka, context, parser, number = self.init_parser("number")
+        text = "def concept sha512 from bnf r'^[a-f0-9]{128}$'"
+        res = parser.parse(context, ParserInput(text))
 
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
 
+        node = res.value.value
+        parsing_expression = RegExMatch("^[a-f0-9]{128}$")
+        parser_result = ParserResultConcept(BnfDefinitionParser(),
+                                            "r'^[a-f0-9]{128}$'",
+                                            None,
+                                            parsing_expression,
+                                            parsing_expression)
+        expected = get_def_concept(name="sha512", bnf_def=parser_result)
+        assert node == expected
+
+    def test_i_can_parse_bnf_concept_with_a_more_complicated_bnf(self):
+        sheerka, context, parser, number = self.init_parser("number")
+        text = "def concept foo from bnf number | r'[a-f0-9]+' | (number r'[a-f0-9]+')+"
+        res = parser.parse(context, ParserInput(text))
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+
+        node = res.value.value
+        parsing_expression = OrderedChoice(
+            ConceptExpression(number, rule_name="number"),
+            RegExMatch("[a-f0-9]+"),
+            OneOrMore(Sequence(ConceptExpression(number, rule_name="number"), RegExMatch("[a-f0-9]+")))
+        )
+        parser_result = ParserResultConcept(BnfDefinitionParser(),
+                                            "number | r'[a-f0-9]+' | (number r'[a-f0-9]+')+",
+                                            None,
+                                            parsing_expression,
+                                            parsing_expression)
+        expected = get_def_concept(name="foo", bnf_def=parser_result)
+        assert node == expected
+
+    def test_i_can_parse_bnf_concept_definition_with_a_variable(self):
+        sheerka, context, parser, number = self.init_parser("number")
+        text = "def concept foo from bnf number x where x"
+        res = parser.parse(context, ParserInput(text))
+
+        node = res.value.value
+        definition = Sequence(ConceptExpression(number, rule_name="number"), VariableExpression("x"))
+        parser_result = ParserResultConcept(BnfDefinitionParser(), "number x", None, definition, definition)
+        expected = get_def_concept(name="foo", bnf_def=parser_result, where="x")
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+        assert node == expected
+
+    def test_i_can_parse_bnf_definition_referencing_unknown_concept(self):
+        text = "def concept name from bnf unknown"
+        sheerka, context, parser, *concepts = self.init_parser()
+        res = parser.parse(context, ParserInput(text))
+
+        node = res.value.value
+        definition = VariableExpression("unknown")
+        parser_result = ParserResultConcept(BnfDefinitionParser(), "unknown", None, definition, definition)
+        expected = get_def_concept(name="name", bnf_def=parser_result)
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+        assert node == expected

tests/parsers/test_parsers_utils.py

@@ -0,0 +1,109 @@
+from core.concept import Concept, ConceptParts, CC
+from core.sheerka.services.SheerkaExecute import ParserInput
+from parsers.BaseNodeParser import CNC
+from parsers.BnfNodeParser import BnfNodeParser
+from parsers.SyaNodeParser import SyaNodeParser
+from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
+from tests.parsers.parsers_utils import get_test_obj
+
+
+class TestParsersUtils(TestUsingMemoryBasedSheerka):
+
+    def test_i_can_get_test_obj_when_CNC_from_sya(self):
+        sheerka, context, one, two, plus = self.init_concepts(
+            "one",
+            "two",
+            Concept("a plus b").def_var("a").def_var("b")
+        )
+
+        parser = SyaNodeParser().init_from_concepts(context, [one, two, plus])
+        cnode = parser.parse(context, ParserInput("one plus two")).body.body[0]
+
+        # compare all attributes
+        cnc_res = get_test_obj(CNC(concept_key="key", start=0, end=1, source="", exclude_body=False), cnode)
+        assert isinstance(cnc_res, CNC)
+        assert cnc_res == CNC("__var__0 plus __var__1", 0, 4, "one plus two", False, **cnode.concept.get_compiled())
+
+        # I can discard start, end and source
+        cnc_res = get_test_obj(CNC(concept_key="key"), cnode)
+        assert isinstance(cnc_res, CNC)
+        assert cnc_res == CNC("__var__0 plus __var__1", None, None, None, False, **cnode.concept.get_compiled())
+
+    def test_i_can_get_test_obj_when_CNC_from_bnf(self):
+        sheerka, context, one, two, plus = self.init_concepts(
+            "one",
+            "two",
+            Concept("twenties", definition="'twenty' (one | two)=unit").def_var("unit").def_var("one").def_var("two")
+        )
+
+        parser = BnfNodeParser().init_from_concepts(context, [one, two, plus])
+        cnode = parser.parse(context, ParserInput("twenty one")).body.body[0]
+
+        # compare all attributes
+        cnc_res = get_test_obj(CNC(concept_key="key", start=0, end=1, source="", exclude_body=False), cnode)
+        assert isinstance(cnc_res, CNC)
+        assert cnc_res == CNC("twenties", 0, 2, "twenty one", False, **cnode.concept.get_compiled())
+
+        # I can exclude body
+        cnc_res = get_test_obj(CNC(concept_key="key", exclude_body=True), cnode)
+        expected_compiled = {k: v for k, v in cnode.concept.get_compiled().items()}
+        del expected_compiled[ConceptParts.BODY]
+        assert isinstance(cnc_res, CNC)
+        assert cnc_res == CNC("twenties", None, None, None, False, **expected_compiled)
+
+    def test_i_can_get_test_obj_when_list(self):
+        sheerka, context, one, two, plus = self.init_concepts(
+            "one",
+            "two",
+            Concept("a plus b").def_var("a").def_var("b")
+        )
+
+        parser = SyaNodeParser().init_from_concepts(context, [one, two, plus])
+        cnode = parser.parse(context, ParserInput("one plus two")).body.body[0]
+
+        res = get_test_obj([CNC("key1"), CNC("key", 0, 1, "")], [cnode, cnode])
+
+        assert len(res) == 2
+        assert isinstance(res[0], CNC)
+        assert res[0] == CNC("__var__0 plus __var__1", None, None, None, False, **cnode.concept.get_compiled())
+        assert isinstance(res[1], CNC)
+        assert res[1] == CNC("__var__0 plus __var__1", 0, 4, "one plus two", False, **cnode.concept.get_compiled())
+
+    def test_i_can_get_test_obj_when_dict(self):
+        sheerka, context, one, two, plus = self.init_concepts(
+            "one",
+            "two",
+            Concept("a plus b").def_var("a").def_var("b")
+        )
+
+        parser = SyaNodeParser().init_from_concepts(context, [one, two, plus])
+        cnode = parser.parse(context, ParserInput("one plus two")).body.body[0]
+
+        res = get_test_obj({"key1": CNC("key1"), "key2": CNC("key", 0, 1, "")}, {"key1": cnode, "key2": cnode})
+        assert len(res) == 2
+        assert isinstance(res["key1"], CNC)
+        assert res["key1"] == CNC("__var__0 plus __var__1", None, None, None, False, **cnode.concept.get_compiled())
+        assert isinstance(res["key2"], CNC)
+        assert res["key2"] == CNC("__var__0 plus __var__1", 0, 4, "one plus two", False, **cnode.concept.get_compiled())
+
+    def test_i_can_get_test_obj_when_CC(self):
+        sheerka, context, one, two, plus = self.init_concepts(
+            "one",
+            "two",
+            Concept("twenties", definition="'twenty' (one | two)=unit").def_var("unit").def_var("one").def_var("two")
+        )
+
+        parser = BnfNodeParser().init_from_concepts(context, [one, two, plus])
+        cc = parser.parse(context, ParserInput("twenty one")).body.body[0].concept
+
+        # compare all attributes
+        cc_res = get_test_obj(CC(concept="key", source="", exclude_body=False), cc)
+        assert isinstance(cc_res, CC)
+        assert cc_res == CC("twenties", "twenty one", False, **cc.get_compiled())
+
+        # I can exclude body
+        cnc_res = get_test_obj(CC(concept="key", exclude_body=True), cc)
+        expected_compiled = {k: v for k, v in cc.get_compiled().items()}
+        del expected_compiled[ConceptParts.BODY]
+        assert isinstance(cnc_res, CC)
+        assert cnc_res == CC("twenties", "twenty one", True, **expected_compiled)

tests/sheerkapickle/test_SheerkaPickler.py

@@ -2,7 +2,7 @@ import logging
 
 import pytest
 from core.concept import Concept
-from core.global_symbols import NotInit, NotFound, Removed
+from core.global_symbols import NotInit, NotFound, Removed, NoFirstToken
 from core.tokenizer import Keywords
 from sheerkapickle import tags
 from sheerkapickle.SheerkaPickler import SheerkaPickler
@@ -68,6 +68,7 @@ class TestSheerkaPickler(TestUsingMemoryBasedSheerka):
         (NotInit, {tags.CUSTOM: NotInit.value}),
         (NotFound, {tags.CUSTOM: NotFound.value}),
         (Removed, {tags.CUSTOM: Removed.value}),
+        (NoFirstToken, {tags.CUSTOM: NoFirstToken.value}),
     ])
     def test_i_can_flatten_and_restore_custom_types(self, obj, expected):
         sheerka = self.get_sheerka()