Fixed #30 : Add variable support in BNF concept definition

Fixed #31 : Add regex support in BNF Concept
Fixed #33 : Do not memorize object during restore

tests/core/test_tokenizer.py

@@ -1,4 +1,5 @@
 import pytest
+
 from core.tokenizer import Tokenizer, Token, TokenKind, LexerError
 
 
@@ -172,6 +173,7 @@ def test_i_can_parse_concept_token(text, expected):
     assert tokens[0].type == TokenKind.CONCEPT
     assert tokens[0].value == expected
 
+
 @pytest.mark.parametrize("text, expected", [
     ("r:key:", ("key", None)),
     ("r:key|id:", ("key", "id")),
@@ -197,3 +199,6 @@ def test_i_can_parse_regex_token(text, expected):
 
     assert tokens[0].type == TokenKind.REGEX
     assert tokens[0].value == expected
+    assert tokens[0].str_value == "r" + expected
+    assert tokens[0].repr_value == "r" + expected
+    assert tokens[0].strip_quote == expected[1:-1]