Fixed #30 : Add variable support in BNF concept definition

Fixed #31 : Add regex support in BNF Concept
Fixed #33 : Do not memorize object during restore

tests/parsers/test_DefConceptParser.py

@@ -2,6 +2,7 @@ import ast
 from dataclasses import dataclass
 
 import pytest
+
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
 from core.concept import DEFINITION_TYPE_BNF, DEFINITION_TYPE_DEF, Concept, CV
 from core.global_symbols import NotInit
@@ -9,13 +10,13 @@ from core.sheerka.services.SheerkaExecute import ParserInput
 from core.tokenizer import Keywords, Tokenizer, LexerError
 from parsers.BaseNodeParser import SCWC
 from parsers.BaseParser import UnexpectedEofParsingError
-from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence
 from parsers.BnfDefinitionParser import BnfDefinitionParser
+from parsers.BnfNodeParser import OrderedChoice, ConceptExpression, StrMatch, Sequence, RegExMatch, OneOrMore, \
+    VariableExpression
 from parsers.DefConceptParser import DefConceptParser, NameNode, SyntaxErrorNode
 from parsers.DefConceptParser import UnexpectedTokenParsingError, DefConceptNode
 from parsers.FunctionParser import FunctionParser
 from parsers.PythonParser import PythonParser, PythonNode
-
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.parsers.parsers_utils import compute_expected_array
 
@@ -332,7 +333,7 @@ def concept add one to a as:
         "def concept name from bnf ",
         "def concept name from bnf as True",
     ])
-    def test_i_cannot_parse_empty_bnf_definition(self, text):
+    def test_i_cannot_parse_empty_bnf_definition_when_no_definition(self, text):
         sheerka, context, parser, *concepts = self.init_parser()
         res = parser.parse(context, ParserInput(text))
         error = res.body
@@ -347,7 +348,8 @@ def concept add one to a as:
 
         node = res.value.value
         definition = OrderedChoice(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("a_string"))
-        parser_result = ParserResultConcept(BnfDefinitionParser(), "a_concept | 'a_string'", None, definition, definition)
+        parser_result = ParserResultConcept(BnfDefinitionParser(), "a_concept | 'a_string'", None, definition,
+                                            definition)
         expected = get_def_concept(name="name", body="__definition[0]", bnf_def=parser_result)
 
         assert res.status
@@ -356,6 +358,22 @@ def concept add one to a as:
         assert isinstance(res.value, ParserResultConcept)
         assert node == expected
 
+    def test_i_can_parse_def_concept_from_bnf_when_using_concept_token(self):
+        text = "def concept name from bnf c:a_concept: 'xxx'"
+        sheerka, context, parser, a_concept = self.init_parser("a_concept")
+        res = parser.parse(context, ParserInput(text))
+
+        node = res.value.value
+        definition = Sequence(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("xxx"))
+        parser_result = ParserResultConcept(BnfDefinitionParser(), "c:a_concept: 'xxx'", None, definition, definition)
+        expected = get_def_concept(name="name", bnf_def=parser_result)
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+        assert node == expected
+
     def test_i_can_parse_def_concept_where_bnf_references_itself(self):
         text = "def concept name from bnf 'a' + name?"
         sheerka, context, parser, a_concept = self.init_parser("a_concept")
@@ -495,15 +513,6 @@ from give me the date !
         assert res.body.body[0].message == error_msg
         assert res.body.body[0].text == error_text
 
-    def test_i_cannot_parse_bnf_definition_referencing_unknown_concept(self):
-        text = "def concept name from bnf unknown"
-        sheerka, context, parser, *concepts = self.init_parser()
-        res = parser.parse(context, ParserInput(text))
-
-        assert not res.status
-        assert context.sheerka.isinstance(res.value, BuiltinConcepts.UNKNOWN_CONCEPT)
-        assert res.value.body == ("key", "unknown")
-
     def test_i_cannot_parse_bnf_definition_referencing_multiple_concepts_sharing_the_same_name(self):
         text = "def concept twenty one from bnf 'twenty' one"
         sheerka, context, parser, *concepts = self.init_parser(Concept("one", body="1"), Concept("one", body="1.0"))
@@ -557,5 +566,78 @@ from give me the date !
         assert isinstance(res.value, ParserResultConcept)
         assert node == expected
 
+    def test_i_can_parse_bnf_concept_with_regex(self):
+        sheerka, context, parser, number = self.init_parser("number")
+        text = "def concept sha512 from bnf r'^[a-f0-9]{128}$'"
+        res = parser.parse(context, ParserInput(text))
 
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
 
+        node = res.value.value
+        parsing_expression = RegExMatch("^[a-f0-9]{128}$")
+        parser_result = ParserResultConcept(BnfDefinitionParser(),
+                                            "r'^[a-f0-9]{128}$'",
+                                            None,
+                                            parsing_expression,
+                                            parsing_expression)
+        expected = get_def_concept(name="sha512", bnf_def=parser_result)
+        assert node == expected
+
+    def test_i_can_parse_bnf_concept_with_a_more_complicated_bnf(self):
+        sheerka, context, parser, number = self.init_parser("number")
+        text = "def concept foo from bnf number | r'[a-f0-9]+' | (number r'[a-f0-9]+')+"
+        res = parser.parse(context, ParserInput(text))
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+
+        node = res.value.value
+        parsing_expression = OrderedChoice(
+            ConceptExpression(number, rule_name="number"),
+            RegExMatch("[a-f0-9]+"),
+            OneOrMore(Sequence(ConceptExpression(number, rule_name="number"), RegExMatch("[a-f0-9]+")))
+        )
+        parser_result = ParserResultConcept(BnfDefinitionParser(),
+                                            "number | r'[a-f0-9]+' | (number r'[a-f0-9]+')+",
+                                            None,
+                                            parsing_expression,
+                                            parsing_expression)
+        expected = get_def_concept(name="foo", bnf_def=parser_result)
+        assert node == expected
+
+    def test_i_can_parse_bnf_concept_definition_with_a_variable(self):
+        sheerka, context, parser, number = self.init_parser("number")
+        text = "def concept foo from bnf number x where x"
+        res = parser.parse(context, ParserInput(text))
+
+        node = res.value.value
+        definition = Sequence(ConceptExpression(number, rule_name="number"), VariableExpression("x"))
+        parser_result = ParserResultConcept(BnfDefinitionParser(), "number x", None, definition, definition)
+        expected = get_def_concept(name="foo", bnf_def=parser_result, where="x")
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+        assert node == expected
+
+    def test_i_can_parse_bnf_definition_referencing_unknown_concept(self):
+        text = "def concept name from bnf unknown"
+        sheerka, context, parser, *concepts = self.init_parser()
+        res = parser.parse(context, ParserInput(text))
+
+        node = res.value.value
+        definition = VariableExpression("unknown")
+        parser_result = ParserResultConcept(BnfDefinitionParser(), "unknown", None, definition, definition)
+        expected = get_def_concept(name="name", bnf_def=parser_result)
+
+        assert res.status
+        assert res.who == parser.name
+        assert res.value.source == text
+        assert isinstance(res.value, ParserResultConcept)
+        assert node == expected