Added basic implentation for where

tests/parsers/test_BaseParser.py

@@ -18,7 +18,7 @@ def test_i_can_get_text_from_tokens(text, expected_text):
 
 
 @pytest.mark.parametrize("text, custom, expected_text", [
-    ("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value}"}, "execute(__C__concept_name)")
+    ("execute(c:concept_name:)", {TokenKind.CONCEPT: lambda t: f"__C__{t.value[0]}"}, "execute(__C__concept_name)")
 ])
 def test_i_can_get_text_from_tokens_with_custom_switcher(text, custom, expected_text):
     tokens = list(Tokenizer(text))

tests/parsers/test_BnfParser.py

@@ -2,7 +2,7 @@ import pytest
 
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept
-from core.tokenizer import Tokenizer, TokenKind, LexerError
+from core.tokenizer import Tokenizer, TokenKind, LexerError, Token
 from parsers.BaseParser import UnexpectedTokenErrorNode
 from parsers.BnfParser import BnfParser, UnexpectedEndOfFileError
 from parsers.ConceptLexerParser import StrMatch, Optional, ZeroOrMore, OrderedChoice, Sequence, OneOrMore, \
@@ -16,6 +16,14 @@ class ClassWithName:
         self.name = name
 
 
+def c(name):
+    concept = Concept(name).init_key()
+    return ConceptExpression(concept, rule_name=name)
+
+
+eof_token = Token(TokenKind.EOF, "", 0, 0, 0)
+
+
 class TestBnfParser(TestUsingMemoryBasedSheerka):
 
     @pytest.mark.parametrize("expression, expected", [
@@ -33,9 +41,10 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         ("1 2 | 3 4+", OrderedChoice(
             Sequence(StrMatch("1"), StrMatch("2")),
             Sequence(StrMatch("3"), OneOrMore(StrMatch("4"))))),
-        (
-            "1 (2 | 3) 4+",
-            Sequence(StrMatch("1"), OrderedChoice(StrMatch("2"), StrMatch("3")), OneOrMore(StrMatch("4")))),
+        ("1 (2 | 3) 4+", Sequence(
+            StrMatch("1"),
+            OrderedChoice(StrMatch("2"), StrMatch("3")),
+            OneOrMore(StrMatch("4")))),
         ("(1|2)+", OneOrMore(OrderedChoice(StrMatch("1"), StrMatch("2")))),
         ("(1 2)+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")))),
         ("1 *", Sequence(StrMatch("1"), StrMatch("*"))),
@@ -61,6 +70,13 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         ("(1 2)=var", Sequence(StrMatch("1"), StrMatch("2"), rule_name="var")),
         ("(1 2)+=var", OneOrMore(Sequence(StrMatch("1"), StrMatch("2")), rule_name="var")),
         ("(1 2)=var+", OneOrMore(Sequence(StrMatch("1"), StrMatch("2"), rule_name="var"))),
+        ("(1=a 2=b)=c", Sequence(StrMatch("1", rule_name="a"), StrMatch("2", rule_name="b"), rule_name="c")),
+        ("(1*=a)", ZeroOrMore(StrMatch("1"), rule_name="a")),
+        ("'a'* 'b'+", Sequence(ZeroOrMore(StrMatch("a")), OneOrMore(StrMatch("b")))),
+        ("('a'* 'b'+)", Sequence(ZeroOrMore(StrMatch("a")), OneOrMore(StrMatch("b")))),
+        ("('a'*=x 'b'+=y)=z", Sequence(
+            ZeroOrMore(StrMatch("a"), rule_name="x"),
+            OneOrMore(StrMatch("b"), rule_name="y"), rule_name="z")),
     ])
     def test_i_can_parse_regex(self, expression, expected):
         parser = BnfParser()
@@ -72,12 +88,12 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
         assert res.value.source == expression
 
     @pytest.mark.parametrize("expression, expected", [
-        ("foo", Concept("foo").init_key()),
-        ("foo*", ZeroOrMore(Concept("foo").init_key())),
-        ("foo 'and' bar+", Sequence(Concept("foo").init_key(), StrMatch("and"), OneOrMore(Concept("bar").init_key()))),
-        ("foo | bar?", OrderedChoice(Concept("foo").init_key(), Optional(Concept("bar").init_key()))),
-        ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
-        ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), Concept("var").init_key())),
+        ("foo", c("foo")),
+        ("foo*", ZeroOrMore(c("foo"))),
+        ("foo 'and' bar+", Sequence(c("foo"), StrMatch("and"), OneOrMore(c("bar")))),
+        ("foo | bar?", OrderedChoice(c("foo"), Optional(c("bar")))),
+        ("'str' = var", Sequence(StrMatch("str"), StrMatch("="), c("var"))),
+        ("'str''='var", Sequence(StrMatch("str"), StrMatch("="), c("var"))),
     ])
     def test_i_can_parse_regex_with_concept(self, expression, expected):
         foo = Concept("foo")
@@ -113,8 +129,8 @@ class TestBnfParser(TestUsingMemoryBasedSheerka):
     @pytest.mark.parametrize("expression, error", [
         ("1 ", UnexpectedEndOfFileError()),
         ("1|", UnexpectedEndOfFileError()),
-        ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.RPAR])),
-        ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", [TokenKind.IDENTIFIER])),
+        ("(1|)", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", eof_token, [TokenKind.RPAR])),
+        ("1=", UnexpectedTokenErrorNode("Unexpected token 'Token(<EOF>)'", eof_token, [TokenKind.IDENTIFIER])),
         ("'name", LexerError("Missing Trailing quote", "'name", 5, 1, 6))
     ])
     def test_i_can_detect_errors(self, expression, error):

tests/parsers/test_ConceptLexerParser.py

@@ -1,10 +1,13 @@
+from ast import Str
+
 import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, ConceptParts, DoNotResolve
 from core.tokenizer import Tokenizer, TokenKind, Token
+from parsers.BnfParser import BnfParser
 from parsers.ConceptLexerParser import ConceptLexerParser, ConceptNode, Sequence, StrMatch, OrderedChoice, Optional, \
     ParsingExpressionVisitor, TerminalNode, NonTerminalNode, ZeroOrMore, OneOrMore, \
-    UnrecognizedTokensNode, cnode, short_cnode
+    UnrecognizedTokensNode, cnode, short_cnode, ConceptExpression, ConceptGroupExpression
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -75,6 +78,7 @@ class TestConceptLexerParser(TestUsingMemoryBasedSheerka):
         context = self.get_context()
         for c in concepts:
             context.sheerka.add_in_cache(c)
+            context.sheerka.set_id_if_needed(c, False)
 
         parser = ConceptLexerParser()
         parser.initialize(context, grammar)
@@ -586,6 +590,32 @@ class TestConceptLexerParser(TestUsingMemoryBasedSheerka):
         assert res.status
         assert res.value.body == [cnode("foo", 0, 2, "twenty one")]
 
+    def test_i_can_initialize_when_cyclic_reference(self):
+        foo = Concept(name="foo")
+        grammar = {foo: Optional("one", ConceptExpression("foo"))}
+        context, parser = self.init([foo], grammar)
+
+        assert parser.concepts_grammars[foo] == Optional("one", ConceptExpression(foo, rule_name="foo"))
+
+    def test_i_cannot_initialize_when_cyclic_reference_when_concept_is_under_construction_and_not_known(self):
+        foo = Concept(name="foo").init_key()
+        grammar = {foo: Optional("one", ConceptExpression("foo"))}
+
+        context = self.get_context()
+        parser = ConceptLexerParser()
+        parser.initialize(context, grammar)
+        assert parser.concepts_grammars[foo] == Optional("one", ConceptExpression("foo", rule_name="foo"))
+
+    def test_i_can_initialize_when_cyclic_reference_when_concept_is_under_construction_and_known(self):
+        foo = Concept(name="foo").init_key()
+        grammar = {foo: Optional("one", ConceptExpression("foo"))}
+
+        context = self.get_context()
+        context.concepts["foo"] = foo
+        parser = ConceptLexerParser()
+        parser.initialize(context, grammar)
+        assert parser.concepts_grammars[foo] == Optional("one", ConceptExpression(foo, rule_name="foo"))
+
     def test_i_can_parse_concept_reference_that_is_group(self):
         """
         if one is  number, then number is a 'group'
@@ -1092,6 +1122,109 @@ class TestConceptLexerParser(TestUsingMemoryBasedSheerka):
         assert cprop(concept_found, "seq")[1] == DoNotResolve("un ok")
         assert cprop(concept_found, "seq")[2] == DoNotResolve("uno ok")
 
+    @pytest.mark.parametrize("rule, expected", [
+        (StrMatch("string"), "'string'"),
+        (StrMatch("string", rule_name="rule_name"), "'string'=rule_name"),
+        (Sequence(StrMatch("foo"), StrMatch("bar")), "('foo' 'bar')"),
+        (Sequence(StrMatch("foo"), StrMatch("bar"), rule_name="rule_name"), "('foo' 'bar')=rule_name"),
+        (OrderedChoice(StrMatch("foo"), StrMatch("bar")), "('foo'|'bar')"),
+        (OrderedChoice(StrMatch("foo"), StrMatch("bar"), rule_name="rule_name"), "('foo'|'bar')=rule_name"),
+        (Optional(StrMatch("foo")), "'foo'?"),
+        (Optional(StrMatch("foo"), rule_name="rule_name"), "'foo'?=rule_name"),
+        (ZeroOrMore(StrMatch("foo")), "'foo'*"),
+        (ZeroOrMore(StrMatch("foo"), rule_name="rule_name"), "'foo'*=rule_name"),
+        (OneOrMore(StrMatch("foo")), "'foo'+"),
+        (OneOrMore(StrMatch("foo"), rule_name="rule_name"), "'foo'+=rule_name"),
+        (Sequence(
+            Optional(StrMatch("foo"), rule_name="a"),
+            ZeroOrMore(StrMatch("bar"), rule_name="b"),
+            OneOrMore(StrMatch("baz"), rule_name="c"),
+            rule_name="d"), "('foo'?=a 'bar'*=b 'baz'+=c)=d"),
+        (OrderedChoice(
+            Optional(StrMatch("foo"), rule_name="a"),
+            ZeroOrMore(StrMatch("bar"), rule_name="b"),
+            OneOrMore(StrMatch("baz"), rule_name="c"),
+            rule_name="d"), "('foo'?=a|'bar'*=b|'baz'+=c)=d"),
+        (Sequence(
+            OrderedChoice(StrMatch("foo"), StrMatch("bar"), rule_name="a"),
+            OrderedChoice(StrMatch("x"), StrMatch("y"), rule_name="b"),
+            rule_name="c"), "(('foo'|'bar')=a ('x'|'y')=b)=c")
+    ])
+    def test_i_can_encode_grammar(self, rule, expected):
+        foo = Concept(name="foo")
+        grammar = {foo: rule}
+        context, parser = self.init([foo], grammar)
+
+        encoded = parser.encode_grammar(parser.concepts_grammars)
+        assert encoded["c:foo|1001:"] == expected
+
+        bnf_parser = BnfParser()
+        parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"])
+        assert parse_res.status
+        assert parse_res.value.value == rule
+
+    def test_i_can_encode_grammar_when_concept_simple(self):
+        foo = Concept(name="foo")
+        bar = Concept(name="bar")
+        grammar = {foo: ConceptExpression(bar)}
+        context, parser = self.init([foo, bar], grammar)
+
+        encoded = parser.encode_grammar(parser.concepts_grammars)
+        assert encoded["c:foo|1001:"] == "c:bar|1002:=bar"
+
+        bnf_parser = BnfParser()
+        parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"])
+        assert parse_res.status
+        assert parse_res.value.value == grammar[foo]
+
+    def test_i_can_encode_grammar_when_concepts(self):
+        foo = Concept(name="foo")
+        bar = Concept(name="bar")
+        baz = Concept(name="baz")
+        grammar = {foo: Sequence(
+            StrMatch("a"),
+            OrderedChoice(ConceptExpression(bar),
+                          OneOrMore(ConceptExpression(baz)), rule_name="oc"), rule_name="s")}
+        context, parser = self.init([foo, bar, baz], grammar)
+
+        encoded = parser.encode_grammar(parser.concepts_grammars)
+        assert encoded["c:foo|1001:"] == "('a' (c:bar|1002:=bar|c:baz|1003:=baz+)=oc)=s"
+
+        bnf_parser = BnfParser()
+        parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"])
+        assert parse_res.status
+        assert parse_res.value.value == grammar[foo]
+
+    def test_i_can_encode_grammar_when_set_concepts(self):
+        foo = Concept(name="foo")
+        bar = Concept(name="bar")
+        baz = Concept(name="baz")
+        grammar = {foo: Sequence(
+            StrMatch("a"),
+            OrderedChoice(bar,
+                          OneOrMore(ConceptExpression(baz)), rule_name="oc"), rule_name="s")}
+        context = self.get_context()
+        for c in [foo, bar, baz]:
+            context.sheerka.add_in_cache(c)
+            context.sheerka.set_id_if_needed(c, False)
+        context.sheerka.add_concept_to_set(context, baz, bar)
+
+        parser = ConceptLexerParser()
+        parser.initialize(context, grammar)
+
+        encoded = parser.encode_grammar(parser.concepts_grammars)
+        assert encoded["c:foo|1001:"] == "('a' (c:bar|1002:=bar|c:baz|1003:=baz+)=oc)=s"
+
+        bnf_parser = BnfParser()
+        parse_res = bnf_parser.parse(context, encoded["c:foo|1001:"])
+        assert parse_res.status
+
+        expected = Sequence(
+            StrMatch("a"),
+            OrderedChoice(ConceptGroupExpression(bar, rule_name="bar"),
+                          OneOrMore(ConceptExpression(baz, rule_name="baz")), rule_name="oc"), rule_name="s")
+        assert parse_res.value.value == expected
+
     #
     # def test_i_can_parse_basic_arithmetic_operations_and_resolve_properties(self):
     #     context = self.get_context()

tests/parsers/test_DefaultParser.py

@@ -3,7 +3,7 @@ import ast
 
 from core.builtin_concepts import ParserResultConcept, BuiltinConcepts, ReturnValueConcept
 from core.concept import Concept
-from parsers.ConceptLexerParser import OrderedChoice, StrMatch
+from parsers.ConceptLexerParser import OrderedChoice, StrMatch, ConceptExpression
 from parsers.PythonParser import PythonParser, PythonNode
 from core.tokenizer import Keywords, Tokenizer, LexerError
 from parsers.DefaultParser import DefaultParser, NameNode, SyntaxErrorNode, CannotHandleErrorNode, IsaConceptNode
@@ -246,7 +246,7 @@ def concept add one to a as
         parser = DefaultParser()
         res = parser.parse(context, text)
         node = res.value.value
-        definition = OrderedChoice(a_concept, StrMatch("a_string"))
+        definition = OrderedChoice(ConceptExpression(a_concept, rule_name="a_concept"), StrMatch("a_string"))
         parser_result = ParserResultConcept(BnfParser(), "a_concept | 'a_string'", definition, definition)
         expected = get_def_concept(name="name", body="__definition[0]", definition=parser_result)
 
@@ -321,7 +321,7 @@ def concept add one to a as
         ("def concept 'name", "Missing Trailing quote", "'name"),
         ("def concept name as 'body", "Missing Trailing quote", "'body"),
         ("def concept name from bnf 'expression", "Missing Trailing quote", "'expression"),
-        ("def concept c::", "Concept name not found", ""),
+        ("def concept c::", "Concept identifiers not found", ""),
     ])
     def test_i_cannot_parse_when_tokenizer_fails(self, text, error_msg, error_text):
         parser = DefaultParser()

tests/parsers/test_PythonParser.py

@@ -3,6 +3,7 @@ import pytest
 from core.builtin_concepts import ParserResultConcept
 from core.tokenizer import Tokenizer, LexerError
 from parsers.PythonParser import PythonNode, PythonParser, PythonErrorNode
+import core.utils
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 
@@ -52,8 +53,8 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
         assert isinstance(res.value.value[0].exception, SyntaxError)
 
     @pytest.mark.parametrize("text, error_msg, error_text", [
-        ("c::", "Concept name not found", ""),
-        ("c:: + 1", "Concept name not found", ""),
+        ("c::", "Concept identifiers not found", ""),
+        ("c:: + 1", "Concept identifiers not found", ""),
     ])
     def test_i_can_detect_lexer_errors(self, text, error_msg, error_text):
         parser = PythonParser()
@@ -66,12 +67,12 @@ class TestPythonParser(TestUsingMemoryBasedSheerka):
         assert res.body.body[0].text == error_text
 
     def test_i_can_parse_a_concept(self):
-        text = "c:concept_name: + 1"
+        text = "c:name|key: + 1"
 
         parser = PythonParser()
         res = parser.parse(self.get_context(), text)
 
         assert res
         assert res.value.value == PythonNode(
-            "c:concept_name: + 1",
-            ast.parse("__C__USE_CONCEPT__concept_name__C__+1", mode="eval"))
+            "c:name|key: + 1",
+            ast.parse(core.utils.encode_concept(("name", "key"), True) + "+1", mode="eval"))