Enhanced AtomNode parsing by name

tests/non_reg/test_sheerka_non_reg.py

@@ -192,18 +192,16 @@ as:
         assert sheerka.isinstance(res[0].value, BuiltinConcepts.NOP)
 
     def test_i_can_recognize_concept_with_variable(self):
-        sheerka = self.get_sheerka()
-        concept_hello = Concept(name="hello a").def_var("a")
-        concept_foo = Concept(name="foo")
-        sheerka.add_in_cache(concept_hello)
-        sheerka.add_in_cache(concept_foo)
+        sheerka, context,  concept_foo, concept_hello = self.init_concepts(
+            "foo",
+            Concept(name="hello a").def_var("a"),
+            create_new=True)
 
         res = sheerka.evaluate_user_input("hello foo")
         return_value = res[0].value
         assert len(res) == 1
         assert res[0].status
         assert sheerka.isinstance(return_value, concept_hello)
-        assert return_value.metadata.variables[0] == ('a', "foo")
 
         # sanity check
         evaluated = sheerka.evaluate_concept(self.get_context(eval_body=True), return_value)
@@ -864,12 +862,10 @@ as:
         sheerka = self.init_scenario(definitions)
 
         res = sheerka.evaluate_user_input("eval mult")
-
         assert res[0].status
         assert isinstance(res[0].body, Concept)
 
         # res = sheerka.evaluate_user_input("eval a mult b")
-        #
         # assert res[0].status
         # assert isinstance(res[0].body, Concept)
 

tests/parsers/test_AtomsParser.py

@@ -2,7 +2,7 @@ import pytest
 from core.builtin_concepts import BuiltinConcepts
 from core.concept import Concept, DEFINITION_TYPE_DEF
 from parsers.AtomNodeParser import AtomNodeParser
-from parsers.BaseNodeParser import cnode, utnode, CNC, SCN
+from parsers.BaseNodeParser import cnode, utnode, CNC, SCN, CN
 
 from tests.TestUsingMemoryBasedSheerka import TestUsingMemoryBasedSheerka
 from tests.parsers.parsers_utils import compute_expected_array
@@ -35,16 +35,16 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         ("foo", ["foo"]),
         ("foo bar", ["foo", "bar"]),
         ("foo bar twenties", ["foo", "bar", "twenties"]),
-        # ("plus", ["plus"]),
-        # ("++", ["++"]),
-        # ("a++ foo", ["++", "foo"]),
+        ("a plus b", [CN("plus", 0, 4)]),
+        ("mult", [CN("mult", 0, 0, "mult")]),
     ])
     def test_i_can_parse_simple_sequences(self, text, expected):
         concepts_map = {
             "foo": Concept("foo"),
             "bar": Concept("bar"),
             "plus": Concept("a plus b").def_var("a").def_var("b"),
-            "++": Concept("++", definition="a++", definition_type=DEFINITION_TYPE_DEF).def_var("a"),
+            "mult": Concept("mult", definition="a mult b", definition_type=DEFINITION_TYPE_DEF).def_var("a").def_var(
+                "b"),
             "twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_var("unit"),
         }
 
@@ -286,19 +286,19 @@ class TestAtomsParser(TestUsingMemoryBasedSheerka):
         assert lexer_nodes == expected_array
 
     @pytest.mark.parametrize("text, expected_is_evaluated", [
-        ("foo",  False),
-        ("bar", False ),
+        ("foo", False),
+        ("bar", False),
         ("twenties", True),
-        ("plus", True),
-        # ("plus", ["plus"]),
-        # ("++", ["++"]),
-        # ("a++ foo", ["++", "foo"]),
+        ("a plus b", True),
+        ("mult", True),
     ])
     def test_concepts_with_variables_must_not_be_evaluated(self, text, expected_is_evaluated):
         concepts_map = {
             "foo": Concept("foo"),
             "bar": Concept("bar", body="'bar'"),
-            "plus": Concept("plus", definition="a plus b", definition_type=DEFINITION_TYPE_DEF).def_var("a").def_var("b"),
+            "plus": Concept("a plus b").def_var("a").def_var("b"),
+            "mult": Concept("mult", definition="a mult b", definition_type=DEFINITION_TYPE_DEF).def_var("a").def_var(
+                "b"),
             "twenties": Concept("twenties", definition="'twenty' ('one'|'two')=unit").def_var("unit"),
         }
 

tests/parsers/test_BaseParser.py

@@ -65,3 +65,19 @@ def test_i_can_test_split_iter_parser_indexes():
     assert res[5] == Token(TokenKind.LPAR, "(", 20, 2, 12)
     assert res[6] == Token(TokenKind.RPAR, ")", 21, 2, 13)
     assert res[7] == Token(TokenKind.COMMA, ",", 22, 2, 14)
+
+
+@pytest.mark.parametrize("tokens, expected", [
+    (None, None),
+    ([], (0, 0)),
+    (list(Tokenizer("")), (0, 0)),
+    (list(Tokenizer("", yield_eof=False)), (0, 0)),
+    (list(Tokenizer(" a")), (1, 1)),
+    (list(Tokenizer(" a", yield_eof=False)), (1, 1)),
+    (list(Tokenizer("a ")), (0, 0)),
+    (list(Tokenizer("a ", yield_eof=False)), (0, 0)),
+    (list(Tokenizer(" a ")), (1, 1)),
+    (list(Tokenizer(" a ", yield_eof=False)), (1, 1)),
+])
+def test_i_can_get_tokens_boundaries(tokens, expected):
+    assert BaseParser.get_tokens_boundaries(tokens) == expected

tests/parsers/test_SyaNodeParser.py

@@ -632,18 +632,18 @@ class TestSyaNodeParser(TestUsingMemoryBasedSheerka):
         # I can't manage source code functions :-(
         # ("function(one plus three) minus two", []),
 
-        ("(one plus two) ", ["one", "two", "plus"]),
-        ("(one prefixed) ", ["one", "prefixed"]),
-        ("(suffixed one) ", ["one", "suffixed"]),
-        ("(one ? two : three)", ["one", "two", "three", "?"]),
-        ("square(square(one))", ["one", ("square", 1), "square"]),
-        ("square ( square ( one ) )", ["one", ("square", 1), "square"]),
-
-        ("square(one plus three) minus two", ["one", "three", "plus", "square", "two", "minus"]),
-        ("square( one plus three ) minus two", ["one", "three", "plus", "square", "two", "minus"]),
-        ("one minus square( two plus three ) ", ["one", "two", "three", "plus", "square", "minus"]),
-
-        ("((one prefixed) prefixed)", ["one", "prefixed", ("prefixed", 1)]),
+        # ("(one plus two) ", ["one", "two", "plus"]),
+        # ("(one prefixed) ", ["one", "prefixed"]),
+        # ("(suffixed one) ", ["one", "suffixed"]),
+        # ("(one ? two : three)", ["one", "two", "three", "?"]),
+        # ("square(square(one))", ["one", ("square", 1), "square"]),
+        # ("square ( square ( one ) )", ["one", ("square", 1), "square"]),
+        #
+        # ("square(one plus three) minus two", ["one", "three", "plus", "square", "two", "minus"]),
+        # ("square( one plus three ) minus two", ["one", "three", "plus", "square", "two", "minus"]),
+        # ("one minus square( two plus three ) ", ["one", "two", "three", "plus", "square", "minus"]),
+        #
+        # ("((one prefixed) prefixed)", ["one", "prefixed", ("prefixed", 1)]),
         ("( ( one prefixed ) prefixed)", ["one", "prefixed", ("prefixed", 1)]),
         ("( ( square( one ) prefixed ) prefixed)", ["one", "square", "prefixed", ("prefixed", 1)]),