Added bnf when adding a new concept + Started logging filtering

tests/test_utils.py

@@ -1,6 +1,8 @@
 import core.utils
 import pytest
 
+from core.tokenizer import Token, TokenKind
+
 
 @pytest.mark.parametrize("lst, as_string", [
     (None, "",),
@@ -76,3 +78,63 @@ def test_i_can_get_sub_classes():
 def test_i_can_product(a, b, expected):
     res = core.utils.product(a, b)
     assert res == expected
+
+
+@pytest.mark.parametrize("input_as_list, expected_as_list", [
+    ([" "], []),
+    ([" ", "one"], ["one"]),
+    (["one", " "], ["one"]),
+    ([" ", "one", " "], ["one"]),
+
+    (["\n", "one"], ["one"]),
+    (["one", "\n"], ["one"]),
+    (["\n", "one", "\n"], ["one"]),
+
+    ([" ", "\n", "one"], ["one"]),
+    (["one", " ", "\n"], ["one"]),
+    ([" ", "\n", "one", " ", "\n"], ["one"]),
+
+    (["\n", " ", "one"], ["one"]),
+    (["one", "\n", " "], ["one"]),
+    (["\n", " ", "one", "\n", " "], ["one"]),
+
+    ([" ", "\n", " ", "one"], ["one"]),
+    (["one", " ", "\n", " "], ["one"]),
+    ([" ", "\n", " ", "one", " ", "\n", " "], ["one"]),
+
+    (["\n", " ", "\n", "one"], ["one"]),
+    (["one", "\n", " ", "\n"], ["one"]),
+    (["\n", " ", "\n", "one", "\n", " ", "\n"], ["one"]),
+
+])
+def test_i_can_strip(input_as_list, expected_as_list):
+    actual = core.utils.strip_tokens(get_tokens(input_as_list))
+    expected = get_tokens(expected_as_list)
+    assert actual == expected
+
+
+def test_by_default_eof_is_not_stripped():
+    actual = core.utils.strip_tokens(get_tokens(["one", "two", " ", "\n", "<EOF>"]))
+    expected = get_tokens(["one", "two", " ", "\n", "<EOF>"])
+    assert actual == expected
+
+
+def test_i_can_strip_eof():
+    actual = core.utils.strip_tokens(get_tokens(["one", "two", " ", "\n", "<EOF>"]), True)
+    expected = get_tokens(["one", "two"])
+    assert actual == expected
+
+
+def get_tokens(lst):
+    res = []
+    for e in lst:
+        if e == " ":
+            res.append(Token(TokenKind.WHITESPACE, " ", 0, 0, 0))
+        elif e == "\n":
+            res.append(Token(TokenKind.NEWLINE, "\n", 0, 0, 0))
+        elif e == "<EOF>":
+            res.append(Token(TokenKind.EOF, "\n", 0, 0, 0))
+        else:
+            res.append(Token(TokenKind.IDENTIFIER, e, 0, 0, 0))
+
+    return res