Fixed first token recognition when creating bnf concepts

tests/parsers/test_BnfNodeParser.py

@@ -1,6 +1,6 @@
 import pytest
 from core.builtin_concepts import BuiltinConcepts
-from core.concept import Concept, ConceptParts, DoNotResolve, DEFINITION_TYPE_DEF, CC
+from core.concept import Concept, ConceptParts, DoNotResolve
 from core.sheerka.services.SheerkaExecute import ParserInput
 from parsers.BaseNodeParser import CNC, UTN, CN
 from parsers.BnfNodeParser import BnfNodeParser, StrMatch, TerminalNode, NonTerminalNode, Sequence, OrderedChoice, \
@@ -30,6 +30,8 @@ cmap = {
     # sequence of keywords using def definition
     # "def_concept_def": Concept("def_concept_def", definition="def concept", definition_type=DEFINITION_TYPE_DEF),
     # "def concept_def number": Concept("def number", definition="def_concept_def (one|two|three)=number"),
+
+    "filter": Concept("filter", definition="'--filter' (one | two)")
 }
 
 
@@ -125,7 +127,7 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         assert res.body.reason == BuiltinConcepts.IS_EMPTY
 
     @pytest.mark.parametrize("expr, text", [
-        # (StrMatch("foo"), "foo"),
+        (StrMatch("foo"), "foo"),
         (StrMatch("'foo'"), "'foo'"),
         (StrMatch("1"), "1"),
         (StrMatch("3.14"), "3.14"),
@@ -171,6 +173,33 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         expected = [CNC("foo", source=text)]
         self.validate_get_concepts_sequences(my_map, text, expected)
 
+    def test_i_can_use_skip_whitespace_when_mixing_sequence_and_strmatch(self):
+        my_map = {
+            "filter": self.bnf_concept("filter",
+                                       Sequence(StrMatch("-", skip_whitespace=False),
+                                                StrMatch("-", skip_whitespace=False),
+                                                "filter")),
+        }
+
+        sheerka, context, *updated = self.init_concepts(*my_map.values(), create_new=False, singleton=True)
+        parser = BnfNodeParser()
+        parser.init_from_concepts(context, updated)
+
+        text = "--filter"
+        expected = [CN("filter", source="--filter")]
+        expected_array = compute_expected_array(my_map, text, expected)
+
+        parser.reset_parser(context, ParserInput(text))
+        bnf_parsers_helpers = parser.get_concepts_sequences()
+        assert bnf_parsers_helpers[0].sequence == expected_array
+        assert not bnf_parsers_helpers[0].has_unrecognized
+
+        # but I cannot parse
+        text = "- - filter"
+        parser.reset_parser(context, ParserInput(text))
+        bnf_parsers_helpers = parser.get_concepts_sequences()
+        assert bnf_parsers_helpers[0].has_unrecognized
+
     def test_i_can_match_multiple_sequences(self):
         my_map = {
             "foo": self.bnf_concept("foo", Sequence(StrMatch("one"), StrMatch("two"), StrMatch("three"))),
@@ -760,6 +789,20 @@ class TestBnfNodeParser(TestUsingMemoryBasedSheerka):
         assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
         assert concepts_nodes == expected_array
 
+    def test_i_can_parse_filter(self):
+        sheerka, context, parser = self.init_parser(init_from_sheerka=True)
+
+        expression = "--filter one"
+        expected = [CNC("filter", source="--filter one", one="one")]
+
+        res = parser.parse(context, ParserInput(expression))
+        expected_array = compute_expected_array(cmap, expression, expected)
+        parser_result = res.value
+        concepts_nodes = res.value.value
+
+        assert res.status == True
+        assert sheerka.isinstance(parser_result, BuiltinConcepts.PARSER_RESULT)
+        assert concepts_nodes == expected_array
 
     # @pytest.mark.parametrize("parser_input, expected", [
     #     ("one", [