Added SyaNodeParser (finally, after one month)

src/core/utils.py

@@ -163,7 +163,7 @@ def remove_list_from_list(lst, to_remove):
 def product(a, b):
     """
     Kind of cartesian product between lists a and b
-    knowing that a is also a list
+    knowing that a is also a list : a is a list of list !!!
 
     So it's a cartesian product between a list of list and a list
     """
@@ -176,7 +176,12 @@ def product(a, b):
     res = []
     for item_b in b:
         for item_a in a:
-            items = item_a + [item_b]
+            #items = item_a + [item_b]
+            items = item_a[:]
+            if hasattr(item_b, "__iter__"):
+                items.extend(item_b)
+            else:
+                items.append(item_b)
             res.append(items)
 
     return res
@@ -276,6 +281,7 @@ def str_concept(t):
     >>> assert str_concept((None, "id")) == "c:|id:"
     >>> assert str_concept(("key", None)) == "c:key:"
     >>> assert str_concept((None, None)) == ""
+    >>> assert str_concept(Concept(key="foo", id="bar")) == "c:foo|bar:"
     :param t:
     :return:
     """
@@ -297,6 +303,12 @@ def unstr_concept(concept_repr):
     """
     if concept_repr is like :c:key:id:
     return the key and the id
+    >>> assert unstr_concept("c:key:") == "key"
+    >>> assert unstr_concept("c:key|id:") == ("key", "id")
+    >>> assert unstr_concept("c:|id:") == ("None", "id")
+    >>> assert unstr_concept("c:key|:") == ("key", "None")
+    >>> # Otherwise, return (None,None)
+
     :param concept_repr:
     :return:
     """
@@ -371,3 +383,25 @@ def decode_concept(text):
         return key, id_, use_concept
 
     return None, None, None
+
+
+def tokens_index(tokens, sub_tokens, skip=0):
+    """
+    Index of the sub tokens in tokens
+    :param tokens: tokens
+    :param sub_tokens: sub tokens to search
+    :param skip: number of found to skip
+    :return:
+    """
+    expected = [token.value for token in sub_tokens if token.type != TokenKind.EOF]
+    for i in range(0, len(tokens) - len(expected) + 1):
+        for j in range(len(expected)):
+            if tokens[i + j].value != expected[j]:
+                break
+        else:
+            if skip == 0:
+                return i
+            else:
+                skip -= 1
+
+    raise ValueError(f"sub tokens '{sub_tokens}' not found")