Refactored Caching, Refactored BnfNodeParser, Introduced Sphinx

2020-05-12 17:21:10 +02:00
parent 7d3a490bc5
commit 6e343ba996
110 changed files with 13865 additions and 7540 deletions
@@ -52,11 +52,20 @@ class UnrecognizedNodeParser(BaseParser):
                res = only_successful(context, res)
                if res.status:
                    lexer_nodes = get_lexer_nodes(res.body.body, node.start, node.tokens)
-                    sequences_found = core.utils.product(sequences_found, lexer_nodes)
+                    if lexer_nodes:
+                        # make lexer_nodes is not empty (for example, some Python result are discarded)
+                        sequences_found = core.utils.product(sequences_found, lexer_nodes)
+                    else:
+                        sequences_found = core.utils.product(sequences_found, [node])
+                        has_unrecognized = True
                else:
                    sequences_found = core.utils.product(sequences_found, [node])
                    has_unrecognized = True

+            elif isinstance(node, SourceCodeNode):
+                sequences_found = core.utils.product(sequences_found, [node])
+                has_unrecognized = True # never trust source code not. I may be an invalid source code
+
            else:  # cannot happen as of today :-)
                raise NotImplementedError()