fix: remove empty segment in splitter (#68)

2023-05-17 15:02:58 +08:00 · 2023-05-17 15:02:58 +08:00 · 0587ff0fba
commit 0587ff0fba
parent ce492d13f1
1 changed files with 1 additions and 1 deletions
--- a/api/core/indexing_runner.py
+++ b/api/core/indexing_runner.py
@ -343,7 +343,7 @@ class IndexingRunner:

            # parse document to nodes
            nodes = node_parser.get_nodes_from_documents([text_doc])
-
+            nodes = [node for node in nodes if node.text is not None and node.text.strip()]
            all_nodes.extend(nodes)

        return all_nodes