jyapayne/dify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

optimize _merge_splits function by using enumerate instead of manual index tracking (#25680)

Browse source 
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
ChasePassion 2025-09-15 09:41:16 +08:00 • committed by GitHub
commit a3f2c05632
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 1 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

5
api/core/rag/splitter/text_splitter.py
View file

@ -110,9 +110,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):
docs = [] docs = []
current_doc: list[str] = [] current_doc: list[str] = []
total = 0 total = 0
index = 0 for d, _len in zip(splits, lengths):
for d in splits:
_len = lengths[index]
if total + _len + (separator_len if len(current_doc) > 0 else 0) > self._chunk_size: if total + _len + (separator_len if len(current_doc) > 0 else 0) > self._chunk_size:
if total > self._chunk_size: if total > self._chunk_size:
logger.warning( logger.warning(
@ -134,7 +132,6 @@ class TextSplitter(BaseDocumentTransformer, ABC):
current_doc = current_doc[1:] current_doc = current_doc[1:]
current_doc.append(d) current_doc.append(d)
total += _len + (separator_len if len(current_doc) > 1 else 0) total += _len + (separator_len if len(current_doc) > 1 else 0)
index += 1
doc = self._join_docs(current_doc, separator) doc = self._join_docs(current_doc, separator)
if doc is not None: if doc is not None:
docs.append(doc) docs.append(doc)