jyapayne/dify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix spliter length missed (#7987)

Browse source
This commit is contained in:
Jyong 2024-09-04 21:47:12 +08:00 • committed by GitHub
commit 0e71f6db84
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 11 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

5
api/core/rag/splitter/text_splitter.py
View file

@ -243,7 +243,10 @@ class CharacterTextSplitter(TextSplitter):
# First we naively split the large input into a bunch of smaller ones.
splits = _split_text_with_regex(text, self._separator, self._keep_separator)
_separator = "" if self._keep_separator else self._separator
return self._merge_splits(splits, _separator)
_good_splits_lengths = [] # cache the lengths of the splits
for split in splits:
_good_splits_lengths.append(self._length_function(split))
return self._merge_splits(splits, _separator, _good_splits_lengths)
class LineType(TypedDict):