jyapayne/dify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Feat: chunk overlap supported (#2209)

Browse source 
Co-authored-by: jyong <jyong@dify.ai>
This commit is contained in:
KVOJJJin 2024-01-26 13:24:40 +08:00 • committed by GitHub
commit 89fcf4ea7c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 53 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

3
web/i18n/lang/dataset-creation.en.ts
View file

@ -59,6 +59,9 @@ const translation = {
separator: 'Segment identifier',
separatorPlaceholder: 'For example, newline (\\\\n) or special separator (such as "***")',
maxLength: 'Maximum chunk length',
overlap: 'Chunk overlap',
overlapTip: 'Setting the chunk overlap can maintain the semantic relevance between them, enhancing the retrieve effect. It is recommended to set 10%-25% of the maximum chunk size.',
overlapCheck: 'chunk overlap should not bigger than maximun chunk length',
rules: 'Text preprocessing rules',
removeExtraSpaces: 'Replace consecutive spaces, newlines and tabs',
removeUrlEmails: 'Delete all URLs and email addresses',