Feat: chunk overlap supported (#2209)

Co-authored-by: jyong <jyong@dify.ai>
2024-01-26 13:24:40 +08:00 · 2024-01-26 13:24:40 +08:00 · 89fcf4ea7c
commit 89fcf4ea7c
parent 3322710dac
9 changed files with 53 additions and 8 deletions
--- a/web/i18n/lang/dataset-creation.zh.ts
+++ b/web/i18n/lang/dataset-creation.zh.ts
@ -59,6 +59,9 @@ const translation = {
    separator: '分段标识符',
    separatorPlaceholder: '例如换行符（\n）或特定的分隔符（如 "***"）',
    maxLength: '分段最大长度',
+    overlap: '分段重叠长度',
+    overlapTip: '设置分段之间的重叠长度可以保留分段之间的语义关系，提升召回效果。建议设置为最大分段长度的10%-25%',
+    overlapCheck: '分段重叠长度不能大于分段最大长度',
    rules: '文本预处理规则',
    removeExtraSpaces: '替换掉连续的空格、换行符和制表符',
    removeUrlEmails: '删除所有 URL 和电子邮件地址',