Feat: chunk overlap supported (#2209)

Co-authored-by: jyong <jyong@dify.ai>
2024-01-26 13:24:40 +08:00 · 2024-01-26 13:24:40 +08:00 · 89fcf4ea7c
commit 89fcf4ea7c
parent 3322710dac
9 changed files with 53 additions and 8 deletions
--- a/api/models/dataset.py
+++ b/api/models/dataset.py
@ -134,7 +134,8 @@ class DatasetProcessRule(db.Model):
        ],
        'segmentation': {
            'delimiter': '\n',
-            'max_tokens': 1000
+            'max_tokens': 500,
+            'chunk_overlap': 50
        }
    }