Feat/add unstructured support (#1780)

Co-authored-by: jyong <jyong@dify.ai>
2023-12-18 23:24:06 +08:00 · 2023-12-18 23:24:06 +08:00 · 5e34f938c1
commit 5e34f938c1
parent 2fd56cb01c
15 changed files with 361 additions and 14 deletions
--- a/api/models/dataset.py
+++ b/api/models/dataset.py
@ -135,7 +135,7 @@ class DatasetProcessRule(db.Model):
        ],
        'segmentation': {
            'delimiter': '\n',
-            'max_tokens': 512
+            'max_tokens': 1000
        }
    }