Feat: chunk overlap supported (#2209)

Co-authored-by: jyong <jyong@dify.ai>

api/core/indexing_runner.py

@@ -562,7 +562,7 @@ class IndexingRunner:
 
             character_splitter = FixedRecursiveCharacterTextSplitter.from_encoder(
                 chunk_size=segmentation["max_tokens"],
-                chunk_overlap=0,
+                chunk_overlap=segmentation.get('chunk_overlap', 0),
                 fixed_separator=separator,
                 separators=["\n\n", "。", ".", " ", ""],
                 embedding_model_instance=embedding_model_instance
@@ -571,7 +571,7 @@ class IndexingRunner:
             # Automatic segmentation
             character_splitter = EnhanceRecursiveCharacterTextSplitter.from_encoder(
                 chunk_size=DatasetProcessRule.AUTOMATIC_RULES['segmentation']['max_tokens'],
-                chunk_overlap=0,
+                chunk_overlap=DatasetProcessRule.AUTOMATIC_RULES['segmentation']['chunk_overlap'],
                 separators=["\n\n", "。", ".", " ", ""],
                 embedding_model_instance=embedding_model_instance
             )

api/models/dataset.py

@@ -134,7 +134,8 @@ class DatasetProcessRule(db.Model):
         ],
         'segmentation': {
             'delimiter': '\n',
-            'max_tokens': 1000
+            'max_tokens': 500,
+            'chunk_overlap': 50
         }
     }
 

api/services/dataset_service.py

@@ -241,7 +241,8 @@ class DocumentService:
             ],
             'segmentation': {
                 'delimiter': '\n',
-                'max_tokens': 500
+                'max_tokens': 500,
+                'chunk_overlap': 50
             }
         }
     }