This commit is contained in:
parent
0f23e3d9ab
commit
18757d07c9
1 changed files with 3 additions and 1 deletions
|
|
@ -507,13 +507,15 @@ const StepTwo = ({
|
||||||
const separator = rules.segmentation.separator
|
const separator = rules.segmentation.separator
|
||||||
const max = rules.segmentation.max_tokens
|
const max = rules.segmentation.max_tokens
|
||||||
const overlap = rules.segmentation.chunk_overlap
|
const overlap = rules.segmentation.chunk_overlap
|
||||||
|
const isHierarchicalDocument = documentDetail.doc_form === ChunkingMode.parentChild
|
||||||
|
|| (rules.parent_mode && rules.subchunk_segmentation)
|
||||||
setSegmentIdentifier(separator)
|
setSegmentIdentifier(separator)
|
||||||
setMaxChunkLength(max)
|
setMaxChunkLength(max)
|
||||||
setOverlap(overlap!)
|
setOverlap(overlap!)
|
||||||
setRules(rules.pre_processing_rules)
|
setRules(rules.pre_processing_rules)
|
||||||
setDefaultConfig(rules)
|
setDefaultConfig(rules)
|
||||||
|
|
||||||
if (documentDetail.dataset_process_rule.mode === 'hierarchical') {
|
if (isHierarchicalDocument) {
|
||||||
setParentChildConfig({
|
setParentChildConfig({
|
||||||
chunkForContext: rules.parent_mode || 'paragraph',
|
chunkForContext: rules.parent_mode || 'paragraph',
|
||||||
parent: {
|
parent: {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue