Feat/parent child retrieval (#12086)

Signed-off-by: yihong0618 <zouzou0208@gmail.com>
Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: AkaraChen <akarachen@outlook.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Warren Chen <warren.chen830@gmail.com>
Co-authored-by: crazywoola <427733928@qq.com>
Co-authored-by: Yi Xiao <54782454+YIXIAO0@users.noreply.github.com>
Co-authored-by: yihong <zouzou0208@gmail.com>
Co-authored-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: KVOJJJin <jzongcode@gmail.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: JzoNgKVO <27049666+JzoNgKVO@users.noreply.github.com>
Co-authored-by: Charlie.Wei <luowei@cvte.com>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: huayaoyue6 <huayaoyue@163.com>
Co-authored-by: kurokobo <kuro664@gmail.com>
Co-authored-by: Matsuda <yiyth.fcb6@gmail.com>
Co-authored-by: shirochan <s.yusuke0711@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: Huỳnh Gia Bôi <boihuynh147@gmail.com>
Co-authored-by: Julian Huynh <julian.huynh@immersio.io>
Co-authored-by: Hash Brown <hi@xzd.me>
Co-authored-by: 非法操作 <hjlarry@163.com>
Co-authored-by: Kazuki Takamatsu <kazuki.takamatsu@chowagiken.co.jp>
Co-authored-by: Trey Dong <1346650911@qq.com>
Co-authored-by: VoidIsVoid <343750470@qq.com>
Co-authored-by: Gimling <huangjl@ruyi.ai>
Co-authored-by: xiandan-erizo <xiandan.erizo@gmail.com>
Co-authored-by: Muneyuki Noguchi <nogu.dev@gmail.com>
Co-authored-by: zhaobingshuang <1475195565@qq.com>
Co-authored-by: zhaobs <zhaobs@cailian.net>
Co-authored-by: suzuki.sh <s2terminal@users.noreply.github.com>
Co-authored-by: Yingchun Lai <laiyingchun@apache.org>
Co-authored-by: huanshare <huanshare@live.com>
Co-authored-by: huanshare <liuhuan101@longfor.com>
Co-authored-by: orangeclk <orangeclk@users.noreply.github.com>
Co-authored-by: 문정현 <120004247+JungHyunMoon@users.noreply.github.com>
Co-authored-by: barabicu <kztk533@gmail.com>
Co-authored-by: Wei Mingzhi <whistler_wmz@users.sf.net>
Co-authored-by: Paul van Oorschot <20116814+pvoo@users.noreply.github.com>
Co-authored-by: zkyTech <zhangkunyuan@hotmail.com>
Co-authored-by: zhangkunyuan <zhangkunyuan@cmhi.chinamobile.com>
Co-authored-by: Tommy <34446820+Asterovim@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Novice <857526207@qq.com>
Co-authored-by: Novice Lee <novicelee@NovicedeMacBook-Pro.local>
Co-authored-by: Novice Lee <novicelee@NoviPro.local>
Co-authored-by: zxhlyh <16177003+zxhlyh@users.noreply.github.com>
Co-authored-by: liuzhenghua <1090179900@qq.com>
Co-authored-by: Jiang <65766008+AlwaysBluer@users.noreply.github.com>
Co-authored-by: jiangzhijie <jiangzhijie.jzj@alibaba-inc.com>
Co-authored-by: Joe <79627742+ZhouhaoJiang@users.noreply.github.com>
Co-authored-by: Alok Shrivastwa <alok.shrivastwa@gmail.com>
Co-authored-by: Alok Shrivastwa <Alok.Shrivastwa@microland.com>
Co-authored-by: JasonVV <jasonwangiii@outlook.com>
Co-authored-by: Hiroshi Fujita <fujita-h@users.noreply.github.com>
Co-authored-by: Kevin9703 <51311316+Kevin9703@users.noreply.github.com>
Co-authored-by: NFish <douxc512@gmail.com>
Co-authored-by: Junyan Qin <1010553892@qq.com>
Co-authored-by: IWAI, Masaharu <iwaim.sub@gmail.com>
Co-authored-by: IWAI, Masaharu <iwai_masaharu@funkit.co.jp>
Co-authored-by: Bowen Liang <liangbowen@gf.com.cn>
Co-authored-by: luckylhb90 <luckylhb90@gmail.com>
Co-authored-by: hobo.l <hobo.l@binance.com>
Co-authored-by: douxc <7553076+douxc@users.noreply.github.com>

web/i18n/zh-Hans/dataset-creation.ts

@@ -47,6 +47,7 @@ const translation = {
     notionSyncTitle: 'Notion 未绑定',
     notionSyncTip: '同步 Notion 内容前，须先绑定 Notion 空间',
     connect: '去绑定',
+    cancel: '取消',
     button: '下一步',
     emptyDatasetCreation: '创建一个空知识库',
     modal: {
@@ -99,6 +100,16 @@ const translation = {
     autoDescription: '自动设置分段规则与预处理规则，如果不了解这些参数建议选择此项',
     custom: '自定义',
     customDescription: '自定义分段规则、分段长度以及预处理规则等参数',
+    general: '通用',
+    generalTip: '通用文本分块模式，检索和回忆的块是相同的',
+    parentChild: '父子分段',
+    parentChildTip: '使用父子模式时，子块用于检索，父块用作上下文',
+    parentChunkForContext: '父块用作上下文',
+    childChunkForRetrieval: '子块用于检索',
+    paragraph: '段落',
+    paragraphTip: '此模式根据分隔符和最大块长度将文本拆分为段落，使用拆分文本作为检索的父块',
+    fullDoc: '全文',
+    fullDocTip: '整个文档用作父块并直接检索。请注意，出于性能原因，超过10000个标记的文本将被自动截断。',
     separator: '分段标识符',
     separatorTip: '分隔符是用于分隔文本的字符。\\n\\n 和 \\n 是常用于分隔段落和行的分隔符。用逗号连接分隔符（\\n\\n,\\n），当段落超过最大块长度时，会按行进行分割。你也可以使用自定义的特殊分隔符（例如 ***）。',
     separatorPlaceholder: '\\n\\n 用于分段；\\n 用于分行',
@@ -111,19 +122,22 @@ const translation = {
     removeExtraSpaces: '替换掉连续的空格、换行符和制表符',
     removeUrlEmails: '删除所有 URL 和电子邮件地址',
     removeStopwords: '去除停用词，例如 “a”，“an”，“the” 等',
-    preview: '确认并预览',
+    preview: '预览',
+    previewChunk: '预览块',
     reset: '重置',
     indexMode: '索引方式',
     qualified: '高质量',
+    highQualityTip: '使用高质量模式进行嵌入后，无法切换回经济模式。',
     recommend: '推荐',
-    qualifiedTip: '调用系统默认的嵌入接口进行处理，以在用户查询时提供更高的准确度',
+    qualifiedTip: '调用嵌入模型处理文档以实现更精确的检索，可以帮助LLM生成高质量的答案。',
     warning: '请先完成模型供应商的 API KEY 设置。.',
     click: '前往设置',
     economical: '经济',
-    economicalTip: '使用离线的向量引擎、关键词索引等方式，降低了准确度但无需花费 Token',
+    economicalTip: '每个数据块使用10个关键词进行检索，不会消耗任何tokens，但会以降低检索准确性为代价。',
     QATitle: '采用 Q&A 分段模式',
     QATip: '开启后将会消耗额外的 token',
     QALanguage: '分段使用',
+    useQALanguage: '使用 Q&A 分段，语言',
     estimateCost: '执行嵌入预估消耗',
     estimateSegment: '预估分段数',
     segmentCount: '段',
@@ -153,6 +167,15 @@ const translation = {
     indexSettingTip: '要更改索引方法和 embedding 模型，请转到',
     retrievalSettingTip: '要更改检索方法，请转到',
     datasetSettingLink: '知识库设置。',
+    previewChunkTip: '点击左侧的“预览块”按钮来加载预览',
+    previewChunkCount: '{{count}} 预估块',
+    switch: '切换',
+    qaSwitchHighQualityTipTitle: 'Q&A 格式需要高质量的索引方法',
+    qaSwitchHighQualityTipContent: '目前，只有高质量的索引方法支持 Q&A 格式分块。您要切换到高质量模式吗？',
+    notAvailableForParentChild: '不支持父子索引',
+    notAvailableForQA: '不支持 Q&A 索引',
+    parentChildDelimiterTip: '文本分隔符是用于分隔文本的字符。建议用 \n\n 将原始文档划分为较大的父级片段。您也可以自定义特殊分隔符。',
+    parentChildChunkDelimiterTip: '文本分隔符是用于分隔文本的字符。建议使用 \n 将父级片段拆分为较小的子级片段。您也可以自定义特殊分隔符。',
   },
   stepThree: {
     creationTitle: '🎉 知识库已创建',