external knowledge api (#8913)

Co-authored-by: Yi <yxiaoisme@gmail.com>
2024-09-30 15:38:43 +08:00 · 2024-09-30 15:38:43 +08:00 · 9d221a5e19
commit 9d221a5e19
parent 77aef9ff1d
90 changed files with 4623 additions and 1171 deletions
--- a/web/app/components/app/configuration/dataset-config/card-item/item.tsx
+++ b/web/app/components/app/configuration/dataset-config/card-item/item.tsx
@ -5,6 +5,7 @@ import {
  RiDeleteBinLine,
  RiEditLine,
 } from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
 import SettingsModal from '../settings-modal'
 import type { DataSet } from '@/models/datasets'
 import { DataSourceType } from '@/models/datasets'
@ -33,6 +34,7 @@ const Item: FC<ItemProps> = ({
  const isMobile = media === MediaType.mobile
  const [showSettingsModal, setShowSettingsModal] = useState(false)
  const { formatIndexingTechniqueAndMethod } = useKnowledge()
+  const { t } = useTranslation()

  const handleSave = (newDataset: DataSet) => {
    onSave(newDataset)
@ -65,9 +67,11 @@ const Item: FC<ItemProps> = ({
      <div className='grow'>
        <div className='flex items-center h-[18px]'>
          <div className='grow text-[13px] font-medium text-gray-800 truncate' title={config.name}>{config.name}</div>
-          <Badge
-            text={formatIndexingTechniqueAndMethod(config.indexing_technique, config.retrieval_model_dict?.search_method)}
-          />
+          {config.provider === 'external'
+            ? <Badge text={t('dataset.externalTag')}></Badge>
+            : <Badge
+              text={formatIndexingTechniqueAndMethod(config.indexing_technique, config.retrieval_model_dict?.search_method)}
+            />}
        </div>
      </div>
      <div className='hidden rounded-lg group-hover:flex items-center justify-end absolute right-0 top-0 bottom-0 pr-2 w-[124px] bg-gradient-to-r from-white/50 to-white to-50%'>
--- a/web/app/components/app/configuration/dataset-config/params-config/config-content.tsx
+++ b/web/app/components/app/configuration/dataset-config/params-config/config-content.tsx
@ -174,6 +174,20 @@ const ConfigContent: FC<Props> = ({
              </div>
            )
          }
+          {
+            selectedDatasetsMode.mixtureInternalAndExternal && (
+              <div className='mt-4 system-xs-medium text-text-warning'>
+                {t('dataset.mixtureInternalAndExternalTip')}
+              </div>
+            )
+          }
+          {
+            selectedDatasetsMode.allExternal && (
+              <div className='mt-4 system-xs-medium text-text-warning'>
+                {t('dataset.allExternalTip')}
+              </div>
+            )
+          }
          {
            selectedDatasetsMode.mixtureHighQualityAndEconomic
            && (
@ -229,15 +243,15 @@ const ConfigContent: FC<Props> = ({
                      />
                    )
                  }
-                  <div className='ml-2 leading-[32px] text-[13px] font-medium text-gray-900'>{t('common.modelProvider.rerankModel.key')}</div>
+                  <div className='leading-[32px] text-text-secondary system-sm-semibold'>{t('common.modelProvider.rerankModel.key')}</div>
                  <Tooltip
                    popupContent={
                      <div className="w-[200px]">
                        {t('common.modelProvider.rerankModel.tip')}
                      </div>
                    }
-                    popupClassName='ml-0.5'
-                    triggerClassName='ml-0.5 w-3.5 h-3.5'
+                    popupClassName='ml-1'
+                    triggerClassName='ml-1 w-4 h-4'
                  />
                </div>
                <div>
--- a/web/app/components/app/configuration/dataset-config/params-config/index.tsx
+++ b/web/app/components/app/configuration/dataset-config/params-config/index.tsx
@ -39,13 +39,26 @@ const ParamsConfig = ({
  useEffect(() => {
    const {
      allEconomic,
+      allHighQuality,
+      allHighQualityFullTextSearch,
+      allHighQualityVectorSearch,
+      allExternal,
+      mixtureHighQualityAndEconomic,
+      inconsistentEmbeddingModel,
+      mixtureInternalAndExternal,
    } = getSelectedDatasetsMode(selectedDatasets)
    const { datasets, retrieval_model, score_threshold_enabled, ...restConfigs } = datasetConfigs
    let rerankEnable = restConfigs.reranking_enable

-    if (allEconomic && !restConfigs.reranking_model?.reranking_provider_name && rerankEnable === undefined)
+    if ((allEconomic && !restConfigs.reranking_model?.reranking_provider_name && rerankEnable === undefined) || allExternal)
      rerankEnable = false

+    if (allEconomic || allHighQuality || allHighQualityFullTextSearch || allHighQualityVectorSearch || (allExternal && selectedDatasets.length === 1))
+      setRerankSettingModalOpen(false)
+
+    if (mixtureHighQualityAndEconomic || inconsistentEmbeddingModel || mixtureInternalAndExternal || (allExternal && selectedDatasets.length > 1))
+      setRerankSettingModalOpen(true)
+
    setTempDataSetConfigs({
      ...getMultipleRetrievalConfig({
        top_k: restConfigs.top_k,
--- a/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx
+++ b/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx
@ -47,7 +47,7 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
        const { data, has_more } = await fetchDatasets({ url: '/datasets', params: { page } })
        setPage(getPage() + 1)
        setIsNoMore(!has_more)
-        const newList = [...(datasets || []), ...data.filter(item => item.indexing_technique)]
+        const newList = [...(datasets || []), ...data.filter(item => item.indexing_technique || item.provider === 'external')]
        setDataSets(newList)
        setLoaded(true)
        if (!selected.find(item => !item.name))
@ -145,6 +145,11 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
                    />
                  )
                }
+                {
+                  item.provider === 'external' && (
+                    <Badge text={t('dataset.externalTag')} />
+                  )
+                }
              </div>
            ))}
          </div>
--- a/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx
+++ b/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx
@ -5,8 +5,10 @@ import { useTranslation } from 'react-i18next'
 import { isEqual } from 'lodash-es'
 import { RiCloseLine } from '@remixicon/react'
 import { BookOpenIcon } from '@heroicons/react/24/outline'
+import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development'
 import cn from '@/utils/classnames'
 import IndexMethodRadio from '@/app/components/datasets/settings/index-method-radio'
+import Divider from '@/app/components/base/divider'
 import Button from '@/app/components/base/button'
 import type { DataSet } from '@/models/datasets'
 import { useToastContext } from '@/app/components/base/toast'
@ -14,6 +16,7 @@ import { updateDatasetSetting } from '@/service/datasets'
 import { useAppContext } from '@/context/app-context'
 import { useModalContext } from '@/context/modal-context'
 import type { RetrievalConfig } from '@/types/app'
+import RetrievalSettings from '@/app/components/datasets/external-knowledge-base/create/RetrievalSettings'
 import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
 import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config'
 import { ensureRerankModelSelected, isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model'
@ -56,6 +59,9 @@ const SettingsModal: FC<SettingsModalProps> = ({
  const { t } = useTranslation()
  const { notify } = useToastContext()
  const ref = useRef(null)
+  const [topK, setTopK] = useState(currentDataset?.external_retrieval_model.top_k ?? 2)
+  const [scoreThreshold, setScoreThreshold] = useState(currentDataset?.external_retrieval_model.score_threshold ?? 0.5)
+  const [scoreThresholdEnabled, setScoreThresholdEnabled] = useState(currentDataset?.external_retrieval_model.score_threshold_enabled ?? false)

  const { setShowAccountSettingModal } = useModalContext()
  const [loading, setLoading] = useState(false)
@ -73,6 +79,15 @@ const SettingsModal: FC<SettingsModalProps> = ({
  const [isHideChangedTip, setIsHideChangedTip] = useState(false)
  const isRetrievalChanged = !isEqual(retrievalConfig, localeCurrentDataset?.retrieval_model_dict) || indexMethod !== localeCurrentDataset?.indexing_technique

+  const handleSettingsChange = (data: { top_k?: number; score_threshold?: number; score_threshold_enabled?: boolean }) => {
+    if (data.top_k !== undefined)
+      setTopK(data.top_k)
+    if (data.score_threshold !== undefined)
+      setScoreThreshold(data.score_threshold)
+    if (data.score_threshold_enabled !== undefined)
+      setScoreThresholdEnabled(data.score_threshold_enabled)
+  }
+
  const handleSave = async () => {
    if (loading)
      return
@ -107,10 +122,17 @@ const SettingsModal: FC<SettingsModalProps> = ({
          description,
          permission,
          indexing_technique: indexMethod,
+          external_retrieval_model: {
+            top_k: topK,
+            score_threshold: scoreThreshold,
+            score_threshold_enabled: scoreThresholdEnabled,
+          },
          retrieval_model: {
            ...postRetrievalConfig,
            score_threshold: postRetrievalConfig.score_threshold_enabled ? postRetrievalConfig.score_threshold : 0,
          },
+          external_knowledge_id: currentDataset!.external_knowledge_info.external_knowledge_id,
+          external_knowledge_api_id: currentDataset!.external_knowledge_info.external_knowledge_api_id,
          embedding_model: localeCurrentDataset.embedding_model,
          embedding_model_provider: localeCurrentDataset.embedding_model_provider,
        },
@ -178,7 +200,7 @@ const SettingsModal: FC<SettingsModalProps> = ({
      }}>
        <div className={cn(rowClass, 'items-center')}>
          <div className={labelClass}>
-            {t('datasetSettings.form.name')}
+            <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.name')}</div>
          </div>
          <input
            value={localeCurrentDataset.name}
@ -189,7 +211,7 @@ const SettingsModal: FC<SettingsModalProps> = ({
        </div>
        <div className={cn(rowClass)}>
          <div className={labelClass}>
-            {t('datasetSettings.form.desc')}
+            <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.desc')}</div>
          </div>
          <div className='w-full'>
            <textarea
@ -206,7 +228,7 @@ const SettingsModal: FC<SettingsModalProps> = ({
        </div>
        <div className={rowClass}>
          <div className={labelClass}>
-            <div>{t('datasetSettings.form.permissions')}</div>
+            <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.permissions')}</div>
          </div>
          <div className='w-full'>
            <PermissionSelector
@ -219,24 +241,25 @@ const SettingsModal: FC<SettingsModalProps> = ({
            />
          </div>
        </div>
-        <div className="w-full h-0 border-b-[0.5px] border-b-gray-200 my-2"></div>
-        <div className={cn(rowClass)}>
-          <div className={labelClass}>
-            {t('datasetSettings.form.indexMethod')}
+        {currentDataset && currentDataset.indexing_technique && (
+          <div className={cn(rowClass)}>
+            <div className={labelClass}>
+              <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.indexMethod')}</div>
+            </div>
+            <div className='grow'>
+              <IndexMethodRadio
+                disable={!localeCurrentDataset?.embedding_available}
+                value={indexMethod}
+                onChange={v => setIndexMethod(v!)}
+                itemClassName='sm:!w-[280px]'
+              />
+            </div>
          </div>
-          <div className='grow'>
-            <IndexMethodRadio
-              disable={!localeCurrentDataset?.embedding_available}
-              value={indexMethod}
-              onChange={v => setIndexMethod(v!)}
-              itemClassName='sm:!w-[280px]'
-            />
-          </div>
-        </div>
+        )}
        {indexMethod === 'high_quality' && (
          <div className={cn(rowClass)}>
            <div className={labelClass}>
-              {t('datasetSettings.form.embeddingModel')}
+              <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.embeddingModel')}</div>
            </div>
            <div className='w-full'>
              <div className='w-full h-9 rounded-lg bg-gray-100 opacity-60'>
@ -258,32 +281,75 @@ const SettingsModal: FC<SettingsModalProps> = ({
        )}

        {/* Retrieval Method Config */}
-        <div className={rowClass}>
-          <div className={cn(labelClass, 'w-auto min-w-[168px]')}>
-            <div>
-              <div>{t('datasetSettings.form.retrievalSetting.title')}</div>
-              <div className='leading-[18px] text-xs font-normal text-gray-500'>
-                <a target='_blank' rel='noopener noreferrer' href='https://docs.dify.ai/guides/knowledge-base/create-knowledge-and-upload-documents#id-4-retrieval-settings' className='text-[#155eef]'>{t('datasetSettings.form.retrievalSetting.learnMore')}</a>
-                {t('datasetSettings.form.retrievalSetting.description')}
+        {currentDataset?.provider === 'external'
+          ? <>
+            <div className={rowClass}><Divider/></div>
+            <div className={rowClass}>
+              <div className={labelClass}>
+                <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.retrievalSetting.title')}</div>
+              </div>
+              <RetrievalSettings
+                topK={topK}
+                scoreThreshold={scoreThreshold}
+                scoreThresholdEnabled={scoreThresholdEnabled}
+                onChange={handleSettingsChange}
+                isInRetrievalSetting={true}
+              />
+            </div>
+            <div className={rowClass}><Divider/></div>
+            <div className={rowClass}>
+              <div className={labelClass}>
+                <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.externalKnowledgeAPI')}</div>
+              </div>
+              <div className='w-full max-w-[480px]'>
+                <div className='flex h-full px-3 py-2 items-center gap-1 rounded-lg bg-components-input-bg-normal'>
+                  <ApiConnectionMod className='w-4 h-4 text-text-secondary' />
+                  <div className='overflow-hidden text-text-secondary text-ellipsis system-sm-medium'>
+                    {currentDataset?.external_knowledge_info.external_knowledge_api_name}
+                  </div>
+                  <div className='text-text-tertiary system-xs-regular'>·</div>
+                  <div className='text-text-tertiary system-xs-regular'>{currentDataset?.external_knowledge_info.external_knowledge_api_endpoint}</div>
+                </div>
              </div>
            </div>
-          </div>
-          <div>
-            {indexMethod === 'high_quality'
-              ? (
-                <RetrievalMethodConfig
-                  value={retrievalConfig}
-                  onChange={setRetrievalConfig}
-                />
-              )
-              : (
-                <EconomicalRetrievalMethodConfig
-                  value={retrievalConfig}
-                  onChange={setRetrievalConfig}
-                />
-              )}
-          </div>
-        </div>
+            <div className={rowClass}>
+              <div className={labelClass}>
+                <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.externalKnowledgeID')}</div>
+              </div>
+              <div className='w-full max-w-[480px]'>
+                <div className='flex h-full px-3 py-2 items-center gap-1 rounded-lg bg-components-input-bg-normal'>
+                  <div className='text-text-tertiary system-xs-regular'>{currentDataset?.external_knowledge_info.external_knowledge_id}</div>
+                </div>
+              </div>
+            </div>
+            <div className={rowClass}><Divider/></div>
+          </>
+          : <div className={rowClass}>
+            <div className={cn(labelClass, 'w-auto min-w-[168px]')}>
+              <div>
+                <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.retrievalSetting.title')}</div>
+                <div className='leading-[18px] text-xs font-normal text-gray-500'>
+                  <a target='_blank' rel='noopener noreferrer' href='https://docs.dify.ai/guides/knowledge-base/create-knowledge-and-upload-documents#id-4-retrieval-settings' className='text-[#155eef]'>{t('datasetSettings.form.retrievalSetting.learnMore')}</a>
+                  {t('datasetSettings.form.retrievalSetting.description')}
+                </div>
+              </div>
+            </div>
+            <div>
+              {indexMethod === 'high_quality'
+                ? (
+                  <RetrievalMethodConfig
+                    value={retrievalConfig}
+                    onChange={setRetrievalConfig}
+                  />
+                )
+                : (
+                  <EconomicalRetrievalMethodConfig
+                    value={retrievalConfig}
+                    onChange={setRetrievalConfig}
+                  />
+                )}
+            </div>
+          </div>}
      </div>
      {isRetrievalChanged && !isHideChangedTip && (
        <div className='absolute z-10 left-[30px] right-[30px] bottom-[76px] flex h-10 items-center px-3 rounded-lg border border-[#FEF0C7] bg-[#FFFAEB] shadow-lg justify-between'>