feat: upgrade knowledge metadata (#16063)

Support filter knowledge by metadata.

Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: NFish <douxc512@gmail.com>

web/app/components/app/configuration/dataset-config/index.tsx

@@ -1,9 +1,11 @@
 'use client'
 import type { FC } from 'react'
-import React, { useMemo } from 'react'
+import React, { useCallback, useMemo } from 'react'
 import { useTranslation } from 'react-i18next'
+import { intersectionBy } from 'lodash-es'
 import { useContext } from 'use-context-selector'
 import produce from 'immer'
+import { v4 as uuid4 } from 'uuid'
 import { useFormattingChangedDispatcher } from '../debug/hooks'
 import FeaturePanel from '../base/feature-panel'
 import OperationBtn from '../base/operation-btn'
@@ -21,6 +23,19 @@ import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/com
 import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
 import { useSelector as useAppContextSelector } from '@/context/app-context'
 import { hasEditPermissionForDataset } from '@/utils/permission'
+import MetadataFilter from '@/app/components/workflow/nodes/knowledge-retrieval/components/metadata/metadata-filter'
+import type {
+  HandleAddCondition,
+  HandleRemoveCondition,
+  HandleToggleConditionLogicalOperator,
+  HandleUpdateCondition,
+  MetadataFilteringModeEnum,
+} from '@/app/components/workflow/nodes/knowledge-retrieval/types'
+import {
+  ComparisonOperator,
+  LogicalOperator,
+  MetadataFilteringVariableType,
+} from '@/app/components/workflow/nodes/knowledge-retrieval/types'
 
 const DatasetConfig: FC = () => {
   const { t } = useTranslation()
@@ -34,6 +49,7 @@ const DatasetConfig: FC = () => {
     showSelectDataSet,
     isAgent,
     datasetConfigs,
+    datasetConfigsRef,
     setDatasetConfigs,
     setRerankSettingModalOpen,
   } = useContext(ConfigContext)
@@ -115,6 +131,98 @@ const DatasetConfig: FC = () => {
     })
   }, [dataSet, userProfile?.id])
 
+  const metadataList = useMemo(() => {
+    return intersectionBy(...formattedDataset.filter((dataset) => {
+      return !!dataset.doc_metadata
+    }).map((dataset) => {
+      return dataset.doc_metadata!
+    }), 'name')
+  }, [formattedDataset])
+
+  const handleMetadataFilterModeChange = useCallback((newMode: MetadataFilteringModeEnum) => {
+    setDatasetConfigs(produce(datasetConfigsRef.current!, (draft) => {
+      draft.metadata_filtering_mode = newMode
+    }))
+  }, [setDatasetConfigs, datasetConfigsRef])
+
+  const handleAddCondition = useCallback<HandleAddCondition>(({ name, type }) => {
+    let operator: ComparisonOperator = ComparisonOperator.is
+
+    if (type === MetadataFilteringVariableType.number)
+      operator = ComparisonOperator.equal
+
+    const newCondition = {
+      id: uuid4(),
+      name,
+      comparison_operator: operator,
+    }
+
+    const newInputs = produce(datasetConfigsRef.current!, (draft) => {
+      if (draft.metadata_filtering_conditions) {
+        draft.metadata_filtering_conditions.conditions.push(newCondition)
+      }
+      else {
+        draft.metadata_filtering_conditions = {
+          logical_operator: LogicalOperator.and,
+          conditions: [newCondition],
+        }
+      }
+    })
+    setDatasetConfigs(newInputs)
+  }, [setDatasetConfigs, datasetConfigsRef])
+
+  const handleRemoveCondition = useCallback<HandleRemoveCondition>((id) => {
+    const conditions = datasetConfigsRef.current!.metadata_filtering_conditions?.conditions || []
+    const index = conditions.findIndex(c => c.id === id)
+    const newInputs = produce(datasetConfigsRef.current!, (draft) => {
+      if (index > -1)
+        draft.metadata_filtering_conditions?.conditions.splice(index, 1)
+    })
+    setDatasetConfigs(newInputs)
+  }, [setDatasetConfigs, datasetConfigsRef])
+
+  const handleUpdateCondition = useCallback<HandleUpdateCondition>((id, newCondition) => {
+    console.log(newCondition, 'newCondition')
+    const conditions = datasetConfigsRef.current!.metadata_filtering_conditions?.conditions || []
+    const index = conditions.findIndex(c => c.id === id)
+    const newInputs = produce(datasetConfigsRef.current!, (draft) => {
+      if (index > -1)
+        draft.metadata_filtering_conditions!.conditions[index] = newCondition
+    })
+    setDatasetConfigs(newInputs)
+  }, [setDatasetConfigs, datasetConfigsRef])
+
+  const handleToggleConditionLogicalOperator = useCallback<HandleToggleConditionLogicalOperator>(() => {
+    const oldLogicalOperator = datasetConfigsRef.current!.metadata_filtering_conditions?.logical_operator
+    const newLogicalOperator = oldLogicalOperator === LogicalOperator.and ? LogicalOperator.or : LogicalOperator.and
+    const newInputs = produce(datasetConfigsRef.current!, (draft) => {
+      draft.metadata_filtering_conditions!.logical_operator = newLogicalOperator
+    })
+    setDatasetConfigs(newInputs)
+  }, [setDatasetConfigs, datasetConfigsRef])
+
+  const handleMetadataModelChange = useCallback((model: { provider: string; modelId: string; mode?: string }) => {
+    const newInputs = produce(datasetConfigsRef.current!, (draft) => {
+      draft.metadata_model_config = {
+        provider: model.provider,
+        name: model.modelId,
+        mode: model.mode || 'chat',
+        completion_params: draft.metadata_model_config?.completion_params || { temperature: 0.7 },
+      }
+    })
+    setDatasetConfigs(newInputs)
+  }, [setDatasetConfigs, datasetConfigsRef])
+
+  const handleMetadataCompletionParamsChange = useCallback((newParams: Record<string, any>) => {
+    const newInputs = produce(datasetConfigsRef.current!, (draft) => {
+      draft.metadata_model_config = {
+        ...draft.metadata_model_config!,
+        completion_params: newParams,
+      }
+    })
+    setDatasetConfigs(newInputs)
+  }, [setDatasetConfigs, datasetConfigsRef])
+
   return (
     <FeaturePanel
       className='mt-2'
@@ -148,6 +256,26 @@ const DatasetConfig: FC = () => {
           </div>
         )}
 
+      <div className='py-2 border-t border-t-divider-subtle'>
+        <MetadataFilter
+          metadataList={metadataList}
+          selectedDatasetsLoaded
+          metadataFilterMode={datasetConfigs.metadata_filtering_mode}
+          metadataFilteringConditions={datasetConfigs.metadata_filtering_conditions}
+          handleAddCondition={handleAddCondition}
+          handleMetadataFilterModeChange={handleMetadataFilterModeChange}
+          handleRemoveCondition={handleRemoveCondition}
+          handleToggleConditionLogicalOperator={handleToggleConditionLogicalOperator}
+          handleUpdateCondition={handleUpdateCondition}
+          metadataModelConfig={datasetConfigs.metadata_model_config}
+          handleMetadataModelChange={handleMetadataModelChange}
+          handleMetadataCompletionParamsChange={handleMetadataCompletionParamsChange}
+          isCommonVariable
+          availableCommonStringVars={promptVariablesToSelect.filter(item => item.type === MetadataFilteringVariableType.string)}
+          availableCommonNumberVars={promptVariablesToSelect.filter(item => item.type === MetadataFilteringVariableType.number)}
+        />
+      </div>
+
       {mode === AppType.completion && dataSet.length > 0 && (
         <ContextVar
           value={selectedContextVar?.key}

web/app/components/app/configuration/index.tsx

@@ -191,7 +191,6 @@ const Configuration: FC = () => {
     dataSets: [],
     agentConfig: DEFAULT_AGENT_SETTING,
   })
-
   const isAgent = mode === 'agent-chat'
 
   const isOpenAI = modelConfig.provider === 'langgenius/openai/openai'
@@ -200,7 +199,7 @@ const Configuration: FC = () => {
   useEffect(() => {
 
   }, [])
-  const [datasetConfigs, setDatasetConfigs] = useState<DatasetConfigs>({
+  const [datasetConfigs, doSetDatasetConfigs] = useState<DatasetConfigs>({
     retrieval_model: RETRIEVE_TYPE.multiWay,
     reranking_model: {
       reranking_provider_name: '',
@@ -213,6 +212,11 @@ const Configuration: FC = () => {
       datasets: [],
     },
   })
+  const datasetConfigsRef = useRef(datasetConfigs)
+  const setDatasetConfigs = useCallback((newDatasetConfigs: DatasetConfigs) => {
+    doSetDatasetConfigs(newDatasetConfigs)
+    datasetConfigsRef.current = newDatasetConfigs
+  }, [])
 
   const setModelConfig = (newModelConfig: ModelConfig) => {
     doSetModelConfig(newModelConfig)
@@ -292,6 +296,7 @@ const Configuration: FC = () => {
     })
 
     setDatasetConfigs({
+      ...datasetConfigsRef.current,
       ...retrievalConfig,
       reranking_model: {
         reranking_provider_name: retrievalConfig?.reranking_model?.provider || '',
@@ -884,6 +889,7 @@ const Configuration: FC = () => {
       dataSets,
       setDataSets,
       datasetConfigs,
+      datasetConfigsRef,
       setDatasetConfigs,
       hasSetContextVar,
       isShowVisionConfig,