feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

web/app/components/workflow-app/hooks/index.ts

@@ -4,7 +4,10 @@ export * from './use-nodes-sync-draft'
 export * from './use-workflow-run'
 export * from './use-workflow-start-run'
 export * from './use-is-chat-mode'
+export * from './use-available-nodes-meta-data'
 export * from './use-workflow-refresh-draft'
+export * from './use-get-run-and-trace-url'
+export * from './use-DSL'
 export * from '../../workflow/hooks/use-fetch-workflow-inspect-vars'
 export * from './use-inspect-vars-crud'
 export * from './use-configs-map'

web/app/components/workflow-app/hooks/use-DSL.ts

@@ -0,0 +1,82 @@
+import {
+  useCallback,
+  useState,
+} from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+  DSL_EXPORT_CHECK,
+} from '@/app/components/workflow/constants'
+import { useNodesSyncDraft } from './use-nodes-sync-draft'
+import { useEventEmitterContextContext } from '@/context/event-emitter'
+import { fetchWorkflowDraft } from '@/service/workflow'
+import { exportAppConfig } from '@/service/apps'
+import { useToastContext } from '@/app/components/base/toast'
+import { useStore as useAppStore } from '@/app/components/app/store'
+
+export const useDSL = () => {
+  const { t } = useTranslation()
+  const { notify } = useToastContext()
+  const { eventEmitter } = useEventEmitterContextContext()
+  const [exporting, setExporting] = useState(false)
+  const { doSyncWorkflowDraft } = useNodesSyncDraft()
+
+  const appDetail = useAppStore(s => s.appDetail)
+
+  const handleExportDSL = useCallback(async (include = false, workflowId?: string) => {
+    if (!appDetail)
+      return
+
+    if (exporting)
+      return
+
+    try {
+      setExporting(true)
+      await doSyncWorkflowDraft()
+      const { data } = await exportAppConfig({
+        appID: appDetail.id,
+        include,
+        workflowID: workflowId,
+      })
+      const a = document.createElement('a')
+      const file = new Blob([data], { type: 'application/yaml' })
+      const url = URL.createObjectURL(file)
+      a.href = url
+      a.download = `${appDetail.name}.yml`
+      a.click()
+      URL.revokeObjectURL(url)
+    }
+    catch {
+      notify({ type: 'error', message: t('app.exportFailed') })
+    }
+    finally {
+      setExporting(false)
+    }
+  }, [appDetail, notify, t, doSyncWorkflowDraft, exporting])
+
+  const exportCheck = useCallback(async () => {
+    if (!appDetail)
+      return
+    try {
+      const workflowDraft = await fetchWorkflowDraft(`/apps/${appDetail?.id}/workflows/draft`)
+      const list = (workflowDraft.environment_variables || []).filter(env => env.value_type === 'secret')
+      if (list.length === 0) {
+        handleExportDSL()
+        return
+      }
+      eventEmitter?.emit({
+        type: DSL_EXPORT_CHECK,
+        payload: {
+          data: list,
+        },
+      } as any)
+    }
+    catch {
+      notify({ type: 'error', message: t('app.exportFailed') })
+    }
+  }, [appDetail, eventEmitter, handleExportDSL, notify, t])
+
+  return {
+    exportCheck,
+    handleExportDSL,
+  }
+}

web/app/components/workflow-app/hooks/use-available-nodes-meta-data.ts

@@ -0,0 +1,68 @@
+import { useMemo } from 'react'
+import { useTranslation } from 'react-i18next'
+import { useGetLanguage } from '@/context/i18n'
+import StartDefault from '@/app/components/workflow/nodes/start/default'
+import EndDefault from '@/app/components/workflow/nodes/end/default'
+import AnswerDefault from '@/app/components/workflow/nodes/answer/default'
+import { WORKFLOW_COMMON_NODES } from '@/app/components/workflow/constants/node'
+import type { AvailableNodesMetaData } from '@/app/components/workflow/hooks-store/store'
+import { useIsChatMode } from './use-is-chat-mode'
+import { BlockEnum } from '@/app/components/workflow/types'
+
+export const useAvailableNodesMetaData = () => {
+  const { t } = useTranslation()
+  const isChatMode = useIsChatMode()
+  const language = useGetLanguage()
+
+  const mergedNodesMetaData = useMemo(() => [
+    ...WORKFLOW_COMMON_NODES,
+    StartDefault,
+    ...(
+      isChatMode
+        ? [AnswerDefault]
+        : [EndDefault]
+    ),
+  ], [isChatMode])
+
+  const prefixLink = useMemo(() => {
+    if (language === 'zh_Hans')
+      return 'https://docs.dify.ai/zh-hans/guides/workflow/node/'
+
+    return 'https://docs.dify.ai/guides/workflow/node/'
+  }, [language])
+
+  const availableNodesMetaData = useMemo(() => mergedNodesMetaData.map((node) => {
+    const { metaData } = node
+    const title = t(`workflow.blocks.${metaData.type}`)
+    const description = t(`workflow.blocksAbout.${metaData.type}`)
+    return {
+      ...node,
+      metaData: {
+        ...metaData,
+        title,
+        description,
+        helpLinkUri: `${prefixLink}${metaData.helpLinkUri}`,
+      },
+      defaultValue: {
+        ...node.defaultValue,
+        type: metaData.type,
+        title,
+      },
+    }
+  }), [mergedNodesMetaData, t, prefixLink])
+
+  const availableNodesMetaDataMap = useMemo(() => availableNodesMetaData.reduce((acc, node) => {
+    acc![node.metaData.type] = node
+    return acc
+  }, {} as AvailableNodesMetaData['nodesMap']), [availableNodesMetaData])
+
+  return useMemo(() => {
+    return {
+      nodes: availableNodesMetaData,
+      nodesMap: {
+        ...availableNodesMetaDataMap,
+        [BlockEnum.VariableAssigner]: availableNodesMetaDataMap?.[BlockEnum.VariableAggregator],
+      },
+    }
+  }, [availableNodesMetaData, availableNodesMetaDataMap])
+}

web/app/components/workflow-app/hooks/use-configs-map.ts

@@ -1,13 +1,16 @@
 import { useMemo } from 'react'
 import { useStore } from '@/app/components/workflow/store'
+import { FlowType } from '@/types/common'
+import { useFeatures } from '@/app/components/base/features/hooks'
 
 export const useConfigsMap = () => {
   const appId = useStore(s => s.appId)
+  const fileSettings = useFeatures(s => s.features.file)
   return useMemo(() => {
     return {
-      flowId: appId,
-      conversationVarsUrl: `apps/${appId}/workflows/draft/conversation-variables`,
-      systemVarsUrl: `apps/${appId}/workflows/draft/system-variables`,
+      flowId: appId!,
+      flowType: FlowType.appFlow,
+      fileSettings,
     }
   }, [appId])
 }

web/app/components/workflow-app/hooks/use-get-run-and-trace-url.ts

@@ -0,0 +1,18 @@
+import { useCallback } from 'react'
+import { useWorkflowStore } from '@/app/components/workflow/store'
+
+export const useGetRunAndTraceUrl = () => {
+  const workflowStore = useWorkflowStore()
+  const getWorkflowRunAndTraceUrl = useCallback((runId: string) => {
+    const { appId } = workflowStore.getState()
+
+    return {
+      runUrl: `/apps/${appId}/workflow-runs/${runId}`,
+      traceUrl: `/apps/${appId}/workflow-runs/${runId}/node-executions`,
+    }
+  }, [workflowStore])
+
+  return {
+    getWorkflowRunAndTraceUrl,
+  }
+}

web/app/components/workflow-app/hooks/use-inspect-vars-crud.ts

@@ -1,12 +1,9 @@
-import { useStore } from '@/app/components/workflow/store'
 import { useInspectVarsCrudCommon } from '../../workflow/hooks/use-inspect-vars-crud-common'
 import { useConfigsMap } from './use-configs-map'
 
 export const useInspectVarsCrud = () => {
-  const appId = useStore(s => s.appId)
   const configsMap = useConfigsMap()
   const apis = useInspectVarsCrudCommon({
-    flowId: appId,
     ...configsMap,
   })
 

web/app/components/workflow-app/hooks/use-nodes-sync-draft.ts

@@ -28,6 +28,7 @@ export const useNodesSyncDraft = () => {
       edges,
       transform,
     } = store.getState()
+    const nodes = getNodes()
     const [x, y, zoom] = transform
     const {
       appId,
@@ -36,8 +37,7 @@ export const useNodesSyncDraft = () => {
       syncWorkflowDraftHash,
     } = workflowStore.getState()
 
-    if (appId) {
-      const nodes = getNodes()
+    if (appId && !!nodes.length) {
       const hasStartNode = nodes.find(node => node.data.type === BlockEnum.Start)
 
       if (!hasStartNode)

web/app/components/workflow-app/hooks/use-workflow-init.ts

@@ -17,6 +17,8 @@ import {
 } from '@/service/workflow'
 import type { FetchWorkflowDraftResponse } from '@/types/workflow'
 import { useWorkflowConfig } from '@/service/use-workflow'
+import type { FileUploadConfigResponse } from '@/models/common'
+
 export const useWorkflowInit = () => {
   const workflowStore = useWorkflowStore()
   const {
@@ -28,7 +30,7 @@ export const useWorkflowInit = () => {
   const [data, setData] = useState<FetchWorkflowDraftResponse>()
   const [isLoading, setIsLoading] = useState(true)
   useEffect(() => {
-    workflowStore.setState({ appId: appDetail.id })
+    workflowStore.setState({ appId: appDetail.id, appName: appDetail.name })
   }, [appDetail.id, workflowStore])
 
   const handleUpdateWorkflowConfig = useCallback((config: Record<string, any>) => {
@@ -36,7 +38,16 @@ export const useWorkflowInit = () => {
 
     setWorkflowConfig(config)
   }, [workflowStore])
-  useWorkflowConfig(appDetail.id, handleUpdateWorkflowConfig)
+  useWorkflowConfig(`/apps/${appDetail.id}/workflows/draft/config`, handleUpdateWorkflowConfig)
+
+  const handleUpdateWorkflowFileUploadConfig = useCallback((config: FileUploadConfigResponse) => {
+    const { setFileUploadConfig } = workflowStore.getState()
+    setFileUploadConfig(config)
+  }, [workflowStore])
+  const {
+    data: fileUploadConfigResponse,
+    isLoading: isFileUploadConfigLoading,
+  } = useWorkflowConfig('/files/upload', handleUpdateWorkflowFileUploadConfig)
 
   const handleGetInitialWorkflowData = useCallback(async () => {
     try {
@@ -116,6 +127,7 @@ export const useWorkflowInit = () => {
 
   return {
     data,
-    isLoading,
+    isLoading: isLoading || isFileUploadConfigLoading,
+    fileUploadConfigResponse,
   }
 }

web/app/components/workflow-app/hooks/use-workflow-run.ts

@@ -31,9 +31,10 @@ export const useWorkflowRun = () => {
   const { doSyncWorkflowDraft } = useNodesSyncDraft()
   const { handleUpdateWorkflowCanvas } = useWorkflowUpdate()
   const pathname = usePathname()
-  const appId = useAppStore.getState().appDetail?.id
-  const invalidAllLastRun = useInvalidAllLastRun(appId as string)
   const configsMap = useConfigsMap()
+  const { flowId, flowType } = configsMap
+  const invalidAllLastRun = useInvalidAllLastRun(flowType, flowId)
+
   const { fetchInspectVars } = useSetWorkflowVarsWithValue({
     ...configsMap,
   })
@@ -201,7 +202,7 @@ export const useWorkflowRun = () => {
           if (onWorkflowFinished)
             onWorkflowFinished(params)
           if (isInWorkflowDebug) {
-            fetchInspectVars()
+            fetchInspectVars({})
             invalidAllLastRun()
           }
         },

web/app/components/workflow-app/hooks/use-workflow-template.ts

@@ -1,17 +1,25 @@
+import { useTranslation } from 'react-i18next'
 import { generateNewNode } from '@/app/components/workflow/utils'
 import {
   NODE_WIDTH_X_OFFSET,
   START_INITIAL_POSITION,
 } from '@/app/components/workflow/constants'
-import { useNodesInitialData } from '@/app/components/workflow/hooks'
 import { useIsChatMode } from './use-is-chat-mode'
+import type { StartNodeType } from '@/app/components/workflow/nodes/start/types'
+import startDefault from '@/app/components/workflow/nodes/start/default'
+import llmDefault from '@/app/components/workflow/nodes/llm/default'
+import answerDefault from '@/app/components/workflow/nodes/answer/default'
 
 export const useWorkflowTemplate = () => {
   const isChatMode = useIsChatMode()
-  const nodesInitialData = useNodesInitialData()
+  const { t } = useTranslation()
 
   const { newNode: startNode } = generateNewNode({
-    data: nodesInitialData.start,
+    data: {
+      ...startDefault.defaultValue as StartNodeType,
+      type: startDefault.metaData.type,
+      title: t(`workflow.blocks.${startDefault.metaData.type}`),
+    },
     position: START_INITIAL_POSITION,
   })
 
@@ -19,12 +27,14 @@ export const useWorkflowTemplate = () => {
     const { newNode: llmNode } = generateNewNode({
       id: 'llm',
       data: {
-        ...nodesInitialData.llm,
+        ...llmDefault.defaultValue,
         memory: {
           window: { enabled: false, size: 10 },
           query_prompt_template: '{{#sys.query#}}\n\n{{#sys.files#}}',
         },
         selected: true,
+        type: llmDefault.metaData.type,
+        title: t(`workflow.blocks.${llmDefault.metaData.type}`),
       },
       position: {
         x: START_INITIAL_POSITION.x + NODE_WIDTH_X_OFFSET,
@@ -35,8 +45,10 @@ export const useWorkflowTemplate = () => {
     const { newNode: answerNode } = generateNewNode({
       id: 'answer',
       data: {
-        ...nodesInitialData.answer,
+        ...answerDefault.defaultValue,
         answer: `{{#${llmNode.id}.text#}}`,
+        type: answerDefault.metaData.type,
+        title: t(`workflow.blocks.${answerDefault.metaData.type}`),
       },
       position: {
         x: START_INITIAL_POSITION.x + NODE_WIDTH_X_OFFSET * 2,