jyapayne/dify
Watch
1
0
Fork
You've already forked dify
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: support firecrawl frontend code (#5226)

Browse source
This commit is contained in:
Joel 2024-06-14 22:02:41 +08:00 • committed by GitHub
commit 28554350de
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
51 changed files with 1979 additions and 145 deletions
Download patch file Download diff file Expand all files Collapse all files

3
web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx
View file

@ -109,6 +109,9 @@ const AppDetailLayout: FC<IAppDetailLayoutProps> = (props) => {
setAppDetail(res)
setNavigation(getNavigations(appId, isCurrentWorkspaceManager, isCurrentWorkspaceEditor, res.mode))
}
}).catch((e: any) => {
if (e.status === 404)
router.replace('/apps')
})
}, [appId, isCurrentWorkspaceManager, isCurrentWorkspaceEditor])

8
web/app/(commonLayout)/apps/Apps.tsx
View file

@ -73,10 +73,10 @@ const Apps = () => {
const anchorRef = useRef<HTMLDivElement>(null)
const options = [
{ value: 'all', text: t('app.types.all'), icon: <DotsGrid className='w-[14px] h-[14px] mr-1'/> },
{ value: 'chat', text: t('app.types.chatbot'), icon: <ChatBot className='w-[14px] h-[14px] mr-1'/> },
{ value: 'agent-chat', text: t('app.types.agent'), icon: <CuteRobot className='w-[14px] h-[14px] mr-1'/> },
{ value: 'workflow', text: t('app.types.workflow'), icon: <Route className='w-[14px] h-[14px] mr-1'/> },
{ value: 'all', text: t('app.types.all'), icon: <DotsGrid className='w-[14px] h-[14px] mr-1' /> },
{ value: 'chat', text: t('app.types.chatbot'), icon: <ChatBot className='w-[14px] h-[14px] mr-1' /> },
{ value: 'agent-chat', text: t('app.types.agent'), icon: <CuteRobot className='w-[14px] h-[14px] mr-1' /> },
{ value: 'workflow', text: t('app.types.workflow'), icon: <Route className='w-[14px] h-[14px] mr-1' /> },
]
useEffect(() => {

5
web/app/components/base/icons/assets/public/other/row-struct.svg Normal file
View file

@ -0,0 +1,5 @@
<svg width="624" height="48" viewBox="0 0 624 48" fill="none" xmlns="http://www.w3.org/2000/svg">
<rect x="8" y="7" width="16" height="16" rx="5" fill="#F2F4F7"/>
<rect x="32" y="10" width="233" height="10" rx="3" fill="#EAECF0"/>
<rect x="32" y="31" width="345" height="6" rx="3" fill="#F2F4F7"/>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 305 B

5
web/app/components/base/icons/assets/vender/line/others/icon-3-dots.svg Normal file
View file

@ -0,0 +1,5 @@
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<g id="Icon-3-dots">
<path id="Icon" d="M5 6.5V5M8.93934 7.56066L10 6.5M10.0103 11.5H11.5103" stroke="#374151" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
</g>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 285 B

56
web/app/components/base/icons/src/public/other/RowStruct.json Normal file
View file

@ -0,0 +1,56 @@
{
"icon": {
"type": "element",
"isRootNode": true,
"name": "svg",
"attributes": {
"width": "624",
"height": "48",
"viewBox": "0 0 624 48",
"fill": "none",
"xmlns": "http://www.w3.org/2000/svg"
},
"children": [
{
"type": "element",
"name": "rect",
"attributes": {
"x": "8",
"y": "7",
"width": "16",
"height": "16",
"rx": "5",
"fill": "#F2F4F7"
},
"children": []
},
{
"type": "element",
"name": "rect",
"attributes": {
"x": "32",
"y": "10",
"width": "233",
"height": "10",
"rx": "3",
"fill": "#EAECF0"
},
"children": []
},
{
"type": "element",
"name": "rect",
"attributes": {
"x": "32",
"y": "31",
"width": "345",
"height": "6",
"rx": "3",
"fill": "#F2F4F7"
},
"children": []
}
]
},
"name": "RowStruct"
}

16
web/app/components/base/icons/src/public/other/RowStruct.tsx Normal file
View file

@ -0,0 +1,16 @@
// GENERATE BY script
// DON NOT EDIT IT MANUALLY
import * as React from 'react'
import data from './RowStruct.json'
import IconBase from '@/app/components/base/icons/IconBase'
import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase'
const Icon = React.forwardRef<React.MutableRefObject<SVGElement>, Omit<IconBaseProps, 'data'>>((
props,
ref,
) => <IconBase {...props} ref={ref} data={data as IconData} />)
Icon.displayName = 'RowStruct'
export default Icon

1
web/app/components/base/icons/src/public/other/index.ts
View file

@ -1,2 +1,3 @@
export { default as Icon3Dots } from './Icon3Dots'
export { default as DefaultToolIcon } from './DefaultToolIcon'
export { default as RowStruct } from './RowStruct'

39
web/app/components/base/icons/src/vender/line/others/Icon3Dots.json Normal file
View file

@ -0,0 +1,39 @@
{
"icon": {
"type": "element",
"isRootNode": true,
"name": "svg",
"attributes": {
"width": "16",
"height": "16",
"viewBox": "0 0 16 16",
"fill": "none",
"xmlns": "http://www.w3.org/2000/svg"
},
"children": [
{
"type": "element",
"name": "g",
"attributes": {
"id": "Icon-3-dots"
},
"children": [
{
"type": "element",
"name": "path",
"attributes": {
"id": "Icon",
"d": "M5 6.5V5M8.93934 7.56066L10 6.5M10.0103 11.5H11.5103",
"stroke": "currentColor",
"stroke-width": "2",
"stroke-linecap": "round",
"stroke-linejoin": "round"
},
"children": []
}
]
}
]
},
"name": "Icon3Dots"
}

16
web/app/components/base/icons/src/vender/line/others/Icon3Dots.tsx Normal file
View file

@ -0,0 +1,16 @@
// GENERATE BY script
// DON NOT EDIT IT MANUALLY
import * as React from 'react'
import data from './Icon3Dots.json'
import IconBase from '@/app/components/base/icons/IconBase'
import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase'
const Icon = React.forwardRef<React.MutableRefObject<SVGElement>, Omit<IconBaseProps, 'data'>>((
props,
ref,
) => <IconBase {...props} ref={ref} data={data as IconData} />)
Icon.displayName = 'Icon3Dots'
export default Icon

1
web/app/components/base/icons/src/vender/line/others/index.ts
View file

@ -3,4 +3,5 @@ export { default as Colors } from './Colors'
export { default as DragHandle } from './DragHandle'
export { default as Exchange02 } from './Exchange02'
export { default as FileCode } from './FileCode'
export { default as Icon3Dots } from './Icon3Dots'
export { default as Tools } from './Tools'

53
web/app/components/datasets/create/index.tsx
View file

@ -8,7 +8,7 @@ import StepOne from './step-one'
import StepTwo from './step-two'
import StepThree from './step-three'
import { DataSourceType } from '@/models/datasets'
import type { DataSet, FileItem, createDocumentResponse } from '@/models/datasets'
import type { CrawlOptions, CrawlResultItem, DataSet, FileItem, createDocumentResponse } from '@/models/datasets'
import { fetchDataSource } from '@/service/common'
import { fetchDatasetDetail } from '@/service/datasets'
import type { NotionPage } from '@/models/common'
@ -19,6 +19,15 @@ type DatasetUpdateFormProps = {
datasetId?: string
}
const DEFAULT_CRAWL_OPTIONS: CrawlOptions = {
crawl_sub_pages: true,
only_main_content: true,
includes: '',
excludes: '',
limit: 10,
max_depth: '',
}
const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
const { t } = useTranslation()
const { setShowAccountSettingModal } = useModalContext()
@ -36,9 +45,13 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
setNotionPages(value)
}
const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
const [crawlOptions, setCrawlOptions] = useState<CrawlOptions>(DEFAULT_CRAWL_OPTIONS)
const updateFileList = (preparedFiles: FileItem[]) => {
setFiles(preparedFiles)
}
const [fireCrawlJobId, setFireCrawlJobId] = useState('')
const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
const targetIndex = list.findIndex(file => file.fileID === fileItem.fileID)
@ -108,20 +121,27 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
<StepsNavBar step={step} datasetId={datasetId} />
</div>
<div className="grow bg-white">
{step === 1 && <StepOne
hasConnection={hasConnection}
onSetting={() => setShowAccountSettingModal({ payload: 'data-source' })}
datasetId={datasetId}
dataSourceType={dataSourceType}
dataSourceTypeDisable={!!detail?.data_source_type}
changeType={setDataSourceType}
files={fileList}
updateFile={updateFile}
updateFileList={updateFileList}
notionPages={notionPages}
updateNotionPages={updateNotionPages}
onStepChange={nextStep}
/>}
<div className={step === 1 ? 'block h-full' : 'hidden'}>
<StepOne
hasConnection={hasConnection}
onSetting={() => setShowAccountSettingModal({ payload: 'data-source' })}
datasetId={datasetId}
dataSourceType={dataSourceType}
dataSourceTypeDisable={!!detail?.data_source_type}
changeType={setDataSourceType}
files={fileList}
updateFile={updateFile}
updateFileList={updateFileList}
notionPages={notionPages}
updateNotionPages={updateNotionPages}
onStepChange={nextStep}
websitePages={websitePages}
updateWebsitePages={setWebsitePages}
onFireCrawlJobIdChange={setFireCrawlJobId}
crawlOptions={crawlOptions}
onCrawlOptionsChange={setCrawlOptions}
/>
</div>
{(step === 2 && (!datasetId || (datasetId && !!detail))) && <StepTwo
isAPIKeySet={!!embeddingsDefaultModel}
onSetting={() => setShowAccountSettingModal({ payload: 'provider' })}
@ -130,9 +150,12 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
dataSourceType={dataSourceType}
files={fileList.map(file => file.file)}
notionPages={notionPages}
websitePages={websitePages}
fireCrawlJobId={fireCrawlJobId}
onStepChange={changeStep}
updateIndexingTypeCache={updateIndexingTypeCache}
updateResultCache={updateResultCache}
crawlOptions={crawlOptions}
/>}
{step === 3 && <StepThree
datasetId={datasetId}

54
web/app/components/datasets/create/step-one/index.tsx
View file

@ -6,8 +6,10 @@ import FilePreview from '../file-preview'
import FileUploader from '../file-uploader'
import NotionPagePreview from '../notion-page-preview'
import EmptyDatasetCreationModal from '../empty-dataset-creation-modal'
import Website from '../website'
import WebsitePreview from '../website/preview'
import s from './index.module.css'
import type { FileItem } from '@/models/datasets'
import type { CrawlOptions, CrawlResultItem, FileItem } from '@/models/datasets'
import type { NotionPage } from '@/models/common'
import { DataSourceType } from '@/models/datasets'
import Button from '@/app/components/base/button'
@ -29,6 +31,11 @@ type IStepOneProps = {
updateNotionPages: (value: NotionPage[]) => void
onStepChange: () => void
changeType: (type: DataSourceType) => void
websitePages?: CrawlResultItem[]
updateWebsitePages: (value: CrawlResultItem[]) => void
onFireCrawlJobIdChange: (jobId: string) => void
crawlOptions: CrawlOptions
onCrawlOptionsChange: (payload: CrawlOptions) => void
}
type NotionConnectorProps = {
@ -49,7 +56,7 @@ export const NotionConnector = ({ onSetting }: NotionConnectorProps) => {
const StepOne = ({
datasetId,
dataSourceType,
dataSourceType: inCreatePageDataSourceType,
dataSourceTypeDisable,
changeType,
hasConnection,
@ -60,11 +67,17 @@ const StepOne = ({
updateFile,
notionPages = [],
updateNotionPages,
websitePages = [],
updateWebsitePages,
onFireCrawlJobIdChange,
crawlOptions,
onCrawlOptionsChange,
}: IStepOneProps) => {
const { dataset } = useDatasetDetailContext()
const [showModal, setShowModal] = useState(false)
const [currentFile, setCurrentFile] = useState<File | undefined>()
const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
const [currentWebsite, setCurrentWebsite] = useState<CrawlResultItem | undefined>()
const { t } = useTranslation()
const modalShowHandle = () => setShowModal(true)
@ -85,8 +98,13 @@ const StepOne = ({
setCurrentNotionPage(undefined)
}
const shouldShowDataSourceTypeList = !datasetId || (datasetId && !dataset?.data_source_type)
const hideWebsitePreview = () => {
setCurrentWebsite(undefined)
}
const shouldShowDataSourceTypeList = !datasetId || (datasetId && !dataset?.data_source_type)
const isInCreatePage = shouldShowDataSourceTypeList
const dataSourceType = isInCreatePage ? inCreatePageDataSourceType : dataset?.data_source_type
const { plan, enableBilling } = useProviderContext()
const allFileLoaded = (files.length > 0 && files.every(file => file.file.id))
const hasNotin = notionPages.length > 0
@ -150,10 +168,13 @@ const StepOne = ({
{t('datasetCreation.stepOne.dataSourceType.notion')}
</div>
<div
className={cn(s.dataSourceItem, s.disabled, dataSourceType === DataSourceType.WEB && s.active)}
// onClick={() => changeType(DataSourceType.WEB)}
className={cn(
s.dataSourceItem,
dataSourceType === DataSourceType.WEB && s.active,
dataSourceTypeDisable && dataSourceType !== DataSourceType.WEB && s.disabled,
)}
onClick={() => changeType(DataSourceType.WEB)}
>
<span className={s.comingTag}>Coming soon</span>
<span className={cn(s.datasetIcon, s.web)} />
{t('datasetCreation.stepOne.dataSourceType.web')}
</div>
@ -201,6 +222,26 @@ const StepOne = ({
)}
</>
)}
{dataSourceType === DataSourceType.WEB && (
<>
<div className={cn('mb-8 w-[640px]', !shouldShowDataSourceTypeList && 'mt-12')}>
<Website
onPreview={setCurrentWebsite}
checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={updateWebsitePages}
onJobIdChange={onFireCrawlJobIdChange}
crawlOptions={crawlOptions}
onCrawlOptionsChange={onCrawlOptionsChange}
/>
</div>
{isShowVectorSpaceFull && (
<div className='max-w-[640px] mb-4'>
<VectorSpaceFull />
</div>
)}
<Button disabled={isShowVectorSpaceFull || !websitePages.length} className={s.submitButton} type='primary' onClick={onStepChange}>{t('datasetCreation.stepOne.button')}</Button>
</>
)}
{!datasetId && (
<>
<div className={s.dividerLine} />
@ -212,6 +253,7 @@ const StepOne = ({
</div>
{currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
{currentNotionPage && <NotionPagePreview currentPage={currentNotionPage} hidePreview={hideNotionPagePreview} />}
{currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
</div>
)
}

1
web/app/components/datasets/create/step-two/index.module.css
View file

@ -323,6 +323,7 @@
}
.sourceContent {
width: 0;
flex: 1 1 auto;
}

54
web/app/components/datasets/create/step-two/index.tsx
View file

@ -12,7 +12,7 @@ import RetrievalMethodInfo from '../../common/retrieval-method-info'
import PreviewItem, { PreviewType } from './preview-item'
import LanguageSelect from './language-select'
import s from './index.module.css'
import type { CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, IndexingEstimateResponse, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, IndexingEstimateResponse, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
import {
createDocument,
createFirstDocument,
@ -44,6 +44,7 @@ import TooltipPlus from '@/app/components/base/tooltip-plus'
import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
import { LanguagesSupported } from '@/i18n/language'
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
import { Globe01 } from '@/app/components/base/icons/src/vender/line/mapsAndTravel'
type ValueOf<T> = T[keyof T]
type StepTwoProps = {
@ -56,6 +57,9 @@ type StepTwoProps = {
dataSourceType: DataSourceType
files: CustomFile[]
notionPages?: NotionPage[]
websitePages?: CrawlResultItem[]
crawlOptions?: CrawlOptions
fireCrawlJobId?: string
onStepChange?: (delta: number) => void
updateIndexingTypeCache?: (type: string) => void
updateResultCache?: (res: createDocumentResponse) => void
@ -79,9 +83,12 @@ const StepTwo = ({
onSetting,
datasetId,
indexingType,
dataSourceType,
dataSourceType: inCreatePageDataSourceType,
files,
notionPages = [],
websitePages = [],
crawlOptions,
fireCrawlJobId = '',
onStepChange,
updateIndexingTypeCache,
updateResultCache,
@ -94,6 +101,8 @@ const StepTwo = ({
const isMobile = media === MediaType.mobile
const { dataset: currentDataset, mutateDatasetRes } = useDatasetDetailContext()
const isInCreatePage = !datasetId || (datasetId && !currentDataset?.data_source_type)
const dataSourceType = isInCreatePage ? inCreatePageDataSourceType : currentDataset?.data_source_type
const scrollRef = useRef<HTMLDivElement>(null)
const [scrolled, setScrolled] = useState(false)
const previewScrollRef = useRef<HTMLDivElement>(null)
@ -242,6 +251,15 @@ const StepTwo = ({
}) as NotionInfo[]
}
const getWebsiteInfo = () => {
return {
provider: 'firecrawl',
job_id: fireCrawlJobId,
urls: websitePages.map(page => page.source_url),
only_main_content: crawlOptions?.only_main_content,
}
}
const getFileIndexingEstimateParams = (docForm: DocForm): IndexingEstimateParams | undefined => {
if (dataSourceType === DataSourceType.FILE) {
return {
@ -271,6 +289,19 @@ const StepTwo = ({
dataset_id: datasetId as string,
}
}
if (dataSourceType === DataSourceType.WEB) {
return {
info_list: {
data_source_type: dataSourceType,
website_info_list: getWebsiteInfo(),
},
indexing_technique: getIndexing_technique() as string,
process_rule: getProcessRule(),
doc_form: docForm,
doc_language: docLanguage,
dataset_id: datasetId as string,
}
}
}
const {
modelList: rerankModelList,
@ -335,6 +366,9 @@ const StepTwo = ({
}
if (dataSourceType === DataSourceType.NOTION)
params.data_source.info_list.notion_info_list = getNotionInfo()
if (dataSourceType === DataSourceType.WEB)
params.data_source.info_list.website_info_list = getWebsiteInfo()
}
return params
}
@ -819,6 +853,22 @@ const StepTwo = ({
</div>
</>
)}
{dataSourceType === DataSourceType.WEB && (
<>
<div className='mb-2 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.websiteSource')}</div>
<div className='flex items-center text-sm leading-6 font-medium text-gray-800'>
<Globe01 className='shrink-0 mr-1' />
<span className='grow w-0 truncate'>{websitePages[0].source_url}</span>
{websitePages.length > 1 && (
<span className={s.sourceCount}>
<span>{t('datasetCreation.stepTwo.other')}</span>
<span>{websitePages.length - 1}</span>
<span>{t('datasetCreation.stepTwo.webpageUnit')}</span>
</span>
)}
</div>
</>
)}
</div>
<div className={s.divider} />
<div className={s.segmentCount}>

29
web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx Normal file
View file

@ -0,0 +1,29 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import cn from 'classnames'
import Checkbox from '@/app/components/base/checkbox'
type Props = {
className?: string
isChecked: boolean
onChange: (isChecked: boolean) => void
label: string
labelClassName?: string
}
const CheckboxWithLabel: FC<Props> = ({
className = '',
isChecked,
onChange,
label,
labelClassName,
}) => {
return (
<label className={cn(className, 'flex items-center h-7 space-x-2')}>
<Checkbox checked={isChecked} onCheck={() => onChange(!isChecked)} />
<div className={cn(labelClassName, 'text-sm font-normal text-gray-800')}>{label}</div>
</label>
)
}
export default React.memo(CheckboxWithLabel)

30
web/app/components/datasets/create/website/firecrawl/base/error-message.tsx Normal file
View file

@ -0,0 +1,30 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import cn from 'classnames'
import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
type Props = {
className?: string
title: string
errorMsg?: string
}
const ErrorMessage: FC<Props> = ({
className,
title,
errorMsg,
}) => {
return (
<div className={cn(className, 'py-2 px-4 border-t border-gray-200 bg-[#FFFAEB]')}>
<div className='flex items-center h-5'>
<AlertTriangle className='mr-2 w-4 h-4 text-[#F79009]' />
<div className='text-sm font-medium text-[#DC6803]'>{title}</div>
</div>
{errorMsg && (
<div className='mt-1 pl-6 leading-[18px] text-xs font-normal text-gray-700'>{errorMsg}</div>
)}
</div>
)
}
export default React.memo(ErrorMessage)

54
web/app/components/datasets/create/website/firecrawl/base/field.tsx Normal file
View file

@ -0,0 +1,54 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import cn from 'classnames'
import Input from './input'
import TooltipPlus from '@/app/components/base/tooltip-plus'
import { HelpCircle } from '@/app/components/base/icons/src/vender/line/general'
type Props = {
className?: string
label: string
labelClassName?: string
value: string | number
onChange: (value: string | number) => void
isRequired?: boolean
placeholder?: string
isNumber?: boolean
tooltip?: string
}
const Field: FC<Props> = ({
className,
label,
labelClassName,
value,
onChange,
isRequired = false,
placeholder = '',
isNumber = false,
tooltip,
}) => {
return (
<div className={cn(className)}>
<div className='flex py-[7px]'>
<div className={cn(labelClassName, 'flex items-center h-[18px] text-[13px] font-medium text-gray-900')}>{label} </div>
{isRequired && <span className='ml-0.5 text-xs font-semibold text-[#D92D20]'>*</span>}
{tooltip && (
<TooltipPlus popupContent={
<div className='w-[200px]'>{tooltip}</div>
}>
<HelpCircle className='relative top-[3px] w-3 h-3 ml-1 text-gray-500' />
</TooltipPlus>
)}
</div>
<Input
value={value}
onChange={onChange}
placeholder={placeholder}
isNumber={isNumber}
/>
</div>
)
}
export default React.memo(Field)

58
web/app/components/datasets/create/website/firecrawl/base/input.tsx Normal file
View file

@ -0,0 +1,58 @@
'use client'
import type { FC } from 'react'
import React, { useCallback } from 'react'
type Props = {
value: string | number
onChange: (value: string | number) => void
placeholder?: string
isNumber?: boolean
}
const MIN_VALUE = 1
const Input: FC<Props> = ({
value,
onChange,
placeholder = '',
isNumber = false,
}) => {
const handleChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
const value = e.target.value
if (isNumber) {
let numberValue = parseInt(value, 10) // integer only
if (isNaN(numberValue)) {
onChange('')
return
}
if (numberValue < MIN_VALUE)
numberValue = MIN_VALUE
onChange(numberValue)
return
}
onChange(value)
}, [isNumber, onChange])
const otherOption = (() => {
if (isNumber) {
return {
min: MIN_VALUE,
}
}
return {
}
})()
return (
<input
type={isNumber ? 'number' : 'text'}
{...otherOption}
value={value}
onChange={handleChange}
className='flex h-9 w-full py-1 px-2 rounded-lg text-xs leading-normal bg-gray-100 caret-primary-600 hover:bg-gray-100 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none focus:bg-gray-50 placeholder:text-gray-400'
placeholder={placeholder}
/>
)
}
export default React.memo(Input)

55
web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx Normal file
View file

@ -0,0 +1,55 @@
'use client'
import { useBoolean } from 'ahooks'
import type { FC } from 'react'
import React, { useEffect } from 'react'
import { useTranslation } from 'react-i18next'
import cn from 'classnames'
import { Settings04 } from '@/app/components/base/icons/src/vender/line/general'
import { ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows'
const I18N_PREFIX = 'datasetCreation.stepOne.website'
type Props = {
className?: string
children: React.ReactNode
controlFoldOptions?: number
}
const OptionsWrap: FC<Props> = ({
className = '',
children,
controlFoldOptions,
}) => {
const { t } = useTranslation()
const [fold, {
toggle: foldToggle,
setTrue: foldHide,
}] = useBoolean(false)
useEffect(() => {
if (controlFoldOptions)
foldHide()
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [controlFoldOptions])
return (
<div className={cn(className, !fold ? 'mb-0' : 'mb-3')}>
<div
className='flex justify-between items-center h-[26px] py-1 cursor-pointer select-none'
onClick={foldToggle}
>
<div className='flex items-center text-gray-700'>
<Settings04 className='mr-1 w-4 h-4' />
<div className='text-[13px] font-semibold text-gray-800 uppercase'>{t(`${I18N_PREFIX}.options`)}</div>
</div>
<ChevronRight className={cn(!fold && 'rotate-90', 'w-4 h-4 text-gray-500')} />
</div>
{!fold && (
<div className='mb-4'>
{children}
</div>
)}
</div>
)
}
export default React.memo(OptionsWrap)

48
web/app/components/datasets/create/website/firecrawl/base/url-input.tsx Normal file
View file

@ -0,0 +1,48 @@
'use client'
import type { FC } from 'react'
import React, { useCallback, useState } from 'react'
import { useTranslation } from 'react-i18next'
import Input from './input'
import Button from '@/app/components/base/button'
const I18N_PREFIX = 'datasetCreation.stepOne.website'
type Props = {
isRunning: boolean
onRun: (url: string) => void
}
const UrlInput: FC<Props> = ({
isRunning,
onRun,
}) => {
const { t } = useTranslation()
const [url, setUrl] = useState('')
const handleUrlChange = useCallback((url: string | number) => {
setUrl(url as string)
}, [])
const handleOnRun = useCallback(() => {
if (isRunning)
return
onRun(url)
}, [isRunning, onRun, url])
return (
<div className='flex items-center justify-between'>
<Input
value={url}
onChange={handleUrlChange}
placeholder='https://docs.dify.ai'
/>
<Button
type='primary'
onClick={handleOnRun}
className='ml-2 !h-8 text-[13px] font-medium'
loading={isRunning}
>
{!isRunning ? t(`${I18N_PREFIX}.run`) : ''}
</Button>
</div>
)
}
export default React.memo(UrlInput)

Some files were not shown because too many files have changed in this diff Show more