feat(website-crawl): add jina reader as additional alternative for website crawling (#8761)
This commit is contained in:
parent
fb49413a41
commit
369e1e6f58
38 changed files with 927 additions and 75 deletions
|
|
@ -10,7 +10,7 @@ import WebsitePreview from '../website/preview'
|
|||
import s from './index.module.css'
|
||||
import cn from '@/utils/classnames'
|
||||
import type { CrawlOptions, CrawlResultItem, FileItem } from '@/models/datasets'
|
||||
import type { NotionPage } from '@/models/common'
|
||||
import type { DataSourceProvider, NotionPage } from '@/models/common'
|
||||
import { DataSourceType } from '@/models/datasets'
|
||||
import Button from '@/app/components/base/button'
|
||||
import { NotionPageSelector } from '@/app/components/base/notion-page-selector'
|
||||
|
|
@ -33,7 +33,8 @@ type IStepOneProps = {
|
|||
changeType: (type: DataSourceType) => void
|
||||
websitePages?: CrawlResultItem[]
|
||||
updateWebsitePages: (value: CrawlResultItem[]) => void
|
||||
onFireCrawlJobIdChange: (jobId: string) => void
|
||||
onWebsiteCrawlProviderChange: (provider: DataSourceProvider) => void
|
||||
onWebsiteCrawlJobIdChange: (jobId: string) => void
|
||||
crawlOptions: CrawlOptions
|
||||
onCrawlOptionsChange: (payload: CrawlOptions) => void
|
||||
}
|
||||
|
|
@ -69,7 +70,8 @@ const StepOne = ({
|
|||
updateNotionPages,
|
||||
websitePages = [],
|
||||
updateWebsitePages,
|
||||
onFireCrawlJobIdChange,
|
||||
onWebsiteCrawlProviderChange,
|
||||
onWebsiteCrawlJobIdChange,
|
||||
crawlOptions,
|
||||
onCrawlOptionsChange,
|
||||
}: IStepOneProps) => {
|
||||
|
|
@ -229,7 +231,8 @@ const StepOne = ({
|
|||
onPreview={setCurrentWebsite}
|
||||
checkedCrawlResult={websitePages}
|
||||
onCheckedCrawlResultChange={updateWebsitePages}
|
||||
onJobIdChange={onFireCrawlJobIdChange}
|
||||
onCrawlProviderChange={onWebsiteCrawlProviderChange}
|
||||
onJobIdChange={onWebsiteCrawlJobIdChange}
|
||||
crawlOptions={crawlOptions}
|
||||
onCrawlOptionsChange={onCrawlOptionsChange}
|
||||
/>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue