feat(website-crawl): add jina reader as additional alternative for website crawling (#8761)

This commit is contained in:
Zhaofeng Miao 2024-09-30 09:57:19 +08:00 • committed by GitHub
commit 369e1e6f58
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
38 changed files with 927 additions and 75 deletions

View file

@ -16,6 +16,11 @@ const translation = {
apiKeyPlaceholder: '从 firecrawl.dev 获取 API Key',
getApiKeyLinkText: '从 firecrawl.dev 获取您的 API Key',
},
jinaReader: {
configJinaReader: '配置 Jina Reader',
apiKeyPlaceholder: '从 jina.ai 获取 API Key',
getApiKeyLinkText: '从 jina.ai 获取您的免费 API Key',
},
stepOne: {
filePreview: '文件预览',
pagePreview: '页面预览',
@ -56,13 +61,21 @@ const translation = {
failed: '创建失败',
},
website: {
chooseProvider: '选择工具',
fireCrawlNotConfigured: 'Firecrawl 未配置',
fireCrawlNotConfiguredDescription: '请配置 Firecrawl 的 API 密钥以使用它。',
jinaReaderNotConfigured: 'Jina Reader 未配置',
jinaReaderNotConfiguredDescription: '请配置 Jina Reader 的免费 API 密钥以访问它。',
configure: '配置',
run: '运行',
firecrawlTitle: '使用 🔥Firecrawl 提取网页内容',
firecrawlDoc: 'Firecrawl 文档',
firecrawlDocLink: 'https://docs.dify.ai/v/zh-hans/guides/knowledge-base/sync-from-website',
jinaReaderTitle: '将整个站点内容转换为 Markdown 格式',
jinaReaderDoc: '了解更多关于 Jina Reader',
jinaReaderDocLink: 'https://jina.ai/reader',
useSitemap: '使用 sitemap',
useSitemapTooltip: '根据 sitemap 爬取站点。否则,Jina Reader 将基于页面相关性迭代爬取,抓取较少的页面,但质量更高。',
options: '选项',
crawlSubPage: '爬取子页面',
limit: '限制数量',
@ -70,7 +83,7 @@ const translation = {
excludePaths: '排除路径',
includeOnlyPaths: '仅包含路径',
extractOnlyMainContent: '仅提取主要内容(无标题、导航、页脚等)',
exceptionErrorTitle: '运行 Firecrawl 时发生异常:',
exceptionErrorTitle: '运行时发生异常:',
unknownError: '未知错误',
totalPageScraped: '抓取页面总数:',
selectAll: '全选',