jyapayne/dify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(website-crawl): add jina reader as additional alternative for website crawling (#8761)

Browse source
This commit is contained in:
Zhaofeng Miao 2024-09-30 09:57:19 +08:00 • committed by GitHub
commit 369e1e6f58
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
38 changed files with 927 additions and 75 deletions
Download patch file Download diff file Expand all files Collapse all files

15
web/i18n/en-US/dataset-creation.ts
View file

@ -16,6 +16,11 @@ const translation = {
apiKeyPlaceholder: 'API key from firecrawl.dev',
getApiKeyLinkText: 'Get your API key from firecrawl.dev',
},
jinaReader: {
configJinaReader: 'Configure Jina Reader',
apiKeyPlaceholder: 'API key from jina.ai',
getApiKeyLinkText: 'Get your free API key at jina.ai',
},
stepOne: {
filePreview: 'File Preview',
pagePreview: 'Page Preview',
@ -56,13 +61,21 @@ const translation = {
failed: 'Creation failed',
},
website: {
chooseProvider: 'Select a provider',
fireCrawlNotConfigured: 'Firecrawl is not configured',
fireCrawlNotConfiguredDescription: 'Configure Firecrawl with API key to use it.',
jinaReaderNotConfigured: 'Jina Reader is not configured',
jinaReaderNotConfiguredDescription: 'Set up Jina Reader by entering your free API key for access.',
configure: 'Configure',
run: 'Run',
firecrawlTitle: 'Extract web content with 🔥Firecrawl',
firecrawlDoc: 'Firecrawl docs',
firecrawlDocLink: 'https://docs.dify.ai/guides/knowledge-base/sync-from-website',
jinaReaderTitle: 'Convert the entire site to Markdown',
jinaReaderDoc: 'Learn more about Jina Reader',
jinaReaderDocLink: 'https://jina.ai/reader',
useSitemap: 'Use sitemap',
useSitemapTooltip: 'Follow the sitemap to crawl the site. If not, Jina Reader will crawl iteratively based on page relevance, yielding fewer but higher-quality pages.',
options: 'Options',
crawlSubPage: 'Crawl sub-pages',
limit: 'Limit',
@ -70,7 +83,7 @@ const translation = {
excludePaths: 'Exclude paths',
includeOnlyPaths: 'Include only paths',
extractOnlyMainContent: 'Extract only main content (no headers, navs, footers, etc.)',
exceptionErrorTitle: 'An exception occurred while running Firecrawl job:',
exceptionErrorTitle: 'An exception occurred while running crawling job:',
unknownError: 'Unknown error',
totalPageScraped: 'Total pages scraped:',
selectAll: 'Select All',