jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

feat: add Spider Web Scraper & Crawler (#2439)

Browse source 
* spider files

* rebuild required

* add spider-client here

* Feat: Spider Web Crawler & Scraper

* Feat: spider integration

* new input not working

* [autofix.ci] apply automated fixes

* [autofix.ci] apply automated fixes (attempt 2/3)

* fix: add outputs and configure build method

* style: run ruff

* Refactor SpiderTool to use 'crawl' instead of 'build' for generating Markdown content

* chore: add type ignore

* chore: new lock

* chore: Update mem0ai dependency to version 0.0.5

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Gabriel Luiz Freitas Almeida <gabriel@langflow.org>
This commit is contained in:
William Espegren 2024-08-08 14:53:05 +02:00 • committed by GitHub
commit 7a36cc9ebf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 2580 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

18
src/frontend/src/icons/Spider/SpiderIcon.jsx Normal file
View file

@ -0,0 +1,18 @@
const SvgSpiderIcon = (props) => (
<svg
height="30"
width="30"
viewBox="0 0 36 34"
xmlns="http://www.w3.org/2000/svg"
className="fill-accent-foreground transition-all group-hover:scale-110"
{...props}
>
<title>Spider v0 Logo</title>
<path
fillRule="evenodd"
clipRule="evenodd"
d="M9.13883 7.06589V0.164429L13.0938 0.164429V6.175L14.5178 7.4346C15.577 6.68656 16.7337 6.27495 17.945 6.27495C19.1731 6.27495 20.3451 6.69807 21.4163 7.46593L22.8757 6.175V0.164429L26.8307 0.164429V7.06589V7.95679L26.1634 8.54706L24.0775 10.3922C24.3436 10.8108 24.5958 11.2563 24.8327 11.7262L26.0467 11.4215L28.6971 8.08749L31.793 10.5487L28.7257 14.407L28.3089 14.9313L27.6592 15.0944L26.2418 15.4502C26.3124 15.7082 26.3793 15.9701 26.4422 16.2355L28.653 16.6566L29.092 16.7402L29.4524 17.0045L35.3849 21.355L33.0461 24.5444L27.474 20.4581L27.0719 20.3816C27.1214 21.0613 27.147 21.7543 27.147 22.4577C27.147 22.5398 27.1466 22.6214 27.1459 22.7024L29.5889 23.7911L30.3219 24.1177L30.62 24.8629L33.6873 32.5312L30.0152 34L27.246 27.0769L26.7298 26.8469C25.5612 32.2432 22.0701 33.8808 17.945 33.8808C13.8382 33.8808 10.3598 32.2577 9.17593 26.9185L8.82034 27.0769L6.05109 34L2.37897 32.5312L5.44629 24.8629L5.74435 24.1177L6.47743 23.7911L8.74487 22.7806C8.74366 22.6739 8.74305 22.5663 8.74305 22.4577C8.74305 21.7616 8.76804 21.0758 8.81654 20.4028L8.52606 20.4581L2.95395 24.5444L0.615112 21.355L6.54761 17.0045L6.908 16.7402L7.34701 16.6566L9.44264 16.2575C9.50917 15.9756 9.5801 15.6978 9.65528 15.4242L8.34123 15.0944L7.69155 14.9313L7.27471 14.407L4.20739 10.5487L7.30328 8.08749L9.95376 11.4215L11.0697 11.7016C11.3115 11.2239 11.5692 10.7716 11.8412 10.3473L9.80612 8.54706L9.13883 7.95679V7.06589Z"
></path>
</svg>
);
export default SvgSpiderIcon;

9
src/frontend/src/icons/Spider/index.tsx Normal file
View file

@ -0,0 +1,9 @@
import React, { forwardRef } from "react";
import SvgSpiderIcon from "./SpiderIcon";
export const SpiderIcon = forwardRef<
SVGSVGElement,
React.PropsWithChildren<{}>
>((props, ref) => {
return <SvgSpiderIcon ref={ref} {...props} />;
});

1
src/frontend/src/icons/Spider/spider_logo.svg Normal file
View file

@ -0,0 +1 @@
<svg height="30" width="30" viewBox="0 0 36 34" xml:space="preserve" xmlns="http://www.w3.org/2000/svg" class="fill-accent-foreground transition-all group-hover:scale-110"><title>Spider v1 Logo</title><path fill-rule="evenodd" clip-rule="evenodd" d="M9.13883 7.06589V0.164429L13.0938 0.164429V6.175L14.5178 7.4346C15.577 6.68656 16.7337 6.27495 17.945 6.27495C19.1731 6.27495 20.3451 6.69807 21.4163 7.46593L22.8757 6.175V0.164429L26.8307 0.164429V7.06589V7.95679L26.1634 8.54706L24.0775 10.3922C24.3436 10.8108 24.5958 11.2563 24.8327 11.7262L26.0467 11.4215L28.6971 8.08749L31.793 10.5487L28.7257 14.407L28.3089 14.9313L27.6592 15.0944L26.2418 15.4502C26.3124 15.7082 26.3793 15.9701 26.4422 16.2355L28.653 16.6566L29.092 16.7402L29.4524 17.0045L35.3849 21.355L33.0461 24.5444L27.474 20.4581L27.0719 20.3816C27.1214 21.0613 27.147 21.7543 27.147 22.4577C27.147 22.5398 27.1466 22.6214 27.1459 22.7024L29.5889 23.7911L30.3219 24.1177L30.62 24.8629L33.6873 32.5312L30.0152 34L27.246 27.0769L26.7298 26.8469C25.5612 32.2432 22.0701 33.8808 17.945 33.8808C13.8382 33.8808 10.3598 32.2577 9.17593 26.9185L8.82034 27.0769L6.05109 34L2.37897 32.5312L5.44629 24.8629L5.74435 24.1177L6.47743 23.7911L8.74487 22.7806C8.74366 22.6739 8.74305 22.5663 8.74305 22.4577C8.74305 21.7616 8.76804 21.0758 8.81654 20.4028L8.52606 20.4581L2.95395 24.5444L0.615112 21.355L6.54761 17.0045L6.908 16.7402L7.34701 16.6566L9.44264 16.2575C9.50917 15.9756 9.5801 15.6978 9.65528 15.4242L8.34123 15.0944L7.69155 14.9313L7.27471 14.407L4.20739 10.5487L7.30328 8.08749L9.95376 11.4215L11.0697 11.7016C11.3115 11.2239 11.5692 10.7716 11.8412 10.3473L9.80612 8.54706L9.13883 7.95679V7.06589Z"></path></svg>
Side by side

After

Width:  |  Height:  |  Size: 1.6 KiB

2
src/frontend/src/utils/styleUtils.ts
View file

@ -211,6 +211,7 @@ import { SearxIcon } from "../icons/Searx";
import { ShareIcon } from "../icons/Share";
import { Share2Icon } from "../icons/Share2";
import SvgSlackIcon from "../icons/Slack/SlackIcon";
import { SpiderIcon } from "../icons/Spider";
import { Streamlit } from "../icons/Streamlit";
import { UpstashSvgIcon } from "../icons/Upstash";
import { VectaraIcon } from "../icons/VectaraIcon";
@ -412,6 +413,7 @@ export const nodeIconsLucide: iconsType = {
Weaviate: WeaviateIcon,
Searx: SearxIcon,
SlackDirectoryLoader: SvgSlackIcon,
SpiderTool: SpiderIcon,
SupabaseVectorStore: SupabaseIcon,
Supabase: SupabaseIcon,
VertexAI: VertexAIIcon,