jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

fix: update parameters in spider components (#3280)

Browse source 
* fix: make spider tool work

* upgrade spider-client

* [autofix.ci] apply automated fixes

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
William Espegren 2024-08-12 21:45:20 +02:00 • committed by GitHub
commit e09f157af6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 16 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

2
poetry.lock generated
View file

@ -5236,7 +5236,7 @@ python-multipart = "^0.0.7"
rich = "^13.7.0" rich = "^13.7.0"
sentry-sdk = {version = "^2.5.1", extras = ["fastapi", "loguru"]} sentry-sdk = {version = "^2.5.1", extras = ["fastapi", "loguru"]}
setuptools = ">=70" setuptools = ">=70"
spider-client = "^0.0.27" spider-client = "^0.0.60"
sqlmodel = "^0.0.18" sqlmodel = "^0.0.18"
typer = "^0.12.0" typer = "^0.12.0"
uncurl = "^0.0.11" uncurl = "^0.0.11"

25
src/backend/base/langflow/components/langchain_utilities/SpiderTool.py
View file

@ -1,4 +1,4 @@
from spider.spider import Spider # type: ignore from spider.spider import Spider
from langflow.base.langchain_utilities.spider_constants import MODES from langflow.base.langchain_utilities.spider_constants import MODES
from langflow.custom import Component from langflow.custom import Component
@ -59,7 +59,7 @@ class SpiderTool(Component):
advanced=True, advanced=True,
), ),
BoolInput( BoolInput(
name="use_readability", name="readability",
display_name="Use Readability", display_name="Use Readability",
info="Use readability to pre-process the content for reading.", info="Use readability to pre-process the content for reading.",
advanced=True, advanced=True,
@ -89,15 +89,15 @@ class SpiderTool(Component):
def crawl(self) -> list[Data]: def crawl(self) -> list[Data]:
if self.params: if self.params:
parameters = self.params.data parameters = self.params["data"]
else: else:
parameters = { parameters = {
"limit": self.limit, "limit": self.limit if self.limit else None,
"depth": self.depth, "depth": self.depth if self.depth else None,
"blacklist": self.blacklist, "blacklist": self.blacklist if self.blacklist else None,
"whitelist": self.whitelist, "whitelist": self.whitelist if self.whitelist else None,
"use_readability": self.use_readability, "readability": self.readability,
"request_timeout": self.request_timeout, "request_timeout": self.request_timeout if self.request_timeout else None,
"metadata": self.metadata, "metadata": self.metadata,
"return_format": "markdown", "return_format": "markdown",
} }
@ -117,5 +117,10 @@ class SpiderTool(Component):
records = [] records = []
for record in result: for record in result:
records.append(Data(data={"content": record["content"], "url": record["url"]})) if self.metadata:
records.append(
Data(data={"content": record["content"], "url": record["url"], "metadata": record["metadata"]})
)
else:
records.append(Data(data={"content": record["content"], "url": record["url"]}))
return records return records