jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Merge branch 'main' into dev

Browse source
This commit is contained in:
Gabriel Luiz Freitas Almeida 2024-06-26 16:28:45 -03:00
commit 68c4485770
17 changed files with 467 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

49
src/backend/base/langflow/components/embeddings/AstraVectorize.py
View file

@ -1,6 +1,6 @@
from typing import Any
from langflow.custom import Component
from langflow.inputs.inputs import DictInput, SecretStrInput, MessageTextInput
from langflow.inputs.inputs import DictInput, SecretStrInput, MessageTextInput, DropdownInput
from langflow.template.field.base import Output
@ -10,32 +10,58 @@ class AstraVectorize(Component):
documentation: str = "https://docs.datastax.com/en/astra-db-serverless/databases/embedding-generation.html"
icon = "AstraDB"
VECTORIZE_PROVIDERS_MAPPING = {
"Azure OpenAI": ["azureOpenAI", ["text-embedding-3-small", "text-embedding-3-large", "text-embedding-ada-002"]],
"Hugging Face - Dedicated": ["huggingfaceDedicated", ["endpoint-defined-model"]],
"Hugging Face - Serverless": ["huggingface",
["sentence-transformers/all-MiniLM-L6-v2", "intfloat/multilingual-e5-large",
"intfloat/multilingual-e5-large-instruct", "BAAI/bge-small-en-v1.5",
"BAAI/bge-base-en-v1.5", "BAAI/bge-large-en-v1.5"]],
"Jina AI": ["jinaAI", ["jina-embeddings-v2-base-en", "jina-embeddings-v2-base-de", "jina-embeddings-v2-base-es",
"jina-embeddings-v2-base-code", "jina-embeddings-v2-base-zh"]],
"Mistral AI": ["mistral", ["mistral-embed"]],
"NVIDIA": ["nvidia", ["NV-Embed-QA"]],
"OpenAI": ["openai", ["text-embedding-3-small", "text-embedding-3-large", "text-embedding-ada-002"]],
"Upstage": ["upstageAI", ["solar-embedding-1-large"]],
"Voyage AI": ["voyageAI",
["voyage-large-2-instruct", "voyage-law-2", "voyage-code-2", "voyage-large-2", "voyage-2"]]
}
VECTORIZE_MODELS_STR = "\n\n".join([provider + ": " + (', '.join(models[1])) for provider, models in VECTORIZE_PROVIDERS_MAPPING.items()])
inputs = [
MessageTextInput(
DropdownInput(
name="provider",
display_name="Provider name",
info="The embedding provider to use.",
options=VECTORIZE_PROVIDERS_MAPPING.keys(),
value="",
),
MessageTextInput(
name="model_name",
display_name="Model name",
info="The embedding model to use.",
info=f"The embedding model to use for the selected provider. Each provider has a different set of models "
f"available (full list at https://docs.datastax.com/en/astra-db-serverless/databases/embedding-generation.html):\n\n{VECTORIZE_MODELS_STR}",
required=True
),
MessageTextInput(
name="api_key_name",
display_name="API Key name",
info="The name of the embeddings provider API key stored on Astra. If set, it will override the 'ProviderKey' in the authentication parameters."
),
DictInput(
name="authentication",
display_name="Authentication",
info="Authentication parameters. Use the Astra Portal to add the embedding provider integration to your Astra organization.",
display_name="Authentication parameters",
is_list=True,
advanced=True,
),
SecretStrInput(
name="provider_api_key",
display_name="Provider API Key",
info="An alternative to the Astra Authentication that let you use directly the API key of the provider.",
advanced=True,
),
DictInput(
name="model_parameters",
display_name="Model parameters",
info="Additional model parameters.",
advanced=True,
is_list=True,
),
@ -45,12 +71,17 @@ class AstraVectorize(Component):
]
def build_options(self) -> dict[str, Any]:
provider_value = self.VECTORIZE_PROVIDERS_MAPPING[self.provider][0]
authentication = {**self.authentication}
api_key_name = self.api_key_name
if api_key_name:
authentication["providerKey"] = api_key_name
return {
# must match exactly astra CollectionVectorServiceOptions
"collection_vector_service_options": {
"provider": self.provider,
"provider": provider_value,
"modelName": self.model_name,
"authentication": self.authentication,
"authentication": authentication,
"parameters": self.model_parameters,
},
"collection_embedding_api_key": self.provider_api_key,

81
src/backend/base/langflow/components/langchain_utilities/FirecrawlCrawlApi.py Normal file
View file

@ -0,0 +1,81 @@
from typing import Optional
from firecrawl.firecrawl import FirecrawlApp
from langflow.custom import CustomComponent
from langflow.schema import Data
import uuid
class FirecrawlCrawlApi(CustomComponent):
display_name: str = "FirecrawlCrawlApi"
description: str = "Firecrawl Crawl API."
output_types: list[str] = ["Document"]
documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/crawl"
field_config = {
"api_key": {
"display_name": "API Key",
"field_type": "str",
"required": True,
"password": True,
"info": "The API key to use Firecrawl API.",
},
"url": {
"display_name": "URL",
"field_type": "str",
"required": True,
"info": "The base URL to start crawling from.",
},
"timeout": {
"display_name": "Timeout",
"field_type": "int",
"info": "The timeout in milliseconds.",
},
"crawlerOptions": {
"display_name": "Crawler Options",
"info": "Options for the crawler behavior.",
},
"pageOptions": {
"display_name": "Page Options",
"info": "The page options to send with the request.",
},
"idempotency_key": {
"display_name": "Idempotency Key",
"field_type": "str",
"info": "Optional idempotency key to ensure unique requests.",
},
}
def build(
self,
api_key: str,
url: str,
timeout: Optional[int] = 30000,
crawlerOptions: Optional[Data] = None,
pageOptions: Optional[Data] = None,
idempotency_key: Optional[str] = None,
) -> Data:
if crawlerOptions:
crawler_options_dict = crawlerOptions.__dict__['data']['text']
else:
crawler_options_dict = {}
if pageOptions:
page_options_dict = pageOptions.__dict__['data']['text']
else:
page_options_dict = {}
if not idempotency_key:
idempotency_key = str(uuid.uuid4())
app = FirecrawlApp(api_key=api_key)
crawl_result = app.crawl_url(
url,
{
"crawlerOptions": crawler_options_dict,
"pageOptions": page_options_dict,
},
True,
int(timeout / 1000),
idempotency_key
)
records = Data(data={"results": crawl_result})
return records

70
src/backend/base/langflow/components/langchain_utilities/FirecrawlScrapeApi.py Normal file
View file

@ -0,0 +1,70 @@
from typing import Optional
from firecrawl.firecrawl import FirecrawlApp
from langflow.custom import CustomComponent
from langflow.schema import Data
class FirecrawlScrapeApi(CustomComponent):
display_name: str = "FirecrawlScrapeApi"
description: str = "Firecrawl Scrape API."
output_types: list[str] = ["Document"]
documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/scrape"
field_config = {
"api_key": {
"display_name": "API Key",
"field_type": "str",
"required": True,
"password": True,
"info": "The API key to use Firecrawl API.",
},
"url": {
"display_name": "URL",
"field_type": "str",
"required": True,
"info": "The URL to scrape.",
},
"timeout": {
"display_name": "Timeout",
"info": "Timeout in milliseconds for the request.",
"field_type": "int",
"default_value": 10000,
},
"pageOptions": {
"display_name": "Page Options",
"info": "The page options to send with the request.",
},
"extractorOptions": {
"display_name": "Extractor Options",
"info": "The extractor options to send with the request.",
},
}
def build(
self,
api_key: str,
url: str,
timeout: Optional[int] = 10000,
pageOptions: Optional[Data] = None,
extractorOptions: Optional[Data] = None,
) -> Data:
if extractorOptions:
extractor_options_dict = extractorOptions.__dict__['data']['text']
else:
extractor_options_dict = {}
if pageOptions:
page_options_dict = pageOptions.__dict__['data']['text']
else:
page_options_dict = {}
app = FirecrawlApp(api_key=api_key)
results = app.scrape_url(
url,
{
"timeout": str(timeout),
"extractorOptions": extractor_options_dict,
"pageOptions": page_options_dict,
},
)
record = Data(data=results)
return record

71
src/backend/base/langflow/components/vectorstores/Cassandra.py
View file

@ -4,6 +4,7 @@ from langchain_community.vectorstores import Cassandra
from langflow.base.vectorstores.model import LCVectorStoreComponent
from langflow.helpers.data import docs_to_data
from langflow.inputs import DictInput
from langflow.io import (
DataInput,
DropdownInput,
@ -23,24 +24,30 @@ class CassandraVectorStoreComponent(LCVectorStoreComponent):
icon = "Cassandra"
inputs = [
MessageTextInput(name="database_ref",
display_name="Contact Points / Astra Database ID",
info="Contact points for the database (or AstraDB database ID)",
required=True),
MessageTextInput(name="username",
display_name="Username",
info="Username for the database (leave empty for AstraDB)."),
SecretStrInput(
name="token",
display_name="Token",
info="Authentication token for accessing Cassandra on Astra DB.",
required=True,
),
MessageTextInput(name="database_id", display_name="Database ID", info="The Astra database ID.", required=True),
MessageTextInput(
name="table_name",
display_name="Table Name",
info="The name of the table where vectors will be stored.",
required=True,
display_name="Password / AstraDB Token",
info="User password for the database (or AstraDB token).",
required=True
),
MessageTextInput(
name="keyspace",
display_name="Keyspace",
info="Optional key space within Astra DB. The keyspace should already be created.",
advanced=False,
info="Table Keyspace (or AstraDB namespace).",
required=True,
),
MessageTextInput(
name="table_name",
display_name="Table Name",
info="The name of the table (or AstraDB collection) where vectors will be stored.",
required=True,
),
IntInput(
name="ttl_seconds",
@ -69,6 +76,13 @@ class CassandraVectorStoreComponent(LCVectorStoreComponent):
value="Sync",
advanced=True,
),
DictInput(
name="cluster_kwargs",
display_name="Cluster arguments",
info="Optional dictionary of additional keyword arguments for the Cassandra cluster.",
advanced=True,
is_list=True
),
MultilineInput(name="search_query", display_name="Search Query"),
DataInput(
name="ingest_data",
@ -96,10 +110,35 @@ class CassandraVectorStoreComponent(LCVectorStoreComponent):
"Could not import cassio integration package. " "Please install it with `pip install cassio`."
)
cassio.init(
database_id=self.database_id,
token=self.token,
)
from uuid import UUID
database_ref = self.database_ref
try:
UUID(self.database_ref)
is_astra = True
except ValueError:
is_astra = False
if "," in self.database_ref:
# use a copy because we can't change the type of the parameter
database_ref = self.database_ref.split(",")
if is_astra:
cassio.init(
database_id=database_ref,
token=self.token,
cluster_kwargs=self.cluster_kwargs,
)
else:
cassio.init(
contact_points=database_ref,
username=self.username,
password=self.token,
cluster_kwargs=self.cluster_kwargs,
)
if not self.ttl_seconds:
self.ttl_seconds = None
documents = []

16
src/backend/base/poetry.lock generated
View file

@ -739,6 +739,20 @@ typer = ">=0.12.3"
[package.extras]
standard = ["fastapi", "uvicorn[standard] (>=0.15.0)"]
[[package]]
name = "firecrawl-py"
version = "0.0.16"
description = "Python SDK for Firecrawl API"
optional = false
python-versions = ">=3.8"
files = [
{file = "firecrawl_py-0.0.16-py3-none-any.whl", hash = "sha256:9024f483b501852a6b9c4e6cdfc9e8dde452d922afac357080bb278a0c9c2a26"},
{file = "firecrawl_py-0.0.16.tar.gz", hash = "sha256:6c662fa0a549bc7f5c0acb704baba6731869ca0451094034264dfc1b4eb086e4"},
]
[package.dependencies]
requests = "*"
[[package]]
name = "frozenlist"
version = "1.4.1"
@ -3235,4 +3249,4 @@ local = []
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<3.13"
content-hash = "4f566531a8539ddc81cb91a7e7f9b723c84679f0af5bb8619f7b02f9ffc6cfaa"
content-hash = "7e46144d27c633214f00e73e496c0e4d56db1fb47032a21861677ec275b79d86"

1
src/backend/base/pyproject.toml
View file

@ -64,6 +64,7 @@ pyperclip = "^1.8.2"
uncurl = "^0.0.11"
sentry-sdk = {extras = ["fastapi", "loguru"], version = "^2.5.1"}
chardet = "^5.2.0"
firecrawl-py = "^0.0.16"
[tool.poetry.extras]