diff --git a/src/backend/langflow/api/v1/flows.py b/src/backend/langflow/api/v1/flows.py index 371e320f4..c58168d34 100644 --- a/src/backend/langflow/api/v1/flows.py +++ b/src/backend/langflow/api/v1/flows.py @@ -11,12 +11,8 @@ from sqlmodel import Session, select from langflow.api.utils import remove_api_keys, validate_is_component from langflow.api.v1.schemas import FlowListCreate, FlowListRead from langflow.services.auth.utils import get_current_active_user -from langflow.services.database.models.flow import ( - Flow, - FlowCreate, - FlowRead, - FlowUpdate, -) +from langflow.services.database.models.flow import (Flow, FlowCreate, FlowRead, + FlowUpdate) from langflow.services.database.models.user.model import User from langflow.services.deps import get_session, get_settings_service diff --git a/src/backend/langflow/components/data/URL.py b/src/backend/langflow/components/data/URL.py index 08eafeaa3..d3af63b1c 100644 --- a/src/backend/langflow/components/data/URL.py +++ b/src/backend/langflow/components/data/URL.py @@ -1,6 +1,6 @@ -from typing import Any, Dict, Optional +from typing import Any, Dict -from langchain_community.document_loaders.url import UnstructuredURLLoader +from langchain_community.document_loaders.web_base import WebBaseLoader from langflow import CustomComponent from langflow.schema import Record @@ -8,7 +8,7 @@ from langflow.schema import Record class URLComponent(CustomComponent): display_name = "URL" - description = "Load a URL." + description = "Load URLs and convert them to records." def build_config(self) -> Dict[str, Any]: return { @@ -18,9 +18,9 @@ class URLComponent(CustomComponent): async def build( self, urls: list[str], - ) -> Optional[Record]: + ) -> Record: - loader = UnstructuredURLLoader(urls=urls) + loader = WebBaseLoader(web_paths=urls) docs = loader.load() records = self.to_records(docs) return records diff --git a/src/backend/langflow/components/helpers/APIRequest.py b/src/backend/langflow/components/helpers/APIRequest.py index dfaf2ed07..4d9f3ce9c 100644 --- a/src/backend/langflow/components/helpers/APIRequest.py +++ b/src/backend/langflow/components/helpers/APIRequest.py @@ -52,7 +52,7 @@ class APIRequest(CustomComponent): if method not in ["GET", "POST", "PATCH", "PUT"]: raise ValueError(f"Unsupported method: {method}") - data = record.text if record else None + data = record.data if record else None try: response = await client.request( method, url, headers=headers, content=data, timeout=timeout diff --git a/src/backend/langflow/components/helpers/IDGenerator.py b/src/backend/langflow/components/helpers/IDGenerator.py index ceb937a6c..1a8cecbfd 100644 --- a/src/backend/langflow/components/helpers/IDGenerator.py +++ b/src/backend/langflow/components/helpers/IDGenerator.py @@ -1,5 +1,5 @@ import uuid -from typing import Text +from typing import Any, Text from langflow import CustomComponent @@ -9,11 +9,20 @@ class UUIDGeneratorComponent(CustomComponent): display_name = "Unique ID Generator" description = "Generates a unique ID." - def generate(self, *args, **kwargs): - return Text(uuid.uuid4().hex) + def update_build_config( + self, build_config: dict, field_name: Text, field_value: Any + ): + if field_name == "unique_id": + build_config[field_name]["value"] = str(uuid.uuid4()) + return build_config def build_config(self): - return {"unique_id": {"display_name": "Value", "value": self.generate}} + return { + "unique_id": { + "display_name": "Value", + "refresh": True, + } + } def build(self, unique_id: str) -> str: return unique_id diff --git a/src/backend/langflow/components/memories/MessageHistory.py b/src/backend/langflow/components/helpers/MessageHistory.py similarity index 100% rename from src/backend/langflow/components/memories/MessageHistory.py rename to src/backend/langflow/components/helpers/MessageHistory.py diff --git a/src/backend/langflow/components/helpers/RecordsAsText.py b/src/backend/langflow/components/helpers/RecordsAsText.py index 18bf8be8c..f7750bdba 100644 --- a/src/backend/langflow/components/helpers/RecordsAsText.py +++ b/src/backend/langflow/components/helpers/RecordsAsText.py @@ -6,7 +6,7 @@ from langflow.schema import Record class RecordsAsTextComponent(CustomComponent): display_name = "Records to Text" - description = "Converts Records a list of Records to text using a template." + description = "Converts Records into single piece of text using a template." def build_config(self): return { @@ -16,7 +16,7 @@ class RecordsAsTextComponent(CustomComponent): }, "template": { "display_name": "Template", - "info": "The template to use for formatting the records. It must contain the keys {text} and {data}.", + "info": "The template to use for formatting the records. It can contain the keys {text}, {data} or any other key in the Record.", }, } diff --git a/src/backend/langflow/components/textsplitters/CharacterTextSplitter.py b/src/backend/langflow/components/textsplitters/CharacterTextSplitter.py index d165f47fd..96576a4a3 100644 --- a/src/backend/langflow/components/textsplitters/CharacterTextSplitter.py +++ b/src/backend/langflow/components/textsplitters/CharacterTextSplitter.py @@ -1,8 +1,9 @@ from typing import List from langchain.text_splitter import CharacterTextSplitter -from langchain_core.documents.base import Document + from langflow import CustomComponent +from langflow.schema.schema import Record class CharacterTextSplitterComponent(CustomComponent): @@ -11,7 +12,7 @@ class CharacterTextSplitterComponent(CustomComponent): def build_config(self): return { - "documents": {"display_name": "Documents"}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "chunk_overlap": {"display_name": "Chunk Overlap", "default": 200}, "chunk_size": {"display_name": "Chunk Size", "default": 1000}, "separator": {"display_name": "Separator", "default": "\n"}, @@ -19,17 +20,24 @@ class CharacterTextSplitterComponent(CustomComponent): def build( self, - documents: List[Document], + inputs: List[Record], chunk_overlap: int = 200, chunk_size: int = 1000, separator: str = "\n", - ) -> List[Document]: + ) -> List[Record]: # separator may come escaped from the frontend separator = separator.encode().decode("unicode_escape") + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) docs = CharacterTextSplitter( chunk_overlap=chunk_overlap, chunk_size=chunk_size, separator=separator, ).split_documents(documents) - self.status = docs - return docs + records = self.to_records(docs) + self.status = records + return records diff --git a/src/backend/langflow/components/textsplitters/LanguageRecursiveTextSplitter.py b/src/backend/langflow/components/textsplitters/LanguageRecursiveTextSplitter.py index d1494f4d0..b86b834d8 100644 --- a/src/backend/langflow/components/textsplitters/LanguageRecursiveTextSplitter.py +++ b/src/backend/langflow/components/textsplitters/LanguageRecursiveTextSplitter.py @@ -1,23 +1,22 @@ -from typing import Optional +from typing import List, Optional from langchain.text_splitter import Language -from langchain_core.documents import Document from langflow import CustomComponent +from langflow.schema.schema import Record class LanguageRecursiveTextSplitterComponent(CustomComponent): display_name: str = "Language Recursive Text Splitter" description: str = "Split text into chunks of a specified length based on language." - documentation: str = "https://docs.langflow.org/components/text-splitters#languagerecursivetextsplitter" + documentation: str = ( + "https://docs.langflow.org/components/text-splitters#languagerecursivetextsplitter" + ) def build_config(self): options = [x.value for x in Language] return { - "documents": { - "display_name": "Documents", - "info": "The documents to split.", - }, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "separator_type": { "display_name": "Separator Type", "info": "The type of separator to use.", @@ -47,11 +46,11 @@ class LanguageRecursiveTextSplitterComponent(CustomComponent): def build( self, - documents: list[Document], + inputs: List[Record], chunk_size: Optional[int] = 1000, chunk_overlap: Optional[int] = 200, separator_type: str = "Python", - ) -> list[Document]: + ) -> list[Record]: """ Split text into chunks of a specified length. @@ -77,6 +76,12 @@ class LanguageRecursiveTextSplitterComponent(CustomComponent): chunk_size=chunk_size, chunk_overlap=chunk_overlap, ) - + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) docs = splitter.split_documents(documents) - return docs + records = self.to_records(docs) + return records diff --git a/src/backend/langflow/components/textsplitters/RecursiveCharacterTextSplitter.py b/src/backend/langflow/components/textsplitters/RecursiveCharacterTextSplitter.py index d07ae3ebe..a15600394 100644 --- a/src/backend/langflow/components/textsplitters/RecursiveCharacterTextSplitter.py +++ b/src/backend/langflow/components/textsplitters/RecursiveCharacterTextSplitter.py @@ -1,22 +1,26 @@ from typing import Optional +from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_core.documents import Document from langflow import CustomComponent +from langflow.schema import Record from langflow.utils.util import build_loader_repr_from_documents -from langchain.text_splitter import RecursiveCharacterTextSplitter class RecursiveCharacterTextSplitterComponent(CustomComponent): display_name: str = "Recursive Character Text Splitter" description: str = "Split text into chunks of a specified length." - documentation: str = "https://docs.langflow.org/components/text-splitters#recursivecharactertextsplitter" + documentation: str = ( + "https://docs.langflow.org/components/text-splitters#recursivecharactertextsplitter" + ) def build_config(self): return { - "documents": { - "display_name": "Documents", - "info": "The documents to split.", + "inputs": { + "display_name": "Input", + "info": "The texts to split.", + "input_types": ["Document", "Record"], }, "separators": { "display_name": "Separators", @@ -40,11 +44,11 @@ class RecursiveCharacterTextSplitterComponent(CustomComponent): def build( self, - documents: list[Document], + inputs: list[Document], separators: Optional[list[str]] = None, chunk_size: Optional[int] = 1000, chunk_overlap: Optional[int] = 200, - ) -> list[Document]: + ) -> list[Record]: """ Split text into chunks of a specified length. @@ -75,7 +79,12 @@ class RecursiveCharacterTextSplitterComponent(CustomComponent): chunk_size=chunk_size, chunk_overlap=chunk_overlap, ) - + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) docs = splitter.split_documents(documents) self.repr_value = build_loader_repr_from_documents(docs) - return docs + return self.to_records(docs) diff --git a/src/backend/langflow/components/vectorstores/Chroma.py b/src/backend/langflow/components/vectorstores/Chroma.py index b1756e777..2a247e8d6 100644 --- a/src/backend/langflow/components/vectorstores/Chroma.py +++ b/src/backend/langflow/components/vectorstores/Chroma.py @@ -2,11 +2,12 @@ from typing import List, Optional, Union import chromadb # type: ignore from langchain.embeddings.base import Embeddings -from langchain.schema import BaseRetriever, Document +from langchain.schema import BaseRetriever from langchain_community.vectorstores import VectorStore from langchain_community.vectorstores.chroma import Chroma from langflow import CustomComponent +from langflow.schema.schema import Record class ChromaComponent(CustomComponent): @@ -31,7 +32,7 @@ class ChromaComponent(CustomComponent): "collection_name": {"display_name": "Collection Name", "value": "langflow"}, "index_directory": {"display_name": "Persist Directory"}, "code": {"advanced": True, "display_name": "Code"}, - "documents": {"display_name": "Documents", "is_list": True}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "chroma_server_cors_allow_origins": { "display_name": "Server CORS Allow Origins", @@ -55,7 +56,7 @@ class ChromaComponent(CustomComponent): embedding: Embeddings, chroma_server_ssl_enabled: bool, index_directory: Optional[str] = None, - documents: Optional[List[Document]] = None, + inputs: Optional[List[Record]] = None, chroma_server_cors_allow_origins: Optional[str] = None, chroma_server_host: Optional[str] = None, chroma_server_port: Optional[int] = None, @@ -84,7 +85,8 @@ class ChromaComponent(CustomComponent): if chroma_server_host is not None: chroma_settings = chromadb.config.Settings( - chroma_server_cors_allow_origins=chroma_server_cors_allow_origins or None, + chroma_server_cors_allow_origins=chroma_server_cors_allow_origins + or None, chroma_server_host=chroma_server_host, chroma_server_port=chroma_server_port or None, chroma_server_grpc_port=chroma_server_grpc_port or None, @@ -97,9 +99,17 @@ class ChromaComponent(CustomComponent): if index_directory is not None: index_directory = self.resolve_path(index_directory) + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) if documents is not None and embedding is not None: if len(documents) == 0: - raise ValueError("If documents are provided, there must be at least one document.") + raise ValueError( + "If documents are provided, there must be at least one document." + ) chroma = Chroma.from_documents( documents=documents, # type: ignore persist_directory=index_directory, diff --git a/src/backend/langflow/components/vectorstores/ChromaSearch.py b/src/backend/langflow/components/vectorstores/ChromaSearch.py index 3a6d283b3..cf98ee987 100644 --- a/src/backend/langflow/components/vectorstores/ChromaSearch.py +++ b/src/backend/langflow/components/vectorstores/ChromaSearch.py @@ -35,7 +35,6 @@ class ChromaSearchComponent(LCVectorStoreComponent): # "persist": {"display_name": "Persist"}, "index_directory": {"display_name": "Index Directory"}, "code": {"show": False, "display_name": "Code"}, - "documents": {"display_name": "Documents", "is_list": True}, "embedding": { "display_name": "Embedding", "info": "Embedding model to vectorize inputs (make sure to use same as index)", diff --git a/src/backend/langflow/components/vectorstores/FAISS.py b/src/backend/langflow/components/vectorstores/FAISS.py index a0324456e..7cdadccdb 100644 --- a/src/backend/langflow/components/vectorstores/FAISS.py +++ b/src/backend/langflow/components/vectorstores/FAISS.py @@ -5,7 +5,8 @@ from langchain_community.vectorstores import VectorStore from langchain_community.vectorstores.faiss import FAISS from langflow import CustomComponent -from langflow.field_typing import Document, Embeddings +from langflow.field_typing import Embeddings +from langflow.schema.schema import Record class FAISSComponent(CustomComponent): @@ -15,7 +16,7 @@ class FAISSComponent(CustomComponent): def build_config(self): return { - "documents": {"display_name": "Documents"}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "folder_path": { "display_name": "Folder Path", @@ -27,10 +28,16 @@ class FAISSComponent(CustomComponent): def build( self, embedding: Embeddings, - documents: List[Document], + inputs: List[Record], folder_path: str, index_name: str = "langflow_index", ) -> Union[VectorStore, FAISS, BaseRetriever]: + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) vector_store = FAISS.from_documents(documents=documents, embedding=embedding) if not folder_path: raise ValueError("Folder path is required to save the FAISS index.") diff --git a/src/backend/langflow/components/vectorstores/FAISSSearch.py b/src/backend/langflow/components/vectorstores/FAISSSearch.py index f6ddf4f7a..4544c300b 100644 --- a/src/backend/langflow/components/vectorstores/FAISSSearch.py +++ b/src/backend/langflow/components/vectorstores/FAISSSearch.py @@ -14,7 +14,6 @@ class FAISSSearchComponent(LCVectorStoreComponent): def build_config(self): return { - "documents": {"display_name": "Documents"}, "embedding": {"display_name": "Embedding"}, "folder_path": { "display_name": "Folder Path", diff --git a/src/backend/langflow/components/vectorstores/MongoDBAtlasVector.py b/src/backend/langflow/components/vectorstores/MongoDBAtlasVector.py index e15368f7d..4488e4e92 100644 --- a/src/backend/langflow/components/vectorstores/MongoDBAtlasVector.py +++ b/src/backend/langflow/components/vectorstores/MongoDBAtlasVector.py @@ -3,17 +3,20 @@ from typing import List, Optional from langchain_community.vectorstores.mongodb_atlas import MongoDBAtlasVectorSearch from langflow import CustomComponent -from langflow.field_typing import Document, Embeddings, NestedDict +from langflow.field_typing import Embeddings, NestedDict +from langflow.schema.schema import Record class MongoDBAtlasComponent(CustomComponent): display_name = "MongoDB Atlas" - description = "Construct a `MongoDB Atlas Vector Search` vector store from raw documents." + description = ( + "Construct a `MongoDB Atlas Vector Search` vector store from raw documents." + ) icon = "MongoDB" def build_config(self): return { - "documents": {"display_name": "Documents"}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "collection_name": {"display_name": "Collection Name"}, "db_name": {"display_name": "Database Name"}, @@ -25,7 +28,7 @@ class MongoDBAtlasComponent(CustomComponent): def build( self, embedding: Embeddings, - documents: List[Document], + inputs: List[Record], collection_name: str = "", db_name: str = "", index_name: str = "", @@ -36,12 +39,20 @@ class MongoDBAtlasComponent(CustomComponent): try: from pymongo import MongoClient except ImportError: - raise ImportError("Please install pymongo to use MongoDB Atlas Vector Store") + raise ImportError( + "Please install pymongo to use MongoDB Atlas Vector Store" + ) try: mongo_client: MongoClient = MongoClient(mongodb_atlas_cluster_uri) collection = mongo_client[db_name][collection_name] except Exception as e: raise ValueError(f"Failed to connect to MongoDB Atlas: {e}") + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) if documents: vector_store = MongoDBAtlasVectorSearch.from_documents( documents=documents, diff --git a/src/backend/langflow/components/vectorstores/Pinecone.py b/src/backend/langflow/components/vectorstores/Pinecone.py index 54222b133..c71048266 100644 --- a/src/backend/langflow/components/vectorstores/Pinecone.py +++ b/src/backend/langflow/components/vectorstores/Pinecone.py @@ -7,7 +7,8 @@ from langchain_community.vectorstores import VectorStore from langchain_community.vectorstores.pinecone import Pinecone from langflow import CustomComponent -from langflow.field_typing import Document, Embeddings +from langflow.field_typing import Embeddings +from langflow.schema.schema import Record class PineconeComponent(CustomComponent): @@ -17,7 +18,7 @@ class PineconeComponent(CustomComponent): def build_config(self): return { - "documents": {"display_name": "Documents"}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "index_name": {"display_name": "Index Name"}, "namespace": {"display_name": "Namespace"}, @@ -44,7 +45,7 @@ class PineconeComponent(CustomComponent): self, embedding: Embeddings, pinecone_env: str, - documents: List[Document], + inputs: List[Record], text_key: str = "text", pool_threads: int = 4, index_name: Optional[str] = None, @@ -59,6 +60,12 @@ class PineconeComponent(CustomComponent): pinecone.init(api_key=pinecone_api_key, environment=pinecone_env) # type: ignore if not index_name: raise ValueError("Index Name is required.") + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) if documents: return Pinecone.from_documents( documents=documents, diff --git a/src/backend/langflow/components/vectorstores/Qdrant.py b/src/backend/langflow/components/vectorstores/Qdrant.py index 23ee70b11..e1773268b 100644 --- a/src/backend/langflow/components/vectorstores/Qdrant.py +++ b/src/backend/langflow/components/vectorstores/Qdrant.py @@ -3,8 +3,10 @@ from typing import Optional, Union from langchain.schema import BaseRetriever from langchain_community.vectorstores import VectorStore from langchain_community.vectorstores.qdrant import Qdrant + from langflow import CustomComponent -from langflow.field_typing import Document, Embeddings, NestedDict +from langflow.field_typing import Embeddings, NestedDict +from langflow.schema.schema import Record class QdrantComponent(CustomComponent): @@ -14,17 +16,23 @@ class QdrantComponent(CustomComponent): def build_config(self): return { - "documents": {"display_name": "Documents"}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "api_key": {"display_name": "API Key", "password": True, "advanced": True}, "collection_name": {"display_name": "Collection Name"}, - "content_payload_key": {"display_name": "Content Payload Key", "advanced": True}, + "content_payload_key": { + "display_name": "Content Payload Key", + "advanced": True, + }, "distance_func": {"display_name": "Distance Function", "advanced": True}, "grpc_port": {"display_name": "gRPC Port", "advanced": True}, "host": {"display_name": "Host", "advanced": True}, "https": {"display_name": "HTTPS", "advanced": True}, "location": {"display_name": "Location", "advanced": True}, - "metadata_payload_key": {"display_name": "Metadata Payload Key", "advanced": True}, + "metadata_payload_key": { + "display_name": "Metadata Payload Key", + "advanced": True, + }, "path": {"display_name": "Path", "advanced": True}, "port": {"display_name": "Port", "advanced": True}, "prefer_grpc": {"display_name": "Prefer gRPC", "advanced": True}, @@ -38,7 +46,7 @@ class QdrantComponent(CustomComponent): self, embedding: Embeddings, collection_name: str, - documents: Optional[Document] = None, + inputs: Optional[Record] = None, api_key: Optional[str] = None, content_payload_key: str = "page_content", distance_func: str = "Cosine", @@ -55,6 +63,12 @@ class QdrantComponent(CustomComponent): timeout: Optional[int] = None, url: Optional[str] = None, ) -> Union[VectorStore, Qdrant, BaseRetriever]: + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) if documents is None: from qdrant_client import QdrantClient diff --git a/src/backend/langflow/components/vectorstores/Redis.py b/src/backend/langflow/components/vectorstores/Redis.py index b2d7e4542..bbf04f1a4 100644 --- a/src/backend/langflow/components/vectorstores/Redis.py +++ b/src/backend/langflow/components/vectorstores/Redis.py @@ -3,9 +3,10 @@ from typing import Optional, Union from langchain.embeddings.base import Embeddings from langchain_community.vectorstores import VectorStore from langchain_community.vectorstores.redis import Redis -from langchain_core.documents import Document from langchain_core.retrievers import BaseRetriever + from langflow import CustomComponent +from langflow.schema.schema import Record class RedisComponent(CustomComponent): @@ -28,7 +29,7 @@ class RedisComponent(CustomComponent): return { "index_name": {"display_name": "Index Name", "value": "your_index"}, "code": {"show": False, "display_name": "Code"}, - "documents": {"display_name": "Documents", "is_list": True}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "schema": {"display_name": "Schema", "file_types": [".yaml"]}, "redis_server_url": { @@ -44,7 +45,7 @@ class RedisComponent(CustomComponent): redis_server_url: str, redis_index_name: str, schema: Optional[str] = None, - documents: Optional[Document] = None, + inputs: Optional[Record] = None, ) -> Union[VectorStore, BaseRetriever]: """ Builds the Vector Store or BaseRetriever object. @@ -58,9 +59,17 @@ class RedisComponent(CustomComponent): Returns: - VectorStore: The Vector Store object. """ - if documents is None: + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) + if not documents: if schema is None: - raise ValueError("If no documents are provided, a schema must be provided.") + raise ValueError( + "If no documents are provided, a schema must be provided." + ) redis_vs = Redis.from_existing_index( embedding=embedding, index_name=redis_index_name, diff --git a/src/backend/langflow/components/vectorstores/RedisSearch.py b/src/backend/langflow/components/vectorstores/RedisSearch.py index 4089d4f47..63fc46ddb 100644 --- a/src/backend/langflow/components/vectorstores/RedisSearch.py +++ b/src/backend/langflow/components/vectorstores/RedisSearch.py @@ -33,7 +33,7 @@ class RedisSearchComponent(RedisComponent, LCVectorStoreComponent): "input_value": {"display_name": "Input"}, "index_name": {"display_name": "Index Name", "value": "your_index"}, "code": {"show": False, "display_name": "Code"}, - "documents": {"display_name": "Documents", "is_list": True}, + "embedding": {"display_name": "Embedding"}, "schema": {"display_name": "Schema", "file_types": [".yaml"]}, "redis_server_url": { diff --git a/src/backend/langflow/components/vectorstores/SupabaseVectorStore.py b/src/backend/langflow/components/vectorstores/SupabaseVectorStore.py index 2ec6dfabc..7f6296c03 100644 --- a/src/backend/langflow/components/vectorstores/SupabaseVectorStore.py +++ b/src/backend/langflow/components/vectorstores/SupabaseVectorStore.py @@ -3,10 +3,12 @@ from typing import List, Union from langchain.schema import BaseRetriever from langchain_community.vectorstores import VectorStore from langchain_community.vectorstores.supabase import SupabaseVectorStore -from langflow import CustomComponent -from langflow.field_typing import Document, Embeddings, NestedDict from supabase.client import Client, create_client +from langflow import CustomComponent +from langflow.field_typing import Embeddings, NestedDict +from langflow.schema.schema import Record + class SupabaseComponent(CustomComponent): display_name = "Supabase" @@ -14,7 +16,7 @@ class SupabaseComponent(CustomComponent): def build_config(self): return { - "documents": {"display_name": "Documents"}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "query_name": {"display_name": "Query Name"}, "search_kwargs": {"display_name": "Search Kwargs", "advanced": True}, @@ -26,14 +28,22 @@ class SupabaseComponent(CustomComponent): def build( self, embedding: Embeddings, - documents: List[Document], + inputs: List[Record], query_name: str = "", search_kwargs: NestedDict = {}, supabase_service_key: str = "", supabase_url: str = "", table_name: str = "", ) -> Union[VectorStore, SupabaseVectorStore, BaseRetriever]: - supabase: Client = create_client(supabase_url, supabase_key=supabase_service_key) + supabase: Client = create_client( + supabase_url, supabase_key=supabase_service_key + ) + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) return SupabaseVectorStore.from_documents( documents=documents, embedding=embedding, diff --git a/src/backend/langflow/components/vectorstores/Vectara.py b/src/backend/langflow/components/vectorstores/Vectara.py index 0a396918c..8f1f7250e 100644 --- a/src/backend/langflow/components/vectorstores/Vectara.py +++ b/src/backend/langflow/components/vectorstores/Vectara.py @@ -8,13 +8,16 @@ from langchain_community.vectorstores.vectara import Vectara from langchain_core.vectorstores import VectorStore from langflow import CustomComponent -from langflow.field_typing import BaseRetriever, Document +from langflow.field_typing import BaseRetriever +from langflow.schema.schema import Record class VectaraComponent(CustomComponent): display_name: str = "Vectara" description: str = "Implementation of Vector Store using Vectara" - documentation = "https://python.langchain.com/docs/integrations/vectorstores/vectara" + documentation = ( + "https://python.langchain.com/docs/integrations/vectorstores/vectara" + ) beta = True icon = "Vectara" field_config = { @@ -28,8 +31,9 @@ class VectaraComponent(CustomComponent): "display_name": "Vectara API Key", "password": True, }, - "documents": { - "display_name": "Documents", + "inputs": { + "display_name": "Input", + "input_types": ["Document", "Record"], "info": "If provided, will be upserted to corpus (optional)", }, "files_url": { @@ -44,11 +48,18 @@ class VectaraComponent(CustomComponent): vectara_corpus_id: str, vectara_api_key: str, files_url: Optional[List[str]] = None, - documents: Optional[Document] = None, + inputs: Optional[Record] = None, ) -> Union[VectorStore, BaseRetriever]: source = "Langflow" - if documents is not None: + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) + + if documents: return Vectara.from_documents( documents=documents, # type: ignore embedding=FakeEmbeddings(size=768), diff --git a/src/backend/langflow/components/vectorstores/VectaraSearch.py b/src/backend/langflow/components/vectorstores/VectaraSearch.py index ae2d442be..cbc876f86 100644 --- a/src/backend/langflow/components/vectorstores/VectaraSearch.py +++ b/src/backend/langflow/components/vectorstores/VectaraSearch.py @@ -33,10 +33,6 @@ class VectaraSearchComponent(VectaraComponent, LCVectorStoreComponent): "display_name": "Vectara API Key", "password": True, }, - "documents": { - "display_name": "Documents", - "info": "If provided, will be upserted to corpus (optional)", - }, "files_url": { "display_name": "Files Url", "info": "Make vectara object using url of files (optional)", diff --git a/src/backend/langflow/components/vectorstores/Weaviate.py b/src/backend/langflow/components/vectorstores/Weaviate.py index 3d804255a..a85febcd5 100644 --- a/src/backend/langflow/components/vectorstores/Weaviate.py +++ b/src/backend/langflow/components/vectorstores/Weaviate.py @@ -2,16 +2,19 @@ from typing import Optional, Union import weaviate # type: ignore from langchain.embeddings.base import Embeddings -from langchain.schema import BaseRetriever, Document +from langchain.schema import BaseRetriever from langchain_community.vectorstores import VectorStore, Weaviate from langflow import CustomComponent +from langflow.schema.schema import Record class WeaviateVectorStoreComponent(CustomComponent): display_name: str = "Weaviate" description: str = "Implementation of Vector Store using Weaviate" - documentation = "https://python.langchain.com/docs/integrations/vectorstores/weaviate" + documentation = ( + "https://python.langchain.com/docs/integrations/vectorstores/weaviate" + ) beta = True field_config = { "url": {"display_name": "Weaviate URL", "value": "http://localhost:8080"}, @@ -30,7 +33,7 @@ class WeaviateVectorStoreComponent(CustomComponent): "advanced": True, "value": "text", }, - "documents": {"display_name": "Documents", "is_list": True}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "attributes": { "display_name": "Attributes", @@ -55,7 +58,7 @@ class WeaviateVectorStoreComponent(CustomComponent): index_name: Optional[str] = None, text_key: str = "text", embedding: Optional[Embeddings] = None, - documents: Optional[Document] = None, + inputs: Optional[Record] = None, attributes: Optional[list] = None, ) -> Union[VectorStore, BaseRetriever]: if api_key: @@ -78,8 +81,14 @@ class WeaviateVectorStoreComponent(CustomComponent): return pascal_case_word index_name = _to_pascal_case(index_name) if index_name else None + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) - if documents is not None and embedding is not None: + if documents and embedding is not None: return Weaviate.from_documents( client=client, index_name=index_name, diff --git a/src/backend/langflow/components/vectorstores/WeaviateSearch.py b/src/backend/langflow/components/vectorstores/WeaviateSearch.py index 6eee202c9..9b14ca779 100644 --- a/src/backend/langflow/components/vectorstores/WeaviateSearch.py +++ b/src/backend/langflow/components/vectorstores/WeaviateSearch.py @@ -39,7 +39,6 @@ class WeaviateSearchVectorStore(WeaviateVectorStoreComponent, LCVectorStoreCompo "advanced": True, "value": "text", }, - "documents": {"display_name": "Documents", "is_list": True}, "embedding": {"display_name": "Embedding"}, "attributes": { "display_name": "Attributes", diff --git a/src/backend/langflow/components/vectorstores/pgvector.py b/src/backend/langflow/components/vectorstores/pgvector.py index 2baf6dae6..e3b528313 100644 --- a/src/backend/langflow/components/vectorstores/pgvector.py +++ b/src/backend/langflow/components/vectorstores/pgvector.py @@ -3,9 +3,10 @@ from typing import Optional, Union from langchain.embeddings.base import Embeddings from langchain_community.vectorstores import VectorStore from langchain_community.vectorstores.pgvector import PGVector -from langchain_core.documents import Document from langchain_core.retrievers import BaseRetriever + from langflow import CustomComponent +from langflow.schema.schema import Record class PGVectorComponent(CustomComponent): @@ -15,7 +16,9 @@ class PGVectorComponent(CustomComponent): display_name: str = "PGVector" description: str = "Implementation of Vector Store using PostgreSQL" - documentation = "https://python.langchain.com/docs/integrations/vectorstores/pgvector" + documentation = ( + "https://python.langchain.com/docs/integrations/vectorstores/pgvector" + ) def build_config(self): """ @@ -26,7 +29,7 @@ class PGVectorComponent(CustomComponent): """ return { "code": {"show": False}, - "documents": {"display_name": "Documents", "is_list": True}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "pg_server_url": { "display_name": "PostgreSQL Server Connection String", @@ -40,7 +43,7 @@ class PGVectorComponent(CustomComponent): embedding: Embeddings, pg_server_url: str, collection_name: str, - documents: Optional[Document] = None, + inputs: Optional[Record] = None, ) -> Union[VectorStore, BaseRetriever]: """ Builds the Vector Store or BaseRetriever object. @@ -55,6 +58,12 @@ class PGVectorComponent(CustomComponent): - VectorStore: The Vector Store object. """ + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) try: if documents is None: vector_store = PGVector.from_existing_index( diff --git a/src/backend/langflow/components/io/__init__.py b/src/backend/langflow/initial_setup/__init__.py similarity index 100% rename from src/backend/langflow/components/io/__init__.py rename to src/backend/langflow/initial_setup/__init__.py diff --git a/src/backend/langflow/initial_setup/setup.py b/src/backend/langflow/initial_setup/setup.py new file mode 100644 index 000000000..1f24ddab1 --- /dev/null +++ b/src/backend/langflow/initial_setup/setup.py @@ -0,0 +1,115 @@ +from datetime import datetime +from pathlib import Path + +import orjson +from loguru import logger +from sqlmodel import select + +from langflow.services.database.models.flow.model import Flow +from langflow.services.deps import session_scope + +STARTER_FOLDER_NAME = "Starter Projects" + + +# In the folder ./starter_projects we have a few JSON files that represent +# starter projects. We want to load these into the database so that users +# can use them as a starting point for their own projects. + + +def load_starter_projects(): + starter_projects = [] + folder = Path(__file__).parent / "starter_projects" + for file in folder.glob("*.json"): + project = orjson.loads(file.read_text()) + starter_projects.append(project) + logger.info(f"Loaded starter project {file}") + return starter_projects + + +def get_project_data(project): + project_name = project.get("name") + project_description = project.get("description") + project_is_component = project.get("is_component") + project_updated_at = project.get("updated_at") + updated_at_datetime = datetime.strptime(project_updated_at, "%Y-%m-%dT%H:%M:%S.%f") + project_data = project.get("data") + return ( + project_name, + project_description, + project_is_component, + updated_at_datetime, + project_data, + ) + + +def update_existing_project( + existing_project, + project_name, + project_description, + project_is_component, + updated_at_datetime, + project_data, +): + logger.info(f"Updating starter project {project_name}") + existing_project.data = project_data + existing_project.folder = STARTER_FOLDER_NAME + existing_project.description = project_description + existing_project.is_component = project_is_component + existing_project.updated_at = updated_at_datetime + + +def create_new_project( + session, + project_name, + project_description, + project_is_component, + updated_at_datetime, + project_data, +): + logger.info(f"Creating starter project {project_name}") + new_project = Flow( + name=project_name, + description=project_description, + is_component=project_is_component, + updated_at=updated_at_datetime, + folder=STARTER_FOLDER_NAME, + data=project_data, + ) + session.add(new_project) + + +def create_or_update_starter_projects(): + with session_scope() as session: + starter_projects = load_starter_projects() + for project in starter_projects: + ( + project_name, + project_description, + project_is_component, + updated_at_datetime, + project_data, + ) = get_project_data(project) + if project_name and project_data: + existing_project = session.exec( + select(Flow).where( + Flow.name == project_name, Flow.folder == STARTER_FOLDER_NAME + ) + ).first() + if existing_project: + update_existing_project( + existing_project, + project_name, + project_description, + project_is_component, + updated_at_datetime, + project_data, + ) + else: + create_new_project( + session, + project_name, + project_description, + project_is_component, + updated_at_datetime, + project_data, + ) diff --git a/src/backend/langflow/initial_setup/starter_projects/Langflow Data Ingestion.json b/src/backend/langflow/initial_setup/starter_projects/Langflow Data Ingestion.json new file mode 100644 index 000000000..de8a8c1df --- /dev/null +++ b/src/backend/langflow/initial_setup/starter_projects/Langflow Data Ingestion.json @@ -0,0 +1,1085 @@ +{ + "name": "Data Ingestion", + "description": "This project is the starting point to insert data into a Vector Store. \n\nWe use the Vector Store Chroma but you can replace it with any other Vector Store. \n\nYou start by deciding what type of data you want to load, then you pick a place where you want to store the vectors and run it.\n\nThis will create a vector store in your local environment which you can query using the Chroma Search component.", + "data": { + "nodes": [ + { + "id": "RecursiveCharacterTextSplitter-jwfyG", + "type": "genericNode", + "position": { + "x": 1042.4388767006992, + "y": 633.2204634490822 + }, + "data": { + "type": "RecursiveCharacterTextSplitter", + "node": { + "template": { + "inputs": { + "type": "Document", + "required": true, + "placeholder": "", + "list": true, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "inputs", + "display_name": "Input", + "advanced": false, + "input_types": [ + "Document", + "Record" + ], + "dynamic": false, + "info": "The texts to split.", + "title_case": false + }, + "chunk_overlap": { + "type": "int", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": 200, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chunk_overlap", + "display_name": "Chunk Overlap", + "advanced": false, + "dynamic": false, + "info": "The amount of overlap between chunks.", + "title_case": false + }, + "chunk_size": { + "type": "int", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": 1000, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chunk_size", + "display_name": "Chunk Size", + "advanced": false, + "dynamic": false, + "info": "The maximum length of each chunk.", + "title_case": false + }, + "code": { + "type": "code", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": true, + "value": "from typing import Optional\n\nfrom langchain.text_splitter import RecursiveCharacterTextSplitter\nfrom langchain_core.documents import Document\n\nfrom langflow import CustomComponent\nfrom langflow.schema import Record\nfrom langflow.utils.util import build_loader_repr_from_documents\n\n\nclass RecursiveCharacterTextSplitterComponent(CustomComponent):\n display_name: str = \"Recursive Character Text Splitter\"\n description: str = \"Split text into chunks of a specified length.\"\n documentation: str = (\n \"https://docs.langflow.org/components/text-splitters#recursivecharactertextsplitter\"\n )\n\n def build_config(self):\n return {\n \"inputs\": {\n \"display_name\": \"Input\",\n \"info\": \"The texts to split.\",\n \"input_types\": [\"Document\", \"Record\"],\n },\n \"separators\": {\n \"display_name\": \"Separators\",\n \"info\": 'The characters to split on.\\nIf left empty defaults to [\"\\\\n\\\\n\", \"\\\\n\", \" \", \"\"].',\n \"is_list\": True,\n },\n \"chunk_size\": {\n \"display_name\": \"Chunk Size\",\n \"info\": \"The maximum length of each chunk.\",\n \"field_type\": \"int\",\n \"value\": 1000,\n },\n \"chunk_overlap\": {\n \"display_name\": \"Chunk Overlap\",\n \"info\": \"The amount of overlap between chunks.\",\n \"field_type\": \"int\",\n \"value\": 200,\n },\n \"code\": {\"show\": False},\n }\n\n def build(\n self,\n inputs: list[Document],\n separators: Optional[list[str]] = None,\n chunk_size: Optional[int] = 1000,\n chunk_overlap: Optional[int] = 200,\n ) -> list[Record]:\n \"\"\"\n Split text into chunks of a specified length.\n\n Args:\n separators (list[str]): The characters to split on.\n chunk_size (int): The maximum length of each chunk.\n chunk_overlap (int): The amount of overlap between chunks.\n length_function (function): The function to use to calculate the length of the text.\n\n Returns:\n list[str]: The chunks of text.\n \"\"\"\n\n if separators == \"\":\n separators = None\n elif separators:\n # check if the separators list has escaped characters\n # if there are escaped characters, unescape them\n separators = [x.encode().decode(\"unicode-escape\") for x in separators]\n\n # Make sure chunk_size and chunk_overlap are ints\n if isinstance(chunk_size, str):\n chunk_size = int(chunk_size)\n if isinstance(chunk_overlap, str):\n chunk_overlap = int(chunk_overlap)\n splitter = RecursiveCharacterTextSplitter(\n separators=separators,\n chunk_size=chunk_size,\n chunk_overlap=chunk_overlap,\n )\n documents = []\n for _input in inputs:\n if isinstance(_input, Record):\n documents.append(_input.to_lc_document())\n else:\n documents.append(_input)\n docs = splitter.split_documents(documents)\n self.repr_value = build_loader_repr_from_documents(docs)\n return self.to_records(docs)\n", + "fileTypes": [], + "file_path": "", + "password": false, + "name": "code", + "advanced": false, + "dynamic": true, + "info": "", + "title_case": false + }, + "separators": { + "type": "str", + "required": false, + "placeholder": "", + "list": true, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "separators", + "display_name": "Separators", + "advanced": false, + "dynamic": false, + "info": "The characters to split on.\nIf left empty defaults to [\"\\n\\n\", \"\\n\", \" \", \"\"].", + "title_case": false, + "input_types": [ + "Text" + ], + "value": [ + "\\n" + ] + }, + "_type": "CustomComponent" + }, + "description": "Split text into chunks of a specified length.", + "base_classes": [ + "Record" + ], + "display_name": "Recursive Character Text Splitter", + "documentation": "https://docs.langflow.org/components/text-splitters#recursivecharactertextsplitter", + "custom_fields": { + "inputs": null, + "separators": null, + "chunk_size": null, + "chunk_overlap": null + }, + "output_types": [ + "Record" + ], + "field_formatters": {}, + "frozen": false, + "beta": true + }, + "id": "RecursiveCharacterTextSplitter-jwfyG" + }, + "selected": false, + "width": 384, + "height": 509, + "positionAbsolute": { + "x": 1042.4388767006992, + "y": 633.2204634490822 + }, + "dragging": false + }, + { + "id": "Chroma-aFGHF", + "type": "genericNode", + "position": { + "x": 1641.280676720732, + "y": 356.94961598422196 + }, + "data": { + "type": "Chroma", + "node": { + "template": { + "embedding": { + "type": "Embeddings", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "embedding", + "display_name": "Embedding", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false + }, + "inputs": { + "type": "Record", + "required": false, + "placeholder": "", + "list": true, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "inputs", + "display_name": "Input", + "advanced": false, + "input_types": [ + "Document", + "Record" + ], + "dynamic": false, + "info": "", + "title_case": false + }, + "chroma_server_cors_allow_origins": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chroma_server_cors_allow_origins", + "display_name": "Server CORS Allow Origins", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "chroma_server_grpc_port": { + "type": "int", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chroma_server_grpc_port", + "display_name": "Server gRPC Port", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "chroma_server_host": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chroma_server_host", + "display_name": "Server Host", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "chroma_server_port": { + "type": "int", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chroma_server_port", + "display_name": "Server Port", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "chroma_server_ssl_enabled": { + "type": "bool", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chroma_server_ssl_enabled", + "display_name": "Server SSL Enabled", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "code": { + "type": "code", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": true, + "value": "from typing import List, Optional, Union\n\nimport chromadb # type: ignore\nfrom langchain.embeddings.base import Embeddings\nfrom langchain.schema import BaseRetriever\nfrom langchain_community.vectorstores import VectorStore\nfrom langchain_community.vectorstores.chroma import Chroma\n\nfrom langflow import CustomComponent\nfrom langflow.schema.schema import Record\n\n\nclass ChromaComponent(CustomComponent):\n \"\"\"\n A custom component for implementing a Vector Store using Chroma.\n \"\"\"\n\n display_name: str = \"Chroma\"\n description: str = \"Implementation of Vector Store using Chroma\"\n documentation = \"https://python.langchain.com/docs/integrations/vectorstores/chroma\"\n beta: bool = True\n icon = \"Chroma\"\n\n def build_config(self):\n \"\"\"\n Builds the configuration for the component.\n\n Returns:\n - dict: A dictionary containing the configuration options for the component.\n \"\"\"\n return {\n \"collection_name\": {\"display_name\": \"Collection Name\", \"value\": \"langflow\"},\n \"index_directory\": {\"display_name\": \"Persist Directory\"},\n \"code\": {\"advanced\": True, \"display_name\": \"Code\"},\n \"inputs\": {\"display_name\": \"Input\", \"input_types\": [\"Document\", \"Record\"]},\n \"embedding\": {\"display_name\": \"Embedding\"},\n \"chroma_server_cors_allow_origins\": {\n \"display_name\": \"Server CORS Allow Origins\",\n \"advanced\": True,\n },\n \"chroma_server_host\": {\"display_name\": \"Server Host\", \"advanced\": True},\n \"chroma_server_port\": {\"display_name\": \"Server Port\", \"advanced\": True},\n \"chroma_server_grpc_port\": {\n \"display_name\": \"Server gRPC Port\",\n \"advanced\": True,\n },\n \"chroma_server_ssl_enabled\": {\n \"display_name\": \"Server SSL Enabled\",\n \"advanced\": True,\n },\n }\n\n def build(\n self,\n collection_name: str,\n embedding: Embeddings,\n chroma_server_ssl_enabled: bool,\n index_directory: Optional[str] = None,\n inputs: Optional[List[Record]] = None,\n chroma_server_cors_allow_origins: Optional[str] = None,\n chroma_server_host: Optional[str] = None,\n chroma_server_port: Optional[int] = None,\n chroma_server_grpc_port: Optional[int] = None,\n ) -> Union[VectorStore, BaseRetriever]:\n \"\"\"\n Builds the Vector Store or BaseRetriever object.\n\n Args:\n - collection_name (str): The name of the collection.\n - index_directory (Optional[str]): The directory to persist the Vector Store to.\n - chroma_server_ssl_enabled (bool): Whether to enable SSL for the Chroma server.\n - embedding (Optional[Embeddings]): The embeddings to use for the Vector Store.\n - documents (Optional[Document]): The documents to use for the Vector Store.\n - chroma_server_cors_allow_origins (Optional[str]): The CORS allow origins for the Chroma server.\n - chroma_server_host (Optional[str]): The host for the Chroma server.\n - chroma_server_port (Optional[int]): The port for the Chroma server.\n - chroma_server_grpc_port (Optional[int]): The gRPC port for the Chroma server.\n\n Returns:\n - Union[VectorStore, BaseRetriever]: The Vector Store or BaseRetriever object.\n \"\"\"\n\n # Chroma settings\n chroma_settings = None\n\n if chroma_server_host is not None:\n chroma_settings = chromadb.config.Settings(\n chroma_server_cors_allow_origins=chroma_server_cors_allow_origins\n or None,\n chroma_server_host=chroma_server_host,\n chroma_server_port=chroma_server_port or None,\n chroma_server_grpc_port=chroma_server_grpc_port or None,\n chroma_server_ssl_enabled=chroma_server_ssl_enabled,\n )\n\n # If documents, then we need to create a Chroma instance using .from_documents\n\n # Check index_directory and expand it if it is a relative path\n if index_directory is not None:\n index_directory = self.resolve_path(index_directory)\n\n documents = []\n for _input in inputs:\n if isinstance(_input, Record):\n documents.append(_input.to_lc_document())\n else:\n documents.append(_input)\n if documents is not None and embedding is not None:\n if len(documents) == 0:\n raise ValueError(\n \"If documents are provided, there must be at least one document.\"\n )\n chroma = Chroma.from_documents(\n documents=documents, # type: ignore\n persist_directory=index_directory,\n collection_name=collection_name,\n embedding=embedding,\n client_settings=chroma_settings,\n )\n else:\n chroma = Chroma(\n persist_directory=index_directory,\n client_settings=chroma_settings,\n embedding_function=embedding,\n )\n return chroma\n", + "fileTypes": [], + "file_path": "", + "password": false, + "name": "code", + "advanced": true, + "dynamic": true, + "info": "", + "title_case": false + }, + "collection_name": { + "type": "str", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": "langflow_contrib", + "fileTypes": [], + "file_path": "", + "password": false, + "name": "collection_name", + "display_name": "Collection Name", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "index_directory": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "index_directory", + "display_name": "Persist Directory", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ], + "value": "./chroma_langflow" + }, + "_type": "CustomComponent" + }, + "description": "Implementation of Vector Store using Chroma", + "icon": "Chroma", + "base_classes": [ + "Serializable", + "VectorStore", + "object", + "Runnable", + "BaseRetriever", + "RunnableSerializable", + "Generic" + ], + "display_name": "Chroma", + "documentation": "https://python.langchain.com/docs/integrations/vectorstores/chroma", + "custom_fields": { + "collection_name": null, + "embedding": null, + "chroma_server_ssl_enabled": null, + "index_directory": null, + "inputs": null, + "chroma_server_cors_allow_origins": null, + "chroma_server_host": null, + "chroma_server_port": null, + "chroma_server_grpc_port": null + }, + "output_types": [ + "VectorStore", + "BaseRetriever" + ], + "field_formatters": {}, + "frozen": false, + "beta": true + }, + "id": "Chroma-aFGHF" + }, + "selected": true, + "width": 384, + "height": 495, + "positionAbsolute": { + "x": 1641.280676720732, + "y": 356.94961598422196 + }, + "dragging": false + }, + { + "id": "OpenAIEmbeddings-rbMk3", + "type": "genericNode", + "position": { + "x": 1053.9472627140208, + "y": -2.5921878249999963 + }, + "data": { + "type": "OpenAIEmbeddings", + "node": { + "template": { + "allowed_special": { + "type": "str", + "required": true, + "placeholder": "", + "list": true, + "show": true, + "multiline": false, + "value": [], + "fileTypes": [], + "file_path": "", + "password": false, + "name": "allowed_special", + "display_name": "Allowed Special", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "chunk_size": { + "type": "int", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": 1000, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chunk_size", + "display_name": "Chunk Size", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "client": { + "type": "Any", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "client", + "display_name": "Client", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "code": { + "type": "code", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": true, + "value": "from typing import Any, Callable, Dict, List, Optional, Union\n\nfrom langchain_openai.embeddings.base import OpenAIEmbeddings\nfrom langflow import CustomComponent\nfrom langflow.field_typing import NestedDict\nfrom pydantic.v1.types import SecretStr\n\n\nclass OpenAIEmbeddingsComponent(CustomComponent):\n display_name = \"OpenAIEmbeddings\"\n description = \"OpenAI embedding models\"\n\n def build_config(self):\n return {\n \"allowed_special\": {\n \"display_name\": \"Allowed Special\",\n \"advanced\": True,\n \"field_type\": \"str\",\n \"is_list\": True,\n },\n \"default_headers\": {\n \"display_name\": \"Default Headers\",\n \"advanced\": True,\n \"field_type\": \"dict\",\n },\n \"default_query\": {\n \"display_name\": \"Default Query\",\n \"advanced\": True,\n \"field_type\": \"NestedDict\",\n },\n \"disallowed_special\": {\n \"display_name\": \"Disallowed Special\",\n \"advanced\": True,\n \"field_type\": \"str\",\n \"is_list\": True,\n },\n \"chunk_size\": {\"display_name\": \"Chunk Size\", \"advanced\": True},\n \"client\": {\"display_name\": \"Client\", \"advanced\": True},\n \"deployment\": {\"display_name\": \"Deployment\", \"advanced\": True},\n \"embedding_ctx_length\": {\n \"display_name\": \"Embedding Context Length\",\n \"advanced\": True,\n },\n \"max_retries\": {\"display_name\": \"Max Retries\", \"advanced\": True},\n \"model\": {\n \"display_name\": \"Model\",\n \"advanced\": False,\n \"options\": [\"text-embedding-3-small\", \"text-embedding-3-large\", \"text-embedding-ada-002\"],\n },\n \"model_kwargs\": {\"display_name\": \"Model Kwargs\", \"advanced\": True},\n \"openai_api_base\": {\"display_name\": \"OpenAI API Base\", \"password\": True, \"advanced\": True},\n \"openai_api_key\": {\"display_name\": \"OpenAI API Key\", \"password\": True},\n \"openai_api_type\": {\"display_name\": \"OpenAI API Type\", \"advanced\": True, \"password\": True},\n \"openai_api_version\": {\n \"display_name\": \"OpenAI API Version\",\n \"advanced\": True,\n },\n \"openai_organization\": {\n \"display_name\": \"OpenAI Organization\",\n \"advanced\": True,\n },\n \"openai_proxy\": {\"display_name\": \"OpenAI Proxy\", \"advanced\": True},\n \"request_timeout\": {\"display_name\": \"Request Timeout\", \"advanced\": True},\n \"show_progress_bar\": {\n \"display_name\": \"Show Progress Bar\",\n \"advanced\": True,\n },\n \"skip_empty\": {\"display_name\": \"Skip Empty\", \"advanced\": True},\n \"tiktoken_model_name\": {\"display_name\": \"TikToken Model Name\"},\n \"tikToken_enable\": {\"display_name\": \"TikToken Enable\", \"advanced\": True},\n }\n\n def build(\n self,\n default_headers: Optional[Dict[str, str]] = None,\n default_query: Optional[NestedDict] = {},\n allowed_special: List[str] = [],\n disallowed_special: List[str] = [\"all\"],\n chunk_size: int = 1000,\n client: Optional[Any] = None,\n deployment: str = \"text-embedding-3-small\",\n embedding_ctx_length: int = 8191,\n max_retries: int = 6,\n model: str = \"text-embedding-3-small\",\n model_kwargs: NestedDict = {},\n openai_api_base: Optional[str] = None,\n openai_api_key: Optional[str] = \"\",\n openai_api_type: Optional[str] = None,\n openai_api_version: Optional[str] = None,\n openai_organization: Optional[str] = None,\n openai_proxy: Optional[str] = None,\n request_timeout: Optional[float] = None,\n show_progress_bar: bool = False,\n skip_empty: bool = False,\n tiktoken_enable: bool = True,\n tiktoken_model_name: Optional[str] = None,\n ) -> Union[OpenAIEmbeddings, Callable]:\n # This is to avoid errors with Vector Stores (e.g Chroma)\n if disallowed_special == [\"all\"]:\n disallowed_special = \"all\" # type: ignore\n\n api_key = SecretStr(openai_api_key) if openai_api_key else None\n\n return OpenAIEmbeddings(\n tiktoken_enabled=tiktoken_enable,\n default_headers=default_headers,\n default_query=default_query,\n allowed_special=set(allowed_special),\n disallowed_special=\"all\",\n chunk_size=chunk_size,\n client=client,\n deployment=deployment,\n embedding_ctx_length=embedding_ctx_length,\n max_retries=max_retries,\n model=model,\n model_kwargs=model_kwargs,\n base_url=openai_api_base,\n api_key=api_key,\n openai_api_type=openai_api_type,\n api_version=openai_api_version,\n organization=openai_organization,\n openai_proxy=openai_proxy,\n timeout=request_timeout,\n show_progress_bar=show_progress_bar,\n skip_empty=skip_empty,\n tiktoken_model_name=tiktoken_model_name,\n )\n", + "fileTypes": [], + "file_path": "", + "password": false, + "name": "code", + "advanced": false, + "dynamic": true, + "info": "", + "title_case": false + }, + "default_headers": { + "type": "dict", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "default_headers", + "display_name": "Default Headers", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "default_query": { + "type": "NestedDict", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": {}, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "default_query", + "display_name": "Default Query", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "deployment": { + "type": "str", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": "text-embedding-3-small", + "fileTypes": [], + "file_path": "", + "password": false, + "name": "deployment", + "display_name": "Deployment", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "disallowed_special": { + "type": "str", + "required": true, + "placeholder": "", + "list": true, + "show": true, + "multiline": false, + "value": [ + "all" + ], + "fileTypes": [], + "file_path": "", + "password": false, + "name": "disallowed_special", + "display_name": "Disallowed Special", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "embedding_ctx_length": { + "type": "int", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": 8191, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "embedding_ctx_length", + "display_name": "Embedding Context Length", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "max_retries": { + "type": "int", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": 6, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "max_retries", + "display_name": "Max Retries", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "model": { + "type": "str", + "required": true, + "placeholder": "", + "list": true, + "show": true, + "multiline": false, + "value": "text-embedding-3-small", + "fileTypes": [], + "file_path": "", + "password": false, + "options": [ + "text-embedding-3-small", + "text-embedding-3-large", + "text-embedding-ada-002" + ], + "name": "model", + "display_name": "Model", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "model_kwargs": { + "type": "NestedDict", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": {}, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "model_kwargs", + "display_name": "Model Kwargs", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "openai_api_base": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": true, + "name": "openai_api_base", + "display_name": "OpenAI API Base", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ], + "value": "" + }, + "openai_api_key": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": "", + "fileTypes": [], + "file_path": "", + "password": true, + "name": "openai_api_key", + "display_name": "OpenAI API Key", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "openai_api_type": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": true, + "name": "openai_api_type", + "display_name": "OpenAI API Type", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ], + "value": "" + }, + "openai_api_version": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "openai_api_version", + "display_name": "OpenAI API Version", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "openai_organization": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "openai_organization", + "display_name": "OpenAI Organization", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "openai_proxy": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "openai_proxy", + "display_name": "OpenAI Proxy", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "request_timeout": { + "type": "float", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "request_timeout", + "display_name": "Request Timeout", + "advanced": true, + "dynamic": false, + "info": "", + "rangeSpec": { + "min": -1, + "max": 1, + "step": 0.1 + }, + "title_case": false + }, + "show_progress_bar": { + "type": "bool", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "show_progress_bar", + "display_name": "Show Progress Bar", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "skip_empty": { + "type": "bool", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "skip_empty", + "display_name": "Skip Empty", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "tiktoken_enable": { + "type": "bool", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": true, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "tiktoken_enable", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false + }, + "tiktoken_model_name": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "tiktoken_model_name", + "display_name": "TikToken Model Name", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "_type": "CustomComponent" + }, + "description": "OpenAI embedding models", + "base_classes": [ + "Embeddings", + "OpenAIEmbeddings", + "Callable" + ], + "display_name": "OpenAIEmbeddings", + "documentation": "", + "custom_fields": { + "default_headers": null, + "default_query": null, + "allowed_special": null, + "disallowed_special": null, + "chunk_size": null, + "client": null, + "deployment": null, + "embedding_ctx_length": null, + "max_retries": null, + "model": null, + "model_kwargs": null, + "openai_api_base": null, + "openai_api_key": null, + "openai_api_type": null, + "openai_api_version": null, + "openai_organization": null, + "openai_proxy": null, + "request_timeout": null, + "show_progress_bar": null, + "skip_empty": null, + "tiktoken_enable": null, + "tiktoken_model_name": null + }, + "output_types": [ + "OpenAIEmbeddings", + "Callable" + ], + "field_formatters": {}, + "frozen": false, + "beta": true + }, + "id": "OpenAIEmbeddings-rbMk3" + }, + "selected": false, + "width": 384, + "height": 573, + "positionAbsolute": { + "x": 1053.9472627140208, + "y": -2.5921878249999963 + }, + "dragging": false + }, + { + "id": "URL-5zjQH", + "type": "genericNode", + "position": { + "x": 567.0838444398559, + "y": 596.6568151511171 + }, + "data": { + "type": "URL", + "node": { + "template": { + "code": { + "type": "code", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": true, + "value": "from typing import Any, Dict\n\nfrom langchain_community.document_loaders.web_base import WebBaseLoader\n\nfrom langflow import CustomComponent\nfrom langflow.schema import Record\n\n\nclass URLComponent(CustomComponent):\n display_name = \"URL\"\n description = \"Load URLs and convert them to records.\"\n\n def build_config(self) -> Dict[str, Any]:\n return {\n \"urls\": {\"display_name\": \"URL\"},\n }\n\n async def build(\n self,\n urls: list[str],\n ) -> Record:\n\n loader = WebBaseLoader(web_paths=urls)\n docs = loader.load()\n records = self.to_records(docs)\n return records\n", + "fileTypes": [], + "file_path": "", + "password": false, + "name": "code", + "advanced": false, + "dynamic": true, + "info": "", + "title_case": false + }, + "urls": { + "type": "str", + "required": true, + "placeholder": "", + "list": true, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "urls", + "display_name": "URL", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ], + "value": [ + "https://raw.githubusercontent.com/logspace-ai/langflow/dev/CONTRIBUTING.md" + ] + }, + "_type": "CustomComponent" + }, + "description": "Load URLs and convert them to records.", + "base_classes": [ + "Record" + ], + "display_name": "URL", + "documentation": "", + "custom_fields": { + "urls": null + }, + "output_types": [ + "Record" + ], + "field_formatters": {}, + "frozen": false, + "beta": true + }, + "id": "URL-5zjQH" + }, + "selected": false, + "width": 384, + "height": 289, + "dragging": false, + "positionAbsolute": { + "x": 567.0838444398559, + "y": 596.6568151511171 + } + } + ], + "edges": [ + { + "source": "RecursiveCharacterTextSplitter-jwfyG", + "sourceHandle": "{œbaseClassesœ:[œRecordœ],œdataTypeœ:œRecursiveCharacterTextSplitterœ,œidœ:œRecursiveCharacterTextSplitter-jwfyGœ}", + "target": "Chroma-aFGHF", + "targetHandle": "{œfieldNameœ:œinputsœ,œidœ:œChroma-aFGHFœ,œinputTypesœ:[œDocumentœ,œRecordœ],œtypeœ:œRecordœ}", + "data": { + "targetHandle": { + "fieldName": "inputs", + "id": "Chroma-aFGHF", + "inputTypes": [ + "Document", + "Record" + ], + "type": "Record" + }, + "sourceHandle": { + "baseClasses": [ + "Record" + ], + "dataType": "RecursiveCharacterTextSplitter", + "id": "RecursiveCharacterTextSplitter-jwfyG" + } + }, + "style": { + "stroke": "#555" + }, + "className": "stroke-gray-900 stroke-connection", + "id": "reactflow__edge-RecursiveCharacterTextSplitter-jwfyG{œbaseClassesœ:[œRecordœ],œdataTypeœ:œRecursiveCharacterTextSplitterœ,œidœ:œRecursiveCharacterTextSplitter-jwfyGœ}-Chroma-aFGHF{œfieldNameœ:œinputsœ,œidœ:œChroma-aFGHFœ,œinputTypesœ:[œDocumentœ,œRecordœ],œtypeœ:œRecordœ}" + }, + { + "source": "OpenAIEmbeddings-rbMk3", + "sourceHandle": "{œbaseClassesœ:[œEmbeddingsœ,œOpenAIEmbeddingsœ,œCallableœ],œdataTypeœ:œOpenAIEmbeddingsœ,œidœ:œOpenAIEmbeddings-rbMk3œ}", + "target": "Chroma-aFGHF", + "targetHandle": "{œfieldNameœ:œembeddingœ,œidœ:œChroma-aFGHFœ,œinputTypesœ:null,œtypeœ:œEmbeddingsœ}", + "data": { + "targetHandle": { + "fieldName": "embedding", + "id": "Chroma-aFGHF", + "inputTypes": null, + "type": "Embeddings" + }, + "sourceHandle": { + "baseClasses": [ + "Embeddings", + "OpenAIEmbeddings", + "Callable" + ], + "dataType": "OpenAIEmbeddings", + "id": "OpenAIEmbeddings-rbMk3" + } + }, + "style": { + "stroke": "#555" + }, + "className": "stroke-gray-900 stroke-connection", + "id": "reactflow__edge-OpenAIEmbeddings-rbMk3{œbaseClassesœ:[œEmbeddingsœ,œOpenAIEmbeddingsœ,œCallableœ],œdataTypeœ:œOpenAIEmbeddingsœ,œidœ:œOpenAIEmbeddings-rbMk3œ}-Chroma-aFGHF{œfieldNameœ:œembeddingœ,œidœ:œChroma-aFGHFœ,œinputTypesœ:null,œtypeœ:œEmbeddingsœ}" + }, + { + "source": "URL-5zjQH", + "sourceHandle": "{œbaseClassesœ:[œRecordœ],œdataTypeœ:œURLœ,œidœ:œURL-5zjQHœ}", + "target": "RecursiveCharacterTextSplitter-jwfyG", + "targetHandle": "{œfieldNameœ:œinputsœ,œidœ:œRecursiveCharacterTextSplitter-jwfyGœ,œinputTypesœ:[œDocumentœ,œRecordœ],œtypeœ:œDocumentœ}", + "data": { + "targetHandle": { + "fieldName": "inputs", + "id": "RecursiveCharacterTextSplitter-jwfyG", + "inputTypes": [ + "Document", + "Record" + ], + "type": "Document" + }, + "sourceHandle": { + "baseClasses": [ + "Record" + ], + "dataType": "URL", + "id": "URL-5zjQH" + } + }, + "style": { + "stroke": "#555" + }, + "className": "stroke-foreground stroke-connection", + "id": "reactflow__edge-URL-5zjQH{œbaseClassesœ:[œRecordœ],œdataTypeœ:œURLœ,œidœ:œURL-5zjQHœ}-RecursiveCharacterTextSplitter-jwfyG{œfieldNameœ:œinputsœ,œidœ:œRecursiveCharacterTextSplitter-jwfyGœ,œinputTypesœ:[œDocumentœ,œRecordœ],œtypeœ:œDocumentœ}" + } + ], + "viewport": { + "x": -160.3219973143573, + "y": 117.63775645863632, + "zoom": 0.48903173672366845 + } + }, + "is_component": false, + "updated_at": "2024-03-05T21:59:59.738081", + "folder": null, + "id": "7f90dc54-717d-49fe-a43f-c4dc055daa4e", + "user_id": "9365dbda-e8cf-4e95-8c84-49f8b6edb44f" +} \ No newline at end of file diff --git a/src/backend/langflow/main.py b/src/backend/langflow/main.py index 4b4c19a25..7c34d9b08 100644 --- a/src/backend/langflow/main.py +++ b/src/backend/langflow/main.py @@ -8,7 +8,9 @@ from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles + from langflow.api import router +from langflow.initial_setup.setup import create_or_update_starter_projects from langflow.interface.utils import setup_llm_caching from langflow.services.plugins.langfuse_plugin import LangfuseInstance from langflow.services.utils import initialize_services, teardown_services @@ -18,9 +20,12 @@ from langflow.utils.logger import configure def get_lifespan(fix_migration=False, socketio_server=None): @asynccontextmanager async def lifespan(app: FastAPI): - initialize_services(fix_migration=fix_migration, socketio_server=socketio_server) + initialize_services( + fix_migration=fix_migration, socketio_server=socketio_server + ) setup_llm_caching() LangfuseInstance.update() + create_or_update_starter_projects() yield teardown_services() @@ -31,7 +36,9 @@ def create_app(): """Create the FastAPI app and include the router.""" configure() - socketio_server = socketio.AsyncServer(async_mode="asgi", cors_allowed_origins="*", logger=True) + socketio_server = socketio.AsyncServer( + async_mode="asgi", cors_allowed_origins="*", logger=True + ) lifespan = get_lifespan(socketio_server=socketio_server) app = FastAPI(lifespan=lifespan) origins = ["*"] @@ -98,7 +105,9 @@ def get_static_files_dir(): return frontend_path / "frontend" -def setup_app(static_files_dir: Optional[Path] = None, backend_only: bool = False) -> FastAPI: +def setup_app( + static_files_dir: Optional[Path] = None, backend_only: bool = False +) -> FastAPI: """Setup the FastAPI app.""" # get the directory of the current file if not static_files_dir: @@ -114,6 +123,7 @@ def setup_app(static_files_dir: Optional[Path] = None, backend_only: bool = Fals if __name__ == "__main__": import uvicorn + from langflow.__main__ import get_number_of_workers configure() diff --git a/src/backend/langflow/services/database/service.py b/src/backend/langflow/services/database/service.py index 3cecdd340..c1e214700 100644 --- a/src/backend/langflow/services/database/service.py +++ b/src/backend/langflow/services/database/service.py @@ -169,12 +169,12 @@ class DatabaseService(Service): try: command.check(alembic_cfg) - except util.exc.AutogenerateDiffsDetected as e: + except util.exc.AutogenerateDiffsDetected as exc: logger.error(f"AutogenerateDiffsDetected: {exc}") if not fix: raise RuntimeError( "Something went wrong running migrations. Please, run `langflow migration --fix`" - ) from e + ) from exc if fix: self.try_downgrade_upgrade_until_success(alembic_cfg) diff --git a/src/backend/langflow/services/deps.py b/src/backend/langflow/services/deps.py index 19f3dcbf0..7d2338b04 100644 --- a/src/backend/langflow/services/deps.py +++ b/src/backend/langflow/services/deps.py @@ -1,3 +1,4 @@ +from contextlib import contextmanager from typing import TYPE_CHECKING, Generator from langflow.services import ServiceType, service_manager @@ -54,6 +55,19 @@ def get_session() -> Generator["Session", None, None]: yield from db_service.get_session() +@contextmanager +def session_scope(): + session = next(get_session()) + try: + yield session + session.commit() + except: + session.rollback() + raise + finally: + session.close() + + def get_cache_service() -> "BaseCacheService": return service_manager.get(ServiceType.CACHE_SERVICE) # type: ignore diff --git a/src/frontend/src/CustomNodes/GenericNode/index.tsx b/src/frontend/src/CustomNodes/GenericNode/index.tsx index ddfc04aef..80e5ad0f7 100644 --- a/src/frontend/src/CustomNodes/GenericNode/index.tsx +++ b/src/frontend/src/CustomNodes/GenericNode/index.tsx @@ -44,7 +44,6 @@ export default function GenericNode({ const buildFlow = useFlowStore((state) => state.buildFlow); const setNode = useFlowStore((state) => state.setNode); const name = nodeIconsLucide[data.type] ? data.type : types[data.type]; - console.log(types[data.type]) const [inputName, setInputName] = useState(false); const [nodeName, setNodeName] = useState(data.node!.display_name); const [inputDescription, setInputDescription] = useState(false); @@ -158,7 +157,7 @@ export default function GenericNode({ const iconElement = data?.node?.icon; const iconColor = nodeColors[types[data.type]]; const iconName = - iconElement || (data.node?.flow ? "group_components" : name); + iconElement || (data.node?.flow ? "group_components" : name); const iconClassName = `generic-node-icon ${ !showNode ? " absolute inset-x-6 h-12 w-12 " : "" }`; diff --git a/src/frontend/src/components/NewFlowCardComponent/index.tsx b/src/frontend/src/components/NewFlowCardComponent/index.tsx new file mode 100644 index 000000000..131fb3543 --- /dev/null +++ b/src/frontend/src/components/NewFlowCardComponent/index.tsx @@ -0,0 +1,54 @@ +import { useEffect, useState } from "react"; +import { getComponent, postLikeComponent } from "../../controllers/API"; +import DeleteConfirmationModal from "../../modals/DeleteConfirmationModal"; +import useAlertStore from "../../stores/alertStore"; +import useFlowsManagerStore from "../../stores/flowsManagerStore"; +import { useStoreStore } from "../../stores/storeStore"; +import { storeComponent } from "../../types/store"; +import cloneFLowWithParent from "../../utils/storeUtils"; +import { cn } from "../../utils/utils"; +import ShadTooltip from "../ShadTooltipComponent"; +import IconComponent from "../genericIconComponent"; +import { Badge } from "../ui/badge"; +import { Button } from "../ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "../ui/card"; +import { FlowType } from "../../types/flow"; +import { useNavigate } from "react-router-dom"; + +export default function NewFlowCardComponent({ +}: { + }) { + const addFlow = useFlowsManagerStore((state) => state.addFlow); + const navigate = useNavigate(); + + return ( + + + + + + ); +} diff --git a/src/frontend/src/components/exampleComponent/index.tsx b/src/frontend/src/components/exampleComponent/index.tsx new file mode 100644 index 000000000..fb6d570aa --- /dev/null +++ b/src/frontend/src/components/exampleComponent/index.tsx @@ -0,0 +1,90 @@ +import { useEffect, useState } from "react"; +import { getComponent, postLikeComponent } from "../../controllers/API"; +import DeleteConfirmationModal from "../../modals/DeleteConfirmationModal"; +import useAlertStore from "../../stores/alertStore"; +import useFlowsManagerStore from "../../stores/flowsManagerStore"; +import { useStoreStore } from "../../stores/storeStore"; +import { storeComponent } from "../../types/store"; +import cloneFLowWithParent from "../../utils/storeUtils"; +import { cn } from "../../utils/utils"; +import ShadTooltip from "../ShadTooltipComponent"; +import IconComponent from "../genericIconComponent"; +import { Badge } from "../ui/badge"; +import { Button } from "../ui/button"; +import { + Card, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "../ui/card"; +import { FlowType } from "../../types/flow"; +import { updateIds } from "../../utils/reactflowUtils"; +import { useNavigate } from "react-router-dom"; + +export default function CollectionCardComponent({ + flow, +}: { + flow: FlowType; + authorized?: boolean; +}) { + const addFlow = useFlowsManagerStore((state) => state.addFlow); + const navigate = useNavigate(); + + return ( + +
+ +
+ + + +
{flow.name}
+ + +
+ + +
{flow.description}
+ + + +
+ + +
+
+ +
+
+ + + ); +} diff --git a/src/frontend/src/constants/constants.ts b/src/frontend/src/constants/constants.ts index 10b6e41df..897a1b4b3 100644 --- a/src/frontend/src/constants/constants.ts +++ b/src/frontend/src/constants/constants.ts @@ -727,7 +727,7 @@ export const STATUS_BUILD = "Build to validate status."; export const STATUS_BUILDING = "Building..."; export const SAVED_HOVER = "Last saved at "; export const RUN_TIMESTAMP_PREFIX = "Last Run: "; - +export const STARTER_FOLDER_NAME = "Starter Projects"; export const PRIORITY_SIDEBAR_ORDER = [ "saved_components", "inputs", diff --git a/src/frontend/src/pages/MainPage/components/components/index.tsx b/src/frontend/src/pages/MainPage/components/components/index.tsx index 86ee65d0c..234852388 100644 --- a/src/frontend/src/pages/MainPage/components/components/index.tsx +++ b/src/frontend/src/pages/MainPage/components/components/index.tsx @@ -14,6 +14,7 @@ import { import useAlertStore from "../../../../stores/alertStore"; import useFlowsManagerStore from "../../../../stores/flowsManagerStore"; import { FlowType } from "../../../../types/flow"; +import { STARTER_FOLDER_NAME } from "../../../../constants/constants"; export default function ComponentsComponent({ is_component = true, @@ -24,6 +25,7 @@ export default function ComponentsComponent({ const uploadFlow = useFlowsManagerStore((state) => state.uploadFlow); const removeFlow = useFlowsManagerStore((state) => state.removeFlow); const isLoading = useFlowsManagerStore((state) => state.isLoading); + const setExamples = useFlowsManagerStore((state) => state.setExamples); const flows = useFlowsManagerStore((state) => state.flows); const setSuccessData = useAlertStore((state) => state.setSuccessData); const setErrorData = useAlertStore((state) => state.setErrorData); @@ -35,7 +37,7 @@ export default function ComponentsComponent({ useEffect(() => { if (isLoading) return; - const all = flows + let all = flows .filter((f) => (f.is_component ?? false) === is_component) .sort((a, b) => { if (a?.updated_at && b?.updated_at) { diff --git a/src/frontend/src/pages/MainPage/index.tsx b/src/frontend/src/pages/MainPage/index.tsx index cc4115e7e..2574f03af 100644 --- a/src/frontend/src/pages/MainPage/index.tsx +++ b/src/frontend/src/pages/MainPage/index.tsx @@ -1,5 +1,5 @@ import { Group, ToyBrick } from "lucide-react"; -import { useEffect } from "react"; +import { useEffect, useState } from "react"; import { Outlet, useLocation, useNavigate } from "react-router-dom"; import DropdownButton from "../../components/DropdownButtonComponent"; import IconComponent from "../../components/genericIconComponent"; @@ -14,6 +14,9 @@ import { import useAlertStore from "../../stores/alertStore"; import useFlowsManagerStore from "../../stores/flowsManagerStore"; import { downloadFlows } from "../../utils/reactflowUtils"; +import BaseModal from "../../modals/baseModal"; +import ExampleCardComponent from "../../components/exampleComponent"; +import NewFlowCardComponent from "../../components/NewFlowCardComponent"; export default function HomePage(): JSX.Element { const addFlow = useFlowsManagerStore((state) => state.addFlow); const uploadFlow = useFlowsManagerStore((state) => state.uploadFlow); @@ -25,6 +28,8 @@ export default function HomePage(): JSX.Element { const setErrorData = useAlertStore((state) => state.setErrorData); const location = useLocation(); const pathname = location.pathname; + const [openModal, setOpenModal] = useState(false); + const examples = useFlowsManagerStore((state) => state.examples); const is_component = pathname === "/components"; const dropdownOptions = [ { @@ -36,9 +41,8 @@ export default function HomePage(): JSX.Element { }) .then((id) => { setSuccessData({ - title: `${ - is_component ? "Component" : "Flow" - } uploaded successfully`, + title: `${is_component ? "Component" : "Flow" + } uploaded successfully`, }); if (!is_component) navigate("/flow/" + id); }) @@ -98,11 +102,7 @@ export default function HomePage(): JSX.Element { { - addFlow(true).then((id) => { - navigate("/flow/" + id); - }); - }} + onFirstBtnClick={() => setOpenModal(true)} options={dropdownOptions} /> @@ -116,6 +116,27 @@ export default function HomePage(): JSX.Element { + + + + Create a New Flow + + + +
+ {examples.map((example, idx) => { + return( + ) + })} + +
+ + ); } diff --git a/src/frontend/src/stores/flowStore.ts b/src/frontend/src/stores/flowStore.ts index cbf183289..e3cfbb20e 100644 --- a/src/frontend/src/stores/flowStore.ts +++ b/src/frontend/src/stores/flowStore.ts @@ -563,7 +563,6 @@ const useFlowStore = create((set, get) => ({ }); }, updateBuildStatus: (nodeIdList: string[], status: BuildStatus) => { - console.log("updateBuildStatus", nodeIdList, status); const newFlowBuildStatus = { ...get().flowBuildStatus }; nodeIdList.forEach((id) => { newFlowBuildStatus[id] = { @@ -573,7 +572,6 @@ const useFlowStore = create((set, get) => ({ const timestamp_string = new Date(Date.now()).toLocaleString(); newFlowBuildStatus[id].timestamp = timestamp_string; } - console.log("updateBuildStatus", newFlowBuildStatus); }); set({ flowBuildStatus: newFlowBuildStatus }); }, diff --git a/src/frontend/src/stores/flowsManagerStore.ts b/src/frontend/src/stores/flowsManagerStore.ts index fac806c5f..d993f40b0 100644 --- a/src/frontend/src/stores/flowsManagerStore.ts +++ b/src/frontend/src/stores/flowsManagerStore.ts @@ -25,6 +25,7 @@ import useAlertStore from "./alertStore"; import { useDarkStore } from "./darkStore"; import useFlowStore from "./flowStore"; import { useTypesStore } from "./typesStore"; +import { STARTER_FOLDER_NAME } from "../constants/constants"; let saveTimeoutId: NodeJS.Timeout | null = null; @@ -37,6 +38,10 @@ const past = {}; const future = {}; const useFlowsManagerStore = create((set, get) => ({ + examples:[], + setExamples: (examples: FlowType[]) => { + set({ examples }); + }, currentFlowId: "", setCurrentFlowId: (currentFlowId: string) => { set((state) => ({ @@ -62,7 +67,8 @@ const useFlowsManagerStore = create((set, get) => ({ .then((dbData) => { if (dbData) { const { data, flows } = processFlows(dbData, false); - get().setFlows(flows); + get().setExamples(flows.filter(f=>(f.folder===STARTER_FOLDER_NAME && !f.user_id))); + get().setFlows(flows.filter(f=>!(f.folder===STARTER_FOLDER_NAME && !f.user_id))); useTypesStore.setState((state) => ({ data: { ...state.data, ["saved_components"]: data }, })); diff --git a/src/frontend/src/types/flow/index.ts b/src/frontend/src/types/flow/index.ts index 967d4e424..ffb364d61 100644 --- a/src/frontend/src/types/flow/index.ts +++ b/src/frontend/src/types/flow/index.ts @@ -13,6 +13,8 @@ export type FlowType = { updated_at?: string; date_created?: string; parent?: string; + folder?: string; + user_id?: string; }; export type NodeType = { diff --git a/src/frontend/src/types/zustand/flowsManager/index.ts b/src/frontend/src/types/zustand/flowsManager/index.ts index 471d4cf17..87fbf9a22 100644 --- a/src/frontend/src/types/zustand/flowsManager/index.ts +++ b/src/frontend/src/types/zustand/flowsManager/index.ts @@ -44,6 +44,8 @@ export type FlowsManagerStoreType = { undo: () => void; redo: () => void; takeSnapshot: () => void; + examples: Array; + setExamples: (examples: FlowType[]) => void; }; export type UseUndoRedoOptions = { diff --git a/src/frontend/src/utils/styleUtils.ts b/src/frontend/src/utils/styleUtils.ts index cabae91b6..ac88826ca 100644 --- a/src/frontend/src/utils/styleUtils.ts +++ b/src/frontend/src/utils/styleUtils.ts @@ -87,6 +87,8 @@ import { Pin, Play, Plus, + PlusCircle, + PlusSquare, PocketKnife, Redo, RefreshCcw, @@ -393,6 +395,8 @@ export const nodeIconsLucide: iconsType = { Circle, CircleDot, Clipboard, + PlusCircle, + PlusSquare, Code2, Variable, Snowflake,