From 064e16cb7758db7283b031e7978a5dd806fd7122 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 17:41:39 -0300 Subject: [PATCH 01/26] Refactor flow reading logic and handle exception when retrieving example flows --- src/backend/langflow/api/v1/flows.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/backend/langflow/api/v1/flows.py b/src/backend/langflow/api/v1/flows.py index de021d091..c58168d34 100644 --- a/src/backend/langflow/api/v1/flows.py +++ b/src/backend/langflow/api/v1/flows.py @@ -5,17 +5,14 @@ from uuid import UUID import orjson from fastapi import APIRouter, Depends, File, HTTPException, UploadFile from fastapi.encoders import jsonable_encoder +from loguru import logger from sqlmodel import Session, select from langflow.api.utils import remove_api_keys, validate_is_component from langflow.api.v1.schemas import FlowListCreate, FlowListRead from langflow.services.auth.utils import get_current_active_user -from langflow.services.database.models.flow import ( - Flow, - FlowCreate, - FlowRead, - FlowUpdate, -) +from langflow.services.database.models.flow import (Flow, FlowCreate, FlowRead, + FlowUpdate) from langflow.services.database.models.user.model import User from langflow.services.deps import get_session, get_settings_service @@ -54,8 +51,11 @@ def read_flows( flows = current_user.flows flows = validate_is_component(flows) # with the session get the flows that DO NOT have a user_id - example_flows = session.exec(select(Flow).where(Flow.user_id == None)).all() - flows.extend(example_flows) + try: + example_flows = session.exec(select(Flow).where(Flow.user_id == None)).all() + flows.extend(example_flows) + except Exception as e: + logger.error(e) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) from e return [jsonable_encoder(flow) for flow in flows] From cbf6dfae4d2b4c8d8c34230165e72c655a1f8d03 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 17:44:04 -0300 Subject: [PATCH 02/26] Fix exception variable name in DatabaseService --- src/backend/langflow/services/database/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/langflow/services/database/service.py b/src/backend/langflow/services/database/service.py index 3cecdd340..c4b06b90f 100644 --- a/src/backend/langflow/services/database/service.py +++ b/src/backend/langflow/services/database/service.py @@ -169,7 +169,7 @@ class DatabaseService(Service): try: command.check(alembic_cfg) - except util.exc.AutogenerateDiffsDetected as e: + except util.exc.AutogenerateDiffsDetected as exc: logger.error(f"AutogenerateDiffsDetected: {exc}") if not fix: raise RuntimeError( From 1d2364935823c8c090ea7462d17d47f06a339a0f Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 17:44:48 -0300 Subject: [PATCH 03/26] Fix exception variable name in database service --- src/backend/langflow/services/database/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/langflow/services/database/service.py b/src/backend/langflow/services/database/service.py index c4b06b90f..c1e214700 100644 --- a/src/backend/langflow/services/database/service.py +++ b/src/backend/langflow/services/database/service.py @@ -174,7 +174,7 @@ class DatabaseService(Service): if not fix: raise RuntimeError( "Something went wrong running migrations. Please, run `langflow migration --fix`" - ) from e + ) from exc if fix: self.try_downgrade_upgrade_until_success(alembic_cfg) From ac8f2b7070289af54ad366132428fd22ba883b64 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 17:46:17 -0300 Subject: [PATCH 04/26] Update user_id field to be optional in FlowRead class --- src/backend/langflow/services/database/models/flow/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/langflow/services/database/models/flow/model.py b/src/backend/langflow/services/database/models/flow/model.py index 1211c40ed..c7ee08141 100644 --- a/src/backend/langflow/services/database/models/flow/model.py +++ b/src/backend/langflow/services/database/models/flow/model.py @@ -80,7 +80,7 @@ class FlowCreate(FlowBase): class FlowRead(FlowBase): id: UUID - user_id: UUID = Field() + user_id: Optional[UUID] = Field() class FlowUpdate(SQLModel): From dce9901222bd4dd9ca9f8fa14d492e08648732bf Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 18:46:18 -0300 Subject: [PATCH 05/26] Remove "documents" configuration from vector store components --- .../textsplitters/CharacterTextSplitter.py | 20 +++++++++----- .../LanguageRecursiveTextSplitter.py | 27 +++++++++++-------- .../RecursiveCharacterTextSplitter.py | 27 ++++++++++++------- .../components/vectorstores/Chroma.py | 9 ++++--- .../components/vectorstores/ChromaSearch.py | 1 - .../langflow/components/vectorstores/FAISS.py | 13 ++++++--- .../components/vectorstores/FAISSSearch.py | 1 - .../vectorstores/MongoDBAtlasVector.py | 21 +++++++++++---- .../components/vectorstores/Pinecone.py | 13 ++++++--- .../components/vectorstores/Qdrant.py | 24 +++++++++++++---- .../langflow/components/vectorstores/Redis.py | 19 +++++++++---- .../components/vectorstores/RedisSearch.py | 2 +- .../vectorstores/SupabaseVectorStore.py | 20 ++++++++++---- .../components/vectorstores/Vectara.py | 23 +++++++++++----- .../components/vectorstores/VectaraSearch.py | 4 --- .../components/vectorstores/Weaviate.py | 19 +++++++++---- .../components/vectorstores/WeaviateSearch.py | 1 - .../components/vectorstores/pgvector.py | 17 +++++++++--- 18 files changed, 183 insertions(+), 78 deletions(-) diff --git a/src/backend/langflow/components/textsplitters/CharacterTextSplitter.py b/src/backend/langflow/components/textsplitters/CharacterTextSplitter.py index d165f47fd..96576a4a3 100644 --- a/src/backend/langflow/components/textsplitters/CharacterTextSplitter.py +++ b/src/backend/langflow/components/textsplitters/CharacterTextSplitter.py @@ -1,8 +1,9 @@ from typing import List from langchain.text_splitter import CharacterTextSplitter -from langchain_core.documents.base import Document + from langflow import CustomComponent +from langflow.schema.schema import Record class CharacterTextSplitterComponent(CustomComponent): @@ -11,7 +12,7 @@ class CharacterTextSplitterComponent(CustomComponent): def build_config(self): return { - "documents": {"display_name": "Documents"}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "chunk_overlap": {"display_name": "Chunk Overlap", "default": 200}, "chunk_size": {"display_name": "Chunk Size", "default": 1000}, "separator": {"display_name": "Separator", "default": "\n"}, @@ -19,17 +20,24 @@ class CharacterTextSplitterComponent(CustomComponent): def build( self, - documents: List[Document], + inputs: List[Record], chunk_overlap: int = 200, chunk_size: int = 1000, separator: str = "\n", - ) -> List[Document]: + ) -> List[Record]: # separator may come escaped from the frontend separator = separator.encode().decode("unicode_escape") + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) docs = CharacterTextSplitter( chunk_overlap=chunk_overlap, chunk_size=chunk_size, separator=separator, ).split_documents(documents) - self.status = docs - return docs + records = self.to_records(docs) + self.status = records + return records diff --git a/src/backend/langflow/components/textsplitters/LanguageRecursiveTextSplitter.py b/src/backend/langflow/components/textsplitters/LanguageRecursiveTextSplitter.py index d1494f4d0..b86b834d8 100644 --- a/src/backend/langflow/components/textsplitters/LanguageRecursiveTextSplitter.py +++ b/src/backend/langflow/components/textsplitters/LanguageRecursiveTextSplitter.py @@ -1,23 +1,22 @@ -from typing import Optional +from typing import List, Optional from langchain.text_splitter import Language -from langchain_core.documents import Document from langflow import CustomComponent +from langflow.schema.schema import Record class LanguageRecursiveTextSplitterComponent(CustomComponent): display_name: str = "Language Recursive Text Splitter" description: str = "Split text into chunks of a specified length based on language." - documentation: str = "https://docs.langflow.org/components/text-splitters#languagerecursivetextsplitter" + documentation: str = ( + "https://docs.langflow.org/components/text-splitters#languagerecursivetextsplitter" + ) def build_config(self): options = [x.value for x in Language] return { - "documents": { - "display_name": "Documents", - "info": "The documents to split.", - }, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "separator_type": { "display_name": "Separator Type", "info": "The type of separator to use.", @@ -47,11 +46,11 @@ class LanguageRecursiveTextSplitterComponent(CustomComponent): def build( self, - documents: list[Document], + inputs: List[Record], chunk_size: Optional[int] = 1000, chunk_overlap: Optional[int] = 200, separator_type: str = "Python", - ) -> list[Document]: + ) -> list[Record]: """ Split text into chunks of a specified length. @@ -77,6 +76,12 @@ class LanguageRecursiveTextSplitterComponent(CustomComponent): chunk_size=chunk_size, chunk_overlap=chunk_overlap, ) - + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) docs = splitter.split_documents(documents) - return docs + records = self.to_records(docs) + return records diff --git a/src/backend/langflow/components/textsplitters/RecursiveCharacterTextSplitter.py b/src/backend/langflow/components/textsplitters/RecursiveCharacterTextSplitter.py index d07ae3ebe..a15600394 100644 --- a/src/backend/langflow/components/textsplitters/RecursiveCharacterTextSplitter.py +++ b/src/backend/langflow/components/textsplitters/RecursiveCharacterTextSplitter.py @@ -1,22 +1,26 @@ from typing import Optional +from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain_core.documents import Document from langflow import CustomComponent +from langflow.schema import Record from langflow.utils.util import build_loader_repr_from_documents -from langchain.text_splitter import RecursiveCharacterTextSplitter class RecursiveCharacterTextSplitterComponent(CustomComponent): display_name: str = "Recursive Character Text Splitter" description: str = "Split text into chunks of a specified length." - documentation: str = "https://docs.langflow.org/components/text-splitters#recursivecharactertextsplitter" + documentation: str = ( + "https://docs.langflow.org/components/text-splitters#recursivecharactertextsplitter" + ) def build_config(self): return { - "documents": { - "display_name": "Documents", - "info": "The documents to split.", + "inputs": { + "display_name": "Input", + "info": "The texts to split.", + "input_types": ["Document", "Record"], }, "separators": { "display_name": "Separators", @@ -40,11 +44,11 @@ class RecursiveCharacterTextSplitterComponent(CustomComponent): def build( self, - documents: list[Document], + inputs: list[Document], separators: Optional[list[str]] = None, chunk_size: Optional[int] = 1000, chunk_overlap: Optional[int] = 200, - ) -> list[Document]: + ) -> list[Record]: """ Split text into chunks of a specified length. @@ -75,7 +79,12 @@ class RecursiveCharacterTextSplitterComponent(CustomComponent): chunk_size=chunk_size, chunk_overlap=chunk_overlap, ) - + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) docs = splitter.split_documents(documents) self.repr_value = build_loader_repr_from_documents(docs) - return docs + return self.to_records(docs) diff --git a/src/backend/langflow/components/vectorstores/Chroma.py b/src/backend/langflow/components/vectorstores/Chroma.py index b1756e777..139a877e9 100644 --- a/src/backend/langflow/components/vectorstores/Chroma.py +++ b/src/backend/langflow/components/vectorstores/Chroma.py @@ -31,7 +31,7 @@ class ChromaComponent(CustomComponent): "collection_name": {"display_name": "Collection Name", "value": "langflow"}, "index_directory": {"display_name": "Persist Directory"}, "code": {"advanced": True, "display_name": "Code"}, - "documents": {"display_name": "Documents", "is_list": True}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "chroma_server_cors_allow_origins": { "display_name": "Server CORS Allow Origins", @@ -84,7 +84,8 @@ class ChromaComponent(CustomComponent): if chroma_server_host is not None: chroma_settings = chromadb.config.Settings( - chroma_server_cors_allow_origins=chroma_server_cors_allow_origins or None, + chroma_server_cors_allow_origins=chroma_server_cors_allow_origins + or None, chroma_server_host=chroma_server_host, chroma_server_port=chroma_server_port or None, chroma_server_grpc_port=chroma_server_grpc_port or None, @@ -99,7 +100,9 @@ class ChromaComponent(CustomComponent): if documents is not None and embedding is not None: if len(documents) == 0: - raise ValueError("If documents are provided, there must be at least one document.") + raise ValueError( + "If documents are provided, there must be at least one document." + ) chroma = Chroma.from_documents( documents=documents, # type: ignore persist_directory=index_directory, diff --git a/src/backend/langflow/components/vectorstores/ChromaSearch.py b/src/backend/langflow/components/vectorstores/ChromaSearch.py index 3a6d283b3..cf98ee987 100644 --- a/src/backend/langflow/components/vectorstores/ChromaSearch.py +++ b/src/backend/langflow/components/vectorstores/ChromaSearch.py @@ -35,7 +35,6 @@ class ChromaSearchComponent(LCVectorStoreComponent): # "persist": {"display_name": "Persist"}, "index_directory": {"display_name": "Index Directory"}, "code": {"show": False, "display_name": "Code"}, - "documents": {"display_name": "Documents", "is_list": True}, "embedding": { "display_name": "Embedding", "info": "Embedding model to vectorize inputs (make sure to use same as index)", diff --git a/src/backend/langflow/components/vectorstores/FAISS.py b/src/backend/langflow/components/vectorstores/FAISS.py index a0324456e..7cdadccdb 100644 --- a/src/backend/langflow/components/vectorstores/FAISS.py +++ b/src/backend/langflow/components/vectorstores/FAISS.py @@ -5,7 +5,8 @@ from langchain_community.vectorstores import VectorStore from langchain_community.vectorstores.faiss import FAISS from langflow import CustomComponent -from langflow.field_typing import Document, Embeddings +from langflow.field_typing import Embeddings +from langflow.schema.schema import Record class FAISSComponent(CustomComponent): @@ -15,7 +16,7 @@ class FAISSComponent(CustomComponent): def build_config(self): return { - "documents": {"display_name": "Documents"}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "folder_path": { "display_name": "Folder Path", @@ -27,10 +28,16 @@ class FAISSComponent(CustomComponent): def build( self, embedding: Embeddings, - documents: List[Document], + inputs: List[Record], folder_path: str, index_name: str = "langflow_index", ) -> Union[VectorStore, FAISS, BaseRetriever]: + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) vector_store = FAISS.from_documents(documents=documents, embedding=embedding) if not folder_path: raise ValueError("Folder path is required to save the FAISS index.") diff --git a/src/backend/langflow/components/vectorstores/FAISSSearch.py b/src/backend/langflow/components/vectorstores/FAISSSearch.py index f6ddf4f7a..4544c300b 100644 --- a/src/backend/langflow/components/vectorstores/FAISSSearch.py +++ b/src/backend/langflow/components/vectorstores/FAISSSearch.py @@ -14,7 +14,6 @@ class FAISSSearchComponent(LCVectorStoreComponent): def build_config(self): return { - "documents": {"display_name": "Documents"}, "embedding": {"display_name": "Embedding"}, "folder_path": { "display_name": "Folder Path", diff --git a/src/backend/langflow/components/vectorstores/MongoDBAtlasVector.py b/src/backend/langflow/components/vectorstores/MongoDBAtlasVector.py index e15368f7d..4488e4e92 100644 --- a/src/backend/langflow/components/vectorstores/MongoDBAtlasVector.py +++ b/src/backend/langflow/components/vectorstores/MongoDBAtlasVector.py @@ -3,17 +3,20 @@ from typing import List, Optional from langchain_community.vectorstores.mongodb_atlas import MongoDBAtlasVectorSearch from langflow import CustomComponent -from langflow.field_typing import Document, Embeddings, NestedDict +from langflow.field_typing import Embeddings, NestedDict +from langflow.schema.schema import Record class MongoDBAtlasComponent(CustomComponent): display_name = "MongoDB Atlas" - description = "Construct a `MongoDB Atlas Vector Search` vector store from raw documents." + description = ( + "Construct a `MongoDB Atlas Vector Search` vector store from raw documents." + ) icon = "MongoDB" def build_config(self): return { - "documents": {"display_name": "Documents"}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "collection_name": {"display_name": "Collection Name"}, "db_name": {"display_name": "Database Name"}, @@ -25,7 +28,7 @@ class MongoDBAtlasComponent(CustomComponent): def build( self, embedding: Embeddings, - documents: List[Document], + inputs: List[Record], collection_name: str = "", db_name: str = "", index_name: str = "", @@ -36,12 +39,20 @@ class MongoDBAtlasComponent(CustomComponent): try: from pymongo import MongoClient except ImportError: - raise ImportError("Please install pymongo to use MongoDB Atlas Vector Store") + raise ImportError( + "Please install pymongo to use MongoDB Atlas Vector Store" + ) try: mongo_client: MongoClient = MongoClient(mongodb_atlas_cluster_uri) collection = mongo_client[db_name][collection_name] except Exception as e: raise ValueError(f"Failed to connect to MongoDB Atlas: {e}") + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) if documents: vector_store = MongoDBAtlasVectorSearch.from_documents( documents=documents, diff --git a/src/backend/langflow/components/vectorstores/Pinecone.py b/src/backend/langflow/components/vectorstores/Pinecone.py index 54222b133..c71048266 100644 --- a/src/backend/langflow/components/vectorstores/Pinecone.py +++ b/src/backend/langflow/components/vectorstores/Pinecone.py @@ -7,7 +7,8 @@ from langchain_community.vectorstores import VectorStore from langchain_community.vectorstores.pinecone import Pinecone from langflow import CustomComponent -from langflow.field_typing import Document, Embeddings +from langflow.field_typing import Embeddings +from langflow.schema.schema import Record class PineconeComponent(CustomComponent): @@ -17,7 +18,7 @@ class PineconeComponent(CustomComponent): def build_config(self): return { - "documents": {"display_name": "Documents"}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "index_name": {"display_name": "Index Name"}, "namespace": {"display_name": "Namespace"}, @@ -44,7 +45,7 @@ class PineconeComponent(CustomComponent): self, embedding: Embeddings, pinecone_env: str, - documents: List[Document], + inputs: List[Record], text_key: str = "text", pool_threads: int = 4, index_name: Optional[str] = None, @@ -59,6 +60,12 @@ class PineconeComponent(CustomComponent): pinecone.init(api_key=pinecone_api_key, environment=pinecone_env) # type: ignore if not index_name: raise ValueError("Index Name is required.") + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) if documents: return Pinecone.from_documents( documents=documents, diff --git a/src/backend/langflow/components/vectorstores/Qdrant.py b/src/backend/langflow/components/vectorstores/Qdrant.py index 23ee70b11..e1773268b 100644 --- a/src/backend/langflow/components/vectorstores/Qdrant.py +++ b/src/backend/langflow/components/vectorstores/Qdrant.py @@ -3,8 +3,10 @@ from typing import Optional, Union from langchain.schema import BaseRetriever from langchain_community.vectorstores import VectorStore from langchain_community.vectorstores.qdrant import Qdrant + from langflow import CustomComponent -from langflow.field_typing import Document, Embeddings, NestedDict +from langflow.field_typing import Embeddings, NestedDict +from langflow.schema.schema import Record class QdrantComponent(CustomComponent): @@ -14,17 +16,23 @@ class QdrantComponent(CustomComponent): def build_config(self): return { - "documents": {"display_name": "Documents"}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "api_key": {"display_name": "API Key", "password": True, "advanced": True}, "collection_name": {"display_name": "Collection Name"}, - "content_payload_key": {"display_name": "Content Payload Key", "advanced": True}, + "content_payload_key": { + "display_name": "Content Payload Key", + "advanced": True, + }, "distance_func": {"display_name": "Distance Function", "advanced": True}, "grpc_port": {"display_name": "gRPC Port", "advanced": True}, "host": {"display_name": "Host", "advanced": True}, "https": {"display_name": "HTTPS", "advanced": True}, "location": {"display_name": "Location", "advanced": True}, - "metadata_payload_key": {"display_name": "Metadata Payload Key", "advanced": True}, + "metadata_payload_key": { + "display_name": "Metadata Payload Key", + "advanced": True, + }, "path": {"display_name": "Path", "advanced": True}, "port": {"display_name": "Port", "advanced": True}, "prefer_grpc": {"display_name": "Prefer gRPC", "advanced": True}, @@ -38,7 +46,7 @@ class QdrantComponent(CustomComponent): self, embedding: Embeddings, collection_name: str, - documents: Optional[Document] = None, + inputs: Optional[Record] = None, api_key: Optional[str] = None, content_payload_key: str = "page_content", distance_func: str = "Cosine", @@ -55,6 +63,12 @@ class QdrantComponent(CustomComponent): timeout: Optional[int] = None, url: Optional[str] = None, ) -> Union[VectorStore, Qdrant, BaseRetriever]: + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) if documents is None: from qdrant_client import QdrantClient diff --git a/src/backend/langflow/components/vectorstores/Redis.py b/src/backend/langflow/components/vectorstores/Redis.py index b2d7e4542..bbf04f1a4 100644 --- a/src/backend/langflow/components/vectorstores/Redis.py +++ b/src/backend/langflow/components/vectorstores/Redis.py @@ -3,9 +3,10 @@ from typing import Optional, Union from langchain.embeddings.base import Embeddings from langchain_community.vectorstores import VectorStore from langchain_community.vectorstores.redis import Redis -from langchain_core.documents import Document from langchain_core.retrievers import BaseRetriever + from langflow import CustomComponent +from langflow.schema.schema import Record class RedisComponent(CustomComponent): @@ -28,7 +29,7 @@ class RedisComponent(CustomComponent): return { "index_name": {"display_name": "Index Name", "value": "your_index"}, "code": {"show": False, "display_name": "Code"}, - "documents": {"display_name": "Documents", "is_list": True}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "schema": {"display_name": "Schema", "file_types": [".yaml"]}, "redis_server_url": { @@ -44,7 +45,7 @@ class RedisComponent(CustomComponent): redis_server_url: str, redis_index_name: str, schema: Optional[str] = None, - documents: Optional[Document] = None, + inputs: Optional[Record] = None, ) -> Union[VectorStore, BaseRetriever]: """ Builds the Vector Store or BaseRetriever object. @@ -58,9 +59,17 @@ class RedisComponent(CustomComponent): Returns: - VectorStore: The Vector Store object. """ - if documents is None: + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) + if not documents: if schema is None: - raise ValueError("If no documents are provided, a schema must be provided.") + raise ValueError( + "If no documents are provided, a schema must be provided." + ) redis_vs = Redis.from_existing_index( embedding=embedding, index_name=redis_index_name, diff --git a/src/backend/langflow/components/vectorstores/RedisSearch.py b/src/backend/langflow/components/vectorstores/RedisSearch.py index 4089d4f47..63fc46ddb 100644 --- a/src/backend/langflow/components/vectorstores/RedisSearch.py +++ b/src/backend/langflow/components/vectorstores/RedisSearch.py @@ -33,7 +33,7 @@ class RedisSearchComponent(RedisComponent, LCVectorStoreComponent): "input_value": {"display_name": "Input"}, "index_name": {"display_name": "Index Name", "value": "your_index"}, "code": {"show": False, "display_name": "Code"}, - "documents": {"display_name": "Documents", "is_list": True}, + "embedding": {"display_name": "Embedding"}, "schema": {"display_name": "Schema", "file_types": [".yaml"]}, "redis_server_url": { diff --git a/src/backend/langflow/components/vectorstores/SupabaseVectorStore.py b/src/backend/langflow/components/vectorstores/SupabaseVectorStore.py index 2ec6dfabc..7f6296c03 100644 --- a/src/backend/langflow/components/vectorstores/SupabaseVectorStore.py +++ b/src/backend/langflow/components/vectorstores/SupabaseVectorStore.py @@ -3,10 +3,12 @@ from typing import List, Union from langchain.schema import BaseRetriever from langchain_community.vectorstores import VectorStore from langchain_community.vectorstores.supabase import SupabaseVectorStore -from langflow import CustomComponent -from langflow.field_typing import Document, Embeddings, NestedDict from supabase.client import Client, create_client +from langflow import CustomComponent +from langflow.field_typing import Embeddings, NestedDict +from langflow.schema.schema import Record + class SupabaseComponent(CustomComponent): display_name = "Supabase" @@ -14,7 +16,7 @@ class SupabaseComponent(CustomComponent): def build_config(self): return { - "documents": {"display_name": "Documents"}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "query_name": {"display_name": "Query Name"}, "search_kwargs": {"display_name": "Search Kwargs", "advanced": True}, @@ -26,14 +28,22 @@ class SupabaseComponent(CustomComponent): def build( self, embedding: Embeddings, - documents: List[Document], + inputs: List[Record], query_name: str = "", search_kwargs: NestedDict = {}, supabase_service_key: str = "", supabase_url: str = "", table_name: str = "", ) -> Union[VectorStore, SupabaseVectorStore, BaseRetriever]: - supabase: Client = create_client(supabase_url, supabase_key=supabase_service_key) + supabase: Client = create_client( + supabase_url, supabase_key=supabase_service_key + ) + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) return SupabaseVectorStore.from_documents( documents=documents, embedding=embedding, diff --git a/src/backend/langflow/components/vectorstores/Vectara.py b/src/backend/langflow/components/vectorstores/Vectara.py index 0a396918c..8f1f7250e 100644 --- a/src/backend/langflow/components/vectorstores/Vectara.py +++ b/src/backend/langflow/components/vectorstores/Vectara.py @@ -8,13 +8,16 @@ from langchain_community.vectorstores.vectara import Vectara from langchain_core.vectorstores import VectorStore from langflow import CustomComponent -from langflow.field_typing import BaseRetriever, Document +from langflow.field_typing import BaseRetriever +from langflow.schema.schema import Record class VectaraComponent(CustomComponent): display_name: str = "Vectara" description: str = "Implementation of Vector Store using Vectara" - documentation = "https://python.langchain.com/docs/integrations/vectorstores/vectara" + documentation = ( + "https://python.langchain.com/docs/integrations/vectorstores/vectara" + ) beta = True icon = "Vectara" field_config = { @@ -28,8 +31,9 @@ class VectaraComponent(CustomComponent): "display_name": "Vectara API Key", "password": True, }, - "documents": { - "display_name": "Documents", + "inputs": { + "display_name": "Input", + "input_types": ["Document", "Record"], "info": "If provided, will be upserted to corpus (optional)", }, "files_url": { @@ -44,11 +48,18 @@ class VectaraComponent(CustomComponent): vectara_corpus_id: str, vectara_api_key: str, files_url: Optional[List[str]] = None, - documents: Optional[Document] = None, + inputs: Optional[Record] = None, ) -> Union[VectorStore, BaseRetriever]: source = "Langflow" - if documents is not None: + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) + + if documents: return Vectara.from_documents( documents=documents, # type: ignore embedding=FakeEmbeddings(size=768), diff --git a/src/backend/langflow/components/vectorstores/VectaraSearch.py b/src/backend/langflow/components/vectorstores/VectaraSearch.py index ae2d442be..cbc876f86 100644 --- a/src/backend/langflow/components/vectorstores/VectaraSearch.py +++ b/src/backend/langflow/components/vectorstores/VectaraSearch.py @@ -33,10 +33,6 @@ class VectaraSearchComponent(VectaraComponent, LCVectorStoreComponent): "display_name": "Vectara API Key", "password": True, }, - "documents": { - "display_name": "Documents", - "info": "If provided, will be upserted to corpus (optional)", - }, "files_url": { "display_name": "Files Url", "info": "Make vectara object using url of files (optional)", diff --git a/src/backend/langflow/components/vectorstores/Weaviate.py b/src/backend/langflow/components/vectorstores/Weaviate.py index 3d804255a..a85febcd5 100644 --- a/src/backend/langflow/components/vectorstores/Weaviate.py +++ b/src/backend/langflow/components/vectorstores/Weaviate.py @@ -2,16 +2,19 @@ from typing import Optional, Union import weaviate # type: ignore from langchain.embeddings.base import Embeddings -from langchain.schema import BaseRetriever, Document +from langchain.schema import BaseRetriever from langchain_community.vectorstores import VectorStore, Weaviate from langflow import CustomComponent +from langflow.schema.schema import Record class WeaviateVectorStoreComponent(CustomComponent): display_name: str = "Weaviate" description: str = "Implementation of Vector Store using Weaviate" - documentation = "https://python.langchain.com/docs/integrations/vectorstores/weaviate" + documentation = ( + "https://python.langchain.com/docs/integrations/vectorstores/weaviate" + ) beta = True field_config = { "url": {"display_name": "Weaviate URL", "value": "http://localhost:8080"}, @@ -30,7 +33,7 @@ class WeaviateVectorStoreComponent(CustomComponent): "advanced": True, "value": "text", }, - "documents": {"display_name": "Documents", "is_list": True}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "attributes": { "display_name": "Attributes", @@ -55,7 +58,7 @@ class WeaviateVectorStoreComponent(CustomComponent): index_name: Optional[str] = None, text_key: str = "text", embedding: Optional[Embeddings] = None, - documents: Optional[Document] = None, + inputs: Optional[Record] = None, attributes: Optional[list] = None, ) -> Union[VectorStore, BaseRetriever]: if api_key: @@ -78,8 +81,14 @@ class WeaviateVectorStoreComponent(CustomComponent): return pascal_case_word index_name = _to_pascal_case(index_name) if index_name else None + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) - if documents is not None and embedding is not None: + if documents and embedding is not None: return Weaviate.from_documents( client=client, index_name=index_name, diff --git a/src/backend/langflow/components/vectorstores/WeaviateSearch.py b/src/backend/langflow/components/vectorstores/WeaviateSearch.py index 6eee202c9..9b14ca779 100644 --- a/src/backend/langflow/components/vectorstores/WeaviateSearch.py +++ b/src/backend/langflow/components/vectorstores/WeaviateSearch.py @@ -39,7 +39,6 @@ class WeaviateSearchVectorStore(WeaviateVectorStoreComponent, LCVectorStoreCompo "advanced": True, "value": "text", }, - "documents": {"display_name": "Documents", "is_list": True}, "embedding": {"display_name": "Embedding"}, "attributes": { "display_name": "Attributes", diff --git a/src/backend/langflow/components/vectorstores/pgvector.py b/src/backend/langflow/components/vectorstores/pgvector.py index 2baf6dae6..e3b528313 100644 --- a/src/backend/langflow/components/vectorstores/pgvector.py +++ b/src/backend/langflow/components/vectorstores/pgvector.py @@ -3,9 +3,10 @@ from typing import Optional, Union from langchain.embeddings.base import Embeddings from langchain_community.vectorstores import VectorStore from langchain_community.vectorstores.pgvector import PGVector -from langchain_core.documents import Document from langchain_core.retrievers import BaseRetriever + from langflow import CustomComponent +from langflow.schema.schema import Record class PGVectorComponent(CustomComponent): @@ -15,7 +16,9 @@ class PGVectorComponent(CustomComponent): display_name: str = "PGVector" description: str = "Implementation of Vector Store using PostgreSQL" - documentation = "https://python.langchain.com/docs/integrations/vectorstores/pgvector" + documentation = ( + "https://python.langchain.com/docs/integrations/vectorstores/pgvector" + ) def build_config(self): """ @@ -26,7 +29,7 @@ class PGVectorComponent(CustomComponent): """ return { "code": {"show": False}, - "documents": {"display_name": "Documents", "is_list": True}, + "inputs": {"display_name": "Input", "input_types": ["Document", "Record"]}, "embedding": {"display_name": "Embedding"}, "pg_server_url": { "display_name": "PostgreSQL Server Connection String", @@ -40,7 +43,7 @@ class PGVectorComponent(CustomComponent): embedding: Embeddings, pg_server_url: str, collection_name: str, - documents: Optional[Document] = None, + inputs: Optional[Record] = None, ) -> Union[VectorStore, BaseRetriever]: """ Builds the Vector Store or BaseRetriever object. @@ -55,6 +58,12 @@ class PGVectorComponent(CustomComponent): - VectorStore: The Vector Store object. """ + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) try: if documents is None: vector_store = PGVector.from_existing_index( From b0d4a67863d4c97db98492f180ed82278a704bb7 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 19:16:57 -0300 Subject: [PATCH 06/26] Refactor URLComponent to use WebBaseLoader instead of UnstructuredURLLoader --- src/backend/langflow/components/data/URL.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/backend/langflow/components/data/URL.py b/src/backend/langflow/components/data/URL.py index 08eafeaa3..d3af63b1c 100644 --- a/src/backend/langflow/components/data/URL.py +++ b/src/backend/langflow/components/data/URL.py @@ -1,6 +1,6 @@ -from typing import Any, Dict, Optional +from typing import Any, Dict -from langchain_community.document_loaders.url import UnstructuredURLLoader +from langchain_community.document_loaders.web_base import WebBaseLoader from langflow import CustomComponent from langflow.schema import Record @@ -8,7 +8,7 @@ from langflow.schema import Record class URLComponent(CustomComponent): display_name = "URL" - description = "Load a URL." + description = "Load URLs and convert them to records." def build_config(self) -> Dict[str, Any]: return { @@ -18,9 +18,9 @@ class URLComponent(CustomComponent): async def build( self, urls: list[str], - ) -> Optional[Record]: + ) -> Record: - loader = UnstructuredURLLoader(urls=urls) + loader = WebBaseLoader(web_paths=urls) docs = loader.load() records = self.to_records(docs) return records From 7c91e6900433174dfbe6ba331fb95464ed09bb26 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 19:17:04 -0300 Subject: [PATCH 07/26] Refactor ChromaComponent to accept inputs as Records --- .../langflow/components/vectorstores/Chroma.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/backend/langflow/components/vectorstores/Chroma.py b/src/backend/langflow/components/vectorstores/Chroma.py index 139a877e9..2a247e8d6 100644 --- a/src/backend/langflow/components/vectorstores/Chroma.py +++ b/src/backend/langflow/components/vectorstores/Chroma.py @@ -2,11 +2,12 @@ from typing import List, Optional, Union import chromadb # type: ignore from langchain.embeddings.base import Embeddings -from langchain.schema import BaseRetriever, Document +from langchain.schema import BaseRetriever from langchain_community.vectorstores import VectorStore from langchain_community.vectorstores.chroma import Chroma from langflow import CustomComponent +from langflow.schema.schema import Record class ChromaComponent(CustomComponent): @@ -55,7 +56,7 @@ class ChromaComponent(CustomComponent): embedding: Embeddings, chroma_server_ssl_enabled: bool, index_directory: Optional[str] = None, - documents: Optional[List[Document]] = None, + inputs: Optional[List[Record]] = None, chroma_server_cors_allow_origins: Optional[str] = None, chroma_server_host: Optional[str] = None, chroma_server_port: Optional[int] = None, @@ -98,6 +99,12 @@ class ChromaComponent(CustomComponent): if index_directory is not None: index_directory = self.resolve_path(index_directory) + documents = [] + for _input in inputs: + if isinstance(_input, Record): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) if documents is not None and embedding is not None: if len(documents) == 0: raise ValueError( From afb3cfbc0589117959f705f81808e5c4678199e0 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 19:17:20 -0300 Subject: [PATCH 08/26] Remove console.log statements in updateBuildStatus function --- src/frontend/src/stores/flowStore.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/frontend/src/stores/flowStore.ts b/src/frontend/src/stores/flowStore.ts index cbf183289..e3cfbb20e 100644 --- a/src/frontend/src/stores/flowStore.ts +++ b/src/frontend/src/stores/flowStore.ts @@ -563,7 +563,6 @@ const useFlowStore = create((set, get) => ({ }); }, updateBuildStatus: (nodeIdList: string[], status: BuildStatus) => { - console.log("updateBuildStatus", nodeIdList, status); const newFlowBuildStatus = { ...get().flowBuildStatus }; nodeIdList.forEach((id) => { newFlowBuildStatus[id] = { @@ -573,7 +572,6 @@ const useFlowStore = create((set, get) => ({ const timestamp_string = new Date(Date.now()).toLocaleString(); newFlowBuildStatus[id].timestamp = timestamp_string; } - console.log("updateBuildStatus", newFlowBuildStatus); }); set({ flowBuildStatus: newFlowBuildStatus }); }, From 8d1dfdc2b60c2502c0ad000528fce95d9fa7b049 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 19:17:30 -0300 Subject: [PATCH 09/26] Add initial setup for loading starter projects into the database --- .../langflow/initial_setup/__init__.py | 0 src/backend/langflow/initial_setup/setup.py | 73 +++++++++++++++++++ src/backend/langflow/main.py | 16 +++- 3 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 src/backend/langflow/initial_setup/__init__.py create mode 100644 src/backend/langflow/initial_setup/setup.py diff --git a/src/backend/langflow/initial_setup/__init__.py b/src/backend/langflow/initial_setup/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/backend/langflow/initial_setup/setup.py b/src/backend/langflow/initial_setup/setup.py new file mode 100644 index 000000000..3b9ed8d5e --- /dev/null +++ b/src/backend/langflow/initial_setup/setup.py @@ -0,0 +1,73 @@ +import json +from datetime import datetime +from pathlib import Path + +from loguru import logger +from sqlmodel import select + +from langflow.services.database.models.flow.model import Flow +from langflow.services.deps import get_session + +STARTER_FOLDER_NAME = "Starter Projects" + + +# In the folder ./starter_projects we have a few JSON files that represent +# starter projects. We want to load these into the database so that users +# can use them as a starting point for their own projects. +def load_starter_projects(): + # Load the starter projects from the JSON files + # using Pathlib's glob method + starter_projects = [] + folder = Path(__file__).parent / "starter_projects" + for file in folder.glob("*.json"): + with open(file, "r") as f: + starter_projects.append(json.load(f)) + logger.info(f"Loaded starter project {file}") + return starter_projects + + +# We want to load the starter projects into the database +def create_or_update_starter_projects(): + session = next(get_session()) + starter_projects = load_starter_projects() + for project in starter_projects: + # Check if the project already exists in the database + project_name = project.get("name") + project_description = project.get("description") + project_is_component = project.get("is_component") + project_updated_at = project.get("updated_at") + # 2024-03-05T21:59:59.738081 + updated_at_datetime = datetime.strptime( + project_updated_at, "%Y-%m-%dT%H:%M:%S.%f" + ) + project_data = project.get("data") + if project_name and project_data: + existing_project = session.exec( + select(Flow).where( + Flow.name == project_name, Flow.folder == STARTER_FOLDER_NAME + ) + ).first() + if existing_project: + logger.info(f"Updating starter project {project_name}") + existing_project.data = project_data + existing_project.folder = STARTER_FOLDER_NAME + existing_project.description = project_description + existing_project.is_component = project_is_component + existing_project.updated_at = updated_at_datetime + # Now we need to update the project in the database + session.add(existing_project) + else: + logger.info(f"Creating starter project {project_name}") + session.add( + Flow( + name=project_name, + description=project_description, + is_component=project_is_component, + updated_at=updated_at_datetime, + folder=STARTER_FOLDER_NAME, + data=project_data, + ) + ) + session.commit() + session.close() + logger.info("Starter projects loaded into database") diff --git a/src/backend/langflow/main.py b/src/backend/langflow/main.py index 4b4c19a25..7c34d9b08 100644 --- a/src/backend/langflow/main.py +++ b/src/backend/langflow/main.py @@ -8,7 +8,9 @@ from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles + from langflow.api import router +from langflow.initial_setup.setup import create_or_update_starter_projects from langflow.interface.utils import setup_llm_caching from langflow.services.plugins.langfuse_plugin import LangfuseInstance from langflow.services.utils import initialize_services, teardown_services @@ -18,9 +20,12 @@ from langflow.utils.logger import configure def get_lifespan(fix_migration=False, socketio_server=None): @asynccontextmanager async def lifespan(app: FastAPI): - initialize_services(fix_migration=fix_migration, socketio_server=socketio_server) + initialize_services( + fix_migration=fix_migration, socketio_server=socketio_server + ) setup_llm_caching() LangfuseInstance.update() + create_or_update_starter_projects() yield teardown_services() @@ -31,7 +36,9 @@ def create_app(): """Create the FastAPI app and include the router.""" configure() - socketio_server = socketio.AsyncServer(async_mode="asgi", cors_allowed_origins="*", logger=True) + socketio_server = socketio.AsyncServer( + async_mode="asgi", cors_allowed_origins="*", logger=True + ) lifespan = get_lifespan(socketio_server=socketio_server) app = FastAPI(lifespan=lifespan) origins = ["*"] @@ -98,7 +105,9 @@ def get_static_files_dir(): return frontend_path / "frontend" -def setup_app(static_files_dir: Optional[Path] = None, backend_only: bool = False) -> FastAPI: +def setup_app( + static_files_dir: Optional[Path] = None, backend_only: bool = False +) -> FastAPI: """Setup the FastAPI app.""" # get the directory of the current file if not static_files_dir: @@ -114,6 +123,7 @@ def setup_app(static_files_dir: Optional[Path] = None, backend_only: bool = Fals if __name__ == "__main__": import uvicorn + from langflow.__main__ import get_number_of_workers configure() From 83c65fd3dfd5023df490f5651d0c2e0245771587 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 19:17:51 -0300 Subject: [PATCH 10/26] Add Data Ingestion Starter Project --- .../Langflow Data Ingestion.json | 1085 +++++++++++++++++ 1 file changed, 1085 insertions(+) create mode 100644 src/backend/langflow/initial_setup/starter_projects/Langflow Data Ingestion.json diff --git a/src/backend/langflow/initial_setup/starter_projects/Langflow Data Ingestion.json b/src/backend/langflow/initial_setup/starter_projects/Langflow Data Ingestion.json new file mode 100644 index 000000000..de8a8c1df --- /dev/null +++ b/src/backend/langflow/initial_setup/starter_projects/Langflow Data Ingestion.json @@ -0,0 +1,1085 @@ +{ + "name": "Data Ingestion", + "description": "This project is the starting point to insert data into a Vector Store. \n\nWe use the Vector Store Chroma but you can replace it with any other Vector Store. \n\nYou start by deciding what type of data you want to load, then you pick a place where you want to store the vectors and run it.\n\nThis will create a vector store in your local environment which you can query using the Chroma Search component.", + "data": { + "nodes": [ + { + "id": "RecursiveCharacterTextSplitter-jwfyG", + "type": "genericNode", + "position": { + "x": 1042.4388767006992, + "y": 633.2204634490822 + }, + "data": { + "type": "RecursiveCharacterTextSplitter", + "node": { + "template": { + "inputs": { + "type": "Document", + "required": true, + "placeholder": "", + "list": true, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "inputs", + "display_name": "Input", + "advanced": false, + "input_types": [ + "Document", + "Record" + ], + "dynamic": false, + "info": "The texts to split.", + "title_case": false + }, + "chunk_overlap": { + "type": "int", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": 200, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chunk_overlap", + "display_name": "Chunk Overlap", + "advanced": false, + "dynamic": false, + "info": "The amount of overlap between chunks.", + "title_case": false + }, + "chunk_size": { + "type": "int", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": 1000, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chunk_size", + "display_name": "Chunk Size", + "advanced": false, + "dynamic": false, + "info": "The maximum length of each chunk.", + "title_case": false + }, + "code": { + "type": "code", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": true, + "value": "from typing import Optional\n\nfrom langchain.text_splitter import RecursiveCharacterTextSplitter\nfrom langchain_core.documents import Document\n\nfrom langflow import CustomComponent\nfrom langflow.schema import Record\nfrom langflow.utils.util import build_loader_repr_from_documents\n\n\nclass RecursiveCharacterTextSplitterComponent(CustomComponent):\n display_name: str = \"Recursive Character Text Splitter\"\n description: str = \"Split text into chunks of a specified length.\"\n documentation: str = (\n \"https://docs.langflow.org/components/text-splitters#recursivecharactertextsplitter\"\n )\n\n def build_config(self):\n return {\n \"inputs\": {\n \"display_name\": \"Input\",\n \"info\": \"The texts to split.\",\n \"input_types\": [\"Document\", \"Record\"],\n },\n \"separators\": {\n \"display_name\": \"Separators\",\n \"info\": 'The characters to split on.\\nIf left empty defaults to [\"\\\\n\\\\n\", \"\\\\n\", \" \", \"\"].',\n \"is_list\": True,\n },\n \"chunk_size\": {\n \"display_name\": \"Chunk Size\",\n \"info\": \"The maximum length of each chunk.\",\n \"field_type\": \"int\",\n \"value\": 1000,\n },\n \"chunk_overlap\": {\n \"display_name\": \"Chunk Overlap\",\n \"info\": \"The amount of overlap between chunks.\",\n \"field_type\": \"int\",\n \"value\": 200,\n },\n \"code\": {\"show\": False},\n }\n\n def build(\n self,\n inputs: list[Document],\n separators: Optional[list[str]] = None,\n chunk_size: Optional[int] = 1000,\n chunk_overlap: Optional[int] = 200,\n ) -> list[Record]:\n \"\"\"\n Split text into chunks of a specified length.\n\n Args:\n separators (list[str]): The characters to split on.\n chunk_size (int): The maximum length of each chunk.\n chunk_overlap (int): The amount of overlap between chunks.\n length_function (function): The function to use to calculate the length of the text.\n\n Returns:\n list[str]: The chunks of text.\n \"\"\"\n\n if separators == \"\":\n separators = None\n elif separators:\n # check if the separators list has escaped characters\n # if there are escaped characters, unescape them\n separators = [x.encode().decode(\"unicode-escape\") for x in separators]\n\n # Make sure chunk_size and chunk_overlap are ints\n if isinstance(chunk_size, str):\n chunk_size = int(chunk_size)\n if isinstance(chunk_overlap, str):\n chunk_overlap = int(chunk_overlap)\n splitter = RecursiveCharacterTextSplitter(\n separators=separators,\n chunk_size=chunk_size,\n chunk_overlap=chunk_overlap,\n )\n documents = []\n for _input in inputs:\n if isinstance(_input, Record):\n documents.append(_input.to_lc_document())\n else:\n documents.append(_input)\n docs = splitter.split_documents(documents)\n self.repr_value = build_loader_repr_from_documents(docs)\n return self.to_records(docs)\n", + "fileTypes": [], + "file_path": "", + "password": false, + "name": "code", + "advanced": false, + "dynamic": true, + "info": "", + "title_case": false + }, + "separators": { + "type": "str", + "required": false, + "placeholder": "", + "list": true, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "separators", + "display_name": "Separators", + "advanced": false, + "dynamic": false, + "info": "The characters to split on.\nIf left empty defaults to [\"\\n\\n\", \"\\n\", \" \", \"\"].", + "title_case": false, + "input_types": [ + "Text" + ], + "value": [ + "\\n" + ] + }, + "_type": "CustomComponent" + }, + "description": "Split text into chunks of a specified length.", + "base_classes": [ + "Record" + ], + "display_name": "Recursive Character Text Splitter", + "documentation": "https://docs.langflow.org/components/text-splitters#recursivecharactertextsplitter", + "custom_fields": { + "inputs": null, + "separators": null, + "chunk_size": null, + "chunk_overlap": null + }, + "output_types": [ + "Record" + ], + "field_formatters": {}, + "frozen": false, + "beta": true + }, + "id": "RecursiveCharacterTextSplitter-jwfyG" + }, + "selected": false, + "width": 384, + "height": 509, + "positionAbsolute": { + "x": 1042.4388767006992, + "y": 633.2204634490822 + }, + "dragging": false + }, + { + "id": "Chroma-aFGHF", + "type": "genericNode", + "position": { + "x": 1641.280676720732, + "y": 356.94961598422196 + }, + "data": { + "type": "Chroma", + "node": { + "template": { + "embedding": { + "type": "Embeddings", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "embedding", + "display_name": "Embedding", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false + }, + "inputs": { + "type": "Record", + "required": false, + "placeholder": "", + "list": true, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "inputs", + "display_name": "Input", + "advanced": false, + "input_types": [ + "Document", + "Record" + ], + "dynamic": false, + "info": "", + "title_case": false + }, + "chroma_server_cors_allow_origins": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chroma_server_cors_allow_origins", + "display_name": "Server CORS Allow Origins", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "chroma_server_grpc_port": { + "type": "int", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chroma_server_grpc_port", + "display_name": "Server gRPC Port", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "chroma_server_host": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chroma_server_host", + "display_name": "Server Host", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "chroma_server_port": { + "type": "int", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chroma_server_port", + "display_name": "Server Port", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "chroma_server_ssl_enabled": { + "type": "bool", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chroma_server_ssl_enabled", + "display_name": "Server SSL Enabled", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "code": { + "type": "code", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": true, + "value": "from typing import List, Optional, Union\n\nimport chromadb # type: ignore\nfrom langchain.embeddings.base import Embeddings\nfrom langchain.schema import BaseRetriever\nfrom langchain_community.vectorstores import VectorStore\nfrom langchain_community.vectorstores.chroma import Chroma\n\nfrom langflow import CustomComponent\nfrom langflow.schema.schema import Record\n\n\nclass ChromaComponent(CustomComponent):\n \"\"\"\n A custom component for implementing a Vector Store using Chroma.\n \"\"\"\n\n display_name: str = \"Chroma\"\n description: str = \"Implementation of Vector Store using Chroma\"\n documentation = \"https://python.langchain.com/docs/integrations/vectorstores/chroma\"\n beta: bool = True\n icon = \"Chroma\"\n\n def build_config(self):\n \"\"\"\n Builds the configuration for the component.\n\n Returns:\n - dict: A dictionary containing the configuration options for the component.\n \"\"\"\n return {\n \"collection_name\": {\"display_name\": \"Collection Name\", \"value\": \"langflow\"},\n \"index_directory\": {\"display_name\": \"Persist Directory\"},\n \"code\": {\"advanced\": True, \"display_name\": \"Code\"},\n \"inputs\": {\"display_name\": \"Input\", \"input_types\": [\"Document\", \"Record\"]},\n \"embedding\": {\"display_name\": \"Embedding\"},\n \"chroma_server_cors_allow_origins\": {\n \"display_name\": \"Server CORS Allow Origins\",\n \"advanced\": True,\n },\n \"chroma_server_host\": {\"display_name\": \"Server Host\", \"advanced\": True},\n \"chroma_server_port\": {\"display_name\": \"Server Port\", \"advanced\": True},\n \"chroma_server_grpc_port\": {\n \"display_name\": \"Server gRPC Port\",\n \"advanced\": True,\n },\n \"chroma_server_ssl_enabled\": {\n \"display_name\": \"Server SSL Enabled\",\n \"advanced\": True,\n },\n }\n\n def build(\n self,\n collection_name: str,\n embedding: Embeddings,\n chroma_server_ssl_enabled: bool,\n index_directory: Optional[str] = None,\n inputs: Optional[List[Record]] = None,\n chroma_server_cors_allow_origins: Optional[str] = None,\n chroma_server_host: Optional[str] = None,\n chroma_server_port: Optional[int] = None,\n chroma_server_grpc_port: Optional[int] = None,\n ) -> Union[VectorStore, BaseRetriever]:\n \"\"\"\n Builds the Vector Store or BaseRetriever object.\n\n Args:\n - collection_name (str): The name of the collection.\n - index_directory (Optional[str]): The directory to persist the Vector Store to.\n - chroma_server_ssl_enabled (bool): Whether to enable SSL for the Chroma server.\n - embedding (Optional[Embeddings]): The embeddings to use for the Vector Store.\n - documents (Optional[Document]): The documents to use for the Vector Store.\n - chroma_server_cors_allow_origins (Optional[str]): The CORS allow origins for the Chroma server.\n - chroma_server_host (Optional[str]): The host for the Chroma server.\n - chroma_server_port (Optional[int]): The port for the Chroma server.\n - chroma_server_grpc_port (Optional[int]): The gRPC port for the Chroma server.\n\n Returns:\n - Union[VectorStore, BaseRetriever]: The Vector Store or BaseRetriever object.\n \"\"\"\n\n # Chroma settings\n chroma_settings = None\n\n if chroma_server_host is not None:\n chroma_settings = chromadb.config.Settings(\n chroma_server_cors_allow_origins=chroma_server_cors_allow_origins\n or None,\n chroma_server_host=chroma_server_host,\n chroma_server_port=chroma_server_port or None,\n chroma_server_grpc_port=chroma_server_grpc_port or None,\n chroma_server_ssl_enabled=chroma_server_ssl_enabled,\n )\n\n # If documents, then we need to create a Chroma instance using .from_documents\n\n # Check index_directory and expand it if it is a relative path\n if index_directory is not None:\n index_directory = self.resolve_path(index_directory)\n\n documents = []\n for _input in inputs:\n if isinstance(_input, Record):\n documents.append(_input.to_lc_document())\n else:\n documents.append(_input)\n if documents is not None and embedding is not None:\n if len(documents) == 0:\n raise ValueError(\n \"If documents are provided, there must be at least one document.\"\n )\n chroma = Chroma.from_documents(\n documents=documents, # type: ignore\n persist_directory=index_directory,\n collection_name=collection_name,\n embedding=embedding,\n client_settings=chroma_settings,\n )\n else:\n chroma = Chroma(\n persist_directory=index_directory,\n client_settings=chroma_settings,\n embedding_function=embedding,\n )\n return chroma\n", + "fileTypes": [], + "file_path": "", + "password": false, + "name": "code", + "advanced": true, + "dynamic": true, + "info": "", + "title_case": false + }, + "collection_name": { + "type": "str", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": "langflow_contrib", + "fileTypes": [], + "file_path": "", + "password": false, + "name": "collection_name", + "display_name": "Collection Name", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "index_directory": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "index_directory", + "display_name": "Persist Directory", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ], + "value": "./chroma_langflow" + }, + "_type": "CustomComponent" + }, + "description": "Implementation of Vector Store using Chroma", + "icon": "Chroma", + "base_classes": [ + "Serializable", + "VectorStore", + "object", + "Runnable", + "BaseRetriever", + "RunnableSerializable", + "Generic" + ], + "display_name": "Chroma", + "documentation": "https://python.langchain.com/docs/integrations/vectorstores/chroma", + "custom_fields": { + "collection_name": null, + "embedding": null, + "chroma_server_ssl_enabled": null, + "index_directory": null, + "inputs": null, + "chroma_server_cors_allow_origins": null, + "chroma_server_host": null, + "chroma_server_port": null, + "chroma_server_grpc_port": null + }, + "output_types": [ + "VectorStore", + "BaseRetriever" + ], + "field_formatters": {}, + "frozen": false, + "beta": true + }, + "id": "Chroma-aFGHF" + }, + "selected": true, + "width": 384, + "height": 495, + "positionAbsolute": { + "x": 1641.280676720732, + "y": 356.94961598422196 + }, + "dragging": false + }, + { + "id": "OpenAIEmbeddings-rbMk3", + "type": "genericNode", + "position": { + "x": 1053.9472627140208, + "y": -2.5921878249999963 + }, + "data": { + "type": "OpenAIEmbeddings", + "node": { + "template": { + "allowed_special": { + "type": "str", + "required": true, + "placeholder": "", + "list": true, + "show": true, + "multiline": false, + "value": [], + "fileTypes": [], + "file_path": "", + "password": false, + "name": "allowed_special", + "display_name": "Allowed Special", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "chunk_size": { + "type": "int", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": 1000, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "chunk_size", + "display_name": "Chunk Size", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "client": { + "type": "Any", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "client", + "display_name": "Client", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "code": { + "type": "code", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": true, + "value": "from typing import Any, Callable, Dict, List, Optional, Union\n\nfrom langchain_openai.embeddings.base import OpenAIEmbeddings\nfrom langflow import CustomComponent\nfrom langflow.field_typing import NestedDict\nfrom pydantic.v1.types import SecretStr\n\n\nclass OpenAIEmbeddingsComponent(CustomComponent):\n display_name = \"OpenAIEmbeddings\"\n description = \"OpenAI embedding models\"\n\n def build_config(self):\n return {\n \"allowed_special\": {\n \"display_name\": \"Allowed Special\",\n \"advanced\": True,\n \"field_type\": \"str\",\n \"is_list\": True,\n },\n \"default_headers\": {\n \"display_name\": \"Default Headers\",\n \"advanced\": True,\n \"field_type\": \"dict\",\n },\n \"default_query\": {\n \"display_name\": \"Default Query\",\n \"advanced\": True,\n \"field_type\": \"NestedDict\",\n },\n \"disallowed_special\": {\n \"display_name\": \"Disallowed Special\",\n \"advanced\": True,\n \"field_type\": \"str\",\n \"is_list\": True,\n },\n \"chunk_size\": {\"display_name\": \"Chunk Size\", \"advanced\": True},\n \"client\": {\"display_name\": \"Client\", \"advanced\": True},\n \"deployment\": {\"display_name\": \"Deployment\", \"advanced\": True},\n \"embedding_ctx_length\": {\n \"display_name\": \"Embedding Context Length\",\n \"advanced\": True,\n },\n \"max_retries\": {\"display_name\": \"Max Retries\", \"advanced\": True},\n \"model\": {\n \"display_name\": \"Model\",\n \"advanced\": False,\n \"options\": [\"text-embedding-3-small\", \"text-embedding-3-large\", \"text-embedding-ada-002\"],\n },\n \"model_kwargs\": {\"display_name\": \"Model Kwargs\", \"advanced\": True},\n \"openai_api_base\": {\"display_name\": \"OpenAI API Base\", \"password\": True, \"advanced\": True},\n \"openai_api_key\": {\"display_name\": \"OpenAI API Key\", \"password\": True},\n \"openai_api_type\": {\"display_name\": \"OpenAI API Type\", \"advanced\": True, \"password\": True},\n \"openai_api_version\": {\n \"display_name\": \"OpenAI API Version\",\n \"advanced\": True,\n },\n \"openai_organization\": {\n \"display_name\": \"OpenAI Organization\",\n \"advanced\": True,\n },\n \"openai_proxy\": {\"display_name\": \"OpenAI Proxy\", \"advanced\": True},\n \"request_timeout\": {\"display_name\": \"Request Timeout\", \"advanced\": True},\n \"show_progress_bar\": {\n \"display_name\": \"Show Progress Bar\",\n \"advanced\": True,\n },\n \"skip_empty\": {\"display_name\": \"Skip Empty\", \"advanced\": True},\n \"tiktoken_model_name\": {\"display_name\": \"TikToken Model Name\"},\n \"tikToken_enable\": {\"display_name\": \"TikToken Enable\", \"advanced\": True},\n }\n\n def build(\n self,\n default_headers: Optional[Dict[str, str]] = None,\n default_query: Optional[NestedDict] = {},\n allowed_special: List[str] = [],\n disallowed_special: List[str] = [\"all\"],\n chunk_size: int = 1000,\n client: Optional[Any] = None,\n deployment: str = \"text-embedding-3-small\",\n embedding_ctx_length: int = 8191,\n max_retries: int = 6,\n model: str = \"text-embedding-3-small\",\n model_kwargs: NestedDict = {},\n openai_api_base: Optional[str] = None,\n openai_api_key: Optional[str] = \"\",\n openai_api_type: Optional[str] = None,\n openai_api_version: Optional[str] = None,\n openai_organization: Optional[str] = None,\n openai_proxy: Optional[str] = None,\n request_timeout: Optional[float] = None,\n show_progress_bar: bool = False,\n skip_empty: bool = False,\n tiktoken_enable: bool = True,\n tiktoken_model_name: Optional[str] = None,\n ) -> Union[OpenAIEmbeddings, Callable]:\n # This is to avoid errors with Vector Stores (e.g Chroma)\n if disallowed_special == [\"all\"]:\n disallowed_special = \"all\" # type: ignore\n\n api_key = SecretStr(openai_api_key) if openai_api_key else None\n\n return OpenAIEmbeddings(\n tiktoken_enabled=tiktoken_enable,\n default_headers=default_headers,\n default_query=default_query,\n allowed_special=set(allowed_special),\n disallowed_special=\"all\",\n chunk_size=chunk_size,\n client=client,\n deployment=deployment,\n embedding_ctx_length=embedding_ctx_length,\n max_retries=max_retries,\n model=model,\n model_kwargs=model_kwargs,\n base_url=openai_api_base,\n api_key=api_key,\n openai_api_type=openai_api_type,\n api_version=openai_api_version,\n organization=openai_organization,\n openai_proxy=openai_proxy,\n timeout=request_timeout,\n show_progress_bar=show_progress_bar,\n skip_empty=skip_empty,\n tiktoken_model_name=tiktoken_model_name,\n )\n", + "fileTypes": [], + "file_path": "", + "password": false, + "name": "code", + "advanced": false, + "dynamic": true, + "info": "", + "title_case": false + }, + "default_headers": { + "type": "dict", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "default_headers", + "display_name": "Default Headers", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "default_query": { + "type": "NestedDict", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": {}, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "default_query", + "display_name": "Default Query", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "deployment": { + "type": "str", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": "text-embedding-3-small", + "fileTypes": [], + "file_path": "", + "password": false, + "name": "deployment", + "display_name": "Deployment", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "disallowed_special": { + "type": "str", + "required": true, + "placeholder": "", + "list": true, + "show": true, + "multiline": false, + "value": [ + "all" + ], + "fileTypes": [], + "file_path": "", + "password": false, + "name": "disallowed_special", + "display_name": "Disallowed Special", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "embedding_ctx_length": { + "type": "int", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": 8191, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "embedding_ctx_length", + "display_name": "Embedding Context Length", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "max_retries": { + "type": "int", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": 6, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "max_retries", + "display_name": "Max Retries", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "model": { + "type": "str", + "required": true, + "placeholder": "", + "list": true, + "show": true, + "multiline": false, + "value": "text-embedding-3-small", + "fileTypes": [], + "file_path": "", + "password": false, + "options": [ + "text-embedding-3-small", + "text-embedding-3-large", + "text-embedding-ada-002" + ], + "name": "model", + "display_name": "Model", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "model_kwargs": { + "type": "NestedDict", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": {}, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "model_kwargs", + "display_name": "Model Kwargs", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "openai_api_base": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": true, + "name": "openai_api_base", + "display_name": "OpenAI API Base", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ], + "value": "" + }, + "openai_api_key": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": "", + "fileTypes": [], + "file_path": "", + "password": true, + "name": "openai_api_key", + "display_name": "OpenAI API Key", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "openai_api_type": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": true, + "name": "openai_api_type", + "display_name": "OpenAI API Type", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ], + "value": "" + }, + "openai_api_version": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "openai_api_version", + "display_name": "OpenAI API Version", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "openai_organization": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "openai_organization", + "display_name": "OpenAI Organization", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "openai_proxy": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "openai_proxy", + "display_name": "OpenAI Proxy", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "request_timeout": { + "type": "float", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "request_timeout", + "display_name": "Request Timeout", + "advanced": true, + "dynamic": false, + "info": "", + "rangeSpec": { + "min": -1, + "max": 1, + "step": 0.1 + }, + "title_case": false + }, + "show_progress_bar": { + "type": "bool", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "show_progress_bar", + "display_name": "Show Progress Bar", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "skip_empty": { + "type": "bool", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "skip_empty", + "display_name": "Skip Empty", + "advanced": true, + "dynamic": false, + "info": "", + "title_case": false + }, + "tiktoken_enable": { + "type": "bool", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "value": true, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "tiktoken_enable", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false + }, + "tiktoken_model_name": { + "type": "str", + "required": false, + "placeholder": "", + "list": false, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "tiktoken_model_name", + "display_name": "TikToken Model Name", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ] + }, + "_type": "CustomComponent" + }, + "description": "OpenAI embedding models", + "base_classes": [ + "Embeddings", + "OpenAIEmbeddings", + "Callable" + ], + "display_name": "OpenAIEmbeddings", + "documentation": "", + "custom_fields": { + "default_headers": null, + "default_query": null, + "allowed_special": null, + "disallowed_special": null, + "chunk_size": null, + "client": null, + "deployment": null, + "embedding_ctx_length": null, + "max_retries": null, + "model": null, + "model_kwargs": null, + "openai_api_base": null, + "openai_api_key": null, + "openai_api_type": null, + "openai_api_version": null, + "openai_organization": null, + "openai_proxy": null, + "request_timeout": null, + "show_progress_bar": null, + "skip_empty": null, + "tiktoken_enable": null, + "tiktoken_model_name": null + }, + "output_types": [ + "OpenAIEmbeddings", + "Callable" + ], + "field_formatters": {}, + "frozen": false, + "beta": true + }, + "id": "OpenAIEmbeddings-rbMk3" + }, + "selected": false, + "width": 384, + "height": 573, + "positionAbsolute": { + "x": 1053.9472627140208, + "y": -2.5921878249999963 + }, + "dragging": false + }, + { + "id": "URL-5zjQH", + "type": "genericNode", + "position": { + "x": 567.0838444398559, + "y": 596.6568151511171 + }, + "data": { + "type": "URL", + "node": { + "template": { + "code": { + "type": "code", + "required": true, + "placeholder": "", + "list": false, + "show": true, + "multiline": true, + "value": "from typing import Any, Dict\n\nfrom langchain_community.document_loaders.web_base import WebBaseLoader\n\nfrom langflow import CustomComponent\nfrom langflow.schema import Record\n\n\nclass URLComponent(CustomComponent):\n display_name = \"URL\"\n description = \"Load URLs and convert them to records.\"\n\n def build_config(self) -> Dict[str, Any]:\n return {\n \"urls\": {\"display_name\": \"URL\"},\n }\n\n async def build(\n self,\n urls: list[str],\n ) -> Record:\n\n loader = WebBaseLoader(web_paths=urls)\n docs = loader.load()\n records = self.to_records(docs)\n return records\n", + "fileTypes": [], + "file_path": "", + "password": false, + "name": "code", + "advanced": false, + "dynamic": true, + "info": "", + "title_case": false + }, + "urls": { + "type": "str", + "required": true, + "placeholder": "", + "list": true, + "show": true, + "multiline": false, + "fileTypes": [], + "file_path": "", + "password": false, + "name": "urls", + "display_name": "URL", + "advanced": false, + "dynamic": false, + "info": "", + "title_case": false, + "input_types": [ + "Text" + ], + "value": [ + "https://raw.githubusercontent.com/logspace-ai/langflow/dev/CONTRIBUTING.md" + ] + }, + "_type": "CustomComponent" + }, + "description": "Load URLs and convert them to records.", + "base_classes": [ + "Record" + ], + "display_name": "URL", + "documentation": "", + "custom_fields": { + "urls": null + }, + "output_types": [ + "Record" + ], + "field_formatters": {}, + "frozen": false, + "beta": true + }, + "id": "URL-5zjQH" + }, + "selected": false, + "width": 384, + "height": 289, + "dragging": false, + "positionAbsolute": { + "x": 567.0838444398559, + "y": 596.6568151511171 + } + } + ], + "edges": [ + { + "source": "RecursiveCharacterTextSplitter-jwfyG", + "sourceHandle": "{œbaseClassesœ:[œRecordœ],œdataTypeœ:œRecursiveCharacterTextSplitterœ,œidœ:œRecursiveCharacterTextSplitter-jwfyGœ}", + "target": "Chroma-aFGHF", + "targetHandle": "{œfieldNameœ:œinputsœ,œidœ:œChroma-aFGHFœ,œinputTypesœ:[œDocumentœ,œRecordœ],œtypeœ:œRecordœ}", + "data": { + "targetHandle": { + "fieldName": "inputs", + "id": "Chroma-aFGHF", + "inputTypes": [ + "Document", + "Record" + ], + "type": "Record" + }, + "sourceHandle": { + "baseClasses": [ + "Record" + ], + "dataType": "RecursiveCharacterTextSplitter", + "id": "RecursiveCharacterTextSplitter-jwfyG" + } + }, + "style": { + "stroke": "#555" + }, + "className": "stroke-gray-900 stroke-connection", + "id": "reactflow__edge-RecursiveCharacterTextSplitter-jwfyG{œbaseClassesœ:[œRecordœ],œdataTypeœ:œRecursiveCharacterTextSplitterœ,œidœ:œRecursiveCharacterTextSplitter-jwfyGœ}-Chroma-aFGHF{œfieldNameœ:œinputsœ,œidœ:œChroma-aFGHFœ,œinputTypesœ:[œDocumentœ,œRecordœ],œtypeœ:œRecordœ}" + }, + { + "source": "OpenAIEmbeddings-rbMk3", + "sourceHandle": "{œbaseClassesœ:[œEmbeddingsœ,œOpenAIEmbeddingsœ,œCallableœ],œdataTypeœ:œOpenAIEmbeddingsœ,œidœ:œOpenAIEmbeddings-rbMk3œ}", + "target": "Chroma-aFGHF", + "targetHandle": "{œfieldNameœ:œembeddingœ,œidœ:œChroma-aFGHFœ,œinputTypesœ:null,œtypeœ:œEmbeddingsœ}", + "data": { + "targetHandle": { + "fieldName": "embedding", + "id": "Chroma-aFGHF", + "inputTypes": null, + "type": "Embeddings" + }, + "sourceHandle": { + "baseClasses": [ + "Embeddings", + "OpenAIEmbeddings", + "Callable" + ], + "dataType": "OpenAIEmbeddings", + "id": "OpenAIEmbeddings-rbMk3" + } + }, + "style": { + "stroke": "#555" + }, + "className": "stroke-gray-900 stroke-connection", + "id": "reactflow__edge-OpenAIEmbeddings-rbMk3{œbaseClassesœ:[œEmbeddingsœ,œOpenAIEmbeddingsœ,œCallableœ],œdataTypeœ:œOpenAIEmbeddingsœ,œidœ:œOpenAIEmbeddings-rbMk3œ}-Chroma-aFGHF{œfieldNameœ:œembeddingœ,œidœ:œChroma-aFGHFœ,œinputTypesœ:null,œtypeœ:œEmbeddingsœ}" + }, + { + "source": "URL-5zjQH", + "sourceHandle": "{œbaseClassesœ:[œRecordœ],œdataTypeœ:œURLœ,œidœ:œURL-5zjQHœ}", + "target": "RecursiveCharacterTextSplitter-jwfyG", + "targetHandle": "{œfieldNameœ:œinputsœ,œidœ:œRecursiveCharacterTextSplitter-jwfyGœ,œinputTypesœ:[œDocumentœ,œRecordœ],œtypeœ:œDocumentœ}", + "data": { + "targetHandle": { + "fieldName": "inputs", + "id": "RecursiveCharacterTextSplitter-jwfyG", + "inputTypes": [ + "Document", + "Record" + ], + "type": "Document" + }, + "sourceHandle": { + "baseClasses": [ + "Record" + ], + "dataType": "URL", + "id": "URL-5zjQH" + } + }, + "style": { + "stroke": "#555" + }, + "className": "stroke-foreground stroke-connection", + "id": "reactflow__edge-URL-5zjQH{œbaseClassesœ:[œRecordœ],œdataTypeœ:œURLœ,œidœ:œURL-5zjQHœ}-RecursiveCharacterTextSplitter-jwfyG{œfieldNameœ:œinputsœ,œidœ:œRecursiveCharacterTextSplitter-jwfyGœ,œinputTypesœ:[œDocumentœ,œRecordœ],œtypeœ:œDocumentœ}" + } + ], + "viewport": { + "x": -160.3219973143573, + "y": 117.63775645863632, + "zoom": 0.48903173672366845 + } + }, + "is_component": false, + "updated_at": "2024-03-05T21:59:59.738081", + "folder": null, + "id": "7f90dc54-717d-49fe-a43f-c4dc055daa4e", + "user_id": "9365dbda-e8cf-4e95-8c84-49f8b6edb44f" +} \ No newline at end of file From 9d61ece65dc3e4245cf9f7c4e1652ccbdf39f229 Mon Sep 17 00:00:00 2001 From: anovazzi1 Date: Tue, 5 Mar 2024 19:25:39 -0300 Subject: [PATCH 11/26] Add new components and update styleUtils.ts --- .../components/NewFlowCardComponent/index.tsx | 54 +++++++++++++ .../src/components/exampleComponent/index.tsx | 78 +++++++++++++++++++ src/frontend/src/pages/MainPage/index.tsx | 38 ++++++--- src/frontend/src/utils/styleUtils.ts | 6 ++ 4 files changed, 167 insertions(+), 9 deletions(-) create mode 100644 src/frontend/src/components/NewFlowCardComponent/index.tsx create mode 100644 src/frontend/src/components/exampleComponent/index.tsx diff --git a/src/frontend/src/components/NewFlowCardComponent/index.tsx b/src/frontend/src/components/NewFlowCardComponent/index.tsx new file mode 100644 index 000000000..131fb3543 --- /dev/null +++ b/src/frontend/src/components/NewFlowCardComponent/index.tsx @@ -0,0 +1,54 @@ +import { useEffect, useState } from "react"; +import { getComponent, postLikeComponent } from "../../controllers/API"; +import DeleteConfirmationModal from "../../modals/DeleteConfirmationModal"; +import useAlertStore from "../../stores/alertStore"; +import useFlowsManagerStore from "../../stores/flowsManagerStore"; +import { useStoreStore } from "../../stores/storeStore"; +import { storeComponent } from "../../types/store"; +import cloneFLowWithParent from "../../utils/storeUtils"; +import { cn } from "../../utils/utils"; +import ShadTooltip from "../ShadTooltipComponent"; +import IconComponent from "../genericIconComponent"; +import { Badge } from "../ui/badge"; +import { Button } from "../ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "../ui/card"; +import { FlowType } from "../../types/flow"; +import { useNavigate } from "react-router-dom"; + +export default function NewFlowCardComponent({ +}: { + }) { + const addFlow = useFlowsManagerStore((state) => state.addFlow); + const navigate = useNavigate(); + + return ( + + + + + + ); +} diff --git a/src/frontend/src/components/exampleComponent/index.tsx b/src/frontend/src/components/exampleComponent/index.tsx new file mode 100644 index 000000000..af0cf56ea --- /dev/null +++ b/src/frontend/src/components/exampleComponent/index.tsx @@ -0,0 +1,78 @@ +import { useEffect, useState } from "react"; +import { getComponent, postLikeComponent } from "../../controllers/API"; +import DeleteConfirmationModal from "../../modals/DeleteConfirmationModal"; +import useAlertStore from "../../stores/alertStore"; +import useFlowsManagerStore from "../../stores/flowsManagerStore"; +import { useStoreStore } from "../../stores/storeStore"; +import { storeComponent } from "../../types/store"; +import cloneFLowWithParent from "../../utils/storeUtils"; +import { cn } from "../../utils/utils"; +import ShadTooltip from "../ShadTooltipComponent"; +import IconComponent from "../genericIconComponent"; +import { Badge } from "../ui/badge"; +import { Button } from "../ui/button"; +import { + Card, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "../ui/card"; +import { FlowType } from "../../types/flow"; + +export default function CollectionCardComponent({ + data, +}: { + data: FlowType; + authorized?: boolean; +}) { + const addFlow = useFlowsManagerStore((state) => state.addFlow); + + return ( + +
+ +
+ + + +
{data.name}
+ + +
+ +
{data.description}
+ + +
+ + +
+
+ +
+
+ + + ); +} diff --git a/src/frontend/src/pages/MainPage/index.tsx b/src/frontend/src/pages/MainPage/index.tsx index cc4115e7e..fc8747094 100644 --- a/src/frontend/src/pages/MainPage/index.tsx +++ b/src/frontend/src/pages/MainPage/index.tsx @@ -1,5 +1,5 @@ import { Group, ToyBrick } from "lucide-react"; -import { useEffect } from "react"; +import { useEffect, useState } from "react"; import { Outlet, useLocation, useNavigate } from "react-router-dom"; import DropdownButton from "../../components/DropdownButtonComponent"; import IconComponent from "../../components/genericIconComponent"; @@ -14,6 +14,9 @@ import { import useAlertStore from "../../stores/alertStore"; import useFlowsManagerStore from "../../stores/flowsManagerStore"; import { downloadFlows } from "../../utils/reactflowUtils"; +import BaseModal from "../../modals/baseModal"; +import ExampleCardComponent from "../../components/exampleComponent"; +import NewFlowCardComponent from "../../components/NewFlowCardComponent"; export default function HomePage(): JSX.Element { const addFlow = useFlowsManagerStore((state) => state.addFlow); const uploadFlow = useFlowsManagerStore((state) => state.uploadFlow); @@ -25,6 +28,7 @@ export default function HomePage(): JSX.Element { const setErrorData = useAlertStore((state) => state.setErrorData); const location = useLocation(); const pathname = location.pathname; + const [openModal, setOpenModal] = useState(false); const is_component = pathname === "/components"; const dropdownOptions = [ { @@ -36,9 +40,8 @@ export default function HomePage(): JSX.Element { }) .then((id) => { setSuccessData({ - title: `${ - is_component ? "Component" : "Flow" - } uploaded successfully`, + title: `${is_component ? "Component" : "Flow" + } uploaded successfully`, }); if (!is_component) navigate("/flow/" + id); }) @@ -98,11 +101,7 @@ export default function HomePage(): JSX.Element { { - addFlow(true).then((id) => { - navigate("/flow/" + id); - }); - }} + onFirstBtnClick={() => setOpenModal(true)} options={dropdownOptions} /> @@ -116,6 +115,27 @@ export default function HomePage(): JSX.Element { + + + + Create a New Flow + + + +
+ + +
+ + ); } diff --git a/src/frontend/src/utils/styleUtils.ts b/src/frontend/src/utils/styleUtils.ts index 601066407..96d754e85 100644 --- a/src/frontend/src/utils/styleUtils.ts +++ b/src/frontend/src/utils/styleUtils.ts @@ -5,8 +5,11 @@ import { Bell, BookMarked, BookmarkPlus, + PlusCircle, + PlusSquare, Bot, Boxes, + GitCompare, Braces, Check, CheckCircle2, @@ -392,6 +395,8 @@ export const nodeIconsLucide: iconsType = { Circle, CircleDot, Clipboard, + PlusCircle, + PlusSquare, Code2, Variable, Snowflake, @@ -469,3 +474,4 @@ export const nodeIconsLucide: iconsType = { Delete, Command, }; + From 57afaae666a96b1f6e3b8f6615cff160b1e5751d Mon Sep 17 00:00:00 2001 From: anovazzi1 Date: Tue, 5 Mar 2024 19:26:47 -0300 Subject: [PATCH 12/26] Add STARTER_FOLDER_NAME constant --- src/frontend/src/constants/constants.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/frontend/src/constants/constants.ts b/src/frontend/src/constants/constants.ts index 3f4584102..fb9d38451 100644 --- a/src/frontend/src/constants/constants.ts +++ b/src/frontend/src/constants/constants.ts @@ -727,3 +727,4 @@ export const STATUS_BUILD = "Build to validate status."; export const STATUS_BUILDING = "Building..."; export const SAVED_HOVER = "Last saved at "; export const RUN_TIMESTAMP_PREFIX = "Last Run: "; +export const STARTER_FOLDER_NAME = "Starter Projects"; From 8a1d48336a672c52eb19083644ff9bf884cc654a Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 19:33:25 -0300 Subject: [PATCH 13/26] Refactor starter project loading and database update --- src/backend/langflow/initial_setup/setup.py | 142 +++++++++++++------- src/backend/langflow/services/deps.py | 14 ++ 2 files changed, 106 insertions(+), 50 deletions(-) diff --git a/src/backend/langflow/initial_setup/setup.py b/src/backend/langflow/initial_setup/setup.py index 3b9ed8d5e..1f24ddab1 100644 --- a/src/backend/langflow/initial_setup/setup.py +++ b/src/backend/langflow/initial_setup/setup.py @@ -1,12 +1,12 @@ -import json from datetime import datetime from pathlib import Path +import orjson from loguru import logger from sqlmodel import select from langflow.services.database.models.flow.model import Flow -from langflow.services.deps import get_session +from langflow.services.deps import session_scope STARTER_FOLDER_NAME = "Starter Projects" @@ -14,60 +14,102 @@ STARTER_FOLDER_NAME = "Starter Projects" # In the folder ./starter_projects we have a few JSON files that represent # starter projects. We want to load these into the database so that users # can use them as a starting point for their own projects. + + def load_starter_projects(): - # Load the starter projects from the JSON files - # using Pathlib's glob method starter_projects = [] folder = Path(__file__).parent / "starter_projects" for file in folder.glob("*.json"): - with open(file, "r") as f: - starter_projects.append(json.load(f)) - logger.info(f"Loaded starter project {file}") + project = orjson.loads(file.read_text()) + starter_projects.append(project) + logger.info(f"Loaded starter project {file}") return starter_projects -# We want to load the starter projects into the database +def get_project_data(project): + project_name = project.get("name") + project_description = project.get("description") + project_is_component = project.get("is_component") + project_updated_at = project.get("updated_at") + updated_at_datetime = datetime.strptime(project_updated_at, "%Y-%m-%dT%H:%M:%S.%f") + project_data = project.get("data") + return ( + project_name, + project_description, + project_is_component, + updated_at_datetime, + project_data, + ) + + +def update_existing_project( + existing_project, + project_name, + project_description, + project_is_component, + updated_at_datetime, + project_data, +): + logger.info(f"Updating starter project {project_name}") + existing_project.data = project_data + existing_project.folder = STARTER_FOLDER_NAME + existing_project.description = project_description + existing_project.is_component = project_is_component + existing_project.updated_at = updated_at_datetime + + +def create_new_project( + session, + project_name, + project_description, + project_is_component, + updated_at_datetime, + project_data, +): + logger.info(f"Creating starter project {project_name}") + new_project = Flow( + name=project_name, + description=project_description, + is_component=project_is_component, + updated_at=updated_at_datetime, + folder=STARTER_FOLDER_NAME, + data=project_data, + ) + session.add(new_project) + + def create_or_update_starter_projects(): - session = next(get_session()) - starter_projects = load_starter_projects() - for project in starter_projects: - # Check if the project already exists in the database - project_name = project.get("name") - project_description = project.get("description") - project_is_component = project.get("is_component") - project_updated_at = project.get("updated_at") - # 2024-03-05T21:59:59.738081 - updated_at_datetime = datetime.strptime( - project_updated_at, "%Y-%m-%dT%H:%M:%S.%f" - ) - project_data = project.get("data") - if project_name and project_data: - existing_project = session.exec( - select(Flow).where( - Flow.name == project_name, Flow.folder == STARTER_FOLDER_NAME - ) - ).first() - if existing_project: - logger.info(f"Updating starter project {project_name}") - existing_project.data = project_data - existing_project.folder = STARTER_FOLDER_NAME - existing_project.description = project_description - existing_project.is_component = project_is_component - existing_project.updated_at = updated_at_datetime - # Now we need to update the project in the database - session.add(existing_project) - else: - logger.info(f"Creating starter project {project_name}") - session.add( - Flow( - name=project_name, - description=project_description, - is_component=project_is_component, - updated_at=updated_at_datetime, - folder=STARTER_FOLDER_NAME, - data=project_data, + with session_scope() as session: + starter_projects = load_starter_projects() + for project in starter_projects: + ( + project_name, + project_description, + project_is_component, + updated_at_datetime, + project_data, + ) = get_project_data(project) + if project_name and project_data: + existing_project = session.exec( + select(Flow).where( + Flow.name == project_name, Flow.folder == STARTER_FOLDER_NAME + ) + ).first() + if existing_project: + update_existing_project( + existing_project, + project_name, + project_description, + project_is_component, + updated_at_datetime, + project_data, + ) + else: + create_new_project( + session, + project_name, + project_description, + project_is_component, + updated_at_datetime, + project_data, ) - ) - session.commit() - session.close() - logger.info("Starter projects loaded into database") diff --git a/src/backend/langflow/services/deps.py b/src/backend/langflow/services/deps.py index 19f3dcbf0..7d2338b04 100644 --- a/src/backend/langflow/services/deps.py +++ b/src/backend/langflow/services/deps.py @@ -1,3 +1,4 @@ +from contextlib import contextmanager from typing import TYPE_CHECKING, Generator from langflow.services import ServiceType, service_manager @@ -54,6 +55,19 @@ def get_session() -> Generator["Session", None, None]: yield from db_service.get_session() +@contextmanager +def session_scope(): + session = next(get_session()) + try: + yield session + session.commit() + except: + session.rollback() + raise + finally: + session.close() + + def get_cache_service() -> "BaseCacheService": return service_manager.get(ServiceType.CACHE_SERVICE) # type: ignore From f2395aaee11748c571a03770c19829e917d6f9d1 Mon Sep 17 00:00:00 2001 From: anovazzi1 Date: Tue, 5 Mar 2024 20:44:11 -0300 Subject: [PATCH 14/26] fix(exampleComponent): add Link component to wrap Button for navigation feat(exampleComponent): add Link to navigate to specific flow based on data id feat(components): add support for setting examples based on STARTER_FOLDER_NAME constant feat(components): filter out examples from all flows and set them separately feat(components): log examples and set them using setExamples function feat(MainPage): add examples state to store and use it to render ExampleCardComponent feat(MainPage): map over examples to render ExampleCardComponent for each example feat(flowsManagerStore): add examples array to store state feat(flowsManagerStore): add setExamples function to set examples in store feat(flow): add folder and user_id fields to FlowType feat(zustand): add examples array and setExamples function to FlowsManagerStoreType --- src/frontend/src/components/exampleComponent/index.tsx | 3 +++ .../src/pages/MainPage/components/components/index.tsx | 8 +++++++- src/frontend/src/pages/MainPage/index.tsx | 9 +++++---- src/frontend/src/stores/flowsManagerStore.ts | 4 ++++ src/frontend/src/types/flow/index.ts | 2 ++ src/frontend/src/types/zustand/flowsManager/index.ts | 2 ++ 6 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/frontend/src/components/exampleComponent/index.tsx b/src/frontend/src/components/exampleComponent/index.tsx index af0cf56ea..7dff31254 100644 --- a/src/frontend/src/components/exampleComponent/index.tsx +++ b/src/frontend/src/components/exampleComponent/index.tsx @@ -19,6 +19,7 @@ import { CardTitle, } from "../ui/card"; import { FlowType } from "../../types/flow"; +import { Link } from "react-router-dom"; export default function CollectionCardComponent({ data, @@ -58,6 +59,7 @@ export default function CollectionCardComponent({
+ +
diff --git a/src/frontend/src/pages/MainPage/components/components/index.tsx b/src/frontend/src/pages/MainPage/components/components/index.tsx index 86ee65d0c..0073447c0 100644 --- a/src/frontend/src/pages/MainPage/components/components/index.tsx +++ b/src/frontend/src/pages/MainPage/components/components/index.tsx @@ -14,6 +14,7 @@ import { import useAlertStore from "../../../../stores/alertStore"; import useFlowsManagerStore from "../../../../stores/flowsManagerStore"; import { FlowType } from "../../../../types/flow"; +import { STARTER_FOLDER_NAME } from "../../../../constants/constants"; export default function ComponentsComponent({ is_component = true, @@ -24,6 +25,7 @@ export default function ComponentsComponent({ const uploadFlow = useFlowsManagerStore((state) => state.uploadFlow); const removeFlow = useFlowsManagerStore((state) => state.removeFlow); const isLoading = useFlowsManagerStore((state) => state.isLoading); + const setExamples = useFlowsManagerStore((state) => state.setExamples); const flows = useFlowsManagerStore((state) => state.flows); const setSuccessData = useAlertStore((state) => state.setSuccessData); const setErrorData = useAlertStore((state) => state.setErrorData); @@ -35,7 +37,7 @@ export default function ComponentsComponent({ useEffect(() => { if (isLoading) return; - const all = flows + let all = flows .filter((f) => (f.is_component ?? false) === is_component) .sort((a, b) => { if (a?.updated_at && b?.updated_at) { @@ -56,6 +58,10 @@ export default function ComponentsComponent({ }); const start = (pageIndex - 1) * pageSize; const end = start + pageSize; + const examples = all.filter(f=>(f.folder===STARTER_FOLDER_NAME && !f.user_id)); + console.log(examples); + setExamples(examples); + all = all.filter(f=>!(f.folder===STARTER_FOLDER_NAME && !f.user_id));; setData(all.slice(start, end)); }, [flows, isLoading, pageIndex, pageSize]); diff --git a/src/frontend/src/pages/MainPage/index.tsx b/src/frontend/src/pages/MainPage/index.tsx index fc8747094..88055ff79 100644 --- a/src/frontend/src/pages/MainPage/index.tsx +++ b/src/frontend/src/pages/MainPage/index.tsx @@ -29,6 +29,7 @@ export default function HomePage(): JSX.Element { const location = useLocation(); const pathname = location.pathname; const [openModal, setOpenModal] = useState(false); + const examples = useFlowsManagerStore((state) => state.examples); const is_component = pathname === "/components"; const dropdownOptions = [ { @@ -128,10 +129,10 @@ export default function HomePage(): JSX.Element {
- + {examples.map((example, idx) => { + return( + ) + })}
diff --git a/src/frontend/src/stores/flowsManagerStore.ts b/src/frontend/src/stores/flowsManagerStore.ts index fac806c5f..96b443152 100644 --- a/src/frontend/src/stores/flowsManagerStore.ts +++ b/src/frontend/src/stores/flowsManagerStore.ts @@ -37,6 +37,10 @@ const past = {}; const future = {}; const useFlowsManagerStore = create((set, get) => ({ + examples:[], + setExamples: (examples: FlowType[]) => { + set({ examples }); + }, currentFlowId: "", setCurrentFlowId: (currentFlowId: string) => { set((state) => ({ diff --git a/src/frontend/src/types/flow/index.ts b/src/frontend/src/types/flow/index.ts index 967d4e424..ffb364d61 100644 --- a/src/frontend/src/types/flow/index.ts +++ b/src/frontend/src/types/flow/index.ts @@ -13,6 +13,8 @@ export type FlowType = { updated_at?: string; date_created?: string; parent?: string; + folder?: string; + user_id?: string; }; export type NodeType = { diff --git a/src/frontend/src/types/zustand/flowsManager/index.ts b/src/frontend/src/types/zustand/flowsManager/index.ts index 471d4cf17..87fbf9a22 100644 --- a/src/frontend/src/types/zustand/flowsManager/index.ts +++ b/src/frontend/src/types/zustand/flowsManager/index.ts @@ -44,6 +44,8 @@ export type FlowsManagerStoreType = { undo: () => void; redo: () => void; takeSnapshot: () => void; + examples: Array; + setExamples: (examples: FlowType[]) => void; }; export type UseUndoRedoOptions = { From 781d2705aa56a60f88a14c2157b4024b3657f1ea Mon Sep 17 00:00:00 2001 From: anovazzi1 Date: Tue, 5 Mar 2024 20:54:07 -0300 Subject: [PATCH 15/26] Update exampleComponent and MainPage components --- .../src/components/exampleComponent/index.tsx | 47 +++++++++++-------- src/frontend/src/pages/MainPage/index.tsx | 2 +- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/src/frontend/src/components/exampleComponent/index.tsx b/src/frontend/src/components/exampleComponent/index.tsx index 7dff31254..fdaa36766 100644 --- a/src/frontend/src/components/exampleComponent/index.tsx +++ b/src/frontend/src/components/exampleComponent/index.tsx @@ -19,15 +19,17 @@ import { CardTitle, } from "../ui/card"; import { FlowType } from "../../types/flow"; -import { Link } from "react-router-dom"; +import { updateIds } from "../../utils/reactflowUtils"; +import { useNavigate } from "react-router-dom"; export default function CollectionCardComponent({ - data, + flow, }: { - data: FlowType; + flow: FlowType; authorized?: boolean; }) { const addFlow = useFlowsManagerStore((state) => state.addFlow); + const navigate = useNavigate(); return ( - -
{data.name}
+ +
{flow.name}
 -
{data.description}
+
{flow.description}
 @@ -59,20 +61,25 @@ export default function CollectionCardComponent({
- - - +
diff --git a/src/frontend/src/pages/MainPage/index.tsx b/src/frontend/src/pages/MainPage/index.tsx index 88055ff79..de546befa 100644 --- a/src/frontend/src/pages/MainPage/index.tsx +++ b/src/frontend/src/pages/MainPage/index.tsx @@ -131,7 +131,7 @@ export default function HomePage(): JSX.Element {
{examples.map((example, idx) => { return( - ) + ) })}
From 3b7f1928ff8d511cc28489d87e32f1cc3375453d Mon Sep 17 00:00:00 2001 From: anovazzi1 Date: Tue, 5 Mar 2024 20:59:15 -0300 Subject: [PATCH 16/26] update examples in MainPage using zustand --- .../src/pages/MainPage/components/components/index.tsx | 4 ---- src/frontend/src/pages/MainPage/index.tsx | 2 +- src/frontend/src/stores/flowsManagerStore.ts | 4 +++- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/frontend/src/pages/MainPage/components/components/index.tsx b/src/frontend/src/pages/MainPage/components/components/index.tsx index 0073447c0..234852388 100644 --- a/src/frontend/src/pages/MainPage/components/components/index.tsx +++ b/src/frontend/src/pages/MainPage/components/components/index.tsx @@ -58,10 +58,6 @@ export default function ComponentsComponent({ }); const start = (pageIndex - 1) * pageSize; const end = start + pageSize; - const examples = all.filter(f=>(f.folder===STARTER_FOLDER_NAME && !f.user_id)); - console.log(examples); - setExamples(examples); - all = all.filter(f=>!(f.folder===STARTER_FOLDER_NAME && !f.user_id));; setData(all.slice(start, end)); }, [flows, isLoading, pageIndex, pageSize]); diff --git a/src/frontend/src/pages/MainPage/index.tsx b/src/frontend/src/pages/MainPage/index.tsx index de546befa..9b35681eb 100644 --- a/src/frontend/src/pages/MainPage/index.tsx +++ b/src/frontend/src/pages/MainPage/index.tsx @@ -131,7 +131,7 @@ export default function HomePage(): JSX.Element {
{examples.map((example, idx) => { return( - ) + ) })}
diff --git a/src/frontend/src/stores/flowsManagerStore.ts b/src/frontend/src/stores/flowsManagerStore.ts index 96b443152..d993f40b0 100644 --- a/src/frontend/src/stores/flowsManagerStore.ts +++ b/src/frontend/src/stores/flowsManagerStore.ts @@ -25,6 +25,7 @@ import useAlertStore from "./alertStore"; import { useDarkStore } from "./darkStore"; import useFlowStore from "./flowStore"; import { useTypesStore } from "./typesStore"; +import { STARTER_FOLDER_NAME } from "../constants/constants"; let saveTimeoutId: NodeJS.Timeout | null = null; @@ -66,7 +67,8 @@ const useFlowsManagerStore = create((set, get) => ({ .then((dbData) => { if (dbData) { const { data, flows } = processFlows(dbData, false); - get().setFlows(flows); + get().setExamples(flows.filter(f=>(f.folder===STARTER_FOLDER_NAME && !f.user_id))); + get().setFlows(flows.filter(f=>!(f.folder===STARTER_FOLDER_NAME && !f.user_id))); useTypesStore.setState((state) => ({ data: { ...state.data, ["saved_components"]: data }, })); From e6672189538cdee2c3b049e71929bfd17ae5c8ae Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 17:36:09 -0300 Subject: [PATCH 17/26] Update Langflow documentation with new component descriptions --- docs/docs/getting-started/creating-flows.mdx | 23 +++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/docs/getting-started/creating-flows.mdx b/docs/docs/getting-started/creating-flows.mdx index aecc3ea16..9c16d225f 100644 --- a/docs/docs/getting-started/creating-flows.mdx +++ b/docs/docs/getting-started/creating-flows.mdx @@ -7,7 +7,8 @@ import ReactPlayer from "react-player"; ## Compose -Creating flows with Langflow is easy. Drag sidebar components onto the canvas and connect them together to create your pipeline. Langflow provides a range of [LangChain components](https://python.langchain.com/docs/modules/) to choose from, including LLMs, prompt serializers, agents, and chains. +Creating flows with Langflow is easy. Drag sidebar components onto the canvas and connect them together to create your pipeline. +Langflow provides a range of Components to choose from, including **Chat Input**, **Chat Output**, **API Request** and **Prompt**. -## Fork +## Starter Flows -The easiest way to start with Langflow is by forking a **community example**. Forking an example stores a copy in your project collection, allowing you to edit and save the modified version as a new flow. +Langflow provides a range of starter flows to help you get started. These flows are pre-built and can be used as a starting point for your own flows.
-## Build +## Defining Inputs and Outputs + +Each flow can have multiple inputs and outputs. These can be defined by placing **Inputs** and **Outputs** components on the canvas. + +The **Inputs** components define the inputs to the flow. +Whenever you place an Input component on the canvas, it will allow you to interactively define change its value +from the Interactive Panel. + +The **Text Input** component allows you to define a text input, and the **Chat Input** component allows you to use the chat input from the Interactive Panel. + +The **Outputs** components define the outputs of the flow and work similarly to the Inputs components. + +Both Inputs and Outputs components can be connected to other components on the canvas and are used to define how the API works too. + -Building a flow means validating if the components have prerequisites fulfilled and are properly instantiated. When a chat message is sent, the flow will run for the first time, executing the pipeline.
Date: Tue, 5 Mar 2024 21:02:15 -0300 Subject: [PATCH 18/26] Update description and info in RecordsAsTextComponent --- src/backend/langflow/components/utilities/RecordsAsText.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/langflow/components/utilities/RecordsAsText.py b/src/backend/langflow/components/utilities/RecordsAsText.py index 18bf8be8c..f7750bdba 100644 --- a/src/backend/langflow/components/utilities/RecordsAsText.py +++ b/src/backend/langflow/components/utilities/RecordsAsText.py @@ -6,7 +6,7 @@ from langflow.schema import Record class RecordsAsTextComponent(CustomComponent): display_name = "Records to Text" - description = "Converts Records a list of Records to text using a template." + description = "Converts Records into single piece of text using a template." def build_config(self): return { @@ -16,7 +16,7 @@ class RecordsAsTextComponent(CustomComponent): }, "template": { "display_name": "Template", - "info": "The template to use for formatting the records. It must contain the keys {text} and {data}.", + "info": "The template to use for formatting the records. It can contain the keys {text}, {data} or any other key in the Record.", }, } From f304f7ef38e7d66fdead09dfe679dfc17420c4d4 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 21:16:34 -0300 Subject: [PATCH 19/26] Delete io/__init__.py file --- src/backend/langflow/components/io/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/backend/langflow/components/io/__init__.py diff --git a/src/backend/langflow/components/io/__init__.py b/src/backend/langflow/components/io/__init__.py deleted file mode 100644 index e69de29bb..000000000 From a2dfadcf4aaa23e3df2b994432c65e3f4f20eb02 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 21:24:53 -0300 Subject: [PATCH 20/26] Update sidebar component and styleUtils --- .../components/extraSidebarComponent/utils.tsx | 3 ++- src/frontend/src/utils/styleUtils.ts | 10 ++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/frontend/src/pages/FlowPage/components/extraSidebarComponent/utils.tsx b/src/frontend/src/pages/FlowPage/components/extraSidebarComponent/utils.tsx index f8b5a0af9..998ed8cd7 100644 --- a/src/frontend/src/pages/FlowPage/components/extraSidebarComponent/utils.tsx +++ b/src/frontend/src/pages/FlowPage/components/extraSidebarComponent/utils.tsx @@ -6,8 +6,9 @@ export function sortKeys(a: string, b: string) { "outputs", "prompts", "data", + "prompts", "models", - "utilities", + "helpers", ]; const indexA = order.indexOf(a.toLowerCase()); const indexB = order.indexOf(b.toLowerCase()); diff --git a/src/frontend/src/utils/styleUtils.ts b/src/frontend/src/utils/styleUtils.ts index c9425dbe4..ac88826ca 100644 --- a/src/frontend/src/utils/styleUtils.ts +++ b/src/frontend/src/utils/styleUtils.ts @@ -6,11 +6,8 @@ import { Binary, BookMarked, BookmarkPlus, - PlusCircle, - PlusSquare, Bot, Boxes, - GitCompare, Braces, BrainCircuit, Check, @@ -90,6 +87,8 @@ import { Pin, Play, Plus, + PlusCircle, + PlusSquare, PocketKnife, Redo, RefreshCcw, @@ -231,7 +230,7 @@ export const nodeColors: { [char: string]: string } = { textsplitters: "#B47CB5", toolkits: "#DB2C2C", wrappers: "#E6277A", - utilities: "#31A3CC", + helpers: "#31A3CC", langchain_utilities: "#31A3CC", output_parsers: "#E6A627", str: "#31a3cc", @@ -354,7 +353,7 @@ export const nodeIconsLucide: iconsType = { toolkits: Package2, textsplitters: Scissors, wrappers: Gift, - utilities: Wand2, + helpers: Wand2, langchain_utilities: PocketKnife, WolframAlphaAPIWrapper: SvgWolfram, output_parsers: Compass, @@ -475,4 +474,3 @@ export const nodeIconsLucide: iconsType = { Delete, Command, }; - From 6af16272fd407ce399a92d75fb569f0426e2d495 Mon Sep 17 00:00:00 2001 From: anovazzi1 Date: Tue, 5 Mar 2024 21:34:36 -0300 Subject: [PATCH 21/26] Add ShadTooltip component and update styling in exampleComponent and MainPage --- .../src/components/exampleComponent/index.tsx | 40 ++++++++++--------- src/frontend/src/pages/MainPage/index.tsx | 2 +- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/frontend/src/components/exampleComponent/index.tsx b/src/frontend/src/components/exampleComponent/index.tsx index fdaa36766..fb6d570aa 100644 --- a/src/frontend/src/components/exampleComponent/index.tsx +++ b/src/frontend/src/components/exampleComponent/index.tsx @@ -53,7 +53,9 @@ export default function CollectionCardComponent({
-
{flow.description}
+ +
{flow.description}
+ @@ -61,25 +63,25 @@ export default function CollectionCardComponent({
- + navigate("/flow/" + id); + }); + }} + tabIndex={-1} + variant="outline" + size="sm" + className="whitespace-nowrap " + > + + Select Flow +
diff --git a/src/frontend/src/pages/MainPage/index.tsx b/src/frontend/src/pages/MainPage/index.tsx index 9b35681eb..2574f03af 100644 --- a/src/frontend/src/pages/MainPage/index.tsx +++ b/src/frontend/src/pages/MainPage/index.tsx @@ -128,7 +128,7 @@ export default function HomePage(): JSX.Element { /> -
+
{examples.map((example, idx) => { return( ) From f1eb6ee6730e6fb7ab03e7a2a3ec95f0f5f6010f Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 21:54:48 -0300 Subject: [PATCH 22/26] Add STARTER_FOLDER_NAME constant --- src/frontend/src/constants/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/src/constants/constants.ts b/src/frontend/src/constants/constants.ts index 10b6e41df..897a1b4b3 100644 --- a/src/frontend/src/constants/constants.ts +++ b/src/frontend/src/constants/constants.ts @@ -727,7 +727,7 @@ export const STATUS_BUILD = "Build to validate status."; export const STATUS_BUILDING = "Building..."; export const SAVED_HOVER = "Last saved at "; export const RUN_TIMESTAMP_PREFIX = "Last Run: "; - +export const STARTER_FOLDER_NAME = "Starter Projects"; export const PRIORITY_SIDEBAR_ORDER = [ "saved_components", "inputs", From abf9bbd4a2c00b600dacd98b9177c23428100b18 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 22:05:57 -0300 Subject: [PATCH 23/26] Update IDGenerator.py to generate a unique ID in the build_config --- .../langflow/components/helpers/IDGenerator.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/backend/langflow/components/helpers/IDGenerator.py b/src/backend/langflow/components/helpers/IDGenerator.py index ceb937a6c..1a8cecbfd 100644 --- a/src/backend/langflow/components/helpers/IDGenerator.py +++ b/src/backend/langflow/components/helpers/IDGenerator.py @@ -1,5 +1,5 @@ import uuid -from typing import Text +from typing import Any, Text from langflow import CustomComponent @@ -9,11 +9,20 @@ class UUIDGeneratorComponent(CustomComponent): display_name = "Unique ID Generator" description = "Generates a unique ID." - def generate(self, *args, **kwargs): - return Text(uuid.uuid4().hex) + def update_build_config( + self, build_config: dict, field_name: Text, field_value: Any + ): + if field_name == "unique_id": + build_config[field_name]["value"] = str(uuid.uuid4()) + return build_config def build_config(self): - return {"unique_id": {"display_name": "Value", "value": self.generate}} + return { + "unique_id": { + "display_name": "Value", + "refresh": True, + } + } def build(self, unique_id: str) -> str: return unique_id From 930a2c01ecd1930547561bb367ebe0ee600c7e95 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 22:06:06 -0300 Subject: [PATCH 24/26] Remove console.log statement and format code --- src/frontend/src/CustomNodes/GenericNode/index.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/frontend/src/CustomNodes/GenericNode/index.tsx b/src/frontend/src/CustomNodes/GenericNode/index.tsx index ddfc04aef..80e5ad0f7 100644 --- a/src/frontend/src/CustomNodes/GenericNode/index.tsx +++ b/src/frontend/src/CustomNodes/GenericNode/index.tsx @@ -44,7 +44,6 @@ export default function GenericNode({ const buildFlow = useFlowStore((state) => state.buildFlow); const setNode = useFlowStore((state) => state.setNode); const name = nodeIconsLucide[data.type] ? data.type : types[data.type]; - console.log(types[data.type]) const [inputName, setInputName] = useState(false); const [nodeName, setNodeName] = useState(data.node!.display_name); const [inputDescription, setInputDescription] = useState(false); @@ -158,7 +157,7 @@ export default function GenericNode({ const iconElement = data?.node?.icon; const iconColor = nodeColors[types[data.type]]; const iconName = - iconElement || (data.node?.flow ? "group_components" : name); + iconElement || (data.node?.flow ? "group_components" : name); const iconClassName = `generic-node-icon ${ !showNode ? " absolute inset-x-6 h-12 w-12 " : "" }`; From 1f77b49191ffb6e600d09fbbeea0707f47225128 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 22:24:08 -0300 Subject: [PATCH 25/26] Add MessageHistoryComponent to retrieve stored messages --- .../langflow/components/{memories => helpers}/MessageHistory.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/backend/langflow/components/{memories => helpers}/MessageHistory.py (100%) diff --git a/src/backend/langflow/components/memories/MessageHistory.py b/src/backend/langflow/components/helpers/MessageHistory.py similarity index 100% rename from src/backend/langflow/components/memories/MessageHistory.py rename to src/backend/langflow/components/helpers/MessageHistory.py From bc37bc9982c4ac2e3d105486a2a9fe66bacc9dd3 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 5 Mar 2024 22:24:18 -0300 Subject: [PATCH 26/26] Update APIRequest.py to use record.data instead of record.text --- src/backend/langflow/components/helpers/APIRequest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/langflow/components/helpers/APIRequest.py b/src/backend/langflow/components/helpers/APIRequest.py index dfaf2ed07..4d9f3ce9c 100644 --- a/src/backend/langflow/components/helpers/APIRequest.py +++ b/src/backend/langflow/components/helpers/APIRequest.py @@ -52,7 +52,7 @@ class APIRequest(CustomComponent): if method not in ["GET", "POST", "PATCH", "PUT"]: raise ValueError(f"Unsupported method: {method}") - data = record.text if record else None + data = record.data if record else None try: response = await client.request( method, url, headers=headers, content=data, timeout=timeout