Merge branch 'release' into add_params_to_llms

src/backend/langflow/__main__.py

@@ -1,6 +1,5 @@
 import sys
 import time
-from fastapi import FastAPI
 import httpx
 from multiprocess import Process, cpu_count  # type: ignore
 import platform
@@ -11,9 +10,7 @@ from rich.panel import Panel
 from rich import box
 from rich import print as rprint
 import typer
-from fastapi.staticfiles import StaticFiles
-from fastapi.responses import FileResponse
-from langflow.main import create_app
+from langflow.main import setup_app
 from langflow.settings import settings
 from langflow.utils.logger import configure, logger
 import webbrowser
@@ -30,6 +27,7 @@ def get_number_of_workers(workers=None):
 
 def update_settings(
     config: str,
+    cache: str,
     dev: bool = False,
     database_url: Optional[str] = None,
     remove_api_keys: bool = False,
@@ -41,6 +39,8 @@ def update_settings(
         settings.update_settings(database_url=database_url)
     if remove_api_keys:
         settings.update_settings(remove_api_keys=remove_api_keys)
+    if cache:
+        settings.update_settings(cache=cache)
 
 
 def serve_on_jcloud():
@@ -102,6 +102,11 @@ def serve(
     ),
     log_level: str = typer.Option("critical", help="Logging level."),
     log_file: Path = typer.Option("logs/langflow.log", help="Path to the log file."),
+    cache: str = typer.Argument(
+        envvar="LANGCHAIN_CACHE",
+        help="Type of cache to use. (InMemoryCache, SQLiteCache)",
+        default="SQLiteCache",
+    ),
     jcloud: bool = typer.Option(False, help="Deploy on Jina AI Cloud"),
     dev: bool = typer.Option(False, help="Run in development mode (may contain bugs)"),
     database_url: str = typer.Option(
@@ -130,17 +135,15 @@ def serve(
 
     configure(log_level=log_level, log_file=log_file)
     update_settings(
-        config, dev=dev, database_url=database_url, remove_api_keys=remove_api_keys
+        config,
+        dev=dev,
+        database_url=database_url,
+        remove_api_keys=remove_api_keys,
+        cache=cache,
     )
-    # get the directory of the current file
-    if not path:
-        frontend_path = Path(__file__).parent
-        static_files_dir = frontend_path / "frontend"
-    else:
-        static_files_dir = Path(path)
-
-    app = create_app()
-    setup_static_files(app, static_files_dir)
+    # create path object if path is provided
+    static_files_dir: Optional[Path] = Path(path) if path else None
+    app = setup_app(static_files_dir=static_files_dir)
     # check if port is being used
     if is_port_in_use(port, host):
         port = get_free_port(port)
@@ -188,29 +191,6 @@ def run_on_windows(host, port, log_level, options, app):
     run_langflow(host, port, log_level, options, app)
 
 
-def setup_static_files(app: FastAPI, static_files_dir: Path):
-    """
-    Setup the static files directory.
-
-    Args:
-        app (FastAPI): FastAPI app.
-        path (str): Path to the static files directory.
-    """
-    app.mount(
-        "/",
-        StaticFiles(directory=static_files_dir, html=True),
-        name="static",
-    )
-
-    @app.exception_handler(404)
-    async def custom_404_handler(request, __):
-        path = static_files_dir / "index.html"
-
-        if not path.exists():
-            raise RuntimeError(f"File at path {path} does not exist.")
-        return FileResponse(path)
-
-
 def is_port_in_use(port, host="localhost"):
     """
     Check if a port is in use.

src/backend/langflow/api/v1/endpoints.py

@@ -1,12 +1,14 @@
+from typing import Optional
+from langflow.cache.utils import save_uploaded_file
 from langflow.database.models.flow import Flow
 from langflow.processing.process import process_graph_cached, process_tweaks
 from langflow.utils.logger import logger
 
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, UploadFile
 
 from langflow.api.v1.schemas import (
-    PredictRequest,
-    PredictResponse,
+    ProcessResponse,
+    UploadFileResponse,
 )
 
 from langflow.interface.types import build_langchain_types_dict
@@ -22,14 +24,17 @@ def get_all():
     return build_langchain_types_dict()
 
 
-@router.post("/predict/{flow_id}", response_model=PredictResponse)
-async def predict_flow(
-    predict_request: PredictRequest,
+# For backwards compatibility we will keep the old endpoint
+@router.post("/predict/{flow_id}", response_model=ProcessResponse)
+@router.post("/process/{flow_id}", response_model=ProcessResponse)
+async def process_flow(
     flow_id: str,
+    inputs: Optional[dict] = None,
+    tweaks: Optional[dict] = None,
     session: Session = Depends(get_session),
 ):
     """
-    Endpoint to process a message using the flow passed in the bearer token.
+    Endpoint to process an input with a given flow_id.
     """
 
     try:
@@ -40,15 +45,14 @@ async def predict_flow(
         if flow.data is None:
             raise ValueError(f"Flow {flow_id} has no data")
         graph_data = flow.data
-        if predict_request.tweaks:
+        if tweaks:
             try:
-                graph_data = process_tweaks(graph_data, predict_request.tweaks)
+                graph_data = process_tweaks(graph_data, tweaks)
             except Exception as exc:
                 logger.error(f"Error processing tweaks: {exc}")
-        response = process_graph_cached(graph_data, predict_request.message)
-        return PredictResponse(
-            result=response.get("result", ""),
-            intermediate_steps=response.get("thought", ""),
+        response = process_graph_cached(graph_data, inputs)
+        return ProcessResponse(
+            result=response,
         )
     except Exception as e:
         # Log stack trace
@@ -56,6 +60,21 @@ async def predict_flow(
         raise HTTPException(status_code=500, detail=str(e)) from e
 
 
+@router.post("/upload/{flow_id}", response_model=UploadFileResponse, status_code=201)
+async def create_upload_file(file: UploadFile, flow_id: str):
+    # Cache file
+    try:
+        file_path = save_uploaded_file(file.file, folder_name=flow_id)
+
+        return UploadFileResponse(
+            flowId=flow_id,
+            file_path=file_path,
+        )
+    except Exception as exc:
+        logger.error(f"Error saving file: {exc}")
+        raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+
 # get endpoint to return version of langflow
 @router.get("/version")
 def get_version():

src/backend/langflow/api/v1/schemas.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 from langflow.database.models.flow import FlowCreate, FlowRead
 from pydantic import BaseModel, Field, validator
@@ -20,34 +21,22 @@ class ExportedFlow(BaseModel):
     data: GraphData
 
 
-class PredictRequest(BaseModel):
-    """Predict request schema."""
+class InputRequest(BaseModel):
+    input: dict
 
-    message: str
+
+class TweaksRequest(BaseModel):
     tweaks: Optional[Dict[str, Dict[str, str]]] = Field(default_factory=dict)
 
-    class Config:
-        schema_extra = {
-            "example": {
-                "message": "Hello, how are you?",
-                "tweaks": {
-                    "dndnode_986363f0-4677-4035-9f38-74b94af5dd78": {
-                        "name": "A tool name",
-                        "description": "A tool description",
-                    },
-                    "dndnode_986363f0-4677-4035-9f38-74b94af57378": {
-                        "template": "A {template}",
-                    },
-                },
-            }
-        }
+
+class UpdateTemplateRequest(BaseModel):
+    template: dict
 
 
-class PredictResponse(BaseModel):
-    """Predict response schema."""
+class ProcessResponse(BaseModel):
+    """Process response schema."""
 
-    result: str
-    intermediate_steps: str = ""
+    result: dict
 
 
 class ChatMessage(BaseModel):
@@ -104,6 +93,13 @@ class BuiltResponse(BaseModel):
     built: bool
 
 
+class UploadFileResponse(BaseModel):
+    """Upload file response schema."""
+
+    flowId: str
+    file_path: Path
+
+
 class StreamData(BaseModel):
     event: str
     data: dict

src/backend/langflow/api/v1/validate.py

@@ -1,5 +1,3 @@
-import json
-
 from fastapi import APIRouter, HTTPException
 
 from langflow.api.v1.base import (
@@ -9,8 +7,6 @@ from langflow.api.v1.base import (
     PromptValidationResponse,
     validate_prompt,
 )
-from langflow.graph.vertex.types import VectorStoreVertex
-from langflow.graph import Graph
 from langflow.utils.logger import logger
 from langflow.utils.validate import validate_code
 
@@ -37,21 +33,3 @@ def post_validate_prompt(prompt: Prompt):
     except Exception as e:
         logger.exception(e)
         raise HTTPException(status_code=500, detail=str(e)) from e
-
-
-# validate node
-@router.post("/node/{node_id}", status_code=200)
-def post_validate_node(node_id: str, data: dict):
-    try:
-        # build graph
-        graph = Graph.from_payload(data)
-        # validate node
-        node = graph.get_node(node_id)
-        if node is None:
-            raise ValueError(f"Node {node_id} not found")
-        if not isinstance(node, VectorStoreVertex):
-            node.build()
-        return json.dumps({"valid": True, "params": str(node._built_object_repr())})
-    except Exception as e:
-        logger.exception(e)
-        return json.dumps({"valid": False, "params": str(e)})

src/backend/langflow/cache/base.py

@@ -62,9 +62,6 @@ class BaseCache(abc.ABC):
 
         Args:
             key: The key of the item to retrieve.
-
-        Returns:
-            The value associated with the key, or None if the key is not found.
         """
 
     @abc.abstractmethod

src/backend/langflow/cache/utils.py

@@ -8,15 +8,17 @@ import tempfile
 from collections import OrderedDict
 from pathlib import Path
 from typing import Any, Dict
-
+from appdirs import user_cache_dir
 
 CACHE: Dict[str, Any] = {}
 
+CACHE_DIR = user_cache_dir("langflow", "langflow")
+
 
 def create_cache_folder(func):
     def wrapper(*args, **kwargs):
         # Get the destination folder
-        cache_path = Path(tempfile.gettempdir()) / PREFIX
+        cache_path = Path(CACHE_DIR) / PREFIX
 
         # Create the destination folder if it doesn't exist
         os.makedirs(cache_path, exist_ok=True)
@@ -118,7 +120,7 @@ def save_binary_file(content: str, file_name: str, accepted_types: list[str]) ->
         raise ValueError(f"File {file_name} is not accepted")
 
     # Get the destination folder
-    cache_path = Path(tempfile.gettempdir()) / PREFIX
+    cache_path = Path(CACHE_DIR) / PREFIX
     if not content:
         raise ValueError("Please, reload the file in the loader.")
     data = content.split(",")[1]
@@ -132,3 +134,46 @@ def save_binary_file(content: str, file_name: str, accepted_types: list[str]) ->
         file.write(decoded_bytes)
 
     return file_path
+
+
+@create_cache_folder
+def save_uploaded_file(file, folder_name):
+    """
+    Save an uploaded file to the specified folder with a hash of its content as the file name.
+
+    Args:
+        file: The uploaded file object.
+        folder_name: The name of the folder to save the file in.
+
+    Returns:
+        The path to the saved file.
+    """
+    cache_path = Path(CACHE_DIR)
+    folder_path = cache_path / folder_name
+
+    # Create the folder if it doesn't exist
+    if not folder_path.exists():
+        folder_path.mkdir()
+
+    # Create a hash of the file content
+    sha256_hash = hashlib.sha256()
+    # Reset the file cursor to the beginning of the file
+    file.seek(0)
+    # Iterate over the uploaded file in small chunks to conserve memory
+    while chunk := file.read(8192):  # Read 8KB at a time (adjust as needed)
+        sha256_hash.update(chunk)
+
+    # Use the hex digest of the hash as the file name
+    hex_dig = sha256_hash.hexdigest()
+    file_name = hex_dig
+
+    # Reset the file cursor to the beginning of the file
+    file.seek(0)
+
+    # Save the file with the hash as its name
+    file_path = folder_path / file_name
+    with open(file_path, "wb") as new_file:
+        while chunk := file.read(8192):
+            new_file.write(chunk)
+
+    return file_path

src/backend/langflow/chat/config.py

@@ -0,0 +1,2 @@
+class ChatConfig:
+    streaming: bool = True

src/backend/langflow/chat/manager.py

@@ -144,6 +144,8 @@ class ChatManager:
                 if isinstance(msg, FileResponse):
                     if msg.data_type == "image":
                         # Base64 encode the image
+                        if isinstance(msg.data, str):
+                            continue
                         msg.data = pil_to_base64(msg.data)
                     file_responses.append(msg)
                 if msg.type == "start":

src/backend/langflow/config.yaml

@@ -1,136 +1,247 @@
----
 agents:
-  - ZeroShotAgent
-  - JsonAgent
-  - CSVAgent
-  - AgentInitializer
-  - VectorStoreAgent
-  - VectorStoreRouterAgent
-  - SQLAgent
+  ZeroShotAgent:
+    documentation: "https://python.langchain.com/docs/modules/agents/how_to/custom_mrkl_agent"
+  JsonAgent:
+    documentation: "https://python.langchain.com/docs/modules/agents/toolkits/openapi"
+  CSVAgent:
+    documentation: "https://python.langchain.com/docs/modules/agents/toolkits/csv"
+  AgentInitializer:
+    documentation: "https://python.langchain.com/docs/modules/agents/agent_types/"
+  VectorStoreAgent:
+    documentation: ""
+  VectorStoreRouterAgent:
+    documentation: ""
+  SQLAgent:
+    documentation: ""
 chains:
-  - LLMChain
-  - LLMMathChain
-  - LLMCheckerChain
-  - ConversationChain
-  - SeriesCharacterChain
-  - MidJourneyPromptChain
-  - TimeTravelGuideChain
-  - SQLDatabaseChain
-  - RetrievalQA
-  - RetrievalQAWithSourcesChain
-  - ConversationalRetrievalChain
-  - CombineDocsChain
+  LLMChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/foundational/llm_chain"
+  LLMMathChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/additional/llm_math"
+  LLMCheckerChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/additional/llm_checker"
+  ConversationChain:
+    documentation: ""
+  SeriesCharacterChain:
+    documentation: ""
+  MidJourneyPromptChain:
+    documentation: ""
+  TimeTravelGuideChain:
+    documentation: ""
+  SQLDatabaseChain:
+    documentation: ""
+  RetrievalQA:
+    documentation: "https://python.langchain.com/docs/modules/chains/popular/vector_db_qa"
+  RetrievalQAWithSourcesChain:
+    documentation: ""
+  ConversationalRetrievalChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/popular/chat_vector_db"
+  CombineDocsChain:
+    documentation: ""
 documentloaders:
-  - AirbyteJSONLoader
-  - CoNLLULoader
-  - CSVLoader
-  - UnstructuredEmailLoader
-  - EverNoteLoader
-  - FacebookChatLoader
-  - GutenbergLoader
-  - BSHTMLLoader
-  - UnstructuredHTMLLoader
-  # - UnstructuredImageLoader # Issue with Python 3.11 (https://github.com/Unstructured-IO/unstructured-inference/issues/83)
-  - UnstructuredMarkdownLoader
-  - PyPDFLoader
-  - UnstructuredPowerPointLoader
-  - SRTLoader
-  - TelegramChatLoader
-  - TextLoader
-  - UnstructuredWordDocumentLoader
-  - WebBaseLoader
-  - AZLyricsLoader
-  - CollegeConfidentialLoader
-  - HNLoader
-  - IFixitLoader
-  - IMSDbLoader
-  - GitbookLoader
-  - ReadTheDocsLoader
-  - SlackDirectoryLoader
-  - NotionDirectoryLoader
+  AirbyteJSONLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/airbyte_json"
+  CoNLLULoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/conll-u"
+  CSVLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/csv"
+  UnstructuredEmailLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/email"
+  EverNoteLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/evernote"
+  FacebookChatLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/facebook_chat"
+  GutenbergLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/gutenberg"
+  BSHTMLLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/html"
+  UnstructuredHTMLLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/html"
+  UnstructuredMarkdownLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/markdown"
+  PyPDFLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/pdf"
+  UnstructuredPowerPointLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/microsoft_powerpoint"
+  SRTLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/subtitle"
+  TelegramChatLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/telegram"
+  TextLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/"
+  UnstructuredWordDocumentLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/microsoft_word"
+  WebBaseLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/web_base"
+  AZLyricsLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/azlyrics"
+  CollegeConfidentialLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/college_confidential"
+  HNLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/hacker_news"
+  IFixitLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/ifixit"
+  IMSDbLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/imsdb"
+  GitbookLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/gitbook"
+  ReadTheDocsLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/readthedocs_documentation"
+  SlackDirectoryLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/slack"
+  NotionDirectoryLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/notion"
+  DirectoryLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/file_directory"
+  GitLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/git"
 embeddings:
-  - OpenAIEmbeddings
-  - HuggingFaceEmbeddings
-  - CohereEmbeddings
+  OpenAIEmbeddings:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/openai"
+  HuggingFaceEmbeddings:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/sentence_transformers"
+  CohereEmbeddings:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/cohere"
 llms:
-  - OpenAI
-  # - AzureOpenAI
-  # - AzureChatOpenAI
-  - ChatOpenAI
-  - LlamaCpp 
-  - CTransformers
-  - Cohere
-  - Anthropic
-  - ChatAnthropic
-  - HuggingFaceHub
+  OpenAI:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/openai"
+  ChatOpenAI:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/integrations/openai"
+  LlamaCpp:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/llamacpp"
+  CTransformers:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/ctransformers"
+  Cohere:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/cohere"
+  Anthropic:
+    documentation: ""
+  ChatAnthropic:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/integrations/anthropic"
+  HuggingFaceHub:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/huggingface_hub"
 memories:
-  - ConversationBufferMemory
-  - ConversationSummaryMemory
-  - ConversationKGMemory
+  ConversationBufferMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/summary"
+  ConversationSummaryMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/summary"
+  ConversationKGMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/kg"
+  ConversationBufferWindowMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/buffer_window"
+  VectorStoreRetrieverMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/vectorstore_retriever_memory"
 prompts:
-  - PromptTemplate
-  - FewShotPromptTemplate
-  - ZeroShotPrompt
+  PromptTemplate:
+    documentation: "https://python.langchain.com/docs/modules/model_io/prompts/prompt_templates/"
+  ZeroShotPrompt:
+    documentation: "https://python.langchain.com/docs/modules/agents/how_to/custom_mrkl_agent"
 textsplitters:
-  - CharacterTextSplitter
-  - RecursiveCharacterTextSplitter
-  # - LatexTextSplitter
-  # - PythonCodeTextSplitter
+  CharacterTextSplitter:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_transformers/text_splitters/character_text_splitter"
+  RecursiveCharacterTextSplitter:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_transformers/text_splitters/recursive_text_splitter"
 toolkits:
-  - OpenAPIToolkit
-  - JsonToolkit
-  - VectorStoreInfo
-  - VectorStoreRouterToolkit
-  - VectorStoreToolkit
+  OpenAPIToolkit:
+    documentation: ""
+  JsonToolkit:
+    documentation: ""
+  VectorStoreInfo:
+    documentation: ""
+  VectorStoreRouterToolkit:
+    documentation: ""
+  VectorStoreToolkit:
+    documentation: ""
 tools:
-  - Search
-  - PAL-MATH
-  - Calculator
-  - Serper Search
-  - Tool
-  - PythonFunctionTool
-  - PythonFunction
-  - JsonSpec
-  - News API
-  - TMDB API
-  - Podcast API
-  - QuerySQLDataBaseTool
-  - InfoSQLDatabaseTool
-  - ListSQLDatabaseTool
-  # - QueryCheckerTool
-  - BingSearchRun
-  - GoogleSearchRun
-  - GoogleSearchResults
-  - GoogleSerperRun
-  - JsonListKeysTool
-  - JsonGetValueTool
-  - PythonREPLTool
-  - PythonAstREPLTool
-  - RequestsGetTool
-  - RequestsPostTool
-  - RequestsPatchTool
-  - RequestsPutTool
-  - RequestsDeleteTool
-  - WikipediaQueryRun
-  - WolframAlphaQueryRun
+  Search:
+    documentation: ""
+  PAL-MATH:
+    documentation: ""
+  Calculator:
+    documentation: ""
+  Serper Search:
+    documentation: ""
+  Tool:
+    documentation: ""
+  PythonFunctionTool:
+    documentation: ""
+  PythonFunction:
+    documentation: ""
+  JsonSpec:
+    documentation: ""
+  News API:
+    documentation: ""
+  TMDB API:
+    documentation: ""
+  Podcast API:
+    documentation: ""
+  QuerySQLDataBaseTool:
+    documentation: ""
+  InfoSQLDatabaseTool:
+    documentation: ""
+  ListSQLDatabaseTool:
+    documentation: ""
+  BingSearchRun:
+    documentation: ""
+  GoogleSearchRun:
+    documentation: ""
+  GoogleSearchResults:
+    documentation: ""
+  GoogleSerperRun:
+    documentation: ""
+  JsonListKeysTool:
+    documentation: ""
+  JsonGetValueTool:
+    documentation: ""
+  PythonREPLTool:
+    documentation: ""
+  PythonAstREPLTool:
+    documentation: ""
+  RequestsGetTool:
+    documentation: ""
+  RequestsPostTool:
+    documentation: ""
+  RequestsPatchTool:
+    documentation: ""
+  RequestsPutTool:
+    documentation: ""
+  RequestsDeleteTool:
+    documentation: ""
+  WikipediaQueryRun:
+    documentation: ""
+  WolframAlphaQueryRun:
+    documentation: ""
 utilities:
-  - BingSearchAPIWrapper
-  - GoogleSearchAPIWrapper
-  - GoogleSerperAPIWrapper
-  - SearxResults
-  - SearxSearchWrapper
-  - SerpAPIWrapper
-  - WikipediaAPIWrapper
-  - WolframAlphaAPIWrapper
-  # - ZapierNLAWrapper
-  - SQLDatabase
+  BingSearchAPIWrapper:
+    documentation: ""
+  GoogleSearchAPIWrapper:
+    documentation: ""
+  GoogleSerperAPIWrapper:
+    documentation: ""
+  SearxResults:
+    documentation: ""
+  SearxSearchWrapper:
+    documentation: ""
+  SerpAPIWrapper:
+    documentation: ""
+  WikipediaAPIWrapper:
+    documentation: ""
+  WolframAlphaAPIWrapper:
+    documentation: ""
 vectorstores:
-  - Chroma
-  - Qdrant
-  - Weaviate
-  - FAISS
+  Chroma:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/chroma"
+  Qdrant:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/qdrant"
+  Weaviate:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/weaviate"
+  FAISS:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/faiss"
+  Pinecone:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/pinecone"
+  SupabaseVectorStore:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/supabase"
+  MongoDBAtlasVectorSearch:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/mongodb_atlas_vector_search"
 wrappers:
-  - RequestsWrapper
-  # - ChatPromptTemplate
-  # - SystemMessagePromptTemplate
-  # - HumanMessagePromptTemplate
+  RequestsWrapper:
+    documentation: ""

src/backend/langflow/graph/graph/base.py

@@ -146,7 +146,7 @@ class Graph:
     def generator_build(self) -> Generator:
         """Builds each vertex in the graph and yields it."""
         sorted_vertices = self.topological_sort()
-        logger.info("Sorted vertices: %s", sorted_vertices)
+        logger.debug("Sorted vertices: %s", sorted_vertices)
         yield from sorted_vertices
 
     def get_node_neighbors(self, node: Vertex) -> Dict[Vertex, int]:

src/backend/langflow/graph/graph/constants.py

@@ -30,9 +30,6 @@ from langflow.interface.wrappers.base import wrapper_creator
 from typing import Dict, Type
 
 
-DIRECT_TYPES = ["str", "bool", "code", "int", "float", "Any", "prompt"]
-
-
 VERTEX_TYPE_MAP: Dict[str, Type[Vertex]] = {
     **{t: PromptVertex for t in prompt_creator.to_list()},
     **{t: AgentVertex for t in agent_creator.to_list()},

src/backend/langflow/graph/vertex/base.py

@@ -1,6 +1,5 @@
-from langflow.cache import utils as cache_utils
-from langflow.graph.vertex.constants import DIRECT_TYPES
-from langflow.interface import loading
+from langflow.utils.constants import DIRECT_TYPES
+from langflow.interface.initialize import loading
 from langflow.interface.listing import ALL_TYPES_DICT
 from langflow.utils.logger import logger
 from langflow.utils.util import sync_to_async
@@ -90,12 +89,7 @@ class Vertex:
                 # Load the type in value.get('suffixes') using
                 # what is inside value.get('content')
                 # value.get('value') is the file name
-                file_name = value.get("value")
-                content = value.get("content")
-                type_to_load = value.get("suffixes")
-                file_path = cache_utils.save_binary_file(
-                    content=content, file_name=file_name, accepted_types=type_to_load
-                )
+                file_path = value.get("file_path")
 
                 params[key] = file_path
 

src/backend/langflow/graph/vertex/constants.py

@@ -1 +1 @@
-DIRECT_TYPES = ["str", "bool", "code", "int", "float", "Any", "prompt"]
+

src/backend/langflow/graph/vertex/types.py

@@ -106,9 +106,6 @@ class VectorStoreVertex(Vertex):
     def __init__(self, data: Dict):
         super().__init__(data, base_type="vectorstores")
 
-    def _built_object_repr(self):
-        return "Vector stores can take time to build. It will build on the first query."
-
 
 class MemoryVertex(Vertex):
     def __init__(self, data: Dict):

src/backend/langflow/interface/agents/custom.py

@@ -6,6 +6,7 @@ from langchain.agents import (
     Tool,
     ZeroShotAgent,
     initialize_agent,
+    AgentType,
 )
 from langchain.agents.agent_toolkits import (
     SQLDatabaseToolkit,
@@ -192,7 +193,7 @@ class SQLAgent(CustomAgentExecutor):
         from langchain.tools.sql_database.tool import (
             InfoSQLDatabaseTool,
             ListSQLDatabaseTool,
-            QueryCheckerTool,
+            QuerySQLCheckerTool,
             QuerySQLDataBaseTool,
         )
 
@@ -207,7 +208,7 @@ class SQLAgent(CustomAgentExecutor):
             QuerySQLDataBaseTool(db=db),  # type: ignore
             InfoSQLDatabaseTool(db=db),  # type: ignore
             ListSQLDatabaseTool(db=db),  # type: ignore
-            QueryCheckerTool(db=db, llm_chain=llmchain, llm=llm),  # type: ignore
+            QuerySQLCheckerTool(db=db, llm_chain=llmchain, llm=llm),  # type: ignore
         ]
 
         prefix = SQL_PREFIX.format(dialect=toolkit.dialect, top_k=10)
@@ -297,6 +298,9 @@ class InitializeAgent(CustomAgentExecutor):
         agent: str,
         memory: Optional[BaseChatMemory] = None,
     ):
+        # Find which value in the AgentType enum corresponds to the string
+        # passed in as agent
+        agent = AgentType(agent)
         return initialize_agent(
             tools=tools,
             llm=llm,

src/backend/langflow/interface/base.py

@@ -8,6 +8,7 @@ from langflow.template.field.base import TemplateField
 from langflow.template.frontend_node.base import FrontendNode
 from langflow.template.template.base import Template
 from langflow.utils.logger import logger
+from langflow.settings import settings
 
 # Assuming necessary imports for Field, Template, and FrontendNode classes
 
@@ -15,12 +16,29 @@ from langflow.utils.logger import logger
 class LangChainTypeCreator(BaseModel, ABC):
     type_name: str
     type_dict: Optional[Dict] = None
+    name_docs_dict: Optional[Dict[str, str]] = None
 
     @property
     def frontend_node_class(self) -> Type[FrontendNode]:
         """The class type of the FrontendNode created in frontend_node."""
         return FrontendNode
 
+    @property
+    def docs_map(self) -> Dict[str, str]:
+        """A dict with the name of the component as key and the documentation link as value."""
+        if self.name_docs_dict is None:
+            try:
+                type_settings = getattr(settings, self.type_name)
+                self.name_docs_dict = {
+                    name: value_dict["documentation"]
+                    for name, value_dict in type_settings.items()
+                }
+            except AttributeError as exc:
+                logger.error(exc)
+
+                self.name_docs_dict = {}
+        return self.name_docs_dict
+
     @property
     @abstractmethod
     def type_to_loader_dict(self) -> Dict:
@@ -68,7 +86,7 @@ class LangChainTypeCreator(BaseModel, ABC):
                     value=value.get("value", None),
                     suffixes=value.get("suffixes", []),
                     file_types=value.get("fileTypes", []),
-                    content=value.get("content", None),
+                    file_path=value.get("file_path", None),
                 )
                 for key, value in signature["template"].items()
                 if key != "_type"
@@ -83,7 +101,7 @@ class LangChainTypeCreator(BaseModel, ABC):
 
         signature.add_extra_fields()
         signature.add_extra_base_classes()
-
+        signature.set_documentation(self.docs_map.get(name, ""))
         return signature
 
 

src/backend/langflow/interface/initialize/loading.py

@@ -1,31 +1,26 @@
 import json
-from typing import Any, Callable, Dict, Optional
+from typing import Any, Callable, Dict, Sequence, Type
 
 from langchain.agents import ZeroShotAgent
 from langchain.agents import agent as agent_module
 from langchain.agents.agent import AgentExecutor
 from langchain.agents.agent_toolkits.base import BaseToolkit
-from langchain.agents.load_tools import (
-    _BASE_TOOLS,
-    _EXTRA_LLM_TOOLS,
-    _EXTRA_OPTIONAL_TOOLS,
-    _LLM_TOOLS,
-)
-from langchain.agents.loading import load_agent_from_config
-from langchain.agents.tools import Tool
-from langchain.base_language import BaseLanguageModel
-from langchain.callbacks.base import BaseCallbackManager
-from langchain.chains.loading import load_chain_from_config
-from langchain.llms.loading import load_llm_from_config
+from langchain.agents.tools import BaseTool
+
+from langflow.interface.initialize.vector_store import vecstore_initializer
+
 from pydantic import ValidationError
 
 from langflow.interface.custom_lists import CUSTOM_NODES
 from langflow.interface.importing.utils import get_function, import_by_type
 from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.chains.base import chain_creator
-from langflow.interface.types import get_type_list
 from langflow.interface.utils import load_file_into_dict
-from langflow.utils import util, validate
+from langflow.utils import validate
+from langchain.chains.base import Chain
+from langchain.vectorstores.base import VectorStore
+from langchain.document_loaders.base import BaseLoader
+from langchain.prompts.base import BasePromptTemplate
 
 
 def instantiate_class(node_type: str, base_type: str, params: Dict) -> Any:
@@ -88,11 +83,11 @@ def instantiate_based_on_type(class_object, base_type, node_type, params):
         return class_object(**params)
 
 
-def instantiate_llm(node_type, class_object, params):
+def instantiate_llm(node_type, class_object, params: Dict):
     return class_object(**params)
 
 
-def instantiate_chains(node_type, class_object, params):
+def instantiate_chains(node_type, class_object: Type[Chain], params: Dict):
     if "retriever" in params and hasattr(params["retriever"], "as_retriever"):
         params["retriever"] = params["retriever"].as_retriever()
     if node_type in chain_creator.from_method_nodes:
@@ -104,11 +99,11 @@ def instantiate_chains(node_type, class_object, params):
     return class_object(**params)
 
 
-def instantiate_agent(class_object, params):
+def instantiate_agent(class_object: Type[agent_module.Agent], params: Dict):
     return load_agent_executor(class_object, params)
 
 
-def instantiate_prompt(node_type, class_object, params):
+def instantiate_prompt(node_type, class_object: Type[BasePromptTemplate], params: Dict):
     if node_type == "ZeroShotPrompt":
         if "tools" not in params:
             params["tools"] = []
@@ -116,7 +111,7 @@ def instantiate_prompt(node_type, class_object, params):
     return class_object(**params)
 
 
-def instantiate_tool(node_type, class_object, params):
+def instantiate_tool(node_type, class_object: Type[BaseTool], params: Dict):
     if node_type == "JsonSpec":
         params["dict_"] = load_file_into_dict(params.pop("path"))
         return class_object(**params)
@@ -134,7 +129,7 @@ def instantiate_tool(node_type, class_object, params):
     return class_object(**params)
 
 
-def instantiate_toolkit(node_type, class_object, params):
+def instantiate_toolkit(node_type, class_object: Type[BaseToolkit], params: Dict):
     loaded_toolkit = class_object(**params)
     # Commenting this out for now to use toolkits as normal tools
     # if toolkits_creator.has_create_function(node_type):
@@ -144,7 +139,7 @@ def instantiate_toolkit(node_type, class_object, params):
     return loaded_toolkit
 
 
-def instantiate_embedding(class_object, params):
+def instantiate_embedding(class_object, params: Dict):
     params.pop("model", None)
     params.pop("headers", None)
     try:
@@ -158,40 +153,81 @@ def instantiate_embedding(class_object, params):
         return class_object(**params)
 
 
-def instantiate_vectorstore(class_object, params):
-    if len(params.get("documents", [])) == 0:
-        raise ValueError(
-            "The source you provided did not load correctly or was empty."
-            "This may cause an error in the vectorstore."
+def instantiate_vectorstore(class_object: Type[VectorStore], params: Dict):
+    search_kwargs = params.pop("search_kwargs", {})
+    if initializer := vecstore_initializer.get(class_object.__name__):
+        vecstore = initializer(class_object, params)
+    else:
+        if "texts" in params:
+            params["documents"] = params.pop("texts")
+        vecstore = class_object.from_documents(**params)
+
+    # ! This might not work. Need to test
+    if search_kwargs and hasattr(vecstore, "as_retriever"):
+        vecstore = vecstore.as_retriever(search_kwargs=search_kwargs)
+
+    return vecstore
+
+
+def instantiate_documentloader(class_object: Type[BaseLoader], params: Dict):
+    if "file_filter" in params:
+        # file_filter will be a string but we need a function
+        # that will be used to filter the files using file_filter
+        # like lambda x: x.endswith(".txt") but as we don't know
+        # anything besides the string, we will simply check if the string is
+        # in x and if it is, we will return True
+        file_filter = params.pop("file_filter", None)
+        extensions = file_filter.split(",")
+        params["file_filter"] = lambda x: any(
+            extension.strip() in x for extension in extensions
         )
-    # Chroma requires all metadata values to not be None
-    if class_object.__name__ == "Chroma":
-        for doc in params["documents"]:
-            if doc.metadata is None:
-                doc.metadata = {}
-            for key, value in doc.metadata.items():
-                if value is None:
-                    doc.metadata[key] = ""
-    return class_object.from_documents(**params)
+    metadata = params.pop("metadata", None)
+    if metadata and isinstance(metadata, str):
+        try:
+            metadata = json.loads(metadata)
+        except json.JSONDecodeError as exc:
+            raise ValueError(
+                "The metadata you provided is not a valid JSON string."
+            ) from exc
+    docs = class_object(**params).load()
+    # Now if metadata is an empty dict, we will not add it to the documents
+    if metadata:
+        for doc in docs:
+            # If the document already has metadata, we will not overwrite it
+            if not doc.metadata:
+                doc.metadata = metadata
+            else:
+                doc.metadata.update(metadata)
+
+    return docs
 
 
-def instantiate_documentloader(class_object, params):
-    return class_object(**params).load()
-
-
-def instantiate_textsplitter(class_object, params):
+def instantiate_textsplitter(
+    class_object,
+    params: Dict,
+):
     try:
         documents = params.pop("documents")
-    except KeyError as e:
+    except KeyError as exc:
         raise ValueError(
             "The source you provided did not load correctly or was empty."
             "Try changing the chunk_size of the Text Splitter."
-        ) from e
-    text_splitter = class_object(**params)
+        ) from exc
+
+    if (
+        "separator_type" in params
+        and params["separator_type"] == "Text"
+        or "separator_type" not in params
+    ):
+        text_splitter = class_object(**params)
+    else:
+        params["language"] = params.pop("separator_type", None)
+        params.pop("separators", None)
+        text_splitter = class_object.from_language(**params)
     return text_splitter.split_documents(documents)
 
 
-def instantiate_utility(node_type, class_object, params):
+def instantiate_utility(node_type, class_object, params: Dict):
     if node_type == "SQLDatabase":
         return class_object.from_uri(params.pop("uri"))
     return class_object(**params)
@@ -213,48 +249,14 @@ def replace_zero_shot_prompt_with_prompt_template(nodes):
     return nodes
 
 
-def load_langchain_type_from_config(config: Dict[str, Any]):
-    """Load langchain type from config"""
-    # Get type list
-    type_list = get_type_list()
-    if config["_type"] in type_list["agents"]:
-        config = util.update_verbose(config, new_value=False)
-        return load_agent_executor_from_config(config, verbose=True)
-    elif config["_type"] in type_list["chains"]:
-        config = util.update_verbose(config, new_value=False)
-        return load_chain_from_config(config, verbose=True)
-    elif config["_type"] in type_list["llms"]:
-        config = util.update_verbose(config, new_value=True)
-        return load_llm_from_config(config)
-    else:
-        raise ValueError("Type should be either agent, chain or llm")
-
-
-def load_agent_executor_from_config(
-    config: dict,
-    llm: Optional[BaseLanguageModel] = None,
-    tools: Optional[list[Tool]] = None,
-    callback_manager: Optional[BaseCallbackManager] = None,
-    **kwargs: Any,
-):
-    tools = load_tools_from_config(config["allowed_tools"])
-    config["allowed_tools"] = [tool.name for tool in tools] if tools else []
-    agent_obj = load_agent_from_config(config, llm, tools, **kwargs)
-
-    return AgentExecutor.from_agent_and_tools(
-        agent=agent_obj,
-        tools=tools,
-        callback_manager=callback_manager,
-        **kwargs,
-    )
-
-
 def load_agent_executor(agent_class: type[agent_module.Agent], params, **kwargs):
     """Load agent executor from agent class, tools and chain"""
-    allowed_tools = params.get("allowed_tools", [])
+    allowed_tools: Sequence[BaseTool] = params.get("allowed_tools", [])
     llm_chain = params["llm_chain"]
     # if allowed_tools is not a list or set, make it a list
-    if not isinstance(allowed_tools, (list, set)):
+    if not isinstance(allowed_tools, (list, set)) and isinstance(
+        allowed_tools, BaseTool
+    ):
         allowed_tools = [allowed_tools]
     tool_names = [tool.name for tool in allowed_tools]
     # Agent class requires an output_parser but Agent classes
@@ -273,46 +275,6 @@ def load_toolkits_executor(node_type: str, toolkit: BaseToolkit, params: dict):
         return create_function(llm=llm, toolkit=toolkit)
 
 
-def load_tools_from_config(tool_list: list[dict]) -> list:
-    """Load tools based on a config list.
-
-    Args:
-        config: config list.
-
-    Returns:
-        List of tools.
-    """
-    tools = []
-    for tool in tool_list:
-        tool_type = tool.pop("_type")
-        llm_config = tool.pop("llm", None)
-        llm = load_llm_from_config(llm_config) if llm_config else None
-        kwargs = tool
-        if tool_type in _BASE_TOOLS:
-            tools.append(_BASE_TOOLS[tool_type]())
-        elif tool_type in _LLM_TOOLS:
-            if llm is None:
-                raise ValueError(f"Tool {tool_type} requires an LLM to be provided")
-            tools.append(_LLM_TOOLS[tool_type](llm))
-        elif tool_type in _EXTRA_LLM_TOOLS:
-            if llm is None:
-                raise ValueError(f"Tool {tool_type} requires an LLM to be provided")
-            _get_llm_tool_func, extra_keys = _EXTRA_LLM_TOOLS[tool_type]
-            if missing_keys := set(extra_keys).difference(kwargs):
-                raise ValueError(
-                    f"Tool {tool_type} requires some parameters that were not "
-                    f"provided: {missing_keys}"
-                )
-            tools.append(_get_llm_tool_func(llm=llm, **kwargs))
-        elif tool_type in _EXTRA_OPTIONAL_TOOLS:
-            _get_tool_func, extra_keys = _EXTRA_OPTIONAL_TOOLS[tool_type]
-            kwargs = {k: value for k, value in kwargs.items() if value}
-            tools.append(_get_tool_func(**kwargs))
-        else:
-            raise ValueError(f"Got unknown tool {tool_type}")
-    return tools
-
-
 def build_prompt_template(prompt, tools):
     """Build PromptTemplate from ZeroShotPrompt"""
     prefix = prompt["node"]["template"]["prefix"]["value"]

src/backend/langflow/interface/initialize/vector_store.py

@@ -0,0 +1,223 @@
+import json
+from typing import Any, Callable, Dict, Type
+from langchain.vectorstores import (
+    Pinecone,
+    Qdrant,
+    Chroma,
+    FAISS,
+    Weaviate,
+    SupabaseVectorStore,
+    MongoDBAtlasVectorSearch,
+)
+
+import os
+
+
+def docs_in_params(params: dict) -> bool:
+    """Check if params has documents OR texts and one of them is not an empty list,
+    If any of them is not an empty list, return True, else return False"""
+    return ("documents" in params and params["documents"]) or (
+        "texts" in params and params["texts"]
+    )
+
+
+def initialize_mongodb(class_object: Type[MongoDBAtlasVectorSearch], params: dict):
+    """Initialize mongodb and return the class object"""
+
+    MONGODB_ATLAS_CLUSTER_URI = params.pop("mongodb_atlas_cluster_uri")
+    if not MONGODB_ATLAS_CLUSTER_URI:
+        raise ValueError("Mongodb atlas cluster uri must be provided in the params")
+    from pymongo import MongoClient
+    import certifi
+
+    client: MongoClient = MongoClient(
+        MONGODB_ATLAS_CLUSTER_URI, tlsCAFile=certifi.where()
+    )
+    db_name = params.pop("db_name", None)
+    collection_name = params.pop("collection_name", None)
+    if not db_name or not collection_name:
+        raise ValueError("db_name and collection_name must be provided in the params")
+
+    index_name = params.pop("index_name", None)
+    if not index_name:
+        raise ValueError("index_name must be provided in the params")
+
+    collection = client[db_name][collection_name]
+    if not docs_in_params(params):
+        # __init__ requires collection, embedding and index_name
+        init_args = {
+            "collection": collection,
+            "index_name": index_name,
+            "embedding": params.get("embedding"),
+        }
+
+        return class_object(**init_args)
+
+    if "texts" in params:
+        params["documents"] = params.pop("texts")
+
+    params["collection"] = collection
+    params["index_name"] = index_name
+
+    return class_object.from_documents(**params)
+
+
+def initialize_supabase(class_object: Type[SupabaseVectorStore], params: dict):
+    """Initialize supabase and return the class object"""
+    from supabase.client import Client, create_client
+
+    if "supabase_url" not in params or "supabase_service_key" not in params:
+        raise ValueError("Supabase url and service key must be provided in the params")
+    if "texts" in params:
+        params["documents"] = params.pop("texts")
+
+    client_kwargs = {
+        "supabase_url": params.pop("supabase_url"),
+        "supabase_key": params.pop("supabase_service_key"),
+    }
+
+    supabase: Client = create_client(**client_kwargs)
+    if not docs_in_params(params):
+        params.pop("documents", None)
+        params.pop("texts", None)
+        return class_object(client=supabase, **params)
+    # If there are docs in the params, create a new index
+
+    return class_object.from_documents(client=supabase, **params)
+
+
+def initialize_weaviate(class_object: Type[Weaviate], params: dict):
+    """Initialize weaviate and return the class object"""
+    if not docs_in_params(params):
+        import weaviate  # type: ignore
+
+        client_kwargs_json = params.get("client_kwargs", "{}")
+        client_kwargs = json.loads(client_kwargs_json)
+        client_params = {
+            "url": params.get("weaviate_url"),
+        }
+        client_params.update(client_kwargs)
+        weaviate_client = weaviate.Client(**client_params)
+
+        new_params = {
+            "client": weaviate_client,
+            "index_name": params.get("index_name"),
+            "text_key": params.get("text_key"),
+        }
+        return class_object(**new_params)
+    # If there are docs in the params, create a new index
+    if "texts" in params:
+        params["documents"] = params.pop("texts")
+
+    return class_object.from_documents(**params)
+
+
+def initialize_faiss(class_object: Type[FAISS], params: dict):
+    """Initialize faiss and return the class object"""
+
+    if not docs_in_params(params):
+        return class_object.load_local
+
+    save_local = params.get("save_local")
+    faiss_index = class_object(**params)
+    if save_local:
+        faiss_index.save_local(folder_path=save_local)
+    return faiss_index
+
+
+def initialize_pinecone(class_object: Type[Pinecone], params: dict):
+    """Initialize pinecone and return the class object"""
+
+    import pinecone  # type: ignore
+
+    pinecone_api_key = params.get("pinecone_api_key")
+    pinecone_env = params.get("pinecone_env")
+
+    if pinecone_api_key is None or pinecone_env is None:
+        if os.getenv("PINECONE_API_KEY") is not None:
+            pinecone_api_key = os.getenv("PINECONE_API_KEY")
+        if os.getenv("PINECONE_ENV") is not None:
+            pinecone_env = os.getenv("PINECONE_ENV")
+
+    if pinecone_api_key is None or pinecone_env is None:
+        raise ValueError(
+            "Pinecone API key and environment must be provided in the params"
+        )
+
+    # initialize pinecone
+    pinecone.init(
+        api_key=pinecone_api_key,  # find at app.pinecone.io
+        environment=pinecone_env,  # next to api key in console
+    )
+
+    # If there are no docs in the params, return an existing index
+    # but first remove any texts or docs keys from the params
+    if not docs_in_params(params):
+        existing_index_params = {
+            "embedding": params.pop("embedding"),
+        }
+        if "index_name" in params:
+            existing_index_params["index_name"] = params.pop("index_name")
+        if "namespace" in params:
+            existing_index_params["namespace"] = params.pop("namespace")
+
+        return class_object.from_existing_index(**existing_index_params)
+    # If there are docs in the params, create a new index
+    if "texts" in params:
+        params["documents"] = params.pop("texts")
+    return class_object.from_documents(**params)
+
+
+def initialize_chroma(class_object: Type[Chroma], params: dict):
+    """Initialize a ChromaDB object from the params"""
+    persist = params.pop("persist", False)
+    if not docs_in_params(params):
+        params.pop("documents", None)
+        params.pop("texts", None)
+        params["embedding_function"] = params.pop("embedding")
+        chromadb = class_object(**params)
+    else:
+        if "texts" in params:
+            params["documents"] = params.pop("texts")
+        for doc in params["documents"]:
+            if doc.metadata is None:
+                doc.metadata = {}
+            for key, value in doc.metadata.items():
+                if value is None:
+                    doc.metadata[key] = ""
+        chromadb = class_object.from_documents(**params)
+    if persist:
+        chromadb.persist()
+    return chromadb
+
+
+def initialize_qdrant(class_object: Type[Qdrant], params: dict):
+    if not docs_in_params(params):
+        if "location" not in params and "api_key" not in params:
+            raise ValueError("Location and API key must be provided in the params")
+        from qdrant_client import QdrantClient
+
+        client_params = {
+            "location": params.pop("location"),
+            "api_key": params.pop("api_key"),
+        }
+        lc_params = {
+            "collection_name": params.pop("collection_name"),
+            "embeddings": params.pop("embedding"),
+        }
+        client = QdrantClient(**client_params)
+
+        return class_object(client=client, **lc_params)
+
+    return class_object.from_documents(**params)
+
+
+vecstore_initializer: Dict[str, Callable[[Type[Any], dict], Any]] = {
+    "Pinecone": initialize_pinecone,
+    "Chroma": initialize_chroma,
+    "Qdrant": initialize_qdrant,
+    "Weaviate": initialize_weaviate,
+    "FAISS": initialize_faiss,
+    "SupabaseVectorStore": initialize_supabase,
+    "MongoDBAtlasVectorSearch": initialize_mongodb,
+}

src/backend/langflow/interface/utils.py

@@ -4,9 +4,12 @@ import os
 from io import BytesIO
 import re
 
+
 import yaml
 from langchain.base_language import BaseLanguageModel
 from PIL.Image import Image
+from langflow.utils.logger import logger
+from langflow.chat.config import ChatConfig
 
 
 def load_file_into_dict(file_path: str) -> dict:
@@ -48,9 +51,9 @@ def try_setting_streaming_options(langchain_object, websocket):
 
     if isinstance(llm, BaseLanguageModel):
         if hasattr(llm, "streaming") and isinstance(llm.streaming, bool):
-            llm.streaming = True
+            llm.streaming = ChatConfig.streaming
         elif hasattr(llm, "stream") and isinstance(llm.stream, bool):
-            llm.stream = True
+            llm.stream = ChatConfig.streaming
 
     return langchain_object
 
@@ -58,3 +61,22 @@ def try_setting_streaming_options(langchain_object, websocket):
 def extract_input_variables_from_prompt(prompt: str) -> list[str]:
     """Extract input variables from prompt."""
     return re.findall(r"{(.*?)}", prompt)
+
+
+def setup_llm_caching():
+    """Setup LLM caching."""
+
+    try:
+        import langchain
+        from langflow.settings import settings
+        from langflow.interface.importing.utils import import_class
+
+        cache_class = import_class(f"langchain.cache.{settings.cache}")
+
+        logger.debug(f"Setting up LLM caching with {cache_class.__name__}")
+        langchain.llm_cache = cache_class()
+        logger.info(f"LLM caching setup with {cache_class.__name__}")
+    except ImportError:
+        logger.warning(f"Could not import {settings.cache}. ")
+    except Exception as exc:
+        logger.warning(f"Could not setup LLM caching. Error: {exc}")

src/backend/langflow/main.py

@@ -1,8 +1,13 @@
+from pathlib import Path
+from typing import Optional
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
 
 from langflow.api import router
 from langflow.database.base import create_db_and_tables
+from langflow.interface.utils import setup_llm_caching
 
 
 def create_app():
@@ -28,6 +33,43 @@ def create_app():
 
     app.include_router(router)
     app.on_event("startup")(create_db_and_tables)
+    app.on_event("startup")(setup_llm_caching)
+    return app
+
+
+def setup_static_files(app: FastAPI, static_files_dir: Path):
+    """
+    Setup the static files directory.
+    Args:
+        app (FastAPI): FastAPI app.
+        path (str): Path to the static files directory.
+    """
+    app.mount(
+        "/",
+        StaticFiles(directory=static_files_dir, html=True),
+        name="static",
+    )
+
+    @app.exception_handler(404)
+    async def custom_404_handler(request, __):
+        path = static_files_dir / "index.html"
+
+        if not path.exists():
+            raise RuntimeError(f"File at path {path} does not exist.")
+        return FileResponse(path)
+
+
+# app = create_app()
+# setup_static_files(app, static_files_dir)
+def setup_app(static_files_dir: Optional[Path]) -> FastAPI:
+    """Setup the FastAPI app."""
+    # get the directory of the current file
+    if not static_files_dir:
+        frontend_path = Path(__file__).parent
+        static_files_dir = frontend_path / "frontend"
+
+    app = create_app()
+    setup_static_files(app, static_files_dir)
     return app
 
 

src/backend/langflow/processing/process.py

@@ -1,5 +1,3 @@
-import contextlib
-import io
 from pathlib import Path
 from langchain.schema import AgentAction
 import json
@@ -10,7 +8,8 @@ from langflow.interface.run import (
 )
 from langflow.utils.logger import logger
 from langflow.graph import Graph
-
+from langchain.chains.base import Chain
+from langchain.vectorstores.base import VectorStore
 from typing import Any, Dict, List, Optional, Tuple, Union
 
 
@@ -55,69 +54,42 @@ def format_actions(actions: List[Tuple[AgentAction, str]]) -> str:
     return "\n".join(output)
 
 
-def get_result_and_thought(langchain_object, message: str):
+def get_result_and_thought(langchain_object: Any, inputs: dict):
     """Get result and thought from extracted json"""
     try:
         if hasattr(langchain_object, "verbose"):
             langchain_object.verbose = True
-        chat_input = None
-        memory_key = ""
-        if hasattr(langchain_object, "memory") and langchain_object.memory is not None:
-            memory_key = langchain_object.memory.memory_key
-
-        if hasattr(langchain_object, "input_keys"):
-            for key in langchain_object.input_keys:
-                if key not in [memory_key, "chat_history"]:
-                    chat_input = {key: message}
-        else:
-            chat_input = message  # type: ignore
 
         if hasattr(langchain_object, "return_intermediate_steps"):
-            # https://github.com/hwchase17/langchain/issues/2068
-            # Deactivating until we have a frontend solution
-            # to display intermediate steps
-            langchain_object.return_intermediate_steps = False
+            langchain_object.return_intermediate_steps = True
 
         fix_memory_inputs(langchain_object)
 
-        with io.StringIO() as output_buffer, contextlib.redirect_stdout(output_buffer):
-            try:
-                # if hasattr(langchain_object, "acall"):
-                #     output = await langchain_object.acall(chat_input)
-                # else:
-                output = langchain_object(chat_input)
-            except ValueError as exc:
-                # make the error message more informative
-                logger.debug(f"Error: {str(exc)}")
-                output = langchain_object.run(chat_input)
-
-            intermediate_steps = (
-                output.get("intermediate_steps", []) if isinstance(output, dict) else []
-            )
-
-            result = (
-                output.get(langchain_object.output_keys[0])
-                if isinstance(output, dict)
-                else output
-            )
-            if intermediate_steps:
-                thought = format_actions(intermediate_steps)
-            else:
-                thought = output_buffer.getvalue()
+        try:
+            output = langchain_object(inputs, return_only_outputs=True)
+        except ValueError as exc:
+            # make the error message more informative
+            logger.debug(f"Error: {str(exc)}")
+            output = langchain_object.run(inputs)
 
     except Exception as exc:
         raise ValueError(f"Error: {str(exc)}") from exc
-    return result, thought
+    return output
 
 
-def process_graph_cached(data_graph: Dict[str, Any], message: str):
+def get_input_str_if_only_one_input(inputs: dict) -> Optional[str]:
+    """Get input string if only one input is provided"""
+    return list(inputs.values())[0] if len(inputs) == 1 else None
+
+
+def process_graph_cached(data_graph: Dict[str, Any], inputs: Optional[dict] = None):
     """
     Process graph by extracting input variables and replacing ZeroShotPrompt
     with PromptTemplate,then run the graph and return the result and thought.
     """
     # Load langchain object
     langchain_object = build_langchain_object_with_caching(data_graph)
-    logger.debug("Loaded langchain object")
+    logger.debug("Loaded LangChain object")
 
     if langchain_object is None:
         # Raise user facing error
@@ -126,10 +98,16 @@ def process_graph_cached(data_graph: Dict[str, Any], message: str):
         )
 
     # Generate result and thought
-    logger.debug("Generating result and thought")
-    result, thought = get_result_and_thought(langchain_object, message)
-    logger.debug("Generated result and thought")
-    return {"result": str(result), "thought": thought.strip()}
+    if isinstance(langchain_object, Chain):
+        if inputs is None:
+            raise ValueError("Inputs must be provided for a Chain")
+        logger.debug("Generating result and thought")
+        result = get_result_and_thought(langchain_object, inputs)
+        logger.debug("Generated result and thought")
+    elif isinstance(langchain_object, VectorStore):
+        class_name = langchain_object.__class__.__name__
+        result = {"message": f"Processed {class_name} successfully"}
+    return result
 
 
 def load_flow_from_json(
@@ -206,7 +184,8 @@ def apply_tweaks(node: Dict[str, Any], node_tweaks: Dict[str, Any]) -> None:
 
     for tweak_name, tweak_value in node_tweaks.items():
         if tweak_name and tweak_value and tweak_name in template_data:
-            template_data[tweak_name]["value"] = tweak_value
+            key = tweak_name if tweak_name == "file_path" else "value"
+            template_data[tweak_name][key] = tweak_value
 
 
 def process_tweaks(

src/backend/langflow/settings.py

@@ -1,26 +1,26 @@
 import os
-from typing import List
 
 import yaml
 from pydantic import BaseSettings, root_validator
 
 
 class Settings(BaseSettings):
-    chains: List[str] = []
-    agents: List[str] = []
-    prompts: List[str] = []
-    llms: List[str] = []
-    tools: List[str] = []
-    memories: List[str] = []
-    embeddings: List[str] = []
-    vectorstores: List[str] = []
-    documentloaders: List[str] = []
-    wrappers: List[str] = []
-    toolkits: List[str] = []
-    textsplitters: List[str] = []
-    utilities: List[str] = []
+    chains: dict = {}
+    agents: dict = {}
+    prompts: dict = {}
+    llms: dict = {}
+    tools: dict = {}
+    memories: dict = {}
+    embeddings: dict = {}
+    vectorstores: dict = {}
+    documentloaders: dict = {}
+    wrappers: dict = {}
+    toolkits: dict = {}
+    textsplitters: dict = {}
+    utilities: dict = {}
     dev: bool = False
     database_url: str = "sqlite:///./langflow.db"
+    cache: str = "InMemoryCache"
     remove_api_keys: bool = False
 
     class Config:
@@ -37,16 +37,16 @@ class Settings(BaseSettings):
 
     def update_from_yaml(self, file_path: str, dev: bool = False):
         new_settings = load_settings_from_yaml(file_path)
-        self.chains = new_settings.chains or []
-        self.agents = new_settings.agents or []
-        self.prompts = new_settings.prompts or []
-        self.llms = new_settings.llms or []
-        self.tools = new_settings.tools or []
-        self.memories = new_settings.memories or []
-        self.wrappers = new_settings.wrappers or []
-        self.toolkits = new_settings.toolkits or []
-        self.textsplitters = new_settings.textsplitters or []
-        self.utilities = new_settings.utilities or []
+        self.chains = new_settings.chains or {}
+        self.agents = new_settings.agents or {}
+        self.prompts = new_settings.prompts or {}
+        self.llms = new_settings.llms or {}
+        self.tools = new_settings.tools or {}
+        self.memories = new_settings.memories or {}
+        self.wrappers = new_settings.wrappers or {}
+        self.toolkits = new_settings.toolkits or {}
+        self.textsplitters = new_settings.textsplitters or {}
+        self.utilities = new_settings.utilities or {}
         self.dev = dev
 
     def update_settings(self, **kwargs):

src/backend/langflow/template/field/base.py

@@ -15,12 +15,13 @@ class TemplateFieldCreator(BaseModel, ABC):
     suffixes: list[str] = []
     fileTypes: list[str] = []
     file_types: list[str] = []
-    content: Union[str, None] = None
+    file_path: Union[str, None] = None
     password: bool = False
     options: list[str] = []
     name: str = ""
     display_name: Optional[str] = None
     advanced: bool = False
+    info: Optional[str] = ""
 
     def to_dict(self):
         result = self.dict()
@@ -35,7 +36,7 @@ class TemplateFieldCreator(BaseModel, ABC):
             result["fileTypes"] = result.pop("file_types")
 
         if self.field_type == "file":
-            result["content"] = self.content
+            result["file_path"] = self.file_path
         return result
 
 

src/backend/langflow/template/frontend_node/base.py

@@ -15,14 +15,21 @@ class FrontendNode(BaseModel):
     base_classes: List[str]
     name: str = ""
     display_name: str = ""
+    documentation: str = ""
+
+    def set_documentation(self, documentation: str) -> None:
+        """Sets the documentation of the frontend node."""
+        self.documentation = documentation
 
     def to_dict(self) -> dict:
+        """Returns a dict representation of the frontend node."""
         return {
             self.name: {
                 "template": self.template.to_dict(self.format_field),
                 "description": self.description,
                 "base_classes": self.base_classes,
                 "display_name": self.display_name or self.name,
+                "documentation": self.documentation,
             },
         }
 

src/backend/langflow/template/frontend_node/chains.py

@@ -13,12 +13,37 @@ class ChainFrontendNode(FrontendNode):
             self.template.add_field(
                 TemplateField(
                     field_type="BaseChatMemory",
-                    required=False,
+                    required=True,
                     show=True,
                     name="memory",
                     advanced=False,
                 )
             )
+            # add return_source_documents
+            self.template.add_field(
+                TemplateField(
+                    field_type="bool",
+                    required=False,
+                    show=True,
+                    name="return_source_documents",
+                    advanced=False,
+                    value=True,
+                    display_name="Return source documents",
+                )
+            )
+            self.template.add_field(
+                TemplateField(
+                    field_type="str",
+                    required=True,
+                    is_list=True,
+                    show=True,
+                    multiline=False,
+                    options=QA_CHAIN_TYPES,
+                    value=QA_CHAIN_TYPES[0],
+                    name="chain_type",
+                    advanced=False,
+                )
+            )
 
     @staticmethod
     def format_field(field: TemplateField, name: Optional[str] = None) -> None:
@@ -47,7 +72,7 @@ class ChainFrontendNode(FrontendNode):
             field.show = True
             field.advanced = False
         if field.name == "memory":
-            field.required = False
+            # field.required = False
             field.show = True
             field.advanced = False
         if field.name == "verbose":
@@ -59,6 +84,12 @@ class ChainFrontendNode(FrontendNode):
             field.show = True
             field.advanced = False
 
+        if field.name == "return_source_documents":
+            field.required = False
+            field.show = True
+            field.advanced = True
+            field.value = True
+
 
 class SeriesCharacterChainNode(FrontendNode):
     name: str = "SeriesCharacterChain"

src/backend/langflow/template/frontend_node/constants.py

@@ -49,3 +49,12 @@ CTRANSFORMERS_DEFAULT_CONFIG = {
     "context_length": -1,
     "gpu_layers": 0,
 }
+
+# This variable is used to tell the user
+# that it can be changed to use other APIs
+# like Prem and LocalAI
+OPENAI_API_BASE_INFO = """
+The base URL of the OpenAI API. Defaults to https://api.openai.com/v1.
+
+You can change this to use other APIs like JinaChat, LocalAI and Prem.
+"""

src/backend/langflow/template/frontend_node/documentloaders.py

@@ -1,8 +1,9 @@
+from typing import Optional
 from langflow.template.field.base import TemplateField
 from langflow.template.frontend_node.base import FrontendNode
 
 
-def build_template(
+def build_file_field(
     suffixes: list, fileTypes: list, name: str = "file_path"
 ) -> TemplateField:
     """Build a template field for a document loader."""
@@ -19,33 +20,35 @@ def build_template(
 
 class DocumentLoaderFrontNode(FrontendNode):
     file_path_templates = {
-        "AirbyteJSONLoader": build_template(suffixes=[".json"], fileTypes=["json"]),
-        "CoNLLULoader": build_template(suffixes=[".csv"], fileTypes=["csv"]),
-        "CSVLoader": build_template(suffixes=[".csv"], fileTypes=["csv"]),
-        "UnstructuredEmailLoader": build_template(suffixes=[".eml"], fileTypes=["eml"]),
-        "EverNoteLoader": build_template(suffixes=[".xml"], fileTypes=["xml"]),
-        "FacebookChatLoader": build_template(suffixes=[".json"], fileTypes=["json"]),
-        "GutenbergLoader": build_template(suffixes=[".txt"], fileTypes=["txt"]),
-        "BSHTMLLoader": build_template(suffixes=[".html"], fileTypes=["html"]),
-        "UnstructuredHTMLLoader": build_template(
+        "AirbyteJSONLoader": build_file_field(suffixes=[".json"], fileTypes=["json"]),
+        "CoNLLULoader": build_file_field(suffixes=[".csv"], fileTypes=["csv"]),
+        "CSVLoader": build_file_field(suffixes=[".csv"], fileTypes=["csv"]),
+        "UnstructuredEmailLoader": build_file_field(
+            suffixes=[".eml"], fileTypes=["eml"]
+        ),
+        "SlackDirectoryLoader": build_file_field(suffixes=[".zip"], fileTypes=["zip"]),
+        "EverNoteLoader": build_file_field(suffixes=[".xml"], fileTypes=["xml"]),
+        "FacebookChatLoader": build_file_field(suffixes=[".json"], fileTypes=["json"]),
+        "GutenbergLoader": build_file_field(suffixes=[".txt"], fileTypes=["txt"]),
+        "BSHTMLLoader": build_file_field(suffixes=[".html"], fileTypes=["html"]),
+        "UnstructuredHTMLLoader": build_file_field(
             suffixes=[".html"], fileTypes=["html"]
         ),
-        "UnstructuredImageLoader": build_template(
+        "UnstructuredImageLoader": build_file_field(
             suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"],
             fileTypes=["jpg", "jpeg", "png", "gif", "bmp"],
         ),
-        "UnstructuredMarkdownLoader": build_template(
+        "UnstructuredMarkdownLoader": build_file_field(
             suffixes=[".md"], fileTypes=["md"]
         ),
-        "PyPDFLoader": build_template(suffixes=[".pdf"], fileTypes=["pdf"]),
-        "UnstructuredPowerPointLoader": build_template(
+        "PyPDFLoader": build_file_field(suffixes=[".pdf"], fileTypes=["pdf"]),
+        "UnstructuredPowerPointLoader": build_file_field(
             suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"]
         ),
-        "SlackDirectoryLoader": build_template(suffixes=[".zip"], fileTypes=["zip"]),
-        "SRTLoader": build_template(suffixes=[".srt"], fileTypes=["srt"]),
-        "TelegramChatLoader": build_template(suffixes=[".json"], fileTypes=["json"]),
-        "TextLoader": build_template(suffixes=[".txt"], fileTypes=["txt"]),
-        "UnstructuredWordDocumentLoader": build_template(
+        "SRTLoader": build_file_field(suffixes=[".srt"], fileTypes=["srt"]),
+        "TelegramChatLoader": build_file_field(suffixes=[".json"], fileTypes=["json"]),
+        "TextLoader": build_file_field(suffixes=[".txt"], fileTypes=["txt"]),
+        "UnstructuredWordDocumentLoader": build_file_field(
             suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"]
         ),
     }
@@ -53,7 +56,54 @@ class DocumentLoaderFrontNode(FrontendNode):
     def add_extra_fields(self) -> None:
         name = None
         display_name = "Web Page"
-        if self.template.type_name in self.file_path_templates:
+        if self.template.type_name in {"GitLoader"}:
+            # Add fields repo_path, clone_url, branch and file_filter
+            self.template.add_field(
+                TemplateField(
+                    field_type="str",
+                    required=True,
+                    show=True,
+                    name="repo_path",
+                    value="",
+                    display_name="Path to repository",
+                    advanced=False,
+                )
+            )
+            self.template.add_field(
+                TemplateField(
+                    field_type="str",
+                    required=False,
+                    show=True,
+                    name="clone_url",
+                    value="",
+                    display_name="Clone URL",
+                    advanced=False,
+                )
+            )
+            self.template.add_field(
+                TemplateField(
+                    field_type="str",
+                    required=True,
+                    show=True,
+                    name="branch",
+                    value="",
+                    display_name="Branch",
+                    advanced=False,
+                )
+            )
+            self.template.add_field(
+                TemplateField(
+                    field_type="str",
+                    required=False,
+                    show=True,
+                    name="file_filter",
+                    value="",
+                    display_name="File extensions (comma-separated)",
+                    advanced=False,
+                )
+            )
+
+        elif self.template.type_name in self.file_path_templates:
             self.template.add_field(self.file_path_templates[self.template.type_name])
         elif self.template.type_name in {
             "WebBaseLoader",
@@ -67,8 +117,9 @@ class DocumentLoaderFrontNode(FrontendNode):
         elif self.template.type_name in {"GitbookLoader"}:
             name = "web_page"
         elif self.template.type_name in {
-            "NotionDirectoryLoader",
+            "DirectoryLoader",
             "ReadTheDocsLoader",
+            "NotionDirectoryLoader",
         }:
             name = "path"
             display_name = "Local directory"
@@ -83,3 +134,34 @@ class DocumentLoaderFrontNode(FrontendNode):
                     display_name=display_name,
                 )
             )
+            if self.template.type_name in {"DirectoryLoader"}:
+                self.template.add_field(
+                    TemplateField(
+                        field_type="str",
+                        required=True,
+                        show=True,
+                        name="glob",
+                        value="**/*.txt",
+                        display_name="glob",
+                    )
+                )
+            # add a metadata field of type dict
+        self.template.add_field(
+            TemplateField(
+                field_type="code",
+                required=True,
+                show=True,
+                name="metadata",
+                value="{}",
+                display_name="Metadata",
+                multiline=False,
+            )
+        )
+
+    @staticmethod
+    def format_field(field: TemplateField, name: Optional[str] = None) -> None:
+        FrontendNode.format_field(field, name)
+        if field.name == "metadata":
+            field.show = True
+            field.advanced = False
+        field.show = True

src/backend/langflow/template/frontend_node/llms.py

@@ -4,6 +4,7 @@ from typing import Optional
 from langflow.template.field.base import TemplateField
 from langflow.template.frontend_node.base import FrontendNode
 from langflow.template.frontend_node.constants import CTRANSFORMERS_DEFAULT_CONFIG
+from langflow.template.frontend_node.constants import OPENAI_API_BASE_INFO
 
 
 class LLMFrontendNode(FrontendNode):
@@ -17,6 +18,9 @@ class LLMFrontendNode(FrontendNode):
         if "key" not in field.name.lower() and "token" not in field.name.lower():
             field.password = False
 
+        if field.name == "openai_api_base":
+            field.info = OPENAI_API_BASE_INFO
+
     @staticmethod
     def format_azure_field(field: TemplateField):
         if field.name == "model_name":

src/backend/langflow/template/frontend_node/memories.py

@@ -18,6 +18,27 @@ class MemoryFrontendNode(FrontendNode):
                 value=False,
             )
         )
+        # add input_key and output_key str fields
+        self.template.add_field(
+            TemplateField(
+                field_type="str",
+                required=False,
+                show=True,
+                name="input_key",
+                advanced=True,
+                value="",
+            )
+        )
+        self.template.add_field(
+            TemplateField(
+                field_type="str",
+                required=False,
+                show=True,
+                name="output_key",
+                advanced=True,
+                value="",
+            )
+        )
 
     @staticmethod
     def format_field(field: TemplateField, name: Optional[str] = None) -> None:
@@ -36,3 +57,10 @@ class MemoryFrontendNode(FrontendNode):
             field.required = False
             field.show = True
             field.advanced = False
+        if field.name in ["input_key", "output_key"]:
+            field.required = False
+            field.show = True
+            field.advanced = False
+            field.value = ""
+        if field.name == "memory_key":
+            field.value = "chat_history"

src/backend/langflow/template/frontend_node/textsplitters.py

@@ -1,5 +1,6 @@
 from langflow.template.field.base import TemplateField
 from langflow.template.frontend_node.base import FrontendNode
+from langchain.text_splitter import Language
 
 
 class TextSplittersFrontendNode(FrontendNode):
@@ -17,6 +18,22 @@ class TextSplittersFrontendNode(FrontendNode):
             name = "separator"
         elif self.template.type_name == "RecursiveCharacterTextSplitter":
             name = "separators"
+            # Add a field for type of separator
+            # which will have Text or any value from the
+            # Language enum
+            self.template.add_field(
+                TemplateField(
+                    field_type="str",
+                    required=True,
+                    show=True,
+                    name="separator_type",
+                    advanced=False,
+                    is_list=True,
+                    options=[x.value for x in Language],
+                    value="Text",
+                    display_name="Separator Type",
+                )
+            )
         self.template.add_field(
             TemplateField(
                 field_type="str",

src/backend/langflow/template/frontend_node/tools.py

@@ -96,6 +96,16 @@ class PythonFunctionToolNode(FrontendNode):
                 name="code",
                 advanced=False,
             ),
+            TemplateField(
+                field_type="bool",
+                required=True,
+                placeholder="",
+                is_list=False,
+                show=True,
+                multiline=False,
+                value=False,
+                name="return_direct",
+            ),
         ],
     )
     description: str = "Python function to be executed."

src/backend/langflow/template/frontend_node/vectorstores.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import List, Optional
 
 from langflow.template.field.base import TemplateField
 from langflow.template.frontend_node.base import FrontendNode
@@ -6,6 +6,19 @@ from langflow.template.frontend_node.base import FrontendNode
 
 class VectorStoreFrontendNode(FrontendNode):
     def add_extra_fields(self) -> None:
+        extra_fields: List[TemplateField] = []
+        # Add search_kwargs field
+        extra_field = TemplateField(
+            name="search_kwargs",
+            field_type="code",
+            required=False,
+            placeholder="",
+            show=True,
+            advanced=True,
+            multiline=False,
+            value="{}",
+        )
+        extra_fields.append(extra_field)
         if self.template.type_name == "Weaviate":
             extra_field = TemplateField(
                 name="weaviate_url",
@@ -17,17 +30,201 @@ class VectorStoreFrontendNode(FrontendNode):
                 multiline=False,
                 value="http://localhost:8080",
             )
+            # Add client_kwargs field
+            extra_field2 = TemplateField(
+                name="client_kwargs",
+                field_type="code",
+                required=False,
+                placeholder="",
+                show=True,
+                advanced=True,
+                multiline=False,
+                value="{}",
+            )
+            extra_fields.extend((extra_field, extra_field2))
 
-            self.template.add_field(extra_field)
+        elif self.template.type_name == "Chroma":
+            # New bool field for persist parameter
+            extra_field = TemplateField(
+                name="persist",
+                field_type="bool",
+                required=False,
+                show=True,
+                advanced=False,
+                value=True,
+                display_name="Persist",
+            )
+            extra_fields.append(extra_field)
+        elif self.template.type_name == "Pinecone":
+            # add pinecone_api_key and pinecone_env
+            extra_field = TemplateField(
+                name="pinecone_api_key",
+                field_type="str",
+                required=False,
+                placeholder="",
+                show=True,
+                advanced=True,
+                multiline=False,
+                value="",
+            )
+            extra_field2 = TemplateField(
+                name="pinecone_env",
+                field_type="str",
+                required=False,
+                placeholder="",
+                show=True,
+                advanced=True,
+                multiline=False,
+                value="",
+            )
+            extra_fields.extend((extra_field, extra_field2))
+        elif self.template.type_name == "FAISS":
+            extra_field = TemplateField(
+                name="folder_path",
+                field_type="str",
+                required=False,
+                placeholder="",
+                show=True,
+                advanced=True,
+                multiline=False,
+                display_name="Local Path",
+                value="",
+            )
+            extra_field2 = TemplateField(
+                name="index_name",
+                field_type="str",
+                required=False,
+                show=True,
+                advanced=False,
+                value="",
+                display_name="Index Name",
+            )
+            extra_fields.extend((extra_field, extra_field2))
+        elif self.template.type_name == "SupabaseVectorStore":
+            self.display_name = "Supabase"
+            # Add table_name and query_name
+            extra_field = TemplateField(
+                name="table_name",
+                field_type="str",
+                required=False,
+                placeholder="",
+                show=True,
+                advanced=True,
+                multiline=False,
+                value="",
+            )
+            extra_field2 = TemplateField(
+                name="query_name",
+                field_type="str",
+                required=False,
+                placeholder="",
+                show=True,
+                advanced=True,
+                multiline=False,
+                value="",
+            )
+            # Add supabase_url and supabase_service_key
+            extra_field3 = TemplateField(
+                name="supabase_url",
+                field_type="str",
+                required=False,
+                placeholder="",
+                show=True,
+                advanced=True,
+                multiline=False,
+                value="",
+            )
+            extra_field4 = TemplateField(
+                name="supabase_service_key",
+                field_type="str",
+                required=False,
+                placeholder="",
+                show=True,
+                advanced=True,
+                multiline=False,
+                value="",
+            )
+            extra_fields.extend((extra_field, extra_field2, extra_field3, extra_field4))
+
+        elif self.template.type_name == "MongoDBAtlasVectorSearch":
+            self.display_name = "MongoDB Atlas"
+
+            extra_field = TemplateField(
+                name="mongodb_atlas_cluster_uri",
+                field_type="str",
+                required=False,
+                placeholder="",
+                show=True,
+                advanced=True,
+                multiline=False,
+                display_name="MongoDB Atlas Cluster URI",
+                value="",
+            )
+            extra_field2 = TemplateField(
+                name="collection_name",
+                field_type="str",
+                required=False,
+                placeholder="",
+                show=True,
+                advanced=True,
+                multiline=False,
+                display_name="Collection Name",
+                value="",
+            )
+            extra_field3 = TemplateField(
+                name="db_name",
+                field_type="str",
+                required=False,
+                placeholder="",
+                show=True,
+                advanced=True,
+                multiline=False,
+                display_name="Database Name",
+                value="",
+            )
+            extra_field4 = TemplateField(
+                name="index_name",
+                field_type="str",
+                required=False,
+                placeholder="",
+                show=True,
+                advanced=True,
+                multiline=False,
+                display_name="Index Name",
+                value="",
+            )
+            extra_fields.extend((extra_field, extra_field2, extra_field3, extra_field4))
+
+        if extra_fields:
+            for field in extra_fields:
+                self.template.add_field(field)
 
     def add_extra_base_classes(self) -> None:
-        self.base_classes.append("BaseRetriever")
+        self.base_classes.extend(("BaseRetriever", "VectorStoreRetriever"))
 
     @staticmethod
     def format_field(field: TemplateField, name: Optional[str] = None) -> None:
         FrontendNode.format_field(field, name)
         # Define common field attributes
-        basic_fields = ["work_dir", "collection_name", "api_key", "location"]
+        basic_fields = [
+            "work_dir",
+            "collection_name",
+            "api_key",
+            "location",
+            "persist_directory",
+            "persist",
+            "weaviate_url",
+            "index_name",
+            "namespace",
+            "folder_path",
+            "table_name",
+            "query_name",
+            "supabase_url",
+            "supabase_service_key",
+            "mongodb_atlas_cluster_uri",
+            "collection_name",
+            "db_name",
+        ]
         advanced_fields = [
             "n_dim",
             "key",
@@ -43,14 +240,21 @@ class VectorStoreFrontendNode(FrontendNode):
             "https",
             "prefer_grpc",
             "grpc_port",
+            "pinecone_api_key",
+            "pinecone_env",
+            "client_kwargs",
+            "search_kwargs",
         ]
 
         # Check and set field attributes
         if field.name == "texts":
+            # if field.name is "texts" it has to be replaced
+            # when instantiating the vectorstores
             field.name = "documents"
+
             field.field_type = "TextSplitter"
-            field.display_name = "Text Splitter"
-            field.required = True
+            field.display_name = "Documents"
+            field.required = False
             field.show = True
             field.advanced = False
 
@@ -78,5 +282,6 @@ class VectorStoreFrontendNode(FrontendNode):
             field.advanced = True
             if "key" in field.name:
                 field.password = False
-        # TODO: Weaviate requires weaviate_url to be passed as it is not part of
-        # the class or from_texts method. We need the add_extra_fields to fix this
+
+        elif field.name == "text_key":
+            field.show = False

src/backend/langflow/template/template/base.py

@@ -3,6 +3,7 @@ from typing import Callable, Optional, Union
 from pydantic import BaseModel
 
 from langflow.template.field.base import TemplateField
+from langflow.utils.constants import DIRECT_TYPES
 
 
 class Template(BaseModel):
@@ -18,8 +19,15 @@ class Template(BaseModel):
             for field in self.fields:
                 format_field_func(field, name)
 
+    def sort_fields(self):
+        # first sort alphabetically
+        # then sort fields so that fields that have .field_type in DIRECT_TYPES are first
+        self.fields.sort(key=lambda x: x.name)
+        self.fields.sort(key=lambda x: x.field_type in DIRECT_TYPES, reverse=False)
+
     def to_dict(self, format_field_func=None):
         self.process_fields(self.type_name, format_field_func)
+        self.sort_fields()
         result = {field.name: field.to_dict() for field in self.fields}
         result["_type"] = self.type_name  # type: ignore
         return result

src/backend/langflow/utils/constants.py

@@ -36,3 +36,4 @@ def python_function(text: str) -> str:
     \"\"\"This is a default python function that returns the input text\"\"\"
     return text
 """
+DIRECT_TYPES = ["str", "bool", "code", "int", "float", "Any", "prompt"]

src/backend/langflow/utils/util.py

@@ -299,12 +299,15 @@ def format_dict(d, name: Optional[str] = None):
         if name == "OpenAI" and key == "model_name":
             value["options"] = constants.OPENAI_MODELS
             value["list"] = True
+            value["value"] = constants.OPENAI_MODELS[0]
         elif name == "ChatOpenAI" and key == "model_name":
             value["options"] = constants.CHAT_OPENAI_MODELS
             value["list"] = True
+            value["value"] = constants.CHAT_OPENAI_MODELS[0]
         elif (name == "Anthropic" or name == "ChatAnthropic") and key == "model_name":
             value["options"] = constants.ANTHROPIC_MODELS
             value["list"] = True
+            value["value"] = constants.ANTHROPIC_MODELS[0]
     return d