merging branch release into dev

2023-06-30 13:49:49 -03:00 · 2023-06-30 13:49:49 -03:00 · 168a1b9e0f
commit 168a1b9e0f
parent 683c69340b
75 changed files with 2958 additions and 850 deletions
--- a/poetry.lock
+++ b/poetry.lock
--- a/pyproject.toml
+++ b/pyproject.toml
@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langflow"
-version = "0.2.2"
+version = "0.2.7"
 description = "A Python package with a built-in web application"
 authors = ["Logspace <contact@logspace.ai>"]
 maintainers = [
@ -25,22 +25,21 @@ langflow = "langflow.__main__:main"
 python = ">=3.9,<3.12"
 fastapi = "^0.98.0"
 uvicorn = "^0.22.0"
-beautifulsoup4 = "^4.11.2"
+beautifulsoup4 = "^4.12.2"
 google-search-results = "^2.4.1"
 google-api-python-client = "^2.79.0"
 typer = "^0.9.0"
 gunicorn = "^20.1.0"
-langchain = "^0.0.215"
+langchain = "^0.0.219"
 openai = "^0.27.8"
-types-pyyaml = "^6.0.12.8"
-pandas = "^1.5.3"
+pandas = "^2.0.0"
 chromadb = "^0.3.21"
-huggingface-hub = "^0.13.3"
+huggingface-hub = "^0.15.0"
 rich = "^13.4.2"
 llama-cpp-python = "~0.1.0"
 networkx = "^3.1"
-unstructured = "^0.5.11"
-pypdf = "^3.7.1"
+unstructured = "^0.7.0"
+pypdf = "^3.11.0"
 lxml = "^4.9.2"
 pysrt = "^1.1.2"
 fake-useragent = "^1.1.3"
@ -49,18 +48,18 @@ psycopg2-binary = "^2.9.6"
 pyarrow = "^12.0.0"
 tiktoken = "~0.4.0"
 wikipedia = "^1.4.0"
-langchain-serve = { version = ">0.0.39", optional = true }
-qdrant-client = "^1.2.0"
+langchain-serve = { version = ">0.0.47", optional = true }
+qdrant-client = "^1.3.0"
 websockets = "^10.3"
 weaviate-client = "^3.21.0"
 jina = "3.15.2"
 sentence-transformers = "^2.2.2"
-ctransformers = "^0.2.2"
-cohere = "^4.6.0"
+ctransformers = "^0.2.10"
+cohere = "^4.11.0"
 python-multipart = "^0.0.6"
 sqlmodel = "^0.0.8"
 faiss-cpu = "^1.7.4"
-anthropic = "^0.2.10"
+anthropic = "^0.3.0"
 orjson = "^3.9.1"
 multiprocess = "^0.70.14"
 cachetools = "^5.3.1"
@ -70,7 +69,8 @@ pinecone-client = "^2.2.2"
 supabase = "^1.0.3"
 pymongo = "^4.4.0"
 certifi = "^2023.5.7"
-
+psycopg = "^3.1.9"
+psycopg-binary = "^3.1.9"

 [tool.poetry.dev-dependencies]
 black = "^23.1.0"
@ -85,6 +85,7 @@ pytest-cov = "^4.0.0"
 pandas-stubs = "^2.0.0.230412"
 types-pillow = "^9.5.0.2"
 types-appdirs = "^1.4.3.5"
+types-pyyaml = "^6.0.12.8"


 [tool.poetry.extras]
--- a/src/backend/langflow/main.py
+++ b/src/backend/langflow/main.py
@ -1,6 +1,5 @@
 import sys
 import time
-from fastapi import FastAPI
 import httpx
 from multiprocess import Process, cpu_count  # type: ignore
 import platform
@ -11,9 +10,7 @@ from rich.panel import Panel
 from rich import box
 from rich import print as rprint
 import typer
-from fastapi.staticfiles import StaticFiles
-from fastapi.responses import FileResponse
-from langflow.main import create_app
+from langflow.main import setup_app
 from langflow.settings import settings
 from langflow.utils.logger import configure, logger
 import webbrowser
@ -144,15 +141,9 @@ def serve(
        remove_api_keys=remove_api_keys,
        cache=cache,
    )
-    # get the directory of the current file
-    if not path:
-        frontend_path = Path(__file__).parent
-        static_files_dir = frontend_path / "frontend"
-    else:
-        static_files_dir = Path(path)
-
-    app = create_app()
-    setup_static_files(app, static_files_dir)
+    # create path object if path is provided
+    static_files_dir: Optional[Path] = Path(path) if path else None
+    app = setup_app(static_files_dir=static_files_dir)
    # check if port is being used
    if is_port_in_use(port, host):
        port = get_free_port(port)
@ -200,29 +191,6 @@ def run_on_windows(host, port, log_level, options, app):
    run_langflow(host, port, log_level, options, app)


-def setup_static_files(app: FastAPI, static_files_dir: Path):
-    """
-    Setup the static files directory.
-
-    Args:
-        app (FastAPI): FastAPI app.
-        path (str): Path to the static files directory.
-    """
-    app.mount(
-        "/",
-        StaticFiles(directory=static_files_dir, html=True),
-        name="static",
-    )
-
-    @app.exception_handler(404)
-    async def custom_404_handler(request, __):
-        path = static_files_dir / "index.html"
-
-        if not path.exists():
-            raise RuntimeError(f"File at path {path} does not exist.")
-        return FileResponse(path)
-
-
 def is_port_in_use(port, host="localhost"):
    """
    Check if a port is in use.
--- a/src/backend/langflow/api/v1/chat.py
+++ b/src/backend/langflow/api/v1/chat.py
@ -1,12 +1,6 @@
-from fastapi import (
-    APIRouter,
-    HTTPException,
-    WebSocket,
-    WebSocketException,
-    status,
-)
+from fastapi import APIRouter, HTTPException, WebSocket, WebSocketException, status
 from fastapi.responses import StreamingResponse
-from langflow.api.v1.schemas import BuiltResponse, InitResponse, StreamData
+from langflow.api.v1.schemas import BuildStatus, BuiltResponse, InitResponse, StreamData

 from langflow.chat.manager import ChatManager
 from langflow.graph.graph.base import Graph
@ -32,15 +26,29 @@ async def chat(client_id: str, websocket: WebSocket):
        await websocket.close(code=status.WS_1011_INTERNAL_ERROR, reason=str(exc))


-@router.post("/build/init", response_model=InitResponse, status_code=201)
-async def init_build(graph_data: dict):
+@router.post("/build/init/{flow_id}", response_model=InitResponse, status_code=201)
+async def init_build(graph_data: dict, flow_id: str):
    """Initialize the build by storing graph data and returning a unique session ID."""

    try:
-        flow_id = graph_data.get("id")
        if flow_id is None:
            raise ValueError("No ID provided")
-        flow_data_store[flow_id] = graph_data
+        # Check if already building
+        if (
+            flow_id in flow_data_store
+            and flow_data_store[flow_id]["status"] == BuildStatus.IN_PROGRESS
+        ):
+            return InitResponse(flowId=flow_id)
+
+        # Delete from cache if already exists
+        if flow_id in chat_manager.in_memory_cache:
+            with chat_manager.in_memory_cache._lock:
+                chat_manager.in_memory_cache.delete(flow_id)
+                logger.debug(f"Deleted flow {flow_id} from cache")
+        flow_data_store[flow_id] = {
+            "graph_data": graph_data,
+            "status": BuildStatus.STARTED,
+        }

        return InitResponse(flowId=flow_id)
    except Exception as exc:
@ -52,8 +60,9 @@ async def init_build(graph_data: dict):
 async def build_status(flow_id: str):
    """Check the flow_id is in the flow_data_store."""
    try:
-        built = flow_id in flow_data_store and not isinstance(
-            flow_data_store[flow_id], dict
+        built = (
+            flow_id in flow_data_store
+            and flow_data_store[flow_id]["status"] == BuildStatus.SUCCESS
        )

        return BuiltResponse(
@ -77,7 +86,12 @@ async def stream_build(flow_id: str):
                yield str(StreamData(event="error", data={"error": error_message}))
                return

-            graph_data = flow_data_store[flow_id].get("data")
+            if flow_data_store[flow_id].get("status") == BuildStatus.IN_PROGRESS:
+                error_message = "Already building"
+                yield str(StreamData(event="error", data={"error": error_message}))
+                return
+
+            graph_data = flow_data_store[flow_id].get("graph_data")

            if not graph_data:
                error_message = "No data provided"
@ -95,6 +109,7 @@ async def stream_build(flow_id: str):
                return

            number_of_nodes = len(graph.nodes)
+            flow_data_store[flow_id]["status"] = BuildStatus.IN_PROGRESS
            for i, vertex in enumerate(graph.generator_build(), 1):
                try:
                    log_dict = {
@ -110,6 +125,7 @@ async def stream_build(flow_id: str):
                except Exception as exc:
                    params = str(exc)
                    valid = False
+                    flow_data_store[flow_id]["status"] = BuildStatus.FAILURE

                response = {
                    "valid": valid,
@ -121,8 +137,10 @@ async def stream_build(flow_id: str):
                yield str(StreamData(event="message", data=response))

            chat_manager.set_cache(flow_id, graph.build())
+            flow_data_store[flow_id]["status"] = BuildStatus.SUCCESS
        except Exception as exc:
            logger.error("Error while building the flow: %s", exc)
+            flow_data_store[flow_id]["status"] = BuildStatus.FAILURE
            yield str(StreamData(event="error", data={"error": str(exc)}))
        finally:
            yield str(StreamData(event="message", data=final_response))
--- a/src/backend/langflow/api/v1/endpoints.py
+++ b/src/backend/langflow/api/v1/endpoints.py
@ -11,7 +11,7 @@ from langflow.api.v1.schemas import (
    UploadFileResponse,
 )

-from langflow.interface.types import build_langchain_types_dict
+from langflow.interface.types import langchain_types_dict
 from langflow.database.base import get_session
 from sqlmodel import Session

@ -21,7 +21,7 @@ router = APIRouter(tags=["Base"])

@router.get("/all")
 def get_all():
-    return build_langchain_types_dict()
+    return langchain_types_dict


 # For backwards compatibility we will keep the old endpoint
--- a/src/backend/langflow/api/v1/schemas.py
+++ b/src/backend/langflow/api/v1/schemas.py
@ -1,3 +1,4 @@
+from enum import Enum
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 from langflow.database.models.flow import FlowCreate, FlowRead
@ -5,6 +6,15 @@ from pydantic import BaseModel, Field, validator
 import json


+class BuildStatus(Enum):
+    """Status of the build."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    STARTED = "started"
+    IN_PROGRESS = "in_progress"
+
+
 class GraphData(BaseModel):
    """Data inside the exported flow."""

--- a/src/backend/langflow/chat/config.py
+++ b/src/backend/langflow/chat/config.py
@ -0,0 +1,2 @@
+class ChatConfig:
+    streaming: bool = True
--- a/src/backend/langflow/config.yaml
+++ b/src/backend/langflow/config.yaml
@ -1,141 +1,264 @@
---
 agents:
-  - ZeroShotAgent
-  - JsonAgent
-  - CSVAgent
-  - AgentInitializer
-  - VectorStoreAgent
-  - VectorStoreRouterAgent
-  - SQLAgent
+  ZeroShotAgent:
+    documentation: "https://python.langchain.com/docs/modules/agents/how_to/custom_mrkl_agent"
+  JsonAgent:
+    documentation: "https://python.langchain.com/docs/modules/agents/toolkits/openapi"
+  CSVAgent:
+    documentation: "https://python.langchain.com/docs/modules/agents/toolkits/csv"
+  AgentInitializer:
+    documentation: "https://python.langchain.com/docs/modules/agents/agent_types/"
+  VectorStoreAgent:
+    documentation: ""
+  VectorStoreRouterAgent:
+    documentation: ""
+  SQLAgent:
+    documentation: ""
 chains:
-  - LLMChain
-  - LLMMathChain
-  - LLMCheckerChain
-  - ConversationChain
-  - SeriesCharacterChain
-  - MidJourneyPromptChain
-  - TimeTravelGuideChain
-  - SQLDatabaseChain
-  - RetrievalQA
-  - RetrievalQAWithSourcesChain
-  - ConversationalRetrievalChain
-  - CombineDocsChain
+  LLMChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/foundational/llm_chain"
+  LLMMathChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/additional/llm_math"
+  LLMCheckerChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/additional/llm_checker"
+  ConversationChain:
+    documentation: ""
+  SeriesCharacterChain:
+    documentation: ""
+  MidJourneyPromptChain:
+    documentation: ""
+  TimeTravelGuideChain:
+    documentation: ""
+  SQLDatabaseChain:
+    documentation: ""
+  RetrievalQA:
+    documentation: "https://python.langchain.com/docs/modules/chains/popular/vector_db_qa"
+  RetrievalQAWithSourcesChain:
+    documentation: ""
+  ConversationalRetrievalChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/popular/chat_vector_db"
+  CombineDocsChain:
+    documentation: ""
 documentloaders:
-  - AirbyteJSONLoader
-  - CoNLLULoader
-  - CSVLoader
-  - UnstructuredEmailLoader
-  - EverNoteLoader
-  - FacebookChatLoader
-  - GutenbergLoader
-  - BSHTMLLoader
-  - UnstructuredHTMLLoader
-  # - UnstructuredImageLoader # Issue with Python 3.11 (https://github.com/Unstructured-IO/unstructured-inference/issues/83)
-  - UnstructuredMarkdownLoader
-  - PyPDFLoader
-  - UnstructuredPowerPointLoader
-  - SRTLoader
-  - TelegramChatLoader
-  - TextLoader
-  - UnstructuredWordDocumentLoader
-  - WebBaseLoader
-  - AZLyricsLoader
-  - CollegeConfidentialLoader
-  - HNLoader
-  - IFixitLoader
-  - IMSDbLoader
-  - GitbookLoader
-  - ReadTheDocsLoader
-  - SlackDirectoryLoader
-  - NotionDirectoryLoader
-  - DirectoryLoader
-  - GitLoader
+  AirbyteJSONLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/airbyte_json"
+  CoNLLULoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/conll-u"
+  CSVLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/csv"
+  UnstructuredEmailLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/email"
+  EverNoteLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/evernote"
+  FacebookChatLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/facebook_chat"
+  GutenbergLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/gutenberg"
+  BSHTMLLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/html"
+  UnstructuredHTMLLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/html"
+  UnstructuredMarkdownLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/markdown"
+  PyPDFLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/pdf"
+  UnstructuredPowerPointLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/microsoft_powerpoint"
+  SRTLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/subtitle"
+  TelegramChatLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/telegram"
+  TextLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/"
+  UnstructuredWordDocumentLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/microsoft_word"
+  WebBaseLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/web_base"
+  AZLyricsLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/azlyrics"
+  CollegeConfidentialLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/college_confidential"
+  HNLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/hacker_news"
+  IFixitLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/ifixit"
+  IMSDbLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/imsdb"
+  GitbookLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/gitbook"
+  ReadTheDocsLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/readthedocs_documentation"
+  SlackDirectoryLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/slack"
+  NotionDirectoryLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/notion"
+  DirectoryLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/file_directory"
+  GitLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/git"
 embeddings:
-  - OpenAIEmbeddings
-  - HuggingFaceEmbeddings
-  - CohereEmbeddings
+  OpenAIEmbeddings:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/openai"
+  HuggingFaceEmbeddings:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/sentence_transformers"
+  CohereEmbeddings:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/cohere"
 llms:
-  - OpenAI
-  # - AzureOpenAI
-  # - AzureChatOpenAI
-  - ChatOpenAI
-  - LlamaCpp
-  - CTransformers
-  - Cohere
-  - Anthropic
-  - ChatAnthropic
-  - HuggingFaceHub
+  OpenAI:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/openai"
+  ChatOpenAI:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/integrations/openai"
+  LlamaCpp:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/llamacpp"
+  CTransformers:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/ctransformers"
+  Cohere:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/cohere"
+  Anthropic:
+    documentation: ""
+  ChatAnthropic:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/integrations/anthropic"
+  HuggingFaceHub:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/huggingface_hub"
 memories:
-  - ConversationBufferMemory
-  - ConversationSummaryMemory
-  - ConversationKGMemory
+  # https://github.com/supabase-community/supabase-py/issues/482
+  # ZepChatMessageHistory:
+  #   documentation: "https://python.langchain.com/docs/modules/memory/integrations/zep_memory"
+  ConversationEntityMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/integrations/entity_memory_with_sqlite"
+  # https://github.com/hwchase17/langchain/issues/6091
+  # SQLiteEntityStore:
+  #   documentation: "https://python.langchain.com/docs/modules/memory/integrations/entity_memory_with_sqlite"
+  PostgresChatMessageHistory:
+    documentation: "https://python.langchain.com/docs/modules/memory/integrations/postgres_chat_message_history"
+  ConversationBufferMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/buffer"
+  ConversationSummaryMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/summary"
+  ConversationKGMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/kg"
+  ConversationBufferWindowMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/buffer_window"
+  VectorStoreRetrieverMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/vectorstore_retriever_memory"
+
 prompts:
-  - PromptTemplate
-  - FewShotPromptTemplate
-  - ZeroShotPrompt
+  PromptTemplate:
+    documentation: "https://python.langchain.com/docs/modules/model_io/prompts/prompt_templates/"
+  ZeroShotPrompt:
+    documentation: "https://python.langchain.com/docs/modules/agents/how_to/custom_mrkl_agent"
 textsplitters:
-  - CharacterTextSplitter
-  - RecursiveCharacterTextSplitter
-  # - LatexTextSplitter
-  # - PythonCodeTextSplitter
+  CharacterTextSplitter:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_transformers/text_splitters/character_text_splitter"
+  RecursiveCharacterTextSplitter:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_transformers/text_splitters/recursive_text_splitter"
 toolkits:
-  - OpenAPIToolkit
-  - JsonToolkit
-  - VectorStoreInfo
-  - VectorStoreRouterToolkit
-  - VectorStoreToolkit
+  OpenAPIToolkit:
+    documentation: ""
+  JsonToolkit:
+    documentation: ""
+  VectorStoreInfo:
+    documentation: ""
+  VectorStoreRouterToolkit:
+    documentation: ""
+  VectorStoreToolkit:
+    documentation: ""
 tools:
-  - Search
-  - PAL-MATH
-  - Calculator
-  - Serper Search
-  - Tool
-  - PythonFunctionTool
-  - PythonFunction
-  - JsonSpec
-  - News API
-  - TMDB API
-  - Podcast API
-  - QuerySQLDataBaseTool
-  - InfoSQLDatabaseTool
-  - ListSQLDatabaseTool
-  # - QueryCheckerTool
-  - BingSearchRun
-  - GoogleSearchRun
-  - GoogleSearchResults
-  - GoogleSerperRun
-  - JsonListKeysTool
-  - JsonGetValueTool
-  - PythonREPLTool
-  - PythonAstREPLTool
-  - RequestsGetTool
-  - RequestsPostTool
-  - RequestsPatchTool
-  - RequestsPutTool
-  - RequestsDeleteTool
-  - WikipediaQueryRun
-  - WolframAlphaQueryRun
+  Search:
+    documentation: ""
+  PAL-MATH:
+    documentation: ""
+  Calculator:
+    documentation: ""
+  Serper Search:
+    documentation: ""
+  Tool:
+    documentation: ""
+  PythonFunctionTool:
+    documentation: ""
+  PythonFunction:
+    documentation: ""
+  JsonSpec:
+    documentation: ""
+  News API:
+    documentation: ""
+  TMDB API:
+    documentation: ""
+  Podcast API:
+    documentation: ""
+  QuerySQLDataBaseTool:
+    documentation: ""
+  InfoSQLDatabaseTool:
+    documentation: ""
+  ListSQLDatabaseTool:
+    documentation: ""
+  BingSearchRun:
+    documentation: ""
+  GoogleSearchRun:
+    documentation: ""
+  GoogleSearchResults:
+    documentation: ""
+  GoogleSerperRun:
+    documentation: ""
+  JsonListKeysTool:
+    documentation: ""
+  JsonGetValueTool:
+    documentation: ""
+  PythonREPLTool:
+    documentation: ""
+  PythonAstREPLTool:
+    documentation: ""
+  RequestsGetTool:
+    documentation: ""
+  RequestsPostTool:
+    documentation: ""
+  RequestsPatchTool:
+    documentation: ""
+  RequestsPutTool:
+    documentation: ""
+  RequestsDeleteTool:
+    documentation: ""
+  WikipediaQueryRun:
+    documentation: ""
+  WolframAlphaQueryRun:
+    documentation: ""
 utilities:
-  - BingSearchAPIWrapper
-  - GoogleSearchAPIWrapper
-  - GoogleSerperAPIWrapper
-  - SearxResults
-  - SearxSearchWrapper
-  - SerpAPIWrapper
-  - WikipediaAPIWrapper
-  - WolframAlphaAPIWrapper
-  # - ZapierNLAWrapper
-  - SQLDatabase
+  BingSearchAPIWrapper:
+    documentation: ""
+  GoogleSearchAPIWrapper:
+    documentation: ""
+  GoogleSerperAPIWrapper:
+    documentation: ""
+  SearxResults:
+    documentation: ""
+  SearxSearchWrapper:
+    documentation: ""
+  SerpAPIWrapper:
+    documentation: ""
+  WikipediaAPIWrapper:
+    documentation: ""
+  WolframAlphaAPIWrapper:
+    documentation: ""
+retrievers:
+  MultiQueryRetriever:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/retrievers/how_to/MultiQueryRetriever"
+  # https://github.com/supabase-community/supabase-py/issues/482
+  # ZepRetriever:
+  #   documentation: "https://python.langchain.com/docs/modules/data_connection/retrievers/integrations/zep_memorystore"
 vectorstores:
-  - Chroma
-  - Qdrant
-  - Weaviate
-  - FAISS
-  - Pinecone
-  - SupabaseVectorStore
-  - MongoDBAtlasVectorSearch
+  Chroma:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/chroma"
+  Qdrant:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/qdrant"
+  Weaviate:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/weaviate"
+  FAISS:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/faiss"
+  Pinecone:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/pinecone"
+  SupabaseVectorStore:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/supabase"
+  MongoDBAtlasVectorSearch:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/mongodb_atlas_vector_search"
 wrappers:
-  - RequestsWrapper
-  # - ChatPromptTemplate
-  # - SystemMessagePromptTemplate
-  # - HumanMessagePromptTemplate
+  RequestsWrapper:
+    documentation: ""
--- a/src/backend/langflow/custom/customs.py
+++ b/src/backend/langflow/custom/customs.py
@ -21,6 +21,9 @@ CUSTOM_NODES = {
    "utilities": {
        "SQLDatabase": frontend_node.agents.SQLDatabaseNode(),
    },
+    "memories": {
+        "PostgresChatMessageHistory": frontend_node.memories.PostgresChatMessageHistoryFrontendNode(),
+    },
    "chains": {
        "SeriesCharacterChain": frontend_node.chains.SeriesCharacterChainNode(),
        "TimeTravelGuideChain": frontend_node.chains.TimeTravelGuideChainNode(),
--- a/src/backend/langflow/graph/init.py
+++ b/src/backend/langflow/graph/init.py
@ -14,6 +14,7 @@ from langflow.graph.vertex.types import (
    ToolkitVertex,
    VectorStoreVertex,
    WrapperVertex,
+    RetrieverVertex,
 )

 __all__ = [
@ -32,4 +33,5 @@ __all__ = [
    "ToolkitVertex",
    "VectorStoreVertex",
    "WrapperVertex",
+    "RetrieverVertex",
 ]
--- a/src/backend/langflow/graph/graph/constants.py
+++ b/src/backend/langflow/graph/graph/constants.py
@ -1,18 +1,5 @@
 from langflow.graph.vertex.base import Vertex
-from langflow.graph.vertex.types import (
-    AgentVertex,
-    ChainVertex,
-    DocumentLoaderVertex,
-    EmbeddingVertex,
-    LLMVertex,
-    MemoryVertex,
-    PromptVertex,
-    TextSplitterVertex,
-    ToolVertex,
-    ToolkitVertex,
-    VectorStoreVertex,
-    WrapperVertex,
-)
+from langflow.graph.vertex import types
 from langflow.interface.agents.base import agent_creator
 from langflow.interface.chains.base import chain_creator
 from langflow.interface.document_loaders.base import documentloader_creator
@ -25,22 +12,23 @@ from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.tools.base import tool_creator
 from langflow.interface.vector_store.base import vectorstore_creator
 from langflow.interface.wrappers.base import wrapper_creator
-
+from langflow.interface.retrievers.base import retriever_creator

 from typing import Dict, Type


 VERTEX_TYPE_MAP: Dict[str, Type[Vertex]] = {
-    **{t: PromptVertex for t in prompt_creator.to_list()},
-    **{t: AgentVertex for t in agent_creator.to_list()},
-    **{t: ChainVertex for t in chain_creator.to_list()},
-    **{t: ToolVertex for t in tool_creator.to_list()},
-    **{t: ToolkitVertex for t in toolkits_creator.to_list()},
-    **{t: WrapperVertex for t in wrapper_creator.to_list()},
-    **{t: LLMVertex for t in llm_creator.to_list()},
-    **{t: MemoryVertex for t in memory_creator.to_list()},
-    **{t: EmbeddingVertex for t in embedding_creator.to_list()},
-    **{t: VectorStoreVertex for t in vectorstore_creator.to_list()},
-    **{t: DocumentLoaderVertex for t in documentloader_creator.to_list()},
-    **{t: TextSplitterVertex for t in textsplitter_creator.to_list()},
+    **{t: types.PromptVertex for t in prompt_creator.to_list()},
+    **{t: types.AgentVertex for t in agent_creator.to_list()},
+    **{t: types.ChainVertex for t in chain_creator.to_list()},
+    **{t: types.ToolVertex for t in tool_creator.to_list()},
+    **{t: types.ToolkitVertex for t in toolkits_creator.to_list()},
+    **{t: types.WrapperVertex for t in wrapper_creator.to_list()},
+    **{t: types.LLMVertex for t in llm_creator.to_list()},
+    **{t: types.MemoryVertex for t in memory_creator.to_list()},
+    **{t: types.EmbeddingVertex for t in embedding_creator.to_list()},
+    **{t: types.VectorStoreVertex for t in vectorstore_creator.to_list()},
+    **{t: types.DocumentLoaderVertex for t in documentloader_creator.to_list()},
+    **{t: types.TextSplitterVertex for t in textsplitter_creator.to_list()},
+    **{t: types.RetrieverVertex for t in retriever_creator.to_list()},
 }
--- a/src/backend/langflow/graph/vertex/types.py
+++ b/src/backend/langflow/graph/vertex/types.py
@ -112,6 +112,11 @@ class MemoryVertex(Vertex):
        super().__init__(data, base_type="memory")


+class RetrieverVertex(Vertex):
+    def __init__(self, data: Dict):
+        super().__init__(data, base_type="retrievers")
+
+
 class TextSplitterVertex(Vertex):
    def __init__(self, data: Dict):
        super().__init__(data, base_type="textsplitters")
--- a/src/backend/langflow/interface/base.py
+++ b/src/backend/langflow/interface/base.py
@ -8,6 +8,7 @@ from langflow.template.field.base import TemplateField
 from langflow.template.frontend_node.base import FrontendNode
 from langflow.template.template.base import Template
 from langflow.utils.logger import logger
+from langflow.settings import settings

 # Assuming necessary imports for Field, Template, and FrontendNode classes

@ -15,12 +16,29 @@ from langflow.utils.logger import logger
 class LangChainTypeCreator(BaseModel, ABC):
    type_name: str
    type_dict: Optional[Dict] = None
+    name_docs_dict: Optional[Dict[str, str]] = None

    @property
    def frontend_node_class(self) -> Type[FrontendNode]:
        """The class type of the FrontendNode created in frontend_node."""
        return FrontendNode

+    @property
+    def docs_map(self) -> Dict[str, str]:
+        """A dict with the name of the component as key and the documentation link as value."""
+        if self.name_docs_dict is None:
+            try:
+                type_settings = getattr(settings, self.type_name)
+                self.name_docs_dict = {
+                    name: value_dict["documentation"]
+                    for name, value_dict in type_settings.items()
+                }
+            except AttributeError as exc:
+                logger.error(exc)
+
+                self.name_docs_dict = {}
+        return self.name_docs_dict
+
    @property
    @abstractmethod
    def type_to_loader_dict(self) -> Dict:
@ -83,7 +101,7 @@ class LangChainTypeCreator(BaseModel, ABC):

        signature.add_extra_fields()
        signature.add_extra_base_classes()
-
+        signature.set_documentation(self.docs_map.get(name, ""))
        return signature


--- a/src/backend/langflow/interface/importing/utils.py
+++ b/src/backend/langflow/interface/importing/utils.py
@ -44,6 +44,7 @@ def import_by_type(_type: str, name: str) -> Any:
        "documentloaders": import_documentloader,
        "textsplitters": import_textsplitter,
        "utilities": import_utility,
+        "retrievers": import_retriever,
    }
    if _type == "llms":
        key = "chat" if "chat" in name.lower() else "llm"
@ -59,6 +60,11 @@ def import_chat_llm(llm: str) -> BaseChatModel:
    return import_class(f"langchain.chat_models.{llm}")


+def import_retriever(retriever: str) -> Any:
+    """Import retriever from retriever name"""
+    return import_module(f"from langchain.retrievers import {retriever}")
+
+
 def import_memory(memory: str) -> Any:
    """Import memory from memory name"""
    return import_module(f"from langchain.memory import {memory}")
--- a/src/backend/langflow/interface/initialize/loading.py
+++ b/src/backend/langflow/interface/initialize/loading.py
@ -1,11 +1,12 @@
 import json
-from typing import Any, Callable, Dict, Sequence
+from typing import Any, Callable, Dict, Sequence, Type

 from langchain.agents import ZeroShotAgent
 from langchain.agents import agent as agent_module
 from langchain.agents.agent import AgentExecutor
 from langchain.agents.agent_toolkits.base import BaseToolkit
 from langchain.agents.tools import BaseTool
+
 from langflow.interface.initialize.vector_store import vecstore_initializer

 from pydantic import ValidationError
@ -14,8 +15,13 @@ from langflow.interface.custom_lists import CUSTOM_NODES
 from langflow.interface.importing.utils import get_function, import_by_type
 from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.chains.base import chain_creator
+from langflow.interface.retrievers.base import retriever_creator
 from langflow.interface.utils import load_file_into_dict
 from langflow.utils import validate
+from langchain.chains.base import Chain
+from langchain.vectorstores.base import VectorStore
+from langchain.document_loaders.base import BaseLoader
+from langchain.prompts.base import BasePromptTemplate


 def instantiate_class(node_type: str, base_type: str, params: Dict) -> Any:
@ -43,8 +49,8 @@ def convert_params_to_sets(params):

 def convert_kwargs(params):
    # if *kwargs are passed as a string, convert to dict
-    # first find any key that has kwargs in it
-    kwargs_keys = [key for key in params.keys() if "kwargs" in key]
+    # first find any key that has kwargs or config in it
+    kwargs_keys = [key for key in params.keys() if "kwargs" in key or "config" in key]
    for key in kwargs_keys:
        if isinstance(params[key], str):
            params[key] = json.loads(params[key])
@ -72,11 +78,50 @@ def instantiate_based_on_type(class_object, base_type, node_type, params):
        return instantiate_utility(node_type, class_object, params)
    elif base_type == "chains":
        return instantiate_chains(node_type, class_object, params)
+    elif base_type == "llms":
+        return instantiate_llm(node_type, class_object, params)
+    elif base_type == "retrievers":
+        return instantiate_retriever(node_type, class_object, params)
+    elif base_type == "memory":
+        return instantiate_memory(node_type, class_object, params)
    else:
        return class_object(**params)


-def instantiate_chains(node_type, class_object, params):
+def instantiate_memory(node_type, class_object, params):
+    try:
+        return class_object(**params)
+    # I want to catch a specific attribute error that happens
+    # when the object does not have a cursor attribute
+    except Exception as exc:
+        if "object has no attribute 'cursor'" in str(
+            exc
+        ) or 'object has no field "conn"' in str(exc):
+            raise AttributeError(
+                (
+                    "Failed to build connection to database."
+                    f" Please check your connection string and try again. Error: {exc}"
+                )
+            ) from exc
+        raise exc
+
+
+def instantiate_retriever(node_type, class_object, params):
+    if "retriever" in params and hasattr(params["retriever"], "as_retriever"):
+        params["retriever"] = params["retriever"].as_retriever()
+    if node_type in retriever_creator.from_method_nodes:
+        method = retriever_creator.from_method_nodes[node_type]
+        if class_method := getattr(class_object, method, None):
+            return class_method(**params)
+        raise ValueError(f"Method {method} not found in {class_object}")
+    return class_object(**params)
+
+
+def instantiate_llm(node_type, class_object, params: Dict):
+    return class_object(**params)
+
+
+def instantiate_chains(node_type, class_object: Type[Chain], params: Dict):
    if "retriever" in params and hasattr(params["retriever"], "as_retriever"):
        params["retriever"] = params["retriever"].as_retriever()
    if node_type in chain_creator.from_method_nodes:
@ -88,11 +133,11 @@ def instantiate_chains(node_type, class_object, params):
    return class_object(**params)


-def instantiate_agent(class_object, params):
+def instantiate_agent(class_object: Type[agent_module.Agent], params: Dict):
    return load_agent_executor(class_object, params)


-def instantiate_prompt(node_type, class_object, params):
+def instantiate_prompt(node_type, class_object: Type[BasePromptTemplate], params: Dict):
    if node_type == "ZeroShotPrompt":
        if "tools" not in params:
            params["tools"] = []
@ -100,7 +145,7 @@ def instantiate_prompt(node_type, class_object, params):
    return class_object(**params)


-def instantiate_tool(node_type, class_object, params):
+def instantiate_tool(node_type, class_object: Type[BaseTool], params: Dict):
    if node_type == "JsonSpec":
        params["dict_"] = load_file_into_dict(params.pop("path"))
        return class_object(**params)
@ -118,7 +163,7 @@ def instantiate_tool(node_type, class_object, params):
    return class_object(**params)


-def instantiate_toolkit(node_type, class_object, params):
+def instantiate_toolkit(node_type, class_object: Type[BaseToolkit], params: Dict):
    loaded_toolkit = class_object(**params)
    # Commenting this out for now to use toolkits as normal tools
    # if toolkits_creator.has_create_function(node_type):
@ -128,7 +173,7 @@ def instantiate_toolkit(node_type, class_object, params):
    return loaded_toolkit


-def instantiate_embedding(class_object, params):
+def instantiate_embedding(class_object, params: Dict):
    params.pop("model", None)
    params.pop("headers", None)
    try:
@ -142,7 +187,7 @@ def instantiate_embedding(class_object, params):
        return class_object(**params)


-def instantiate_vectorstore(class_object, params):
+def instantiate_vectorstore(class_object: Type[VectorStore], params: Dict):
    search_kwargs = params.pop("search_kwargs", {})
    if initializer := vecstore_initializer.get(class_object.__name__):
        vecstore = initializer(class_object, params)
@ -158,7 +203,7 @@ def instantiate_vectorstore(class_object, params):
    return vecstore


-def instantiate_documentloader(class_object, params):
+def instantiate_documentloader(class_object: Type[BaseLoader], params: Dict):
    if "file_filter" in params:
        # file_filter will be a string but we need a function
        # that will be used to filter the files using file_filter
@ -171,35 +216,55 @@ def instantiate_documentloader(class_object, params):
            extension.strip() in x for extension in extensions
        )
    metadata = params.pop("metadata", None)
+    if metadata and isinstance(metadata, str):
+        try:
+            metadata = json.loads(metadata)
+        except json.JSONDecodeError as exc:
+            raise ValueError(
+                "The metadata you provided is not a valid JSON string."
+            ) from exc
    docs = class_object(**params).load()
+    # Now if metadata is an empty dict, we will not add it to the documents
    if metadata:
-        if isinstance(metadata, str):
-            try:
-                metadata = json.loads(metadata)
-            except json.JSONDecodeError as exc:
-                raise ValueError(
-                    "The metadata you provided is not a valid JSON string."
-                ) from exc
-
        for doc in docs:
-            doc.metadata = metadata
+            # If the document already has metadata, we will not overwrite it
+            if not doc.metadata:
+                doc.metadata = metadata
+            else:
+                doc.metadata.update(metadata)

    return docs


-def instantiate_textsplitter(class_object, params):
+def instantiate_textsplitter(
+    class_object,
+    params: Dict,
+):
    try:
        documents = params.pop("documents")
-    except KeyError as e:
+    except KeyError as exc:
        raise ValueError(
            "The source you provided did not load correctly or was empty."
            "Try changing the chunk_size of the Text Splitter."
-        ) from e
-    text_splitter = class_object(**params)
+        ) from exc
+
+    if (
+        "separator_type" in params and params["separator_type"] == "Text"
+    ) or "separator_type" not in params:
+        params.pop("separator_type", None)
+        text_splitter = class_object(**params)
+    else:
+        from langchain.text_splitter import Language
+
+        language = params.pop("separator_type", None)
+        params["language"] = Language(language)
+        params.pop("separators", None)
+
+        text_splitter = class_object.from_language(**params)
    return text_splitter.split_documents(documents)


-def instantiate_utility(node_type, class_object, params):
+def instantiate_utility(node_type, class_object, params: Dict):
    if node_type == "SQLDatabase":
        return class_object.from_uri(params.pop("uri"))
    return class_object(**params)
@ -225,6 +290,8 @@ def load_agent_executor(agent_class: type[agent_module.Agent], params, **kwargs)
    """Load agent executor from agent class, tools and chain"""
    allowed_tools: Sequence[BaseTool] = params.get("allowed_tools", [])
    llm_chain = params["llm_chain"]
+    # agent has hidden args for memory. might need to be support
+    # memory = params["memory"]
    # if allowed_tools is not a list or set, make it a list
    if not isinstance(allowed_tools, (list, set)) and isinstance(
        allowed_tools, BaseTool
@ -237,6 +304,7 @@ def load_agent_executor(agent_class: type[agent_module.Agent], params, **kwargs)
    return AgentExecutor.from_agent_and_tools(
        agent=agent,
        tools=allowed_tools,
+        # memory=memory,
        **kwargs,
    )

--- a/src/backend/langflow/interface/listing.py
+++ b/src/backend/langflow/interface/listing.py
@ -11,6 +11,7 @@ from langflow.interface.tools.base import tool_creator
 from langflow.interface.utilities.base import utility_creator
 from langflow.interface.vector_store.base import vectorstore_creator
 from langflow.interface.wrappers.base import wrapper_creator
+from langflow.interface.retrievers.base import retriever_creator


 def get_type_dict():
@ -28,6 +29,7 @@ def get_type_dict():
        "embeddings": embedding_creator.to_list(),
        "textSplitters": textsplitter_creator.to_list(),
        "utilities": utility_creator.to_list(),
+        "retrievers": retriever_creator.to_list(),
    }


--- a/src/backend/langflow/interface/memories/base.py
+++ b/src/backend/langflow/interface/memories/base.py
@ -6,12 +6,18 @@ from langflow.settings import settings
 from langflow.template.frontend_node.base import FrontendNode
 from langflow.template.frontend_node.memories import MemoryFrontendNode
 from langflow.utils.logger import logger
-from langflow.utils.util import build_template_from_class
+from langflow.utils.util import build_template_from_class, build_template_from_method
+from langflow.custom.customs import get_custom_nodes


 class MemoryCreator(LangChainTypeCreator):
    type_name: str = "memories"

+    from_method_nodes = {
+        "ZepChatMessageHistory": "__init__",
+        "SQLiteEntityStore": "__init__",
+    }
+
    @property
    def frontend_node_class(self) -> Type[FrontendNode]:
        """The class type of the FrontendNode created in frontend_node."""
@ -26,6 +32,14 @@ class MemoryCreator(LangChainTypeCreator):
    def get_signature(self, name: str) -> Optional[Dict]:
        """Get the signature of a memory."""
        try:
+            if name in get_custom_nodes(self.type_name).keys():
+                return get_custom_nodes(self.type_name)[name]
+            elif name in self.from_method_nodes:
+                return build_template_from_method(
+                    name,
+                    type_to_cls_dict=memory_type_to_cls_dict,
+                    method_name=self.from_method_nodes[name],
+                )
            return build_template_from_class(name, memory_type_to_cls_dict)
        except ValueError as exc:
            raise ValueError("Memory not found") from exc
--- a/src/backend/langflow/interface/retrievers/init.py
+++ b/src/backend/langflow/interface/retrievers/init.py
--- a/src/backend/langflow/interface/retrievers/base.py
+++ b/src/backend/langflow/interface/retrievers/base.py
@ -0,0 +1,58 @@
+from typing import Any, Dict, List, Optional, Type
+
+from langchain import retrievers
+
+from langflow.interface.base import LangChainTypeCreator
+from langflow.interface.importing.utils import import_class
+from langflow.settings import settings
+from langflow.template.frontend_node.retrievers import RetrieverFrontendNode
+from langflow.utils.logger import logger
+from langflow.utils.util import build_template_from_method, build_template_from_class
+
+
+class RetrieverCreator(LangChainTypeCreator):
+    type_name: str = "retrievers"
+
+    from_method_nodes = {"MultiQueryRetriever": "from_llm", "ZepRetriever": "__init__"}
+
+    @property
+    def frontend_node_class(self) -> Type[RetrieverFrontendNode]:
+        return RetrieverFrontendNode
+
+    @property
+    def type_to_loader_dict(self) -> Dict:
+        if self.type_dict is None:
+            self.type_dict: dict[str, Any] = {
+                retriever_name: import_class(f"langchain.retrievers.{retriever_name}")
+                for retriever_name in retrievers.__all__
+            }
+        return self.type_dict
+
+    def get_signature(self, name: str) -> Optional[Dict]:
+        """Get the signature of an embedding."""
+        try:
+            if name in self.from_method_nodes:
+                return build_template_from_method(
+                    name,
+                    type_to_cls_dict=self.type_to_loader_dict,
+                    method_name=self.from_method_nodes[name],
+                )
+            else:
+                return build_template_from_class(
+                    name, type_to_cls_dict=self.type_to_loader_dict
+                )
+        except ValueError as exc:
+            raise ValueError(f"Retriever {name} not found") from exc
+        except AttributeError as exc:
+            logger.error(f"Retriever {name} not loaded: {exc}")
+            return None
+
+    def to_list(self) -> List[str]:
+        return [
+            retriever
+            for retriever in self.type_to_loader_dict.keys()
+            if retriever in settings.retrievers or settings.dev
+        ]
+
+
+retriever_creator = RetrieverCreator()
--- a/src/backend/langflow/interface/types.py
+++ b/src/backend/langflow/interface/types.py
@ -11,6 +11,7 @@ from langflow.interface.tools.base import tool_creator
 from langflow.interface.utilities.base import utility_creator
 from langflow.interface.vector_store.base import vectorstore_creator
 from langflow.interface.wrappers.base import wrapper_creator
+from langflow.interface.retrievers.base import retriever_creator


 def get_type_list():
@ -44,6 +45,7 @@ def build_langchain_types_dict():  # sourcery skip: dict-assign-update-to-union
        documentloader_creator,
        textsplitter_creator,
        utility_creator,
+        retriever_creator,
    ]

    all_types = {}
@ -52,3 +54,6 @@ def build_langchain_types_dict():  # sourcery skip: dict-assign-update-to-union
        if created_types[creator.type_name].values():
            all_types.update(created_types)
    return all_types
+
+
+langchain_types_dict = build_langchain_types_dict()
--- a/Show more
+++ b/Show more