adjust edge tooltips

2023-07-04 19:11:15 -03:00 · 2023-07-04 19:11:15 -03:00 · 80efed1d91
commit 80efed1d91
parent 4f33f02d4f 8a4f1406de
135 changed files with 4884 additions and 2532 deletions
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@ -0,0 +1,66 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ 'dev', 'main' ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ 'dev' ]
+  schedule:
+    - cron: '17 2 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python', 'javascript' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Use only 'java' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #     echo "Run, Build Application using script"
+    #     ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"
--- a/.gitignore
+++ b/.gitignore
@ -245,3 +245,4 @@ dmypy.json
 .testenv/*
 langflow.db
 .githooks/prepare-commit-msg
+langchain.db
--- a/README.md
+++ b/README.md
@ -163,7 +163,7 @@ print(run_flow("Your message", flow_id=FLOW_ID, tweaks=TWEAKS))

 ## 🎨 Creating Flows

-Creating flows with LangFlow is easy. Simply drag sidebar components onto the canvas and connect them together to create your pipeline. LangFlow provides a range of [LangChain components](https://langchain.readthedocs.io/en/latest/reference.html) to choose from, including LLMs, prompt serializers, agents, and chains.
+Creating flows with LangFlow is easy. Simply drag sidebar components onto the canvas and connect them together to create your pipeline. LangFlow provides a range of [LangChain components](https://api.python.langchain.com/en/latest/index.html) to choose from, including LLMs, prompt serializers, agents, and chains.

 Explore by editing prompt parameters, link chains and agents, track an agent's thought process, and export your flow.

--- a/poetry.lock
+++ b/poetry.lock
--- a/pyproject.toml
+++ b/pyproject.toml
@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langflow"
-version = "0.2.1"
+version = "0.2.11"
 description = "A Python package with a built-in web application"
 authors = ["Logspace <contact@logspace.ai>"]
 maintainers = [
@ -22,25 +22,24 @@ include = ["src/backend/langflow/*", "src/backend/langflow/**/*"]
 langflow = "langflow.__main__:main"

 [tool.poetry.dependencies]
-python = ">=3.9,<3.12"
+python = ">=3.9,<3.11"
 fastapi = "^0.99.0"
 uvicorn = "^0.22.0"
-beautifulsoup4 = "^4.11.2"
+beautifulsoup4 = "^4.12.2"
 google-search-results = "^2.4.1"
 google-api-python-client = "^2.79.0"
 typer = "^0.9.0"
 gunicorn = "^20.1.0"
-langchain = "^0.0.215"
+langchain = "^0.0.219"
 openai = "^0.27.8"
-types-pyyaml = "^6.0.12.8"
-pandas = "^1.5.3"
+pandas = "^2.0.0"
 chromadb = "^0.3.21"
-huggingface-hub = "^0.13.3"
+huggingface-hub = "^0.15.0"
 rich = "^13.4.2"
 llama-cpp-python = "~0.1.0"
 networkx = "^3.1"
-unstructured = "^0.5.11"
-pypdf = "^3.7.1"
+unstructured = "^0.7.0"
+pypdf = "^3.11.0"
 lxml = "^4.9.2"
 pysrt = "^1.1.2"
 fake-useragent = "^1.1.3"
@ -49,18 +48,18 @@ psycopg2-binary = "^2.9.6"
 pyarrow = "^12.0.0"
 tiktoken = "~0.4.0"
 wikipedia = "^1.4.0"
-langchain-serve = { version = ">0.0.39", optional = true }
-qdrant-client = "^1.2.0"
+langchain-serve = { version = ">0.0.49", optional = true }
+qdrant-client = "^1.3.0"
 websockets = "^10.3"
 weaviate-client = "^3.21.0"
 jina = "3.15.2"
 sentence-transformers = "^2.2.2"
-ctransformers = "^0.2.2"
-cohere = "^4.6.0"
+ctransformers = "^0.2.10"
+cohere = "^4.11.0"
 python-multipart = "^0.0.6"
 sqlmodel = "^0.0.8"
 faiss-cpu = "^1.7.4"
-anthropic = "^0.2.10"
+anthropic = "^0.3.0"
 orjson = "^3.9.1"
 multiprocess = "^0.70.14"
 cachetools = "^5.3.1"
@ -70,7 +69,9 @@ pinecone-client = "^2.2.2"
 supabase = "^1.0.3"
 pymongo = "^4.4.0"
 certifi = "^2023.5.7"
-
+google-cloud-aiplatform = "^1.26.1"
+psycopg = "^3.1.9"
+psycopg-binary = "^3.1.9"

 [tool.poetry.dev-dependencies]
 black = "^23.1.0"
@ -85,6 +86,7 @@ pytest-cov = "^4.0.0"
 pandas-stubs = "^2.0.0.230412"
 types-pillow = "^9.5.0.2"
 types-appdirs = "^1.4.3.5"
+types-pyyaml = "^6.0.12.8"


 [tool.poetry.extras]
--- a/src/backend/langflow/main.py
+++ b/src/backend/langflow/main.py
@ -1,6 +1,5 @@
 import sys
 import time
-from fastapi import FastAPI
 import httpx
 from multiprocess import Process, cpu_count  # type: ignore
 import platform
@ -11,9 +10,7 @@ from rich.panel import Panel
 from rich import box
 from rich import print as rprint
 import typer
-from fastapi.staticfiles import StaticFiles
-from fastapi.responses import FileResponse
-from langflow.main import create_app
+from langflow.main import setup_app
 from langflow.settings import settings
 from langflow.utils.logger import configure, logger
 import webbrowser
@ -144,15 +141,9 @@ def serve(
        remove_api_keys=remove_api_keys,
        cache=cache,
    )
-    # get the directory of the current file
-    if not path:
-        frontend_path = Path(__file__).parent
-        static_files_dir = frontend_path / "frontend"
-    else:
-        static_files_dir = Path(path)
-
-    app = create_app()
-    setup_static_files(app, static_files_dir)
+    # create path object if path is provided
+    static_files_dir: Optional[Path] = Path(path) if path else None
+    app = setup_app(static_files_dir=static_files_dir)
    # check if port is being used
    if is_port_in_use(port, host):
        port = get_free_port(port)
@ -200,29 +191,6 @@ def run_on_windows(host, port, log_level, options, app):
    run_langflow(host, port, log_level, options, app)


-def setup_static_files(app: FastAPI, static_files_dir: Path):
-    """
-    Setup the static files directory.
-
-    Args:
-        app (FastAPI): FastAPI app.
-        path (str): Path to the static files directory.
-    """
-    app.mount(
-        "/",
-        StaticFiles(directory=static_files_dir, html=True),
-        name="static",
-    )
-
-    @app.exception_handler(404)
-    async def custom_404_handler(request, __):
-        path = static_files_dir / "index.html"
-
-        if not path.exists():
-            raise RuntimeError(f"File at path {path} does not exist.")
-        return FileResponse(path)
-
-
 def is_port_in_use(port, host="localhost"):
    """
    Check if a port is in use.
@ -303,7 +271,7 @@ def run_langflow(host, port, log_level, options, app):
    except KeyboardInterrupt:
        pass
    except Exception as e:
-        logger.error(e)
+        logger.exception(e)
        sys.exit(1)


--- a/src/backend/langflow/api/v1/base.py
+++ b/src/backend/langflow/api/v1/base.py
@ -18,6 +18,7 @@ class FrontendNodeRequest(FrontendNode):


 class ValidatePromptRequest(BaseModel):
+    name: str
    template: str
    frontend_node: FrontendNodeRequest

--- a/src/backend/langflow/api/v1/chat.py
+++ b/src/backend/langflow/api/v1/chat.py
@ -1,13 +1,7 @@
-from fastapi import (
-    APIRouter,
-    HTTPException,
-    WebSocket,
-    WebSocketException,
-    status,
-)
+from fastapi import APIRouter, HTTPException, WebSocket, WebSocketException, status
 from fastapi.responses import StreamingResponse
 from langflow.api.utils import build_input_keys_response
-from langflow.api.v1.schemas import BuiltResponse, InitResponse, StreamData
+from langflow.api.v1.schemas import BuildStatus, BuiltResponse, InitResponse, StreamData

 from langflow.chat.manager import ChatManager
 from langflow.graph.graph.base import Graph
@ -26,22 +20,39 @@ async def chat(client_id: str, websocket: WebSocket):
        if client_id in chat_manager.in_memory_cache:
            await chat_manager.handle_websocket(client_id, websocket)
        else:
+            # We accept the connection but close it immediately
+            # if the flow is not built yet
+            await websocket.accept()
            message = "Please, build the flow before sending messages"
-            await websocket.close(code=status.WS_1008_POLICY_VIOLATION, reason=message)
+            await websocket.close(code=status.WS_1011_INTERNAL_ERROR, reason=message)
    except WebSocketException as exc:
        logger.error(exc)
        await websocket.close(code=status.WS_1011_INTERNAL_ERROR, reason=str(exc))


-@router.post("/build/init", response_model=InitResponse, status_code=201)
-async def init_build(graph_data: dict):
+@router.post("/build/init/{flow_id}", response_model=InitResponse, status_code=201)
+async def init_build(graph_data: dict, flow_id: str):
    """Initialize the build by storing graph data and returning a unique session ID."""

    try:
-        flow_id = graph_data.get("id")
        if flow_id is None:
            raise ValueError("No ID provided")
-        flow_data_store[flow_id] = graph_data
+        # Check if already building
+        if (
+            flow_id in flow_data_store
+            and flow_data_store[flow_id]["status"] == BuildStatus.IN_PROGRESS
+        ):
+            return InitResponse(flowId=flow_id)
+
+        # Delete from cache if already exists
+        if flow_id in chat_manager.in_memory_cache:
+            with chat_manager.in_memory_cache._lock:
+                chat_manager.in_memory_cache.delete(flow_id)
+                logger.debug(f"Deleted flow {flow_id} from cache")
+        flow_data_store[flow_id] = {
+            "graph_data": graph_data,
+            "status": BuildStatus.STARTED,
+        }

        return InitResponse(flowId=flow_id)
    except Exception as exc:
@ -53,8 +64,9 @@ async def init_build(graph_data: dict):
 async def build_status(flow_id: str):
    """Check the flow_id is in the flow_data_store."""
    try:
-        built = flow_id in flow_data_store and not isinstance(
-            flow_data_store[flow_id], dict
+        built = (
+            flow_id in flow_data_store
+            and flow_data_store[flow_id]["status"] == BuildStatus.SUCCESS
        )

        return BuiltResponse(
@ -79,7 +91,12 @@ async def stream_build(flow_id: str):
                yield str(StreamData(event="error", data={"error": error_message}))
                return

-            graph_data = flow_data_store[flow_id].get("data")
+            if flow_data_store[flow_id].get("status") == BuildStatus.IN_PROGRESS:
+                error_message = "Already building"
+                yield str(StreamData(event="error", data={"error": error_message}))
+                return
+
+            graph_data = flow_data_store[flow_id].get("graph_data")

            if not graph_data:
                error_message = "No data provided"
@ -97,6 +114,15 @@ async def stream_build(flow_id: str):
                return

            number_of_nodes = len(graph.nodes)
+            flow_data_store[flow_id]["status"] = BuildStatus.IN_PROGRESS
+            # To deal with the ZeroShotAgent case
+            # we need to build the root node first
+            # and then the rest of the graph
+            # This is a big problem because certain nodes require
+            # params that are not connected to it.
+            # We should consider connecting the tools to the ZeroShotPrompt
+            graph.build()
+
            for i, vertex in enumerate(graph.generator_build(), 1):
                try:
                    log_dict = {
@ -117,6 +143,7 @@ async def stream_build(flow_id: str):
                except Exception as exc:
                    params = str(exc)
                    valid = False
+                    flow_data_store[flow_id]["status"] = BuildStatus.FAILURE

                response = {
                    "valid": valid,
@ -138,9 +165,11 @@ async def stream_build(flow_id: str):
            chat_manager.set_cache(flow_id, langchain_object)
            # We need to reset the chat history
            chat_manager.chat_history.empty_history(flow_id)
+            flow_data_store[flow_id]["status"] = BuildStatus.SUCCESS
        except Exception as exc:
            logger.exception(exc)
            logger.error("Error while building the flow: %s", exc)
+            flow_data_store[flow_id]["status"] = BuildStatus.FAILURE
            yield str(StreamData(event="error", data={"error": str(exc)}))
        finally:
            yield str(StreamData(event="message", data=final_response))
--- a/src/backend/langflow/api/v1/endpoints.py
+++ b/src/backend/langflow/api/v1/endpoints.py
@ -11,9 +11,7 @@ from langflow.api.v1.schemas import (
    UploadFileResponse,
 )

-from langflow.interface.types import (
-    build_langchain_types_dict,
-)
+from langflow.interface.types import langchain_types_dict
 from langflow.database.base import get_session
 from sqlmodel import Session

@ -23,7 +21,7 @@ router = APIRouter(tags=["Base"])

@router.get("/all")
 def get_all():
-    return build_langchain_types_dict()
+    return langchain_types_dict


 # For backwards compatibility we will keep the old endpoint
--- a/src/backend/langflow/api/v1/schemas.py
+++ b/src/backend/langflow/api/v1/schemas.py
@ -1,3 +1,4 @@
+from enum import Enum
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 from langflow.database.models.flow import FlowCreate, FlowRead
@ -5,6 +6,15 @@ from pydantic import BaseModel, Field, validator
 import json


+class BuildStatus(Enum):
+    """Status of the build."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    STARTED = "started"
+    IN_PROGRESS = "in_progress"
+
+
 class GraphData(BaseModel):
    """Data inside the exported flow."""

--- a/src/backend/langflow/api/v1/validate.py
+++ b/src/backend/langflow/api/v1/validate.py
@ -28,49 +28,93 @@ def post_validate_code(code: Code):


@router.post("/prompt", status_code=200, response_model=PromptValidationResponse)
-def post_validate_prompt(prompt: ValidatePromptRequest):
+def post_validate_prompt(prompt_request: ValidatePromptRequest):
    try:
-        input_variables = validate_prompt(prompt.template)
-        # Reinitialize custom_fields
-        old_custom_fields = prompt.frontend_node.custom_fields.copy()
-        prompt.frontend_node.custom_fields = []
-        # Add new variables to the template
-        for variable in input_variables:
-            try:
-                template_field = TemplateField(
-                    name=variable,
-                    display_name=variable,
-                    field_type="str",
-                    show=True,
-                    advanced=False,
-                    input_types=["Document", "BaseOutputParser"],
+        input_variables = validate_prompt(prompt_request.template)
+
+        old_custom_fields = get_old_custom_fields(prompt_request)
+
+        add_new_variables_to_template(input_variables, prompt_request)
+
+        remove_old_variables_from_template(
+            old_custom_fields, input_variables, prompt_request
+        )
+
+        update_input_variables_field(input_variables, prompt_request)
+
+        return PromptValidationResponse(
+            input_variables=input_variables,
+            frontend_node=prompt_request.frontend_node,
+        )
+    except Exception as e:
+        logger.exception(e)
+        raise HTTPException(status_code=500, detail=str(e)) from e
+
+
+def get_old_custom_fields(prompt_request):
+    try:
+        old_custom_fields = prompt_request.frontend_node.custom_fields[
+            prompt_request.name
+        ].copy()
+    except KeyError:
+        old_custom_fields = []
+    prompt_request.frontend_node.custom_fields[prompt_request.name] = []
+    return old_custom_fields
+
+
+def add_new_variables_to_template(input_variables, prompt_request):
+    for variable in input_variables:
+        try:
+            template_field = TemplateField(
+                name=variable,
+                display_name=variable,
+                field_type="str",
+                show=True,
+                advanced=False,
+                input_types=["Document", "BaseOutputParser"],
+            )
+
+            prompt_request.frontend_node.template[variable] = template_field.to_dict()
+
+            # Check if variable is not already in the list before appending
+            if (
+                variable
+                not in prompt_request.frontend_node.custom_fields[prompt_request.name]
+            ):
+                prompt_request.frontend_node.custom_fields[prompt_request.name].append(
+                    variable
                )

-                prompt.frontend_node.template[variable] = template_field.to_dict()
-                prompt.frontend_node.custom_fields.append(variable)
+        except Exception as exc:
+            logger.exception(exc)
+            raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+
+def remove_old_variables_from_template(
+    old_custom_fields, input_variables, prompt_request
+):
+    for variable in old_custom_fields:
+        if variable not in input_variables:
+            try:
+                # Remove the variable from custom_fields associated with the given name
+                if (
+                    variable
+                    in prompt_request.frontend_node.custom_fields[prompt_request.name]
+                ):
+                    prompt_request.frontend_node.custom_fields[
+                        prompt_request.name
+                    ].remove(variable)
+
+                # Remove the variable from the template
+                prompt_request.frontend_node.template.pop(variable, None)

            except Exception as exc:
                logger.exception(exc)
                raise HTTPException(status_code=500, detail=str(exc)) from exc

-        # Remove variables that are not in the template anymore
-        for variable in old_custom_fields:
-            if variable not in input_variables:
-                try:
-                    prompt.frontend_node.template.pop(variable, None)
-                except Exception as exc:
-                    logger.exception(exc)
-                    raise HTTPException(status_code=500, detail=str(exc)) from exc

-        # Now we will set the field "input_variables" to the new list of variables
-        # if it exists
-        if "input_variables" in prompt.frontend_node.template:
-            prompt.frontend_node.template["input_variables"]["value"] = input_variables
-
-        return PromptValidationResponse(
-            input_variables=input_variables,
-            frontend_node=prompt.frontend_node,
-        )
-    except Exception as e:
-        logger.exception(e)
-        raise HTTPException(status_code=500, detail=str(e)) from e
+def update_input_variables_field(input_variables, prompt_request):
+    if "input_variables" in prompt_request.frontend_node.template:
+        prompt_request.frontend_node.template["input_variables"][
+            "value"
+        ] = input_variables
--- a/src/backend/langflow/chat/config.py
+++ b/src/backend/langflow/chat/config.py
@ -0,0 +1,2 @@
+class ChatConfig:
+    streaming: bool = True
--- a/src/backend/langflow/config.yaml
+++ b/src/backend/langflow/config.yaml
@ -1,146 +1,289 @@
---
 agents:
-  - ZeroShotAgent
-  - JsonAgent
-  - CSVAgent
-  - AgentInitializer
-  - VectorStoreAgent
-  - VectorStoreRouterAgent
-  - SQLAgent
+  ZeroShotAgent:
+    documentation: "https://python.langchain.com/docs/modules/agents/how_to/custom_mrkl_agent"
+  JsonAgent:
+    documentation: "https://python.langchain.com/docs/modules/agents/toolkits/openapi"
+  CSVAgent:
+    documentation: "https://python.langchain.com/docs/modules/agents/toolkits/csv"
+  AgentInitializer:
+    documentation: "https://python.langchain.com/docs/modules/agents/agent_types/"
+  VectorStoreAgent:
+    documentation: ""
+  VectorStoreRouterAgent:
+    documentation: ""
+  SQLAgent:
+    documentation: ""
 chains:
-  - LLMChain
-  - LLMMathChain
-  - LLMCheckerChain
-  - ConversationChain
-  - SeriesCharacterChain
-  - MidJourneyPromptChain
-  - TimeTravelGuideChain
-  - SQLDatabaseChain
-  - RetrievalQA
-  - RetrievalQAWithSourcesChain
-  - ConversationalRetrievalChain
-  - CombineDocsChain
+  LLMChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/foundational/llm_chain"
+  LLMMathChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/additional/llm_math"
+  LLMCheckerChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/additional/llm_checker"
+  ConversationChain:
+    documentation: ""
+  SeriesCharacterChain:
+    documentation: ""
+  MidJourneyPromptChain:
+    documentation: ""
+  TimeTravelGuideChain:
+    documentation: ""
+  SQLDatabaseChain:
+    documentation: ""
+  RetrievalQA:
+    documentation: "https://python.langchain.com/docs/modules/chains/popular/vector_db_qa"
+  RetrievalQAWithSourcesChain:
+    documentation: ""
+  ConversationalRetrievalChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/popular/chat_vector_db"
+  CombineDocsChain:
+    documentation: ""
 documentloaders:
-  - AirbyteJSONLoader
-  - CoNLLULoader
-  - CSVLoader
-  - UnstructuredEmailLoader
-  - EverNoteLoader
-  - FacebookChatLoader
-  - GutenbergLoader
-  - BSHTMLLoader
-  - UnstructuredHTMLLoader
-  # - UnstructuredImageLoader # Issue with Python 3.11 (https://github.com/Unstructured-IO/unstructured-inference/issues/83)
-  - UnstructuredMarkdownLoader
-  - PyPDFLoader
-  - UnstructuredPowerPointLoader
-  - SRTLoader
-  - TelegramChatLoader
-  - TextLoader
-  - UnstructuredWordDocumentLoader
-  - WebBaseLoader
-  - AZLyricsLoader
-  - CollegeConfidentialLoader
-  - HNLoader
-  - IFixitLoader
-  - IMSDbLoader
-  - GitbookLoader
-  - ReadTheDocsLoader
-  - SlackDirectoryLoader
-  - NotionDirectoryLoader
-  - DirectoryLoader
-  - GitLoader
+  AirbyteJSONLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/airbyte_json"
+  CoNLLULoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/conll-u"
+  CSVLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/csv"
+  UnstructuredEmailLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/email"
+  EverNoteLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/evernote"
+  FacebookChatLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/facebook_chat"
+  GutenbergLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/gutenberg"
+  BSHTMLLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/html"
+  UnstructuredHTMLLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/html"
+  UnstructuredMarkdownLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/markdown"
+  PyPDFDirectoryLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/pdf"
+  PyPDFLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/pdf"
+  UnstructuredPowerPointLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/microsoft_powerpoint"
+  SRTLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/subtitle"
+  TelegramChatLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/telegram"
+  TextLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/"
+  UnstructuredWordDocumentLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/microsoft_word"
+  WebBaseLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/web_base"
+  AZLyricsLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/azlyrics"
+  CollegeConfidentialLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/college_confidential"
+  HNLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/hacker_news"
+  IFixitLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/ifixit"
+  IMSDbLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/imsdb"
+  GitbookLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/gitbook"
+  ReadTheDocsLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/readthedocs_documentation"
+  SlackDirectoryLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/slack"
+  NotionDirectoryLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/notion"
+  DirectoryLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/file_directory"
+  GitLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/git"
 embeddings:
-  - OpenAIEmbeddings
-  - HuggingFaceEmbeddings
-  - CohereEmbeddings
+  OpenAIEmbeddings:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/openai"
+  HuggingFaceEmbeddings:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/sentence_transformers"
+  CohereEmbeddings:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/cohere"
 llms:
-  - OpenAI
-  # - AzureOpenAI
-  # - AzureChatOpenAI
-  - ChatOpenAI
-  - LlamaCpp
-  - CTransformers
-  - Cohere
-  - Anthropic
-  - ChatAnthropic
-  - HuggingFaceHub
+  OpenAI:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/openai"
+  ChatOpenAI:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/integrations/openai"
+  LlamaCpp:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/llamacpp"
+  CTransformers:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/ctransformers"
+  Cohere:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/cohere"
+  Anthropic:
+    documentation: ""
+  ChatAnthropic:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/integrations/anthropic"
+  HuggingFaceHub:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/huggingface_hub"
+  VertexAI:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/google_vertex_ai_palm"
+  ###
+  # There's a bug in this component deactivating until we get it sorted: _language_models.py", line 804, in send_message
+  #     is_blocked=safety_attributes.get("blocked", False),
+  # AttributeError: 'list' object has no attribute 'get'
+  # ChatVertexAI:
+  #   documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/integrations/google_vertex_ai_palm"
+  ###
 memories:
-  - ConversationBufferMemory
-  - ConversationSummaryMemory
-  - ConversationKGMemory
+  # https://github.com/supabase-community/supabase-py/issues/482
+  # ZepChatMessageHistory:
+  #   documentation: "https://python.langchain.com/docs/modules/memory/integrations/zep_memory"
+  ConversationEntityMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/integrations/entity_memory_with_sqlite"
+  # https://github.com/hwchase17/langchain/issues/6091
+  # SQLiteEntityStore:
+  #   documentation: "https://python.langchain.com/docs/modules/memory/integrations/entity_memory_with_sqlite"
+  PostgresChatMessageHistory:
+    documentation: "https://python.langchain.com/docs/modules/memory/integrations/postgres_chat_message_history"
+  ConversationBufferMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/buffer"
+  ConversationSummaryMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/summary"
+  ConversationKGMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/kg"
+  ConversationBufferWindowMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/buffer_window"
+  VectorStoreRetrieverMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/vectorstore_retriever_memory"
+  MongoDBChatMessageHistory:
+    documentation: "https://python.langchain.com/docs/modules/memory/integrations/mongodb_chat_message_history"
+  ChatMessagePromptTemplate:
+    documentation: "https://python.langchain.com/docs/modules/model_io/prompts/prompt_templates/msg_prompt_templates"
+  HumanMessagePromptTemplate:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/how_to/prompts"
+  SystemMessagePromptTemplate:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/how_to/prompts"
+  ChatPromptTemplate:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/how_to/prompts"
 prompts:
-  - PromptTemplate
-  - FewShotPromptTemplate
-  - ZeroShotPrompt
-  - ChatPromptTemplate
-  - SystemMessagePromptTemplate
-  - AIMessagePromptTemplate
-  - HumanMessagePromptTemplate
-  - ChatMessagePromptTemplate
+  PromptTemplate:
+    documentation: "https://python.langchain.com/docs/modules/model_io/prompts/prompt_templates/"
+  ZeroShotPrompt:
+    documentation: "https://python.langchain.com/docs/modules/agents/how_to/custom_mrkl_agent"
 textsplitters:
-  - CharacterTextSplitter
-  - RecursiveCharacterTextSplitter
-  # - LatexTextSplitter
-  # - PythonCodeTextSplitter
+  CharacterTextSplitter:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_transformers/text_splitters/character_text_splitter"
+  RecursiveCharacterTextSplitter:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_transformers/text_splitters/recursive_text_splitter"
 toolkits:
-  - OpenAPIToolkit
-  - JsonToolkit
-  - VectorStoreInfo
-  - VectorStoreRouterToolkit
-  - VectorStoreToolkit
+  OpenAPIToolkit:
+    documentation: ""
+  JsonToolkit:
+    documentation: ""
+  VectorStoreInfo:
+    documentation: ""
+  VectorStoreRouterToolkit:
+    documentation: ""
+  VectorStoreToolkit:
+    documentation: ""
 tools:
-  - Search
-  - PAL-MATH
-  - Calculator
-  - Serper Search
-  - Tool
-  - PythonFunctionTool
-  - PythonFunction
-  - JsonSpec
-  - News API
-  - TMDB API
-  - Podcast API
-  - QuerySQLDataBaseTool
-  - InfoSQLDatabaseTool
-  - ListSQLDatabaseTool
-  # - QueryCheckerTool
-  - BingSearchRun
-  - GoogleSearchRun
-  - GoogleSearchResults
-  - GoogleSerperRun
-  - JsonListKeysTool
-  - JsonGetValueTool
-  - PythonREPLTool
-  - PythonAstREPLTool
-  - RequestsGetTool
-  - RequestsPostTool
-  - RequestsPatchTool
-  - RequestsPutTool
-  - RequestsDeleteTool
-  - WikipediaQueryRun
-  - WolframAlphaQueryRun
+  Search:
+    documentation: ""
+  PAL-MATH:
+    documentation: ""
+  Calculator:
+    documentation: ""
+  Serper Search:
+    documentation: ""
+  Tool:
+    documentation: ""
+  PythonFunctionTool:
+    documentation: ""
+  PythonFunction:
+    documentation: ""
+  JsonSpec:
+    documentation: ""
+  News API:
+    documentation: ""
+  TMDB API:
+    documentation: ""
+  Podcast API:
+    documentation: ""
+  QuerySQLDataBaseTool:
+    documentation: ""
+  InfoSQLDatabaseTool:
+    documentation: ""
+  ListSQLDatabaseTool:
+    documentation: ""
+  BingSearchRun:
+    documentation: ""
+  GoogleSearchRun:
+    documentation: ""
+  GoogleSearchResults:
+    documentation: ""
+  GoogleSerperRun:
+    documentation: ""
+  JsonListKeysTool:
+    documentation: ""
+  JsonGetValueTool:
+    documentation: ""
+  PythonREPLTool:
+    documentation: ""
+  PythonAstREPLTool:
+    documentation: ""
+  RequestsGetTool:
+    documentation: ""
+  RequestsPostTool:
+    documentation: ""
+  RequestsPatchTool:
+    documentation: ""
+  RequestsPutTool:
+    documentation: ""
+  RequestsDeleteTool:
+    documentation: ""
+  WikipediaQueryRun:
+    documentation: ""
+  WolframAlphaQueryRun:
+    documentation: ""
 utilities:
-  - BingSearchAPIWrapper
-  - GoogleSearchAPIWrapper
-  - GoogleSerperAPIWrapper
-  - SearxResults
-  - SearxSearchWrapper
-  - SerpAPIWrapper
-  - WikipediaAPIWrapper
-  - WolframAlphaAPIWrapper
-  # - ZapierNLAWrapper
-  - SQLDatabase
+  BingSearchAPIWrapper:
+    documentation: ""
+  GoogleSearchAPIWrapper:
+    documentation: ""
+  GoogleSerperAPIWrapper:
+    documentation: ""
+  SearxResults:
+    documentation: ""
+  SearxSearchWrapper:
+    documentation: ""
+  SerpAPIWrapper:
+    documentation: ""
+  WikipediaAPIWrapper:
+    documentation: ""
+  WolframAlphaAPIWrapper:
+    documentation: ""
+retrievers:
+  MultiQueryRetriever:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/retrievers/how_to/MultiQueryRetriever"
+  # https://github.com/supabase-community/supabase-py/issues/482
+  # ZepRetriever:
+  #   documentation: "https://python.langchain.com/docs/modules/data_connection/retrievers/integrations/zep_memorystore"
 vectorstores:
-  - Chroma
-  - Qdrant
-  - Weaviate
-  - FAISS
-  - Pinecone
-  - SupabaseVectorStore
-  - MongoDBAtlasVectorSearch
+  Chroma:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/chroma"
+  Qdrant:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/qdrant"
+  Weaviate:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/weaviate"
+  FAISS:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/faiss"
+  Pinecone:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/pinecone"
+  SupabaseVectorStore:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/supabase"
+  MongoDBAtlasVectorSearch:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/mongodb_atlas"
 wrappers:
-  - RequestsWrapper
+  RequestsWrapper:
+    documentation: ""
 output_parsers:
-  - StructuredOutputParser
-  - ResponseSchema
+  StructuredOutputParser:
+    documentation: "https://python.langchain.com/docs/modules/model_io/output_parsers/structured"
+  ResponseSchema:
+    documentation: "https://python.langchain.com/docs/modules/model_io/output_parsers/structured"
--- a/src/backend/langflow/custom/customs.py
+++ b/src/backend/langflow/custom/customs.py
@ -21,6 +21,9 @@ CUSTOM_NODES = {
    "utilities": {
        "SQLDatabase": frontend_node.agents.SQLDatabaseNode(),
    },
+    "memories": {
+        "PostgresChatMessageHistory": frontend_node.memories.PostgresChatMessageHistoryFrontendNode(),
+    },
    "chains": {
        "SeriesCharacterChain": frontend_node.chains.SeriesCharacterChainNode(),
        "TimeTravelGuideChain": frontend_node.chains.TimeTravelGuideChainNode(),
--- a/src/backend/langflow/database/base.py
+++ b/src/backend/langflow/database/base.py
@ -1,6 +1,6 @@
 from langflow.settings import settings
 from sqlmodel import SQLModel, Session, create_engine
-
+from langflow.utils.logger import logger

 if settings.database_url.startswith("sqlite"):
    connect_args = {"check_same_thread": False}
@ -10,7 +10,9 @@ engine = create_engine(settings.database_url, connect_args=connect_args)


 def create_db_and_tables():
+    logger.debug("Creating database and tables")
    SQLModel.metadata.create_all(engine)
+    logger.debug("Database and tables created")


 def get_session():
--- a/src/backend/langflow/graph/init.py
+++ b/src/backend/langflow/graph/init.py
@ -14,6 +14,7 @@ from langflow.graph.vertex.types import (
    ToolkitVertex,
    VectorStoreVertex,
    WrapperVertex,
+    RetrieverVertex,
 )

 __all__ = [
@ -32,4 +33,5 @@ __all__ = [
    "ToolkitVertex",
    "VectorStoreVertex",
    "WrapperVertex",
+    "RetrieverVertex",
 ]
--- a/src/backend/langflow/graph/edge/base.py
+++ b/src/backend/langflow/graph/edge/base.py
@ -33,12 +33,7 @@ class Edge:
        # Get what type of input the target node is expecting

        self.matched_type = next(
-            (
-                output
-                for output in self.source_types
-                for target_req in self.target_reqs
-                if output in target_req
-            ),
+            (output for output in self.source_types if output in self.target_reqs),
            None,
        )
        no_matched_type = self.matched_type is None
--- a/src/backend/langflow/graph/graph/constants.py
+++ b/src/backend/langflow/graph/graph/constants.py
@ -13,7 +13,7 @@ from langflow.interface.tools.base import tool_creator
 from langflow.interface.vector_store.base import vectorstore_creator
 from langflow.interface.wrappers.base import wrapper_creator
 from langflow.interface.output_parsers.base import output_parser_creator
-
+from langflow.interface.retrievers.base import retriever_creator

 from typing import Dict, Type

@ -32,4 +32,5 @@ VERTEX_TYPE_MAP: Dict[str, Type[Vertex]] = {
    **{t: types.DocumentLoaderVertex for t in documentloader_creator.to_list()},
    **{t: types.TextSplitterVertex for t in textsplitter_creator.to_list()},
    **{t: types.OutputParserVertex for t in output_parser_creator.to_list()},
+    **{t: types.RetrieverVertex for t in retriever_creator.to_list()},
 }
--- a/src/backend/langflow/graph/vertex/types.py
+++ b/src/backend/langflow/graph/vertex/types.py
@ -113,6 +113,11 @@ class MemoryVertex(Vertex):
        super().__init__(data, base_type="memory")


+class RetrieverVertex(Vertex):
+    def __init__(self, data: Dict):
+        super().__init__(data, base_type="retrievers")
+
+
 class TextSplitterVertex(Vertex):
    def __init__(self, data: Dict):
        super().__init__(data, base_type="textsplitters")
--- a/src/backend/langflow/interface/base.py
+++ b/src/backend/langflow/interface/base.py
@ -8,6 +8,7 @@ from langflow.template.field.base import TemplateField
 from langflow.template.frontend_node.base import FrontendNode
 from langflow.template.template.base import Template
 from langflow.utils.logger import logger
+from langflow.settings import settings

 # Assuming necessary imports for Field, Template, and FrontendNode classes

@ -15,12 +16,29 @@ from langflow.utils.logger import logger
 class LangChainTypeCreator(BaseModel, ABC):
    type_name: str
    type_dict: Optional[Dict] = None
+    name_docs_dict: Optional[Dict[str, str]] = None

    @property
    def frontend_node_class(self) -> Type[FrontendNode]:
        """The class type of the FrontendNode created in frontend_node."""
        return FrontendNode

+    @property
+    def docs_map(self) -> Dict[str, str]:
+        """A dict with the name of the component as key and the documentation link as value."""
+        if self.name_docs_dict is None:
+            try:
+                type_settings = getattr(settings, self.type_name)
+                self.name_docs_dict = {
+                    name: value_dict["documentation"]
+                    for name, value_dict in type_settings.items()
+                }
+            except AttributeError as exc:
+                logger.error(exc)
+
+                self.name_docs_dict = {}
+        return self.name_docs_dict
+
    @property
    @abstractmethod
    def type_to_loader_dict(self) -> Dict:
@ -83,7 +101,7 @@ class LangChainTypeCreator(BaseModel, ABC):

        signature.add_extra_fields()
        signature.add_extra_base_classes()
-
+        signature.set_documentation(self.docs_map.get(name, ""))
        return signature


--- a/Show more
+++ b/Show more