Merge from form_io to python_custom_node_component

src/backend/langflow/__main__.py

@@ -1,6 +1,5 @@
 import sys
 import time
-from fastapi import FastAPI
 import httpx
 from multiprocess import Process, cpu_count  # type: ignore
 import platform
@@ -11,9 +10,7 @@ from rich.panel import Panel
 from rich import box
 from rich import print as rprint
 import typer
-from fastapi.staticfiles import StaticFiles
-from fastapi.responses import FileResponse
-from langflow.main import create_app
+from langflow.main import setup_app
 from langflow.settings import settings
 from langflow.utils.logger import configure, logger
 import webbrowser
@@ -144,15 +141,9 @@ def serve(
         remove_api_keys=remove_api_keys,
         cache=cache,
     )
-    # get the directory of the current file
-    if not path:
-        frontend_path = Path(__file__).parent
-        static_files_dir = frontend_path / "frontend"
-    else:
-        static_files_dir = Path(path)
-
-    app = create_app()
-    setup_static_files(app, static_files_dir)
+    # create path object if path is provided
+    static_files_dir: Optional[Path] = Path(path) if path else None
+    app = setup_app(static_files_dir=static_files_dir)
     # check if port is being used
     if is_port_in_use(port, host):
         port = get_free_port(port)
@@ -200,29 +191,6 @@ def run_on_windows(host, port, log_level, options, app):
     run_langflow(host, port, log_level, options, app)
 
 
-def setup_static_files(app: FastAPI, static_files_dir: Path):
-    """
-    Setup the static files directory.
-
-    Args:
-        app (FastAPI): FastAPI app.
-        path (str): Path to the static files directory.
-    """
-    app.mount(
-        "/",
-        StaticFiles(directory=static_files_dir, html=True),
-        name="static",
-    )
-
-    @app.exception_handler(404)
-    async def custom_404_handler(request, __):
-        path = static_files_dir / "index.html"
-
-        if not path.exists():
-            raise RuntimeError(f"File at path {path} does not exist.")
-        return FileResponse(path)
-
-
 def is_port_in_use(port, host="localhost"):
     """
     Check if a port is in use.
@@ -303,7 +271,7 @@ def run_langflow(host, port, log_level, options, app):
     except KeyboardInterrupt:
         pass
     except Exception as e:
-        logger.error(e)
+        logger.exception(e)
         sys.exit(1)
 
 

src/backend/langflow/api/utils.py

@@ -51,7 +51,9 @@ def build_input_keys_response(langchain_object, artifacts):
         # Add memory variables to memory_keys
         input_keys_response["memory_keys"] = langchain_object.memory.memory_variables
 
-    if hasattr(langchain_object, "prompt"):
+    if hasattr(langchain_object, "prompt") and hasattr(
+        langchain_object.prompt, "template"
+    ):
         input_keys_response["template"] = langchain_object.prompt.template
 
     return input_keys_response

src/backend/langflow/api/v1/base.py

@@ -18,6 +18,7 @@ class FrontendNodeRequest(FrontendNode):
 
 
 class ValidatePromptRequest(BaseModel):
+    name: str
     template: str
     frontend_node: FrontendNodeRequest
 

src/backend/langflow/api/v1/chat.py

@@ -1,13 +1,7 @@
-from fastapi import (
-    APIRouter,
-    HTTPException,
-    WebSocket,
-    WebSocketException,
-    status,
-)
+from fastapi import APIRouter, HTTPException, WebSocket, WebSocketException, status
 from fastapi.responses import StreamingResponse
 from langflow.api.utils import build_input_keys_response
-from langflow.api.v1.schemas import BuiltResponse, InitResponse, StreamData
+from langflow.api.v1.schemas import BuildStatus, BuiltResponse, InitResponse, StreamData
 
 from langflow.chat.manager import ChatManager
 from langflow.graph.graph.base import Graph
@@ -26,22 +20,39 @@ async def chat(client_id: str, websocket: WebSocket):
         if client_id in chat_manager.in_memory_cache:
             await chat_manager.handle_websocket(client_id, websocket)
         else:
+            # We accept the connection but close it immediately
+            # if the flow is not built yet
+            await websocket.accept()
             message = "Please, build the flow before sending messages"
-            await websocket.close(code=status.WS_1008_POLICY_VIOLATION, reason=message)
+            await websocket.close(code=status.WS_1011_INTERNAL_ERROR, reason=message)
     except WebSocketException as exc:
         logger.error(exc)
         await websocket.close(code=status.WS_1011_INTERNAL_ERROR, reason=str(exc))
 
 
-@router.post("/build/init", response_model=InitResponse, status_code=201)
-async def init_build(graph_data: dict):
+@router.post("/build/init/{flow_id}", response_model=InitResponse, status_code=201)
+async def init_build(graph_data: dict, flow_id: str):
     """Initialize the build by storing graph data and returning a unique session ID."""
 
     try:
-        flow_id = graph_data.get("id")
         if flow_id is None:
             raise ValueError("No ID provided")
-        flow_data_store[flow_id] = graph_data
+        # Check if already building
+        if (
+            flow_id in flow_data_store
+            and flow_data_store[flow_id]["status"] == BuildStatus.IN_PROGRESS
+        ):
+            return InitResponse(flowId=flow_id)
+
+        # Delete from cache if already exists
+        if flow_id in chat_manager.in_memory_cache:
+            with chat_manager.in_memory_cache._lock:
+                chat_manager.in_memory_cache.delete(flow_id)
+                logger.debug(f"Deleted flow {flow_id} from cache")
+        flow_data_store[flow_id] = {
+            "graph_data": graph_data,
+            "status": BuildStatus.STARTED,
+        }
 
         return InitResponse(flowId=flow_id)
     except Exception as exc:
@@ -53,8 +64,9 @@ async def init_build(graph_data: dict):
 async def build_status(flow_id: str):
     """Check the flow_id is in the flow_data_store."""
     try:
-        built = flow_id in flow_data_store and not isinstance(
-            flow_data_store[flow_id], dict
+        built = (
+            flow_id in flow_data_store
+            and flow_data_store[flow_id]["status"] == BuildStatus.SUCCESS
         )
 
         return BuiltResponse(
@@ -79,7 +91,12 @@ async def stream_build(flow_id: str):
                 yield str(StreamData(event="error", data={"error": error_message}))
                 return
 
-            graph_data = flow_data_store[flow_id].get("data")
+            if flow_data_store[flow_id].get("status") == BuildStatus.IN_PROGRESS:
+                error_message = "Already building"
+                yield str(StreamData(event="error", data={"error": error_message}))
+                return
+
+            graph_data = flow_data_store[flow_id].get("graph_data")
 
             if not graph_data:
                 error_message = "No data provided"
@@ -97,6 +114,15 @@ async def stream_build(flow_id: str):
                 return
 
             number_of_nodes = len(graph.nodes)
+            flow_data_store[flow_id]["status"] = BuildStatus.IN_PROGRESS
+            # To deal with the ZeroShotAgent case
+            # we need to build the root node first
+            # and then the rest of the graph
+            # This is a big problem because certain nodes require
+            # params that are not connected to it.
+            # We should consider connecting the tools to the ZeroShotPrompt
+            graph.build()
+
             for i, vertex in enumerate(graph.generator_build(), 1):
                 try:
                     log_dict = {
@@ -117,6 +143,7 @@ async def stream_build(flow_id: str):
                 except Exception as exc:
                     params = str(exc)
                     valid = False
+                    flow_data_store[flow_id]["status"] = BuildStatus.FAILURE
 
                 response = {
                     "valid": valid,
@@ -138,9 +165,11 @@ async def stream_build(flow_id: str):
             chat_manager.set_cache(flow_id, langchain_object)
             # We need to reset the chat history
             chat_manager.chat_history.empty_history(flow_id)
+            flow_data_store[flow_id]["status"] = BuildStatus.SUCCESS
         except Exception as exc:
             logger.exception(exc)
             logger.error("Error while building the flow: %s", exc)
+            flow_data_store[flow_id]["status"] = BuildStatus.FAILURE
             yield str(StreamData(event="error", data={"error": str(exc)}))
         finally:
             yield str(StreamData(event="message", data=final_response))

src/backend/langflow/api/v1/endpoints.py

@@ -19,10 +19,10 @@ from langflow.api.v1.schemas import (
 )
 
 from langflow.interface.types import (
-    build_langchain_types_dict,
-    build_langchain_template_custom_component
+    build_langchain_template_custom_component,
 )
 
+from langflow.interface.types import langchain_types_dict
 from langflow.database.base import get_session
 from sqlmodel import Session
 
@@ -32,7 +32,7 @@ router = APIRouter(tags=["Base"])
 
 @router.get("/all")
 def get_all():
-    return build_langchain_types_dict()
+    return langchain_types_dict
 
 
 # For backwards compatibility we will keep the old endpoint
@@ -71,7 +71,11 @@ async def process_flow(
         raise HTTPException(status_code=500, detail=str(e)) from e
 
 
-@router.post("/upload/{flow_id}", response_model=UploadFileResponse, status_code=HTTPStatus.CREATED)
+@router.post(
+    "/upload/{flow_id}",
+    response_model=UploadFileResponse,
+    status_code=HTTPStatus.CREATED,
+)
 async def create_upload_file(file: UploadFile, flow_id: str):
     # Cache file
     try:
@@ -108,23 +112,25 @@ async def custom_component(
 
 
 # TODO: Just for test - will be remove
-@router.get("/custom_component_error",
-            response_model=CustomComponentResponseError,
-            status_code=HTTPStatus.BAD_REQUEST)
+@router.get(
+    "/custom_component_error",
+    response_model=CustomComponentResponseError,
+    status_code=HTTPStatus.BAD_REQUEST,
+)
 async def custom_component_error():
     error1 = {
         "detail": "'int' object has no attribute 'get'",
-        "traceback": "Traceback (most recent call last):\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/errors.py\", line 162, in __call__\n    await self.app(scope, receive, _send)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/cors.py\", line 83, in __call__\n    await self.app(scope, receive, send)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/exceptions.py\", line 79, in __call__\n    raise exc\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/exceptions.py\", line 68, in __call__\n    await self.app(scope, receive, sender)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py\", line 20, in __call__\n    raise e\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py\", line 17, in __call__\n    await self.app(scope, receive, send)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py\", line 718, in __call__\n    await route.handle(scope, receive, send)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py\", line 276, in handle\n    await self.app(scope, receive, send)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py\", line 66, in app\n    response = await func(request)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/routing.py\", line 241, in app\n    raw_response = await run_endpoint_function(\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/routing.py\", line 167, in run_endpoint_function\n    return await dependant.call(**values)\n  File \"/Users/gustavopoa/Documents/Langspace/langflow/src/backend/langflow/api/v1/endpoints.py\", line 124, in custom_component_error\n    c = x.get(\"a\")\nAttributeError: 'int' object has no attribute 'get'\n"
+        "traceback": 'Traceback (most recent call last):\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/errors.py", line 162, in __call__\n    await self.app(scope, receive, _send)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/cors.py", line 83, in __call__\n    await self.app(scope, receive, send)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 79, in __call__\n    raise exc\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 68, in __call__\n    await self.app(scope, receive, sender)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 20, in __call__\n    raise e\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 17, in __call__\n    await self.app(scope, receive, send)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py", line 718, in __call__\n    await route.handle(scope, receive, send)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py", line 276, in handle\n    await self.app(scope, receive, send)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py", line 66, in app\n    response = await func(request)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/routing.py", line 241, in app\n    raw_response = await run_endpoint_function(\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/routing.py", line 167, in run_endpoint_function\n    return await dependant.call(**values)\n  File "/Users/gustavopoa/Documents/Langspace/langflow/src/backend/langflow/api/v1/endpoints.py", line 124, in custom_component_error\n    c = x.get("a")\nAttributeError: \'int\' object has no attribute \'get\'\n',
     }
 
     error2 = {
         "detail": "division by zero",
-        "traceback": "Traceback (most recent call last):\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/errors.py\", line 162, in __call__\n    await self.app(scope, receive, _send)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/cors.py\", line 83, in __call__\n    await self.app(scope, receive, send)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/exceptions.py\", line 79, in __call__\n    raise exc\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/exceptions.py\", line 68, in __call__\n    await self.app(scope, receive, sender)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py\", line 20, in __call__\n    raise e\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py\", line 17, in __call__\n    await self.app(scope, receive, send)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py\", line 718, in __call__\n    await route.handle(scope, receive, send)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py\", line 276, in handle\n    await self.app(scope, receive, send)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py\", line 66, in app\n    response = await func(request)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/routing.py\", line 241, in app\n    raw_response = await run_endpoint_function(\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/routing.py\", line 167, in run_endpoint_function\n    return await dependant.call(**values)\n  File \"/Users/gustavopoa/Documents/Langspace/langflow/src/backend/langflow/api/v1/endpoints.py\", line 130, in custom_component_error\n    return 1/0\nZeroDivisionError: division by zero\n"
+        "traceback": 'Traceback (most recent call last):\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/errors.py", line 162, in __call__\n    await self.app(scope, receive, _send)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/cors.py", line 83, in __call__\n    await self.app(scope, receive, send)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 79, in __call__\n    raise exc\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 68, in __call__\n    await self.app(scope, receive, sender)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 20, in __call__\n    raise e\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 17, in __call__\n    await self.app(scope, receive, send)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py", line 718, in __call__\n    await route.handle(scope, receive, send)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py", line 276, in handle\n    await self.app(scope, receive, send)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py", line 66, in app\n    response = await func(request)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/routing.py", line 241, in app\n    raw_response = await run_endpoint_function(\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/routing.py", line 167, in run_endpoint_function\n    return await dependant.call(**values)\n  File "/Users/gustavopoa/Documents/Langspace/langflow/src/backend/langflow/api/v1/endpoints.py", line 130, in custom_component_error\n    return 1/0\nZeroDivisionError: division by zero\n',
     }
 
     error3 = {
         "detail": "name 'CreateObject' is not defined",
-        "traceback": "Traceback (most recent call last):\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/errors.py\", line 162, in __call__\n    await self.app(scope, receive, _send)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/cors.py\", line 83, in __call__\n    await self.app(scope, receive, send)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/exceptions.py\", line 79, in __call__\n    raise exc\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/exceptions.py\", line 68, in __call__\n    await self.app(scope, receive, sender)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py\", line 20, in __call__\n    raise e\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py\", line 17, in __call__\n    await self.app(scope, receive, send)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py\", line 718, in __call__\n    await route.handle(scope, receive, send)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py\", line 276, in handle\n    await self.app(scope, receive, send)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py\", line 66, in app\n    response = await func(request)\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/routing.py\", line 241, in app\n    raw_response = await run_endpoint_function(\n  File \"/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/routing.py\", line 167, in run_endpoint_function\n    return await dependant.call(**values)\n  File \"/Users/gustavopoa/Documents/Langspace/langflow/src/backend/langflow/api/v1/endpoints.py\", line 130, in custom_component_error\n    error3 = CreateObject()\nNameError: name 'CreateObject' is not defined\n"
+        "traceback": 'Traceback (most recent call last):\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/errors.py", line 162, in __call__\n    await self.app(scope, receive, _send)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/cors.py", line 83, in __call__\n    await self.app(scope, receive, send)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 79, in __call__\n    raise exc\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 68, in __call__\n    await self.app(scope, receive, sender)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 20, in __call__\n    raise e\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 17, in __call__\n    await self.app(scope, receive, send)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py", line 718, in __call__\n    await route.handle(scope, receive, send)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py", line 276, in handle\n    await self.app(scope, receive, send)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/starlette/routing.py", line 66, in app\n    response = await func(request)\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/routing.py", line 241, in app\n    raw_response = await run_endpoint_function(\n  File "/Users/gustavopoa/Library/Caches/pypoetry/virtualenvs/langflow-3LyDxlRJ-py3.10/lib/python3.10/site-packages/fastapi/routing.py", line 167, in run_endpoint_function\n    return await dependant.call(**values)\n  File "/Users/gustavopoa/Documents/Langspace/langflow/src/backend/langflow/api/v1/endpoints.py", line 130, in custom_component_error\n    error3 = CreateObject()\nNameError: name \'CreateObject\' is not defined\n',
     }
 
     error = [error1, error2, error3]

src/backend/langflow/api/v1/schemas.py

@@ -1,3 +1,4 @@
+from enum import Enum
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 from langflow.database.models.flow import FlowCreate, FlowRead
@@ -5,6 +6,15 @@ from pydantic import BaseModel, Field, validator
 import json
 
 
+class BuildStatus(Enum):
+    """Status of the build."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    STARTED = "started"
+    IN_PROGRESS = "in_progress"
+
+
 class GraphData(BaseModel):
     """Data inside the exported flow."""
 
@@ -58,8 +68,7 @@ class ChatResponse(ChatMessage):
     @validator("type")
     def validate_message_type(cls, v):
         if v not in ["start", "stream", "end", "error", "info", "file"]:
-            raise ValueError(
-                "type must be start, stream, end, error, info, or file")
+            raise ValueError("type must be start, stream, end, error, info, or file")
         return v
 
 

src/backend/langflow/api/v1/validate.py

@@ -28,49 +28,98 @@ def post_validate_code(code: Code):
 
 
 @router.post("/prompt", status_code=200, response_model=PromptValidationResponse)
-def post_validate_prompt(prompt: ValidatePromptRequest):
+def post_validate_prompt(prompt_request: ValidatePromptRequest):
     try:
-        input_variables = validate_prompt(prompt.template)
-        # Reinitialize custom_fields
-        old_custom_fields = prompt.frontend_node.custom_fields.copy()
-        prompt.frontend_node.custom_fields = []
-        # Add new variables to the template
-        for variable in input_variables:
-            try:
-                template_field = TemplateField(
-                    name=variable,
-                    display_name=variable,
-                    field_type="str",
-                    show=True,
-                    advanced=False,
-                    input_types=["BaseLoader", "BaseOutputParser"],
+        input_variables = validate_prompt(prompt_request.template)
+
+        old_custom_fields = get_old_custom_fields(prompt_request)
+
+        add_new_variables_to_template(input_variables, prompt_request)
+
+        remove_old_variables_from_template(
+            old_custom_fields, input_variables, prompt_request
+        )
+
+        update_input_variables_field(input_variables, prompt_request)
+
+        return PromptValidationResponse(
+            input_variables=input_variables,
+            frontend_node=prompt_request.frontend_node,
+        )
+    except Exception as e:
+        logger.exception(e)
+        raise HTTPException(status_code=500, detail=str(e)) from e
+
+
+def get_old_custom_fields(prompt_request):
+    try:
+        old_custom_fields = prompt_request.frontend_node.custom_fields[
+            prompt_request.name
+        ].copy()
+    except KeyError:
+        old_custom_fields = []
+    prompt_request.frontend_node.custom_fields[prompt_request.name] = []
+    return old_custom_fields
+
+
+def add_new_variables_to_template(input_variables, prompt_request):
+    for variable in input_variables:
+        try:
+            template_field = TemplateField(
+                name=variable,
+                display_name=variable,
+                field_type="str",
+                show=True,
+                advanced=False,
+                multiline=True,
+                input_types=["Document", "BaseOutputParser"],
+            )
+            if variable in prompt_request.frontend_node.template:
+                # Set the new field with the old value
+                template_field.value = prompt_request.frontend_node.template[variable][
+                    "value"
+                ]
+            prompt_request.frontend_node.template[variable] = template_field.to_dict()
+
+            # Check if variable is not already in the list before appending
+            if (
+                variable
+                not in prompt_request.frontend_node.custom_fields[prompt_request.name]
+            ):
+                prompt_request.frontend_node.custom_fields[prompt_request.name].append(
+                    variable
                 )
 
-                prompt.frontend_node.template[variable] = template_field.to_dict()
-                prompt.frontend_node.custom_fields.append(variable)
+        except Exception as exc:
+            logger.exception(exc)
+            raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+
+def remove_old_variables_from_template(
+    old_custom_fields, input_variables, prompt_request
+):
+    for variable in old_custom_fields:
+        if variable not in input_variables:
+            try:
+                # Remove the variable from custom_fields associated with the given name
+                if (
+                    variable
+                    in prompt_request.frontend_node.custom_fields[prompt_request.name]
+                ):
+                    prompt_request.frontend_node.custom_fields[
+                        prompt_request.name
+                    ].remove(variable)
+
+                # Remove the variable from the template
+                prompt_request.frontend_node.template.pop(variable, None)
 
             except Exception as exc:
                 logger.exception(exc)
                 raise HTTPException(status_code=500, detail=str(exc)) from exc
 
-        # Remove variables that are not in the template anymore
-        for variable in old_custom_fields:
-            if variable not in input_variables:
-                try:
-                    prompt.frontend_node.template.pop(variable, None)
-                except Exception as exc:
-                    logger.exception(exc)
-                    raise HTTPException(status_code=500, detail=str(exc)) from exc
 
-        # Now we will set the field "input_variables" to the new list of variables
-        # if it exists
-        if "input_variables" in prompt.frontend_node.template:
-            prompt.frontend_node.template["input_variables"]["value"] = input_variables
-
-        return PromptValidationResponse(
-            input_variables=input_variables,
-            frontend_node=prompt.frontend_node,
-        )
-    except Exception as e:
-        logger.exception(e)
-        raise HTTPException(status_code=500, detail=str(e)) from e
+def update_input_variables_field(input_variables, prompt_request):
+    if "input_variables" in prompt_request.frontend_node.template:
+        prompt_request.frontend_node.template["input_variables"][
+            "value"
+        ] = input_variables

src/backend/langflow/chat/config.py

@@ -0,0 +1,2 @@
+class ChatConfig:
+    streaming: bool = True

src/backend/langflow/config.yaml

@@ -1,147 +1,291 @@
----
 agents:
-  - ZeroShotAgent
-  - JsonAgent
-  - CSVAgent
-  - AgentInitializer
-  - VectorStoreAgent
-  - VectorStoreRouterAgent
-  - SQLAgent
+  ZeroShotAgent:
+    documentation: "https://python.langchain.com/docs/modules/agents/how_to/custom_mrkl_agent"
+  JsonAgent:
+    documentation: "https://python.langchain.com/docs/modules/agents/toolkits/openapi"
+  CSVAgent:
+    documentation: "https://python.langchain.com/docs/modules/agents/toolkits/csv"
+  AgentInitializer:
+    documentation: "https://python.langchain.com/docs/modules/agents/agent_types/"
+  VectorStoreAgent:
+    documentation: ""
+  VectorStoreRouterAgent:
+    documentation: ""
+  SQLAgent:
+    documentation: ""
 chains:
-  - LLMChain
-  - LLMMathChain
-  - LLMCheckerChain
-  - ConversationChain
-  - SeriesCharacterChain
-  - MidJourneyPromptChain
-  - TimeTravelGuideChain
-  - SQLDatabaseChain
-  - RetrievalQA
-  - RetrievalQAWithSourcesChain
-  - ConversationalRetrievalChain
-  - CombineDocsChain
+  LLMChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/foundational/llm_chain"
+  LLMMathChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/additional/llm_math"
+  LLMCheckerChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/additional/llm_checker"
+  ConversationChain:
+    documentation: ""
+  SeriesCharacterChain:
+    documentation: ""
+  MidJourneyPromptChain:
+    documentation: ""
+  TimeTravelGuideChain:
+    documentation: ""
+  SQLDatabaseChain:
+    documentation: ""
+  RetrievalQA:
+    documentation: "https://python.langchain.com/docs/modules/chains/popular/vector_db_qa"
+  RetrievalQAWithSourcesChain:
+    documentation: ""
+  ConversationalRetrievalChain:
+    documentation: "https://python.langchain.com/docs/modules/chains/popular/chat_vector_db"
+  CombineDocsChain:
+    documentation: ""
 documentloaders:
-  - AirbyteJSONLoader
-  - CoNLLULoader
-  - CSVLoader
-  - UnstructuredEmailLoader
-  - EverNoteLoader
-  - FacebookChatLoader
-  - GutenbergLoader
-  - BSHTMLLoader
-  - UnstructuredHTMLLoader
-  # - UnstructuredImageLoader # Issue with Python 3.11 (https://github.com/Unstructured-IO/unstructured-inference/issues/83)
-  - UnstructuredMarkdownLoader
-  - PyPDFLoader
-  - UnstructuredPowerPointLoader
-  - SRTLoader
-  - TelegramChatLoader
-  - TextLoader
-  - UnstructuredWordDocumentLoader
-  - WebBaseLoader
-  - AZLyricsLoader
-  - CollegeConfidentialLoader
-  - HNLoader
-  - IFixitLoader
-  - IMSDbLoader
-  - GitbookLoader
-  - ReadTheDocsLoader
-  - SlackDirectoryLoader
-  - NotionDirectoryLoader
-  - DirectoryLoader
-  - GitLoader
+  AirbyteJSONLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/airbyte_json"
+  CoNLLULoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/conll-u"
+  CSVLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/csv"
+  UnstructuredEmailLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/email"
+  EverNoteLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/evernote"
+  FacebookChatLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/facebook_chat"
+  GutenbergLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/gutenberg"
+  BSHTMLLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/html"
+  UnstructuredHTMLLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/html"
+  UnstructuredMarkdownLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/markdown"
+  PyPDFDirectoryLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/pdf"
+  PyPDFLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/pdf"
+  UnstructuredPowerPointLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/microsoft_powerpoint"
+  SRTLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/subtitle"
+  TelegramChatLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/telegram"
+  TextLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/"
+  UnstructuredWordDocumentLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/microsoft_word"
+  WebBaseLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/web_base"
+  AZLyricsLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/azlyrics"
+  CollegeConfidentialLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/college_confidential"
+  HNLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/hacker_news"
+  IFixitLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/ifixit"
+  IMSDbLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/imsdb"
+  GitbookLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/gitbook"
+  ReadTheDocsLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/readthedocs_documentation"
+  SlackDirectoryLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/slack"
+  NotionDirectoryLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/notion"
+  DirectoryLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/how_to/file_directory"
+  GitLoader:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/git"
 embeddings:
-  - OpenAIEmbeddings
-  - HuggingFaceEmbeddings
-  - CohereEmbeddings
+  OpenAIEmbeddings:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/openai"
+  HuggingFaceEmbeddings:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/sentence_transformers"
+  CohereEmbeddings:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/cohere"
 llms:
-  - OpenAI
-  # - AzureOpenAI
-  # - AzureChatOpenAI
-  - ChatOpenAI
-  - LlamaCpp
-  - CTransformers
-  - Cohere
-  - Anthropic
-  - ChatAnthropic
-  - HuggingFaceHub
+  OpenAI:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/openai"
+  ChatOpenAI:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/integrations/openai"
+  LlamaCpp:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/llamacpp"
+  CTransformers:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/ctransformers"
+  Cohere:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/cohere"
+  Anthropic:
+    documentation: ""
+  ChatAnthropic:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/integrations/anthropic"
+  HuggingFaceHub:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/huggingface_hub"
+  VertexAI:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/llms/integrations/google_vertex_ai_palm"
+  ###
+  # There's a bug in this component deactivating until we get it sorted: _language_models.py", line 804, in send_message
+  #     is_blocked=safety_attributes.get("blocked", False),
+  # AttributeError: 'list' object has no attribute 'get'
+  # ChatVertexAI:
+  #   documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/integrations/google_vertex_ai_palm"
+  ###
 memories:
-  - ConversationBufferMemory
-  - ConversationSummaryMemory
-  - ConversationKGMemory
+  # https://github.com/supabase-community/supabase-py/issues/482
+  # ZepChatMessageHistory:
+  #   documentation: "https://python.langchain.com/docs/modules/memory/integrations/zep_memory"
+  ConversationEntityMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/integrations/entity_memory_with_sqlite"
+  # https://github.com/hwchase17/langchain/issues/6091
+  # SQLiteEntityStore:
+  #   documentation: "https://python.langchain.com/docs/modules/memory/integrations/entity_memory_with_sqlite"
+  PostgresChatMessageHistory:
+    documentation: "https://python.langchain.com/docs/modules/memory/integrations/postgres_chat_message_history"
+  ConversationBufferMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/buffer"
+  ConversationSummaryMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/summary"
+  ConversationKGMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/kg"
+  ConversationBufferWindowMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/buffer_window"
+  VectorStoreRetrieverMemory:
+    documentation: "https://python.langchain.com/docs/modules/memory/how_to/vectorstore_retriever_memory"
+  MongoDBChatMessageHistory:
+    documentation: "https://python.langchain.com/docs/modules/memory/integrations/mongodb_chat_message_history"
+  ChatMessagePromptTemplate:
+    documentation: "https://python.langchain.com/docs/modules/model_io/prompts/prompt_templates/msg_prompt_templates"
+  HumanMessagePromptTemplate:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/how_to/prompts"
+  SystemMessagePromptTemplate:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/how_to/prompts"
+  ChatPromptTemplate:
+    documentation: "https://python.langchain.com/docs/modules/model_io/models/chat/how_to/prompts"
 prompts:
-  - PromptTemplate
-  - FewShotPromptTemplate
-  - ZeroShotPrompt
-  - ChatPromptTemplate
-  - SystemMessagePromptTemplate
-  - AIMessagePromptTemplate
-  - HumanMessagePromptTemplate
-  - ChatMessagePromptTemplate
+  PromptTemplate:
+    documentation: "https://python.langchain.com/docs/modules/model_io/prompts/prompt_templates/"
+  ZeroShotPrompt:
+    documentation: "https://python.langchain.com/docs/modules/agents/how_to/custom_mrkl_agent"
 textsplitters:
-  - CharacterTextSplitter
-  - RecursiveCharacterTextSplitter
-  # - LatexTextSplitter
-  # - PythonCodeTextSplitter
+  CharacterTextSplitter:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_transformers/text_splitters/character_text_splitter"
+  RecursiveCharacterTextSplitter:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/document_transformers/text_splitters/recursive_text_splitter"
 toolkits:
-  - OpenAPIToolkit
-  - JsonToolkit
-  - VectorStoreInfo
-  - VectorStoreRouterToolkit
-  - VectorStoreToolkit
+  OpenAPIToolkit:
+    documentation: ""
+  JsonToolkit:
+    documentation: ""
+  VectorStoreInfo:
+    documentation: ""
+  VectorStoreRouterToolkit:
+    documentation: ""
+  VectorStoreToolkit:
+    documentation: ""
 tools:
-  - Search
-  - PAL-MATH
-  - Calculator
-  - Serper Search
-  - Tool
-  - CustomComponent
-  - PythonFunctionTool
-  - PythonFunction
-  - JsonSpec
-  - News API
-  - TMDB API
-  - Podcast API
-  - QuerySQLDataBaseTool
-  - InfoSQLDatabaseTool
-  - ListSQLDatabaseTool
-  # - QueryCheckerTool
-  - BingSearchRun
-  - GoogleSearchRun
-  - GoogleSearchResults
-  - GoogleSerperRun
-  - JsonListKeysTool
-  - JsonGetValueTool
-  - PythonREPLTool
-  - PythonAstREPLTool
-  - RequestsGetTool
-  - RequestsPostTool
-  - RequestsPatchTool
-  - RequestsPutTool
-  - RequestsDeleteTool
-  - WikipediaQueryRun
-  - WolframAlphaQueryRun
+  Search:
+    documentation: ""
+  PAL-MATH:
+    documentation: ""
+  Calculator:
+    documentation: ""
+  CustomComponent:
+    documentation: ""
+  Serper Search:
+    documentation: ""
+  Tool:
+    documentation: ""
+  PythonFunctionTool:
+    documentation: ""
+  PythonFunction:
+    documentation: ""
+  JsonSpec:
+    documentation: ""
+  News API:
+    documentation: ""
+  TMDB API:
+    documentation: ""
+  Podcast API:
+    documentation: ""
+  QuerySQLDataBaseTool:
+    documentation: ""
+  InfoSQLDatabaseTool:
+    documentation: ""
+  ListSQLDatabaseTool:
+    documentation: ""
+  BingSearchRun:
+    documentation: ""
+  GoogleSearchRun:
+    documentation: ""
+  GoogleSearchResults:
+    documentation: ""
+  GoogleSerperRun:
+    documentation: ""
+  JsonListKeysTool:
+    documentation: ""
+  JsonGetValueTool:
+    documentation: ""
+  PythonREPLTool:
+    documentation: ""
+  PythonAstREPLTool:
+    documentation: ""
+  RequestsGetTool:
+    documentation: ""
+  RequestsPostTool:
+    documentation: ""
+  RequestsPatchTool:
+    documentation: ""
+  RequestsPutTool:
+    documentation: ""
+  RequestsDeleteTool:
+    documentation: ""
+  WikipediaQueryRun:
+    documentation: ""
+  WolframAlphaQueryRun:
+    documentation: ""
 utilities:
-  - BingSearchAPIWrapper
-  - GoogleSearchAPIWrapper
-  - GoogleSerperAPIWrapper
-  - SearxResults
-  - SearxSearchWrapper
-  - SerpAPIWrapper
-  - WikipediaAPIWrapper
-  - WolframAlphaAPIWrapper
-  # - ZapierNLAWrapper
-  - SQLDatabase
+  BingSearchAPIWrapper:
+    documentation: ""
+  GoogleSearchAPIWrapper:
+    documentation: ""
+  GoogleSerperAPIWrapper:
+    documentation: ""
+  SearxResults:
+    documentation: ""
+  SearxSearchWrapper:
+    documentation: ""
+  SerpAPIWrapper:
+    documentation: ""
+  WikipediaAPIWrapper:
+    documentation: ""
+  WolframAlphaAPIWrapper:
+    documentation: ""
+retrievers:
+  MultiQueryRetriever:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/retrievers/how_to/MultiQueryRetriever"
+  # https://github.com/supabase-community/supabase-py/issues/482
+  # ZepRetriever:
+  #   documentation: "https://python.langchain.com/docs/modules/data_connection/retrievers/integrations/zep_memorystore"
 vectorstores:
-  - Chroma
-  - Qdrant
-  - Weaviate
-  - FAISS
-  - Pinecone
-  - SupabaseVectorStore
-  - MongoDBAtlasVectorSearch
+  Chroma:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/chroma"
+  Qdrant:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/qdrant"
+  Weaviate:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/weaviate"
+  FAISS:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/faiss"
+  Pinecone:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/pinecone"
+  SupabaseVectorStore:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/supabase"
+  MongoDBAtlasVectorSearch:
+    documentation: "https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/mongodb_atlas"
 wrappers:
-  - RequestsWrapper
+  RequestsWrapper:
+    documentation: ""
 output_parsers:
-  - StructuredOutputParser
-  - ResponseSchema
+  StructuredOutputParser:
+    documentation: "https://python.langchain.com/docs/modules/model_io/output_parsers/structured"
+  ResponseSchema:
+    documentation: "https://python.langchain.com/docs/modules/model_io/output_parsers/structured"

src/backend/langflow/custom/customs.py

@@ -22,12 +22,15 @@ CUSTOM_NODES = {
     "utilities": {
         "SQLDatabase": frontend_node.agents.SQLDatabaseNode(),
     },
+    "memories": {
+        "PostgresChatMessageHistory": frontend_node.memories.PostgresChatMessageHistoryFrontendNode(),
+    },
     "chains": {
         "SeriesCharacterChain": frontend_node.chains.SeriesCharacterChainNode(),
         "TimeTravelGuideChain": frontend_node.chains.TimeTravelGuideChainNode(),
         "MidJourneyPromptChain": frontend_node.chains.MidJourneyPromptChainNode(),
         "load_qa_chain": frontend_node.chains.CombineDocsChainNode(),
-    }
+    },
 }
 
 

src/backend/langflow/database/base.py

@@ -1,6 +1,6 @@
 from langflow.settings import settings
 from sqlmodel import SQLModel, Session, create_engine
-
+from langflow.utils.logger import logger
 
 if settings.database_url.startswith("sqlite"):
     connect_args = {"check_same_thread": False}
@@ -10,7 +10,9 @@ engine = create_engine(settings.database_url, connect_args=connect_args)
 
 
 def create_db_and_tables():
+    logger.debug("Creating database and tables")
     SQLModel.metadata.create_all(engine)
+    logger.debug("Database and tables created")
 
 
 def get_session():

src/backend/langflow/graph/__init__.py

@@ -14,6 +14,7 @@ from langflow.graph.vertex.types import (
     ToolkitVertex,
     VectorStoreVertex,
     WrapperVertex,
+    RetrieverVertex,
 )
 
 __all__ = [
@@ -32,4 +33,5 @@ __all__ = [
     "ToolkitVertex",
     "VectorStoreVertex",
     "WrapperVertex",
+    "RetrieverVertex",
 ]

src/backend/langflow/graph/edge/base.py

@@ -33,12 +33,7 @@ class Edge:
         # Get what type of input the target node is expecting
 
         self.matched_type = next(
-            (
-                output
-                for output in self.source_types
-                for target_req in self.target_reqs
-                if output in target_req
-            ),
+            (output for output in self.source_types if output in self.target_reqs),
             None,
         )
         no_matched_type = self.matched_type is None
@@ -53,6 +48,16 @@ class Edge:
 
     def __repr__(self) -> str:
         return (
-            f"Edge(source={self.source.id}, target={self.target.id}, valid={self.valid}"
+            f"Edge(source={self.source.id}, target={self.target.id}, target_param={self.target_param}"
             f", matched_type={self.matched_type})"
         )
+
+    def __hash__(self) -> int:
+        return hash(self.__repr__())
+
+    def __eq__(self, __value: object) -> bool:
+        return (
+            self.__repr__() == __value.__repr__()
+            if isinstance(__value, Edge)
+            else False
+        )

src/backend/langflow/graph/graph/constants.py

@@ -13,7 +13,7 @@ from langflow.interface.tools.base import tool_creator
 from langflow.interface.vector_store.base import vectorstore_creator
 from langflow.interface.wrappers.base import wrapper_creator
 from langflow.interface.output_parsers.base import output_parser_creator
-
+from langflow.interface.retrievers.base import retriever_creator
 
 from typing import Dict, Type
 
@@ -33,4 +33,5 @@ VERTEX_TYPE_MAP: Dict[str, Type[Vertex]] = {
     **{t: types.TextSplitterVertex for t in textsplitter_creator.to_list()},
     **{t: types.OutputParserVertex for t in output_parser_creator.to_list()},
     **{t: types.CustomComponentVertex for t in tool_creator.to_list()},
+    **{t: types.RetrieverVertex for t in retriever_creator.to_list()},
 }

src/backend/langflow/graph/vertex/base.py

@@ -95,7 +95,7 @@ class Vertex:
                     if param_key not in params:
                         params[param_key] = []
                     params[param_key].append(edge.source)
-                else:
+                elif edge.target.id == self.id:
                     params[param_key] = edge.source
 
         for key, value in template_dict.items():
@@ -110,7 +110,10 @@ class Vertex:
                 file_path = value.get("file_path")
 
                 params[key] = file_path
-            elif value.get("type") in ["code", "str", "prompt"] and params.get(key) is None:
+            elif (
+                value.get("type") in ["code", "str", "prompt"]
+                and params.get(key) is None
+            ):
                 params[key] = value.get("value")
         # Add _type to params
         self.params = params
@@ -204,7 +207,8 @@ class Vertex:
         return self._built_object
 
     def add_edge(self, edge: "Edge") -> None:
-        self.edges.append(edge)
+        if edge not in self.edges:
+            self.edges.append(edge)
 
     def __repr__(self) -> str:
         return f"Node(id={self.id}, data={self.data})"

src/backend/langflow/graph/vertex/types.py

@@ -91,6 +91,7 @@ class DocumentLoaderVertex(Vertex):
     def _built_object_repr(self):
         # This built_object is a list of documents. Maybe we should
         # show how many documents are in the list?
+
         if self._built_object:
             return f"""{self.vertex_type}({len(self._built_object)} documents)
             Documents: {self._built_object[:3]}..."""
@@ -112,6 +113,11 @@ class MemoryVertex(Vertex):
         super().__init__(data, base_type="memory")
 
 
+class RetrieverVertex(Vertex):
+    def __init__(self, data: Dict):
+        super().__init__(data, base_type="retrievers")
+
+
 class TextSplitterVertex(Vertex):
     def __init__(self, data: Dict):
         super().__init__(data, base_type="textsplitters")
@@ -184,8 +190,7 @@ class PromptVertex(Vertex):
             if "prompt" not in self.params and "messages" not in self.params:
                 for param in prompt_params:
                     prompt_text = self.params[param]
-                    variables = extract_input_variables_from_prompt(
-                        prompt_text)
+                    variables = extract_input_variables_from_prompt(prompt_text)
                     self.params["input_variables"].extend(variables)
                 self.params["input_variables"] = list(
                     set(self.params["input_variables"])

src/backend/langflow/interface/base.py

@@ -8,6 +8,7 @@ from langflow.template.field.base import TemplateField
 from langflow.template.frontend_node.base import FrontendNode
 from langflow.template.template.base import Template
 from langflow.utils.logger import logger
+from langflow.settings import settings
 
 # Assuming necessary imports for Field, Template, and FrontendNode classes
 
@@ -15,12 +16,29 @@ from langflow.utils.logger import logger
 class LangChainTypeCreator(BaseModel, ABC):
     type_name: str
     type_dict: Optional[Dict] = None
+    name_docs_dict: Optional[Dict[str, str]] = None
 
     @property
     def frontend_node_class(self) -> Type[FrontendNode]:
         """The class type of the FrontendNode created in frontend_node."""
         return FrontendNode
 
+    @property
+    def docs_map(self) -> Dict[str, str]:
+        """A dict with the name of the component as key and the documentation link as value."""
+        if self.name_docs_dict is None:
+            try:
+                type_settings = getattr(settings, self.type_name)
+                self.name_docs_dict = {
+                    name: value_dict["documentation"]
+                    for name, value_dict in type_settings.items()
+                }
+            except AttributeError as exc:
+                logger.error(exc)
+
+                self.name_docs_dict = {}
+        return self.name_docs_dict
+
     @property
     @abstractmethod
     def type_to_loader_dict(self) -> Dict:
@@ -83,7 +101,7 @@ class LangChainTypeCreator(BaseModel, ABC):
 
         signature.add_extra_fields()
         signature.add_extra_base_classes()
-
+        signature.set_documentation(self.docs_map.get(name, ""))
         return signature
 
 

src/backend/langflow/interface/custom_lists.py

@@ -10,8 +10,12 @@ from langchain import (
     text_splitter,
 )
 from langchain.agents import agent_toolkits
-from langchain.chat_models import AzureChatOpenAI, ChatOpenAI
-from langchain.chat_models import ChatAnthropic
+from langchain.chat_models import (
+    AzureChatOpenAI,
+    ChatOpenAI,
+    ChatVertexAI,
+    ChatAnthropic,
+)
 
 from langflow.interface.importing.utils import import_class
 from langflow.interface.agents.custom import CUSTOM_AGENTS
@@ -22,6 +26,7 @@ llm_type_to_cls_dict = llms.type_to_cls_dict
 llm_type_to_cls_dict["anthropic-chat"] = ChatAnthropic  # type: ignore
 llm_type_to_cls_dict["azure-chat"] = AzureChatOpenAI  # type: ignore
 llm_type_to_cls_dict["openai-chat"] = ChatOpenAI  # type: ignore
+llm_type_to_cls_dict["vertexai-chat"] = ChatVertexAI  # type: ignore
 
 
 # Toolkits

src/backend/langflow/interface/importing/utils.py

@@ -29,8 +29,7 @@ def import_module(module_path: str) -> Any:
 def import_by_type(_type: str, name: str) -> Any:
     """Import class by type and name"""
     if _type is None:
-        raise ValueError(
-            f"Type cannot be None. Check if {name} is in the config file.")
+        raise ValueError(f"Type cannot be None. Check if {name} is in the config file.")
     func_dict = {
         "agents": import_agent,
         "prompts": import_prompt,
@@ -46,6 +45,7 @@ def import_by_type(_type: str, name: str) -> Any:
         "textsplitters": import_textsplitter,
         "utilities": import_utility,
         "output_parsers": import_output_parser,
+        "retrievers": import_retriever,
     }
     if _type == "llms":
         key = "chat" if "chat" in name.lower() else "llm"
@@ -66,6 +66,11 @@ def import_chat_llm(llm: str) -> BaseChatModel:
     return import_class(f"langchain.chat_models.{llm}")
 
 
+def import_retriever(retriever: str) -> Any:
+    """Import retriever from retriever name"""
+    return import_module(f"from langchain.retrievers import {retriever}")
+
+
 def import_memory(memory: str) -> Any:
     """Import memory from memory name"""
     return import_module(f"from langchain.memory import {memory}")

src/backend/langflow/interface/initialize/llm.py

@@ -0,0 +1,9 @@
+def initialize_vertexai(class_object, params):
+    if credentials_path := params.get("credentials"):
+        from google.oauth2 import service_account  # type: ignore
+
+        credentials_object = service_account.Credentials.from_service_account_file(
+            filename=credentials_path
+        )
+        params["credentials"] = credentials_object
+    return class_object(**params)

src/backend/langflow/interface/initialize/loading.py

@@ -1,11 +1,13 @@
 import json
-from typing import Any, Callable, Dict, List, Sequence
+from typing import Any, Callable, Dict, List, Sequence, Type
 
 from langchain.agents import ZeroShotAgent
 from langchain.agents import agent as agent_module
 from langchain.agents.agent import AgentExecutor
 from langchain.agents.agent_toolkits.base import BaseToolkit
 from langchain.agents.tools import BaseTool
+from langflow.interface.initialize.llm import initialize_vertexai
+
 from langflow.interface.initialize.vector_store import vecstore_initializer
 
 from langchain.schema import Document, BaseOutputParser
@@ -14,14 +16,18 @@ from pydantic import ValidationError
 from langflow.interface.importing.utils import (
     get_function,
     import_by_type,
-    get_function_custom
+    get_function_custom,
 )
 from langflow.interface.custom_lists import CUSTOM_NODES
 from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.chains.base import chain_creator
 from langflow.interface.output_parsers.base import output_parser_creator
+from langflow.interface.retrievers.base import retriever_creator
 from langflow.interface.utils import load_file_into_dict
 from langflow.utils import validate
+from langchain.chains.base import Chain
+from langchain.vectorstores.base import VectorStore
+from langchain.document_loaders.base import BaseLoader
 
 
 def instantiate_class(node_type: str, base_type: str, params: Dict) -> Any:
@@ -49,8 +55,8 @@ def convert_params_to_sets(params):
 
 def convert_kwargs(params):
     # if *kwargs are passed as a string, convert to dict
-    # first find any key that has kwargs in it
-    kwargs_keys = [key for key in params.keys() if "kwargs" in key]
+    # first find any key that has kwargs or config in it
+    kwargs_keys = [key for key in params.keys() if "kwargs" in key or "config" in key]
     for key in kwargs_keys:
         if isinstance(params[key], str):
             params[key] = json.loads(params[key])
@@ -80,6 +86,12 @@ def instantiate_based_on_type(class_object, base_type, node_type, params):
         return instantiate_chains(node_type, class_object, params)
     elif base_type == "output_parsers":
         return instantiate_output_parser(node_type, class_object, params)
+    elif base_type == "llms":
+        return instantiate_llm(node_type, class_object, params)
+    elif base_type == "retrievers":
+        return instantiate_retriever(node_type, class_object, params)
+    elif base_type == "memory":
+        return instantiate_memory(node_type, class_object, params)
     else:
         return class_object(**params)
 
@@ -93,7 +105,47 @@ def instantiate_output_parser(node_type, class_object, params):
     return class_object(**params)
 
 
-def instantiate_chains(node_type, class_object, params):
+def instantiate_llm(node_type, class_object, params: Dict):
+    # This is a workaround so JinaChat works until streaming is implemented
+    # if "openai_api_base" in params and "jina" in params["openai_api_base"]:
+    # False if condition is True
+    if node_type == "VertexAI":
+        return initialize_vertexai(class_object=class_object, params=params)
+    return class_object(**params)
+
+
+def instantiate_memory(node_type, class_object, params):
+    try:
+        if "retriever" in params and hasattr(params["retriever"], "as_retriever"):
+            params["retriever"] = params["retriever"].as_retriever()
+        return class_object(**params)
+    # I want to catch a specific attribute error that happens
+    # when the object does not have a cursor attribute
+    except Exception as exc:
+        if "object has no attribute 'cursor'" in str(
+            exc
+        ) or 'object has no field "conn"' in str(exc):
+            raise AttributeError(
+                (
+                    "Failed to build connection to database."
+                    f" Please check your connection string and try again. Error: {exc}"
+                )
+            ) from exc
+        raise exc
+
+
+def instantiate_retriever(node_type, class_object, params):
+    if "retriever" in params and hasattr(params["retriever"], "as_retriever"):
+        params["retriever"] = params["retriever"].as_retriever()
+    if node_type in retriever_creator.from_method_nodes:
+        method = retriever_creator.from_method_nodes[node_type]
+        if class_method := getattr(class_object, method, None):
+            return class_method(**params)
+        raise ValueError(f"Method {method} not found in {class_object}")
+    return class_object(**params)
+
+
+def instantiate_chains(node_type, class_object: Type[Chain], params: Dict):
     if "retriever" in params and hasattr(params["retriever"], "as_retriever"):
         params["retriever"] = params["retriever"].as_retriever()
     if node_type in chain_creator.from_method_nodes:
@@ -105,11 +157,11 @@ def instantiate_chains(node_type, class_object, params):
     return class_object(**params)
 
 
-def instantiate_agent(class_object, params):
+def instantiate_agent(class_object: Type[agent_module.Agent], params: Dict):
     return load_agent_executor(class_object, params)
 
 
-def instantiate_prompt(node_type, class_object, params):
+def instantiate_prompt(node_type, class_object, params: Dict):
     if node_type == "ZeroShotPrompt":
         if "tools" not in params:
             params["tools"] = []
@@ -170,7 +222,7 @@ def instantiate_prompt(node_type, class_object, params):
     return prompt, format_kwargs
 
 
-def instantiate_tool(node_type, class_object, params):
+def instantiate_tool(node_type, class_object: Type[BaseTool], params: Dict):
     if node_type == "JsonSpec":
         params["dict_"] = load_file_into_dict(params.pop("path"))
         return class_object(**params)
@@ -191,7 +243,7 @@ def instantiate_tool(node_type, class_object, params):
     return class_object(**params)
 
 
-def instantiate_toolkit(node_type, class_object, params):
+def instantiate_toolkit(node_type, class_object: Type[BaseToolkit], params: Dict):
     loaded_toolkit = class_object(**params)
     # Commenting this out for now to use toolkits as normal tools
     # if toolkits_creator.has_create_function(node_type):
@@ -201,7 +253,7 @@ def instantiate_toolkit(node_type, class_object, params):
     return loaded_toolkit
 
 
-def instantiate_embedding(class_object, params):
+def instantiate_embedding(class_object, params: Dict):
     params.pop("model", None)
     params.pop("headers", None)
     try:
@@ -215,7 +267,7 @@ def instantiate_embedding(class_object, params):
         return class_object(**params)
 
 
-def instantiate_vectorstore(class_object, params):
+def instantiate_vectorstore(class_object: Type[VectorStore], params: Dict):
     search_kwargs = params.pop("search_kwargs", {})
     if initializer := vecstore_initializer.get(class_object.__name__):
         vecstore = initializer(class_object, params)
@@ -231,48 +283,68 @@ def instantiate_vectorstore(class_object, params):
     return vecstore
 
 
-def instantiate_documentloader(class_object, params):
+def instantiate_documentloader(class_object: Type[BaseLoader], params: Dict):
     if "file_filter" in params:
         # file_filter will be a string but we need a function
         # that will be used to filter the files using file_filter
         # like lambda x: x.endswith(".txt") but as we don't know
         # anything besides the string, we will simply check if the string is
         # in x and if it is, we will return True
-        file_filter = params.pop("file_filter", None)
+        file_filter = params.pop("file_filter")
         extensions = file_filter.split(",")
         params["file_filter"] = lambda x: any(
             extension.strip() in x for extension in extensions
         )
     metadata = params.pop("metadata", None)
+    if metadata and isinstance(metadata, str):
+        try:
+            metadata = json.loads(metadata)
+        except json.JSONDecodeError as exc:
+            raise ValueError(
+                "The metadata you provided is not a valid JSON string."
+            ) from exc
     docs = class_object(**params).load()
+    # Now if metadata is an empty dict, we will not add it to the documents
     if metadata:
-        if isinstance(metadata, str):
-            try:
-                metadata = json.loads(metadata)
-            except json.JSONDecodeError as exc:
-                raise ValueError(
-                    "The metadata you provided is not a valid JSON string."
-                ) from exc
-
         for doc in docs:
-            doc.metadata = metadata
+            # If the document already has metadata, we will not overwrite it
+            if not doc.metadata:
+                doc.metadata = metadata
+            else:
+                doc.metadata.update(metadata)
 
     return docs
 
 
-def instantiate_textsplitter(class_object, params):
+def instantiate_textsplitter(
+    class_object,
+    params: Dict,
+):
     try:
         documents = params.pop("documents")
-    except KeyError as e:
+    except KeyError as exc:
         raise ValueError(
             "The source you provided did not load correctly or was empty."
             "Try changing the chunk_size of the Text Splitter."
-        ) from e
-    text_splitter = class_object(**params)
+        ) from exc
+
+    if (
+        "separator_type" in params and params["separator_type"] == "Text"
+    ) or "separator_type" not in params:
+        params.pop("separator_type", None)
+        text_splitter = class_object(**params)
+    else:
+        from langchain.text_splitter import Language
+
+        language = params.pop("separator_type", None)
+        params["language"] = Language(language)
+        params.pop("separators", None)
+
+        text_splitter = class_object.from_language(**params)
     return text_splitter.split_documents(documents)
 
 
-def instantiate_utility(node_type, class_object, params):
+def instantiate_utility(node_type, class_object, params: Dict):
     if node_type == "SQLDatabase":
         return class_object.from_uri(params.pop("uri"))
     return class_object(**params)
@@ -289,8 +361,7 @@ def replace_zero_shot_prompt_with_prompt_template(nodes):
                 if tool["type"] != "chatOutputNode"
                 and "Tool" in tool["data"]["node"]["base_classes"]
             ]
-            node["data"] = build_prompt_template(
-                prompt=node["data"], tools=tools)
+            node["data"] = build_prompt_template(prompt=node["data"], tools=tools)
             break
     return nodes
 
@@ -299,6 +370,8 @@ def load_agent_executor(agent_class: type[agent_module.Agent], params, **kwargs)
     """Load agent executor from agent class, tools and chain"""
     allowed_tools: Sequence[BaseTool] = params.get("allowed_tools", [])
     llm_chain = params["llm_chain"]
+    # agent has hidden args for memory. might need to be support
+    # memory = params["memory"]
     # if allowed_tools is not a list or set, make it a list
     if not isinstance(allowed_tools, (list, set)) and isinstance(
         allowed_tools, BaseTool
@@ -307,11 +380,11 @@ def load_agent_executor(agent_class: type[agent_module.Agent], params, **kwargs)
     tool_names = [tool.name for tool in allowed_tools]
     # Agent class requires an output_parser but Agent classes
     # have a default output_parser.
-    agent = agent_class(allowed_tools=tool_names,
-                        llm_chain=llm_chain)  # type: ignore
+    agent = agent_class(allowed_tools=tool_names, llm_chain=llm_chain)  # type: ignore
     return AgentExecutor.from_agent_and_tools(
         agent=agent,
         tools=allowed_tools,
+        # memory=memory,
         **kwargs,
     )
 

src/backend/langflow/interface/listing.py

@@ -12,6 +12,7 @@ from langflow.interface.utilities.base import utility_creator
 from langflow.interface.vector_store.base import vectorstore_creator
 from langflow.interface.wrappers.base import wrapper_creator
 from langflow.interface.output_parsers.base import output_parser_creator
+from langflow.interface.retrievers.base import retriever_creator
 
 
 def get_type_dict():
@@ -30,6 +31,7 @@ def get_type_dict():
         "textSplitters": textsplitter_creator.to_list(),
         "utilities": utility_creator.to_list(),
         "outputParsers": output_parser_creator.to_list(),
+        "retrievers": retriever_creator.to_list(),
     }
 
 

src/backend/langflow/interface/memories/base.py

@@ -6,12 +6,18 @@ from langflow.settings import settings
 from langflow.template.frontend_node.base import FrontendNode
 from langflow.template.frontend_node.memories import MemoryFrontendNode
 from langflow.utils.logger import logger
-from langflow.utils.util import build_template_from_class
+from langflow.utils.util import build_template_from_class, build_template_from_method
+from langflow.custom.customs import get_custom_nodes
 
 
 class MemoryCreator(LangChainTypeCreator):
     type_name: str = "memories"
 
+    from_method_nodes = {
+        "ZepChatMessageHistory": "__init__",
+        "SQLiteEntityStore": "__init__",
+    }
+
     @property
     def frontend_node_class(self) -> Type[FrontendNode]:
         """The class type of the FrontendNode created in frontend_node."""
@@ -26,6 +32,14 @@ class MemoryCreator(LangChainTypeCreator):
     def get_signature(self, name: str) -> Optional[Dict]:
         """Get the signature of a memory."""
         try:
+            if name in get_custom_nodes(self.type_name).keys():
+                return get_custom_nodes(self.type_name)[name]
+            elif name in self.from_method_nodes:
+                return build_template_from_method(
+                    name,
+                    type_to_cls_dict=memory_type_to_cls_dict,
+                    method_name=self.from_method_nodes[name],
+                )
             return build_template_from_class(name, memory_type_to_cls_dict)
         except ValueError as exc:
             raise ValueError("Memory not found") from exc

src/backend/langflow/interface/retrievers/base.py

@@ -0,0 +1,58 @@
+from typing import Any, Dict, List, Optional, Type
+
+from langchain import retrievers
+
+from langflow.interface.base import LangChainTypeCreator
+from langflow.interface.importing.utils import import_class
+from langflow.settings import settings
+from langflow.template.frontend_node.retrievers import RetrieverFrontendNode
+from langflow.utils.logger import logger
+from langflow.utils.util import build_template_from_method, build_template_from_class
+
+
+class RetrieverCreator(LangChainTypeCreator):
+    type_name: str = "retrievers"
+
+    from_method_nodes = {"MultiQueryRetriever": "from_llm", "ZepRetriever": "__init__"}
+
+    @property
+    def frontend_node_class(self) -> Type[RetrieverFrontendNode]:
+        return RetrieverFrontendNode
+
+    @property
+    def type_to_loader_dict(self) -> Dict:
+        if self.type_dict is None:
+            self.type_dict: dict[str, Any] = {
+                retriever_name: import_class(f"langchain.retrievers.{retriever_name}")
+                for retriever_name in retrievers.__all__
+            }
+        return self.type_dict
+
+    def get_signature(self, name: str) -> Optional[Dict]:
+        """Get the signature of an embedding."""
+        try:
+            if name in self.from_method_nodes:
+                return build_template_from_method(
+                    name,
+                    type_to_cls_dict=self.type_to_loader_dict,
+                    method_name=self.from_method_nodes[name],
+                )
+            else:
+                return build_template_from_class(
+                    name, type_to_cls_dict=self.type_to_loader_dict
+                )
+        except ValueError as exc:
+            raise ValueError(f"Retriever {name} not found") from exc
+        except AttributeError as exc:
+            logger.error(f"Retriever {name} not loaded: {exc}")
+            return None
+
+    def to_list(self) -> List[str]:
+        return [
+            retriever
+            for retriever in self.type_to_loader_dict.keys()
+            if retriever in settings.retrievers or settings.dev
+        ]
+
+
+retriever_creator = RetrieverCreator()

src/backend/langflow/interface/run.py

@@ -79,6 +79,10 @@ def update_memory_keys(langchain_object, possible_new_mem_key):
         if key not in [langchain_object.memory.memory_key, possible_new_mem_key]
     ][0]
 
-    langchain_object.memory.input_key = input_key
-    langchain_object.memory.output_key = output_key
-    langchain_object.memory.memory_key = possible_new_mem_key
+    keys = [input_key, output_key, possible_new_mem_key]
+    attrs = ["input_key", "output_key", "memory_key"]
+    for key, attr in zip(keys, attrs):
+        try:
+            setattr(langchain_object.memory, attr, key)
+        except ValueError as exc:
+            logger.debug(f"{langchain_object.memory} has no attribute {attr} ({exc})")

src/backend/langflow/interface/types.py

@@ -16,6 +16,7 @@ from langflow.interface.tools.custom import CustomComponent
 
 from langflow.template.field.base import TemplateField
 from langflow.template.frontend_node.tools import CustomComponentNode
+from langflow.interface.retrievers.base import retriever_creator
 
 
 def get_type_list():
@@ -50,6 +51,7 @@ def build_langchain_types_dict():  # sourcery skip: dict-assign-update-to-union
         textsplitter_creator,
         utility_creator,
         output_parser_creator,
+        retriever_creator,
     ]
 
     all_types = {}
@@ -63,14 +65,10 @@ def build_langchain_types_dict():  # sourcery skip: dict-assign-update-to-union
 # TODO: Move to correct place
 def add_new_custom_field(template, field_name: str, field_type: str):
     new_field = TemplateField(
-        name=field_name,
-        field_type=field_type,
-        show=True,
-        required=True,
-        advanced=False
+        name=field_name, field_type=field_type, show=True, required=True, advanced=False
     )
-    template.get('template')[field_name] = new_field.to_dict()
-    template.get('custom_fields').append(field_name)
+    template.get("template")[field_name] = new_field.to_dict()
+    template.get("custom_fields")[field_name] = None
 
     return template
 
@@ -90,10 +88,10 @@ def add_code_field(template, raw_code):
             "name": "code",
             "advanced": False,
             "type": "code",
-            "list": False
+            "list": False,
         }
     }
-    template.get('template')['code'] = code_field.get('code')
+    template.get("template")["code"] = code_field.get("code")
 
     return template
 
@@ -110,29 +108,23 @@ def build_langchain_template_custom_component(extractor: CustomComponent):
         def_field = extra_field[0]
         def_type = extra_field[1]
 
-        if def_field != 'self':
+        if def_field != "self":
             # TODO: Validate type - if is possible to render into frontend
             if not def_type:
-                def_type = 'str'
+                def_type = "str"
 
-            template = add_new_custom_field(
-                template,
-                def_field,
-                def_type
-            )
+            template = add_new_custom_field(template, def_field, def_type)
 
-    template = add_code_field(
-        template,
-        raw_code
-    )
-
-    # TODO: Build a vertex - loading.py
+    template = add_code_field(template, raw_code)
 
     # TODO: Get base classes from "return_type" and add to template.base_classes
-    template.get('base_classes').append("ConversationChain")
-    template.get('base_classes').append("LLMChain")
-    template.get('base_classes').append("Chain")
-    template.get('base_classes').append("Serializable")
-    template.get('base_classes').append("function")
+    template.get("base_classes").append("ConversationChain")
+    template.get("base_classes").append("LLMChain")
+    template.get("base_classes").append("Chain")
+    template.get("base_classes").append("Serializable")
+    template.get("base_classes").append("function")
 
     return template
+
+
+langchain_types_dict = build_langchain_types_dict()

src/backend/langflow/interface/utils.py

@@ -4,10 +4,12 @@ import os
 from io import BytesIO
 import re
 
+
 import yaml
 from langchain.base_language import BaseLanguageModel
 from PIL.Image import Image
 from langflow.utils.logger import logger
+from langflow.chat.config import ChatConfig
 
 
 def load_file_into_dict(file_path: str) -> dict:
@@ -49,9 +51,9 @@ def try_setting_streaming_options(langchain_object, websocket):
 
     if isinstance(llm, BaseLanguageModel):
         if hasattr(llm, "streaming") and isinstance(llm.streaming, bool):
-            llm.streaming = True
+            llm.streaming = ChatConfig.streaming
         elif hasattr(llm, "stream") and isinstance(llm.stream, bool):
-            llm.stream = True
+            llm.stream = ChatConfig.streaming
 
     return langchain_object
 

src/backend/langflow/main.py

@@ -1,5 +1,9 @@
+from pathlib import Path
+from typing import Optional
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
 
 from langflow.api import router
 from langflow.database.base import create_db_and_tables
@@ -33,6 +37,42 @@ def create_app():
     return app
 
 
+def setup_static_files(app: FastAPI, static_files_dir: Path):
+    """
+    Setup the static files directory.
+    Args:
+        app (FastAPI): FastAPI app.
+        path (str): Path to the static files directory.
+    """
+    app.mount(
+        "/",
+        StaticFiles(directory=static_files_dir, html=True),
+        name="static",
+    )
+
+    @app.exception_handler(404)
+    async def custom_404_handler(request, __):
+        path = static_files_dir / "index.html"
+
+        if not path.exists():
+            raise RuntimeError(f"File at path {path} does not exist.")
+        return FileResponse(path)
+
+
+# app = create_app()
+# setup_static_files(app, static_files_dir)
+def setup_app(static_files_dir: Optional[Path]) -> FastAPI:
+    """Setup the FastAPI app."""
+    # get the directory of the current file
+    if not static_files_dir:
+        frontend_path = Path(__file__).parent
+        static_files_dir = frontend_path / "frontend"
+
+    app = create_app()
+    setup_static_files(app, static_files_dir)
+    return app
+
+
 app = create_app()
 
 

src/backend/langflow/processing/base.py

@@ -40,5 +40,6 @@ async def get_result_and_steps(langchain_object, inputs: dict, **kwargs):
         )
         thought = format_actions(intermediate_steps) if intermediate_steps else ""
     except Exception as exc:
+        logger.exception(exc)
         raise ValueError(f"Error: {str(exc)}") from exc
     return result, thought

src/backend/langflow/processing/process.py

@@ -115,6 +115,10 @@ def process_graph_cached(data_graph: Dict[str, Any], inputs: Optional[dict] = No
     elif isinstance(langchain_object, VectorStore):
         class_name = langchain_object.__class__.__name__
         result = {"message": f"Processed {class_name} successfully"}
+    else:
+        raise ValueError(
+            f"Unknown langchain_object type: {type(langchain_object).__name__}"
+        )
     return result
 
 

src/backend/langflow/settings.py

@@ -1,30 +1,42 @@
 import os
-from typing import List
 
 import yaml
 from pydantic import BaseSettings, root_validator
+from langflow.utils.logger import logger
 
 
 class Settings(BaseSettings):
-    chains: List[str] = []
-    agents: List[str] = []
-    prompts: List[str] = []
-    llms: List[str] = []
-    tools: List[str] = []
-    memories: List[str] = []
-    embeddings: List[str] = []
-    vectorstores: List[str] = []
-    documentloaders: List[str] = []
-    wrappers: List[str] = []
-    toolkits: List[str] = []
-    textsplitters: List[str] = []
-    utilities: List[str] = []
-    output_parsers: List[str] = []
+    chains: dict = {}
+    agents: dict = {}
+    prompts: dict = {}
+    llms: dict = {}
+    tools: dict = {}
+    memories: dict = {}
+    embeddings: dict = {}
+    vectorstores: dict = {}
+    documentloaders: dict = {}
+    wrappers: dict = {}
+    retrievers: dict = {}
+    toolkits: dict = {}
+    textsplitters: dict = {}
+    utilities: dict = {}
+    output_parsers: dict = {}
     dev: bool = False
-    database_url: str = "sqlite:///./langflow.db"
+    database_url: str
     cache: str = "InMemoryCache"
     remove_api_keys: bool = False
 
+    @root_validator(pre=True)
+    def set_database_url(cls, values):
+        if "database_url" not in values:
+            logger.debug("No database_url provided, trying DATABASE_URL env variable")
+            if database_url := os.getenv("DATABASE_URL"):
+                values["database_url"] = database_url
+            else:
+                logger.debug("No DATABASE_URL env variable, using sqlite database")
+                values["database_url"] = "sqlite:///./langflow.db"
+        return values
+
     class Config:
         validate_assignment = True
         extra = "ignore"
@@ -39,20 +51,21 @@ class Settings(BaseSettings):
 
     def update_from_yaml(self, file_path: str, dev: bool = False):
         new_settings = load_settings_from_yaml(file_path)
-        self.chains = new_settings.chains or []
-        self.agents = new_settings.agents or []
-        self.prompts = new_settings.prompts or []
-        self.llms = new_settings.llms or []
-        self.tools = new_settings.tools or []
-        self.memories = new_settings.memories or []
-        self.wrappers = new_settings.wrappers or []
-        self.toolkits = new_settings.toolkits or []
-        self.textsplitters = new_settings.textsplitters or []
-        self.utilities = new_settings.utilities or []
-        self.embeddings = new_settings.embeddings or []
-        self.vectorstores = new_settings.vectorstores or []
-        self.documentloaders = new_settings.documentloaders or []
-        self.output_parsers = new_settings.output_parsers or []
+        self.chains = new_settings.chains or {}
+        self.agents = new_settings.agents or {}
+        self.prompts = new_settings.prompts or {}
+        self.llms = new_settings.llms or {}
+        self.tools = new_settings.tools or {}
+        self.memories = new_settings.memories or {}
+        self.wrappers = new_settings.wrappers or {}
+        self.toolkits = new_settings.toolkits or {}
+        self.textsplitters = new_settings.textsplitters or {}
+        self.utilities = new_settings.utilities or {}
+        self.embeddings = new_settings.embeddings or {}
+        self.vectorstores = new_settings.vectorstores or {}
+        self.documentloaders = new_settings.documentloaders or {}
+        self.retrievers = new_settings.retrievers or {}
+        self.output_parsers = new_settings.output_parsers or {}
         self.dev = dev
 
     def update_settings(self, **kwargs):

src/backend/langflow/template/field/base.py

@@ -23,6 +23,7 @@ class TemplateFieldCreator(BaseModel, ABC):
     advanced: bool = False
     input_types: list[str] = []
     dynamic: bool = False
+    info: Optional[str] = ""
 
     def to_dict(self):
         result = self.dict()

src/backend/langflow/template/frontend_node/base.py

@@ -1,14 +1,44 @@
+from collections import defaultdict
 import re
 from typing import List, Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
+from langflow.template.frontend_node.formatter import field_formatters
 from langflow.template.frontend_node.constants import FORCE_SHOW_FIELDS
 from langflow.template.field.base import TemplateField
 from langflow.template.template.base import Template
 from langflow.utils import constants
 
-CLASSES_TO_REMOVE = ["Serializable", "BaseModel"]
+CLASSES_TO_REMOVE = ["Serializable", "BaseModel", "object"]
+
+
+class FieldFormatters(BaseModel):
+    formatters = {
+        "openai_api_key": field_formatters.OpenAIAPIKeyFormatter(),
+    }
+    base_formatters = {
+        "kwargs": field_formatters.KwargsFormatter(),
+        "optional": field_formatters.RemoveOptionalFormatter(),
+        "list": field_formatters.ListTypeFormatter(),
+        "dict": field_formatters.DictTypeFormatter(),
+        "union": field_formatters.UnionTypeFormatter(),
+        "multiline": field_formatters.MultilineFieldFormatter(),
+        "show": field_formatters.ShowFieldFormatter(),
+        "password": field_formatters.PasswordFieldFormatter(),
+        "default": field_formatters.DefaultValueFormatter(),
+        "headers": field_formatters.HeadersDefaultValueFormatter(),
+        "dict_code_file": field_formatters.DictCodeFileFormatter(),
+        "model_fields": field_formatters.ModelSpecificFieldFormatter(),
+    }
+
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        for key, formatter in self.base_formatters.items():
+            formatter.format(field, name)
+
+        for key, formatter in self.formatters.items():
+            if key == field.name:
+                formatter.format(field, name)
 
 
 class FrontendNode(BaseModel):
@@ -17,7 +47,28 @@ class FrontendNode(BaseModel):
     base_classes: List[str]
     name: str = ""
     display_name: str = ""
-    custom_fields: List[str] = []
+    documentation: str = ""
+    custom_fields: defaultdict = defaultdict(list)
+    output_types: List[str] = []
+    field_formatters: FieldFormatters = Field(default_factory=FieldFormatters)
+
+    def process_base_classes(self) -> None:
+        """Removes unwanted base classes from the list of base classes."""
+        self.base_classes = [
+            base_class
+            for base_class in self.base_classes
+            if base_class not in CLASSES_TO_REMOVE
+        ]
+
+    # field formatters is an instance attribute but it is not used in the class
+    # so we need to create a method to get it
+    @staticmethod
+    def get_field_formatters() -> FieldFormatters:
+        return FieldFormatters()
+
+    def set_documentation(self, documentation: str) -> None:
+        """Sets the documentation of the frontend node."""
+        self.documentation = documentation
 
     def process_base_classes(self) -> None:
         """Removes unwanted base classes from the list of base classes."""
@@ -37,6 +88,8 @@ class FrontendNode(BaseModel):
                 "base_classes": self.base_classes,
                 "display_name": self.display_name or self.name,
                 "custom_fields": self.custom_fields,
+                "output_types": self.output_types,
+                "documentation": self.documentation,
             },
         }
 
@@ -49,33 +102,8 @@ class FrontendNode(BaseModel):
     @staticmethod
     def format_field(field: TemplateField, name: Optional[str] = None) -> None:
         """Formats a given field based on its attributes and value."""
-        SPECIAL_FIELD_HANDLERS = {
-            "allowed_tools": lambda field: "Tool",
-            "max_value_length": lambda field: "int",
-        }
 
-        key = field.name
-        value = field.to_dict()
-        _type = value["type"]
-
-        _type = FrontendNode.remove_optional(_type)
-        _type, is_list = FrontendNode.check_for_list_type(_type)
-        field.is_list = is_list or field.is_list
-        _type = FrontendNode.replace_mapping_with_dict(_type)
-        _type = FrontendNode.handle_union_type(_type)
-
-        field.field_type = FrontendNode.handle_special_field(
-            field, key, _type, SPECIAL_FIELD_HANDLERS
-        )
-        field.field_type = FrontendNode.handle_dict_type(field, _type)
-        field.show = FrontendNode.should_show_field(key, field.required)
-        field.password = FrontendNode.should_be_password(key, field.show)
-        field.multiline = FrontendNode.should_be_multiline(key)
-
-        FrontendNode.replace_default_value(field, value)
-        FrontendNode.handle_specific_field_values(field, key, name)
-        FrontendNode.handle_kwargs_field(field)
-        FrontendNode.handle_api_key_field(field, key)
+        FrontendNode.get_field_formatters().format(field, name)
 
     @staticmethod
     def remove_optional(_type: str) -> str:
@@ -198,8 +226,7 @@ class FrontendNode(BaseModel):
     def should_be_password(key: str, show: bool) -> bool:
         """Determines whether the field should be a password field."""
         return (
-            any(text in key.lower()
-                for text in {"password", "token", "api", "key"})
+            any(text in key.lower() for text in {"password", "token", "api", "key"})
             and show
         )
 

src/backend/langflow/template/frontend_node/chains.py

@@ -49,6 +49,10 @@ class ChainFrontendNode(FrontendNode):
     def format_field(field: TemplateField, name: Optional[str] = None) -> None:
         FrontendNode.format_field(field, name)
 
+        if "name" == "RetrievalQA" and field.name == "memory":
+            field.show = False
+            field.required = False
+
         field.advanced = False
         if "key" in field.name:
             field.password = False

src/backend/langflow/template/frontend_node/constants.py

@@ -32,3 +32,29 @@ You are a good listener and you can talk about anything.
 HUMAN_PROMPT = "{input}"
 
 QA_CHAIN_TYPES = ["stuff", "map_reduce", "map_rerank", "refine"]
+
+CTRANSFORMERS_DEFAULT_CONFIG = {
+    "top_k": 40,
+    "top_p": 0.95,
+    "temperature": 0.8,
+    "repetition_penalty": 1.1,
+    "last_n_tokens": 64,
+    "seed": -1,
+    "max_new_tokens": 256,
+    "stop": None,
+    "stream": False,
+    "reset": True,
+    "batch_size": 8,
+    "threads": -1,
+    "context_length": -1,
+    "gpu_layers": 0,
+}
+
+# This variable is used to tell the user
+# that it can be changed to use other APIs
+# like Prem and LocalAI
+OPENAI_API_BASE_INFO = """
+The base URL of the OpenAI API. Defaults to https://api.openai.com/v1.
+
+You can change this to use other APIs like JinaChat, LocalAI and Prem.
+"""

src/backend/langflow/template/frontend_node/documentloaders.py

@@ -19,6 +19,10 @@ def build_file_field(
 
 
 class DocumentLoaderFrontNode(FrontendNode):
+    def add_extra_base_classes(self) -> None:
+        self.base_classes = ["Document"]
+        self.output_types = ["Document"]
+
     file_path_templates = {
         "AirbyteJSONLoader": build_file_field(suffixes=[".json"], fileTypes=["json"]),
         "CoNLLULoader": build_file_field(suffixes=[".csv"], fileTypes=["csv"]),
@@ -120,29 +124,23 @@ class DocumentLoaderFrontNode(FrontendNode):
             "DirectoryLoader",
             "ReadTheDocsLoader",
             "NotionDirectoryLoader",
+            "PyPDFDirectoryLoader",
         }:
             name = "path"
             display_name = "Local directory"
         if name:
-            self.template.add_field(
-                TemplateField(
-                    field_type="str",
-                    required=True,
-                    show=True,
-                    name=name,
-                    value="",
-                    display_name=display_name,
-                )
-            )
             if self.template.type_name in {"DirectoryLoader"}:
+                for field in build_directory_loader_fields():
+                    self.template.add_field(field)
+            else:
                 self.template.add_field(
                     TemplateField(
                         field_type="str",
                         required=True,
                         show=True,
-                        name="glob",
-                        value="**/*.txt",
-                        display_name="glob",
+                        name=name,
+                        value="",
+                        display_name=display_name,
                     )
                 )
             # add a metadata field of type dict
@@ -165,3 +163,101 @@ class DocumentLoaderFrontNode(FrontendNode):
             field.show = True
             field.advanced = False
         field.show = True
+
+
+def build_directory_loader_fields():
+    # if loader_kwargs is None:
+    #         loader_kwargs = {}
+    # self.path = path
+    # self.glob = glob
+    # self.load_hidden = load_hidden
+    # self.loader_cls = loader_cls
+    # self.loader_kwargs = loader_kwargs
+    # self.silent_errors = silent_errors
+    # self.recursive = recursive
+    # self.show_progress = show_progress
+    # self.use_multithreading = use_multithreading
+    # self.max_concurrency = max_concurrency
+    # Based on the above fields, we can build the following fields:
+    # path, glob, load_hidden, silent_errors, recursive, show_progress, use_multithreading, max_concurrency
+    # path
+    path = TemplateField(
+        field_type="str",
+        required=True,
+        show=True,
+        name="path",
+        value="",
+        display_name="Local directory",
+        advanced=False,
+    )
+    # glob
+    glob = TemplateField(
+        field_type="str",
+        required=True,
+        show=True,
+        name="glob",
+        value="**/*.txt",
+        display_name="glob",
+        advanced=False,
+    )
+    # load_hidden
+    load_hidden = TemplateField(
+        field_type="bool",
+        required=False,
+        show=True,
+        name="load_hidden",
+        value="False",
+        display_name="Load hidden files",
+        advanced=True,
+    )
+    # silent_errors
+    silent_errors = TemplateField(
+        field_type="bool",
+        required=False,
+        show=True,
+        name="silent_errors",
+        value="False",
+        display_name="Silent errors",
+        advanced=True,
+    )
+    # recursive
+    recursive = TemplateField(
+        field_type="bool",
+        required=False,
+        show=True,
+        name="recursive",
+        value="True",
+        display_name="Recursive",
+        advanced=True,
+    )
+
+    # use_multithreading
+    use_multithreading = TemplateField(
+        field_type="bool",
+        required=False,
+        show=True,
+        name="use_multithreading",
+        value="True",
+        display_name="Use multithreading",
+        advanced=True,
+    )
+    # max_concurrency
+    max_concurrency = TemplateField(
+        field_type="int",
+        required=False,
+        show=True,
+        name="max_concurrency",
+        value=10,
+        display_name="Max concurrency",
+        advanced=True,
+    )
+
+    return (
+        path,
+        glob,
+        load_hidden,
+        silent_errors,
+        recursive,
+        use_multithreading,
+        max_concurrency,
+    )

src/backend/langflow/template/frontend_node/formatter/base.py

@@ -0,0 +1,10 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from langflow.template.field.base import TemplateField
+
+
+class FieldFormatter(ABC):
+    @abstractmethod
+    def format(self, field: TemplateField, name: Optional[str]) -> None:
+        pass

src/backend/langflow/template/frontend_node/formatter/field_formatters.py

@@ -0,0 +1,162 @@
+from typing import Optional
+from langflow.template.field.base import TemplateField
+from langflow.template.frontend_node.constants import FORCE_SHOW_FIELDS
+from langflow.template.frontend_node.formatter.base import FieldFormatter
+import re
+
+from langflow.utils.constants import (
+    ANTHROPIC_MODELS,
+    CHAT_OPENAI_MODELS,
+    OPENAI_MODELS,
+)
+
+
+class OpenAIAPIKeyFormatter(FieldFormatter):
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        if "api_key" in field.name and "OpenAI" in str(name):
+            field.display_name = "OpenAI API Key"
+            field.required = False
+            if field.value is None:
+                field.value = ""
+
+
+class ModelSpecificFieldFormatter(FieldFormatter):
+    MODEL_DICT = {
+        "OpenAI": OPENAI_MODELS,
+        "ChatOpenAI": CHAT_OPENAI_MODELS,
+        "Anthropic": ANTHROPIC_MODELS,
+        "ChatAnthropic": ANTHROPIC_MODELS,
+    }
+
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        if name in self.MODEL_DICT and field.name == "model_name":
+            field.options = self.MODEL_DICT[name]
+            field.is_list = True
+
+
+class KwargsFormatter(FieldFormatter):
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        if "kwargs" in field.name.lower():
+            field.advanced = True
+            field.required = False
+            field.show = False
+
+
+class APIKeyFormatter(FieldFormatter):
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        if "api" in field.name.lower() and "key" in field.name.lower():
+            field.required = False
+            field.advanced = False
+
+            field.display_name = field.name.replace("_", " ").title()
+            field.display_name = field.display_name.replace("Api", "API")
+
+
+class RemoveOptionalFormatter(FieldFormatter):
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        _type = field.field_type
+        field.field_type = re.sub(r"Optional\[(.*)\]", r"\1", _type)
+
+
+class ListTypeFormatter(FieldFormatter):
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        _type = field.field_type
+        is_list = "List" in _type or "Sequence" in _type
+        if is_list:
+            _type = re.sub(r"(List|Sequence)\[(.*)\]", r"\2", _type)
+            field.is_list = True
+        field.field_type = _type
+
+
+class DictTypeFormatter(FieldFormatter):
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        _type = field.field_type
+        _type = _type.replace("Mapping", "dict")
+        field.field_type = _type
+
+
+class UnionTypeFormatter(FieldFormatter):
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        _type = field.field_type
+        if "Union" in _type:
+            _type = _type.replace("Union[", "")[:-1]
+            _type = _type.split(",")[0]
+            _type = _type.replace("]", "").replace("[", "")
+        field.field_type = _type
+
+
+class SpecialFieldFormatter(FieldFormatter):
+    SPECIAL_FIELD_HANDLERS = {
+        "allowed_tools": lambda field: "Tool",
+        "max_value_length": lambda field: "int",
+    }
+
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        handler = self.SPECIAL_FIELD_HANDLERS.get(field.name)
+        field.field_type = handler(field) if handler else field.field_type
+
+
+class ShowFieldFormatter(FieldFormatter):
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        key = field.name
+        required = field.required
+        field.show = (
+            (required and key not in ["input_variables"])
+            or key in FORCE_SHOW_FIELDS
+            or "api" in key
+            or ("key" in key and "input" not in key and "output" not in key)
+        )
+
+
+class PasswordFieldFormatter(FieldFormatter):
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        key = field.name
+        show = field.show
+        if (
+            any(text in key.lower() for text in {"password", "token", "api", "key"})
+            and show
+        ):
+            field.password = True
+
+
+class MultilineFieldFormatter(FieldFormatter):
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        key = field.name
+        if key in {
+            "suffix",
+            "prefix",
+            "template",
+            "examples",
+            "code",
+            "headers",
+            "description",
+        }:
+            field.multiline = True
+
+
+class DefaultValueFormatter(FieldFormatter):
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        value = field.to_dict()
+        if "default" in value:
+            field.value = value["default"]
+
+
+class HeadersDefaultValueFormatter(FieldFormatter):
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        key = field.name
+        if key == "headers":
+            field.value = """{'Authorization': 'Bearer <token>'}"""
+
+
+class DictCodeFileFormatter(FieldFormatter):
+    def format(self, field: TemplateField, name: Optional[str] = None) -> None:
+        key = field.name
+        value = field.to_dict()
+        _type = value["type"]
+        if "dict" in _type.lower():
+            if key == "dict_":
+                field.field_type = "file"
+                field.suffixes = [".json", ".yaml", ".yml"]
+                field.file_types = ["json", "yaml", "yml"]
+            else:
+                field.field_type = "code"

src/backend/langflow/template/frontend_node/llms.py

@@ -1,10 +1,56 @@
+import json
 from typing import Optional
 
 from langflow.template.field.base import TemplateField
 from langflow.template.frontend_node.base import FrontendNode
+from langflow.template.frontend_node.constants import CTRANSFORMERS_DEFAULT_CONFIG
+from langflow.template.frontend_node.constants import OPENAI_API_BASE_INFO
 
 
 class LLMFrontendNode(FrontendNode):
+    def add_extra_fields(self) -> None:
+        if "VertexAI" in self.template.type_name:
+            # Add credentials field which should of type file.
+            self.template.add_field(
+                TemplateField(
+                    field_type="file",
+                    required=False,
+                    show=True,
+                    name="credentials",
+                    value="",
+                    suffixes=[".json"],
+                    fileTypes=["json"],
+                )
+            )
+
+    @staticmethod
+    def format_vertex_field(field: TemplateField, name: str):
+        if "VertexAI" in name:
+            advanced_fields = [
+                "tuned_model_name",
+                "verbose",
+                "top_p",
+                "top_k",
+                "max_output_tokens",
+            ]
+            if field.name in advanced_fields:
+                field.advanced = True
+            show_fields = [
+                "tuned_model_name",
+                "verbose",
+                "project",
+                "location",
+                "credentials",
+                "max_output_tokens",
+                "model_name",
+                "temperature",
+                "top_p",
+                "top_k",
+            ]
+
+            if field.name in show_fields:
+                field.show = True
+
     @staticmethod
     def format_openai_field(field: TemplateField):
         if "openai" in field.name.lower():
@@ -15,6 +61,13 @@ class LLMFrontendNode(FrontendNode):
         if "key" not in field.name.lower() and "token" not in field.name.lower():
             field.password = False
 
+        if field.name == "openai_api_base":
+            field.info = OPENAI_API_BASE_INFO
+
+    def add_extra_base_classes(self) -> None:
+        if "BaseLLM" not in self.base_classes:
+            self.base_classes.append("BaseLLM")
+
     @staticmethod
     def format_azure_field(field: TemplateField):
         if field.name == "model_name":
@@ -31,6 +84,13 @@ class LLMFrontendNode(FrontendNode):
         field.show = True
         field.advanced = not field.required
 
+    @staticmethod
+    def format_ctransformers_field(field: TemplateField):
+        if field.name == "config":
+            field.show = True
+            field.advanced = True
+            field.value = json.dumps(CTRANSFORMERS_DEFAULT_CONFIG, indent=2)
+
     @staticmethod
     def format_field(field: TemplateField, name: Optional[str] = None) -> None:
         display_names_dict = {
@@ -38,10 +98,13 @@ class LLMFrontendNode(FrontendNode):
         }
         FrontendNode.format_field(field, name)
         LLMFrontendNode.format_openai_field(field)
+        LLMFrontendNode.format_ctransformers_field(field)
         if name and "azure" in name.lower():
             LLMFrontendNode.format_azure_field(field)
         if name and "llama" in name.lower():
             LLMFrontendNode.format_llama_field(field)
+        if name and "vertex" in name.lower():
+            LLMFrontendNode.format_vertex_field(field, name)
         SHOW_FIELDS = ["repo_id"]
         if field.name in SHOW_FIELDS:
             field.show = True
@@ -77,6 +140,17 @@ class LLMFrontendNode(FrontendNode):
             "model_file",
             "model_type",
             "deployment_name",
+            "credentials",
         ]:
             field.advanced = False
             field.show = True
+        if field.name == "credentials":
+            field.field_type = "file"
+        if name == "VertexAI" and field.name not in [
+            "callbacks",
+            "client",
+            "stop",
+            "tags",
+            "cache",
+        ]:
+            field.show = True

src/backend/langflow/template/frontend_node/memories.py

@@ -2,11 +2,19 @@ from typing import Optional
 
 from langflow.template.field.base import TemplateField
 from langflow.template.frontend_node.base import FrontendNode
+from langflow.template.template.base import Template
+from langchain.memory.chat_message_histories.postgres import DEFAULT_CONNECTION_STRING
 
 
 class MemoryFrontendNode(FrontendNode):
     #! Needs testing
     def add_extra_fields(self) -> None:
+        # chat history should have another way to add common field?
+        # prevent adding incorect field in ChatMessageHistory
+        base_message_classes = ["BaseEntityStore", "BaseChatMessageHistory"]
+        if any(base_class in self.base_classes for base_class in base_message_classes):
+            return
+
         # add return_messages field
         self.template.add_field(
             TemplateField(
@@ -29,16 +37,17 @@ class MemoryFrontendNode(FrontendNode):
                 value="",
             )
         )
-        self.template.add_field(
-            TemplateField(
-                field_type="str",
-                required=False,
-                show=True,
-                name="output_key",
-                advanced=True,
-                value="",
+        if self.template.type_name not in {"VectorStoreRetrieverMemory"}:
+            self.template.add_field(
+                TemplateField(
+                    field_type="str",
+                    required=False,
+                    show=True,
+                    name="output_key",
+                    advanced=True,
+                    value="",
+                )
             )
-        )
 
     @staticmethod
     def format_field(field: TemplateField, name: Optional[str] = None) -> None:
@@ -64,3 +73,50 @@ class MemoryFrontendNode(FrontendNode):
             field.value = ""
         if field.name == "memory_key":
             field.value = "chat_history"
+        if field.name == "chat_memory":
+            field.show = True
+            field.advanced = False
+            field.required = False
+        if field.name == "url":
+            field.show = True
+        if field.name == "entity_store":
+            field.show = True
+        if name == "SQLiteEntityStore":
+            field.show = True
+
+
+class PostgresChatMessageHistoryFrontendNode(MemoryFrontendNode):
+    name: str = "PostgresChatMessageHistory"
+    template: Template = Template(
+        type_name="PostgresChatMessageHistory",
+        fields=[
+            TemplateField(
+                field_type="str",
+                required=True,
+                placeholder="",
+                is_list=False,
+                show=True,
+                multiline=False,
+                name="session_id",
+            ),
+            TemplateField(
+                field_type="str",
+                required=True,
+                show=True,
+                name="connection_string",
+                value=DEFAULT_CONNECTION_STRING,
+            ),
+            TemplateField(
+                field_type="str",
+                required=True,
+                placeholder="",
+                is_list=False,
+                show=True,
+                multiline=False,
+                value="message_store",
+                name="table_name",
+            ),
+        ],
+    )
+    description: str = "Memory store with Postgres"
+    base_classes: list[str] = ["PostgresChatMessageHistory", "BaseChatMessageHistory"]

src/backend/langflow/template/frontend_node/retrievers.py

@@ -0,0 +1,15 @@
+from typing import Optional
+
+from langflow.template.field.base import TemplateField
+from langflow.template.frontend_node.base import FrontendNode
+
+
+class RetrieverFrontendNode(FrontendNode):
+    @staticmethod
+    def format_field(field: TemplateField, name: Optional[str] = None) -> None:
+        FrontendNode.format_field(field, name)
+        # Define common field attributes
+        field.show = True
+        if field.name == "parser_key":
+            field.display_name = "Parser Key"
+            field.password = False

src/backend/langflow/template/frontend_node/textsplitters.py

@@ -1,12 +1,17 @@
 from langflow.template.field.base import TemplateField
 from langflow.template.frontend_node.base import FrontendNode
+from langchain.text_splitter import Language
 
 
 class TextSplittersFrontendNode(FrontendNode):
+    def add_extra_base_classes(self) -> None:
+        self.base_classes = ["Document"]
+        self.output_types = ["Document"]
+
     def add_extra_fields(self) -> None:
         self.template.add_field(
             TemplateField(
-                field_type="BaseLoader",
+                field_type="Document",
                 required=True,
                 show=True,
                 name="documents",
@@ -17,6 +22,24 @@ class TextSplittersFrontendNode(FrontendNode):
             name = "separator"
         elif self.template.type_name == "RecursiveCharacterTextSplitter":
             name = "separators"
+            # Add a field for type of separator
+            # which will have Text or any value from the
+            # Language enum
+            options = [x.value for x in Language] + ["Text"]
+            options.sort()
+            self.template.add_field(
+                TemplateField(
+                    field_type="str",
+                    required=True,
+                    show=True,
+                    name="separator_type",
+                    advanced=False,
+                    is_list=True,
+                    options=options,
+                    value="Text",
+                    display_name="Separator Type",
+                )
+            )
         self.template.add_field(
             TemplateField(
                 field_type="str",

src/backend/langflow/template/frontend_node/vectorstores.py

@@ -51,7 +51,7 @@ class VectorStoreFrontendNode(FrontendNode):
                 required=False,
                 show=True,
                 advanced=False,
-                value=True,
+                value=False,
                 display_name="Persist",
             )
             extra_fields.append(extra_field)
@@ -200,7 +200,7 @@ class VectorStoreFrontendNode(FrontendNode):
                 self.template.add_field(field)
 
     def add_extra_base_classes(self) -> None:
-        self.base_classes.append("BaseRetriever")
+        self.base_classes.extend(("BaseRetriever", "VectorStoreRetriever"))
 
     @staticmethod
     def format_field(field: TemplateField, name: Optional[str] = None) -> None:
@@ -252,7 +252,7 @@ class VectorStoreFrontendNode(FrontendNode):
             # when instantiating the vectorstores
             field.name = "documents"
 
-            field.field_type = "TextSplitter"
+            field.field_type = "Document"
             field.display_name = "Documents"
             field.required = False
             field.show = True

src/backend/langflow/utils/util.py

@@ -165,6 +165,7 @@ def build_template_from_method(
                         "required": param.default == param.empty,
                     }
                     for name, param in params.items()
+                    if name not in ["self", "kwargs", "args"]
                 },
             }
 
@@ -233,6 +234,9 @@ def format_dict(d, name: Optional[str] = None):
 
         _type = value["type"]
 
+        if not isinstance(_type, str):
+            _type = _type.__name__
+
         # Remove 'Optional' wrapper
         if "Optional" in _type:
             _type = _type.replace("Optional[", "")[:-1]