feat: pagination improvements + tests (#4163)

* 📝 (utils.py): Add Query import and custom_params function to handle custom pagination parameters
📝 (flows.py): Update read_flows function to handle get_all flag and add error handling for missing folders
📝 (folders.py): Add custom_params dependency to read_folder endpoint for custom pagination handling
📝 (model.py): Remove PaginatedFlowResponse class as it is no longer used, add FlowHeader class to represent flow headers without data

✨ (test_database.py): Add pagination support for reading flows with different parameters to improve testing coverage and flexibility.

✅ (test_database.py): add unit tests for reading folders with different scenarios such as pagination, flows, search, and component filter to ensure proper functionality and data retrieval.

* 📝 (utils.py): Add constants MAX_PAGE_SIZE and MIN_PAGE_SIZE for better readability and maintainability
📝 (flows.py): Update get_all parameter default value to True for backward compatibility with frontend
📝 (flows.py): Update error message formatting for clarity
📝 (model.py): Update comments for folder_id, is_component, endpoint_name, and description fields for clarity

* ✨ (test_database.py): Simplify test functions by removing unnecessary parameters and code duplication to improve readability and maintainability.

* formatter

* [autofix.ci] apply automated fixes

* ruff fix

* [autofix.ci] apply automated fixes

---------

Co-authored-by: Gabriel Luiz Freitas Almeida <gabriel@langflow.org>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

src/backend/base/langflow/api/utils.py

@@ -4,7 +4,8 @@ import uuid
 from datetime import timedelta
 from typing import TYPE_CHECKING, Any
 
-from fastapi import HTTPException
+from fastapi import HTTPException, Query
+from fastapi_pagination import Params
 from loguru import logger
 from sqlalchemy import delete
 
@@ -23,6 +24,9 @@ if TYPE_CHECKING:
 
 API_WORDS = ["api", "key", "token"]
 
+MAX_PAGE_SIZE = 50
+MIN_PAGE_SIZE = 1
+
 
 def has_api_terms(word: str):
     return "api" in word and ("key" in word or ("token" in word and "tokens" not in word))
@@ -263,3 +267,12 @@ async def cascade_delete_flow(session: Session, flow: Flow):
     except Exception as e:
         msg = f"Unable to cascade delete flow: ${flow.id}"
         raise RuntimeError(msg, e) from e
+
+
+def custom_params(
+    page: int | None = Query(None),
+    size: int | None = Query(None),
+):
+    if page is None and size is None:
+        return None
+    return Params(page=page or MIN_PAGE_SIZE, size=size or MAX_PAGE_SIZE)

src/backend/base/langflow/api/v1/flows.py

@@ -132,7 +132,7 @@ def read_flows(
     settings_service: SettingsService = Depends(get_settings_service),
     remove_example_flows: bool = False,
     components_only: bool = False,
-    get_all: bool = False,
+    get_all: bool = True,
     folder_id: UUID | None = None,
     params: Params = Depends(),
     header_flows: bool = False,
@@ -144,14 +144,18 @@ def read_flows(
         session (Session): The database session.
         settings_service (SettingsService): The settings service.
         components_only (bool, optional): Whether to return only components. Defaults to False.
-        get_all (bool, optional): Whether to return all flows without pagination. Defaults to False.
+
+        get_all (bool, optional): Whether to return all flows without pagination. Defaults to True.
+        **This field must be True because of backward compatibility with the frontend - Release: 1.0.20**
+
         folder_id (UUID, optional): The folder ID. Defaults to None.
         params (Params): Pagination parameters.
         remove_example_flows (bool, optional): Whether to remove example flows. Defaults to False.
         header_flows (bool, optional): Whether to return only specific headers of the flows. Defaults to False.
 
     Returns:
-        Union[list[FlowRead], Page[FlowRead]]: A list of flows or a paginated response containing the list of flows.
+        list[FlowRead] | Page[FlowRead] | list[FlowHeader]
+        A list of flows or a paginated response containing the list of flows or a list of flow headers.
     """
     try:
         auth_settings = settings_service.auth_settings
@@ -162,6 +166,12 @@ def read_flows(
         starter_folder = session.exec(select(Folder).where(Folder.name == STARTER_FOLDER_NAME)).first()
         starter_folder_id = starter_folder.id if starter_folder else None
 
+        if not starter_folder and not default_folder:
+            raise HTTPException(
+                status_code=404,
+                detail="Starter folder and default folder not found. Please create a folder and add flows to it.",
+            )
+
         if not folder_id:
             folder_id = default_folder_id
 
@@ -178,9 +188,6 @@ def read_flows(
         if components_only:
             stmt = stmt.where(Flow.is_component == True)  # noqa: E712
 
-        if not get_all:
-            stmt = stmt.where(Flow.folder_id == folder_id)
-
         if get_all:
             flows = session.exec(stmt).all()
             flows = validate_is_component(flows)
@@ -194,6 +201,8 @@ def read_flows(
                     for flow in flows
                 ]
             return flows
+
+        stmt = stmt.where(Flow.folder_id == folder_id)
         return paginate(session, stmt, params=params)
 
     except Exception as e:

src/backend/base/langflow/api/v1/folders.py

@@ -5,7 +5,7 @@ from fastapi_pagination.ext.sqlmodel import paginate
 from sqlalchemy import or_, update
 from sqlmodel import Session, select
 
-from langflow.api.utils import cascade_delete_flow
+from langflow.api.utils import cascade_delete_flow, custom_params
 from langflow.api.v1.flows import create_flows
 from langflow.api.v1.schemas import FlowListCreate, FlowListReadWithFolderName
 from langflow.helpers.flow import generate_unique_flow_name
@@ -18,6 +18,7 @@ from langflow.services.database.models.folder.model import (
     Folder,
     FolderCreate,
     FolderRead,
+    FolderReadWithFlows,
     FolderUpdate,
 )
 from langflow.services.database.models.folder.pagination_model import FolderWithPaginatedFlows
@@ -99,13 +100,13 @@ def read_folders(
         raise HTTPException(status_code=500, detail=str(e)) from e
 
 
-@router.get("/{folder_id}", response_model=FolderWithPaginatedFlows, status_code=200)
+@router.get("/{folder_id}", response_model=FolderWithPaginatedFlows | FolderReadWithFlows, status_code=200)
 def read_folder(
     *,
     session: Session = Depends(get_session),
     folder_id: str,
     current_user: User = Depends(get_current_active_user),
-    params: Params = Depends(),
+    params: Params | None = Depends(custom_params),
     is_component: bool = False,
     is_flow: bool = False,
     search: str = "",
@@ -121,19 +122,25 @@ def read_folder(
         raise HTTPException(status_code=404, detail="Folder not found")
 
     try:
-        stmt = select(Flow).where(Flow.folder_id == folder_id, Flow.user_id == current_user.id)
+        if params and params.page and params.size:
+            stmt = select(Flow).where(Flow.folder_id == folder_id)
 
-        if Flow.updated_at is not None:
-            stmt = stmt.order_by(Flow.updated_at.desc())  # type: ignore[attr-defined]
-        if is_component:
-            stmt = stmt.where(Flow.is_component == True)  # noqa: E712
-        if is_flow:
-            stmt = stmt.where(Flow.is_component == False)  # noqa: E712
-        if search:
-            stmt = stmt.where(Flow.name.like(f"%{search}%"))  # type: ignore[attr-defined]
-        paginated_flows = paginate(session, stmt, params=params)
+            if Flow.updated_at is not None:
+                stmt = stmt.order_by(Flow.updated_at.desc())  # type: ignore[attr-defined]
+            if is_component:
+                stmt = stmt.where(Flow.is_component == True)  # noqa: E712
+            if is_flow:
+                stmt = stmt.where(Flow.is_component == False)  # noqa: E712
+            if search:
+                stmt = stmt.where(Flow.name.like(f"%{search}%"))  # type: ignore[attr-defined]
+            paginated_flows = paginate(session, stmt, params=params)
+
+            return FolderWithPaginatedFlows(folder=FolderRead.model_validate(folder), flows=paginated_flows)
+
+        flows_from_current_user_in_folder = [flow for flow in folder.flows if flow.user_id == current_user.id]
+        folder.flows = flows_from_current_user_in_folder
+        return folder  # noqa: TRY300
 
-        return FolderWithPaginatedFlows(folder=FolderRead.model_validate(folder), flows=paginated_flows)
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e)) from e
 

src/backend/base/langflow/services/database/models/flow/model.py

@@ -196,6 +196,24 @@ class FlowRead(FlowBase):
 
 
 class FlowHeader(BaseModel):
+    """Model representing a header for a flow - Without the data.
+
+    Attributes:
+    -----------
+    id : UUID
+        Unique identifier for the flow.
+    name : str
+        The name of the flow.
+    folder_id : UUID | None, optional
+        The ID of the folder containing the flow. None if not associated with a folder.
+    is_component : bool | None, optional
+        Flag indicating whether the flow is a component.
+    endpoint_name : str | None, optional
+        The name of the endpoint associated with this flow.
+    description : str | None, optional
+        A description of the flow.
+    """
+
     id: UUID
     name: str
     folder_id: UUID | None = None
@@ -204,13 +222,6 @@ class FlowHeader(BaseModel):
     description: str | None = None
 
 
-class PaginatedFlowResponse(BaseModel):
-    flows: list[FlowRead]
-    total: int
-    page_size: int
-    page_index: int
-
-
 class FlowUpdate(SQLModel):
     name: str | None = None
     description: str | None = None

src/backend/tests/unit/test_database.py

@@ -67,20 +67,17 @@ async def test_read_flows(client: TestClient, json_flow: str, logged_in_headers)
     assert len(response.json()) > 0
 
 
-@pytest.mark.usefixtures("active_user")
-async def test_read_flows_pagination(client: TestClient, logged_in_headers):
+async def test_read_flows_pagination_without_params(client: TestClient, logged_in_headers):
     response = await client.get("api/v1/flows/", headers=logged_in_headers)
+    response_json = response.json()
     assert response.status_code == 200
-    assert response.json()["page"] == 1
-    assert response.json()["size"] == 50
-    assert response.json()["pages"] == 0
-    assert response.json()["total"] == 0
-    assert len(response.json()["items"]) == 0
+    assert len(response_json) == 0
 
 
-@pytest.mark.usefixtures("active_user")
 async def test_read_flows_pagination_with_params(client: TestClient, logged_in_headers):
-    response = await client.get("api/v1/flows/", headers=logged_in_headers, params={"page": 3, "size": 10})
+    response = await client.get(
+        "api/v1/flows/", headers=logged_in_headers, params={"page": 3, "size": 10, "get_all": False}
+    )
     assert response.status_code == 200
     assert response.json()["page"] == 3
     assert response.json()["size"] == 10
@@ -89,15 +86,135 @@ async def test_read_flows_pagination_with_params(client: TestClient, logged_in_h
     assert len(response.json()["items"]) == 0
 
 
-@pytest.mark.usefixtures("flow_component")
-async def test_read_flows_components_only(client: TestClient, logged_in_headers):
+async def test_read_flows_pagination_with_flows(client: TestClient, logged_in_headers):
+    number_of_flows = 30
+    flows = [FlowCreate(name=f"Flow {i}", description="description", data={}) for i in range(number_of_flows)]
+    flow_ids = []
+    for flow in flows:
+        response = await client.post("api/v1/flows/", json=flow.model_dump(), headers=logged_in_headers)
+        assert response.status_code == 201
+        flow_ids.append(response.json()["id"])
+
     response = await client.get(
-        "api/v1/flows/", headers=logged_in_headers, params={"components_only": True, "get_all": True}
+        "api/v1/flows/", headers=logged_in_headers, params={"page": 3, "size": 10, "get_all": False}
     )
     assert response.status_code == 200
-    names = [flow["name"] for flow in response.json()]
-    assert any("Chat Input Component" in name for name in names)
-    assert all(flow["is_component"] is True for flow in response.json()), [flow["name"] for flow in response.json()]
+    assert response.json()["page"] == 3
+    assert response.json()["size"] == 10
+    assert response.json()["pages"] == 3
+    assert response.json()["total"] == number_of_flows
+    assert len(response.json()["items"]) == 10
+
+    response = await client.get(
+        "api/v1/flows/", headers=logged_in_headers, params={"page": 4, "size": 10, "get_all": False}
+    )
+    assert response.status_code == 200
+    assert response.json()["page"] == 4
+    assert response.json()["size"] == 10
+    assert response.json()["pages"] == 3
+    assert response.json()["total"] == number_of_flows
+    assert len(response.json()["items"]) == 0
+
+
+async def test_read_flows_custom_page_size(client: TestClient, logged_in_headers):
+    number_of_flows = 30
+    flows = [FlowCreate(name=f"Flow {i}", description="description", data={}) for i in range(number_of_flows)]
+    for flow in flows:
+        response = await client.post("api/v1/flows/", json=flow.model_dump(), headers=logged_in_headers)
+        assert response.status_code == 201
+
+    response = await client.get(
+        "api/v1/flows/", headers=logged_in_headers, params={"page": 1, "size": 15, "get_all": False}
+    )
+    assert response.status_code == 200
+    assert response.json()["page"] == 1
+    assert response.json()["size"] == 15
+    assert response.json()["pages"] == 2
+    assert response.json()["total"] == number_of_flows
+    assert len(response.json()["items"]) == 15
+
+
+async def test_read_flows_invalid_page(client: TestClient, logged_in_headers):
+    number_of_flows = 30
+    flows = [FlowCreate(name=f"Flow {i}", description="description", data={}) for i in range(number_of_flows)]
+    flow_ids = []
+    for flow in flows:
+        response = await client.post("api/v1/flows/", json=flow.model_dump(), headers=logged_in_headers)
+        assert response.status_code == 201
+        flow_ids.append(response.json()["id"])
+
+    response = await client.get(
+        "api/v1/flows/", headers=logged_in_headers, params={"page": 0, "size": 10, "get_all": False}
+    )
+    assert response.status_code == 422  # Assuming 422 is the status code for invalid input
+
+
+async def test_read_flows_invalid_size(client: TestClient, logged_in_headers):
+    number_of_flows = 30
+    flows = [FlowCreate(name=f"Flow {i}", description="description", data={}) for i in range(number_of_flows)]
+    flow_ids = []
+    for flow in flows:
+        response = await client.post("api/v1/flows/", json=flow.model_dump(), headers=logged_in_headers)
+        assert response.status_code == 201
+        flow_ids.append(response.json()["id"])
+
+    response = await client.get(
+        "api/v1/flows/", headers=logged_in_headers, params={"page": 1, "size": 0, "get_all": False}
+    )
+    assert response.status_code == 422  # Assuming 422 is the status code for invalid input
+
+
+async def test_read_flows_no_pagination_params(client: TestClient, logged_in_headers):
+    number_of_flows = 30
+    flows = [FlowCreate(name=f"Flow {i}", description="description", data={}) for i in range(number_of_flows)]
+    for flow in flows:
+        response = await client.post("api/v1/flows/", json=flow.model_dump(), headers=logged_in_headers)
+        assert response.status_code == 201
+
+    response = await client.get("api/v1/flows/", headers=logged_in_headers, params={"get_all": False})
+    assert response.status_code == 200
+    # Assert default pagination values, adjust these according to your API's default behavior
+    assert response.json()["page"] == 1
+    assert response.json()["size"] == 50
+    assert response.json()["pages"] == 1
+    assert response.json()["total"] == number_of_flows
+    assert len(response.json()["items"]) == number_of_flows
+
+
+async def test_read_flows_components_only_paginated(client: TestClient, logged_in_headers):
+    number_of_flows = 10
+    flows = [
+        FlowCreate(name=f"Flow {i}", description="description", data={}, is_component=True)
+        for i in range(number_of_flows)
+    ]
+    for flow in flows:
+        response = await client.post("api/v1/flows/", json=flow.model_dump(), headers=logged_in_headers)
+        assert response.status_code == 201
+    response = await client.get(
+        "api/v1/flows/", headers=logged_in_headers, params={"components_only": True, "get_all": False}
+    )
+    assert response.status_code == 200
+    response_json = response.json()
+    assert response_json["total"] == 10
+    assert response_json["pages"] == 1
+    assert response_json["page"] == 1
+    assert response_json["size"] == 50
+    assert all(flow["is_component"] is True for flow in response_json["items"])
+
+
+async def test_read_flows_components_only(client: TestClient, logged_in_headers):
+    number_of_flows = 10
+    flows = [
+        FlowCreate(name=f"Flow {i}", description="description", data={}, is_component=True)
+        for i in range(number_of_flows)
+    ]
+    for flow in flows:
+        response = await client.post("api/v1/flows/", json=flow.model_dump(), headers=logged_in_headers)
+        assert response.status_code == 201
+    response = await client.get("api/v1/flows/", headers=logged_in_headers, params={"components_only": True})
+    assert response.status_code == 200
+    response_json = response.json()
+    assert all(flow["is_component"] is True for flow in response_json)
 
 
 async def test_read_flow(client: TestClient, json_flow: str, logged_in_headers):
@@ -305,7 +422,9 @@ async def test_get_flows_from_folder_pagination(client: TestClient, logged_in_he
     created_folder = response.json()
     folder_id = created_folder["id"]
 
-    response = await client.get(f"api/v1/folders/{folder_id}", headers=logged_in_headers)
+    response = await client.get(
+        f"api/v1/folders/{folder_id}", headers=logged_in_headers, params={"page": 1, "size": 50}
+    )
     assert response.status_code == 200
     assert response.json()["folder"]["name"] == folder_name
     assert response.json()["folder"]["description"] == "Test folder description"
@@ -522,3 +641,156 @@ def test_sqlite_pragmas():
 
         assert session.exec(text("PRAGMA journal_mode;")).scalar() == "wal"
         assert session.exec(text("PRAGMA synchronous;")).scalar() == 1
+
+
+@pytest.mark.usefixtures("active_user")
+async def test_read_folder(client: TestClient, logged_in_headers):
+    # Create a new folder
+    folder_name = f"Test Folder {uuid4()}"
+    folder = FolderCreate(name=folder_name, description="Test folder description")
+    response = await client.post("api/v1/folders/", json=folder.model_dump(), headers=logged_in_headers)
+    assert response.status_code == 201
+    created_folder = response.json()
+    folder_id = created_folder["id"]
+
+    # Read the folder
+    response = await client.get(f"api/v1/folders/{folder_id}", headers=logged_in_headers)
+    assert response.status_code == 200
+    folder_data = response.json()
+    assert folder_data["name"] == folder_name
+    assert folder_data["description"] == "Test folder description"
+    assert "flows" in folder_data
+    assert isinstance(folder_data["flows"], list)
+
+
+@pytest.mark.usefixtures("active_user")
+async def test_read_folder_with_pagination(client: TestClient, logged_in_headers):
+    # Create a new folder
+    folder_name = f"Test Folder {uuid4()}"
+    folder = FolderCreate(name=folder_name, description="Test folder description")
+    response = await client.post("api/v1/folders/", json=folder.model_dump(), headers=logged_in_headers)
+    assert response.status_code == 201
+    created_folder = response.json()
+    folder_id = created_folder["id"]
+
+    # Read the folder with pagination
+    response = await client.get(
+        f"api/v1/folders/{folder_id}", headers=logged_in_headers, params={"page": 1, "size": 10}
+    )
+    assert response.status_code == 200
+    folder_data = response.json()
+    assert isinstance(folder_data, dict)
+    assert "folder" in folder_data
+    assert "flows" in folder_data
+    assert folder_data["folder"]["name"] == folder_name
+    assert folder_data["folder"]["description"] == "Test folder description"
+    assert folder_data["flows"]["page"] == 1
+    assert folder_data["flows"]["size"] == 10
+    assert isinstance(folder_data["flows"]["items"], list)
+
+
+@pytest.mark.usefixtures("active_user")
+async def test_read_folder_with_flows(client: TestClient, json_flow: str, logged_in_headers):
+    # Create a new folder
+    folder_name = f"Test Folder {uuid4()}"
+    flow_name = f"Test Flow {uuid4()}"
+    folder = FolderCreate(name=folder_name, description="Test folder description")
+    response = await client.post("api/v1/folders/", json=folder.model_dump(), headers=logged_in_headers)
+    assert response.status_code == 201
+    created_folder = response.json()
+    folder_id = created_folder["id"]
+
+    # Create a flow in the folder
+    flow_data = orjson.loads(json_flow)
+    data = flow_data["data"]
+    flow = FlowCreate(name=flow_name, description="description", data=data)
+    flow.folder_id = folder_id
+    response = await client.post("api/v1/flows/", json=flow.model_dump(), headers=logged_in_headers)
+    assert response.status_code == 201
+
+    # Read the folder with flows
+    response = await client.get(f"api/v1/folders/{folder_id}", headers=logged_in_headers)
+    assert response.status_code == 200
+    folder_data = response.json()
+    assert folder_data["name"] == folder_name
+    assert folder_data["description"] == "Test folder description"
+    assert len(folder_data["flows"]) == 1
+    assert folder_data["flows"][0]["name"] == flow_name
+
+
+@pytest.mark.usefixtures("active_user")
+async def test_read_nonexistent_folder(client: TestClient, logged_in_headers):
+    nonexistent_id = str(uuid4())
+    response = await client.get(f"api/v1/folders/{nonexistent_id}", headers=logged_in_headers)
+    assert response.status_code == 404
+    assert response.json()["detail"] == "Folder not found"
+
+
+@pytest.mark.usefixtures("active_user")
+async def test_read_folder_with_search(client: TestClient, json_flow: str, logged_in_headers):
+    # Create a new folder
+    folder_name = f"Test Folder {uuid4()}"
+    folder = FolderCreate(name=folder_name, description="Test folder description")
+    response = await client.post("api/v1/folders/", json=folder.model_dump(), headers=logged_in_headers)
+    assert response.status_code == 201
+    created_folder = response.json()
+    folder_id = created_folder["id"]
+
+    # Create two flows in the folder
+    flow_data = orjson.loads(json_flow)
+    flow_name_1 = f"Test Flow 1 {uuid4()}"
+    flow_name_2 = f"Another Flow {uuid4()}"
+
+    flow1 = FlowCreate(
+        name=flow_name_1, description="Test flow description", data=flow_data["data"], folder_id=folder_id
+    )
+    flow2 = FlowCreate(
+        name=flow_name_2, description="Another flow description", data=flow_data["data"], folder_id=folder_id
+    )
+    flow1.folder_id = folder_id
+    flow2.folder_id = folder_id
+    await client.post("api/v1/flows/", json=flow1.model_dump(), headers=logged_in_headers)
+    await client.post("api/v1/flows/", json=flow2.model_dump(), headers=logged_in_headers)
+
+    # Read the folder with search
+    response = await client.get(
+        f"api/v1/folders/{folder_id}", headers=logged_in_headers, params={"search": "Test", "page": 1, "size": 10}
+    )
+    assert response.status_code == 200
+    folder_data = response.json()
+    assert len(folder_data["flows"]["items"]) == 1
+    assert folder_data["flows"]["items"][0]["name"] == flow_name_1
+
+
+@pytest.mark.usefixtures("active_user")
+async def test_read_folder_with_component_filter(client: TestClient, json_flow: str, logged_in_headers):
+    # Create a new folder
+    folder_name = f"Test Folder {uuid4()}"
+    folder = FolderCreate(name=folder_name, description="Test folder description")
+    response = await client.post("api/v1/folders/", json=folder.model_dump(), headers=logged_in_headers)
+    assert response.status_code == 201
+    created_folder = response.json()
+    folder_id = created_folder["id"]
+
+    # Create a component flow in the folder
+    flow_data = orjson.loads(json_flow)
+    component_flow_name = f"Component Flow {uuid4()}"
+    component_flow = FlowCreate(
+        name=component_flow_name,
+        description="Component flow description",
+        data=flow_data["data"],
+        folder_id=folder_id,
+        is_component=True,
+    )
+    component_flow.folder_id = folder_id
+    await client.post("api/v1/flows/", json=component_flow.model_dump(), headers=logged_in_headers)
+
+    # Read the folder with component filter
+    response = await client.get(
+        f"api/v1/folders/{folder_id}", headers=logged_in_headers, params={"is_component": True, "page": 1, "size": 10}
+    )
+    assert response.status_code == 200
+    folder_data = response.json()
+    assert len(folder_data["flows"]["items"]) == 1
+    assert folder_data["flows"]["items"][0]["name"] == component_flow_name
+    assert folder_data["flows"]["items"][0]["is_component"] == True  # noqa: E712