From 376e4cfdd3d20d655c3f04cefea474f74f4a5ddc Mon Sep 17 00:00:00 2001 From: Cristhian Zanforlin Lousa Date: Fri, 11 Oct 2024 18:49:20 -0300 Subject: [PATCH] refactor: add pagination on folders (#4020) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ⬆️ (uv.lock): Update authlib version from 1.3.2 to 1.3.1 ⬆️ (uv.lock): Update httpx version from 0.27.2 to 0.27.0 ⬆️ (uv.lock): Add new package grpcio-health-checking version 1.62.3 ⬆️ (uv.lock): upgrade weaviate-client package from version 3.26.7 to 4.8.1 and add new dependencies grpcio, grpcio-health-checking, grpcio-tools, httpx, and pydantic. Update package source and distribution URLs accordingly. * ✨ (flows.py): Add pagination support for retrieving a list of flows to improve performance and user experience 📝 (flows.py): Update documentation to reflect changes made for pagination and additional query parameters 📝 (folders.py): Add a new endpoint to retrieve a folder with paginated flows for better organization and readability * ✨ (model.py): add PaginatedFlowResponse class to handle paginated flow responses in API calls * ✨ (test_database.py): add support for fetching all flows by adding "get_all" parameter to the request 🐛 (test_database.py): fix endpoint for fetching read-only starter projects to use correct URL path * packages changes * packages changes * 📝 (paginatorComponent/index.tsx): refactor PaginatorComponent to use constants for default values and improve code readability 🐛 (paginatorComponent/index.tsx): fix issue with setting maxPageIndex when pages prop is provided to PaginatorComponent * 🐛 (storeCardComponent): fix height of store card component to prevent overflow and improve layout aesthetics * ✨ (constants.ts): introduce new constants for search tabs, pagination settings, and store pagination settings to enhance user experience and improve functionality * ✨ (use-post-login-user.ts): add queryClient to UseRequestProcessor to enable refetching queries on mutation settled state 📝 (use-post-login-user.ts): update useLoginUser function to include onSettled callback in options to refetch queries after mutation settles * ✨ (use-get-basic-examples.ts): introduce a new query function to fetch basic examples of flows from the API and update the state with the fetched data. * ✨ (use-get-refresh-flows.ts): introduce new functionality to fetch and process flows with query parameters for components_only, get_all, folder_id, remove_example_flows, page, and size. This allows for more flexible and specific retrieval of flow data. * ✨ (use-get-folder.ts): Add support for pagination and filtering options in the useGetFolderQuery function to enhance flexibility and customization for fetching folder data. * 🔧 (use-get-folders.ts): Remove unused code related to refreshFlows and setStarterProjectId to improve code readability and maintainability ✨ (use-get-folders.ts): Update useGetFoldersQuery to setFolders with the fetched data instead of filtering out starter projects to simplify the logic and improve performance * ✨ (use-upload-flow.ts): update refreshFlows function call to include an object with get_all property set to true to fetch all flows when refreshing * 🔧 (codeAreaModal/index.tsx): remove unused import postCustomComponent to clean up code and improve readability * 📝 (AdminPage/index.tsx): Update initial page size and index values to use constants for better maintainability 📝 (AdminPage/index.tsx): Update resetFilter function to set page size and index using constants for consistency 📝 (AdminPage/index.tsx): Update logic to handle loading state and empty user list based on isIdle state for better user experience 📝 (AdminPage/index.tsx): Update PaginatorComponent props to use constant for rows count and simplify paginate function assignment * ✨ (AppInitPage/index.tsx): introduce new queries to fetch basic examples and folders data for improved functionality and user experience * ✨ (FlowPage/index.tsx): update refreshFlows function call to include a parameter to get all flows at once for better performance. * ✨ (frontend): introduce flowsPagination and setFlowsPagination functions to manage pagination for flows in the UtilityStoreType * ✨ (frontend): introduce new 'folders' and 'setFolders' properties to FoldersStoreType to manage folder data efficiently * ✨ (types.ts): introduce Pagination type to define structure for pagination data in frontend application * ✨ (frontend): introduce PaginatedFlowsType to represent paginated flow data structure in the application. * ✨ (frontend): add optional 'pages' property to PaginatorComponentType to allow customization of the number of pages displayed in the paginator * ✨ (utilityStore.ts): introduce flowsPagination object with default values for page and size to manage pagination in the utility store * ✨ (foldersStore.tsx): introduce new state 'folders' and setter function 'setFolders' to manage folder data in the store * ✨ (ViewPage/index.tsx): update refreshFlows function call to include a parameter to get all flows at once, improving efficiency and reducing unnecessary calls to the backend. * 📝 (StorePage/index.tsx): update constants import to include new pagination constants for better organization ♻️ (StorePage/index.tsx): refactor pagination logic to use new pagination constants for clarity and consistency throughout the file * ✨ (componentsComponent/index.tsx): Add support for pagination feature in ComponentsComponent to improve user experience and performance. * ✨ (myCollectionComponent/index.tsx): introduce pagination and search functionality to improve user experience and data handling in the MyCollectionComponent component * ✨ (Playground/index.tsx): Update refreshFlows function call to include a parameter to get all flows at once for better performance. * ✨ (entities/index.tsx): introduce PaginatedFolderType to represent a folder with pagination information for better organization and handling of paginated data. * ✨ (headerTabsSearchComponent/index.tsx): add support for changing tabs and searching functionality in headerTabsSearchComponent 📝 (headerTabsSearchComponent/index.tsx): update tabsOptions to use constant SEARCH_TABS for consistency and reusability * 📝 (folders.py): Remove redundant import and unused code related to FolderWithPaginatedFlows class 🔧 (folders.py): Update query conditions to use explicit boolean values instead of implicit truthiness for better clarity and readability * 📝 (folders.py): reorganize imports to improve readability and maintain consistency * 📝 (flows.py): import FlowSummary model to support new functionality in the API 📝 (flows.py): add header_flows parameter to read_flows function to return a simplified list of flows if set to true * 📝 (folders.py): remove unnecessary empty line to improve code readability * ✨ (model.py): introduce new FlowSummary class to represent a summary of flow data for better organization and readability * ✨ (pagination_model.py): introduce a new model 'FolderWithPaginatedFlows' to represent a folder with paginated flows for better organization and readability. * 📝 (cardComponent/index.tsx): add missing semicolon to improve code consistency ♻️ (cardComponent/index.tsx): refactor logic to use data parameter directly instead of fetching flow by id to simplify code and improve readability * ✨ (use-get-refresh-flows.ts): introduce new query parameter 'header_flows' to support fetching header flows in API requests. * ✨ (use-get-folders.ts): add functionality to refresh flows when fetching folders to ensure data consistency and up-to-date information. * ✨ (use-upload-flow.ts): add support for fetching header flows along with all flows when refreshing flows in useUploadFlow hook * ✨ (use-redirect-flow-card-click.tsx): introduce a new custom hook 'useFlowCardClick' to handle flow card click events in the newFlowModal component. This hook utilizes react-router-dom for navigation, custom analytics tracking, and various utility functions to manage flow data and update the UI. * ✨ (NewFlowCardComponent/index.tsx): refactor onClick event handler into a separate function handleClick for better readability and maintainability * 📝 (undrawCards/index.tsx): Remove unused imports and variables to clean up the code ♻️ (undrawCards/index.tsx): Refactor onClick event handler to use a separate function for handling flow card click events * ✨ (flowsManager/index.ts): introduce new state and setter for flowToCanvas to manage the flow displayed on canvas * ✨ (flowsManagerStore.ts): introduce new feature to set and update the flow to be displayed on the canvas asynchronously * ✨ (ViewPage/index.tsx): add support for fetching header flows along with all flows when refreshing data to improve data retrieval efficiency * ✨ (collectionCard/index.tsx): introduce useFlowsManagerStore to manage flows in the application 📝 (collectionCard/index.tsx): update handleClick function to set flow to canvas before navigating to editFlowLink * ✨ (Playground/index.tsx): add support for fetching header flows along with all flows when refreshing data in the Playground page. * 🐛 (FlowPage/index.tsx): Fix setCurrentFlow logic to correctly set the current flow based on flowToCanvas value. Add flowToCanvas dependency to useEffect to ensure proper rendering. * ✨ (use-get-flow.ts): introduce a new file to handle API queries for fetching flow data in the frontend application. This file defines a custom hook 'useGetFlow' that makes a GET request to the API endpoint to retrieve flow data based on the provided flow ID. * 📝 (flows.py): update import statement to use FlowHeader instead of FlowSummary for better clarity 📝 (flows.py): update response_model in read_flows endpoint to use FlowHeader for consistency and clarity * ✨ (model.py): rename FlowSummary class to FlowHeader for better clarity and consistency in naming conventions 📝 (model.py): add is_component field to FlowHeader class to indicate if the flow is a component or not * ✨ (use-get-folder.ts): introduce processFlows function to process flows data 📝 (use-get-folder.ts): add cloneDeep function to safely clone data before processing * 🔧 (use-get-refresh-flows.ts): refactor useGetRefreshFlows function to simplify processing of dbDataFlows and update state accordingly * ✨ (FlowPage/index.tsx): Add useGetFlow hook to fetch flow data and update current flow on canvas. Add getFlowToAddToCanvas function to handle fetching and setting flow data on canvas. * ✨ (nodeToolbarComponent/index.tsx): improve user experience by automatically closing the override modal after successful flow override * ✨ (use-post-login-user.ts): add queryClient.refetchQueries for "useGetTags" after successful login to update tags data in the frontend. * ✨ (use-get-flow.ts): add processFlows function to process flow data before returning it to improve code readability and maintainability * 🐛 (use-get-refresh-flows.ts): fix asynchronous flow to correctly handle data retrieval and processing before setting state and returning flows * ✨ (use-get-tags.ts): add functionality to set tags in utility store after fetching them from the API * 📝 (shareModal/index.tsx): Update import statement for useUtilityStore to improve code organization and readability 🔧 (shareModal/index.tsx): Replace useGetTagsQuery with useUtilityStore to manage tags state and remove unnecessary API call for tags data 🔧 (shareModal/index.tsx): Replace references to data with tags variable to ensure consistency and avoid potential bugs 🔧 (shareModal/index.tsx): Update TagsSelector component to use tags variable instead of data for better data management * ✨ (AppInitPage/index.tsx): introduce useGetTagsQuery to fetch tags data from the store API for AppInitPage. * ✨ (StorePage/index.tsx): refactor to use useUtilityStore for tags state management instead of useGetTagsQuery for better separation of concerns and code organization. Update TagsSelector component to use tags from useUtilityStore hook. * ✨ (frontend): introduce Tag type to UtilityStoreType and add tags and setTags functions to manage tags in the store. * ✨ (types.ts): introduce new Tag type to represent a tag with id and name properties * ✨ (utilityStore.ts): introduce new 'tags' array and 'setTags' function to manage tags in the utility store * 📝 (App.css): add a blank line for better readability and consistency in the CSS file * ✨ (test_database.py): add pagination support for reading flows and folders to improve data retrieval efficiency and user experience * ✨ (tabsComponent/index.tsx): refactor changeLocation function to use useCallback hook for better performance and stability ✨ (tabsComponent/index.tsx): update onClick event handler to directly call changeLocation function for cleaner code and improved readability * ✨ (headerTabsSearchComponent/index.tsx): refactor handleChangeTab, handleSearch, handleInputChange, and handleKeyDown functions to use useCallback for better performance and memoization 📝 (headerTabsSearchComponent/index.tsx): update tabActive prop to use the activeTab prop passed from parent component for consistency and clarity * ✨ (myCollectionComponent/index.tsx): refactor filter state initialization to dynamically set based on current location pathname ♻️ (myCollectionComponent/index.tsx): refactor onSearch and onChangeTab functions to use useCallback for better performance and memoization * ✨ (create-query-param-string.ts): introduce a new utility function to build query parameter strings for URLs in the frontend controllers. * ✨ (use-get-folder.ts): introduce buildQueryStringUrl function to create query parameter strings for API requests 📝 (use-get-folder.ts): add comments to explain the purpose of the code and improve code readability 🔧 (use-get-folder.ts): update useGetFolderQuery function to include additional configuration options for the query, such as refetchOnWindowFocus: false * ✨ (use-delete-folders.ts): add functionality to update local store after deleting a folder to keep it in sync with the server data * 📝 (use-post-add-flow.ts): import useFolderStore to access myCollectionId state for refetching queries with the correct folder_id 🐛 (use-post-add-flow.ts): fix queryClient.refetchQueries to include the correct queryKey with folder_id or myCollectionId if response.folder_id is null * ✨ (use-get-refresh-flows.ts): refactor addQueryParams function to use buildQueryStringUrl utility function for better code readability and maintainability * ♻️ (flows.py): remove unnecessary commented out code and add pagination functionality to the router for better code organization and readability * 🔧 (NodeDescription/index.tsx): remove console.log statement for better code cleanliness and readability * ✨ (index.tsx): Add DialogClose component from @radix-ui/react-dialog to handle cancel action in ConfirmationModal. Refactor handleCancel function to improve code readability and maintainability. * ♻️ (use-redirect-flow-card-click.tsx): remove unused setFlowToCanvas function to clean up code and improve maintainability * 📝 (use-patch-update-flow.ts): update onSettled callback to refetch useGetFolders query with the updated folder_id after patching a flow * 🔧 (use-delete-folders.ts): update onSettled function to correctly refetch queries with the specific folder id when deleting folders * ✨ (use-get-folder.ts): update useGetFolderQuery to include additional query parameters for pagination and filtering options * 🔧 (use-post-upload-to-folder.ts): update onSettled callback to refetch useGetFolders query with the correct queryKey and folder_id parameter * ✨ (use-add-flow.ts): add functionality to refresh flows after adding a new flow to ensure the UI is up to date with the latest data. * ✨ (AppInitPage/index.tsx): enable fetching basic examples and tags only when isFetched is true to improve performance and reduce unnecessary API calls * ✨ (myCollectionComponent/index.tsx): refactor onPaginate function to handlePageChange callback for better code organization and readability. Update key prop in ComponentsComponent to include filter and search variables for proper re-rendering. * ✨ (use-get-folder.ts): add placeholderData option to useGetFolderQuery to provide initial data while fetching folder information * [autofix.ci] apply automated fixes * 🐛 (use-post-upload-to-folder.ts): fix queryKey values to correctly refetch queries after uploading a file to a folder * ⬆️ (folders.spec.ts): increase timeout for waiting in test to improve reliability and prevent flakiness * ✨ (folders.spec.ts): update selectors to target specific elements by using the first() method to improve test reliability * ✅ (auto-save-off.spec.ts): update test to match changes in UI for auto-save feature and improve test coverage for hover functionality. * 📝 (model.py): update model fields to set default values for folder_id, is_component, endpoint_name, and description to None for consistency and clarity * ♻️ (index.tsx): refactor isUpdatingFolder logic to include isFetchingFolder variable for better accuracy and readability * 🐛 (index.tsx): fix variable name typo in isLoadingFolder assignment to correctly reference isFetchingFolder * 🐛 (use-patch-update-flow.ts): fix issue with missing closing parenthesis in queryClient.refetchQueries() method 📝 (use-patch-update-flow.ts): update queryKey in queryClient.refetchQueries() to ["useGetFolder"] to correctly refetch the folder data after updating a flow * ✨ (use-save-flow.ts): add support for fetching flow data before saving if not already present to ensure data consistency and accuracy * ✅ (folders.spec.ts): update selectors to target specific elements correctly for testing purposes * ✨ (use-on-file-drop.tsx): add support for checking flow names in a specific collection to prevent duplicates ♻️ (use-add-flow.ts): refactor to use the same logic for checking flow names in a specific collection to prevent duplicates * ✨ (index.tsx): Add useIsMutating hook to check if a folder is being deleted to update UI accordingly ♻️ (index.tsx): Refactor logic to determine if a folder is being updated to include check for folder deletion 🔧 (index.tsx): Refactor Select component to use a separate function for handling value change 🔧 (index.tsx): Refactor Button components to disable based on separate variables for first and last page 🔧 (index.ts): Remove unused import and refactor code to use useRef hook for storing the latest folder id in useGetFolderQuery --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- src/backend/base/langflow/api/v1/flows.py | 108 +++++++++++++----- src/backend/base/langflow/api/v1/folders.py | 29 ++++- .../services/database/models/flow/model.py | 18 ++- .../models/folder/pagination_model.py | 10 ++ src/backend/base/pyproject.toml | 3 +- src/backend/tests/unit/test_database.py | 72 +++++++++++- src/frontend/package-lock.json | 1 - src/frontend/src/App.css | 1 + .../components/NodeDescription/index.tsx | 4 - .../src/components/cardComponent/index.tsx | 9 +- .../components/sideBarFolderButtons/index.tsx | 18 ++- .../hooks/use-on-file-drop.tsx | 10 +- .../folderSidebarComponent/index.tsx | 8 +- .../components/paginatorComponent/index.tsx | 43 ++++--- .../components/storeCardComponent/index.tsx | 2 +- src/frontend/src/constants/constants.ts | 10 ++ .../API/queries/auth/use-post-login-user.ts | 10 +- .../queries/flows/use-get-basic-examples.ts | 32 ++++++ .../API/queries/flows/use-get-flow.ts | 31 +++++ .../queries/flows/use-get-refresh-flows.ts | 97 ++++++++++------ .../queries/flows/use-patch-update-flow.ts | 9 +- .../API/queries/flows/use-post-add-flow.ts | 8 +- .../API/queries/folders/use-delete-folders.ts | 17 ++- .../API/queries/folders/use-get-folder.ts | 75 ++++++++++-- .../API/queries/folders/use-get-folders.ts | 20 +--- .../folders/use-post-upload-to-folder.ts | 9 +- .../API/queries/store/use-get-tags.ts | 3 + .../utils/create-query-param-string.ts | 22 ++++ src/frontend/src/hooks/flows/use-add-flow.ts | 28 ++++- src/frontend/src/hooks/flows/use-save-flow.ts | 25 +++- .../src/hooks/flows/use-upload-flow.ts | 2 +- .../src/modals/codeAreaModal/index.tsx | 1 - .../src/modals/confirmationModal/index.tsx | 27 +++-- .../components/NewFlowCardComponent/index.tsx | 14 ++- .../hooks/use-redirect-flow-card-click.tsx | 24 ++++ .../components/undrawCards/index.tsx | 17 +-- src/frontend/src/modals/shareModal/index.tsx | 12 +- src/frontend/src/pages/AdminPage/index.tsx | 24 ++-- src/frontend/src/pages/AppInitPage/index.tsx | 7 ++ .../components/nodeToolbarComponent/index.tsx | 3 +- src/frontend/src/pages/FlowPage/index.tsx | 23 +++- .../components/collectionCard/index.tsx | 8 +- .../components/componentsComponent/index.tsx | 43 +++---- .../headerTabsSearchComponent/index.tsx | 56 ++++++--- .../components/tabsComponent/index.tsx | 86 ++++++-------- .../myCollectionComponent/index.tsx | 80 +++++++++---- .../src/pages/MainPage/entities/index.tsx | 17 +++ .../pages/MainPage/pages/mainPage/index.tsx | 9 +- src/frontend/src/pages/Playground/index.tsx | 2 +- src/frontend/src/pages/StorePage/index.tsx | 28 +++-- src/frontend/src/pages/ViewPage/index.tsx | 2 +- src/frontend/src/stores/flowsManagerStore.ts | 7 ++ src/frontend/src/stores/foldersStore.tsx | 2 + src/frontend/src/stores/utilityStore.ts | 8 ++ src/frontend/src/types/components/index.ts | 1 + src/frontend/src/types/flow/index.ts | 8 ++ src/frontend/src/types/utils/types.ts | 11 ++ .../src/types/zustand/flowsManager/index.ts | 2 + .../src/types/zustand/folders/index.ts | 2 + .../src/types/zustand/utility/index.ts | 6 + .../tests/core/features/folders.spec.ts | 16 +-- .../extended/features/auto-save-off.spec.ts | 15 ++- uv.lock | 15 +++ 63 files changed, 962 insertions(+), 348 deletions(-) create mode 100644 src/backend/base/langflow/services/database/models/folder/pagination_model.py create mode 100644 src/frontend/src/controllers/API/queries/flows/use-get-basic-examples.ts create mode 100644 src/frontend/src/controllers/API/queries/flows/use-get-flow.ts create mode 100644 src/frontend/src/controllers/utils/create-query-param-string.ts create mode 100644 src/frontend/src/modals/newFlowModal/components/hooks/use-redirect-flow-card-click.tsx create mode 100644 src/frontend/src/types/utils/types.ts diff --git a/src/backend/base/langflow/api/v1/flows.py b/src/backend/base/langflow/api/v1/flows.py index 014250107..b6c7896a3 100644 --- a/src/backend/base/langflow/api/v1/flows.py +++ b/src/backend/base/langflow/api/v1/flows.py @@ -12,7 +12,8 @@ import orjson from fastapi import APIRouter, Depends, File, HTTPException, UploadFile from fastapi.encoders import jsonable_encoder from fastapi.responses import StreamingResponse -from loguru import logger +from fastapi_pagination import Page, Params, add_pagination +from fastapi_pagination.ext.sqlalchemy import paginate from sqlmodel import Session, and_, col, select from langflow.api.utils import cascade_delete_flow, remove_api_keys, validate_is_component @@ -20,6 +21,7 @@ from langflow.api.v1.schemas import FlowListCreate from langflow.initial_setup.setup import STARTER_FOLDER_NAME from langflow.services.auth.utils import get_current_active_user from langflow.services.database.models.flow import Flow, FlowCreate, FlowRead, FlowUpdate +from langflow.services.database.models.flow.model import FlowHeader from langflow.services.database.models.flow.utils import get_webhook_component_in_flow from langflow.services.database.models.folder.constants import DEFAULT_FOLDER_NAME from langflow.services.database.models.folder.model import Folder @@ -121,7 +123,7 @@ def create_flow( raise HTTPException(status_code=500, detail=str(e)) from e -@router.get("/", response_model=list[FlowRead], status_code=200) +@router.get("/", response_model=list[FlowRead] | Page[FlowRead] | list[FlowHeader], status_code=200) def read_flows( *, current_user: User = Depends(get_current_active_user), @@ -129,56 +131,72 @@ def read_flows( settings_service: SettingsService = Depends(get_settings_service), remove_example_flows: bool = False, components_only: bool = False, + get_all: bool = False, + folder_id: UUID | None = None, + params: Params = Depends(), + header_flows: bool = False, ): """ - Retrieve a list of flows. + Retrieve a list of flows with pagination support. Args: current_user (User): The current authenticated user. session (Session): The database session. settings_service (SettingsService): The settings service. - remove_example_flows (bool, optional): Whether to remove example flows. Defaults to False. components_only (bool, optional): Whether to return only components. Defaults to False. - + get_all (bool, optional): Whether to return all flows without pagination. Defaults to False. + folder_id (UUID, optional): The folder ID. Defaults to None. + params (Params): Pagination parameters. + remove_example_flows (bool, optional): Whether to remove example flows. Defaults to False. Returns: - List[Dict]: A list of flows in JSON format. + Union[list[FlowRead], Page[FlowRead]]: A list of flows or a paginated response containing the list of flows. """ try: auth_settings = settings_service.auth_settings + + default_folder = session.exec(select(Folder).where(Folder.name == DEFAULT_FOLDER_NAME)).first() + default_folder_id = default_folder.id if default_folder else None + + starter_folder = session.exec(select(Folder).where(Folder.name == STARTER_FOLDER_NAME)).first() + starter_folder_id = starter_folder.id if starter_folder else None + + if not folder_id: + folder_id = default_folder_id + if auth_settings.AUTO_LOGIN: stmt = select(Flow).where( (Flow.user_id == None) | (Flow.user_id == current_user.id) # noqa: E711 ) - if components_only: - stmt = stmt.where(Flow.is_component == True) # noqa: E712 - flows = session.exec(stmt).all() - else: - flows = current_user.flows - - flows = validate_is_component(flows) - if components_only: - flows = [flow for flow in flows if flow.is_component] - flow_ids = [flow.id for flow in flows] - # with the session get the flows that DO NOT have a user_id - folder = session.exec(select(Folder).where(Folder.name == STARTER_FOLDER_NAME)).first() - - if not remove_example_flows and not components_only: - try: - example_flows = folder.flows if folder else [] - for example_flow in example_flows: - if example_flow.id not in flow_ids: - flows.append(example_flow) - except Exception: # noqa: BLE001 - logger.exception("Error getting example flows") + stmt = select(Flow).where(Flow.user_id == current_user.id) if remove_example_flows: - flows = [flow for flow in flows if flow.folder_id != folder.id] + stmt = stmt.where(Flow.folder_id != starter_folder_id) + + if components_only: + stmt = stmt.where(Flow.is_component == True) # noqa: E712 + + if not get_all: + stmt = stmt.where(Flow.folder_id == folder_id) + + if get_all: + flows = session.exec(stmt).all() + flows = validate_is_component(flows) + if components_only: + flows = [flow for flow in flows if flow.is_component] + if remove_example_flows and starter_folder_id: + flows = [flow for flow in flows if flow.folder_id != starter_folder_id] + if header_flows: + return [ + {"id": flow.id, "name": flow.name, "folder_id": flow.folder_id, "is_component": flow.is_component} + for flow in flows + ] + return flows + return paginate(session, stmt, params=params) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) from e - return [jsonable_encoder(flow) for flow in flows] @router.get("/{flow_id}", response_model=FlowRead, status_code=200) @@ -401,3 +419,35 @@ async def download_multiple_file( headers={"Content-Disposition": f"attachment; filename={filename}"}, ) return flows_without_api_keys[0] + + +@router.get("/basic_examples/", response_model=list[FlowRead], status_code=200) +def read_basic_examples( + *, + session: Session = Depends(get_session), +): + """ + Retrieve a list of basic example flows. + + Args: + session (Session): The database session. + + Returns: + list[FlowRead]: A list of basic example flows. + """ + + try: + # Get the starter folder + starter_folder = session.exec(select(Folder).where(Folder.name == STARTER_FOLDER_NAME)).first() + + if not starter_folder: + return [] + + # Get all flows in the starter folder + return session.exec(select(Flow).where(Flow.folder_id == starter_folder.id)).all() + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) from e + + +add_pagination(router) diff --git a/src/backend/base/langflow/api/v1/folders.py b/src/backend/base/langflow/api/v1/folders.py index 4fb9ae2d4..42fc581a4 100644 --- a/src/backend/base/langflow/api/v1/folders.py +++ b/src/backend/base/langflow/api/v1/folders.py @@ -1,5 +1,7 @@ import orjson from fastapi import APIRouter, Depends, File, HTTPException, Response, UploadFile, status +from fastapi_pagination import Params +from fastapi_pagination.ext.sqlmodel import paginate from sqlalchemy import or_, update from sqlmodel import Session, select @@ -8,6 +10,7 @@ from langflow.api.v1.flows import create_flows from langflow.api.v1.schemas import FlowListCreate, FlowListReadWithFolderName from langflow.helpers.flow import generate_unique_flow_name from langflow.helpers.folders import generate_unique_folder_name +from langflow.initial_setup.setup import STARTER_FOLDER_NAME from langflow.services.auth.utils import get_current_active_user from langflow.services.database.models.flow.model import Flow, FlowCreate, FlowRead from langflow.services.database.models.folder.constants import DEFAULT_FOLDER_NAME @@ -15,9 +18,9 @@ from langflow.services.database.models.folder.model import ( Folder, FolderCreate, FolderRead, - FolderReadWithFlows, FolderUpdate, ) +from langflow.services.database.models.folder.pagination_model import FolderWithPaginatedFlows from langflow.services.database.models.user.model import User from langflow.services.deps import get_session @@ -89,25 +92,41 @@ def read_folders( or_(Folder.user_id == current_user.id, Folder.user_id == None) # noqa: E711 ) ).all() + folders = [folder for folder in folders if folder.name != STARTER_FOLDER_NAME] return sorted(folders, key=lambda x: x.name != DEFAULT_FOLDER_NAME) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) from e -@router.get("/{folder_id}", response_model=FolderReadWithFlows, status_code=200) +@router.get("/{folder_id}", response_model=FolderWithPaginatedFlows, status_code=200) def read_folder( *, session: Session = Depends(get_session), folder_id: str, current_user: User = Depends(get_current_active_user), + params: Params = Depends(), + is_component: bool = False, + is_flow: bool = False, + search: str = "", ): try: folder = session.exec(select(Folder).where(Folder.id == folder_id, Folder.user_id == current_user.id)).first() if not folder: raise HTTPException(status_code=404, detail="Folder not found") - flows_from_current_user_in_folder = [flow for flow in folder.flows if flow.user_id == current_user.id] - folder.flows = flows_from_current_user_in_folder - return folder + + stmt = select(Flow).where(Flow.folder_id == folder_id, Flow.user_id == current_user.id) + + if Flow.updated_at is not None: + stmt = stmt.order_by(Flow.updated_at.desc()) # type: ignore[attr-defined] + if is_component: + stmt = stmt.where(Flow.is_component == True) # noqa: E712 + if is_flow: + stmt = stmt.where(Flow.is_component == False) # noqa: E712 + if search: + stmt = stmt.where(Flow.name.like(f"%{search}%")) # type: ignore[attr-defined] + paginated_flows = paginate(session, stmt, params=params) + + return FolderWithPaginatedFlows(folder=FolderRead.model_validate(folder), flows=paginated_flows) except Exception as e: if "No result found" in str(e): raise HTTPException(status_code=404, detail="Folder not found") from e diff --git a/src/backend/base/langflow/services/database/models/flow/model.py b/src/backend/base/langflow/services/database/models/flow/model.py index 100da7681..88dd2b5e8 100644 --- a/src/backend/base/langflow/services/database/models/flow/model.py +++ b/src/backend/base/langflow/services/database/models/flow/model.py @@ -9,7 +9,7 @@ import emoji from emoji import purely_emoji from fastapi import HTTPException, status from loguru import logger -from pydantic import field_serializer, field_validator +from pydantic import BaseModel, field_serializer, field_validator from sqlalchemy import Text, UniqueConstraint from sqlmodel import JSON, Column, Field, Relationship, SQLModel @@ -190,6 +190,22 @@ class FlowRead(FlowBase): folder_id: UUID | None = Field() +class FlowHeader(BaseModel): + id: UUID + name: str + folder_id: UUID | None = None + is_component: bool | None = None + endpoint_name: str | None = None + description: str | None = None + + +class PaginatedFlowResponse(BaseModel): + flows: list[FlowRead] + total: int + page_size: int + page_index: int + + class FlowUpdate(SQLModel): name: str | None = None description: str | None = None diff --git a/src/backend/base/langflow/services/database/models/folder/pagination_model.py b/src/backend/base/langflow/services/database/models/folder/pagination_model.py new file mode 100644 index 000000000..46dcdc687 --- /dev/null +++ b/src/backend/base/langflow/services/database/models/folder/pagination_model.py @@ -0,0 +1,10 @@ +from fastapi_pagination import Page + +from langflow.helpers.base_model import BaseModel +from langflow.services.database.models.flow.model import Flow +from langflow.services.database.models.folder.model import FolderRead + + +class FolderWithPaginatedFlows(BaseModel): + folder: FolderRead + flows: Page[Flow] diff --git a/src/backend/base/pyproject.toml b/src/backend/base/pyproject.toml index 913d51a1d..e4cd7dbb6 100644 --- a/src/backend/base/pyproject.toml +++ b/src/backend/base/pyproject.toml @@ -154,7 +154,8 @@ dependencies = [ "spider-client>=0.0.27", "diskcache>=5.6.3", "clickhouse-connect==0.7.19", - "assemblyai>=0.33.0" + "assemblyai>=0.33.0", + "fastapi-pagination>=0.12.29", ] [project.urls] diff --git a/src/backend/tests/unit/test_database.py b/src/backend/tests/unit/test_database.py index 1b9668e50..0f99d850b 100644 --- a/src/backend/tests/unit/test_database.py +++ b/src/backend/tests/unit/test_database.py @@ -67,8 +67,30 @@ async def test_read_flows(client: TestClient, json_flow: str, active_user, logge assert len(response.json()) > 0 +async def test_read_flows_pagination(client: TestClient, json_flow: str, active_user, logged_in_headers): + response = await client.get("api/v1/flows/", headers=logged_in_headers) + assert response.status_code == 200 + assert response.json()["page"] == 1 + assert response.json()["size"] == 50 + assert response.json()["pages"] == 0 + assert response.json()["total"] == 0 + assert len(response.json()["items"]) == 0 + + +async def test_read_flows_pagination_with_params(client: TestClient, json_flow: str, active_user, logged_in_headers): + response = await client.get("api/v1/flows/", headers=logged_in_headers, params={"page": 3, "size": 10}) + assert response.status_code == 200 + assert response.json()["page"] == 3 + assert response.json()["size"] == 10 + assert response.json()["pages"] == 0 + assert response.json()["total"] == 0 + assert len(response.json()["items"]) == 0 + + async def test_read_flows_components_only(client: TestClient, flow_component: dict, logged_in_headers): - response = await client.get("api/v1/flows/", headers=logged_in_headers, params={"components_only": True}) + response = await client.get( + "api/v1/flows/", headers=logged_in_headers, params={"components_only": True, "get_all": True} + ) assert response.status_code == 200 names = [flow["name"] for flow in response.json()] assert any("Chat Input Component" in name for name in names) @@ -267,6 +289,52 @@ async def test_delete_folder_with_flows_with_transaction_and_build( assert response.json() == {"vertex_builds": {}} +async def test_get_flows_from_folder_pagination(client: TestClient, logged_in_headers): + # Create a new folder + folder_name = f"Test Folder {uuid4()}" + folder = FolderCreate(name=folder_name, description="Test folder description", components_list=[], flows_list=[]) + + response = await client.post("api/v1/folders/", json=folder.model_dump(), headers=logged_in_headers) + assert response.status_code == 201, f"Expected status code 201, but got {response.status_code}" + + created_folder = response.json() + folder_id = created_folder["id"] + + response = await client.get(f"api/v1/folders/{folder_id}", headers=logged_in_headers) + assert response.status_code == 200 + assert response.json()["folder"]["name"] == folder_name + assert response.json()["folder"]["description"] == "Test folder description" + assert response.json()["flows"]["page"] == 1 + assert response.json()["flows"]["size"] == 50 + assert response.json()["flows"]["pages"] == 0 + assert response.json()["flows"]["total"] == 0 + assert len(response.json()["flows"]["items"]) == 0 + + +async def test_get_flows_from_folder_pagination_with_params(client: TestClient, logged_in_headers): + # Create a new folder + folder_name = f"Test Folder {uuid4()}" + folder = FolderCreate(name=folder_name, description="Test folder description", components_list=[], flows_list=[]) + + response = await client.post("api/v1/folders/", json=folder.model_dump(), headers=logged_in_headers) + assert response.status_code == 201, f"Expected status code 201, but got {response.status_code}" + + created_folder = response.json() + folder_id = created_folder["id"] + + response = await client.get( + f"api/v1/folders/{folder_id}", headers=logged_in_headers, params={"page": 3, "size": 10} + ) + assert response.status_code == 200 + assert response.json()["folder"]["name"] == folder_name + assert response.json()["folder"]["description"] == "Test folder description" + assert response.json()["flows"]["page"] == 3 + assert response.json()["flows"]["size"] == 10 + assert response.json()["flows"]["pages"] == 0 + assert response.json()["flows"]["total"] == 0 + assert len(response.json()["flows"]["items"]) == 0 + + async def test_create_flows(client: TestClient, session: Session, json_flow: str, logged_in_headers): flow = orjson.loads(json_flow) data = flow["data"] @@ -412,7 +480,7 @@ async def test_delete_nonexistent_flow(client: TestClient, active_user, logged_i async def test_read_only_starter_projects(client: TestClient, active_user, logged_in_headers): - response = await client.get("api/v1/flows/", headers=logged_in_headers) + response = await client.get("api/v1/flows/basic_examples/", headers=logged_in_headers) starter_projects = load_starter_projects() assert response.status_code == 200 assert len(response.json()) == len(starter_projects) diff --git a/src/frontend/package-lock.json b/src/frontend/package-lock.json index b07ae39d7..e54783c66 100644 --- a/src/frontend/package-lock.json +++ b/src/frontend/package-lock.json @@ -858,7 +858,6 @@ }, "node_modules/@clack/prompts/node_modules/is-unicode-supported": { "version": "1.3.0", - "extraneous": true, "inBundle": true, "license": "MIT", "engines": { diff --git a/src/frontend/src/App.css b/src/frontend/src/App.css index 6a91177c6..e41830102 100644 --- a/src/frontend/src/App.css +++ b/src/frontend/src/App.css @@ -153,6 +153,7 @@ body { width: 100%; height: 100%; } + .react-flow__resize-control.handle { width: 0.75rem !important; height: 0.75rem !important; diff --git a/src/frontend/src/CustomNodes/GenericNode/components/NodeDescription/index.tsx b/src/frontend/src/CustomNodes/GenericNode/components/NodeDescription/index.tsx index 376ce83f6..ab111ce23 100644 --- a/src/frontend/src/CustomNodes/GenericNode/components/NodeDescription/index.tsx +++ b/src/frontend/src/CustomNodes/GenericNode/components/NodeDescription/index.tsx @@ -36,10 +36,6 @@ export default function NodeDescription({ //timeout to wait for the dom to update setTimeout(() => { if (overflowRef.current) { - console.log( - overflowRef.current.clientHeight, - overflowRef.current.scrollHeight, - ); if ( overflowRef.current.clientHeight < overflowRef.current.scrollHeight ) { diff --git a/src/frontend/src/components/cardComponent/index.tsx b/src/frontend/src/components/cardComponent/index.tsx index e207b8853..763e37754 100644 --- a/src/frontend/src/components/cardComponent/index.tsx +++ b/src/frontend/src/components/cardComponent/index.tsx @@ -42,6 +42,7 @@ export default function CollectionCardComponent({ const selectedFlowsComponentsCards = useFlowsManagerStore( (state) => state.selectedFlowsComponentsCards, ); + function hasPlayground(flow?: FlowType) { if (!flow) { return false; @@ -60,9 +61,9 @@ export default function CollectionCardComponent({ e.stopPropagation(); track("Playground Button Clicked", { flowId: data.id }); setLoadingPlayground(true); - const flow = getFlowById(data.id); - if (flow) { - if (!hasPlayground(flow)) { + + if (data) { + if (!hasPlayground(data)) { setErrorData({ title: "Error", list: ["This flow doesn't have a playground."], @@ -70,7 +71,7 @@ export default function CollectionCardComponent({ setLoadingPlayground(false); return; } - setCurrentFlow(flow); + setCurrentFlow(data); setOpenPlayground(true); setLoadingPlayground(false); } else { diff --git a/src/frontend/src/components/folderSidebarComponent/components/sideBarFolderButtons/index.tsx b/src/frontend/src/components/folderSidebarComponent/components/sideBarFolderButtons/index.tsx index 38ac3c1d4..cad16e6cf 100644 --- a/src/frontend/src/components/folderSidebarComponent/components/sideBarFolderButtons/index.tsx +++ b/src/frontend/src/components/folderSidebarComponent/components/sideBarFolderButtons/index.tsx @@ -10,7 +10,7 @@ import { track } from "@/customization/utils/analytics"; import { createFileUpload } from "@/helpers/create-file-upload"; import { getObjectsFromFilelist } from "@/helpers/get-objects-from-filelist"; import useUploadFlow from "@/hooks/flows/use-upload-flow"; -import { useIsFetching } from "@tanstack/react-query"; +import { useIsFetching, useIsMutating } from "@tanstack/react-query"; import { useEffect, useRef, useState } from "react"; import { useParams } from "react-router-dom"; import { FolderType } from "../../../../pages/MainPage/entities"; @@ -230,7 +230,21 @@ const SideBarFoldersButtonsComponent = ({ exact: false, }); - const isUpdatingFolder = isFetchingFolders || isPending || loading; + const isFetchingFolder = !!useIsFetching({ + queryKey: ["useGetFolder"], + exact: false, + }); + + const isDeletingFolder = !!useIsMutating({ + mutationKey: ["useDeleteFolders"], + }); + + const isUpdatingFolder = + isFetchingFolders || + isFetchingFolder || + isPending || + loading || + isDeletingFolder; const HeaderButtons = () => (
diff --git a/src/frontend/src/components/folderSidebarComponent/hooks/use-on-file-drop.tsx b/src/frontend/src/components/folderSidebarComponent/hooks/use-on-file-drop.tsx index 7e2393444..070ec084a 100644 --- a/src/frontend/src/components/folderSidebarComponent/hooks/use-on-file-drop.tsx +++ b/src/frontend/src/components/folderSidebarComponent/hooks/use-on-file-drop.tsx @@ -15,6 +15,7 @@ const useFileDrop = (folderId: string) => { (state) => state.setFolderIdDragging, ); + const myCollectionId = useFolderStore((state) => state.myCollectionId); const setErrorData = useAlertStore((state) => state.setErrorData); const flows = useFlowsManagerStore((state) => state.flows); const saveFlow = useSaveFlow(); @@ -105,7 +106,14 @@ const useFileDrop = (folderId: string) => { } const updatedFlow = { ...selectedFlow, folder_id: folderId }; - const newName = addVersionToDuplicates(updatedFlow, flows ?? []); + const flowsToCheckNames = flows?.filter( + (f) => f.folder_id === myCollectionId, + ); + + const newName = addVersionToDuplicates( + updatedFlow, + flowsToCheckNames ?? [], + ); updatedFlow.name = newName; diff --git a/src/frontend/src/components/folderSidebarComponent/index.tsx b/src/frontend/src/components/folderSidebarComponent/index.tsx index 25df8aa26..499d625aa 100644 --- a/src/frontend/src/components/folderSidebarComponent/index.tsx +++ b/src/frontend/src/components/folderSidebarComponent/index.tsx @@ -1,4 +1,6 @@ import { useGetFoldersQuery } from "@/controllers/API/queries/folders/use-get-folders"; +import { useFolderStore } from "@/stores/foldersStore"; +import { useIsFetching } from "@tanstack/react-query"; import { useLocation } from "react-router-dom"; import { FolderType } from "../../pages/MainPage/entities"; import { cn } from "../../utils/utils"; @@ -19,8 +21,12 @@ export default function FolderSidebarNav({ }: SidebarNavProps) { const location = useLocation(); const pathname = location.pathname; + const folders = useFolderStore((state) => state.folders); - const { data: folders, isPending } = useGetFoldersQuery(); + const isPending = !!useIsFetching({ + queryKey: ["useGetFolders"], + exact: false, + }); return (