tests: Add performance benchmarks for database initialization and app startup (#4367)

* feat: add benchmark markers to multiple test cases

* test: add performance tests for database initialization and app startup

Introduce benchmark tests to measure the performance of database initialization and application startup. This helps ensure efficiency and identify potential bottlenecks in the setup process.

* feat: update CodSpeed workflow to enhance test execution and duration tracking

* refactor: streamline performance tests for database initialization and app startup

* test: enhance app startup performance test with pytest benchmarking

* test: update benchmark for database initialization

Remove unnecessary benchmark call and simplify the database initialization test to enhance clarity and reliability in performance testing.

* chore: remove unnecessary pytest options from CodSpeed workflow

* Add environment setup for test_app_startup benchmark test

* Add benchmark test for app startup with database setup and flow loading

* Add benchmark markers to flow building tests in test_chat_endpoint.py

* perf: add benchmarks for service initialization and caching

Introduce benchmarks for various service initialization functions and LLM caching to improve performance evaluations.

* Remove unused benchmark marker from test function in test_chat_endpoint.py

* perf: initialize services in super user benchmark test

* Add benchmarking to test_create_starter_projects in performance tests

* Add asyncio threading to benchmark tests and remove benchmark fixture usage

* Remove database initialization benchmark test from performance suite

.github/workflows/codspeed.yml

@@ -24,6 +24,9 @@ jobs:
         uses: CodSpeedHQ/action@v3
         with:
             token: ${{ secrets.CODSPEED_TOKEN }}
-            run: make unit_tests args="--codspeed -n auto"
+            run: |
+              uv run pytest src/backend/tests \
+                --ignore=src/backend/tests/integration \
+                --codspeed
       - name: Minimize uv cache
         run: uv cache prune --ci

src/backend/tests/performance/test_server_init.py

@@ -0,0 +1,61 @@
+import asyncio
+
+import pytest
+from langflow.services.deps import get_settings_service
+
+
+@pytest.mark.benchmark
+async def test_initialize_services():
+    """Benchmark the initialization of services."""
+    from langflow.services.utils import initialize_services
+
+    await asyncio.to_thread(initialize_services, fix_migration=False)
+
+
+@pytest.mark.benchmark
+async def test_setup_llm_caching():
+    """Benchmark LLM caching setup."""
+    from langflow.interface.utils import setup_llm_caching
+
+    await asyncio.to_thread(setup_llm_caching)
+
+
+@pytest.mark.benchmark
+async def test_initialize_super_user():
+    """Benchmark super user initialization."""
+    from langflow.initial_setup.setup import initialize_super_user_if_needed
+    from langflow.services.utils import initialize_services
+
+    await asyncio.to_thread(initialize_services, fix_migration=False)
+    await asyncio.to_thread(initialize_super_user_if_needed)
+
+
+@pytest.mark.benchmark
+async def test_get_and_cache_all_types_dict():
+    """Benchmark get_and_cache_all_types_dict function."""
+    from langflow.interface.types import get_and_cache_all_types_dict
+
+    settings_service = await asyncio.to_thread(get_settings_service)
+    result = await asyncio.to_thread(get_and_cache_all_types_dict, settings_service)
+    assert result is not None
+
+
+@pytest.mark.benchmark
+async def test_create_starter_projects():
+    """Benchmark creation of starter projects."""
+    from langflow.initial_setup.setup import create_or_update_starter_projects
+    from langflow.interface.types import get_and_cache_all_types_dict
+    from langflow.services.utils import initialize_services
+
+    await asyncio.to_thread(initialize_services, fix_migration=False)
+    settings_service = await asyncio.to_thread(get_settings_service)
+    types_dict = await get_and_cache_all_types_dict(settings_service)
+    await asyncio.to_thread(create_or_update_starter_projects, types_dict)
+
+
+@pytest.mark.benchmark
+async def test_load_flows():
+    """Benchmark loading flows from directory."""
+    from langflow.initial_setup.setup import load_flows_from_directory
+
+    await asyncio.to_thread(load_flows_from_directory)

src/backend/tests/unit/test_chat_endpoint.py

@@ -1,11 +1,13 @@
 import json
 from uuid import UUID
 
+import pytest
 from langflow.memory import get_messages
 from langflow.services.database.models.flow import FlowCreate, FlowUpdate
 from orjson import orjson
 
 
+@pytest.mark.benchmark
 async def test_build_flow(client, json_memory_chatbot_no_llm, logged_in_headers):
     flow_id = await _create_flow(client, json_memory_chatbot_no_llm, logged_in_headers)
 
@@ -15,6 +17,7 @@ async def test_build_flow(client, json_memory_chatbot_no_llm, logged_in_headers)
     check_messages(flow_id)
 
 
+@pytest.mark.benchmark
 async def test_build_flow_from_request_data(client, json_memory_chatbot_no_llm, logged_in_headers):
     flow_id = await _create_flow(client, json_memory_chatbot_no_llm, logged_in_headers)
     response = await client.get("api/v1/flows/" + str(flow_id), headers=logged_in_headers)

src/backend/tests/unit/test_endpoints.py

@@ -108,6 +108,7 @@ PROMPT_REQUEST = {
 }
 
 
+@pytest.mark.benchmark
 async def test_get_all(client: AsyncClient, logged_in_headers):
     response = await client.get("api/v1/all", headers=logged_in_headers)
     assert response.status_code == 200
@@ -329,6 +330,7 @@ async def test_successful_run_with_output_type_text(client, simple_api_test, cre
     assert all(key in result for result in inner_results for key in expected_keys), outputs_dict
 
 
+@pytest.mark.benchmark
 async def test_successful_run_with_output_type_any(client, simple_api_test, created_api_key):
     # This one should have both the ChatOutput and TextOutput components
     headers = {"x-api-key": created_api_key.api_key}
@@ -360,6 +362,7 @@ async def test_successful_run_with_output_type_any(client, simple_api_test, crea
     assert all(key in result for result in inner_results for key in expected_keys), outputs_dict
 
 
+@pytest.mark.benchmark
 async def test_successful_run_with_output_type_debug(client, simple_api_test, created_api_key):
     # This one should return outputs for all components
     # Let's just check the amount of outputs(there should be 7)
@@ -385,6 +388,7 @@ async def test_successful_run_with_output_type_debug(client, simple_api_test, cr
     assert len(outputs_dict.get("outputs")) == 3
 
 
+@pytest.mark.benchmark
 async def test_successful_run_with_input_type_text(client, simple_api_test, created_api_key):
     headers = {"x-api-key": created_api_key.api_key}
     flow_id = simple_api_test["id"]
@@ -419,6 +423,7 @@ async def test_successful_run_with_input_type_text(client, simple_api_test, crea
 
 
 @pytest.mark.api_key_required
+@pytest.mark.benchmark
 async def test_successful_run_with_input_type_chat(client: AsyncClient, simple_api_test, created_api_key):
     headers = {"x-api-key": created_api_key.api_key}
     flow_id = simple_api_test["id"]
@@ -451,6 +456,7 @@ async def test_successful_run_with_input_type_chat(client: AsyncClient, simple_a
     ), chat_input_outputs
 
 
+@pytest.mark.benchmark
 async def test_invalid_run_with_input_type_chat(client, simple_api_test, created_api_key):
     headers = {"x-api-key": created_api_key.api_key}
     flow_id = simple_api_test["id"]
@@ -465,6 +471,7 @@ async def test_invalid_run_with_input_type_chat(client, simple_api_test, created
     assert "If you pass an input_value to the chat input, you cannot pass a tweak with the same name." in response.text
 
 
+@pytest.mark.benchmark
 async def test_successful_run_with_input_type_any(client, simple_api_test, created_api_key):
     headers = {"x-api-key": created_api_key.api_key}
     flow_id = simple_api_test["id"]
@@ -517,6 +524,7 @@ async def test_invalid_flow_id(client, created_api_key):
     # Check if the error detail is as expected
 
 
+@pytest.mark.benchmark
 async def test_starter_projects(client, created_api_key):
     headers = {"x-api-key": created_api_key.api_key}
     response = await client.get("api/v1/starter-projects/", headers=headers)