tests: Categorize Tests into CI and Scheduled Groups (#3391)

* categoring tests

* ✨ (actionsMainPage-shard-1.spec.ts): Add end-to-end tests for selecting and deleting items, searching flows, and searching components on the main page to ensure proper functionality and user experience.

* 📝 (frontend/tsconfig.json): update file paths in include section to match the correct directory structure for scheduled-end-to-end tests.

* 🔧 (typescript_test.yml): update path in the command to change directory to run end-to-end tests in the frontend folder
📝 (typescript_test.yml): improve comments for better readability and understanding of the workflow logic

* add yml configuration to choose folder to run CI

* 🔧 (ci.yml): Fix formatting issues and add support for running tests in a specific folder
📝 (ci.yml): Update job dependencies to improve readability and maintainability of the workflow configuration

* ♻️ (typescript_test.yml): refactor matrix values for shardIndex and shardTotal to reduce redundancy and improve readability

* categoring tests

* ✨ (actionsMainPage-shard-1.spec.ts): Add end-to-end tests for selecting and deleting items, searching flows, and searching components on the main page to ensure proper functionality and user experience.

* 📝 (frontend/tsconfig.json): update file paths in include section to match the correct directory structure for scheduled-end-to-end tests.

* 🔧 (typescript_test.yml): update path in the command to change directory to run end-to-end tests in the frontend folder
📝 (typescript_test.yml): improve comments for better readability and understanding of the workflow logic

* add yml configuration to choose folder to run CI

* 🔧 (ci.yml): Fix formatting issues and add support for running tests in a specific folder
📝 (ci.yml): Update job dependencies to improve readability and maintainability of the workflow configuration

* ♻️ (typescript_test.yml): refactor matrix values for shardIndex and shardTotal to reduce redundancy and improve readability

* 🐛 (fileUploadComponent.spec.ts): fix file path for file upload to correctly locate the test file in the assets folder

* changing test_folder

* ✨ (fileUploadComponent.spec.ts): update file path for file upload test to match new file location in the project structure

* fix input on playwright command

.github/workflows/ci.yml

@@ -23,13 +23,12 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
-
 jobs:
   set-ci-condition:
     name: Should Run CI
     runs-on: ubuntu-latest
     outputs:
-      should-run-ci:  ${{ (contains( github.event.pull_request.labels.*.name, 'lgtm') && github.event.pull_request.draft == false) || (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') }}
+      should-run-ci: ${{ (contains( github.event.pull_request.labels.*.name, 'lgtm') && github.event.pull_request.draft == false) || (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') }}
     steps:
       #  Do anything just to make the job run
       - run: echo "Debug CI Condition"
@@ -71,6 +70,8 @@ jobs:
     name: Run Frontend Tests
     if: ${{ needs.path-filter.outputs.python == 'true' || needs.path-filter.outputs.frontend == 'true' || needs.path-filter.outputs.tests == 'true' }}
     uses: ./.github/workflows/typescript_test.yml
+    with:
+      tests_folder: "tests/end-to-end"
     secrets:
       OPENAI_API_KEY: "${{ secrets.OPENAI_API_KEY }}"
       STORE_API_KEY: "${{ secrets.STORE_API_KEY }}"
@@ -91,7 +92,14 @@ jobs:
   # https://github.com/langchain-ai/langchain/blob/master/.github/workflows/check_diffs.yml
   ci_success:
     name: "CI Success"
-    needs: [test-backend, test-frontend, lint-backend, test-docs-build, set-ci-condition ]
+    needs:
+      [
+        test-backend,
+        test-frontend,
+        lint-backend,
+        test-docs-build,
+        set-ci-condition,
+      ]
     if: always()
     runs-on: ubuntu-latest
     env:

.github/workflows/typescript_test.yml

@@ -7,12 +7,23 @@ on:
         required: true
       STORE_API_KEY:
         required: true
+    inputs:
+      tests_folder:
+        description: "(Optional) Tests to run"
+        required: false
+        type: string
+        default: "tests"
   workflow_dispatch:
     inputs:
       branch:
         description: "(Optional) Branch to checkout"
         required: false
         type: string
+      tests_folder:
+        description: "(Optional) Tests to run"
+        required: false
+        type: string
+        default: "tests"
 
 env:
   POETRY_VERSION: "1.8.3"
@@ -29,30 +40,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        shardIndex:
+        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-          [
+        shardTotal: [10]
-            1,
-            2,
-            3,
-            4,
-            5,
-            6,
-            7,
-            8,
-            9,
-            10,
-            11,
-            12,
-            13,
-            14,
-            15,
-            16,
-            17,
-            18,
-            19,
-            20,
-          ]
-        shardTotal: [20]
     env:
       OPENAI_API_KEY: ${{ inputs.openai_api_key || secrets.OPENAI_API_KEY }}
       STORE_API_KEY: ${{ inputs.store_api_key || secrets.STORE_API_KEY }}
@@ -138,7 +127,7 @@ jobs:
           max_attempts: 2
           command: |
             cd src/frontend
-            npx playwright test --trace on --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --workers 2
+            npx playwright test ${{ inputs.tests_folder }} --trace on --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --workers 2
 
       - name: Upload blob report to GitHub Actions Artifacts
         if: always()
@@ -158,8 +147,8 @@ jobs:
       EXIT_CODE: ${{!contains(needs.setup-and-test.result, 'failure') && !contains(needs.setup-and-test.result, 'cancelled') && '0' || '1'}}
     steps:
       - name: "Should Merge Reports"
-      # If the CI was successful, we don't need to merge the reports
+        # If the CI was successful, we don't need to merge the reports
-      # so we can skip all the steps below
+        # so we can skip all the steps below
         id: should_merge_reports
         run: |
           if [ "$EXIT_CODE" == "0" ]; then

src/frontend/tests/end-to-end/filterEdge-shard-0.spec.ts

@@ -1,6 +1,8 @@
 import { expect, test } from "@playwright/test";
 
-test("RetrievalQA - Tooltip", async ({ page }) => {
+test("user must see on handle hover a tooltip with possibility connections", async ({
+  page,
+}) => {
   await page.goto("/");
   await page.waitForTimeout(1000);
 

src/frontend/tests/end-to-end/filterSidebar.spec.ts

@@ -1,6 +1,8 @@
 import { expect, test } from "@playwright/test";
 
-test("LLMChain - Filter", async ({ page }) => {
+test("user must see on handle click the possibility connections - LLMChain", async ({
+  page,
+}) => {
   await page.goto("/");
   await page.waitForTimeout(2000);
 

src/frontend/tests/end-to-end/globalVariables.spec.ts

@@ -1,6 +1,8 @@
 import { expect, test } from "@playwright/test";
 
-test("GlobalVariables", async ({ page }) => {
+test("user must be able to save or delete a global variable", async ({
+  page,
+}) => {
   await page.goto("/");
   await page.waitForTimeout(2000);
 

src/frontend/tests/scheduled-end-to-end/chatInputOutputUser-shard-2.spec.ts

@@ -1,6 +1,5 @@
 import { expect, test } from "@playwright/test";
 import * as dotenv from "dotenv";
-import { readFileSync } from "fs";
 import path from "path";
 
 test("user must interact with chat with Input/Output", async ({ page }) => {

src/frontend/tests/scheduled-end-to-end/fileUploadComponent.spec.ts

@@ -47,7 +47,9 @@ test("should be able to upload a file", async ({ page }) => {
   const fileChooserPromise = page.waitForEvent("filechooser");
   await page.getByTestId("icon-FileSearch2").click();
   const fileChooser = await fileChooserPromise;
-  await fileChooser.setFiles(path.join(__dirname, "/assets/test_file.txt"));
+  await fileChooser.setFiles(
+    path.join(__dirname, "../end-to-end/assets/test_file.txt"),
+  );
   await page.getByText("test_file.txt").isVisible();
 
   await page.getByPlaceholder("Search").click();

src/frontend/tests/scheduled-end-to-end/filterEdge-shard-1.spec.ts

@@ -1,6 +1,8 @@
 import { expect, test } from "@playwright/test";
 
-test("RetrievalQA - Filter", async ({ page }) => {
+test("user must see on handle click the possibility connections - RetrievalQA", async ({
+  page,
+}) => {
   await page.goto("/");
   await page.waitForTimeout(2000);
 

src/frontend/tsconfig.json

@@ -25,38 +25,38 @@
   "include": [
     "src",
     "tests/end-to-end/saveComponents.spec.ts",
-    "tests/end-to-end/actionsMainPage-shard-1.spec.ts",
+    "tests/scheduled-end-to-end/actionsMainPage-shard-1.spec.ts",
     "tests/end-to-end/auto_login.spec.ts",
-    "tests/end-to-end/chatInputOutput.spec.ts",
+    "tests/scheduled-end-to-end/chatInputOutput.spec.ts",
-    "tests/end-to-end/chatInputOutputUser-shard-2.spec.ts",
+    "tests/scheduled-end-to-end/chatInputOutputUser-shard-2.spec.ts",
-    "tests/end-to-end/codeAreaModalComponent.spec.ts",
+    "tests/scheduled-end-to-end/codeAreaModalComponent.spec.ts",
-    "tests/end-to-end/curlApiGeneration.spec.ts",
+    "tests/scheduled-end-to-end/curlApiGeneration.spec.ts",
-    "tests/end-to-end/deleteFlows.spec.ts",
+    "tests/scheduled-end-to-end/deleteFlows.spec.ts",
-    "tests/end-to-end/dragAndDrop.spec.ts",
+    "tests/scheduled-end-to-end/dragAndDrop.spec.ts",
-    "tests/end-to-end/dropdownComponent.spec.ts",
+    "tests/scheduled-end-to-end/dropdownComponent.spec.ts",
-    "tests/end-to-end/fileUploadComponent.spec.ts",
+    "tests/scheduled-end-to-end/fileUploadComponent.spec.ts",
     "tests/end-to-end/filterEdge.spec.ts",
-    "tests/end-to-end/floatComponent.spec.ts",
+    "tests/scheduled-end-to-end/floatComponent.spec.ts",
-    "tests/end-to-end/flowPage.spec.ts",
+    "tests/scheduled-end-to-end/flowPage.spec.ts",
-    "tests/end-to-end/flowSettings.spec.ts",
+    "tests/scheduled-end-to-end/flowSettings.spec.ts",
-    "tests/end-to-end/generalBugs-shard-0.spec.ts",
+    "tests/scheduled-end-to-end/generalBugs-shard-0.spec.ts",
     "tests/end-to-end/globalVariables.spec.ts",
     "tests/end-to-end/group.spec.ts",
-    "tests/end-to-end/folders.spec.ts",
+    "tests/scheduled-end-to-end/folders.spec.ts",
-    "tests/end-to-end/inputComponent.spec.ts",
+    "tests/scheduled-end-to-end/inputComponent.spec.ts",
     "tests/end-to-end/inputListComponent.spec.ts",
-    "tests/end-to-end/intComponent.spec.ts",
+    "tests/scheduled-end-to-end/intComponent.spec.ts",
     "tests/end-to-end/keyPairListComponent.spec.ts",
-    "tests/end-to-end/langflowShortcuts.spec.ts",
+    "tests/scheduled-end-to-end/langflowShortcuts.spec.ts",
     "tests/end-to-end/nestedComponent.spec.ts",
-    "tests/end-to-end/promptModalComponent.spec.ts",
+    "tests/scheduled-end-to-end/promptModalComponent.spec.ts",
     "tests/end-to-end/python_api_generation.spec.ts",
-    "tests/end-to-end/textAreaModalComponent.spec.ts",
+    "tests/scheduled-end-to-end/textAreaModalComponent.spec.ts",
     "tests/end-to-end/textInputOutput.spec.ts",
-    "tests/end-to-end/toggleComponent.spec.ts",
+    "tests/scheduled-end-to-end/toggleComponent.spec.ts",
     "tests/end-to-end/tweaks_test.spec.ts",
-    "tests/end-to-end/twoEdges.spec.ts",
+    "tests/scheduled-end-to-end/twoEdges.spec.ts",
-    "tests/end-to-end/userSettings.spec.ts",
+    "tests/scheduled-end-to-end/userSettings.spec.ts",
     "tests/end-to-end/store.spec.ts",
     "tests/end-to-end/logs.spec.ts"
   ]