fix: spec files cant find .env variable on CI (#2514)

* bugfix: add env import to spec files correctly

* 🔧 (ci.yml): fix indentation for consistency and readability in CI workflow

* ci: add inputs to workflow

* bugfix: improve flaky test general bugs to run smoothier on CI

* ci: add more shards to improve tests on CI

.github/workflows/ci.yml

@@ -8,90 +8,99 @@ on:
         description: "(Optional) Branch to checkout"
         required: false
         type: string
+      openai_api_key:
+        description: "OpenAI API Key"
+        required: false
+        type: string
+      store_api_key:
+        description: "Store API Key"
+        required: false
+        type: string
   pull_request:
 
+
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-    path-filter:
+  path-filter:
-        name: Filter Paths
+    name: Filter Paths
-        runs-on: ubuntu-latest
+    runs-on: ubuntu-latest
-        outputs:
+    outputs:
-            python: ${{ steps.filter.outputs.python }}
+      python: ${{ steps.filter.outputs.python }}
-            frontend: ${{ steps.filter.outputs.frontend }}
+      frontend: ${{ steps.filter.outputs.frontend }}
-            docs: ${{ steps.filter.outputs.docs }}
+      docs: ${{ steps.filter.outputs.docs }}
-            tests: ${{ steps.filter.outputs.tests }}
+      tests: ${{ steps.filter.outputs.tests }}
-        steps:
+    steps:
-        - name: Checkout code
+      - name: Checkout code
-          uses: actions/checkout@v4
+        uses: actions/checkout@v4
-          with:
+        with:
-            ref: ${{ inputs.branch || github.ref }}
+          ref: ${{ inputs.branch || github.ref }}
-        - name: Filter Paths
+      - name: Filter Paths
-          id: filter
+        id: filter
-          uses: dorny/paths-filter@v3
+        uses: dorny/paths-filter@v3
-          with:
+        with:
-            filters: |
+          filters: |
-              python:
+            python:
-                - "src/backend/**"
+              - "src/backend/**"
-                - "src/backend/**.py"
+              - "src/backend/**.py"
-                - "pyproject.toml"
+              - "pyproject.toml"
-                - "poetry.lock"
+              - "poetry.lock"
-                - "**/python_test.yml"
+              - "**/python_test.yml"
-              tests:
+            tests:
-                - "tests/**"
+              - "tests/**"
-                - "src/frontend/tests/**"
+              - "src/frontend/tests/**"
-              frontend:
+            frontend:
-                - "src/frontend/**"
+              - "src/frontend/**"
-                - "**/typescript_test.yml"
+              - "**/typescript_test.yml"
-              docs:
+            docs:
-                - "docs/**"
+              - "docs/**"
 
-    test-backend:
+  test-backend:
-        needs: path-filter
+    needs: path-filter
-        name: Run Backend Tests
+    name: Run Backend Tests
-        if: ${{ needs.path-filter.outputs.python == 'true' || needs.path-filter.outputs.tests == 'true' }}
+    if: ${{ needs.path-filter.outputs.python == 'true' || needs.path-filter.outputs.tests == 'true' }}
-        uses: ./.github/workflows/python_test.yml
+    uses: ./.github/workflows/python_test.yml
 
+  test-frontend:
+    needs: path-filter
+    name: Run Frontend Tests
+    if: ${{ needs.path-filter.outputs.python == 'true' || needs.path-filter.outputs.frontend == 'true' || needs.path-filter.outputs.tests == 'true' }}
+    uses: ./.github/workflows/typescript_test.yml
+    secrets:
+      OPENAI_API_KEY: "${{ secrets.OPENAI_API_KEY }}"
+      STORE_API_KEY: "${{ secrets.STORE_API_KEY }}"
 
+  lint-backend:
+    needs: path-filter
+    if: ${{ needs.path-filter.outputs.python == 'true' || needs.path-filter.outputs.tests == 'true' }}
+    name: Lint Backend
+    uses: ./.github/workflows/lint-py.yml
+    # Run only if there are python files changed
 
-    test-frontend:
+  test-docs-build:
-      needs: path-filter
+    needs: path-filter
-      name: Run Frontend Tests
+    if: ${{ needs.path-filter.outputs.docs == 'true' }}
-      if: ${{ needs.path-filter.outputs.python == 'true' || needs.path-filter.outputs.frontend == 'true' || needs.path-filter.outputs.tests == 'true' }}
+    name: Test Docs Build
-      uses: ./.github/workflows/typescript_test.yml
+    uses: ./.github/workflows/docs_test.yml
 
-
+  # https://github.com/langchain-ai/langchain/blob/master/.github/workflows/check_diffs.yml
-    lint-backend:
+  ci_success:
-        needs: path-filter
+    name: "CI Success"
-        if: ${{ needs.path-filter.outputs.python == 'true' || needs.path-filter.outputs.tests == 'true' }}
+    needs: [test-backend, test-frontend, lint-backend, test-docs-build]
-        name: Lint Backend
+    if: |
-        uses: ./.github/workflows/lint-py.yml
+      always()
-        # Run only if there are python files changed
+    runs-on: ubuntu-latest
-
+    env:
-    test-docs-build:
+      JOBS_JSON: ${{ toJSON(needs) }}
-        needs: path-filter
+      RESULTS_JSON: ${{ toJSON(needs.*.result) }}
-        if: ${{ needs.path-filter.outputs.docs == 'true' }}
+      EXIT_CODE: ${{!contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && '0' || '1'}}
-        name: Test Docs Build
+    steps:
-        uses: ./.github/workflows/docs_test.yml
+      - name: "CI Success"
-
+        run: |
-
+          echo $JOBS_JSON
-    # https://github.com/langchain-ai/langchain/blob/master/.github/workflows/check_diffs.yml
+          echo $RESULTS_JSON
-    ci_success:
+          echo "Exiting with $EXIT_CODE"
-        name: "CI Success"
+          exit $EXIT_CODE
-        needs: [test-backend, test-frontend, lint-backend, test-docs-build]
-        if: |
-            always()
-        runs-on: ubuntu-latest
-        env:
-            JOBS_JSON: ${{ toJSON(needs) }}
-            RESULTS_JSON: ${{ toJSON(needs.*.result) }}
-            EXIT_CODE: ${{!contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && '0' || '1'}}
-        steps:
-        - name: "CI Success"
-          run: |
-            echo $JOBS_JSON
-            echo $RESULTS_JSON
-            echo "Exiting with $EXIT_CODE"
-            exit $EXIT_CODE

.github/workflows/typescript_test.yml

@@ -2,6 +2,11 @@ name: Run Frontend Tests
 
 on:
   workflow_call:
+    secrets:
+      OPENAI_API_KEY:
+        required: true
+      STORE_API_KEY:
+        required: true
   workflow_dispatch:
     inputs:
       branch:
@@ -9,7 +14,6 @@ on:
         required: false
         type: string
 
-
 env:
   POETRY_VERSION: "1.8.3"
   NODE_VERSION: "21"
@@ -25,11 +29,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
-        shardTotal: [10]
+        shardTotal: [14]
     env:
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      OPENAI_API_KEY: ${{ inputs.openai_api_key || secrets.OPENAI_API_KEY }}
-      STORE_API_KEY: ${{ secrets.STORE_API_KEY }}
+      STORE_API_KEY: ${{ inputs.store_api_key || secrets.STORE_API_KEY }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

src/frontend/tests/end-to-end/Basic Prompting.spec.ts

@@ -1,4 +1,6 @@
 import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
 
 test("Basic Prompting (Hello, World)", async ({ page }) => {
   test.skip(
@@ -6,6 +8,10 @@ test("Basic Prompting (Hello, World)", async ({ page }) => {
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(2000);
 

src/frontend/tests/end-to-end/Blog Writer.spec.ts

@@ -1,4 +1,5 @@
 import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
 import path from "path";
 
 test("Blog Writer", async ({ page }) => {
@@ -7,6 +8,10 @@ test("Blog Writer", async ({ page }) => {
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(2000);
 

src/frontend/tests/end-to-end/Document QA.spec.ts

@@ -1,4 +1,5 @@
 import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
 import path from "path";
 
 test("Document QA", async ({ page }) => {
@@ -7,6 +8,10 @@ test("Document QA", async ({ page }) => {
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(2000);
 

src/frontend/tests/end-to-end/Memory Chatbot.spec.ts

@@ -1,4 +1,5 @@
 import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
 import path from "path";
 
 test("Memory Chatbot", async ({ page }) => {
@@ -7,6 +8,10 @@ test("Memory Chatbot", async ({ page }) => {
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(2000);
 

src/frontend/tests/end-to-end/chatInputOutputUser-shard-0.spec.ts

@@ -4,14 +4,15 @@ import { readFileSync } from "fs";
 import path from "path";
 
 test("user must be able to send an image on chat", async ({ page }) => {
-  if (!process.env.CI) {
-    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
-  }
   test.skip(
     !process?.env?.OPENAI_API_KEY,
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
 
   await page.waitForTimeout(1000);

src/frontend/tests/end-to-end/chatInputOutputUser-shard-1.spec.ts

@@ -4,14 +4,15 @@ import { readFileSync } from "fs";
 import path from "path";
 
 test("user must be able to see output inspection", async ({ page }) => {
-  if (!process.env.CI) {
-    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
-  }
   test.skip(
     !process?.env?.OPENAI_API_KEY,
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
 
   await page.waitForTimeout(1000);

src/frontend/tests/end-to-end/chatInputOutputUser-shard-2.spec.ts

@@ -4,14 +4,15 @@ import { readFileSync } from "fs";
 import path from "path";
 
 test("user must interact with chat with Input/Output", async ({ page }) => {
-  if (!process.env.CI) {
-    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
-  }
   test.skip(
     !process?.env?.OPENAI_API_KEY,
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
 
   await page.waitForTimeout(1000);

src/frontend/tests/end-to-end/decisionFlow.spec.ts

@@ -1,10 +1,17 @@
 import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
 
 test("should create a flow with decision", async ({ page }) => {
   test.skip(
     !process?.env?.OPENAI_API_KEY,
     "OPENAI_API_KEY required to run this test",
   );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.locator("span").filter({ hasText: "My Collection" }).isVisible();
   await page.waitForTimeout(2000);

src/frontend/tests/end-to-end/deleteComponents.spec.ts

@@ -1,10 +1,17 @@
 import { test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
 
 test("should delete a component", async ({ page }) => {
   test.skip(
     !process?.env?.STORE_API_KEY,
     "STORE_API_KEY required to run this test",
   );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(1000);
 

src/frontend/tests/end-to-end/deleteFlows.spec.ts

@@ -1,10 +1,16 @@
 import { test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
 
 test("should delete a flow", async ({ page }) => {
   test.skip(
     !process?.env?.STORE_API_KEY,
     "STORE_API_KEY required to run this test",
   );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
   await page.goto("/");
   await page.waitForTimeout(1000);
 

src/frontend/tests/end-to-end/generalBugs-shard-0.spec.ts

@@ -43,14 +43,15 @@ test("should interact with api request", async ({ page }) => {
 });
 
 test("erase button should clear the chat messages", async ({ page }) => {
-  if (!process.env.CI) {
-    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
-  }
   test.skip(
     !process?.env?.OPENAI_API_KEY,
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
 
   await page.waitForTimeout(1000);

src/frontend/tests/end-to-end/generalBugs-shard-1.spec.ts

@@ -7,7 +7,9 @@ test("should delete rows from table message", async ({ page }) => {
     !process?.env?.OPENAI_API_KEY,
     "OPENAI_API_KEY required to run this test",
   );
-
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
   await page.goto("/");
   await page.waitForTimeout(2000);
 
@@ -51,30 +53,9 @@ test("should delete rows from table message", async ({ page }) => {
   await page.waitForSelector("text=built successfully", { timeout: 30000 });
 
   await page.getByText("built successfully").last().click({
-    timeout: 15000,
+    timeout: 30000,
   });
 
-  await page.getByText("Playground", { exact: true }).click();
-  await page
-    .getByText("No input message provided.", { exact: true })
-    .last()
-    .isVisible();
-
-  await page.waitForSelector('[data-testid="input-chat-playground"]', {
-    timeout: 100000,
-  });
-
-  await page
-    .getByTestId("input-chat-playground")
-    .last()
-    .fill("Say hello as a pirate");
-  await page.getByTestId("icon-LucideSend").last().click();
-
-  await page.waitForSelector("text=matey", {
-    timeout: 100000,
-  });
-
-  await page.getByText("Close").last().click();
   await page.getByTestId("user-profile-settings").last().click();
   await page.getByText("Settings").last().click();
   await page.getByText("Messages").last().click();

src/frontend/tests/end-to-end/generalBugs-shard-2.spec.ts

@@ -1,11 +1,15 @@
 import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
 
 test("should use webhook component on API", async ({ page }) => {
   test.skip(
     !process?.env?.OPENAI_API_KEY,
     "OPENAI_API_KEY required to run this test",
   );
-
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
   await page.goto("/");
   await page.waitForTimeout(2000);
 

src/frontend/tests/end-to-end/generalBugs-shard-3.spec.ts

@@ -1,6 +1,17 @@
 import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
 
 test("should copy code from playground modal", async ({ page }) => {
+  test.skip(
+    !process?.env?.OPENAI_API_KEY,
+    "OPENAI_API_KEY required to run this test",
+  );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.locator("span").filter({ hasText: "My Collection" }).isVisible();
   await page.waitForTimeout(2000);
@@ -80,11 +91,6 @@ test("should copy code from playground modal", async ({ page }) => {
   await page.getByTitle("zoom out").click();
   await page.getByTitle("zoom out").click();
 
-  test.skip(
-    !process?.env?.OPENAI_API_KEY,
-    "OPENAI_API_KEY required to run this test",
-  );
-
   await page
     .getByTestId("popover-anchor-input-openai_api_key")
     .fill(process.env.OPENAI_API_KEY ?? "");

src/frontend/tests/end-to-end/logs.spec.ts

@@ -1,4 +1,6 @@
 import { test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
 
 test("should able to see and interact with logs", async ({ page }) => {
   test.skip(
@@ -6,6 +8,10 @@ test("should able to see and interact with logs", async ({ page }) => {
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(2000);
 

src/frontend/tests/end-to-end/store-shard-1.spec.ts

@@ -1,10 +1,17 @@
 import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
 
 test("should like and add components and flows", async ({ page }) => {
   test.skip(
     !process?.env?.STORE_API_KEY,
     "STORE_API_KEY required to run this test",
   );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(1000);
 
@@ -76,6 +83,11 @@ test("should find a searched Component on Store", async ({ page }) => {
     !process?.env?.STORE_API_KEY,
     "STORE_API_KEY required to run this test",
   );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(1000);
 

src/frontend/tests/end-to-end/store-shard-2.spec.ts

@@ -1,10 +1,17 @@
 import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
 
 test("should filter by tag", async ({ page }) => {
   test.skip(
     !process?.env?.STORE_API_KEY,
     "STORE_API_KEY required to run this test",
   );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(1000);
 
@@ -50,6 +57,11 @@ test("should share component with share button", async ({ page }) => {
     !process?.env?.STORE_API_KEY,
     "STORE_API_KEY required to run this test",
   );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(1000);
 

src/frontend/tests/end-to-end/store-shard-3.spec.ts

@@ -1,10 +1,17 @@
 import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
 
 test("should order the visualization", async ({ page }) => {
   test.skip(
     !process?.env?.STORE_API_KEY,
     "STORE_API_KEY required to run this test",
   );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(1000);
 
@@ -46,6 +53,11 @@ test("should filter by type", async ({ page }) => {
     !process?.env?.STORE_API_KEY,
     "STORE_API_KEY required to run this test",
   );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(1000);
 

src/frontend/tests/end-to-end/textInputOutput.spec.ts

@@ -3,14 +3,15 @@ import * as dotenv from "dotenv";
 import path from "path";
 
 test("TextInputOutputComponent", async ({ page }) => {
-  if (!process.env.CI) {
-    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
-  }
   test.skip(
     !process?.env?.OPENAI_API_KEY,
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(2000);