test: add new tests based on new basic examples + vector store tests (#2879)

* 🔧 (.github/workflows/typescript_test.yml): add BRAVE_SEARCH_API_KEY secret to workflow environment variables
✨ (frontend/src/modals/IOModal/components/chatView/chatMessage/index.tsx): add data-testid attribute to div element for testing purposes
✨ (frontend/tests/end-to-end/Hierarchical Tasks Agent.spec.ts): add end-to-end test for Hierarchical Tasks Agent
✨ (frontend/tests/end-to-end/Sequential Tasks Agent.spec.ts): add end-to-end test for Sequential Tasks Agent

* updating hierarchical task json

* 🔧 (.github/workflows/typescript_test.yml): add secrets for ASTRA_DB_APPLICATION_TOKEN and ASTRA_DB_API_ENDPOINT
♻️ (playwright.config.ts): refactor timeout value to 3 minutes
📝 (Complex Agent.spec.ts): update test to use BRAVE_SEARCH_API_KEY instead of OPENAI_API_KEY
📝 (Complex Agent.spec.ts): update test to use "apple" instead of "agile"
📝 (Hierarchical Tasks Agent.spec.ts): update test to use "langflow" instead of "agile"
📝 (Sequential Tasks Agent.spec.ts): update test to use waitForTimeout instead of isVisible

✨ (Vector Store.spec.ts): Add checks to skip tests if required environment variables are not set
🔧 (Vector Store.spec.ts): Update test logic to fill in required environment variables before running tests to ensure proper test execution

.github/workflows/typescript_test.yml

@@ -56,6 +56,9 @@ jobs:
     env:
       OPENAI_API_KEY: ${{ inputs.openai_api_key || secrets.OPENAI_API_KEY }}
       STORE_API_KEY: ${{ inputs.store_api_key || secrets.STORE_API_KEY }}
+      BRAVE_SEARCH_API_KEY: "${{ secrets.BRAVE_SEARCH_API_KEY }}"
+      ASTRA_DB_APPLICATION_TOKEN: "${{ secrets.ASTRA_DB_APPLICATION_TOKEN }}"
+      ASTRA_DB_API_ENDPOINT: "${{ secrets.ASTRA_DB_API_ENDPOINT }}"
     outputs:
       failed: ${{ steps.check-failure.outputs.failed }}
     steps:

src/frontend/playwright.config.ts

@@ -23,7 +23,7 @@ export default defineConfig({
   /* Opt out of parallel tests on CI. */
   workers: 2,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  timeout: 120 * 1000,
+  timeout: 3 * 60 * 1000,
   // reporter: [
   //   ["html", { open: "never", outputFolder: "playwright-report/test-results" }],
   // ],

src/frontend/src/modals/IOModal/components/chatView/chatMessage/index.tsx

@@ -173,7 +173,10 @@ export default function ChatMessage({
               )}
               {chat.thought && chat.thought !== "" && !hidden && <br></br>}
               <div className="flex w-full flex-col">
-                <div className="flex w-full flex-col dark:text-white">
+                <div
+                  className="flex w-full flex-col dark:text-white"
+                  data-testid="div-chat-message"
+                >
                   <div
                     data-testid={
                       "chat-message-" + chat.sender_name + "-" + chatMessage

src/frontend/tests/end-to-end/Complex Agent.spec.ts

@@ -0,0 +1,102 @@
+import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
+
+test("Complex Agent", async ({ page }) => {
+  test.skip(
+    !process?.env?.OPENAI_API_KEY,
+    "OPENAI_API_KEY required to run this test",
+  );
+
+  test.skip(
+    !process?.env?.BRAVE_SEARCH_API_KEY,
+    "BRAVE_SEARCH_API_KEY required to run this test",
+  );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
+  await page.goto("/");
+  await page.waitForTimeout(2000);
+
+  let modalCount = 0;
+  try {
+    const modalTitleElement = await page?.getByTestId("modal-title");
+    if (modalTitleElement) {
+      modalCount = await modalTitleElement.count();
+    }
+  } catch (error) {
+    modalCount = 0;
+  }
+
+  while (modalCount === 0) {
+    await page.getByText("New Project", { exact: true }).click();
+    await page.waitForTimeout(5000);
+    modalCount = await page.getByTestId("modal-title")?.count();
+  }
+
+  await page.getByRole("heading", { name: "Complex Agent" }).click();
+
+  await page.waitForSelector('[title="fit view"]', {
+    timeout: 100000,
+  });
+
+  await page.getByTitle("fit view").click();
+  await page.getByTitle("zoom out").click();
+  await page.getByTitle("zoom out").click();
+  await page.getByTitle("zoom out").click();
+
+  let outdatedComponents = await page.getByTestId("icon-AlertTriangle").count();
+
+  while (outdatedComponents > 0) {
+    await page.getByTestId("icon-AlertTriangle").first().click();
+    await page.waitForTimeout(1000);
+    outdatedComponents = await page.getByTestId("icon-AlertTriangle").count();
+  }
+
+  await page
+    .getByTestId("popover-anchor-input-api_key")
+    .last()
+    .fill(process.env.BRAVE_SEARCH_API_KEY ?? "");
+
+  await page.waitForTimeout(2000);
+
+  let openAiLlms = await page.getByText("OpenAI", { exact: true }).count();
+
+  for (let i = 0; i < openAiLlms; i++) {
+    await page
+      .getByTestId("popover-anchor-input-api_key")
+      .nth(i)
+      .fill(process.env.OPENAI_API_KEY ?? "");
+
+    await page.getByTestId("dropdown-model_name").nth(i).click();
+    await page.getByTestId("gpt-4o-1-option").last().click();
+
+    await page.waitForTimeout(1000);
+  }
+
+  await page.getByTestId("button_run_chat output").click();
+  await page.waitForSelector("text=built successfully", { timeout: 60000 * 3 });
+
+  await page.getByText("built successfully").last().click({
+    timeout: 15000,
+  });
+
+  await page.getByText("Playground", { exact: true }).click();
+
+  await page.waitForTimeout(2000);
+
+  expect(
+    page.getByText("Could you search info about AAPL?", { exact: true }).last(),
+  ).toBeVisible();
+
+  const textContents = await page
+    .getByTestId("div-chat-message")
+    .allTextContents();
+
+  const concatAllText = textContents.join(" ");
+  expect(concatAllText.toLocaleLowerCase()).toContain("apple");
+  const allTextLength = concatAllText.length;
+  expect(allTextLength).toBeGreaterThan(500);
+});

src/frontend/tests/end-to-end/Hierarchical Tasks Agent.spec.ts

@@ -0,0 +1,106 @@
+import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
+
+test("Hierarchical Tasks Agent", async ({ page }) => {
+  test.skip(
+    !process?.env?.OPENAI_API_KEY,
+    "OPENAI_API_KEY required to run this test",
+  );
+
+  test.skip(
+    !process?.env?.BRAVE_SEARCH_API_KEY,
+    "BRAVE_SEARCH_API_KEY required to run this test",
+  );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
+  await page.goto("/");
+  await page.waitForTimeout(2000);
+
+  let modalCount = 0;
+  try {
+    const modalTitleElement = await page?.getByTestId("modal-title");
+    if (modalTitleElement) {
+      modalCount = await modalTitleElement.count();
+    }
+  } catch (error) {
+    modalCount = 0;
+  }
+
+  while (modalCount === 0) {
+    await page.getByText("New Project", { exact: true }).click();
+    await page.waitForTimeout(5000);
+    modalCount = await page.getByTestId("modal-title")?.count();
+  }
+
+  await page.getByRole("heading", { name: "Hierarchical Tasks Agent" }).click();
+
+  await page.waitForSelector('[title="fit view"]', {
+    timeout: 100000,
+  });
+
+  await page.getByTitle("fit view").click();
+  await page.getByTitle("zoom out").click();
+  await page.getByTitle("zoom out").click();
+  await page.getByTitle("zoom out").click();
+
+  let outdatedComponents = await page.getByTestId("icon-AlertTriangle").count();
+
+  while (outdatedComponents > 0) {
+    await page.getByTestId("icon-AlertTriangle").first().click();
+    await page.waitForTimeout(1000);
+    outdatedComponents = await page.getByTestId("icon-AlertTriangle").count();
+  }
+
+  await page
+    .getByTestId("popover-anchor-input-api_key")
+    .first()
+    .fill(process.env.OPENAI_API_KEY ?? "");
+
+  await page
+    .getByTestId("popover-anchor-input-api_key")
+    .nth(1)
+    .fill(process.env.OPENAI_API_KEY ?? "");
+
+  await page.getByTestId("dropdown-model_name").first().click();
+  await page.getByTestId("gpt-4o-1-option").first().click();
+
+  await page.waitForTimeout(2000);
+
+  await page.getByTestId("dropdown-model_name").last().click();
+  await page.getByTestId("gpt-4o-1-option").last().click();
+
+  await page.waitForTimeout(2000);
+
+  await page
+    .getByTestId("popover-anchor-input-api_key")
+    .last()
+    .fill(process.env.BRAVE_SEARCH_API_KEY ?? "");
+
+  await page.waitForTimeout(2000);
+
+  await page.getByTestId("button_run_chat output").click();
+  await page.waitForSelector("text=built successfully", { timeout: 60000 * 3 });
+
+  await page.getByText("built successfully").last().click({
+    timeout: 15000,
+  });
+
+  await page.getByText("Playground", { exact: true }).click();
+
+  await page.waitForTimeout(2000);
+
+  expect(page.getByText("What is Langflow?", { exact: true })).toBeVisible();
+
+  const textContents = await page
+    .getByTestId("div-chat-message")
+    .allTextContents();
+
+  const concatAllText = textContents.join(" ");
+  expect(concatAllText.toLocaleLowerCase()).toContain("langflow");
+  const allTextLength = concatAllText.length;
+  expect(allTextLength).toBeGreaterThan(500);
+});

src/frontend/tests/end-to-end/Sequential Tasks Agent.spec.ts

@@ -0,0 +1,106 @@
+import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
+
+test("Sequential Tasks Agent", async ({ page }) => {
+  test.skip(
+    !process?.env?.OPENAI_API_KEY,
+    "OPENAI_API_KEY required to run this test",
+  );
+
+  test.skip(
+    !process?.env?.BRAVE_SEARCH_API_KEY,
+    "BRAVE_SEARCH_API_KEY required to run this test",
+  );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
+  await page.goto("/");
+  await page.waitForTimeout(2000);
+
+  let modalCount = 0;
+  try {
+    const modalTitleElement = await page?.getByTestId("modal-title");
+    if (modalTitleElement) {
+      modalCount = await modalTitleElement.count();
+    }
+  } catch (error) {
+    modalCount = 0;
+  }
+
+  while (modalCount === 0) {
+    await page.getByText("New Project", { exact: true }).click();
+    await page.waitForTimeout(5000);
+    modalCount = await page.getByTestId("modal-title")?.count();
+  }
+
+  await page.getByRole("heading", { name: "Sequential Tasks Agent" }).click();
+
+  await page.waitForSelector('[title="fit view"]', {
+    timeout: 100000,
+  });
+
+  await page.getByTitle("fit view").click();
+  await page.getByTitle("zoom out").click();
+  await page.getByTitle("zoom out").click();
+  await page.getByTitle("zoom out").click();
+
+  let outdatedComponents = await page.getByTestId("icon-AlertTriangle").count();
+
+  while (outdatedComponents > 0) {
+    await page.getByTestId("icon-AlertTriangle").first().click();
+    await page.waitForTimeout(1000);
+    outdatedComponents = await page.getByTestId("icon-AlertTriangle").count();
+  }
+
+  await page
+    .getByTestId("popover-anchor-input-api_key")
+    .first()
+    .fill(process.env.OPENAI_API_KEY ?? "");
+
+  await page.getByTestId("dropdown-model_name").click();
+  await page.getByTestId("gpt-4o-1-option").click();
+
+  await page.waitForTimeout(2000);
+
+  await page
+    .getByTestId("popover-anchor-input-api_key")
+    .last()
+    .fill(process.env.BRAVE_SEARCH_API_KEY ?? "");
+
+  await page.waitForTimeout(2000);
+
+  await page.getByTestId("button_run_chat output").click();
+  await page.waitForSelector("text=built successfully", { timeout: 60000 * 3 });
+
+  await page.getByText("built successfully").last().click({
+    timeout: 15000,
+  });
+
+  await page.getByText("Playground", { exact: true }).click();
+
+  await page.waitForTimeout(2000);
+
+  expect(
+    page
+      .getByPlaceholder("No chat input variables found. Click to run your flow")
+      .last(),
+  ).toBeVisible();
+
+  await page.getByText("Topic", { exact: true }).nth(1).isVisible();
+  await page.getByText("Topic", { exact: true }).nth(1).click();
+  expect(await page.getByPlaceholder("Enter text...").inputValue()).toBe(
+    "Agile",
+  );
+
+  const textContents = await page
+    .getByTestId("div-chat-message")
+    .allTextContents();
+
+  const concatAllText = textContents.join(" ");
+  expect(concatAllText.toLocaleLowerCase()).toContain("agile");
+  const allTextLength = concatAllText.length;
+  expect(allTextLength).toBeGreaterThan(500);
+});

src/frontend/tests/end-to-end/Vector Store.spec.ts

@@ -2,6 +2,21 @@ import { expect, test } from "@playwright/test";
 import path from "path";
 
 test("Vector Store RAG", async ({ page }) => {
+  test.skip(
+    !process?.env?.OPENAI_API_KEY,
+    "OPENAI_API_KEY required to run this test",
+  );
+
+  test.skip(
+    !process?.env?.ASTRA_DB_API_ENDPOINT,
+    "ASTRA_DB_API_ENDPOINT required to run this test",
+  );
+
+  test.skip(
+    !process?.env?.ASTRA_DB_APPLICATION_TOKEN,
+    "ASTRA_DB_APPLICATION_TOKEN required to run this test",
+  );
+
   await page.goto("/");
   await page.waitForTimeout(2000);
 
@@ -31,47 +46,46 @@ test("Vector Store RAG", async ({ page }) => {
   await page.getByTitle("zoom out").click();
   await page.getByTitle("zoom out").click();
 
-  // if (
+  let outdatedComponents = await page.getByTestId("icon-AlertTriangle").count();
-  //   !process.env.OPENAI_API_KEY ||
-  //   !process.env.ASTRA_DB_API_ENDPOINT ||
-  //   !process.env.ASTRA_DB_APPLICATION_TOKEN
-  // ) {
-  //   //You must set the OPENAI_API_KEY, ASTRA_DB_API_ENDPOINT and ASTRA_DB_APPLICATION_TOKEN on .env file to run this test
-  //   expect(false).toBe(true);
-  // }
 
-  // await page
+  while (outdatedComponents > 0) {
-  //   .getByTestId("popover-anchor-input-openai_api_key")
+    await page.getByTestId("icon-AlertTriangle").first().click();
-  //   .nth(0)
+    await page.waitForTimeout(1000);
-  //   .fill(process.env.OPENAI_API_KEY ?? "");
+    outdatedComponents = await page.getByTestId("icon-AlertTriangle").count();
+  }
 
-  // await page
+  await page
-  //   .getByTestId("popover-anchor-input-openai_api_key")
+    .getByTestId("popover-anchor-input-api_key")
-  //   .nth(1)
+    .nth(0)
-  //   .fill(process.env.OPENAI_API_KEY ?? "");
+    .fill(process.env.OPENAI_API_KEY ?? "");
 
-  // await page
+  await page
-  //   .getByTestId("popover-anchor-input-openai_api_key")
+    .getByTestId("popover-anchor-input-openai_api_key")
-  //   .nth(2)
+    .nth(0)
-  //   .fill(process.env.OPENAI_API_KEY ?? "");
+    .fill(process.env.OPENAI_API_KEY ?? "");
 
-  // await page
+  await page
-  //   .getByTestId("popover-anchor-input-token")
+    .getByTestId("popover-anchor-input-openai_api_key")
-  //   .nth(0)
+    .nth(1)
-  //   .fill(process.env.ASTRA_DB_APPLICATION_TOKEN ?? "");
+    .fill(process.env.OPENAI_API_KEY ?? "");
-  // await page
-  //   .getByTestId("popover-anchor-input-token")
-  //   .nth(1)
-  //   .fill(process.env.ASTRA_DB_APPLICATION_TOKEN ?? "");
 
-  // await page
+  await page
-  //   .getByTestId("popover-anchor-input-api_endpoint")
+    .getByTestId("popover-anchor-input-token")
-  //   .nth(0)
+    .nth(0)
-  //   .fill(process.env.ASTRA_DB_API_ENDPOINT ?? "");
+    .fill(process.env.ASTRA_DB_APPLICATION_TOKEN ?? "");
-  // await page
+  await page
-  //   .getByTestId("popover-anchor-input-api_endpoint")
+    .getByTestId("popover-anchor-input-token")
-  //   .nth(1)
+    .nth(1)
-  //   .fill(process.env.ASTRA_DB_API_ENDPOINT ?? "");
+    .fill(process.env.ASTRA_DB_APPLICATION_TOKEN ?? "");
+
+  await page
+    .getByTestId("popover-anchor-input-api_endpoint")
+    .nth(0)
+    .fill(process.env.ASTRA_DB_API_ENDPOINT ?? "");
+  await page
+    .getByTestId("popover-anchor-input-api_endpoint")
+    .nth(1)
+    .fill(process.env.ASTRA_DB_API_ENDPOINT ?? "");
 
   const fileChooserPromise = page.waitForEvent("filechooser");
   await page.getByTestId("icon-FileSearch2").last().click();
@@ -79,51 +93,60 @@ test("Vector Store RAG", async ({ page }) => {
   await fileChooser.setFiles(path.join(__dirname, "/assets/test_file.txt"));
   await page.getByText("test_file.txt").isVisible();
 
-  // await page.getByTestId("button_run_astra db").first().click();
+  await page.waitForTimeout(2000);
-  // await page.waitForSelector("text=built successfully", { timeout: 30000 });
 
-  // await page.getByText("built successfully").last().click({
+  await page.getByTestId("button_run_astra db").first().click();
-  //   timeout: 30000,
+  await page.waitForSelector("text=built successfully", { timeout: 30000 });
-  // });
 
-  // await page.getByTestId("button_run_chat output").click();
+  await page.getByText("built successfully").last().click({
-  // await page.waitForSelector("text=built successfully", { timeout: 30000 });
+    timeout: 30000,
+  });
 
-  // await page.getByText("built successfully").last().click({
+  await page.getByTestId("button_run_chat output").click();
-  //   timeout: 30000,
+  await page.waitForSelector("text=built successfully", { timeout: 30000 });
-  // });
 
-  // await page.getByText("Playground", { exact: true }).click();
+  await page.getByText("built successfully").last().click({
+    timeout: 30000,
+  });
 
-  // await page.waitForSelector('[data-testid="input-chat-playground"]', {
+  await page.getByTestId("button_run_astra db").last().click();
-  //   timeout: 100000,
+  await page.waitForSelector("text=built successfully", { timeout: 30000 });
-  // });
 
-  // await page.getByTestId("input-chat-playground").last().fill("hello");
+  await page.getByText("built successfully").last().click({
+    timeout: 30000,
+  });
 
-  // await page.getByTestId("icon-LucideSend").last().click();
+  await page.getByText("Playground", { exact: true }).click();
 
-  // await page
+  await page.waitForSelector('[data-testid="input-chat-playground"]', {
-  //   .getByText("This is a test file.", { exact: true })
+    timeout: 100000,
-  //   .last()
+  });
-  //   .isVisible();
 
-  // await page.getByText("Memories", { exact: true }).last().click();
+  await page.getByTestId("input-chat-playground").last().fill("hello");
-  // await page.getByText("Default Session").last().click();
 
-  // await page.getByText("timestamp", { exact: true }).last().isVisible();
+  await page.getByTestId("icon-LucideSend").last().click();
-  // await page.getByText("text", { exact: true }).last().isVisible();
-  // await page.getByText("sender", { exact: true }).last().isVisible();
-  // await page.getByText("sender_name", { exact: true }).last().isVisible();
-  // await page.getByText("session_id", { exact: true }).last().isVisible();
-  // await page.getByText("files", { exact: true }).last().isVisible();
 
-  // await page.getByRole("gridcell").last().isVisible();
+  await page
-  // await page.getByTestId("icon-Trash2").first().click();
+    .getByText("This is a test file.", { exact: true })
+    .last()
+    .isVisible();
 
-  // await page.waitForSelector('[data-testid="input-chat-playground"]', {
+  await page.getByText("Memories", { exact: true }).last().click();
-  //   timeout: 100000,
+  await page.getByText("Default Session").last().click();
-  // });
 
-  // await page.getByTestId("input-chat-playground").last().isVisible();
+  await page.getByText("timestamp", { exact: true }).last().isVisible();
+  await page.getByText("text", { exact: true }).last().isVisible();
+  await page.getByText("sender", { exact: true }).last().isVisible();
+  await page.getByText("sender_name", { exact: true }).last().isVisible();
+  await page.getByText("session_id", { exact: true }).last().isVisible();
+  await page.getByText("files", { exact: true }).last().isVisible();
+
+  await page.getByRole("gridcell").last().isVisible();
+  await page.getByTestId("icon-Trash2").first().click();
+
+  await page.waitForSelector('[data-testid="input-chat-playground"]', {
+    timeout: 100000,
+  });
+
+  await page.getByTestId("input-chat-playground").last().isVisible();
 });