From e4cf3e2b9b49c027dbb637f40e7a8c97bd9fb4d3 Mon Sep 17 00:00:00 2001 From: Cristhian Zanforlin Lousa Date: Wed, 4 Dec 2024 13:49:16 -0300 Subject: [PATCH] test: Enhance frontend testability with data attributes and integration tests (#4948) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ (ChatCodeTabComponent.tsx): add data-testid attribute to ChatCodeTabComponent for testing purposes 🔧 (TemplateCardComponent/index.tsx): add data-testid attribute with dynamic value for testing 🔧 (sidebarDraggableComponent/index.tsx): import convertTestName function and add data-testid attribute with dynamic value for testing 🔧 (Custom Component Generator.spec.ts): add test for Custom Component Generator with Playwright 🔧 (Image Sentiment Analysis.spec.ts): add test for Image Sentiment Analysis with Playwright ✅ (Instagram Copywriter.spec.ts, Market Research.spec.ts): Add integration tests for Instagram Copywriter and Market Research functionalities. ✨ (Prompt Chaining.spec.ts): Add integration test for Prompt Chaining feature ✨ (SEO Keyword Generator.spec.ts): Add integration test for SEO Keyword Generator feature ✨ (SaaS Pricing.spec.ts): Add integration test for SaaS Pricing feature ✨ (Twitter Thread Generator.spec.ts): Add integration test for Twitter Thread Generator feature * ✨ (Custom Component Generator.spec.ts): Update environment variable name from OPENAI_API_KEY to ANTHROPIC_API_KEY for consistency and clarity 📝 (Custom Component Generator.spec.ts): Update test descriptions to reflect the correct environment variable names and requirements 📝 (Instagram Copywriter.spec.ts): Add support for TAVILY_API_KEY environment variable in tests and update test descriptions to reflect the change * 📝 Add test utilities for updating components, adding new API keys, adjusting screen view, awaiting bootstrap test, and getting all response messages. * 📝 (Custom Component Generator.spec.ts): Add waitForOpenModalWithChatInput utility function for better code organization and readability 📝 (Image Sentiment Analysis.spec.ts): Add buildDataTransfer utility function for better code organization and readability 📝 (Instagram Copywriter.spec.ts): Add waitForOpenModalWithChatInput utility function for better code organization and readability 📝 (Market Research.spec.ts): Add waitForOpenModalWithChatInput utility function for better code organization and readability 📝 (Prompt Chaining.spec.ts): Add waitForOpenModalWithChatInput utility function for better code organization and readability 📝 (SEO Keyword Generator.spec.ts): Add waitForOpenModalWithoutChatInput utility function for better code organization and readability 📝 (SaaS Pricing.spec.ts): Add waitForOpenModalWithoutChatInput utility function for better code organization and readability 📝 (Twitter Thread Generator.spec.ts): Add waitForOpenModalWithoutChatInput utility function for better code organization and readability 📝 (build-data-transfer.ts): Add buildDataTransfer utility function for better code organization and readability 📝 (wait-for-open-modal.ts): Add waitForOpenModalWithChatInput and waitForOpenModalWithoutChatInput utility functions for better code organization and readability * 🔧 (ci.yml): add ANTHROPIC_API_KEY and TAVILY_API_KEY secrets for CI workflows 🔧 (nightly_build.yml): add ANTHROPIC_API_KEY and TAVILY_API_KEY secrets for nightly build jobs 🔧 (typescript_test.yml): add ANTHROPIC_API_KEY and TAVILY_API_KEY secrets for TypeScript test workflows * feat: Add initialGPTsetup utility function for setting up GPT environment This commit adds the initialGPTsetup utility function to the test/utils directory. The function is responsible for setting up the GPT environment by performing tasks such as adjusting the screen view, updating old components, removing old API keys, adding new API keys, and selecting the GPT model. The function accepts optional parameters to skip specific tasks if needed. * change to utility function * change to utility function * change to utility function * change to utility function * change to utility function * change to utility function * change to utility function * change to utility function --------- Co-authored-by: anovazzi1 --- .github/workflows/ci.yml | 3 +- .github/workflows/nightly_build.yml | 2 + .github/workflows/typescript_test.yml | 8 +- .../src/CustomNodes/GenericNode/index.tsx | 1 + .../ChatCodeTabComponent.tsx | 5 +- .../TemplateCardComponent/index.tsx | 1 + .../components/UpdateAllComponents/index.tsx | 1 + .../Custom Component Generator.spec.ts | 69 +++++++++++++++ .../Image Sentiment Analysis.spec.ts | 87 +++++++++++++++++++ .../integrations/Instagram Copywriter.spec.ts | 68 +++++++++++++++ .../core/integrations/Market Research.spec.ts | 68 +++++++++++++++ .../core/integrations/Prompt Chaining.spec.ts | 57 ++++++++++++ .../SEO Keyword Generator.spec.ts | 59 +++++++++++++ .../core/integrations/SaaS Pricing.spec.ts | 60 +++++++++++++ .../Travel Planning Agent.spec.ts | 69 ++------------- .../Twitter Thread Generator.spec.ts | 61 +++++++++++++ src/frontend/tests/utils/add-new-api-keys.ts | 12 +++ .../tests/utils/adjust-screen-view.ts | 8 ++ .../tests/utils/await-bootstrap-test.ts | 29 +++++++ .../tests/utils/build-data-transfer.ts | 19 ++++ .../tests/utils/get-all-response-message.ts | 21 +++++ src/frontend/tests/utils/initialGPTsetup.ts | 33 +++++++ .../tests/utils/remove-old-api-keys.ts | 9 ++ src/frontend/tests/utils/select-gpt-model.ts | 12 +++ .../tests/utils/update-old-components.ts | 12 +++ .../tests/utils/wait-for-open-modal.ts | 13 +++ 26 files changed, 724 insertions(+), 63 deletions(-) create mode 100644 src/frontend/tests/core/integrations/Custom Component Generator.spec.ts create mode 100644 src/frontend/tests/core/integrations/Image Sentiment Analysis.spec.ts create mode 100644 src/frontend/tests/core/integrations/Instagram Copywriter.spec.ts create mode 100644 src/frontend/tests/core/integrations/Market Research.spec.ts create mode 100644 src/frontend/tests/core/integrations/Prompt Chaining.spec.ts create mode 100644 src/frontend/tests/core/integrations/SEO Keyword Generator.spec.ts create mode 100644 src/frontend/tests/core/integrations/SaaS Pricing.spec.ts create mode 100644 src/frontend/tests/core/integrations/Twitter Thread Generator.spec.ts create mode 100644 src/frontend/tests/utils/add-new-api-keys.ts create mode 100644 src/frontend/tests/utils/adjust-screen-view.ts create mode 100644 src/frontend/tests/utils/await-bootstrap-test.ts create mode 100644 src/frontend/tests/utils/build-data-transfer.ts create mode 100644 src/frontend/tests/utils/get-all-response-message.ts create mode 100644 src/frontend/tests/utils/initialGPTsetup.ts create mode 100644 src/frontend/tests/utils/remove-old-api-keys.ts create mode 100644 src/frontend/tests/utils/select-gpt-model.ts create mode 100644 src/frontend/tests/utils/update-old-components.ts create mode 100644 src/frontend/tests/utils/wait-for-open-modal.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80c9b9290..3a60d6fb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,6 +98,8 @@ jobs: secrets: OPENAI_API_KEY: "${{ secrets.OPENAI_API_KEY }}" STORE_API_KEY: "${{ secrets.STORE_API_KEY }}" + ANTHROPIC_API_KEY: "${{ secrets.ANTHROPIC_API_KEY }}" + TAVILY_API_KEY: "${{ secrets.TAVILY_API_KEY }}" lint-backend: needs: path-filter @@ -105,7 +107,6 @@ jobs: name: Lint Backend uses: ./.github/workflows/lint-py.yml - test-docs-build: needs: path-filter if: ${{ needs.path-filter.outputs.docs == 'true' }} diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index 77691c7ac..70f0c21b4 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -136,6 +136,8 @@ jobs: secrets: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} STORE_API_KEY: ${{ secrets.STORE_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} backend-unit-tests: if: github.repository == 'langflow-ai/langflow' diff --git a/.github/workflows/typescript_test.yml b/.github/workflows/typescript_test.yml index 0a2709b6b..db242b337 100644 --- a/.github/workflows/typescript_test.yml +++ b/.github/workflows/typescript_test.yml @@ -7,12 +7,16 @@ on: required: true STORE_API_KEY: required: true + ANTHROPIC_API_KEY: + required: true + TAVILY_API_KEY: + required: true inputs: suites: description: "Test suites to run (JSON array)" required: false type: string - default: "[]" + default: '[]' release: description: "Whether this is a release build" required: false @@ -223,6 +227,8 @@ jobs: SEARCH_API_KEY: "${{ secrets.SEARCH_API_KEY }}" ASTRA_DB_APPLICATION_TOKEN: "${{ secrets.ASTRA_DB_APPLICATION_TOKEN }}" ASTRA_DB_API_ENDPOINT: "${{ secrets.ASTRA_DB_API_ENDPOINT }}" + ANTHROPIC_API_KEY: "${{ secrets.ANTHROPIC_API_KEY }}" + TAVILY_API_KEY: "${{ secrets.TAVILY_API_KEY }}" UV_CACHE_DIR: /tmp/.uv-cache outputs: failed: ${{ steps.check-failure.outputs.failed }} diff --git a/src/frontend/src/CustomNodes/GenericNode/index.tsx b/src/frontend/src/CustomNodes/GenericNode/index.tsx index b2388ebfe..b8460ad5a 100644 --- a/src/frontend/src/CustomNodes/GenericNode/index.tsx +++ b/src/frontend/src/CustomNodes/GenericNode/index.tsx @@ -365,6 +365,7 @@ export default function GenericNode({ className="shrink-0 px-2.5 text-xs" onClick={handleUpdateCode} loading={loadingUpdate} + data-testid="update-button" > Update diff --git a/src/frontend/src/components/core/codeTabsComponent/ChatCodeTabComponent.tsx b/src/frontend/src/components/core/codeTabsComponent/ChatCodeTabComponent.tsx index 8c2cf5c42..945c6a53a 100644 --- a/src/frontend/src/components/core/codeTabsComponent/ChatCodeTabComponent.tsx +++ b/src/frontend/src/components/core/codeTabsComponent/ChatCodeTabComponent.tsx @@ -31,7 +31,10 @@ export default function SimplifiedCodeTabComponent({ }; return ( -
+
{language} diff --git a/src/frontend/tests/core/integrations/Custom Component Generator.spec.ts b/src/frontend/tests/core/integrations/Custom Component Generator.spec.ts new file mode 100644 index 000000000..9cc731c1e --- /dev/null +++ b/src/frontend/tests/core/integrations/Custom Component Generator.spec.ts @@ -0,0 +1,69 @@ +import { expect, test } from "@playwright/test"; +import * as dotenv from "dotenv"; +import path from "path"; +import { addNewApiKeys } from "../../utils/add-new-api-keys"; +import { adjustScreenView } from "../../utils/adjust-screen-view"; +import { awaitBootstrapTest } from "../../utils/await-bootstrap-test"; +import { getAllResponseMessage } from "../../utils/get-all-response-message"; +import { initialGPTsetup } from "../../utils/initialGPTsetup"; +import { removeOldApiKeys } from "../../utils/remove-old-api-keys"; +import { selectGptModel } from "../../utils/select-gpt-model"; +import { updateOldComponents } from "../../utils/update-old-components"; +import { waitForOpenModalWithChatInput } from "../../utils/wait-for-open-modal"; + +test( + "Custom Component Generator", + { tag: ["@release", "@starter-project"] }, + async ({ page }) => { + test.skip( + !process?.env?.ANTHROPIC_API_KEY, + "OPENAI_API_KEY required to run this test", + ); + + if (!process.env.CI) { + dotenv.config({ path: path.resolve(__dirname, "../../.env") }); + } + + await page.goto("/"); + + await awaitBootstrapTest(page); + + await page.getByTestId("side_nav_options_all-templates").click(); + await page.getByTestId("template-custom-component-generator").click(); + + await page.waitForSelector('[data-testid="fit_view"]', { + timeout: 100000, + }); + + await initialGPTsetup(page); + + const apiKeyInput = page.getByTestId( + "popover-anchor-input-anthropic_api_key", + ); + const isApiKeyInputVisible = await apiKeyInput.isVisible(); + + if (isApiKeyInputVisible) { + await apiKeyInput.fill(process.env.ANTHROPIC_API_KEY ?? ""); + } + + await page.getByTestId("button_run_chat output").click(); + await page.waitForSelector("text=built successfully", { timeout: 30000 }); + + await page.getByText("built successfully").last().click({ + timeout: 15000, + }); + + await page.getByText("Playground", { exact: true }).last().click(); + await page + .getByText("No input message provided.", { exact: true }) + .last() + .isVisible(); + + await waitForOpenModalWithChatInput(page); + + const textContents = await getAllResponseMessage(page); + expect(textContents.length).toBeGreaterThan(100); + expect(await page.getByTestId("chat-code-tab").isVisible()).toBe(true); + expect(textContents).toContain("langflow"); + }, +); diff --git a/src/frontend/tests/core/integrations/Image Sentiment Analysis.spec.ts b/src/frontend/tests/core/integrations/Image Sentiment Analysis.spec.ts new file mode 100644 index 000000000..3e3f1f822 --- /dev/null +++ b/src/frontend/tests/core/integrations/Image Sentiment Analysis.spec.ts @@ -0,0 +1,87 @@ +import { expect, test } from "@playwright/test"; +import * as dotenv from "dotenv"; +import { readFileSync } from "fs"; +import path from "path"; +import { addNewApiKeys } from "../../utils/add-new-api-keys"; +import { adjustScreenView } from "../../utils/adjust-screen-view"; +import { awaitBootstrapTest } from "../../utils/await-bootstrap-test"; +import { buildDataTransfer } from "../../utils/build-data-transfer"; +import { getAllResponseMessage } from "../../utils/get-all-response-message"; +import { initialGPTsetup } from "../../utils/initialGPTsetup"; +import { removeOldApiKeys } from "../../utils/remove-old-api-keys"; +import { selectGptModel } from "../../utils/select-gpt-model"; +import { updateOldComponents } from "../../utils/update-old-components"; +import { waitForOpenModalWithoutChatInput } from "../../utils/wait-for-open-modal"; + +test( + "Image Sentiment Analysis", + { tag: ["@release", "@starter-project"] }, + async ({ page }) => { + test.skip( + !process?.env?.OPENAI_API_KEY, + "OPENAI_API_KEY required to run this test", + ); + + if (!process.env.CI) { + dotenv.config({ path: path.resolve(__dirname, "../../.env") }); + } + + await page.goto("/"); + await awaitBootstrapTest(page); + + await page.getByTestId("side_nav_options_all-templates").click(); + await page + .getByText("Image Sentiment Analysis", { exact: true }) + .last() + .click(); + + await page.waitForSelector('[data-testid="fit_view"]', { + timeout: 100000, + }); + + await page.getByTestId("fit_view").click(); + + await initialGPTsetup(page); + + await page.getByText("Playground", { exact: true }).last().click(); + + await page.waitForSelector('[data-testid="input-chat-playground"]', { + timeout: 100000, + }); + + // Read the image file as a binary string + const filePath = "tests/assets/chain.png"; + const fileContent = readFileSync(filePath, "base64"); + + // Create the DataTransfer and File objects within the browser context + const dataTransfer = await buildDataTransfer(page, fileContent); + + await page.waitForSelector('[data-testid="input-chat-playground"]', { + timeout: 100000, + }); + + // Locate the target element + const element = await page.getByTestId("input-chat-playground"); + + // Dispatch the drop event on the target element + await element.dispatchEvent("drop", { dataTransfer }); + + await waitForOpenModalWithoutChatInput(page); + + await page.getByTestId("button-send").click(); + + await page.waitForSelector("text=chain.png", { timeout: 30000 }); + + await page.getByText("chain.png").isVisible(); + + await page.waitForSelector('[data-testid="div-chat-message"]', { + timeout: 30000, + }); + + const textContents = await getAllResponseMessage(page); + expect(textContents.length).toBeGreaterThan(10); + expect(textContents.toLowerCase()).toContain("sentiment"); + expect(textContents.toLowerCase()).toContain("neutral"); + expect(textContents.toLowerCase()).toContain("description"); + }, +); diff --git a/src/frontend/tests/core/integrations/Instagram Copywriter.spec.ts b/src/frontend/tests/core/integrations/Instagram Copywriter.spec.ts new file mode 100644 index 000000000..0ddb91d6f --- /dev/null +++ b/src/frontend/tests/core/integrations/Instagram Copywriter.spec.ts @@ -0,0 +1,68 @@ +import { expect, test } from "@playwright/test"; +import * as dotenv from "dotenv"; +import path from "path"; +import { addNewApiKeys } from "../../utils/add-new-api-keys"; +import { adjustScreenView } from "../../utils/adjust-screen-view"; +import { awaitBootstrapTest } from "../../utils/await-bootstrap-test"; +import { getAllResponseMessage } from "../../utils/get-all-response-message"; +import { initialGPTsetup } from "../../utils/initialGPTsetup"; +import { removeOldApiKeys } from "../../utils/remove-old-api-keys"; +import { selectGptModel } from "../../utils/select-gpt-model"; +import { updateOldComponents } from "../../utils/update-old-components"; +import { waitForOpenModalWithChatInput } from "../../utils/wait-for-open-modal"; + +test( + "Instagram Copywriter", + { tag: ["@release", "@starter-project"] }, + async ({ page }) => { + test.skip( + !process?.env?.OPENAI_API_KEY, + "OPENAI_API_KEY required to run this test", + ); + + test.skip( + !process?.env?.TAVILY_API_KEY, + "TAVILY_API_KEY required to run this test", + ); + + if (!process.env.CI) { + dotenv.config({ path: path.resolve(__dirname, "../../.env") }); + } + + await page.goto("/"); + await awaitBootstrapTest(page); + + await page.getByTestId("side_nav_options_all-templates").click(); + await page.getByRole("heading", { name: "Instagram Copywriter" }).click(); + + await page.waitForSelector('[data-testid="fit_view"]', { + timeout: 100000, + }); + + await initialGPTsetup(page); + + await page + .getByTestId("popover-anchor-input-api_key") + .nth(2) + .fill(process.env.TAVILY_API_KEY ?? ""); + + await page.getByTestId("button_run_chat output").click(); + await page.waitForSelector("text=built successfully", { timeout: 30000 }); + + await page.getByText("built successfully").last().click({ + timeout: 15000, + }); + + await page.getByText("Playground", { exact: true }).last().click(); + await page + .getByText("No input message provided.", { exact: true }) + .last() + .isVisible(); + + await waitForOpenModalWithChatInput(page); + + const textContents = await getAllResponseMessage(page); + + expect(textContents.length).toBeGreaterThan(300); + }, +); diff --git a/src/frontend/tests/core/integrations/Market Research.spec.ts b/src/frontend/tests/core/integrations/Market Research.spec.ts new file mode 100644 index 000000000..a275f0047 --- /dev/null +++ b/src/frontend/tests/core/integrations/Market Research.spec.ts @@ -0,0 +1,68 @@ +import { expect, test } from "@playwright/test"; +import * as dotenv from "dotenv"; +import path from "path"; +import { addNewApiKeys } from "../../utils/add-new-api-keys"; +import { adjustScreenView } from "../../utils/adjust-screen-view"; +import { awaitBootstrapTest } from "../../utils/await-bootstrap-test"; +import { getAllResponseMessage } from "../../utils/get-all-response-message"; +import { initialGPTsetup } from "../../utils/initialGPTsetup"; +import { removeOldApiKeys } from "../../utils/remove-old-api-keys"; +import { selectGptModel } from "../../utils/select-gpt-model"; +import { updateOldComponents } from "../../utils/update-old-components"; +import { waitForOpenModalWithChatInput } from "../../utils/wait-for-open-modal"; + +test( + "Market Research", + { tag: ["@release", "@starter-project"] }, + async ({ page }) => { + test.skip( + !process?.env?.OPENAI_API_KEY, + "OPENAI_API_KEY required to run this test", + ); + test.skip( + !process?.env?.TAVILY_API_KEY, + "TAVILY_API_KEY required to run this test", + ); + + if (!process.env.CI) { + dotenv.config({ path: path.resolve(__dirname, "../../.env") }); + } + + await page.goto("/"); + await awaitBootstrapTest(page); + + await page.getByTestId("side_nav_options_all-templates").click(); + await page.getByRole("heading", { name: "Market Research" }).click(); + + await page.waitForSelector('[data-testid="fit_view"]', { + timeout: 100000, + }); + + await initialGPTsetup(page); + + await page + .getByTestId("popover-anchor-input-api_key") + .nth(0) + .fill(process.env.TAVILY_API_KEY ?? ""); + + await page.getByTestId("button_run_chat output").click(); + await page.waitForSelector("text=built successfully", { timeout: 30000 }); + + await page.getByText("built successfully").last().click({ + timeout: 15000, + }); + + await page.getByText("Playground", { exact: true }).last().click(); + await page + .getByText("No input message provided.", { exact: true }) + .last() + .isVisible(); + + await waitForOpenModalWithChatInput(page); + + const textContents = await getAllResponseMessage(page); + + expect(textContents.length).toBeGreaterThan(300); + expect(textContents).toContain("amazon"); + }, +); diff --git a/src/frontend/tests/core/integrations/Prompt Chaining.spec.ts b/src/frontend/tests/core/integrations/Prompt Chaining.spec.ts new file mode 100644 index 000000000..be3a3e8cc --- /dev/null +++ b/src/frontend/tests/core/integrations/Prompt Chaining.spec.ts @@ -0,0 +1,57 @@ +import { expect, test } from "@playwright/test"; +import * as dotenv from "dotenv"; +import path from "path"; +import { addNewApiKeys } from "../../utils/add-new-api-keys"; +import { adjustScreenView } from "../../utils/adjust-screen-view"; +import { awaitBootstrapTest } from "../../utils/await-bootstrap-test"; +import { getAllResponseMessage } from "../../utils/get-all-response-message"; +import { initialGPTsetup } from "../../utils/initialGPTsetup"; +import { removeOldApiKeys } from "../../utils/remove-old-api-keys"; +import { selectGptModel } from "../../utils/select-gpt-model"; +import { updateOldComponents } from "../../utils/update-old-components"; +import { waitForOpenModalWithChatInput } from "../../utils/wait-for-open-modal"; + +test( + "Prompt Chaining", + { tag: ["@release", "@starter-project"] }, + async ({ page }) => { + test.skip( + !process?.env?.OPENAI_API_KEY, + "OPENAI_API_KEY required to run this test", + ); + + if (!process.env.CI) { + dotenv.config({ path: path.resolve(__dirname, "../../.env") }); + } + + await page.goto("/"); + await awaitBootstrapTest(page); + + await page.getByTestId("side_nav_options_all-templates").click(); + await page.getByRole("heading", { name: "Prompt Chaining" }).click(); + + await page.waitForSelector('[data-testid="fit_view"]', { + timeout: 100000, + }); + + await initialGPTsetup(page); + + await page.getByTestId("button_run_chat output").click(); + await page.waitForSelector("text=built successfully", { timeout: 30000 }); + + await page.getByText("built successfully").last().click({ + timeout: 15000, + }); + + await page.getByText("Playground", { exact: true }).last().click(); + await page + .getByText("No input message provided.", { exact: true }) + .last() + .isVisible(); + + await waitForOpenModalWithChatInput(page); + + const textContents = await getAllResponseMessage(page); + expect(textContents.length).toBeGreaterThan(100); + }, +); diff --git a/src/frontend/tests/core/integrations/SEO Keyword Generator.spec.ts b/src/frontend/tests/core/integrations/SEO Keyword Generator.spec.ts new file mode 100644 index 000000000..96d9cb326 --- /dev/null +++ b/src/frontend/tests/core/integrations/SEO Keyword Generator.spec.ts @@ -0,0 +1,59 @@ +import { expect, test } from "@playwright/test"; +import * as dotenv from "dotenv"; +import path from "path"; +import { addNewApiKeys } from "../../utils/add-new-api-keys"; +import { adjustScreenView } from "../../utils/adjust-screen-view"; +import { awaitBootstrapTest } from "../../utils/await-bootstrap-test"; +import { getAllResponseMessage } from "../../utils/get-all-response-message"; +import { initialGPTsetup } from "../../utils/initialGPTsetup"; +import { removeOldApiKeys } from "../../utils/remove-old-api-keys"; +import { selectGptModel } from "../../utils/select-gpt-model"; +import { updateOldComponents } from "../../utils/update-old-components"; +import { waitForOpenModalWithoutChatInput } from "../../utils/wait-for-open-modal"; + +test( + "SEO Keyword Generator", + { tag: ["@release", "@starter-project"] }, + async ({ page }) => { + test.skip( + !process?.env?.OPENAI_API_KEY, + "OPENAI_API_KEY required to run this test", + ); + + if (!process.env.CI) { + dotenv.config({ path: path.resolve(__dirname, "../../.env") }); + } + + await page.goto("/"); + await awaitBootstrapTest(page); + + await page.getByTestId("side_nav_options_all-templates").click(); + await page.getByRole("heading", { name: "SEO Keyword Generator" }).click(); + + await page.waitForSelector('[data-testid="fit_view"]', { + timeout: 100000, + }); + + await initialGPTsetup(page); + + await page.getByTestId("button_run_chat output").click(); + await page.waitForSelector("text=built successfully", { timeout: 30000 }); + + await page.getByText("built successfully").last().click({ + timeout: 15000, + }); + + await page.getByText("Playground", { exact: true }).last().click(); + await page + .getByText("No input message provided.", { exact: true }) + .last() + .isVisible(); + + await waitForOpenModalWithoutChatInput(page); + + const textContents = await getAllResponseMessage(page); + + expect(textContents.length).toBeGreaterThan(200); + expect(textContents).toContain("work"); + }, +); diff --git a/src/frontend/tests/core/integrations/SaaS Pricing.spec.ts b/src/frontend/tests/core/integrations/SaaS Pricing.spec.ts new file mode 100644 index 000000000..efadb0c47 --- /dev/null +++ b/src/frontend/tests/core/integrations/SaaS Pricing.spec.ts @@ -0,0 +1,60 @@ +import { expect, test } from "@playwright/test"; +import * as dotenv from "dotenv"; +import path from "path"; +import { addNewApiKeys } from "../../utils/add-new-api-keys"; +import { adjustScreenView } from "../../utils/adjust-screen-view"; +import { awaitBootstrapTest } from "../../utils/await-bootstrap-test"; +import { getAllResponseMessage } from "../../utils/get-all-response-message"; +import { initialGPTsetup } from "../../utils/initialGPTsetup"; +import { removeOldApiKeys } from "../../utils/remove-old-api-keys"; +import { selectGptModel } from "../../utils/select-gpt-model"; +import { updateOldComponents } from "../../utils/update-old-components"; +import { waitForOpenModalWithoutChatInput } from "../../utils/wait-for-open-modal"; + +test( + "SaaS Pricing", + { tag: ["@release", "@starter-project"] }, + async ({ page }) => { + test.skip( + !process?.env?.OPENAI_API_KEY, + "OPENAI_API_KEY required to run this test", + ); + + if (!process.env.CI) { + dotenv.config({ path: path.resolve(__dirname, "../../.env") }); + } + + await page.goto("/"); + await awaitBootstrapTest(page); + + await page.getByTestId("side_nav_options_all-templates").click(); + await page.getByRole("heading", { name: "SaaS Pricing" }).click(); + + await page.waitForSelector('[data-testid="fit_view"]', { + timeout: 100000, + }); + + await initialGPTsetup(page); + + await page.getByTestId("button_run_chat output").click(); + await page.waitForSelector("text=built successfully", { timeout: 30000 }); + + await page.getByText("built successfully").last().click({ + timeout: 15000, + }); + + await page.getByText("Playground", { exact: true }).last().click(); + await page + .getByText("No input message provided.", { exact: true }) + .last() + .isVisible(); + + await waitForOpenModalWithoutChatInput(page); + + const textContents = await getAllResponseMessage(page); + + expect(textContents.length).toBeGreaterThan(100); + expect(textContents).toContain("costs"); + expect(textContents).toContain("subscription"); + }, +); diff --git a/src/frontend/tests/core/integrations/Travel Planning Agent.spec.ts b/src/frontend/tests/core/integrations/Travel Planning Agent.spec.ts index 3d4158cb5..658da4b0e 100644 --- a/src/frontend/tests/core/integrations/Travel Planning Agent.spec.ts +++ b/src/frontend/tests/core/integrations/Travel Planning Agent.spec.ts @@ -1,6 +1,13 @@ import { expect, Page, test } from "@playwright/test"; import * as dotenv from "dotenv"; import path from "path"; +import { addNewApiKeys } from "../../utils/add-new-api-keys"; +import { adjustScreenView } from "../../utils/adjust-screen-view"; +import { awaitBootstrapTest } from "../../utils/await-bootstrap-test"; +import { initialGPTsetup } from "../../utils/initialGPTsetup"; +import { removeOldApiKeys } from "../../utils/remove-old-api-keys"; +import { selectGptModel } from "../../utils/select-gpt-model"; +import { updateOldComponents } from "../../utils/update-old-components"; test( "Travel Planning Agent", @@ -21,29 +28,7 @@ test( } await page.goto("/"); - await page.waitForSelector('[data-testid="mainpage_title"]', { - timeout: 30000, - }); - - await page.waitForSelector('[id="new-project-btn"]', { - timeout: 30000, - }); - - let modalCount = 0; - try { - const modalTitleElement = await page?.getByTestId("modal-title"); - if (modalTitleElement) { - modalCount = await modalTitleElement.count(); - } - } catch (error) { - modalCount = 0; - } - - while (modalCount === 0) { - await page.getByText("New Flow", { exact: true }).click(); - await page.waitForTimeout(3000); - modalCount = await page.getByTestId("modal-title")?.count(); - } + await awaitBootstrapTest(page); await page.getByTestId("side_nav_options_all-templates").click(); await page @@ -55,27 +40,7 @@ test( timeout: 100000, }); - await page.getByTestId("fit_view").click(); - await page.getByTestId("zoom_out").click(); - await page.getByTestId("zoom_out").click(); - await page.getByTestId("zoom_out").click(); - - let outdatedComponents = await page - .getByTestId("icon-AlertTriangle") - .count(); - - while (outdatedComponents > 0) { - await page.getByTestId("icon-AlertTriangle").first().click(); - await page.waitForTimeout(1000); - outdatedComponents = await page.getByTestId("icon-AlertTriangle").count(); - } - - let filledApiKey = await page.getByTestId("remove-icon-badge").count(); - while (filledApiKey > 0) { - await page.getByTestId("remove-icon-badge").first().click(); - await page.waitForTimeout(1000); - filledApiKey = await page.getByTestId("remove-icon-badge").count(); - } + await initialGPTsetup(page); const randomCity = cities[Math.floor(Math.random() * cities.length)]; const randomCity2 = cities[Math.floor(Math.random() * cities.length)]; @@ -88,22 +53,6 @@ test( `Create a travel plan from ${randomCity} to ${randomCity2} with ${randomFood}`, ); - let openAiLlms = await page.getByText("OpenAI", { exact: true }).count(); - await page.waitForSelector('[data-testid="fit_view"]', { - timeout: 100000, - }); - - for (let i = 0; i < openAiLlms; i++) { - await page - .getByTestId("popover-anchor-input-api_key") - .nth(i + 1) - .fill(process.env.OPENAI_API_KEY ?? ""); - await page.getByTestId("zoom_in").click(); - await page.getByTestId("dropdown_str_model_name").nth(i).click(); - await page.getByTestId("gpt-4o-1-option").last().click(); - await page.waitForTimeout(1000); - } - await page .getByTestId("popover-anchor-input-api_key") .first() diff --git a/src/frontend/tests/core/integrations/Twitter Thread Generator.spec.ts b/src/frontend/tests/core/integrations/Twitter Thread Generator.spec.ts new file mode 100644 index 000000000..bced1d125 --- /dev/null +++ b/src/frontend/tests/core/integrations/Twitter Thread Generator.spec.ts @@ -0,0 +1,61 @@ +import { expect, test } from "@playwright/test"; +import * as dotenv from "dotenv"; +import path from "path"; +import { addNewApiKeys } from "../../utils/add-new-api-keys"; +import { adjustScreenView } from "../../utils/adjust-screen-view"; +import { awaitBootstrapTest } from "../../utils/await-bootstrap-test"; +import { getAllResponseMessage } from "../../utils/get-all-response-message"; +import { initialGPTsetup } from "../../utils/initialGPTsetup"; +import { removeOldApiKeys } from "../../utils/remove-old-api-keys"; +import { selectGptModel } from "../../utils/select-gpt-model"; +import { updateOldComponents } from "../../utils/update-old-components"; +import { waitForOpenModalWithoutChatInput } from "../../utils/wait-for-open-modal"; + +test( + "Twitter Thread Generator", + { tag: ["@release", "@starter-project"] }, + async ({ page }) => { + test.skip( + !process?.env?.OPENAI_API_KEY, + "OPENAI_API_KEY required to run this test", + ); + + if (!process.env.CI) { + dotenv.config({ path: path.resolve(__dirname, "../../.env") }); + } + + await page.goto("/"); + await awaitBootstrapTest(page); + + await page.getByTestId("side_nav_options_all-templates").click(); + await page + .getByRole("heading", { name: "Twitter Thread Generator" }) + .click(); + + await page.waitForSelector('[data-testid="fit_view"]', { + timeout: 100000, + }); + + await initialGPTsetup(page); + + await page.getByTestId("button_run_chat output").click(); + await page.waitForSelector("text=built successfully", { timeout: 30000 }); + + await page.getByText("built successfully").last().click({ + timeout: 15000, + }); + + await page.getByText("Playground", { exact: true }).last().click(); + await page + .getByText("No input message provided.", { exact: true }) + .last() + .isVisible(); + + await waitForOpenModalWithoutChatInput(page); + + const textContents = await getAllResponseMessage(page); + + expect(textContents.length).toBeGreaterThan(100); + expect(textContents).toContain("langflow"); + }, +); diff --git a/src/frontend/tests/utils/add-new-api-keys.ts b/src/frontend/tests/utils/add-new-api-keys.ts new file mode 100644 index 000000000..c51cecaa9 --- /dev/null +++ b/src/frontend/tests/utils/add-new-api-keys.ts @@ -0,0 +1,12 @@ +import { Page } from "playwright/test"; + +export async function addNewApiKeys(page: Page) { + const apiKeyInput = page.getByTestId("popover-anchor-input-api_key"); + const isApiKeyInputVisible = await apiKeyInput.count(); + + if (isApiKeyInputVisible > 0) { + for (let i = 0; i < isApiKeyInputVisible; i++) { + await apiKeyInput.nth(i).fill(process.env.OPENAI_API_KEY ?? ""); + } + } +} diff --git a/src/frontend/tests/utils/adjust-screen-view.ts b/src/frontend/tests/utils/adjust-screen-view.ts new file mode 100644 index 000000000..51f0d9720 --- /dev/null +++ b/src/frontend/tests/utils/adjust-screen-view.ts @@ -0,0 +1,8 @@ +import { Page } from "playwright/test"; + +export async function adjustScreenView(page: Page) { + await page.getByTestId("fit_view").click(); + await page.getByTestId("zoom_out").click(); + await page.getByTestId("zoom_out").click(); + await page.getByTestId("zoom_out").click(); +} diff --git a/src/frontend/tests/utils/await-bootstrap-test.ts b/src/frontend/tests/utils/await-bootstrap-test.ts new file mode 100644 index 000000000..e26e54d77 --- /dev/null +++ b/src/frontend/tests/utils/await-bootstrap-test.ts @@ -0,0 +1,29 @@ +import { Page } from "playwright/test"; + +export const awaitBootstrapTest = async (page: Page) => { + await page.waitForSelector('[data-testid="mainpage_title"]', { + timeout: 30000, + }); + + await page.waitForSelector('[id="new-project-btn"]', { + timeout: 30000, + }); + + let modalCount = 0; + try { + const modalTitleElement = await page?.getByTestId("modal-title"); + if (modalTitleElement) { + modalCount = await modalTitleElement.count(); + } + } catch (error) { + modalCount = 0; + } + + while (modalCount === 0) { + await page.getByText("New Flow", { exact: true }).click(); + await page.waitForSelector('[data-testid="modal-title"]', { + timeout: 3000, + }); + modalCount = await page.getByTestId("modal-title")?.count(); + } +}; diff --git a/src/frontend/tests/utils/build-data-transfer.ts b/src/frontend/tests/utils/build-data-transfer.ts new file mode 100644 index 000000000..17758d23a --- /dev/null +++ b/src/frontend/tests/utils/build-data-transfer.ts @@ -0,0 +1,19 @@ +import { Page } from "playwright/test"; + +export const buildDataTransfer = async (page: Page, fileContent: string) => { + return await page.evaluateHandle( + ({ fileContent }) => { + const dt = new DataTransfer(); + const byteCharacters = atob(fileContent); + const byteNumbers = new Array(byteCharacters.length); + for (let i = 0; i < byteCharacters.length; i++) { + byteNumbers[i] = byteCharacters.charCodeAt(i); + } + const byteArray = new Uint8Array(byteNumbers); + const file = new File([byteArray], "chain.png", { type: "image/png" }); + dt.items.add(file); + return dt; + }, + { fileContent }, + ); +}; diff --git a/src/frontend/tests/utils/get-all-response-message.ts b/src/frontend/tests/utils/get-all-response-message.ts new file mode 100644 index 000000000..553beb045 --- /dev/null +++ b/src/frontend/tests/utils/get-all-response-message.ts @@ -0,0 +1,21 @@ +import { Page } from "playwright/test"; + +export const getAllResponseMessage = async (page: Page) => { + const numberOfResponseMessages = await page + .getByTestId("div-chat-message") + .count(); + + const textContents: string[] = []; + for (let i = 0; i < numberOfResponseMessages; i++) { + const textContent = await page + .getByTestId("div-chat-message") + .nth(i) + .textContent(); + + if (textContent) { + textContents.push(textContent); + } + } + + return textContents.join(" ").toLowerCase(); +}; diff --git a/src/frontend/tests/utils/initialGPTsetup.ts b/src/frontend/tests/utils/initialGPTsetup.ts new file mode 100644 index 000000000..64d756143 --- /dev/null +++ b/src/frontend/tests/utils/initialGPTsetup.ts @@ -0,0 +1,33 @@ +import { Page } from "@playwright/test"; +import { addNewApiKeys } from "./add-new-api-keys"; +import { adjustScreenView } from "./adjust-screen-view"; +import { removeOldApiKeys } from "./remove-old-api-keys"; +import { selectGptModel } from "./select-gpt-model"; +import { updateOldComponents } from "./update-old-components"; + +export async function initialGPTsetup( + page: Page, + options?: { + skipAdjustScreenView?: boolean; + skipUpdateOldComponents?: boolean; + skipRemoveOldApiKeys?: boolean; + skipAddNewApiKeys?: boolean; + skipSelectGptModel?: boolean; + }, +) { + if (!options?.skipAdjustScreenView) { + await adjustScreenView(page); + } + if (!options?.skipUpdateOldComponents) { + await updateOldComponents(page); + } + if (!options?.skipRemoveOldApiKeys) { + await removeOldApiKeys(page); + } + if (!options?.skipAddNewApiKeys) { + await addNewApiKeys(page); + } + if (!options?.skipSelectGptModel) { + await selectGptModel(page); + } +} diff --git a/src/frontend/tests/utils/remove-old-api-keys.ts b/src/frontend/tests/utils/remove-old-api-keys.ts new file mode 100644 index 000000000..9504cd2f9 --- /dev/null +++ b/src/frontend/tests/utils/remove-old-api-keys.ts @@ -0,0 +1,9 @@ +import { Page } from "playwright/test"; + +export async function removeOldApiKeys(page: Page) { + let filledApiKey = await page.getByTestId("remove-icon-badge").count(); + while (filledApiKey > 0) { + await page.getByTestId("remove-icon-badge").first().click(); + filledApiKey = await page.getByTestId("remove-icon-badge").count(); + } +} diff --git a/src/frontend/tests/utils/select-gpt-model.ts b/src/frontend/tests/utils/select-gpt-model.ts new file mode 100644 index 000000000..0d6c6eecd --- /dev/null +++ b/src/frontend/tests/utils/select-gpt-model.ts @@ -0,0 +1,12 @@ +import { Page } from "playwright/test"; + +export const selectGptModel = async (page: Page) => { + const gptModelDropdownCount = await page + .getByTestId("dropdown_str_model_name") + .count(); + + if (gptModelDropdownCount > 0) { + await page.getByTestId("dropdown_str_model_name").nth(0).click(); + await page.getByTestId("gpt-4o-1-option").click(); + } +}; diff --git a/src/frontend/tests/utils/update-old-components.ts b/src/frontend/tests/utils/update-old-components.ts new file mode 100644 index 000000000..a858a5273 --- /dev/null +++ b/src/frontend/tests/utils/update-old-components.ts @@ -0,0 +1,12 @@ +import { Page } from "playwright/test"; + +export async function updateOldComponents(page: Page) { + const hasUpdateAllButton = await page + .getByTestId("update-all-button") + .count(); + if (hasUpdateAllButton === 0) { + return; + } + await page.getByTestId("update-all-button").click(); + await page.waitForSelector("text=successfully updated", { timeout: 10000 }); +} diff --git a/src/frontend/tests/utils/wait-for-open-modal.ts b/src/frontend/tests/utils/wait-for-open-modal.ts new file mode 100644 index 000000000..a638b97f9 --- /dev/null +++ b/src/frontend/tests/utils/wait-for-open-modal.ts @@ -0,0 +1,13 @@ +import { Page } from "playwright/test"; + +export const waitForOpenModalWithChatInput = async (page: Page) => { + await page.waitForSelector('[data-testid="input-chat-playground"]', { + timeout: 10000, + }); +}; + +export const waitForOpenModalWithoutChatInput = async (page: Page) => { + await page.waitForSelector('[data-testid="button-send"]', { + timeout: 100000, + }); +};