Add Frontend Tests (#1571)

* Add TypeScript test workflow

* Update follow-redirects and katex versions

* Add Python setup and Poetry installation for backend

* Update Poetry version and setup Python in workflows

* Add Poetry installation step to GitHub Actions workflow

* Add Playwright report artifact upload and improve test script

* Update Playwright test configuration and add global teardown script

* Update path for playwright-report directory

* Update timeout value in playwright.config.ts

* Update page URLs in end-to-end tests

* Update GitHub Actions workflow and Playwright configuration

* Update TypeScript test workflow

* Add pattern and merge-multiple options to artifact download

* Update TypeScript test workflow to install Poetry

* Add cache steps for Playwright and Poetry

* Update PLAYWRIGHT_BROWSERS_PATH in TypeScript test workflow

* Add 'stuff/' to .gitignore

* Remove caching of Poetry virtualenv

* Update frontend tests to use Playwright for UI testing

* Add global teardown for removing temp database

* Add cache-hit condition to setup-node and setup-python steps

* Add new file to .gitignore and update ignored files

* Update playwright cache key in TypeScript test workflow

* Update path for blob-report in GitHub workflow

* Update path for playwright cache

* Update dependency installation in workflows

* Update baseURL in playwright.config.ts

* Update baseURL in playwright.config.ts

* Refactor test timeouts

* Remove playwright-report index.html file

* Add npm run start command to playwright.config.ts

* Update npm start command in playwright.config.ts

* Update Playwright browser caching and installation

* Update playwright cache path

* Update playwright cache path

* Update actions/cache version to v4

* Update Playwright cache key to use package-lock.json

* Update Playwright cache and install dependencies

* Fix typo in Playwright installation command

* Fix npm ci command in TypeScript test workflow

* Update TypeScript test workflow
This commit is contained in:
Gabriel Luiz Freitas Almeida 2024-03-27 14:27:40 -03:00 committed by GitHub
commit f19c449f15
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
26 changed files with 386 additions and 172 deletions

View file

@ -1,27 +0,0 @@
name: Playwright Tests
on:
push:
branches: [ main, master ]
pull_request:
branches: [ main, master ]
jobs:
test:
timeout-minutes: 60
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 18
- name: Install dependencies
run: npm ci
- name: Install Playwright Browsers
run: npx playwright install --with-deps
- name: Run Playwright tests
run: npx playwright test
- uses: actions/upload-artifact@v3
if: always()
with:
name: playwright-report
path: playwright-report/
retention-days: 30

View file

@ -5770,9 +5770,9 @@
}
},
"node_modules/follow-redirects": {
"version": "1.15.5",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz",
"integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==",
"version": "1.15.6",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz",
"integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==",
"funding": [
{
"type": "individual",
@ -7079,9 +7079,9 @@
}
},
"node_modules/katex": {
"version": "0.16.9",
"resolved": "https://registry.npmjs.org/katex/-/katex-0.16.9.tgz",
"integrity": "sha512-fsSYjWS0EEOwvy81j3vRA8TEAhQhKiqO+FQaKWp0m39qwOzHVBgAUBIXWj1pB+O2W3fIpNa6Y9KSKCVbfPhyAQ==",
"version": "0.16.10",
"resolved": "https://registry.npmjs.org/katex/-/katex-0.16.10.tgz",
"integrity": "sha512-ZiqaC04tp2O5utMsl2TEZTXxa6WSC4yo0fv5ML++D3QZv/vx2Mct0mTlRx3O+uUkjfuAgOkzsCmq5MiUEsDDdA==",
"funding": [
"https://opencollective.com/katex",
"https://github.com/sponsors/katex"

View file

@ -1,15 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Document</title>
</head>
<body>
<ul>
<li>
<a href="./e2e/index.html">e2e report</a>
</li>
</ul>
</body>
</html>

View file

@ -12,7 +12,7 @@ import { defineConfig, devices } from "@playwright/test";
export default defineConfig({
testDir: "./tests",
/* Run tests in files in parallel */
fullyParallel: false,
fullyParallel: true,
/* Fail the build on CI if you accidentally left test.only in the source code. */
forbidOnly: !!process.env.CI,
/* Retry on CI only */
@ -20,18 +20,22 @@ export default defineConfig({
/* Opt out of parallel tests on CI. */
workers: process.env.CI ? 2 : undefined,
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
reporter: [
["html", { open: "never", outputFolder: "playwright-report/test-results" }],
],
timeout: 120 * 1000,
// reporter: [
// ["html", { open: "never", outputFolder: "playwright-report/test-results" }],
// ],
reporter: process.env.CI ? "blob" : "html",
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
use: {
/* Base URL to use in actions like `await page.goto('/')`. */
// baseURL: "http://127.0.0.1:3000",
baseURL: "http://localhost:3000/",
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
trace: "on-first-retry",
},
globalTeardown: require.resolve("./tests/globalTeardown.ts"),
/* Configure projects for major browsers */
projects: [
{
@ -48,39 +52,28 @@ export default defineConfig({
// name: "webkit",
// use: { ...devices["Desktop Safari"] },
// },
/* Test against mobile viewports. */
// {
// name: 'Mobile Chrome',
// use: { ...devices['Pixel 5'] },
// },
// {
// name: 'Mobile Safari',
// use: { ...devices['iPhone 12'] },
// },
/* Test against branded browsers. */
// {
// name: 'Microsoft Edge',
// use: { ...devices['Desktop Edge'], channel: 'msedge' },
// },
// {
// name: 'Google Chrome',
// use: { ...devices['Desktop Chrome'], channel: 'chrome' },
// },
],
/* Run your local dev server before starting the tests */
// webServer: [
// {
// command: "npm run backend",
// reuseExistingServer: !process.env.CI,
// timeout: 120 * 1000,
// },
// {
// command: "npm run start",
// url: "http://127.0.0.1:3000",
// reuseExistingServer: !process.env.CI,
// },
// ],
webServer: [
{
command:
"poetry run uvicorn --factory langflow.main:create_app --host 127.0.0.1 --port 7860",
port: 7860,
env: {
LANGFLOW_DATABASE_URL: "sqlite:///./temp",
LANGFLOW_AUTO_LOGIN: "true",
},
stdout: "ignore",
reuseExistingServer: !process.env.CI,
timeout: 120 * 1000,
},
{
command: "npm start",
port: 3000,
env: {
VITE_PROXY_TARGET: "http://127.0.0.1:7860",
},
},
],
});

View file

@ -3,6 +3,17 @@
# Default value for the --ui flag
ui=false
# Absolute path to the project root directory
PROJECT_ROOT="../../"
# Check if necessary commands are available
for cmd in npx poetry fuser; do
if ! command -v $cmd &> /dev/null; then
echo "Error: Required command '$cmd' is not installed. Aborting."
exit 1
fi
done
# Parse command-line arguments
while [[ $# -gt 0 ]]; do
key="$1"
@ -23,54 +34,85 @@ done
terminate_process_by_port() {
port="$1"
echo "Terminating process on port: $port"
fuser -k -n tcp "$port" # Forcefully terminate processes using the specified port
echo "Process terminated."
if ! fuser -k -n tcp "$port"; then
echo "Failed to terminate process on port $port. Please check manually."
else
echo "Process terminated."
fi
}
delete_temp() {
cd ../../
echo "Deleting temp database"
rm temp
echo "Temp database deleted."
if cd "$PROJECT_ROOT"; then
echo "Deleting temp database"
rm -f temp && echo "Temp database deleted." || echo "Failed to delete temp database."
else
echo "Failed to navigate to project root for cleanup."
fi
}
# Trap signals to ensure cleanup on script termination
trap 'terminate_process_by_port 7860; terminate_process_by_port 3000; delete_temp' EXIT
# install playwright if there is not installed yet
npx playwright install
# Ensure the script is executed from the project root directory
if ! cd "$PROJECT_ROOT"; then
echo "Error: Failed to navigate to project root directory. Aborting."
exit 1
fi
# Navigate to the project root directory (where the Makefile is located)
cd ../../
# Install playwright if not installed yet
if ! npx playwright install; then
echo "Error: Failed to install Playwright. Aborting."
exit 1
fi
# Start the frontend using 'make frontend' in the background
make frontend &
# Start the frontend
make frontend > /dev/null 2>&1 &
# Give some time for the frontend to start (adjust sleep duration as needed)
# Adjust sleep duration as needed
sleep 10
#install backend
poetry install --extras deploy
# Install backend dependencies
if ! poetry install; then
echo "Error: Failed to install backend dependencies. Aborting."
exit 1
fi
# Start the backend using 'make backend' in the background
LANGFLOW_DATABASE_URL=sqlite:///./temp LANGFLOW_AUTO_LOGIN=True poetry run langflow run --backend-only --port 7860 --host 0.0.0.0 --no-open-browser --env-file .env &
# Give some time for the backend to start (adjust sleep duration as needed)
# Start the backend
LANGFLOW_DATABASE_URL=sqlite:///./temp LANGFLOW_AUTO_LOGIN=True poetry run langflow run --backend-only --port 7860 --host 0.0.0.0 --no-open-browser > /dev/null 2>&1 &
backend_pid=$! # Capture PID of the backend process
# Adjust sleep duration as needed
sleep 25
# Navigate to the test directory
cd src/frontend
# Run Playwright tests with or without UI based on the --ui flag
if [ "$ui" = true ]; then
PLAYWRIGHT_HTML_REPORT=playwright-report/e2e npx playwright test tests/end-to-end --ui --project=chromium
else
PLAYWRIGHT_HTML_REPORT=playwright-report/e2e npx playwright test tests/end-to-end --project=chromium
if ! cd src/frontend; then
echo "Error: Failed to navigate to test directory. Aborting."
kill $backend_pid # Terminate the backend process if navigation fails
echo "Backend process terminated."
exit 1
fi
npx playwright show-report
# Check if backend is running
if ! lsof -i :7860; then
echo "Error: Backend is not running. Aborting."
exit 1
fi
# After the tests are finished, you can add cleanup or teardown logic here if needed
# Run Playwright tests
if [ "$ui" = true ]; then
TEST_COMMAND="npx playwright test tests/end-to-end --ui --project=chromium"
else
TEST_COMMAND="npx playwright test tests/end-to-end --project=chromium"
fi
# The trap will automatically terminate processes by port on script exit
if ! PLAYWRIGHT_HTML_REPORT=playwright-report/e2e $TEST_COMMAND; then
echo "Error: Playwright tests failed. Aborting."
exit 1
fi
if [ "$ui" = true ]; then
echo "Opening Playwright report..."
npx playwright show-report
fi
trap 'terminate_process_by_port 7860; terminate_process_by_port 3000; delete_temp; kill $backend_pid 2>/dev/null' EXIT

View file

@ -1,28 +1,28 @@
import { test } from "@playwright/test";
test.beforeEach(async ({ page }) => {
await page.waitForTimeout(16000);
test.setTimeout(140000);
// await page.waitForTimeout(16000);
// test.setTimeout(140000);
});
test.describe("Auto_login tests", () => {
test("auto_login sign in", async ({ page }) => {
await page.goto("http:localhost:3000/");
await page.goto("/");
await page.locator('//*[@id="new-project-btn"]').click();
});
test("auto_login block_admin", async ({ page }) => {
await page.goto("http:localhost:3000/");
await page.goto("/");
await page.locator('//*[@id="new-project-btn"]').click();
await page.waitForTimeout(5000);
await page.goto("http:localhost:3000/login");
await page.goto("/login");
await page.locator('//*[@id="new-project-btn"]').click();
await page.waitForTimeout(5000);
await page.goto("http:localhost:3000/admin");
await page.goto("/admin");
await page.locator('//*[@id="new-project-btn"]').click();
await page.waitForTimeout(5000);
await page.goto("http:localhost:3000/admin/login");
await page.goto("/admin/login");
await page.locator('//*[@id="new-project-btn"]').click();
await page.waitForTimeout(5000);
});

View file

@ -1,10 +1,10 @@
import { expect, test } from "@playwright/test";
test.beforeEach(async ({ page }) => {
await page.waitForTimeout(2000);
test.setTimeout(120000);
// await page.waitForTimeout(2000);
// test.setTimeout(120000);
});
test("CodeAreaModalComponent", async ({ page }) => {
await page.goto("http://localhost:3000/");
await page.goto("/");
await page.waitForTimeout(2000);
await page.locator('//*[@id="new-project-btn"]').click();

View file

@ -1,13 +1,13 @@
import { expect, test } from "@playwright/test";
import { readFileSync } from "fs";
test.beforeEach(async ({ page }) => {
await page.waitForTimeout(3000);
test.setTimeout(120000);
// await page.waitForTimeout(3000);
// test.setTimeout(120000);
});
test.describe("drag and drop test", () => {
/// <reference lib="dom"/>
test("drop collection", async ({ page }) => {
await page.goto("http:localhost:3000/");
await page.goto("/");
await page.locator("span").filter({ hasText: "My Collection" }).isVisible();
// Read your file into a buffer.
const jsonContent = readFileSync(

View file

@ -1,10 +1,10 @@
import { expect, test } from "@playwright/test";
test.beforeEach(async ({ page }) => {
await page.waitForTimeout(4000);
test.setTimeout(120000);
// await page.waitForTimeout(4000);
// test.setTimeout(120000);
});
test("dropDownComponent", async ({ page }) => {
await page.goto("http://localhost:3000/");
await page.goto("/");
await page.waitForTimeout(2000);
await page.locator('//*[@id="new-project-btn"]').click();
@ -255,6 +255,6 @@ class AmazonBedrockComponent(CustomComponent):
`;
await page.locator("textarea").fill(emptyOptionsCode);
await page.getByRole('button', { name: 'Check & Save' }).click();
await page.getByRole("button", { name: "Check & Save" }).click();
await page.getByText("No parameters are available for display.").isVisible();
})
});

View file

@ -1,10 +1,10 @@
import { expect, test } from "@playwright/test";
test.beforeEach(async ({ page }) => {
await page.waitForTimeout(5000);
test.setTimeout(120000);
// await page.waitForTimeout(5000);
// test.setTimeout(120000);
});
test("FloatComponent", async ({ page }) => {
await page.goto("http://localhost:3000/");
await page.goto("/");
await page.waitForTimeout(2000);
await page.locator('//*[@id="new-project-btn"]').click();

View file

@ -1,17 +1,17 @@
import { Page, test } from "@playwright/test";
test.beforeEach(async ({ page }) => {
await page.waitForTimeout(6000);
test.setTimeout(120000);
// await page.waitForTimeout(6000);
// test.setTimeout(120000);
});
test.describe("Flow Page tests", () => {
async function goToFlowPage(page: Page) {
await page.goto("http://localhost:3000/");
await page.goto("/");
await page.getByRole("button", { name: "New Project" }).click();
}
test("save", async ({ page }) => {
await page.goto("http://localhost:3000/");
await page.goto("/");
await page.waitForTimeout(2000);
await page.locator('//*[@id="new-project-btn"]').click();

View file

@ -1,13 +1,13 @@
import { expect, test } from "@playwright/test";
import { readFileSync } from "fs";
test.beforeEach(async ({ page }) => {
await page.waitForTimeout(7000);
test.setTimeout(120000);
// await page.waitForTimeout(7000);
// test.setTimeout(120000);
});
test.describe("group node test", () => {
/// <reference lib="dom"/>
test("group and ungroup updating values", async ({ page }) => {
await page.goto("http:localhost:3000/");
await page.goto("/");
await page.locator('//*[@id="new-project-btn"]').click();
await page.getByTestId("blank-flow").click();

View file

@ -1,10 +1,10 @@
import { expect, test } from "@playwright/test";
test.beforeEach(async ({ page }) => {
await page.waitForTimeout(8000);
test.setTimeout(120000);
// await page.waitForTimeout(8000);
// test.setTimeout(120000);
});
test("InputComponent", async ({ page }) => {
await page.goto("http://localhost:3000/");
await page.goto("/");
await page.waitForTimeout(2000);
await page.locator('//*[@id="new-project-btn"]').click();

View file

@ -1,10 +1,10 @@
import { expect, test } from "@playwright/test";
test.beforeEach(async ({ page }) => {
await page.waitForTimeout(9000);
test.setTimeout(120000);
// await page.waitForTimeout(9000);
// test.setTimeout(120000);
});
test("IntComponent", async ({ page }) => {
await page.goto("http://localhost:3000/");
await page.goto("/");
await page.waitForTimeout(2000);
await page.locator('//*[@id="new-project-btn"]').click();

View file

@ -1,10 +1,10 @@
import { expect, test } from "@playwright/test";
test.beforeEach(async ({ page }) => {
await page.waitForTimeout(20000);
test.setTimeout(120000);
// await page.waitForTimeout(20000);
// test.setTimeout(120000);
});
test("KeypairListComponent", async ({ page }) => {
await page.goto("http://localhost:3000/");
await page.goto("/");
await page.waitForTimeout(2000);
await page.locator('//*[@id="new-project-btn"]').click();

View file

@ -1,8 +1,8 @@
import { expect, test } from "@playwright/test";
import uaParser from "ua-parser-js";
test.beforeEach(async ({ page }) => {
await page.waitForTimeout(11000);
test.setTimeout(120000);
// await page.waitForTimeout(11000);
// test.setTimeout(120000);
});
test("LangflowShortcuts", async ({ page }) => {
const getUA = await page.evaluate(() => navigator.userAgent);
@ -13,7 +13,7 @@ test("LangflowShortcuts", async ({ page }) => {
control = "Meta";
}
await page.goto("http://localhost:3000/");
await page.goto("/");
await page.waitForTimeout(1000);
await page.locator('//*[@id="new-project-btn"]').click();

View file

@ -1,10 +1,10 @@
import { expect, test } from "@playwright/test";
test.beforeEach(async ({ page }) => {
await page.waitForTimeout(12000);
test.setTimeout(120000);
// await page.waitForTimeout(12000);
// test.setTimeout(120000);
});
test("NestedComponent", async ({ page }) => {
await page.goto("http://localhost:3000/");
await page.goto("/");
await page.waitForTimeout(2000);
await page.locator('//*[@id="new-project-btn"]').click();

View file

@ -1,10 +1,10 @@
import { expect, test } from "@playwright/test";
test.beforeEach(async ({ page }) => {
await page.waitForTimeout(13000);
test.setTimeout(120000);
// await page.waitForTimeout(13000);
// test.setTimeout(120000);
});
test("PromptTemplateComponent", async ({ page }) => {
await page.goto("http://localhost:3000/");
await page.goto("/");
await page.waitForTimeout(2000);
await page.locator('//*[@id="new-project-btn"]').click();

View file

@ -1,8 +1,8 @@
import { Page, expect, test } from "@playwright/test";
import { readFileSync } from "fs";
test.beforeEach(async ({ page }) => {
await page.waitForTimeout(14000);
test.setTimeout(120000);
// await page.waitForTimeout(14000);
// test.setTimeout(120000);
});
test.describe("save component tests", () => {
async function saveComponent(page: Page, pattern: RegExp, n: number) {
@ -14,7 +14,7 @@ test.describe("save component tests", () => {
/// <reference lib="dom"/>
test("save group component tests", async ({ page }) => {
await page.goto("http:localhost:3000/");
await page.goto("/");
await page.locator('//*[@id="new-project-btn"]').click();
await page.getByTestId("blank-flow").click();

View file

@ -1,10 +1,10 @@
import { expect, test } from "@playwright/test";
test.beforeEach(async ({ page }) => {
await page.waitForTimeout(15000);
test.setTimeout(120000);
// await page.waitForTimeout(15000);
// test.setTimeout(120000);
});
test("ToggleComponent", async ({ page }) => {
await page.goto("http://localhost:3000/");
await page.goto("/");
await page.waitForTimeout(2000);
await page.locator('//*[@id="new-project-btn"]').click();

View file

@ -0,0 +1,25 @@
// tests/globalTeardown.ts
import fs from "fs";
import path from "path";
export default async () => {
try {
console.log("Removing the temp database");
// Check if the file exists in the path
// this file is in src/frontend/tests/globalTeardown.ts
// temp is in src/frontend/temp
const tempDbPath = path.join(__dirname, "..", "temp");
console.log("tempDbPath", tempDbPath);
// Remove the temp database
fs.rmSync(tempDbPath);
// Check if the file is removed
if (!fs.existsSync(tempDbPath)) {
console.log("Successfully removed the temp database");
} else {
console.error("Error while removing the temp database");
}
} catch (error) {
console.error("Error while removing the temp database:", error);
}
};