From a03da10750794b11cc43aa866a108e7e80d0ec4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Est=C3=A9vez?= Date: Thu, 31 Oct 2024 14:57:07 -0400 Subject: [PATCH] fix: langfuse upgrade and crew result usage (#4342) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * lanfuse upgrade and crew result usage * ✅ (Dynamic Agent.spec.ts): Enable tests for checking the presence of required environment variables before running the test suite. ✨ (Hierarchical Agent.spec.ts): Add integration test for Hierarchical Tasks Agent to ensure proper functionality and behavior ✨ (Sequential Task Agent.spec.ts): Add integration test for Sequential Tasks Agent to ensure proper functionality and behavior --------- Co-authored-by: cristhianzl --- pyproject.toml | 2 +- .../base/langflow/base/agents/crewai/crew.py | 2 +- .../core/integrations/Dynamic Agent.spec.ts | 195 +++++++++--------- .../integrations/Hierarchical Agent.spec.ts | 122 +++++++++++ .../Sequential Task Agent.spec.ts | 103 +++++++++ uv.lock | 8 +- 6 files changed, 325 insertions(+), 107 deletions(-) create mode 100644 src/frontend/tests/core/integrations/Hierarchical Agent.spec.ts create mode 100644 src/frontend/tests/core/integrations/Sequential Task Agent.spec.ts diff --git a/pyproject.toml b/pyproject.toml index fbf33a382..30212ddd4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ dependencies = [ "redis>=5.0.1", "metaphor-python>=0.1.11", 'pywin32>=306; sys_platform == "win32"', - "langfuse>=2.33.0", + "langfuse~=2.53.1", "metal-sdk>=2.5.0", "markupsafe>=2.1.3", "boto3~=1.34.162", diff --git a/src/backend/base/langflow/base/agents/crewai/crew.py b/src/backend/base/langflow/base/agents/crewai/crew.py index c2cc01986..8a7819955 100644 --- a/src/backend/base/langflow/base/agents/crewai/crew.py +++ b/src/backend/base/langflow/base/agents/crewai/crew.py @@ -78,6 +78,6 @@ class BaseCrewComponent(Component): async def build_output(self) -> Message: crew = self.build_crew() result = await crew.kickoff_async() - message = Message(text=result, sender=MESSAGE_SENDER_AI) + message = Message(text=result.raw, sender=MESSAGE_SENDER_AI) self.status = message return message diff --git a/src/frontend/tests/core/integrations/Dynamic Agent.spec.ts b/src/frontend/tests/core/integrations/Dynamic Agent.spec.ts index 762bec8ef..9e469296d 100644 --- a/src/frontend/tests/core/integrations/Dynamic Agent.spec.ts +++ b/src/frontend/tests/core/integrations/Dynamic Agent.spec.ts @@ -3,105 +3,98 @@ import * as dotenv from "dotenv"; import path from "path"; test("Dynamic Agent", async ({ page }) => { - // test.skip( - // !process?.env?.OPENAI_API_KEY, - // "OPENAI_API_KEY required to run this test", - // ); - // test.skip( - // !process?.env?.SEARCH_API_KEY, - // "SEARCH_API_KEY required to run this test", - // ); - // if (!process.env.CI) { - // dotenv.config({ path: path.resolve(__dirname, "../../.env") }); - // } - // await page.goto("/"); - // await page.waitForSelector('[data-testid="mainpage_title"]', { - // timeout: 30000, - // }); - // await page.waitForSelector('[id="new-project-btn"]', { - // timeout: 30000, - // }); - // let modalCount = 0; - // try { - // const modalTitleElement = await page?.getByTestId("modal-title"); - // if (modalTitleElement) { - // modalCount = await modalTitleElement.count(); - // } - // } catch (error) { - // modalCount = 0; - // } - // while (modalCount === 0) { - // await page.getByText("New Project", { exact: true }).click(); - // await page.waitForTimeout(3000); - // modalCount = await page.getByTestId("modal-title")?.count(); - // } - // await page.getByTestId("side_nav_options_all-templates").click(); - // await page.getByRole("heading", { name: "Dynamic Agent" }).last().click(); - // await page.waitForSelector('[data-testid="fit_view"]', { - // timeout: 100000, - // }); - // await page.getByTestId("fit_view").click(); - // await page.getByTestId("zoom_out").click(); - // await page.getByTestId("zoom_out").click(); - // await page.getByTestId("zoom_out").click(); - // let outdatedComponents = await page.getByTestId("icon-AlertTriangle").count(); - // while (outdatedComponents > 0) { - // await page.getByTestId("icon-AlertTriangle").first().click(); - // await page.waitForTimeout(1000); - // outdatedComponents = await page.getByTestId("icon-AlertTriangle").count(); - // } - // let filledApiKey = await page.getByTestId("remove-icon-badge").count(); - // while (filledApiKey > 0) { - // await page.getByTestId("remove-icon-badge").first().click(); - // await page.waitForTimeout(1000); - // filledApiKey = await page.getByTestId("remove-icon-badge").count(); - // } - // await page - // .getByTestId("popover-anchor-input-api_key") - // .last() - // .fill(process.env.SEARCH_API_KEY ?? ""); - // await page.waitForTimeout(1000); - // let openAiLlms = await page.getByText("OpenAI", { exact: true }).count(); - // await page.waitForSelector('[data-testid="fit_view"]', { - // timeout: 100000, - // }); - // for (let i = 0; i < openAiLlms; i++) { - // await page - // .getByTestId("popover-anchor-input-api_key") - // .nth(i) - // .fill(process.env.OPENAI_API_KEY ?? ""); - // await page.getByTestId("zoom_in").click(); - // await page.getByTestId("dropdown_str_model_name").nth(i).click(); - // await page.getByTestId("gpt-4o-1-option").last().click(); - // await page.waitForTimeout(1000); - // } - // await page - // .getByTestId("textarea_str_input_value") - // .first() - // .fill("how much is an apple stock today"); - // await page.getByTestId("button_run_chat output").click(); - // const result = await Promise.race([ - // page.waitForSelector("text=OpenAIException", { timeout: 30000 }), - // page.waitForSelector("text=Error building", { timeout: 30000 }), - // ]); - // if (result) { - // test.skip(); - // } - // await page.waitForSelector("text=built successfully", { timeout: 60000 * 3 }); - // await page.getByText("built successfully").last().click({ - // timeout: 15000, - // }); - // await page.getByText("Playground", { exact: true }).last().click(); - // await page.waitForTimeout(1000); - // expect(page.getByText("apple").last()).toBeVisible(); - // const textContents = await page - // .getByTestId("div-chat-message") - // .allTextContents(); - // const concatAllText = textContents.join(" "); - // expect(concatAllText.toLocaleLowerCase()).toContain("apple"); - // expect(concatAllText.toLocaleLowerCase()).not.toContain("error"); - // expect(concatAllText.toLocaleLowerCase()).not.toContain("apologize"); - // expect(concatAllText.toLocaleLowerCase()).not.toContain("unable"); - // const allTextLength = concatAllText.length; - // expect(allTextLength).toBeGreaterThan(100); + test.skip( + !process?.env?.OPENAI_API_KEY, + "OPENAI_API_KEY required to run this test", + ); + test.skip( + !process?.env?.SEARCH_API_KEY, + "SEARCH_API_KEY required to run this test", + ); + if (!process.env.CI) { + dotenv.config({ path: path.resolve(__dirname, "../../.env") }); + } + await page.goto("/"); + await page.waitForSelector('[data-testid="mainpage_title"]', { + timeout: 30000, + }); + await page.waitForSelector('[id="new-project-btn"]', { + timeout: 30000, + }); + let modalCount = 0; + try { + const modalTitleElement = await page?.getByTestId("modal-title"); + if (modalTitleElement) { + modalCount = await modalTitleElement.count(); + } + } catch (error) { + modalCount = 0; + } + while (modalCount === 0) { + await page.getByText("New Project", { exact: true }).click(); + await page.waitForTimeout(3000); + modalCount = await page.getByTestId("modal-title")?.count(); + } + await page.getByTestId("side_nav_options_all-templates").click(); + await page.getByRole("heading", { name: "Dynamic Agent" }).last().click(); + await page.waitForSelector('[data-testid="fit_view"]', { + timeout: 100000, + }); + await page.getByTestId("fit_view").click(); + await page.getByTestId("zoom_out").click(); + await page.getByTestId("zoom_out").click(); + await page.getByTestId("zoom_out").click(); + let outdatedComponents = await page.getByTestId("icon-AlertTriangle").count(); + while (outdatedComponents > 0) { + await page.getByTestId("icon-AlertTriangle").first().click(); + await page.waitForTimeout(1000); + outdatedComponents = await page.getByTestId("icon-AlertTriangle").count(); + } + let filledApiKey = await page.getByTestId("remove-icon-badge").count(); + while (filledApiKey > 0) { + await page.getByTestId("remove-icon-badge").first().click(); + await page.waitForTimeout(1000); + filledApiKey = await page.getByTestId("remove-icon-badge").count(); + } + await page + .getByTestId("popover-anchor-input-api_key") + .last() + .fill(process.env.SEARCH_API_KEY ?? ""); + await page.waitForTimeout(1000); + let openAiLlms = await page.getByText("OpenAI", { exact: true }).count(); + await page.waitForSelector('[data-testid="fit_view"]', { + timeout: 100000, + }); + for (let i = 0; i < openAiLlms; i++) { + await page + .getByTestId("popover-anchor-input-api_key") + .nth(i) + .fill(process.env.OPENAI_API_KEY ?? ""); + await page.getByTestId("zoom_in").click(); + await page.getByTestId("dropdown_str_model_name").nth(i).click(); + await page.getByTestId("gpt-4o-1-option").last().click(); + await page.waitForTimeout(1000); + } + await page + .getByTestId("textarea_str_input_value") + .first() + .fill("how much is an apple stock today"); + await page.getByTestId("button_run_chat output").click(); + + await page.waitForSelector("text=built successfully", { timeout: 60000 * 3 }); + await page.getByText("built successfully").last().click({ + timeout: 15000, + }); + await page.getByText("Playground", { exact: true }).last().click(); + await page.waitForTimeout(1000); + expect(page.getByText("apple").last()).toBeVisible(); + const textContents = await page + .getByTestId("div-chat-message") + .allTextContents(); + const concatAllText = textContents.join(" "); + expect(concatAllText.toLocaleLowerCase()).toContain("apple"); + expect(concatAllText.toLocaleLowerCase()).not.toContain("error"); + expect(concatAllText.toLocaleLowerCase()).not.toContain("apologize"); + const allTextLength = concatAllText.length; + expect(allTextLength).toBeGreaterThan(100); }); diff --git a/src/frontend/tests/core/integrations/Hierarchical Agent.spec.ts b/src/frontend/tests/core/integrations/Hierarchical Agent.spec.ts new file mode 100644 index 000000000..c2ea70aba --- /dev/null +++ b/src/frontend/tests/core/integrations/Hierarchical Agent.spec.ts @@ -0,0 +1,122 @@ +import { expect, test } from "@playwright/test"; +import * as dotenv from "dotenv"; +import path from "path"; +import uaParser from "ua-parser-js"; + +test("Hierarchical Tasks Agent", async ({ page }) => { + test.skip( + !process?.env?.OPENAI_API_KEY, + "OPENAI_API_KEY required to run this test", + ); + + test.skip( + !process?.env?.SEARCH_API_KEY, + "SEARCH_API_KEY required to run this test", + ); + + if (!process.env.CI) { + dotenv.config({ path: path.resolve(__dirname, "../../.env") }); + } + + await page.goto("/"); + await page.waitForSelector('[data-testid="mainpage_title"]', { + timeout: 30000, + }); + + await page.waitForSelector('[id="new-project-btn"]', { + timeout: 30000, + }); + + let modalCount = 0; + try { + const modalTitleElement = await page?.getByTestId("modal-title"); + if (modalTitleElement) { + modalCount = await modalTitleElement.count(); + } + } catch (error) { + modalCount = 0; + } + + while (modalCount === 0) { + await page.getByText("New Project", { exact: true }).click(); + await page.waitForTimeout(3000); + modalCount = await page.getByTestId("modal-title")?.count(); + } + + const getUA = await page.evaluate(() => navigator.userAgent); + const userAgentInfo = uaParser(getUA); + let control = "Control"; + + if (userAgentInfo.os.name.includes("Mac")) { + control = "Meta"; + } + + await page.getByTestId("side_nav_options_all-templates").click(); + await page + .getByRole("heading", { name: "Hierarchical Tasks Agent" }) + .first() + .click(); + + await page.waitForSelector('[data-testid="fit_view"]', { + timeout: 100000, + }); + + await page.getByTestId("fit_view").click(); + await page.getByTestId("zoom_out").click(); + await page.getByTestId("zoom_out").click(); + await page.getByTestId("zoom_out").click(); + + let outdatedComponents = await page.getByTestId("icon-AlertTriangle").count(); + + while (outdatedComponents > 0) { + await page.getByTestId("icon-AlertTriangle").first().click(); + await page.waitForTimeout(1000); + outdatedComponents = await page.getByTestId("icon-AlertTriangle").count(); + } + + let filledApiKey = await page.getByTestId("remove-icon-badge").count(); + while (filledApiKey > 0) { + await page.getByTestId("remove-icon-badge").first().click(); + await page.waitForTimeout(1000); + filledApiKey = await page.getByTestId("remove-icon-badge").count(); + } + + await page + .getByTestId("popover-anchor-input-api_key") + .nth(1) + .fill(process.env.OPENAI_API_KEY ?? ""); + + await page + .getByTestId("popover-anchor-input-api_key") + .nth(0) + .fill(process.env.OPENAI_API_KEY ?? ""); + + await page.getByTestId("dropdown_str_model_name").nth(1).click(); + await page.getByTestId("gpt-4o-1-option").last().click(); + + await page.getByTestId("dropdown_str_model_name").nth(0).click(); + await page.getByTestId("gpt-4o-1-option").first().click(); + + await page + .getByTestId("popover-anchor-input-api_key") + .last() + .fill(process.env.SEARCH_API_KEY ?? ""); + + await page.waitForTimeout(1000); + + await page.getByTestId("button_run_chat output").click(); + + await page.waitForTimeout(1000); + + await page.waitForSelector("text=built successfully", { timeout: 30000 }); + + await page.getByText("built successfully").last().click({ + timeout: 15000, + }); + + await page.getByText("Playground", { exact: true }).last().click(); + + expect(await page.locator(".markdown").count()).toBeGreaterThan(0); + + expect(await page.getByText("Langflow").count()).toBeGreaterThan(2); +}); diff --git a/src/frontend/tests/core/integrations/Sequential Task Agent.spec.ts b/src/frontend/tests/core/integrations/Sequential Task Agent.spec.ts new file mode 100644 index 000000000..f481364ec --- /dev/null +++ b/src/frontend/tests/core/integrations/Sequential Task Agent.spec.ts @@ -0,0 +1,103 @@ +import { expect, test } from "@playwright/test"; +import * as dotenv from "dotenv"; +import path from "path"; +import uaParser from "ua-parser-js"; + +test("Sequential Task Agent", async ({ page }) => { + test.skip( + !process?.env?.OPENAI_API_KEY, + "OPENAI_API_KEY required to run this test", + ); + + if (!process.env.CI) { + dotenv.config({ path: path.resolve(__dirname, "../../.env") }); + } + + await page.goto("/"); + await page.waitForSelector('[data-testid="mainpage_title"]', { + timeout: 30000, + }); + + await page.waitForSelector('[id="new-project-btn"]', { + timeout: 30000, + }); + + let modalCount = 0; + try { + const modalTitleElement = await page?.getByTestId("modal-title"); + if (modalTitleElement) { + modalCount = await modalTitleElement.count(); + } + } catch (error) { + modalCount = 0; + } + + while (modalCount === 0) { + await page.getByText("New Project", { exact: true }).click(); + await page.waitForTimeout(3000); + modalCount = await page.getByTestId("modal-title")?.count(); + } + + const getUA = await page.evaluate(() => navigator.userAgent); + const userAgentInfo = uaParser(getUA); + let control = "Control"; + + if (userAgentInfo.os.name.includes("Mac")) { + control = "Meta"; + } + + await page.getByTestId("side_nav_options_all-templates").click(); + await page + .getByRole("heading", { name: "Sequential Tasks Agent" }) + .first() + .click(); + + await page.waitForSelector('[data-testid="fit_view"]', { + timeout: 100000, + }); + + await page.getByTestId("fit_view").click(); + await page.getByTestId("zoom_out").click(); + await page.getByTestId("zoom_out").click(); + await page.getByTestId("zoom_out").click(); + + let outdatedComponents = await page.getByTestId("icon-AlertTriangle").count(); + + while (outdatedComponents > 0) { + await page.getByTestId("icon-AlertTriangle").first().click(); + await page.waitForTimeout(1000); + outdatedComponents = await page.getByTestId("icon-AlertTriangle").count(); + } + + let filledApiKey = await page.getByTestId("remove-icon-badge").count(); + while (filledApiKey > 0) { + await page.getByTestId("remove-icon-badge").first().click(); + await page.waitForTimeout(1000); + filledApiKey = await page.getByTestId("remove-icon-badge").count(); + } + + await page + .getByTestId("popover-anchor-input-api_key") + .fill(process.env.OPENAI_API_KEY ?? ""); + + await page.getByTestId("dropdown_str_model_name").click(); + await page.getByTestId("gpt-4o-1-option").click(); + + await page.waitForTimeout(1000); + + await page.getByTestId("button_run_chat output").click(); + + await page.waitForTimeout(1000); + + await page.waitForSelector("text=built successfully", { timeout: 30000 }); + + await page.getByText("built successfully").last().click({ + timeout: 15000, + }); + + await page.getByText("Playground", { exact: true }).last().click(); + + expect(await page.locator(".markdown").count()).toBeGreaterThan(0); + + expect(await page.getByText("Agile").count()).toBeGreaterThan(0); +}); diff --git a/uv.lock b/uv.lock index 742546759..00c0f8422 100644 --- a/uv.lock +++ b/uv.lock @@ -3759,7 +3759,7 @@ requires-dist = [ { name = "langchain-pinecone", specifier = "~=0.2.0" }, { name = "langchain-unstructured", specifier = ">=0.1.2" }, { name = "langflow-base", editable = "src/backend/base" }, - { name = "langfuse", specifier = ">=2.33.0" }, + { name = "langfuse", specifier = "~=2.53.1" }, { name = "langsmith", specifier = "~=0.1.136" }, { name = "langwatch", specifier = "==0.1.16" }, { name = "lark", specifier = ">=1.2.2" }, @@ -4066,7 +4066,7 @@ dev = [ [[package]] name = "langfuse" -version = "2.52.0" +version = "2.53.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -4077,9 +4077,9 @@ dependencies = [ { name = "pydantic" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9c/46/6b06df6bfc2a19465e27fa75a02ba2489b0e07367ba11fb23dddeee74a3b/langfuse-2.52.0.tar.gz", hash = "sha256:3ceb8803cb5991dd1127167a2b1288c7c6edc43429547ded1d6dc9e064e6e5cd", size = 122264 } +sdist = { url = "https://files.pythonhosted.org/packages/89/43/ff488a56303106e45d170e0a5b2bee8babc9184296436ae08bef940da6df/langfuse-2.53.3.tar.gz", hash = "sha256:702ee42669f8283c0b4400fdf039f13b896e18c8b94ad20c079148c2894cbc64", size = 123601 } wheels = [ - { url = "https://files.pythonhosted.org/packages/04/47/4e87fa27a30b9da00df26b960cc194d19fb842312cc3723f40c3e144cf9e/langfuse-2.52.0-py3-none-any.whl", hash = "sha256:f2497966bd31adedbb2d643fde3b15cf0387cfbcdac8ebc2db2b6df9413b888a", size = 220663 }, + { url = "https://files.pythonhosted.org/packages/a4/33/cb208078f2721f92af8f90ce387ac739ebf6942c37c2f1f61bf7106e44b7/langfuse-2.53.3-py3-none-any.whl", hash = "sha256:9fce8c70e77d03aa64fca474e254135ba379189bbafa2fdb23b255bda16c5b04", size = 221750 }, ] [[package]]