From c0624c4ad3e47b409e7b880a40541fa7de713500 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Mon, 24 Jun 2024 10:51:43 -0300 Subject: [PATCH 01/16] chore: update linting workflows to include dev branch in merge_group --- .github/workflows/lint-js.yml | 2 ++ .github/workflows/lint-py.yml | 2 ++ .github/workflows/python_test.yml | 20 +++++++++++++++++++- .github/workflows/style-check-py.yml | 2 ++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-js.yml b/.github/workflows/lint-js.yml index d381a467f..11156c8b8 100644 --- a/.github/workflows/lint-js.yml +++ b/.github/workflows/lint-js.yml @@ -4,6 +4,8 @@ on: pull_request: paths: - "src/frontend/**" + merge_group: + branches: [dev] env: NODE_VERSION: "21" diff --git a/.github/workflows/lint-py.yml b/.github/workflows/lint-py.yml index 95fbb15e6..9dab54eb8 100644 --- a/.github/workflows/lint-py.yml +++ b/.github/workflows/lint-py.yml @@ -7,6 +7,8 @@ on: - "pyproject.toml" - "src/backend/**" - "tests/**" + merge_group: + branches: [dev] env: POETRY_VERSION: "1.8.2" diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml index 9a8ef15c8..c3c320aad 100644 --- a/.github/workflows/python_test.yml +++ b/.github/workflows/python_test.yml @@ -1,4 +1,4 @@ -name: test +name: Python tests on: push: @@ -13,6 +13,8 @@ on: - "poetry.lock" - "pyproject.toml" - "src/backend/**" + merge_group: + branches: [dev] env: POETRY_VERSION: "1.8.2" @@ -43,3 +45,19 @@ jobs: - name: Run unit tests run: | make unit_tests args="-n auto" + - name: Test CLI + run: | + poetry run python -m langflow run --host 127.0.0.1 --port 7860 --backend-only & + SERVER_PID=$! + # Wait for the server to start + timeout 120 bash -c 'until curl -f http://127.0.0.1:7860/health; do sleep 2; done' || (echo "Server did not start in time" && kill $SERVER_PID && exit 1) + # Terminate the server + kill $SERVER_PID || (echo "Failed to terminate the server" && exit 1) + sleep 10 # give the server some time to terminate + # Check if the server is still running + if kill -0 $SERVER_PID 2>/dev/null; then + echo "Failed to terminate the server" + exit 1 + else + echo "Server terminated successfully" + fi diff --git a/.github/workflows/style-check-py.yml b/.github/workflows/style-check-py.yml index 3255aa31d..bd7b9dc0e 100644 --- a/.github/workflows/style-check-py.yml +++ b/.github/workflows/style-check-py.yml @@ -7,6 +7,8 @@ on: - "pyproject.toml" - "src/backend/**" - "tests/**" + merge_group: + branches: [dev] env: POETRY_VERSION: "1.8.2" From 71a9ee549b26e1c533df4e4a34cf36c6e18348c0 Mon Sep 17 00:00:00 2001 From: Rodrigo Nader Date: Mon, 24 Jun 2024 11:16:32 -0300 Subject: [PATCH 02/16] Update README.md Add 1.0 banner --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 765835b25..a427e974e 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,10 @@ +
+

Langflow 1.0 is OUT! 🎉

+

Read all about it here!

+
+ + +https://astra.datastax.com/signup?type=langflow # [![Langflow](./docs/static/img/hero.png)](https://www.langflow.org) From 73373016d908b07a2b13c71ccdfd1c962a7d8846 Mon Sep 17 00:00:00 2001 From: Rodrigo Nader Date: Mon, 24 Jun 2024 11:16:46 -0300 Subject: [PATCH 03/16] Update README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index a427e974e..9ba12ae7b 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,6 @@

Read all about it here!

- -https://astra.datastax.com/signup?type=langflow # [![Langflow](./docs/static/img/hero.png)](https://www.langflow.org) From 546f856708f128a8e20b6994af014e18b9ae62d0 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Mon, 24 Jun 2024 11:39:17 -0300 Subject: [PATCH 04/16] chore: update package versions in pyproject.toml files --- pyproject.toml | 2 +- src/backend/base/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d42e6bb9a..9fd1d3e04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langflow" -version = "1.0.0" +version = "1.0.1" description = "A Python package with a built-in web application" authors = ["Langflow "] maintainers = [ diff --git a/src/backend/base/pyproject.toml b/src/backend/base/pyproject.toml index 66d4cadcc..232bdd0aa 100644 --- a/src/backend/base/pyproject.toml +++ b/src/backend/base/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langflow-base" -version = "0.0.76" +version = "0.0.77" description = "A Python package with a built-in web application" authors = ["Langflow "] maintainers = [ From 77e23d94428b8d2e5cd35facb9cbd316f3b281fb Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Mon, 24 Jun 2024 12:39:49 -0700 Subject: [PATCH 05/16] Refactor "created_at" column type for consistency and fix cancel middleware (#2316) * chore: update linting workflows to include dev branch in merge_group * Update README.md Add 1.0 banner * Update README.md * chore: update package versions in pyproject.toml files * refactor: update "created_at" column type to use the "sa" module for consistency * Update README.md Add 1.0 banner * chore: Remove unused import in ToolCallingAgent.py * fix: adapt RequestCancelledMiddleware to handle cancelled requests * chore: Remove unused import in test_helper_components.py * refactor: Declare queue variable with explicit type in RequestCancelledMiddleware --------- Co-authored-by: Rodrigo Nader --- README.md | 2 ++ .../2ac71eb9c3ae_adds_credential_table.py | 2 +- .../components/agents/ToolCallingAgent.py | 1 - src/backend/base/langflow/main.py | 28 ++++++++----------- tests/unit/test_helper_components.py | 22 +++++++-------- 5 files changed, 25 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 9ba12ae7b..21405c493 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ # 📝 Content +- [](#) - [📝 Content](#-content) - [📦 Get Started](#-get-started) - [🎨 Create Flows](#-create-flows) @@ -49,6 +50,7 @@ - [Deploy Langflow on Google Cloud Platform](#deploy-langflow-on-google-cloud-platform) - [Deploy on Railway](#deploy-on-railway) - [Deploy on Render](#deploy-on-render) + - [Deploy on Kubernetes](#deploy-on-kubernetes) - [🖥️ Command Line Interface (CLI)](#️-command-line-interface-cli) - [Usage](#usage) - [Environment Variables](#environment-variables) diff --git a/src/backend/base/langflow/alembic/versions/2ac71eb9c3ae_adds_credential_table.py b/src/backend/base/langflow/alembic/versions/2ac71eb9c3ae_adds_credential_table.py index 617189071..2250a8b8c 100644 --- a/src/backend/base/langflow/alembic/versions/2ac71eb9c3ae_adds_credential_table.py +++ b/src/backend/base/langflow/alembic/versions/2ac71eb9c3ae_adds_credential_table.py @@ -34,7 +34,7 @@ def upgrade() -> None: sa.Column("provider", sqlmodel.sql.sqltypes.AutoString(), nullable=True), sa.Column("user_id", sqlmodel.sql.sqltypes.GUID(), nullable=False), sa.Column("id", sqlmodel.sql.sqltypes.GUID(), nullable=False), - sa.Column("created_at", sqlmodel.sql.sqltypes.DateTime(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), sa.Column("updated_at", sa.DateTime(), nullable=True), sa.PrimaryKeyConstraint("id"), ) diff --git a/src/backend/base/langflow/components/agents/ToolCallingAgent.py b/src/backend/base/langflow/components/agents/ToolCallingAgent.py index 017003aaa..407ab333d 100644 --- a/src/backend/base/langflow/components/agents/ToolCallingAgent.py +++ b/src/backend/base/langflow/components/agents/ToolCallingAgent.py @@ -2,7 +2,6 @@ from typing import Dict, List, cast from langchain.agents import AgentExecutor, BaseSingleActionAgent from langchain.agents.tool_calling_agent.base import create_tool_calling_agent -from langchain_core.messages import BaseMessage from langchain_core.prompts import ChatPromptTemplate from langflow.custom import Component diff --git a/src/backend/base/langflow/main.py b/src/backend/base/langflow/main.py index 45a681ff5..a8a165c95 100644 --- a/src/backend/base/langflow/main.py +++ b/src/backend/base/langflow/main.py @@ -6,7 +6,7 @@ from typing import Optional from urllib.parse import urlencode import nest_asyncio # type: ignore -from fastapi import FastAPI, Request +from fastapi import FastAPI, Request, Response from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles @@ -32,22 +32,17 @@ from langflow.utils.logger import configure warnings.filterwarnings("ignore", category=PydanticDeprecatedSince20) -class RequestCancelledMiddleware: +class RequestCancelledMiddleware(BaseHTTPMiddleware): def __init__(self, app): - self.app = app + super().__init__(app) - async def __call__(self, scope, receive, send): - if scope["type"] != "http": - await self.app(scope, receive, send) - return + async def dispatch(self, request: Request, call_next): + queue: asyncio.Queue = asyncio.Queue() - # Let's make a shared queue for the request messages - queue = asyncio.Queue() - - async def message_poller(sentinel, handler_task): + async def message_poller(sentinel, handler_task, request): nonlocal queue while True: - message = await receive() + message = await request.receive if message["type"] == "http.disconnect": handler_task.cancel() return sentinel # Break the loop @@ -56,13 +51,14 @@ class RequestCancelledMiddleware: await queue.put(message) sentinel = object() - handler_task = asyncio.create_task(self.app(scope, queue.get, send)) - asyncio.create_task(message_poller(sentinel, handler_task)) + handler_task = asyncio.create_task(call_next(request)) + asyncio.create_task(message_poller(sentinel, handler_task, request)) try: - return await handler_task + response = await handler_task + return response except asyncio.CancelledError: - pass + return Response("Request was cancelled", status_code=499) class JavaScriptMIMETypeMiddleware(BaseHTTPMiddleware): diff --git a/tests/unit/test_helper_components.py b/tests/unit/test_helper_components.py index 80c85b787..9e9c4f63a 100644 --- a/tests/unit/test_helper_components.py +++ b/tests/unit/test_helper_components.py @@ -1,5 +1,3 @@ -from langchain_core.documents import Document - from langflow.components import helpers from langflow.custom.utils import build_custom_component_template from langflow.schema import Data @@ -17,18 +15,18 @@ from langflow.schema import Data # assert result.new_key == "new_value" -def test_document_to_data_component(): - # Arrange - document_to_data_component = helpers.DocumentsToDataComponent() +# def test_document_to_data_component(): +# # Arrange +# document_to_data_component = helpers.DocumentsToDataComponent() - # Act - # Replace with your actual test data - document = Document(page_content="key: value", metadata={"url": "https://example.com"}) - result = document_to_data_component.build(document) +# # Act +# # Replace with your actual test data +# document = Document(page_content="key: value", metadata={"url": "https://example.com"}) +# result = document_to_data_component.build(document) - # Assert - # Replace with your actual expected result - assert result == [Data(data={"text": "key: value", "url": "https://example.com"})] +# # Assert +# # Replace with your actual expected result +# assert result == [Data(data={"text": "key: value", "url": "https://example.com"})] def test_uuid_generator_component(): From 6b67c4996d41450854155c0b5dfee7f492e59aa5 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Mon, 24 Jun 2024 17:01:00 -0300 Subject: [PATCH 06/16] chore: Update AstraDB.py imports and method signature for search_documents --- src/backend/base/langflow/components/vectorstores/AstraDB.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/base/langflow/components/vectorstores/AstraDB.py b/src/backend/base/langflow/components/vectorstores/AstraDB.py index 58e1d19ef..3d146c510 100644 --- a/src/backend/base/langflow/components/vectorstores/AstraDB.py +++ b/src/backend/base/langflow/components/vectorstores/AstraDB.py @@ -3,13 +3,13 @@ from loguru import logger from langflow.base.vectorstores.model import LCVectorStoreComponent from langflow.io import ( BoolInput, + DataInput, DropdownInput, HandleInput, IntInput, MultilineInput, SecretStrInput, StrInput, - DataInput, ) from langflow.schema import Data @@ -216,7 +216,7 @@ class AstraVectorStoreComponent(LCVectorStoreComponent): else: logger.debug("No documents to add to the Vector Store.") - def search_documents(self): + def search_documents(self) -> list[Data]: vector_store = self.build_vector_store() logger.debug(f"Search input: {self.search_input}") From 75fa498cac7662e8f4a954a277477940141921c9 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Mon, 24 Jun 2024 17:07:35 -0300 Subject: [PATCH 07/16] chore: Update package versions in pyproject.toml files --- .github/workflows/release.yml | 136 +++++++++++++++++++++----------- pyproject.toml | 2 +- src/backend/base/pyproject.toml | 2 +- 3 files changed, 94 insertions(+), 46 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a764c6cae..d5e599f68 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,6 @@ name: Langflow Release run-name: Langflow (${{inputs.release_type}}) Release by @${{ github.actor }} + on: workflow_dispatch: inputs: @@ -8,25 +9,23 @@ on: required: true type: boolean default: false - release_type: - description: "Type of release (base or main)" - required: true - type: choice - options: - - base - - main pre_release: description: "Pre-release" required: false type: boolean default: true + branch: + description: "Branch to release from" + required: true + type: string + default: "main" env: POETRY_VERSION: "1.8.2" jobs: - release: - name: Release Langflow + release-base: + name: Release Langflow Base if: inputs.release_package == true runs-on: ubuntu-latest outputs: @@ -47,13 +46,8 @@ jobs: - name: Check Version id: check-version run: | - if [ "${{ inputs.release_type }}" == "base" ]; then - version=$(cd src/backend/base && poetry version --short) - last_released_version=$(curl -s "https://pypi.org/pypi/langflow-base/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1) - else - version=$(poetry version --short) - last_released_version=$(curl -s "https://pypi.org/pypi/langflow/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1) - fi + version=$(cd src/backend/base && poetry version --short) + last_released_version=$(curl -s "https://pypi.org/pypi/langflow-base/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1) if [ "$version" = "$last_released_version" ]; then echo "Version $version is already released. Skipping release." exit 1 @@ -61,19 +55,10 @@ jobs: echo version=$version >> $GITHUB_OUTPUT fi - name: Build project for distribution - run: | - if [ "${{ inputs.release_type }}" == "base" ]; then - make build base=true - else - make build main=true - fi + run: make build base=true - name: Test CLI run: | - if [ "${{ inputs.release_type }}" == "base" ]; then - python -m pip install src/backend/base/dist/*.whl - else - python -m pip install dist/*.whl - fi + python -m pip install src/backend/base/dist/*.whl python -m langflow run --host 127.0.0.1 --port 7860 & SERVER_PID=$! # Wait for the server to start @@ -91,37 +76,100 @@ jobs: - name: Publish to PyPI env: POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }} - run: | - if [ "${{ inputs.release_type }}" == "base" ]; then - make publish base=true - else - make publish main=true - fi + run: make publish base=true - name: Upload Artifact uses: actions/upload-artifact@v4 with: - name: dist${{ inputs.release_type }} - path: ${{ inputs.release_type == 'base' && 'src/backend/base/dist' || 'dist' }} + name: dist-base + path: src/backend/base/dist + + release-main: + name: Release Langflow Main + if: inputs.release_package == true + needs: release-base + runs-on: ubuntu-latest + outputs: + version: ${{ steps.check-version.outputs.version }} + steps: + - uses: actions/checkout@v4 + - name: Install poetry + run: pipx install poetry==${{ env.POETRY_VERSION }} + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "poetry" + - name: Set up Nodejs 20 + uses: actions/setup-node@v4 + with: + node-version: "20" + - name: Check Version + id: check-version + run: | + version=$(poetry version --short) + last_released_version=$(curl -s "https://pypi.org/pypi/langflow/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1) + if [ "$version" = "$last_released_version" ]; then + echo "Version $version is already released. Skipping release." + exit 1 + else + echo version=$version >> $GITHUB_OUTPUT + fi + - name: Wait for PyPI Propagation + run: sleep 300 # wait for 5 minutes to ensure PyPI propagation + - name: Build project for distribution + run: make build main=true + - name: Test CLI + run: | + python -m pip install dist/*.whl + python -m langflow run --host 127.0.0.1 --port 7860 & + SERVER_PID=$! + # Wait for the server to start + timeout 120 bash -c 'until curl -f http://127.0.0.1:7860/health; do sleep 2; done' || (echo "Server did not start in time" && kill $SERVER_PID && exit 1) + # Terminate the server + kill $SERVER_PID || (echo "Failed to terminate the server" && exit 1) + sleep 10 # give the server some time to terminate + # Check if the server is still running + if kill -0 $SERVER_PID 2>/dev/null; then + echo "Failed to terminate the server" + exit 1 + else + echo "Server terminated successfully" + fi + - name: Publish to PyPI + env: + POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }} + run: make publish main=true + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: dist-main + path: dist call_docker_build: name: Call Docker Build Workflow - needs: release + needs: [release-base, release-main] uses: langflow-ai/langflow/.github/workflows/docker-build.yml@main + strategy: + matrix: + release_type: + - base + - main with: - version: ${{ needs.release.outputs.version }} - release_type: ${{ inputs.release_type }} + # version should be needs.release-base.outputs.version if release_type is base + # version should be needs.release-main.outputs.version if release_type is main + version: ${{ matrix.release_type == 'base' && needs.release-base.outputs.version || matrix.release_type == 'main' && needs.release-main.outputs.version }} + release_type: ${{ matrix.release_type }} pre_release: ${{ inputs.pre_release }} secrets: inherit create_release: name: Create Release runs-on: ubuntu-latest - needs: [release] - if: ${{ inputs.release_type == 'main' }} + needs: release-main steps: - uses: actions/download-artifact@v4 with: - name: dist${{ inputs.release_type }} + name: dist-main path: dist - name: Create Release uses: ncipollo/release-action@v1 @@ -130,6 +178,6 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} draft: false generateReleaseNotes: true - prerelease: false - tag: v${{ needs.release.outputs.version }} - commit: dev + prerelease: ${{ inputs.pre_release }} + tag: v${{ needs.release-main.outputs.version }} + commit: ${{ inputs.branch }} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9fd1d3e04..5fc120287 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langflow" -version = "1.0.1" +version = "1.0.2" description = "A Python package with a built-in web application" authors = ["Langflow "] maintainers = [ diff --git a/src/backend/base/pyproject.toml b/src/backend/base/pyproject.toml index 232bdd0aa..102097591 100644 --- a/src/backend/base/pyproject.toml +++ b/src/backend/base/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langflow-base" -version = "0.0.77" +version = "0.0.78" description = "A Python package with a built-in web application" authors = ["Langflow "] maintainers = [ From c6ca57c723449e5a806332ba536657f003b19861 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Mon, 24 Jun 2024 17:08:02 -0300 Subject: [PATCH 08/16] chore: Update run-name in release.yml for Langflow Release --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d5e599f68..b6453075e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,5 @@ name: Langflow Release -run-name: Langflow (${{inputs.release_type}}) Release by @${{ github.actor }} +run-name: Langflow Release by @${{ github.actor }} on: workflow_dispatch: From b02462f0921794420564d0d861a15b5d963dac48 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Mon, 24 Jun 2024 17:15:16 -0300 Subject: [PATCH 09/16] fix: add call to _add_documents_to_vector_store in AstraDB component --- src/backend/base/langflow/components/vectorstores/AstraDB.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/backend/base/langflow/components/vectorstores/AstraDB.py b/src/backend/base/langflow/components/vectorstores/AstraDB.py index 3d146c510..aabbb91a7 100644 --- a/src/backend/base/langflow/components/vectorstores/AstraDB.py +++ b/src/backend/base/langflow/components/vectorstores/AstraDB.py @@ -196,6 +196,10 @@ class AstraVectorStoreComponent(LCVectorStoreComponent): except Exception as e: raise ValueError(f"Error initializing AstraDBVectorStore: {str(e)}") from e + if hasattr(self, "ingest_data") and self.ingest_data: + logger.debug("Ingesting data into the Vector Store.") + self._add_documents_to_vector_store(vector_store) + self.status = self._astradb_collection_to_data(vector_store.collection) return vector_store From fcdaeb3b30a44e9e6745f385df09f986c828f2ac Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Mon, 24 Jun 2024 17:22:01 -0300 Subject: [PATCH 10/16] chore: Fix missing parentheses in RequestCancelledMiddleware --- src/backend/base/langflow/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/base/langflow/main.py b/src/backend/base/langflow/main.py index a8a165c95..085d72710 100644 --- a/src/backend/base/langflow/main.py +++ b/src/backend/base/langflow/main.py @@ -42,7 +42,7 @@ class RequestCancelledMiddleware(BaseHTTPMiddleware): async def message_poller(sentinel, handler_task, request): nonlocal queue while True: - message = await request.receive + message = await request.receive() if message["type"] == "http.disconnect": handler_task.cancel() return sentinel # Break the loop From c4d1427b3a07f40504ebff65f4c34067e1c4acf4 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Mon, 24 Jun 2024 17:22:29 -0300 Subject: [PATCH 11/16] chore: Update pydantic-settings and tenacity versions The commit updates the versions of the `pydantic-settings` and `tenacity` packages in the `poetry.lock` file. The `pydantic-settings` version is updated from 2.3.3 to 2.3.4, and the `tenacity` version is updated from 8.4.1 to 8.4.2. --- poetry.lock | 53 +++++++++++++++++---------------- pyproject.toml | 2 +- src/backend/base/poetry.lock | 12 ++++---- src/backend/base/pyproject.toml | 2 +- 4 files changed, 35 insertions(+), 34 deletions(-) diff --git a/poetry.lock b/poetry.lock index c07ae2e94..5ac1608a9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2579,13 +2579,13 @@ protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4 [[package]] name = "google-api-core" -version = "2.19.0" +version = "2.19.1" description = "Google API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google-api-core-2.19.0.tar.gz", hash = "sha256:cf1b7c2694047886d2af1128a03ae99e391108a08804f87cfd35970e49c9cd10"}, - {file = "google_api_core-2.19.0-py3-none-any.whl", hash = "sha256:8661eec4078c35428fd3f69a2c7ee29e342896b70f01d1a1cbcb334372dd6251"}, + {file = "google-api-core-2.19.1.tar.gz", hash = "sha256:f4695f1e3650b316a795108a76a1c416e6afb036199d1c1f1f110916df479ffd"}, + {file = "google_api_core-2.19.1-py3-none-any.whl", hash = "sha256:f12a9b8309b5e21d92483bbd47ce2c445861ec7d269ef6784ecc0ea8c1fa6125"}, ] [package.dependencies] @@ -2600,7 +2600,7 @@ grpcio-status = [ {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, ] proto-plus = ">=1.22.3,<2.0.0dev" -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" requests = ">=2.18.0,<3.0.0.dev0" [package.extras] @@ -2936,18 +2936,18 @@ requests = "*" [[package]] name = "googleapis-common-protos" -version = "1.63.1" +version = "1.63.2" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.63.1.tar.gz", hash = "sha256:c6442f7a0a6b2a80369457d79e6672bb7dcbaab88e0848302497e3ec80780a6a"}, - {file = "googleapis_common_protos-1.63.1-py2.py3-none-any.whl", hash = "sha256:0e1c2cdfcbc354b76e4a211a35ea35d6926a835cba1377073c4861db904a1877"}, + {file = "googleapis-common-protos-1.63.2.tar.gz", hash = "sha256:27c5abdffc4911f28101e635de1533fb4cfd2c37fbaa9174587c799fac90aa87"}, + {file = "googleapis_common_protos-1.63.2-py2.py3-none-any.whl", hash = "sha256:27a2499c7e8aff199665b22741997e485eccc8645aa9176c7c988e6fae507945"}, ] [package.dependencies] grpcio = {version = ">=1.44.0,<2.0.0.dev0", optional = true, markers = "extra == \"grpc\""} -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" +protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" [package.extras] grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] @@ -3070,19 +3070,19 @@ typing-extensions = ">=4.7,<5" [[package]] name = "grpc-google-iam-v1" -version = "0.13.0" +version = "0.13.1" description = "IAM API client library" optional = false python-versions = ">=3.7" files = [ - {file = "grpc-google-iam-v1-0.13.0.tar.gz", hash = "sha256:fad318608b9e093258fbf12529180f400d1c44453698a33509cc6ecf005b294e"}, - {file = "grpc_google_iam_v1-0.13.0-py2.py3-none-any.whl", hash = "sha256:53902e2af7de8df8c1bd91373d9be55b0743ec267a7428ea638db3775becae89"}, + {file = "grpc-google-iam-v1-0.13.1.tar.gz", hash = "sha256:3ff4b2fd9d990965e410965253c0da6f66205d5a8291c4c31c6ebecca18a9001"}, + {file = "grpc_google_iam_v1-0.13.1-py2.py3-none-any.whl", hash = "sha256:c3e86151a981811f30d5e7330f271cee53e73bb87755e88cc3b6f0c7b5fe374e"}, ] [package.dependencies] googleapis-common-protos = {version = ">=1.56.0,<2.0.0dev", extras = ["grpc"]} grpcio = ">=1.44.0,<2.0.0dev" -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" +protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0dev" [[package]] name = "grpcio" @@ -4442,7 +4442,7 @@ six = "*" [[package]] name = "langflow-base" -version = "0.0.76" +version = "0.0.78" description = "A Python package with a built-in web application" optional = false python-versions = ">=3.10,<3.13" @@ -7016,13 +7016,13 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pydantic-settings" -version = "2.3.3" +version = "2.3.4" description = "Settings management using Pydantic" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_settings-2.3.3-py3-none-any.whl", hash = "sha256:e4ed62ad851670975ec11285141db888fd24947f9440bd4380d7d8788d4965de"}, - {file = "pydantic_settings-2.3.3.tar.gz", hash = "sha256:87fda838b64b5039b970cd47c3e8a1ee460ce136278ff672980af21516f6e6ce"}, + {file = "pydantic_settings-2.3.4-py3-none-any.whl", hash = "sha256:11ad8bacb68a045f00e4f862c7a718c8a9ec766aa8fd4c32e39a0594b207b53a"}, + {file = "pydantic_settings-2.3.4.tar.gz", hash = "sha256:c5802e3d62b78e82522319bbc9b8f8ffb28ad1c988a99311d04f2a6051fca0a7"}, ] [package.dependencies] @@ -8783,13 +8783,13 @@ files = [ [[package]] name = "tenacity" -version = "8.4.1" +version = "8.4.2" description = "Retry code until it succeeds" optional = false python-versions = ">=3.8" files = [ - {file = "tenacity-8.4.1-py3-none-any.whl", hash = "sha256:28522e692eda3e1b8f5e99c51464efcc0b9fc86933da92415168bc1c4e2308fa"}, - {file = "tenacity-8.4.1.tar.gz", hash = "sha256:54b1412b878ddf7e1f1577cd49527bad8cdef32421bd599beac0c6c3f10582fd"}, + {file = "tenacity-8.4.2-py3-none-any.whl", hash = "sha256:9e6f7cf7da729125c7437222f8a522279751cdfbe6b67bfe64f75d3a348661b2"}, + {file = "tenacity-8.4.2.tar.gz", hash = "sha256:cd80a53a79336edba8489e767f729e4f391c896956b57140b5d7511a64bbd3ef"}, ] [package.extras] @@ -9541,13 +9541,13 @@ six = "*" [[package]] name = "unstructured" -version = "0.14.7" +version = "0.14.8" description = "A library that prepares raw documents for downstream ML tasks." optional = false python-versions = "<3.13,>=3.9.0" files = [ - {file = "unstructured-0.14.7-py3-none-any.whl", hash = "sha256:78e029eb2dd3dc792989af2133cb5472e6a17a501cf415cd8c095f1a88763738"}, - {file = "unstructured-0.14.7.tar.gz", hash = "sha256:219f6ae8a8c5a23e5e6944d3a78d1c7f162ee01e7c6034afe2e5052eef679918"}, + {file = "unstructured-0.14.8-py3-none-any.whl", hash = "sha256:eb97a0754d9f1a90134af45e4ee2e17f12aced57fda4f73ca4875090fef396a5"}, + {file = "unstructured-0.14.8.tar.gz", hash = "sha256:2a39fd2e92a986c45f9686b1115cce56a022d2600a1aae67b8e09eb0373d85b3"}, ] [package.dependencies] @@ -9576,7 +9576,7 @@ wrapt = "*" [package.extras] airtable = ["pyairtable"] -all-docs = ["effdet", "google-cloud-vision", "markdown", "networkx", "onnx", "openpyxl", "pandas", "pdf2image", "pdfminer.six", "pikepdf", "pillow-heif", "pypandoc", "pypdf", "pytesseract", "python-docx (>=1.1.2)", "python-oxmsg", "python-pptx (<=0.6.23)", "unstructured-inference (==0.7.35)", "unstructured.pytesseract (>=0.3.12)", "xlrd"] +all-docs = ["effdet", "google-cloud-vision", "markdown", "networkx", "onnx", "openpyxl", "pandas", "pdf2image", "pdfminer.six", "pikepdf", "pillow-heif", "pypandoc", "pypdf", "pytesseract", "python-docx (>=1.1.2)", "python-oxmsg", "python-pptx (<=0.6.23)", "unstructured-inference (==0.7.36)", "unstructured.pytesseract (>=0.3.12)", "xlrd"] astra = ["astrapy"] azure = ["adlfs", "fsspec"] azure-cognitive-search = ["azure-search-documents"] @@ -9605,9 +9605,10 @@ gitlab = ["python-gitlab"] google-drive = ["google-api-python-client"] hubspot = ["hubspot-api-client", "urllib3"] huggingface = ["langdetect", "sacremoses", "sentencepiece", "torch", "transformers"] -image = ["effdet", "google-cloud-vision", "onnx", "pdf2image", "pdfminer.six", "pikepdf", "pillow-heif", "pypdf", "pytesseract", "unstructured-inference (==0.7.35)", "unstructured.pytesseract (>=0.3.12)"] +image = ["effdet", "google-cloud-vision", "onnx", "pdf2image", "pdfminer.six", "pikepdf", "pillow-heif", "pypdf", "pytesseract", "unstructured-inference (==0.7.36)", "unstructured.pytesseract (>=0.3.12)"] jira = ["atlassian-python-api"] -local-inference = ["effdet", "google-cloud-vision", "markdown", "networkx", "onnx", "openpyxl", "pandas", "pdf2image", "pdfminer.six", "pikepdf", "pillow-heif", "pypandoc", "pypdf", "pytesseract", "python-docx (>=1.1.2)", "python-oxmsg", "python-pptx (<=0.6.23)", "unstructured-inference (==0.7.35)", "unstructured.pytesseract (>=0.3.12)", "xlrd"] +kafka = ["confluent-kafka"] +local-inference = ["effdet", "google-cloud-vision", "markdown", "networkx", "onnx", "openpyxl", "pandas", "pdf2image", "pdfminer.six", "pikepdf", "pillow-heif", "pypandoc", "pypdf", "pytesseract", "python-docx (>=1.1.2)", "python-oxmsg", "python-pptx (<=0.6.23)", "unstructured-inference (==0.7.36)", "unstructured.pytesseract (>=0.3.12)", "xlrd"] md = ["markdown"] mongodb = ["pymongo"] msg = ["python-oxmsg"] @@ -9619,7 +9620,7 @@ opensearch = ["opensearch-py"] org = ["pypandoc"] outlook = ["Office365-REST-Python-Client", "msal"] paddleocr = ["unstructured.paddleocr (==2.6.1.3)"] -pdf = ["effdet", "google-cloud-vision", "onnx", "pdf2image", "pdfminer.six", "pikepdf", "pillow-heif", "pypdf", "pytesseract", "unstructured-inference (==0.7.35)", "unstructured.pytesseract (>=0.3.12)"] +pdf = ["effdet", "google-cloud-vision", "onnx", "pdf2image", "pdfminer.six", "pikepdf", "pillow-heif", "pypdf", "pytesseract", "unstructured-inference (==0.7.36)", "unstructured.pytesseract (>=0.3.12)"] pinecone = ["pinecone-client (>=3.7.1)"] postgres = ["psycopg2-binary"] ppt = ["python-pptx (<=0.6.23)"] diff --git a/pyproject.toml b/pyproject.toml index 5fc120287..98d00485b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langflow" -version = "1.0.2" +version = "1.0.3" description = "A Python package with a built-in web application" authors = ["Langflow "] maintainers = [ diff --git a/src/backend/base/poetry.lock b/src/backend/base/poetry.lock index a919a0251..379821ebf 100644 --- a/src/backend/base/poetry.lock +++ b/src/backend/base/poetry.lock @@ -2199,13 +2199,13 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pydantic-settings" -version = "2.3.3" +version = "2.3.4" description = "Settings management using Pydantic" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_settings-2.3.3-py3-none-any.whl", hash = "sha256:e4ed62ad851670975ec11285141db888fd24947f9440bd4380d7d8788d4965de"}, - {file = "pydantic_settings-2.3.3.tar.gz", hash = "sha256:87fda838b64b5039b970cd47c3e8a1ee460ce136278ff672980af21516f6e6ce"}, + {file = "pydantic_settings-2.3.4-py3-none-any.whl", hash = "sha256:11ad8bacb68a045f00e4f862c7a718c8a9ec766aa8fd4c32e39a0594b207b53a"}, + {file = "pydantic_settings-2.3.4.tar.gz", hash = "sha256:c5802e3d62b78e82522319bbc9b8f8ffb28ad1c988a99311d04f2a6051fca0a7"}, ] [package.dependencies] @@ -2669,13 +2669,13 @@ full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.7 [[package]] name = "tenacity" -version = "8.4.1" +version = "8.4.2" description = "Retry code until it succeeds" optional = false python-versions = ">=3.8" files = [ - {file = "tenacity-8.4.1-py3-none-any.whl", hash = "sha256:28522e692eda3e1b8f5e99c51464efcc0b9fc86933da92415168bc1c4e2308fa"}, - {file = "tenacity-8.4.1.tar.gz", hash = "sha256:54b1412b878ddf7e1f1577cd49527bad8cdef32421bd599beac0c6c3f10582fd"}, + {file = "tenacity-8.4.2-py3-none-any.whl", hash = "sha256:9e6f7cf7da729125c7437222f8a522279751cdfbe6b67bfe64f75d3a348661b2"}, + {file = "tenacity-8.4.2.tar.gz", hash = "sha256:cd80a53a79336edba8489e767f729e4f391c896956b57140b5d7511a64bbd3ef"}, ] [package.extras] diff --git a/src/backend/base/pyproject.toml b/src/backend/base/pyproject.toml index 102097591..ee8072a05 100644 --- a/src/backend/base/pyproject.toml +++ b/src/backend/base/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langflow-base" -version = "0.0.78" +version = "0.0.79" description = "A Python package with a built-in web application" authors = ["Langflow "] maintainers = [ From 7a6c957f35774abea0e33152989adaa4a738cf3f Mon Sep 17 00:00:00 2001 From: Rodrigo Nader Date: Mon, 24 Jun 2024 11:16:32 -0300 Subject: [PATCH 12/16] Update README.md Add 1.0 banner --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index c4fcde4a2..6a15027aa 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,6 @@ Alternatively, click the **"Open in Cloud Shell"** button below to launch Google Use this template to deploy Langflow 1.0 on Railway: - [![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/JMXEWp?referralCode=MnPSdg) ## Deploy on Render From dfd419e438664653537785a1c0d72f1df12f9800 Mon Sep 17 00:00:00 2001 From: anovazzi1 Date: Mon, 24 Jun 2024 17:33:20 -0300 Subject: [PATCH 13/16] fix fetch data to work even with autologin true --- src/frontend/src/App.tsx | 7 ------- src/frontend/src/contexts/authContext.tsx | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/frontend/src/App.tsx b/src/frontend/src/App.tsx index 1c8274e63..1a7178248 100644 --- a/src/frontend/src/App.tsx +++ b/src/frontend/src/App.tsx @@ -103,13 +103,6 @@ export default function App() { if (isAuthenticated) { try { await setupAxiosDefaults(); - - const res = await getGlobalVariables(); - setGlobalVariables(res); - - checkHasStore(); - fetchApiData(); - resolve(); } catch (error) { console.error("Failed to fetch data:", error); diff --git a/src/frontend/src/contexts/authContext.tsx b/src/frontend/src/contexts/authContext.tsx index bd5c2d02c..76b3ba742 100644 --- a/src/frontend/src/contexts/authContext.tsx +++ b/src/frontend/src/contexts/authContext.tsx @@ -1,11 +1,13 @@ import { createContext, useEffect, useState } from "react"; import { useNavigate } from "react-router-dom"; import Cookies from "universal-cookie"; -import { getLoggedUser, requestLogout } from "../controllers/API"; +import { getGlobalVariables, getLoggedUser, requestLogout } from "../controllers/API"; import useAlertStore from "../stores/alertStore"; import { useFolderStore } from "../stores/foldersStore"; import { Users } from "../types/api"; import { AuthContextType } from "../types/contexts/auth"; +import { useGlobalVariablesStore } from "../stores/globalVariablesStore/globalVariables"; +import { useStoreStore } from "../stores/storeStore"; const initialValue: AuthContextType = { isAdmin: false, @@ -45,6 +47,11 @@ export function AuthProvider({ children }): React.ReactElement { ); const getFoldersApi = useFolderStore((state) => state.getFoldersApi); + const setGlobalVariables = useGlobalVariablesStore( + (state) => state.setGlobalVariables, + ); + const checkHasStore = useStoreStore((state) => state.checkHasStore); + const fetchApiData = useStoreStore((state) => state.fetchApiData); useEffect(() => { const storedAccessToken = cookies.get("access_token_lf"); @@ -66,8 +73,11 @@ export function AuthProvider({ children }): React.ReactElement { setUserData(user); const isSuperUser = user!.is_superuser; setIsAdmin(isSuperUser); - getFoldersApi(true, true); + const res = await getGlobalVariables(); + setGlobalVariables(res); + checkHasStore(); + fetchApiData(); }) .catch((error) => { setLoading(false); From 853f5483a3ffd8e84fa277c92eacd499da673624 Mon Sep 17 00:00:00 2001 From: anovazzi1 Date: Mon, 24 Jun 2024 17:34:03 -0300 Subject: [PATCH 14/16] format code --- src/frontend/src/contexts/authContext.tsx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/frontend/src/contexts/authContext.tsx b/src/frontend/src/contexts/authContext.tsx index 76b3ba742..cfdbdfb4f 100644 --- a/src/frontend/src/contexts/authContext.tsx +++ b/src/frontend/src/contexts/authContext.tsx @@ -1,13 +1,17 @@ import { createContext, useEffect, useState } from "react"; import { useNavigate } from "react-router-dom"; import Cookies from "universal-cookie"; -import { getGlobalVariables, getLoggedUser, requestLogout } from "../controllers/API"; +import { + getGlobalVariables, + getLoggedUser, + requestLogout, +} from "../controllers/API"; import useAlertStore from "../stores/alertStore"; import { useFolderStore } from "../stores/foldersStore"; -import { Users } from "../types/api"; -import { AuthContextType } from "../types/contexts/auth"; import { useGlobalVariablesStore } from "../stores/globalVariablesStore/globalVariables"; import { useStoreStore } from "../stores/storeStore"; +import { Users } from "../types/api"; +import { AuthContextType } from "../types/contexts/auth"; const initialValue: AuthContextType = { isAdmin: false, From 8eae0c5df7e176fb6b4fe3bae0f35a4a3770b1f2 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 25 Jun 2024 05:00:22 -0700 Subject: [PATCH 15/16] deactivate stop button until we have a better solution (#2337) --- src/backend/base/langflow/main.py | 39 +++++++++---------- .../components/menuBar/index.tsx | 5 ++- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/backend/base/langflow/main.py b/src/backend/base/langflow/main.py index 085d72710..5eab2f331 100644 --- a/src/backend/base/langflow/main.py +++ b/src/backend/base/langflow/main.py @@ -37,28 +37,26 @@ class RequestCancelledMiddleware(BaseHTTPMiddleware): super().__init__(app) async def dispatch(self, request: Request, call_next): - queue: asyncio.Queue = asyncio.Queue() - - async def message_poller(sentinel, handler_task, request): - nonlocal queue - while True: - message = await request.receive() - if message["type"] == "http.disconnect": - handler_task.cancel() - return sentinel # Break the loop - - # Puts the message in the queue - await queue.put(message) - sentinel = object() - handler_task = asyncio.create_task(call_next(request)) - asyncio.create_task(message_poller(sentinel, handler_task, request)) - try: - response = await handler_task - return response - except asyncio.CancelledError: + async def cancel_handler(): + while True: + if await request.is_disconnected(): + return sentinel + await asyncio.sleep(0.1) + + handler_task = asyncio.create_task(call_next(request)) + cancel_task = asyncio.create_task(cancel_handler()) + + done, pending = await asyncio.wait([handler_task, cancel_task], return_when=asyncio.FIRST_COMPLETED) + + for task in pending: + task.cancel() + + if cancel_task in done: return Response("Request was cancelled", status_code=499) + else: + return await handler_task class JavaScriptMIMETypeMiddleware(BaseHTTPMiddleware): @@ -126,7 +124,8 @@ def create_app(): allow_headers=["*"], ) app.add_middleware(JavaScriptMIMETypeMiddleware) - app.add_middleware(RequestCancelledMiddleware) + # ! Deactivating this until we find a better solution + # app.add_middleware(RequestCancelledMiddleware) @app.middleware("http") async def flatten_query_string_lists(request: Request, call_next): diff --git a/src/frontend/src/components/headerComponent/components/menuBar/index.tsx b/src/frontend/src/components/headerComponent/components/menuBar/index.tsx index ba3b0e9ad..7376bfc4f 100644 --- a/src/frontend/src/components/headerComponent/components/menuBar/index.tsx +++ b/src/frontend/src/components/headerComponent/components/menuBar/index.tsx @@ -218,7 +218,8 @@ export const MenuBar = ({}: {}): JSX.Element => { />
{printByBuildStatus()}
- + */} )} From d4c18d3c00af186377ec73bd7a6b8ade335f2c07 Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Tue, 25 Jun 2024 17:10:29 -0300 Subject: [PATCH 16/16] added firecrawl integration --- poetry.lock | 26 ++++-- pyproject.toml | 1 + .../langchain_utilities/FirecrawlCrawlApi.py | 81 +++++++++++++++++++ .../langchain_utilities/FirecrawlScrapeApi.py | 72 +++++++++++++++++ src/backend/base/poetry.lock | 16 +++- src/backend/base/pyproject.toml | 1 + .../src/icons/Firecrawl/FirecrawlLogo.jsx | 61 ++++++++++++++ .../src/icons/Firecrawl/firecraw-logo.svg | 28 +++++++ src/frontend/src/icons/Firecrawl/index.tsx | 9 +++ src/frontend/src/utils/styleUtils.ts | 3 + 10 files changed, 291 insertions(+), 7 deletions(-) create mode 100644 src/backend/base/langflow/components/langchain_utilities/FirecrawlCrawlApi.py create mode 100644 src/backend/base/langflow/components/langchain_utilities/FirecrawlScrapeApi.py create mode 100644 src/frontend/src/icons/Firecrawl/FirecrawlLogo.jsx create mode 100644 src/frontend/src/icons/Firecrawl/firecraw-logo.svg create mode 100644 src/frontend/src/icons/Firecrawl/index.tsx diff --git a/poetry.lock b/poetry.lock index 5ac1608a9..a4f0db362 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2112,6 +2112,20 @@ files = [ {file = "filetype-1.2.0.tar.gz", hash = "sha256:66b56cd6474bf41d8c54660347d37afcc3f7d1970648de365c102ef77548aadb"}, ] +[[package]] +name = "firecrawl-py" +version = "0.0.16" +description = "Python SDK for Firecrawl API" +optional = false +python-versions = ">=3.8" +files = [ + {file = "firecrawl_py-0.0.16-py3-none-any.whl", hash = "sha256:9024f483b501852a6b9c4e6cdfc9e8dde452d922afac357080bb278a0c9c2a26"}, + {file = "firecrawl_py-0.0.16.tar.gz", hash = "sha256:6c662fa0a549bc7f5c0acb704baba6731869ca0451094034264dfc1b4eb086e4"}, +] + +[package.dependencies] +requests = "*" + [[package]] name = "flaml" version = "2.1.2" @@ -2432,8 +2446,8 @@ files = [ [package.dependencies] cffi = {version = ">=1.12.2", markers = "platform_python_implementation == \"CPython\" and sys_platform == \"win32\""} greenlet = [ - {version = ">=2.0.0", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.11\""}, {version = ">=3.0rc3", markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\""}, + {version = ">=2.0.0", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.11\""}, ] "zope.event" = "*" "zope.interface" = "*" @@ -2592,12 +2606,12 @@ files = [ google-auth = ">=2.14.1,<3.0.dev0" googleapis-common-protos = ">=1.56.2,<2.0.dev0" grpcio = [ - {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, ] grpcio-status = [ - {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, ] proto-plus = ">=1.22.3,<2.0.0dev" protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" @@ -4611,8 +4625,8 @@ psutil = ">=5.9.1" pywin32 = {version = "*", markers = "platform_system == \"Windows\""} pyzmq = ">=25.0.0" requests = [ - {version = ">=2.26.0", markers = "python_version <= \"3.11\""}, {version = ">=2.32.2", markers = "python_version > \"3.11\""}, + {version = ">=2.26.0", markers = "python_version <= \"3.11\""}, ] tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} Werkzeug = ">=2.0.0" @@ -6076,9 +6090,9 @@ files = [ [package.dependencies] numpy = [ + {version = ">=1.26.0,<2", markers = "python_version >= \"3.12\""}, {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""}, {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, - {version = ">=1.26.0,<2", markers = "python_version >= \"3.12\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -10548,4 +10562,4 @@ local = ["ctransformers", "llama-cpp-python", "sentence-transformers"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "f7377e3a997651cbcec2b9227b0bcde2507afc7d6236b708f4dc62857f150578" +content-hash = "3e72b6faa1c674615a7e5dec3e7d962349e736bf6675c08a49080b7f336cc75b" diff --git a/pyproject.toml b/pyproject.toml index 98d00485b..0ea058281 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,6 +92,7 @@ unstructured = {extras = ["docx", "md", "pptx"], version = "^0.14.4"} langchain-aws = "^0.1.6" langchain-mongodb = "^0.1.6" kubernetes = "^30.1.0" +firecrawl-py = "^0.0.16" [tool.poetry.group.dev.dependencies] diff --git a/src/backend/base/langflow/components/langchain_utilities/FirecrawlCrawlApi.py b/src/backend/base/langflow/components/langchain_utilities/FirecrawlCrawlApi.py new file mode 100644 index 000000000..54355285d --- /dev/null +++ b/src/backend/base/langflow/components/langchain_utilities/FirecrawlCrawlApi.py @@ -0,0 +1,81 @@ +from typing import Optional +from firecrawl.firecrawl import FirecrawlApp +from langflow.custom import CustomComponent +from langflow.schema import Data +import uuid + +class FirecrawlCrawlApi(CustomComponent): + display_name: str = "FirecrawlCrawlApi" + description: str = "Firecrawl Crawl API." + output_types: list[str] = ["Document"] + documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/crawl" + field_config = { + "api_key": { + "display_name": "API Key", + "field_type": "str", + "required": True, + "password": True, + "info": "The API key to use Firecrawl API.", + }, + "url": { + "display_name": "URL", + "field_type": "str", + "required": True, + "info": "The base URL to start crawling from.", + }, + "timeout": { + "display_name": "Timeout", + "field_type": "int", + "info": "The timeout in milliseconds.", + }, + "crawlerOptions": { + "display_name": "Crawler Options", + "info": "Options for the crawler behavior.", + }, + "pageOptions": { + "display_name": "Page Options", + "info": "The page options to send with the request.", + }, + "idempotency_key": { + "display_name": "Idempotency Key", + "field_type": "str", + "info": "Optional idempotency key to ensure unique requests.", + }, + } + + def build( + self, + api_key: str, + url: str, + timeout: Optional[int] = 30000, + crawlerOptions: Optional[Data] = None, + pageOptions: Optional[Data] = None, + idempotency_key: Optional[str] = None, + ) -> Data: + if crawlerOptions: + crawler_options_dict = crawlerOptions.__dict__['data']['text'] + else: + crawler_options_dict = {} + + if pageOptions: + page_options_dict = pageOptions.__dict__['data']['text'] + else: + page_options_dict = {} + + if not idempotency_key: + idempotency_key = str(uuid.uuid4()) + + app = FirecrawlApp(api_key=api_key) + crawl_result = app.crawl_url( + url, + { + "crawlerOptions": crawler_options_dict, + "pageOptions": page_options_dict, + }, + True, + int(timeout / 1000), + idempotency_key + ) + + records = Data(data={"results": crawl_result}) + return records diff --git a/src/backend/base/langflow/components/langchain_utilities/FirecrawlScrapeApi.py b/src/backend/base/langflow/components/langchain_utilities/FirecrawlScrapeApi.py new file mode 100644 index 000000000..5cdfc82ba --- /dev/null +++ b/src/backend/base/langflow/components/langchain_utilities/FirecrawlScrapeApi.py @@ -0,0 +1,72 @@ +from typing import Optional +from firecrawl.firecrawl import FirecrawlApp +from langflow.custom import CustomComponent +from langflow.schema import Data +from langflow.services.database.models.base import orjson_dumps +import json + +class FirecrawlScrapeApi(CustomComponent): + display_name: str = "FirecrawlScrapeApi" + description: str = "Firecrawl Scrape API." + output_types: list[str] = ["Document"] + documentation: str = "https://docs.firecrawl.dev/api-reference/endpoint/scrape" + field_config = { + "api_key": { + "display_name": "API Key", + "field_type": "str", + "required": True, + "password": True, + "info": "The API key to use Firecrawl API.", + }, + "url": { + "display_name": "URL", + "field_type": "str", + "required": True, + "info": "The URL to scrape.", + }, + "timeout": { + "display_name": "Timeout", + "info": "Timeout in milliseconds for the request.", + "field_type": "int", + "default_value": 10000, + }, + "pageOptions": { + "display_name": "Page Options", + "info": "The page options to send with the request.", + }, + "extractorOptions": { + "display_name": "Extractor Options", + "info": "The extractor options to send with the request.", + }, + } + + def build( + self, + api_key: str, + url: str, + timeout: Optional[int] = 10000, + pageOptions: Optional[Data] = None, + extractorOptions: Optional[Data] = None, + ) -> Data: + if extractorOptions: + extractor_options_dict = extractorOptions.__dict__['data']['text'] + else: + extractor_options_dict = {} + + if pageOptions: + page_options_dict = pageOptions.__dict__['data']['text'] + else: + page_options_dict = {} + + app = FirecrawlApp(api_key=api_key) + results = app.scrape_url( + url, + { + "timeout": str(timeout), + "extractorOptions": extractor_options_dict, + "pageOptions": page_options_dict, + }, + ) + + record = Data(data=results) + return record diff --git a/src/backend/base/poetry.lock b/src/backend/base/poetry.lock index 379821ebf..256a5eef9 100644 --- a/src/backend/base/poetry.lock +++ b/src/backend/base/poetry.lock @@ -739,6 +739,20 @@ typer = ">=0.12.3" [package.extras] standard = ["fastapi", "uvicorn[standard] (>=0.15.0)"] +[[package]] +name = "firecrawl-py" +version = "0.0.16" +description = "Python SDK for Firecrawl API" +optional = false +python-versions = ">=3.8" +files = [ + {file = "firecrawl_py-0.0.16-py3-none-any.whl", hash = "sha256:9024f483b501852a6b9c4e6cdfc9e8dde452d922afac357080bb278a0c9c2a26"}, + {file = "firecrawl_py-0.0.16.tar.gz", hash = "sha256:6c662fa0a549bc7f5c0acb704baba6731869ca0451094034264dfc1b4eb086e4"}, +] + +[package.dependencies] +requests = "*" + [[package]] name = "frozenlist" version = "1.4.1" @@ -3232,4 +3246,4 @@ local = [] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "4f566531a8539ddc81cb91a7e7f9b723c84679f0af5bb8619f7b02f9ffc6cfaa" +content-hash = "7e46144d27c633214f00e73e496c0e4d56db1fb47032a21861677ec275b79d86" diff --git a/src/backend/base/pyproject.toml b/src/backend/base/pyproject.toml index ee8072a05..c1eab2bb9 100644 --- a/src/backend/base/pyproject.toml +++ b/src/backend/base/pyproject.toml @@ -64,6 +64,7 @@ pyperclip = "^1.8.2" uncurl = "^0.0.11" sentry-sdk = {extras = ["fastapi", "loguru"], version = "^2.5.1"} chardet = "^5.2.0" +firecrawl-py = "^0.0.16" [tool.poetry.extras] diff --git a/src/frontend/src/icons/Firecrawl/FirecrawlLogo.jsx b/src/frontend/src/icons/Firecrawl/FirecrawlLogo.jsx new file mode 100644 index 000000000..f79173d13 --- /dev/null +++ b/src/frontend/src/icons/Firecrawl/FirecrawlLogo.jsx @@ -0,0 +1,61 @@ +const SvgFirecrawlLogo = (props) => ( + + + + + + + + + + + + + + + +); +export default SvgFirecrawlLogo; diff --git a/src/frontend/src/icons/Firecrawl/firecraw-logo.svg b/src/frontend/src/icons/Firecrawl/firecraw-logo.svg new file mode 100644 index 000000000..ea8d2c7f1 --- /dev/null +++ b/src/frontend/src/icons/Firecrawl/firecraw-logo.svg @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/frontend/src/icons/Firecrawl/index.tsx b/src/frontend/src/icons/Firecrawl/index.tsx new file mode 100644 index 000000000..060d053bb --- /dev/null +++ b/src/frontend/src/icons/Firecrawl/index.tsx @@ -0,0 +1,9 @@ +import React, { forwardRef } from "react"; +import SvgFirecrawlLogo from "./FirecrawlLogo"; + +export const FirecrawlIcon = forwardRef< + SVGSVGElement, + React.PropsWithChildren<{}> +>((props, ref) => { + return ; +}); diff --git a/src/frontend/src/utils/styleUtils.ts b/src/frontend/src/utils/styleUtils.ts index 30a22fd60..b344eaaec 100644 --- a/src/frontend/src/utils/styleUtils.ts +++ b/src/frontend/src/utils/styleUtils.ts @@ -171,6 +171,7 @@ import { CouchbaseIcon } from "../icons/Couchbase"; import { ElasticsearchIcon } from "../icons/ElasticsearchStore"; import { EvernoteIcon } from "../icons/Evernote"; import { FBIcon } from "../icons/FacebookMessenger"; +import { FirecrawlIcon } from "../icons/Firecrawl"; import { GitBookIcon } from "../icons/GitBook"; import { GoogleIcon } from "../icons/Google"; import { GoogleGenerativeAIIcon } from "../icons/GoogleGenerativeAI"; @@ -363,6 +364,8 @@ export const nodeIconsLucide: iconsType = { CohereEmbeddings: CohereIcon, EverNoteLoader: EvernoteIcon, FacebookChatLoader: FBIcon, + FirecrawlCrawlApi: FirecrawlIcon, + FirecrawlScrapeApi: FirecrawlIcon, GitbookLoader: GitBookIcon, GoogleSearchAPIWrapper: GoogleIcon, GoogleSearchResults: GoogleIcon,