From d85657f2141c3ed7cc625de63a1ecb0dae60844c Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Mon, 24 Jun 2024 13:15:54 -0700 Subject: [PATCH] Refactor "created_at" column type for consistency and fix cancel middleware (#2316) (#2320) * chore: update linting workflows to include dev branch in merge_group * Update README.md Add 1.0 banner * Update README.md * chore: update package versions in pyproject.toml files * Refactor "created_at" column type for consistency and fix cancel middleware (#2316) * chore: update linting workflows to include dev branch in merge_group * Update README.md Add 1.0 banner * Update README.md * chore: update package versions in pyproject.toml files * refactor: update "created_at" column type to use the "sa" module for consistency * Update README.md Add 1.0 banner * chore: Remove unused import in ToolCallingAgent.py * fix: adapt RequestCancelledMiddleware to handle cancelled requests * chore: Remove unused import in test_helper_components.py * refactor: Declare queue variable with explicit type in RequestCancelledMiddleware --------- Co-authored-by: Rodrigo Nader * chore: Update AstraDB.py imports and method signature for search_documents * chore: Update package versions in pyproject.toml files * chore: Update run-name in release.yml for Langflow Release * fix: add call to _add_documents_to_vector_store in AstraDB component --------- Co-authored-by: Rodrigo Nader --- .github/workflows/release.yml | 138 ++++++++++++------ README.md | 2 + pyproject.toml | 2 +- .../2ac71eb9c3ae_adds_credential_table.py | 2 +- .../components/agents/ToolCallingAgent.py | 1 - .../components/vectorstores/AstraDB.py | 8 +- src/backend/base/langflow/main.py | 28 ++-- src/backend/base/pyproject.toml | 2 +- tests/unit/test_helper_components.py | 22 ++- 9 files changed, 126 insertions(+), 79 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a764c6cae..b6453075e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,6 @@ name: Langflow Release -run-name: Langflow (${{inputs.release_type}}) Release by @${{ github.actor }} +run-name: Langflow Release by @${{ github.actor }} + on: workflow_dispatch: inputs: @@ -8,25 +9,23 @@ on: required: true type: boolean default: false - release_type: - description: "Type of release (base or main)" - required: true - type: choice - options: - - base - - main pre_release: description: "Pre-release" required: false type: boolean default: true + branch: + description: "Branch to release from" + required: true + type: string + default: "main" env: POETRY_VERSION: "1.8.2" jobs: - release: - name: Release Langflow + release-base: + name: Release Langflow Base if: inputs.release_package == true runs-on: ubuntu-latest outputs: @@ -47,13 +46,8 @@ jobs: - name: Check Version id: check-version run: | - if [ "${{ inputs.release_type }}" == "base" ]; then - version=$(cd src/backend/base && poetry version --short) - last_released_version=$(curl -s "https://pypi.org/pypi/langflow-base/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1) - else - version=$(poetry version --short) - last_released_version=$(curl -s "https://pypi.org/pypi/langflow/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1) - fi + version=$(cd src/backend/base && poetry version --short) + last_released_version=$(curl -s "https://pypi.org/pypi/langflow-base/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1) if [ "$version" = "$last_released_version" ]; then echo "Version $version is already released. Skipping release." exit 1 @@ -61,19 +55,10 @@ jobs: echo version=$version >> $GITHUB_OUTPUT fi - name: Build project for distribution - run: | - if [ "${{ inputs.release_type }}" == "base" ]; then - make build base=true - else - make build main=true - fi + run: make build base=true - name: Test CLI run: | - if [ "${{ inputs.release_type }}" == "base" ]; then - python -m pip install src/backend/base/dist/*.whl - else - python -m pip install dist/*.whl - fi + python -m pip install src/backend/base/dist/*.whl python -m langflow run --host 127.0.0.1 --port 7860 & SERVER_PID=$! # Wait for the server to start @@ -91,37 +76,100 @@ jobs: - name: Publish to PyPI env: POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }} - run: | - if [ "${{ inputs.release_type }}" == "base" ]; then - make publish base=true - else - make publish main=true - fi + run: make publish base=true - name: Upload Artifact uses: actions/upload-artifact@v4 with: - name: dist${{ inputs.release_type }} - path: ${{ inputs.release_type == 'base' && 'src/backend/base/dist' || 'dist' }} + name: dist-base + path: src/backend/base/dist + + release-main: + name: Release Langflow Main + if: inputs.release_package == true + needs: release-base + runs-on: ubuntu-latest + outputs: + version: ${{ steps.check-version.outputs.version }} + steps: + - uses: actions/checkout@v4 + - name: Install poetry + run: pipx install poetry==${{ env.POETRY_VERSION }} + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "poetry" + - name: Set up Nodejs 20 + uses: actions/setup-node@v4 + with: + node-version: "20" + - name: Check Version + id: check-version + run: | + version=$(poetry version --short) + last_released_version=$(curl -s "https://pypi.org/pypi/langflow/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1) + if [ "$version" = "$last_released_version" ]; then + echo "Version $version is already released. Skipping release." + exit 1 + else + echo version=$version >> $GITHUB_OUTPUT + fi + - name: Wait for PyPI Propagation + run: sleep 300 # wait for 5 minutes to ensure PyPI propagation + - name: Build project for distribution + run: make build main=true + - name: Test CLI + run: | + python -m pip install dist/*.whl + python -m langflow run --host 127.0.0.1 --port 7860 & + SERVER_PID=$! + # Wait for the server to start + timeout 120 bash -c 'until curl -f http://127.0.0.1:7860/health; do sleep 2; done' || (echo "Server did not start in time" && kill $SERVER_PID && exit 1) + # Terminate the server + kill $SERVER_PID || (echo "Failed to terminate the server" && exit 1) + sleep 10 # give the server some time to terminate + # Check if the server is still running + if kill -0 $SERVER_PID 2>/dev/null; then + echo "Failed to terminate the server" + exit 1 + else + echo "Server terminated successfully" + fi + - name: Publish to PyPI + env: + POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }} + run: make publish main=true + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: dist-main + path: dist call_docker_build: name: Call Docker Build Workflow - needs: release + needs: [release-base, release-main] uses: langflow-ai/langflow/.github/workflows/docker-build.yml@main + strategy: + matrix: + release_type: + - base + - main with: - version: ${{ needs.release.outputs.version }} - release_type: ${{ inputs.release_type }} + # version should be needs.release-base.outputs.version if release_type is base + # version should be needs.release-main.outputs.version if release_type is main + version: ${{ matrix.release_type == 'base' && needs.release-base.outputs.version || matrix.release_type == 'main' && needs.release-main.outputs.version }} + release_type: ${{ matrix.release_type }} pre_release: ${{ inputs.pre_release }} secrets: inherit create_release: name: Create Release runs-on: ubuntu-latest - needs: [release] - if: ${{ inputs.release_type == 'main' }} + needs: release-main steps: - uses: actions/download-artifact@v4 with: - name: dist${{ inputs.release_type }} + name: dist-main path: dist - name: Create Release uses: ncipollo/release-action@v1 @@ -130,6 +178,6 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} draft: false generateReleaseNotes: true - prerelease: false - tag: v${{ needs.release.outputs.version }} - commit: dev + prerelease: ${{ inputs.pre_release }} + tag: v${{ needs.release-main.outputs.version }} + commit: ${{ inputs.branch }} \ No newline at end of file diff --git a/README.md b/README.md index 361d1e08c..c4fcde4a2 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ # 📝 Content +- [](#) - [📝 Content](#-content) - [📦 Get Started](#-get-started) - [🎨 Create Flows](#-create-flows) @@ -49,6 +50,7 @@ - [Deploy Langflow on Google Cloud Platform](#deploy-langflow-on-google-cloud-platform) - [Deploy on Railway](#deploy-on-railway) - [Deploy on Render](#deploy-on-render) + - [Deploy on Kubernetes](#deploy-on-kubernetes) - [🖥️ Command Line Interface (CLI)](#️-command-line-interface-cli) - [Usage](#usage) - [Environment Variables](#environment-variables) diff --git a/pyproject.toml b/pyproject.toml index 9fd1d3e04..5fc120287 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langflow" -version = "1.0.1" +version = "1.0.2" description = "A Python package with a built-in web application" authors = ["Langflow "] maintainers = [ diff --git a/src/backend/base/langflow/alembic/versions/2ac71eb9c3ae_adds_credential_table.py b/src/backend/base/langflow/alembic/versions/2ac71eb9c3ae_adds_credential_table.py index 617189071..2250a8b8c 100644 --- a/src/backend/base/langflow/alembic/versions/2ac71eb9c3ae_adds_credential_table.py +++ b/src/backend/base/langflow/alembic/versions/2ac71eb9c3ae_adds_credential_table.py @@ -34,7 +34,7 @@ def upgrade() -> None: sa.Column("provider", sqlmodel.sql.sqltypes.AutoString(), nullable=True), sa.Column("user_id", sqlmodel.sql.sqltypes.GUID(), nullable=False), sa.Column("id", sqlmodel.sql.sqltypes.GUID(), nullable=False), - sa.Column("created_at", sqlmodel.sql.sqltypes.DateTime(), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), sa.Column("updated_at", sa.DateTime(), nullable=True), sa.PrimaryKeyConstraint("id"), ) diff --git a/src/backend/base/langflow/components/agents/ToolCallingAgent.py b/src/backend/base/langflow/components/agents/ToolCallingAgent.py index 017003aaa..407ab333d 100644 --- a/src/backend/base/langflow/components/agents/ToolCallingAgent.py +++ b/src/backend/base/langflow/components/agents/ToolCallingAgent.py @@ -2,7 +2,6 @@ from typing import Dict, List, cast from langchain.agents import AgentExecutor, BaseSingleActionAgent from langchain.agents.tool_calling_agent.base import create_tool_calling_agent -from langchain_core.messages import BaseMessage from langchain_core.prompts import ChatPromptTemplate from langflow.custom import Component diff --git a/src/backend/base/langflow/components/vectorstores/AstraDB.py b/src/backend/base/langflow/components/vectorstores/AstraDB.py index 58e1d19ef..aabbb91a7 100644 --- a/src/backend/base/langflow/components/vectorstores/AstraDB.py +++ b/src/backend/base/langflow/components/vectorstores/AstraDB.py @@ -3,13 +3,13 @@ from loguru import logger from langflow.base.vectorstores.model import LCVectorStoreComponent from langflow.io import ( BoolInput, + DataInput, DropdownInput, HandleInput, IntInput, MultilineInput, SecretStrInput, StrInput, - DataInput, ) from langflow.schema import Data @@ -196,6 +196,10 @@ class AstraVectorStoreComponent(LCVectorStoreComponent): except Exception as e: raise ValueError(f"Error initializing AstraDBVectorStore: {str(e)}") from e + if hasattr(self, "ingest_data") and self.ingest_data: + logger.debug("Ingesting data into the Vector Store.") + self._add_documents_to_vector_store(vector_store) + self.status = self._astradb_collection_to_data(vector_store.collection) return vector_store @@ -216,7 +220,7 @@ class AstraVectorStoreComponent(LCVectorStoreComponent): else: logger.debug("No documents to add to the Vector Store.") - def search_documents(self): + def search_documents(self) -> list[Data]: vector_store = self.build_vector_store() logger.debug(f"Search input: {self.search_input}") diff --git a/src/backend/base/langflow/main.py b/src/backend/base/langflow/main.py index 45a681ff5..a8a165c95 100644 --- a/src/backend/base/langflow/main.py +++ b/src/backend/base/langflow/main.py @@ -6,7 +6,7 @@ from typing import Optional from urllib.parse import urlencode import nest_asyncio # type: ignore -from fastapi import FastAPI, Request +from fastapi import FastAPI, Request, Response from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles @@ -32,22 +32,17 @@ from langflow.utils.logger import configure warnings.filterwarnings("ignore", category=PydanticDeprecatedSince20) -class RequestCancelledMiddleware: +class RequestCancelledMiddleware(BaseHTTPMiddleware): def __init__(self, app): - self.app = app + super().__init__(app) - async def __call__(self, scope, receive, send): - if scope["type"] != "http": - await self.app(scope, receive, send) - return + async def dispatch(self, request: Request, call_next): + queue: asyncio.Queue = asyncio.Queue() - # Let's make a shared queue for the request messages - queue = asyncio.Queue() - - async def message_poller(sentinel, handler_task): + async def message_poller(sentinel, handler_task, request): nonlocal queue while True: - message = await receive() + message = await request.receive if message["type"] == "http.disconnect": handler_task.cancel() return sentinel # Break the loop @@ -56,13 +51,14 @@ class RequestCancelledMiddleware: await queue.put(message) sentinel = object() - handler_task = asyncio.create_task(self.app(scope, queue.get, send)) - asyncio.create_task(message_poller(sentinel, handler_task)) + handler_task = asyncio.create_task(call_next(request)) + asyncio.create_task(message_poller(sentinel, handler_task, request)) try: - return await handler_task + response = await handler_task + return response except asyncio.CancelledError: - pass + return Response("Request was cancelled", status_code=499) class JavaScriptMIMETypeMiddleware(BaseHTTPMiddleware): diff --git a/src/backend/base/pyproject.toml b/src/backend/base/pyproject.toml index 232bdd0aa..102097591 100644 --- a/src/backend/base/pyproject.toml +++ b/src/backend/base/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langflow-base" -version = "0.0.77" +version = "0.0.78" description = "A Python package with a built-in web application" authors = ["Langflow "] maintainers = [ diff --git a/tests/unit/test_helper_components.py b/tests/unit/test_helper_components.py index 80c85b787..9e9c4f63a 100644 --- a/tests/unit/test_helper_components.py +++ b/tests/unit/test_helper_components.py @@ -1,5 +1,3 @@ -from langchain_core.documents import Document - from langflow.components import helpers from langflow.custom.utils import build_custom_component_template from langflow.schema import Data @@ -17,18 +15,18 @@ from langflow.schema import Data # assert result.new_key == "new_value" -def test_document_to_data_component(): - # Arrange - document_to_data_component = helpers.DocumentsToDataComponent() +# def test_document_to_data_component(): +# # Arrange +# document_to_data_component = helpers.DocumentsToDataComponent() - # Act - # Replace with your actual test data - document = Document(page_content="key: value", metadata={"url": "https://example.com"}) - result = document_to_data_component.build(document) +# # Act +# # Replace with your actual test data +# document = Document(page_content="key: value", metadata={"url": "https://example.com"}) +# result = document_to_data_component.build(document) - # Assert - # Replace with your actual expected result - assert result == [Data(data={"text": "key: value", "url": "https://example.com"})] +# # Assert +# # Replace with your actual expected result +# assert result == [Data(data={"text": "key: value", "url": "https://example.com"})] def test_uuid_generator_component():