jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

ci: create a nightly build workflow (#3553)

Browse source 
* test poetry install

* Add nightly builds workflow

* remove old comments and fix poetry

* remove old debug statement

* [autofix.ci] apply automated fixes

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
Jordan Frazier 2024-09-05 12:13:04 -07:00 • committed by GitHub
commit 7b3e51f769
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 1770 additions and 1055 deletions
Download patch file Download diff file Expand all files Collapse all files

52
.github/workflows/docker-build.yml vendored
View file

@ -12,22 +12,31 @@ on:
required: false
type: boolean
default: false
nightly_build:
required: false
type: boolean
default: false
workflow_dispatch:
inputs:
version:
required: true
description: "Version to build"
required: false
type: string
release_type:
description: "Type of release"
required: true
type: choice
options:
- base
- main
type: string
pre_release:
description: "Pre-release"
required: false
type: boolean
default: true
default: false
nightly_build:
description: "Build nightly build"
required: false
type: boolean
default: false
env:
POETRY_VERSION: "1.8.2"
TEST_TAG: "langflowai/langflow:test"
@ -48,19 +57,19 @@ jobs:
poetry-version: ${{ env.POETRY_VERSION }}
cache-key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}
- name: Get Version from Input
if : ${{ inputs.version != '' }}
if: ${{ inputs.version != '' }}
id: get-version-input
run: |
version=${{ inputs.version }}
echo version=$version >> $GITHUB_OUTPUT
- name: Get Version Main
if : ${{ inputs.version == '' && inputs.release_type == 'base' }}
- name: Get Version Base
if: ${{ inputs.version == '' && inputs.release_type == 'base' }}
id: get-version-base
run: |
version=$(cd src/backend/base && poetry version --short)
echo version=$version >> $GITHUB_OUTPUT
- name: Get Version Base
if : ${{ inputs.version == '' && inputs.release_type == 'main' }}
- name: Get Version Main
if: ${{ inputs.version == '' && inputs.release_type == 'main' }}
id: get-version-main
run: |
version=$(poetry version --short)
@ -76,14 +85,19 @@ jobs:
- name: Set Dockerfile and Tags
id: set-vars
run: |
nightly_suffix=""
if [[ "${{ inputs.nightly_build }}" == "true" ]]; then
nightly_suffix="-nightly"
fi
if [[ "${{ inputs.release_type }}" == "base" ]]; then
echo "tags=langflowai/langflow:base-${{ needs.get-version.outputs.version }},langflowai/langflow:base-latest" >> $GITHUB_OUTPUT
echo "tags=langflowai/langflow${nightly_suffix}:base-${{ needs.get-version.outputs.version }},langflowai/langflow${nightly_suffix}:base-latest" >> $GITHUB_OUTPUT
echo "file=./docker/build_and_push_base.Dockerfile" >> $GITHUB_OUTPUT
else
if [[ "${{ inputs.pre_release }}" == "true" ]]; then
echo "tags=langflowai/langflow:${{ needs.get-version.outputs.version }}" >> $GITHUB_OUTPUT
echo "tags=langflowai/langflow${nightly_suffix}:${{ needs.get-version.outputs.version }}" >> $GITHUB_OUTPUT
else
echo "tags=langflowai/langflow:${{ needs.get-version.outputs.version }},langflowai/langflow:latest" >> $GITHUB_OUTPUT
echo "tags=langflowai/langflow${nightly_suffix}:${{ needs.get-version.outputs.version }},langflowai/langflow${nightly_suffix}:latest" >> $GITHUB_OUTPUT
fi
echo "file=./docker/build_and_push.Dockerfile" >> $GITHUB_OUTPUT
fi
@ -119,10 +133,10 @@ jobs:
include:
- component: backend
dockerfile: ./docker/build_and_push_backend.Dockerfile
tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-backend:{0}', needs.get-version.outputs.version) || format('langflowai/langflow-backend:{0},langflowai/langflow-backend:latest', needs.get-version.outputs.version) }}
tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-backend{0}:{1}', inputs.nightly_build && '-nightly' || '', needs.get-version.outputs.version) || format('langflowai/langflow-backend{0}:{1},langflowai/langflow-backend{0}:latest{1}', inputs.nightly_build && '-nightly' || '', needs.get-version.outputs.version) }}
- component: frontend
dockerfile: ./docker/frontend/build_and_push_frontend.Dockerfile
tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-frontend:{0}', needs.get-version.outputs.version) || format('langflowai/langflow-frontend:{0},langflowai/langflow-frontend:latest', needs.get-version.outputs.version) }}
tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-frontend{0}:{1}', inputs.nightly_build && '-nightly' || '', needs.get-version.outputs.version) || format('langflowai/langflow-frontend{0}:{1},langflowai/langflow-frontend{0}:latest{1}', inputs.nightly_build && '-nightly' || '', needs.get-version.outputs.version) }}
steps:
- uses: actions/checkout@v4
- name: Set up Docker Buildx
@ -140,7 +154,7 @@ jobs:
context: .
push: true
build-args: |
LANGFLOW_IMAGE=langflowai/langflow:${{ needs.get-version.outputs.version }}
LANGFLOW_IMAGE=langflowai/langflow${{ inputs.nightly_build && '-nightly' || '' }}:${{ needs.get-version.outputs.version }}
file: ${{ matrix.dockerfile }}
tags: ${{ matrix.tags }}
# provenance: false will result in a single manifest for all platforms which makes the image pullable from arm64 machines via the emulation (e.g. Apple Silicon machines)
@ -148,7 +162,7 @@ jobs:
restart-space:
name: Restart HuggingFace Spaces
if: ${{ inputs.release_type == 'main' }}
if: ${{ inputs.release_type == 'main' && inputs.nightly_build == 'false' }}
runs-on: ubuntu-latest
needs: build
strategy:

23
.github/workflows/scheduled_integration_test.yml → .github/workflows/integration_tests.yml vendored
View file

@ -1,20 +1,29 @@
name: Integration tests
name: Integration Tests
on:
workflow_dispatch:
inputs:
branch:
description: "(Optional) Branch to checkout"
ref:
description: "(Optional) ref to checkout"
required: false
type: string
workflow_call:
inputs:
python-versions:
description: "(Optional) Python versions to test"
required: true
type: string
default: "['3.10', '3.11', '3.12']"
ref:
description: "(Optional) ref to checkout"
required: false
type: string
schedule:
- cron: "0 0 */2 * *" # Run every 2 days
env:
POETRY_VERSION: "1.8.2"
jobs:
test-integration:
integration-tests:
name: Run Integration Tests
runs-on: ubuntu-latest
strategy:
@ -32,7 +41,7 @@ jobs:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.branch || github.ref }}
ref: ${{ inputs.ref || github.ref }}
- name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
uses: "./.github/actions/poetry_caching"
with:

137
.github/workflows/nightly_build.yml vendored
View file

@ -3,15 +3,142 @@ name: Nightly Build
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * *" # Run every day at midnight (UTC)
# Run job at 6:30 UTC, 10.30pm PST, or 11.30pm PDT
- cron: "30 6 * * *"
env:
POETRY_VERSION: "1.8.2"
POETRY_VERSION: "1.8.3"
PYTHON_VERSION: "3.12"
jobs:
hello-world:
create-nightly-tag:
runs-on: ubuntu-latest
defaults:
run:
shell: bash
permissions:
# Required to create tag
contents: write
outputs:
main_tag: ${{ steps.create_tag.outputs.main_tag }}
base_tag: ${{ steps.create_tag.outputs.base_tag }}
steps:
- name: Run hello world
- name: Checkout code
uses: actions/checkout@v4
with:
persist-credentials: true
- name: Set up Python ${{ env.PYTHON_VERSION }} + Poetry ${{ env.POETRY_VERSION }}
uses: "./.github/actions/poetry_caching"
with:
python-version: ${{ env.PYTHON_VERSION }}
poetry-version: ${{ env.POETRY_VERSION }}
cache-key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}
- name: Install dependencies
run: |
echo "Hello, world!"
poetry env use ${{ env.PYTHON_VERSION }}
poetry install
- name: Create tag
id: create_tag
run: |
git config --global user.email "bot-nightly-builds@langflow.org"
git config --global user.name "Langflow Bot"
# WARNING: These scripts must be run in this order.
# Poetry will use a different cached virtual environment once the main pyproject.toml
# project-name is updated, which does not have dependencies installed.
BASE_TAG="$(poetry run python ./scripts/ci/pypi_nightly_tag.py base)"
echo "base_tag=$BASE_TAG" >> $GITHUB_OUTPUT
poetry run python ./scripts/ci/update_pyproject_name.py langflow-base-nightly base
poetry run python ./scripts/ci/update_pyproject_version.py $BASE_TAG base
# This updates the dependency of langflow-base to langflow-base-nightly in {project_root}/pyproject.toml
poetry run python ./scripts/ci/update_lf_base_dependency.py $BASE_TAG
MAIN_TAG="$(poetry run python ./scripts/ci/pypi_nightly_tag.py main)"
echo "main_tag=$MAIN_TAG" >> $GITHUB_OUTPUT
poetry run python ./scripts/ci/update_pyproject_version.py $MAIN_TAG main
poetry run python ./scripts/ci/update_pyproject_name.py langflow-nightly main
git add pyproject.toml src/backend/base/pyproject.toml
git commit -m "Update version and project name in files"
git tag -a $MAIN_TAG -m "Langflow nightly $MAIN_TAG"
git push origin $MAIN_TAG || echo "Tag push failed. Check if the tag already exists."
# TODO: notify on failure
frontend-tests:
name: Run Frontend Tests
needs: create-nightly-tag
uses: ./.github/workflows/typescript_test.yml
with:
tests_folder: "tests/end-to-end"
ref: ${{ needs.create-nightly-tag.outputs.tag }}
secrets:
OPENAI_API_KEY: "${{ secrets.OPENAI_API_KEY }}"
STORE_API_KEY: "${{ secrets.STORE_API_KEY }}"
backend-unit-tests:
name: Run Backend Unit Tests
needs: create-nightly-tag
uses: ./.github/workflows/python_test.yml
with:
python-versions: '["3.10", "3.11", "3.12"]'
ref: ${{ needs.create-nightly-tag.outputs.tag }}
# Not making nightly builds dependent on integration test success
# due to inherent flakiness of 3rd party integrations
# Revisit when https://github.com/langflow-ai/langflow/pull/3607 is merged.
# backend-integration-tests:
# name: Run Backend Integration Tests
# needs: create-nightly-tag
# uses: ./.github/workflows/integration_tests.yml
# with:
# python-versions: '["3.10", "3.11", "3.12"]'
# ref: ${{ needs.create-nightly-tag.outputs.tag }}
release-nightly-build:
name: Run Nightly Langflow Build
needs: [frontend-tests, backend-unit-tests, create-nightly-tag]
uses: ./.github/workflows/release_nightly.yml
with:
build_docker_base: false # TODO: Docker builds
build_docker_main: false
nightly_tag_main: ${{ needs.create-nightly-tag.outputs.main_tag }}
nightly_tag_base: ${{ needs.create-nightly-tag.outputs.base_tag }}
secrets: inherit
# slack-notification:
# name: Send Slack Notification
# needs: run-nightly-build
# runs-on: ubuntu-latest
# steps:
# - name: Send success notification to Slack
# if: success()
# uses: slackapi/slack-github-action@v1.26.0
# with:
# payload: |
# {
# "channel": "#langflow-nightly-builds",
# "username": "GitHub Actions",
# "text": "Nightly Build Successful :white_check_mark:",
# "icon_emoji": ":rocket:"
# }
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
# - name: Send failure notification to Slack
# if: failure()
# uses: slackapi/slack-github-action@v1.26.0
# with:
# payload: |
# {
# "channel": "#langflow-nightly-builds",
# "username": "GitHub Actions",
# "text": "Nightly Build Failed :x:",
# "icon_emoji": ":warning:"
# }
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

25
.github/workflows/python_test.yml vendored
View file

@ -8,12 +8,17 @@ on:
required: true
type: string
default: "['3.10', '3.11', '3.12']"
workflow_dispatch:
inputs:
branch:
description: "(Optional) Branch to checkout"
ref:
description: "(Optional) ref to checkout"
required: false
type: string
nightly:
description: "Whether run is from the nightly build"
required: false
type: boolean
default: false
workflow_dispatch:
inputs:
python-versions:
description: "(Optional) Python versions to test"
required: true
@ -35,7 +40,7 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
ref: ${{ inputs.branch || github.ref }}
ref: ${{ inputs.ref || github.ref }}
- name: Setup Node.js
uses: actions/setup-node@v4
id: setup-node
@ -88,6 +93,9 @@ jobs:
python-version: ${{ fromJson(inputs.python-versions || '["3.10", "3.11", "3.12"]') }}
steps:
- uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
uses: "./.github/actions/poetry_caching"
with:
@ -98,7 +106,12 @@ jobs:
id: check-version
run: |
version=$(cd src/backend/base && poetry version --short)
last_released_version=$(curl -s "https://pypi.org/pypi/langflow-base/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1)
url="https://pypi.org/pypi/langflow-base/json"
if [ ${{ inputs.nightly }} == true ]; then
url="https://pypi.org/pypi/langflow-base-nightly/json"
fi
last_released_version=$(curl -s $url | jq -r '.releases | keys | .[]' | sort -V | tail -n 1)
if [ "$version" != "$last_released_version" ]; then
echo "Version $version has not been released yet. Skipping the rest of the job."
echo skipped=true >> $GITHUB_OUTPUT

18
.github/workflows/release.yml vendored
View file

@ -29,7 +29,11 @@ on:
required: false
type: boolean
default: false
create_release:
description: "Whether to create a gh release"
required: false
type: boolean
default: true
env:
POETRY_VERSION: "1.8.2"
@ -155,7 +159,7 @@ jobs:
fi
- name: Wait for PyPI Propagation
if: needs.release-base.outputs.skipped == 'false'
run: sleep 300 # wait for 5 minutes to ensure PyPI propagation
run: sleep 300 # wait for 5 minutes to ensure PyPI propagation
- name: Build project for distribution
run: make build main=true
@ -188,7 +192,7 @@ jobs:
call_docker_build_base:
name: Call Docker Build Workflow for Langflow Base
if : inputs.build_docker_base == true
if: inputs.build_docker_base == true
uses: ./.github/workflows/docker-build.yml
strategy:
matrix:
@ -197,14 +201,14 @@ jobs:
with:
# version should be needs.release-base.outputs.version if release_type is base
# version should be needs.release-main.outputs.version if release_type is main
version: ''
version: ""
release_type: ${{ matrix.release_type }}
pre_release: ${{ inputs.pre_release }}
secrets: inherit
call_docker_build_main:
name: Call Docker Build Workflow for Langflow
if : inputs.build_docker_main == true
if: inputs.build_docker_main == true
uses: ./.github/workflows/docker-build.yml
strategy:
matrix:
@ -213,7 +217,7 @@ jobs:
with:
# version should be needs.release-base.outputs.version if release_type is base
# version should be needs.release-main.outputs.version if release_type is main
version: ''
version: ""
release_type: ${{ matrix.release_type }}
pre_release: ${{ inputs.pre_release }}
secrets: inherit
@ -236,4 +240,4 @@ jobs:
generateReleaseNotes: true
prerelease: ${{ inputs.pre_release }}
tag: v${{ needs.release-main.outputs.version }}
commit: ${{ github.ref }}
commit: ${{ github.ref }}

225
.github/workflows/release_nightly.yml vendored Normal file
View file

@ -0,0 +1,225 @@
name: Langflow Nightly Build
run-name: Langflow Nightly Release by @${{ github.actor }}
on:
workflow_dispatch:
inputs:
build_docker_base:
description: "Build Docker Image for Langflow Nightly Base"
required: true
type: boolean
default: false
build_docker_main:
description: "Build Docker Image for Langflow Nightly"
required: true
type: boolean
default: false
nightly_tag_main:
description: "Tag for the nightly main build"
required: true
type: string
nightly_tag_base:
description: "Tag for the nightly base build"
required: true
type: string
workflow_call:
inputs:
build_docker_base:
description: "Build Docker Image for Langflow Nightly Base"
required: true
type: boolean
default: false
build_docker_main:
description: "Build Docker Image for Langflow Nightly"
required: true
type: boolean
default: false
nightly_tag_main:
description: "Tag for the nightly main build"
required: true
type: string
nightly_tag_base:
description: "Tag for the nightly base build"
required: true
type: string
env:
POETRY_VERSION: "1.8.3"
PYTHON_VERSION: "3.12"
jobs:
release-nightly-base:
name: Release Langflow Nightly Base
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- name: Check out the code at a specific ref (e.g., nightly tag)
uses: actions/checkout@v4
with:
ref: ${{ inputs.nightly_tag_main }}
persist-credentials: true
- name: Install poetry
run: |
pipx install poetry==${{ env.POETRY_VERSION }}
- name: Set up Python 3.12
uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: "poetry"
- name: Set up Nodejs 20
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Verify Nightly Name and Version
working-directory: src/backend/base
run: |
name=$(poetry version | cut -d' ' -f1)
version=$(poetry version --short)
if [ "$name" != "langflow-base-nightly" ]; then
echo "Name $name does not match langflow-base-nightly. Exiting the workflow."
exit 1
fi
if [ "$version" != "${{ inputs.nightly_tag_base }}" ]; then
echo "Version $version does not match nightly tag ${{ inputs.nightly_tag_base }}. Exiting the workflow."
exit 1
fi
- name: Build project for distribution
run: make build base=true
- name: Test CLI
run: |
python -m pip install src/backend/base/dist/*.whl
python -m langflow run --host 127.0.0.1 --port 7860 &
SERVER_PID=$!
# Wait for the server to start
timeout 120 bash -c 'until curl -f http://127.0.0.1:7860/api/v1/auto_login; do sleep 2; done' || (echo "Server did not start in time" && kill $SERVER_PID && exit 1)
# Terminate the server
kill $SERVER_PID || (echo "Failed to terminate the server" && exit 1)
sleep 10 # give the server some time to terminate
# Check if the server is still running
if kill -0 $SERVER_PID 2>/dev/null; then
echo "Failed to terminate the server"
exit 1
else
echo "Server terminated successfully"
fi
- name: Publish to PyPI
env:
POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
run: |
make publish base=true
- name: Upload Artifact
uses: actions/upload-artifact@v4
with:
name: dist-base
path: src/backend/base/dist
release-nightly-main:
name: Release Langflow Nightly Main
needs: [release-nightly-base]
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- name: Check out the code at a specific ref (e.g., nightly tag)
uses: actions/checkout@v4
with:
ref: ${{ inputs.nightly_tag_main }}
- name: Install poetry
run: pipx install poetry==${{ env.POETRY_VERSION }}
- name: Set up Python 3.12
uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: "poetry"
- name: Set up Nodejs 20
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Verify Nightly Name and Version
run: |
name=$(poetry version | cut -d' ' -f1)
version=$(poetry version --short)
if [ "$name" != "langflow-nightly" ]; then
echo "Name $name does not match langflow-nightly. Exiting the workflow."
exit 1
fi
if [ "$version" != "${{ inputs.nightly_tag_main }}" ]; then
echo "Version $version does not match nightly tag ${{ inputs.nightly_tag_main }}. Exiting the workflow."
exit 1
fi
- name: Wait for PyPI Propagation
run: sleep 300 # wait for 5 minutes to ensure PyPI propagation of base
- name: Build project for distribution
run: make build main=true
- name: Test CLI
run: |
python -m pip install dist/*.whl
python -m langflow run --host 127.0.0.1 --port 7860 --backend-only &
SERVER_PID=$!
# Wait for the server to start
timeout 120 bash -c 'until curl -f http://127.0.0.1:7860/health_check; do sleep 2; done' || (echo "Server did not start in time" && kill $SERVER_PID && exit 1)
# Terminate the server
kill $SERVER_PID || (echo "Failed to terminate the server" && exit 1)
sleep 10 # give the server some time to terminate
# Check if the server is still running
if kill -0 $SERVER_PID 2>/dev/null; then
echo "Failed to terminate the server"
exit 1
else
echo "Server terminated successfully"
fi
- name: Publish to PyPI
env:
POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
run: |
make publish main=true
- name: Upload Artifact
uses: actions/upload-artifact@v4
with:
name: dist-main
path: dist
call_docker_build_base:
name: Call Docker Build Workflow for Langflow Base
if: ${{ inputs.build_docker_base == 'true' }}
needs: [release-nightly-base]
uses: ./.github/workflows/docker-build.yml
strategy:
matrix:
release_type:
- base
with:
# version should be needs.release-base.outputs.version if release_type is base
# version should be needs.release-main.outputs.version if release_type is main
version: ""
release_type: ${{ matrix.release_type }}
nightly_build: true
secrets: inherit
call_docker_build_main:
name: Call Docker Build Workflow for Langflow
if: ${{ inputs.build_docker_main == 'true' }}
needs: [release-nightly-main]
uses: ./.github/workflows/docker-build.yml
strategy:
matrix:
release_type:
- main
with:
# version should be needs.release-base.outputs.version if release_type is base
# version should be needs.release-main.outputs.version if release_type is main
version: ""
release_type: ${{ matrix.release_type }}
nightly_build: true
secrets: inherit

14
.github/workflows/typescript_test.yml vendored
View file

@ -13,12 +13,12 @@ on:
required: false
type: string
default: "tests"
workflow_dispatch:
inputs:
branch:
description: "(Optional) Branch to checkout"
ref:
description: "(Optional) ref to checkout"
required: false
type: string
workflow_dispatch:
inputs:
tests_folder:
description: "(Optional) Tests to run"
required: false
@ -54,9 +54,9 @@ jobs:
- name: Checkout code
uses: actions/checkout@v4
with:
# If branch is passed as input, checkout that branch
# else checkout the default branch
ref: ${{ github.event.inputs.branch || github.ref }}
# If ref is passed as input, checkout that ref
# else checkout the default ref
ref: ${{ inputs.ref || github.ref }}
- name: Setup Node.js
uses: actions/setup-node@v4

20
Makefile
View file

@ -409,6 +409,12 @@ publish_base:
publish_langflow:
poetry publish
publish_base_testpypi:
cd src/backend/base && poetry publish --skip-existing -r test-pypi
publish_langflow_testpypi:
poetry publish -r test-pypi
publish: ## build the frontend static files and package the project and publish it to PyPI
@echo 'Publishing the project'
ifdef base
@ -418,3 +424,17 @@ endif
ifdef main
make publish_langflow
endif
publish_testpypi: ## build the frontend static files and package the project and publish it to PyPI
@echo 'Publishing the project'
ifdef base
poetry config repositories.test-pypi https://test.pypi.org/legacy/
make publish_base_testpypi
endif
ifdef main
poetry config repositories.test-pypi https://test.pypi.org/legacy/
make publish_langflow_testpypi
endif

1814
poetry.lock generated
View file

File diff suppressed because it is too large Load diff

1
pyproject.toml
View file

@ -142,6 +142,7 @@ dictdiffer = "^0.9.0"
pytest-split = "^0.9.0"
pytest-flakefinder = "^1.1.0"
types-markdown = "^3.7.0.20240822"
packaging = ">=23.2,<24.0"
[tool.poetry.extras]
deploy = ["celery", "redis", "flower"]

77
scripts/ci/pypi_nightly_tag.py Executable file
View file

@ -0,0 +1,77 @@
#!/usr/bin/env python
"""
Idea from https://github.com/streamlit/streamlit/blob/4841cf91f1c820a392441092390c4c04907f9944/scripts/pypi_nightly_create_tag.py
"""
import sys
import packaging.version
from packaging.version import Version
PYPI_LANGFLOW_URL = "https://pypi.org/pypi/langflow/json"
PYPI_LANGFLOW_NIGHTLY_URL = "https://pypi.org/pypi/langflow-nightly/json"
PYPI_LANGFLOW_BASE_URL = "https://pypi.org/pypi/langflow-base/json"
PYPI_LANGFLOW_BASE_NIGHTLY_URL = "https://pypi.org/pypi/langflow-base-nightly/json"
def get_latest_published_version(build_type: str, is_nightly: bool) -> Version:
import requests
url = ""
if build_type == "base":
url = PYPI_LANGFLOW_BASE_NIGHTLY_URL if is_nightly else PYPI_LANGFLOW_BASE_URL
elif build_type == "main":
url = PYPI_LANGFLOW_NIGHTLY_URL if is_nightly else PYPI_LANGFLOW_URL
else:
raise ValueError(f"Invalid build type: {build_type}")
res = requests.get(url)
try:
version_str = res.json()["info"]["version"]
except Exception as e:
raise RuntimeError("Got unexpected response from PyPI", e)
return Version(version_str)
def create_tag(build_type: str):
current_version = get_latest_published_version(build_type, is_nightly=False)
current_nightly_version = get_latest_published_version(build_type, is_nightly=True)
build_number = "0"
latest_base_version = current_version.base_version
nightly_base_version = current_nightly_version.base_version
if latest_base_version == nightly_base_version:
# If the latest version is the same as the nightly version, increment the build number
build_number = str(current_nightly_version.dev + 1)
new_nightly_version = latest_base_version + ".dev" + build_number
# X.Y.Z.dev.YYYYMMDD
# This takes the base version of the current version and appends the
# current date. If the last release was on the same day, we exit, as
# pypi does not allow for overwriting the same version.
#
# We could use a different versioning scheme, such as just incrementing
# an integer.
# version_with_date = (
# ".".join([str(x) for x in current_version.release])
# + ".dev"
# + "0"
# + datetime.now(pytz.timezone("UTC")).strftime("%Y%m%d")
# )
# Verify if version is PEP440 compliant.
packaging.version.Version(new_nightly_version)
return new_nightly_version
if __name__ == "__main__":
if len(sys.argv) != 2:
raise Exception("Specify base or main")
build_type = sys.argv[1]
tag = create_tag(build_type)
print(tag)

62
scripts/ci/update_lf_base_dependency.py Executable file
View file

@ -0,0 +1,62 @@
import os
import sys
import re
import packaging.version
BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
def update_base_dep(pyproject_path: str, new_version: str) -> None:
"""Update the langflow-base dependency in pyproject.toml."""
filepath = os.path.join(BASE_DIR, pyproject_path)
with open(filepath, "r") as file:
content = file.read()
# Regex to match the langflow-base dep line under [tool.poetry]
# NOTE: this functions similarly to `update_dependencies.py`.
# The order of operations for these scripts is more complex than it needs to be;
# this is a reminder to revisit this process.
# Currently, the process is as follows:
# 1. nightly-build workflow updates all names, version, and the langflow-base
# dependency in pyproject.toml, and commits the changes and creates a tag.
# 2. release-nightly workflow runs, which calls update_dependencies.py to update
# the langflow-base dependency in pyproject.toml _again_. This is redundant, but
# necessary because the release workflow relies on that script to update the base
# dependency.
pattern = re.compile(r'langflow-base = \{ path = "\./src/backend/base", develop = true \}')
if not pattern.search(content):
raise Exception(f'langflow-base dependency not found in "{filepath}"')
replacement = f'langflow-base-nightly = "^{new_version}"'
content = pattern.sub(replacement, content)
with open(filepath, "w") as file:
file.write(content)
def verify_pep440(version):
"""
Verify if version is PEP440 compliant.
https://github.com/pypa/packaging/blob/16.7/packaging/version.py#L191
"""
try:
return packaging.version.Version(version)
except packaging.version.InvalidVersion as e:
raise e
def main() -> None:
if len(sys.argv) != 2:
raise Exception("New version not specified")
base_version = sys.argv[1]
verify_pep440(base_version)
update_base_dep("pyproject.toml", base_version)
if __name__ == "__main__":
main()

41
scripts/ci/update_pyproject_name.py Executable file
View file

@ -0,0 +1,41 @@
import os
import sys
import re
BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
def update_pyproject_name(pyproject_path: str, new_project_name: str) -> None:
"""Update the project name in pyproject.toml."""
filepath = os.path.join(BASE_DIR, pyproject_path)
with open(filepath, "r") as file:
content = file.read()
# Regex to match the version line under [tool.poetry]
pattern = re.compile(r'(?<=^name = ")[^"]+(?=")', re.MULTILINE)
if not pattern.search(content):
raise Exception(f'Project name not found in "{filepath}"')
content = pattern.sub(new_project_name, content)
with open(filepath, "w") as file:
file.write(content)
def main() -> None:
if len(sys.argv) != 3:
raise Exception("Must specify project name and build type, e.g. langflow-nightly base")
new_project_name = sys.argv[1]
build_type = sys.argv[2]
if build_type == "base":
update_pyproject_name("src/backend/base/pyproject.toml", new_project_name)
elif build_type == "main":
update_pyproject_name("pyproject.toml", new_project_name)
else:
raise ValueError(f"Invalid build type: {build_type}")
if __name__ == "__main__":
main()

58
scripts/ci/update_pyproject_version.py Executable file
View file

@ -0,0 +1,58 @@
import os
import sys
import re
import packaging.version
BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
def update_pyproject_version(pyproject_path: str, new_version: str) -> None:
"""Update the version in pyproject.toml."""
filepath = os.path.join(BASE_DIR, pyproject_path)
with open(filepath, "r") as file:
content = file.read()
# Regex to match the version line under [tool.poetry]
pattern = re.compile(r'(?<=^version = ")[^"]+(?=")', re.MULTILINE)
if not pattern.search(content):
raise Exception(f'Project version not found in "{filepath}"')
content = pattern.sub(new_version, content)
with open(filepath, "w") as file:
file.write(content)
def verify_pep440(version):
"""
Verify if version is PEP440 compliant.
https://github.com/pypa/packaging/blob/16.7/packaging/version.py#L191
"""
try:
return packaging.version.Version(version)
except packaging.version.InvalidVersion as e:
raise e
def main() -> None:
if len(sys.argv) != 3:
raise Exception("New version not specified")
new_version = sys.argv[1]
build_type = sys.argv[2]
verify_pep440(new_version)
if build_type == "base":
update_pyproject_version("src/backend/base/pyproject.toml", new_version)
elif build_type == "main":
update_pyproject_version("pyproject.toml", new_version)
else:
raise ValueError(f"Invalid build type: {build_type}")
if __name__ == "__main__":
main()

22
scripts/update_dependencies.py
View file

@ -31,9 +31,22 @@ def get_version_from_pypi(package_name):
return None
def update_pyproject_dependency(pyproject_path, version):
def is_development_release(version):
"""
Determines if the version is a development version based on PEP 440.
We consider a development version (.devN) as our nightly versions
"""
return "dev" in version
def update_pyproject_dependency(pyproject_path, version, is_nightly):
pattern = re.compile(r'langflow-base = \{ path = "\./src/backend/base", develop = true \}')
replacement = f'langflow-base = "^{version}"'
if is_nightly:
# NOTE: This process can be simplified; see the note in update_lf_base_dependency.py
replacement = f'langflow-base-nightly = "{version}"'
else:
replacement = f'langflow-base = "^{version}"'
with open(pyproject_path, "r") as file:
content = file.read()
content = pattern.sub(replacement, content)
@ -57,6 +70,9 @@ if __name__ == "__main__":
langflow_base_path = Path(__file__).resolve().parent / "../src/backend/base/pyproject.toml"
version = read_version_from_pyproject(langflow_base_path)
if version:
update_pyproject_dependency(pyproject_path, version)
# Nightly versions contain "dev"
# WARNING: This will cause issues if we release `.dev` versions to `langflow` or `langflow-base`
is_nightly = is_development_release(version)
update_pyproject_dependency(pyproject_path, version, is_nightly)
else:
print("Error: Version not found.")

80
src/backend/base/langflow/__main__.py
View file

@ -8,6 +8,8 @@ from typing import Optional
import click
import httpx
from langflow.utils.version import get_version_info, fetch_latest_version
from langflow.utils.version import is_pre_release as langflow_is_pre_release
import typer
from dotenv import load_dotenv
from multiprocess import cpu_count # type: ignore
@ -22,7 +24,9 @@ from sqlmodel import select
from langflow.logging.logger import configure, logger
from langflow.main import setup_app
from langflow.services.database.models.folder.utils import create_default_folder_if_it_doesnt_exist
from langflow.services.database.models.folder.utils import (
create_default_folder_if_it_doesnt_exist,
)
from langflow.services.database.utils import session_getter
from langflow.services.deps import get_db_service, get_settings_service, session_scope
from langflow.services.settings.constants import DEFAULT_SUPERUSER
@ -259,13 +263,6 @@ def get_free_port(port):
return port
def version_is_prerelease(version: str):
"""
Check if a version is a pre-release version.
"""
return "a" in version or "b" in version or "rc" in version
def get_letter_from_version(version: str):
"""
Get the letter from a pre-release version.
@ -279,30 +276,10 @@ def get_letter_from_version(version: str):
return None
def is_prerelease(version: str) -> bool:
return "a" in version or "b" in version or "rc" in version
def fetch_latest_version(package_name: str, include_prerelease: bool) -> Optional[str]:
valid_versions = []
try:
response = httpx.get(f"https://pypi.org/pypi/{package_name}/json")
versions = response.json()["releases"].keys()
valid_versions = [v for v in versions if include_prerelease or not is_prerelease(v)]
except Exception as e:
logger.exception(e)
finally:
if not valid_versions:
return None # Handle case where no valid versions are found
return max(valid_versions, key=lambda v: pkg_version.parse(v))
def build_version_notice(current_version: str, package_name: str) -> str:
latest_version = fetch_latest_version(package_name, is_prerelease(current_version))
latest_version = fetch_latest_version(package_name, langflow_is_pre_release(current_version))
if latest_version and pkg_version.parse(current_version) < pkg_version.parse(latest_version):
release_type = "pre-release" if is_prerelease(latest_version) else "version"
release_type = "pre-release" if langflow_is_pre_release(latest_version) else "version"
return f"A new {release_type} of {package_name} is available: {latest_version}"
return ""
@ -331,35 +308,17 @@ def print_banner(host: str, port: int):
is_pre_release = False # Track if any package is a pre-release
package_name = ""
try:
from langflow.version import __version__ as langflow_version # type: ignore
# Use langflow.utils.version to get the version info
version_info = get_version_info()
langflow_version = version_info["version"]
package_name = version_info["package"]
is_pre_release |= langflow_is_pre_release(langflow_version) # Update pre-release status
is_pre_release |= is_prerelease(langflow_version) # Update pre-release status
notice = build_version_notice(langflow_version, "langflow")
notice = stylize_text(notice, "langflow", is_pre_release)
if notice:
notices.append(notice)
package_names.append("langflow")
package_name = "Langflow"
except ImportError:
langflow_version = None
# Attempt to handle langflow-base similarly
if langflow_version is None: # This means langflow.version was not imported
try:
from importlib import metadata
langflow_base_version = metadata.version("langflow-base")
is_pre_release |= is_prerelease(langflow_base_version) # Update pre-release status
notice = build_version_notice(langflow_base_version, "langflow-base")
notice = stylize_text(notice, "langflow-base", is_pre_release)
if notice:
notices.append(notice)
package_names.append("langflow-base")
package_name = "Langflow Base"
except ImportError as e:
logger.exception(e)
raise e
notice = build_version_notice(langflow_version, package_name)
notice = stylize_text(notice, package_name, is_pre_release)
if notice:
notices.append(notice)
package_names.append(package_name)
# Generate pip command based on the collected data
pip_command = generate_pip_command(package_names, is_pre_release)
@ -528,7 +487,10 @@ def api_key(
typer.echo("Default superuser not found. This command requires a superuser and AUTO_LOGIN to be enabled.")
return
from langflow.services.database.models.api_key import ApiKey, ApiKeyCreate
from langflow.services.database.models.api_key.crud import create_api_key, delete_api_key
from langflow.services.database.models.api_key.crud import (
create_api_key,
delete_api_key,
)
api_key = session.exec(select(ApiKey).where(ApiKey.user_id == superuser.id)).first()
if api_key:

6
src/backend/base/langflow/api/utils.py
View file

@ -85,10 +85,12 @@ def get_is_component_from_data(data: dict):
async def check_langflow_version(component: StoreComponentCreate):
from langflow.version.version import __version__ as current_version # type: ignore
from langflow.version import get_version
__version__ = get_version()
if not component.last_tested_version:
component.last_tested_version = current_version
component.last_tested_version = __version__
langflow_version = get_lf_version_from_pypi()
if langflow_version is None:

7
src/backend/base/langflow/main.py
View file

@ -111,12 +111,9 @@ def get_lifespan(fix_migration=False, socketio_server=None, version=None):
def create_app():
"""Create the FastAPI app and include the router."""
try:
from langflow.version import __version__ # type: ignore
except ImportError:
from importlib.metadata import version
from langflow.utils.version import get_version_info
__version__ = version("langflow-base")
__version__ = get_version_info()["version"]
configure()
lifespan = get_lifespan(version=__version__)

11
src/backend/base/langflow/services/settings/base.py
View file

@ -220,13 +220,12 @@ class Settings(BaseSettings):
# if there is a database in that location
if not info.data["config_dir"]:
raise ValueError("config_dir not set, please set it or provide a database_url")
try:
from langflow.version import is_pre_release # type: ignore
except ImportError:
from importlib import metadata
version = metadata.version("langflow-base")
is_pre_release = "a" in version or "b" in version or "rc" in version
from langflow.utils.version import get_version_info
from langflow.utils.version import is_pre_release as langflow_is_pre_release
version = get_version_info()["version"]
is_pre_release = langflow_is_pre_release(version)
if info.data["save_db_in_config_dir"]:
database_dir = info.data["config_dir"]

85
src/backend/base/langflow/utils/version.py
View file

@ -1,3 +1,9 @@
import httpx
from typing import Optional
from langflow.logging.logger import logger
def _compute_non_prerelease_version(prerelease_version: str) -> str:
prerelease_keywords = ["a", "b", "rc", "dev", "post"]
for keyword in prerelease_keywords:
@ -7,23 +13,80 @@ def _compute_non_prerelease_version(prerelease_version: str) -> str:
def _get_version_info():
try:
from langflow.version import __version__ # type: ignore
"""
Retrieves the version of the package from a possible list of package names.
This accounts for after package names are updated for -nightly builds.
prerelease_version = __version__
version = _compute_non_prerelease_version(prerelease_version)
package = "Langflow"
except ImportError:
from importlib import metadata
Returns:
str: The version of the package
prerelease_version = metadata.version("langflow-base")
version = _compute_non_prerelease_version(prerelease_version)
package = "Langflow Base"
return {"version": prerelease_version, "main_version": version, "package": package}
Raises:
ValueError: If the package is not found from the list of package names.
"""
from importlib import metadata
package_options = [
("langflow", "Langflow"),
("langflow-base", "Langflow Base"),
("langflow-nightly", "Langflow Nightly"),
("langflow-base-nightly", "Langflow Base Nightly"),
]
__version__ = None
for pkg_name, display_name in package_options:
try:
__version__ = metadata.version(pkg_name)
prerelease_version = __version__
version = _compute_non_prerelease_version(prerelease_version)
return {
"version": prerelease_version,
"main_version": version,
"package": display_name,
}
except (ImportError, metadata.PackageNotFoundError):
pass
if __version__ is None:
raise ValueError(f"Package not found from options {package_options}")
VERSION_INFO = _get_version_info()
def is_pre_release(v: str) -> bool:
"""
Returns a boolean indicating whether the version is a pre-release version,
as per the definition of a pre-release segment from PEP 440.
"""
return any(label in v for label in ["a", "b", "rc"])
def is_nightly(v: str) -> bool:
"""
Returns a boolean indicating whether the version is a dev (nightly) version,
as per the definition of a dev segment from PEP 440.
"""
return "dev" in v
def fetch_latest_version(package_name: str, include_prerelease: bool) -> Optional[str]:
from packaging import version as pkg_version
package_name = package_name.replace(" ", "-").lower()
valid_versions = []
try:
response = httpx.get(f"https://pypi.org/pypi/{package_name}/json")
versions = response.json()["releases"].keys()
valid_versions = [v for v in versions if include_prerelease or not is_pre_release(v)]
except Exception as e:
logger.exception(e)
finally:
if not valid_versions:
return None # Handle case where no valid versions are found
return max(valid_versions, key=lambda v: pkg_version.parse(v))
def get_version_info():
return VERSION_INFO

1
src/backend/langflow/version/__init__.py
View file

@ -1 +0,0 @@
from .version import __version__, is_pre_release # noqa: F401

46
src/backend/langflow/version/version.py
View file

@ -1,10 +1,38 @@
from importlib import metadata
def get_version() -> str:
"""
Retrieves the version of the package from a possible list of package names.
This accounts for after package names are updated for -nightly builds.
try:
__version__ = metadata.version("langflow")
# Check if the version is a pre-release version
is_pre_release = any(label in __version__ for label in ["a", "b", "rc", "dev", "post"])
except metadata.PackageNotFoundError:
__version__ = ""
is_pre_release = False
del metadata
Returns:
str: The version of the package
Raises:
ValueError: If the package is not found from the list of package names.
"""
from importlib import metadata
pkg_names = [
"langflow",
"langflow-base",
"langflow-nightly",
"langflow-base-nightly",
]
_version = None
for pkg_name in pkg_names:
try:
_version = metadata.version(pkg_name)
except (ImportError, metadata.PackageNotFoundError):
pass
if _version is None:
raise ValueError(f"Package not found from options {pkg_names}")
return _version
def is_pre_release(v: str) -> bool:
"""
Returns a boolean indicating whether the version is a pre-release version,
as per the definition of a pre-release segment from PEP 440.
"""
return any(label in v for label in ["a", "b", "rc"])