jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

ref: simplify docker build workflow (#4294)

Browse source
This commit is contained in:
Jordan Frazier 2024-11-06 12:38:05 -08:00 • committed by GitHub
commit 91ccf1f458
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 95 additions and 125 deletions
Download patch file Download diff file Expand all files Collapse all files

179
.github/workflows/docker-build.yml vendored
View file

@ -2,53 +2,50 @@ name: Docker Build and Push
on: on:
workflow_call: workflow_call:
inputs: inputs:
version: main_version:
required: true
type: string
description: "Main version to tag images with. Required for both main and base releases."
base_version:
required: false required: false
type: string type: string
description: "Main version to build. Does not support base version." description: "Base version to tag images with. Required for base release type."
release_type: release_type:
required: true required: true
type: string type: string
description: "Release type. One of 'main', 'main-ep', 'base', 'nightly-main', 'nightly-base'."
pre_release: pre_release:
required: false required: false
type: boolean type: boolean
default: false default: false
nightly_tag_main: ref:
description: "Tag for the nightly main build"
required: false required: false
type: string type: string
default: '' description: "Ref to check out. If not specified, will default to the main version or current branch."
nightly_tag_base:
description: "Tag for the nightly base build"
required: false
type: string
default: ''
workflow_dispatch: workflow_dispatch:
inputs: inputs:
version: main_version:
description: "Main version to build. Does not support base version." description: "Main version to tag images with. Required for both main and base releases."
required: false
type: string
base_version:
description: "Base version to tag images with. Required for base release type."
required: false required: false
type: string type: string
release_type: release_type:
description: "Type of release" description: "Type of release. One of 'main', 'main-ep', 'base', 'nightly-main', 'nightly-base'."
required: true required: true
type: string type: string
pre_release: pre_release:
description: "Pre-release"
required: false required: false
type: boolean type: boolean
default: false default: false
nightly_tag_main: ref:
description: "Tag for the nightly main build"
required: false required: false
type: string type: string
default: '' description: "Ref to check out. If not specified, will default to the main version or current branch."
nightly_tag_base:
description: "Tag for the nightly base build"
required: false
type: string
default: ''
env: env:
POETRY_VERSION: "1.8.2" POETRY_VERSION: "1.8.2"
@ -60,88 +57,58 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
outputs: outputs:
version: ${{ steps.get-version-input.outputs.version || steps.get-version-base.outputs.version || steps.get-version-main.outputs.version }} version: ${{ steps.get-version-input.outputs.version || steps.get-version-base.outputs.version || steps.get-version-main.outputs.version }}
nightly-tag: ${{ steps.resolve-nightly-tag.outputs.nightly_tag }}
nightly-build: ${{ steps.resolve-nightly-tag.outputs.nightly_build }}
steps: steps:
- name: Resolve nightly tag - name: Verify a main version exists
id: resolve-nightly-tag if: ${{ inputs.main_version == '' }}
run: | run: |
# Note - this is more complex than I'd like. For `main` builds, we just pass the `main` tag. # due to our how we split packages, we need to have a main version to check out.
# For `base` builds, we pass both the `base` and `main` tags. This is because the `main` tag is the echo "Must specify a main version to check out."
# version we need to check out for the build, but the `base` tag is the version we need to build. exit 1
#
# So, you need to check for `base` existence before `main`.
if [[ "${{ inputs.nightly_tag_base }}" != '' ]]; then
if [[ "${{ inputs.release_type }}" != "base" ]]; then
echo "Release type is not 'base'. Exiting the workflow."
exit 1
fi
# Main tag must not be empty, otherwise we have no valid tag to check out.
if [[ "${{ inputs.nightly_tag_main }}" == '' ]]; then
echo "Nightly tag main is empty. Exiting the workflow."
exit 1
fi
echo "nightly_tag=${{ inputs.nightly_tag_base }}" >> $GITHUB_OUTPUT
echo "nightly_build=true" >> $GITHUB_OUTPUT
elif [[ "${{ inputs.nightly_tag_main }}" != '' ]]; then
if [[ "${{ inputs.release_type }}" != "main" ]]; then
echo "Release type is not 'main'. Exiting the workflow."
exit 1
fi
echo "nightly_tag=${{ inputs.nightly_tag_main }}" >> $GITHUB_OUTPUT
echo "nightly_build=true" >> $GITHUB_OUTPUT
else
echo "nightly_tag=" >> $GITHUB_OUTPUT
echo "nightly_build=false" >> $GITHUB_OUTPUT
fi
- name: Check out the code at a specific ref - name: Check out the code at a specific ref
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
ref: ${{ inputs.nightly_tag_main || github.ref }} ref: ${{ inputs.ref || inputs.main_version || github.ref }}
persist-credentials: true persist-credentials: true
- name: "Setup Environment" - name: Get Version to Tag
uses: ./.github/actions/setup-uv if: ${{ inputs.main_version != '' }}
- name: Get Version from Input
if: ${{ inputs.version != '' }}
id: get-version-input id: get-version-input
run: | run: |
# Base version cannot have a version input, since you have to specify a valid main tag to checkout. # Produces the versions we will use to tag the docker images with.
if [[ "${{ inputs.release_type }}" == "base" && "${{ inputs.version }}" != '' ]]; then
echo "Cannot specify version for base release." if [[ "${{ inputs.release_type }}" == "base" && "${{ inputs.base_version }}" == '' ]]; then
echo "Build type is 'base' and version is non-empty. Exiting the workflow." echo "Must specify a base version for base release type."
exit 1 exit 1
fi fi
# If doing a nightly release, just specify the nightly tag. if [[ "${{ inputs.release_type }}" == "nightly-base" && "${{ inputs.base_version }}" == '' ]]; then
if [[ "${{ inputs.version }}" != '' && "${{ steps.resolve-nightly-tag.outputs.nightly-tag }}" != '' ]]; then echo "Must specify a base version for nightly-base release type."
echo "Cannot specify both version and nightly version."
echo "Version is specified and nightly tag is non-empty. Exiting the workflow."
exit 1 exit 1
fi fi
if [[ "${{ steps.resolve-nightly-tag.outputs.nightly-tag }}" != '' ]]; then if [[ "${{ inputs.release_type }}" == "main" && "${{ inputs.main_version }}" == '' ]]; then
if [[ "${{ inputs.nightly_tag_main }}" == '' ]]; then echo "Must specify a main version for main release type."
echo "Must specify a main nightly tag to check out when building base image." exit 1
echo "Nightly tag main is empty. Exiting the workflow." fi
exit 1
fi
echo "main nightly version=${{ inputs.nightly_tag_main }}"
echo "base nightly version=${{ inputs.nightly_tag_base }}"
echo "building image for nightly_version=${{ steps.resolve-nightly-tag.outputs.nightly-tag }}"
# Strip leading v from nightly tag main if present if [[ "${{ inputs.release_type }}" == "main-ep" && "${{ inputs.main_version }}" == '' ]]; then
version=${{ inputs.nightly_tag_main }} echo "Must specify a main version for main-ep release type."
version=${version#v} exit 1
fi
if [[ "${{ inputs.release_type }}" == "nightly-main" && "${{ inputs.main_version }}" == '' ]]; then
echo "Must specify a main version for nightly-main release type."
exit 1
fi
if [[ "${{ inputs.release_type }}" == "base" || "${{ inputs.release_type }}" == "nightly-base" ]]; then
version=${{ inputs.base_version }}
echo "base version=${{ inputs.base_version }}"
echo version=$version
echo version=$version >> $GITHUB_OUTPUT echo version=$version >> $GITHUB_OUTPUT
elif [[ "${{ inputs.version }}" != '' ]]; then elif [[ "${{ inputs.release_type }}" == "main" || "${{ inputs.release_type }}" == "main-ep" || "${{ inputs.release_type }}" == "nightly-main" ]]; then
# strip leading v if present version=${{ inputs.main_version }}
version=${{ inputs.version }}
version=${version#v}
echo version=$version echo version=$version
echo version=$version >> $GITHUB_OUTPUT echo version=$version >> $GITHUB_OUTPUT
else else
@ -149,7 +116,7 @@ jobs:
exit 1 exit 1
fi fi
- name: Get Version Base - name: Get Version Base
if: ${{ inputs.version == '' && inputs.release_type == 'base' }} if: ${{ inputs.base_version == '' && (inputs.release_type == 'base' || inputs.release_type == 'nightly-base') }}
id: get-version-base id: get-version-base
run: | run: |
version=$(uv tree | grep 'langflow-base' | awk '{print $3}' | sed 's/^v//') version=$(uv tree | grep 'langflow-base' | awk '{print $3}' | sed 's/^v//')
@ -160,7 +127,7 @@ jobs:
echo version=$version echo version=$version
echo version=$version >> $GITHUB_OUTPUT echo version=$version >> $GITHUB_OUTPUT
- name: Get Version Main - name: Get Version Main
if: ${{ inputs.version == '' && (inputs.release_type == 'main' || inputs.release_type == 'main-ep') }} if: ${{ inputs.main_version == '' && (inputs.release_type == 'main' || inputs.release_type == 'main-ep' || inputs.release_type == 'nightly-main') }}
id: get-version-main id: get-version-main
run: | run: |
version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//') version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//')
@ -178,11 +145,11 @@ jobs:
id: set-vars id: set-vars
run: | run: |
nightly_suffix='' nightly_suffix=''
if [[ "${{ needs.get-version.outputs.nightly-build }}" == "true" ]]; then if [[ "${{ inputs.release_type }}" == "nightly-base" || "${{ inputs.release_type }}" == "nightly-main" ]]; then
nightly_suffix="-nightly" nightly_suffix="-nightly"
fi fi
if [[ "${{ inputs.release_type }}" == "base" ]]; then if [[ "${{ inputs.release_type }}" == "base" || "${{ inputs.release_type }}" == "nightly-base" ]]; then
# LANGFLOW-BASE RELEASE # LANGFLOW-BASE RELEASE
echo "docker_tags=langflowai/langflow${nightly_suffix}:base-${{ needs.get-version.outputs.version }},langflowai/langflow${nightly_suffix}:base-latest" >> $GITHUB_OUTPUT echo "docker_tags=langflowai/langflow${nightly_suffix}:base-${{ needs.get-version.outputs.version }},langflowai/langflow${nightly_suffix}:base-latest" >> $GITHUB_OUTPUT
echo "ghcr_tags=ghcr.io/langflow-ai/langflow${nightly_suffix}:base-${{ needs.get-version.outputs.version }},ghcr.io/langflow-ai/langflow${nightly_suffix}:base-latest" >> $GITHUB_OUTPUT echo "ghcr_tags=ghcr.io/langflow-ai/langflow${nightly_suffix}:base-${{ needs.get-version.outputs.version }},ghcr.io/langflow-ai/langflow${nightly_suffix}:base-latest" >> $GITHUB_OUTPUT
@ -198,7 +165,7 @@ jobs:
echo "docker_tags=langflowai/langflow-ep${nightly_suffix}:${{ needs.get-version.outputs.version }},langflowai/langflow-ep${nightly_suffix}:latest" >> $GITHUB_OUTPUT echo "docker_tags=langflowai/langflow-ep${nightly_suffix}:${{ needs.get-version.outputs.version }},langflowai/langflow-ep${nightly_suffix}:latest" >> $GITHUB_OUTPUT
echo "ghcr_tags=ghcr.io/langflow-ai/langflow-ep${nightly_suffix}:${{ needs.get-version.outputs.version }},ghcr.io/langflow-ai/langflow-ep${nightly_suffix}:latest" >> $GITHUB_OUTPUT echo "ghcr_tags=ghcr.io/langflow-ai/langflow-ep${nightly_suffix}:${{ needs.get-version.outputs.version }},ghcr.io/langflow-ai/langflow-ep${nightly_suffix}:latest" >> $GITHUB_OUTPUT
echo "file=./docker/build_and_push_ep.Dockerfile" >> $GITHUB_OUTPUT echo "file=./docker/build_and_push_ep.Dockerfile" >> $GITHUB_OUTPUT
elif [[ "${{ inputs.release_type }}" == "main" ]]; then elif [[ "${{ inputs.release_type }}" == "main" || "${{ inputs.release_type }}" == "nightly-main" ]]; then
# LANGFLOW-MAIN RELEASE # LANGFLOW-MAIN RELEASE
echo "docker_tags=langflowai/langflow${nightly_suffix}:${{ needs.get-version.outputs.version }},langflowai/langflow${nightly_suffix}:latest" >> $GITHUB_OUTPUT echo "docker_tags=langflowai/langflow${nightly_suffix}:${{ needs.get-version.outputs.version }},langflowai/langflow${nightly_suffix}:latest" >> $GITHUB_OUTPUT
echo "ghcr_tags=ghcr.io/langflow-ai/langflow${nightly_suffix}:${{ needs.get-version.outputs.version }},ghcr.io/langflow-ai/langflow${nightly_suffix}:latest" >> $GITHUB_OUTPUT echo "ghcr_tags=ghcr.io/langflow-ai/langflow${nightly_suffix}:${{ needs.get-version.outputs.version }},ghcr.io/langflow-ai/langflow${nightly_suffix}:latest" >> $GITHUB_OUTPUT
@ -215,14 +182,13 @@ jobs:
- name: Check out the code at a specific ref - name: Check out the code at a specific ref
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
ref: ${{ inputs.nightly_tag_main || github.ref }} ref: ${{ inputs.ref || inputs.main_version || github.ref }}
#ref: ${{ needs.get-version.outputs.version && format('v{0}', needs.get-version.outputs.version) || github.ref }}
persist-credentials: true persist-credentials: true
- name: "Setup Environment" - name: "Setup Environment"
uses: ./.github/actions/setup-uv uses: ./.github/actions/setup-uv
- name: Install the project - name: Install the project
run: | run: |
if [[ "${{ inputs.release_type }}" == "base" ]]; then if [[ "${{ inputs.release_type }}" == "base" || "${{ inputs.release_type }}" == "nightly-base" ]]; then
uv sync --directory src/backend/base --no-dev --no-sources uv sync --directory src/backend/base --no-dev --no-sources
else else
uv sync --no-dev --no-sources uv sync --no-dev --no-sources
@ -274,25 +240,25 @@ jobs:
include: include:
- component: docker-backend - component: docker-backend
dockerfile: ./docker/build_and_push_backend.Dockerfile dockerfile: ./docker/build_and_push_backend.Dockerfile
tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-backend{0}:{1}', needs.get-version.outputs.nightly-build && '-nightly' || '', needs.get-version.outputs.version) || format('langflowai/langflow-backend{0}:{1},langflowai/langflow-backend{0}:latest', needs.get-version.outputs.nightly-build && '-nightly' || '', needs.get-version.outputs.version) }} tags: langflowai/langflow-backend:${{ needs.get-version.outputs.version }},langflowai/langflow-backend:latest
langflow_image: langflowai/langflow${{ needs.get-version.outputs.nightly-build && '-nightly' || '' }}:${{ needs.get-version.outputs.version }} langflow_image: langflowai/langflow:${{ needs.get-version.outputs.version }}
- component: docker-frontend - component: docker-frontend
dockerfile: ./docker/frontend/build_and_push_frontend.Dockerfile dockerfile: ./docker/frontend/build_and_push_frontend.Dockerfile
tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-frontend{0}:{1}', needs.get-version.outputs.nightly-build && '-nightly' || '', needs.get-version.outputs.version) || format('langflowai/langflow-frontend{0}:{1},langflowai/langflow-frontend{0}:latest', needs.get-version.outputs.nightly-build && '-nightly' || '', needs.get-version.outputs.version) }} tags: langflowai/langflow-frontend:${{ needs.get-version.outputs.version }},langflowai/langflow-frontend:latest
langflow_image: langflowai/langflow${{ needs.get-version.outputs.nightly-build && '-nightly' || '' }}:${{ needs.get-version.outputs.version }} langflow_image: langflowai/langflow:${{ needs.get-version.outputs.version }}
- component: ghcr-backend - component: ghcr-backend
dockerfile: ./docker/build_and_push_backend.Dockerfile dockerfile: ./docker/build_and_push_backend.Dockerfile
tags: ${{ format('ghcr.io/langflow-ai/langflow-backend{0}:{1},ghcr.io/langflow-ai/langflow-backend{0}:latest', needs.get-version.outputs.nightly-build && '-nightly' || '', needs.get-version.outputs.version) }} tags: ghcr.io/langflow-ai/langflow-backend:${{ needs.get-version.outputs.version }},ghcr.io/langflow-ai/langflow-backend:latest
langflow_image: ghcr.io/langflow-ai/langflow${{ needs.get-version.outputs.nightly-build && '-nightly' || '' }}:${{ needs.get-version.outputs.version }} langflow_image: ghcr.io/langflow-ai/langflow:${{ needs.get-version.outputs.version }}
- component: ghcr-frontend - component: ghcr-frontend
dockerfile: ./docker/frontend/build_and_push_frontend.Dockerfile dockerfile: ./docker/frontend/build_and_push_frontend.Dockerfile
tags: ${{ format('ghcr.io/langflow-ai/langflow-frontend{0}:{1},ghcr.io/langflow-ai/langflow-frontend{0}:latest', needs.get-version.outputs.nightly-build && '-nightly' || '', needs.get-version.outputs.version) }} tags: ghcr.io/langflow-ai/langflow-frontend:${{ needs.get-version.outputs.version }},ghcr.io/langflow-ai/langflow-frontend:latest
langflow_image: ghcr.io/langflow-ai/langflow${{ needs.get-version.outputs.nightly-build && '-nightly' || '' }}:${{ needs.get-version.outputs.version }} langflow_image: ghcr.io/langflow-ai/langflow:${{ needs.get-version.outputs.version }}
steps: steps:
- name: Check out the code at a specific ref - name: Check out the code at a specific ref
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
ref: ${{ inputs.nightly_tag_main || github.ref }} ref: ${{ inputs.ref || inputs.main_version || github.ref }}
- name: Set up Docker Buildx - name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3 uses: docker/setup-buildx-action@v3
@ -329,7 +295,7 @@ jobs:
restart-space: restart-space:
name: Restart HuggingFace Spaces name: Restart HuggingFace Spaces
if: ${{ inputs.release_type == 'main' && needs.get-version.outputs.nightly-build == 'false' }} if: ${{ inputs.release_type == 'main' }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [build, get-version] needs: [build, get-version]
strategy: strategy:
@ -340,10 +306,13 @@ jobs:
- name: Check out the code at a specific ref - name: Check out the code at a specific ref
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
ref: ${{ inputs.nightly_tag_main || github.ref }} ref: ${{ inputs.ref || inputs.main_version || github.ref }}
- name: "Setup Environment" - name: "Setup Environment"
uses: ./.github/actions/setup-uv uses: ./.github/actions/setup-uv
- name: Restart HuggingFace Spaces Build - name: Restart HuggingFace Spaces Build
run: | run: |
uv run ./scripts/factory_restart_space.py --space "Langflow/Langflow" --token ${{ secrets.HUGGINGFACE_API_TOKEN }} uv run ./scripts/factory_restart_space.py --space "Langflow/Langflow" --token ${{ secrets.HUGGINGFACE_API_TOKEN }}

12
.github/workflows/release.yml vendored
View file

@ -189,10 +189,11 @@ jobs:
call_docker_build_base: call_docker_build_base:
name: Call Docker Build Workflow for Langflow Base name: Call Docker Build Workflow for Langflow Base
if: inputs.build_docker_base == true if: inputs.build_docker_base == true
needs: release-base needs: [release-base, release-main]
uses: ./.github/workflows/docker-build.yml uses: ./.github/workflows/docker-build.yml
with: with:
version: ${{ needs.release-base.outputs.version }} base_version: ${{ needs.release-base.outputs.version }}
main_version: ${{ needs.release-main.outputs.version }}
release_type: base release_type: base
pre_release: ${{ inputs.pre_release }} pre_release: ${{ inputs.pre_release }}
secrets: inherit secrets: inherit
@ -200,10 +201,10 @@ jobs:
call_docker_build_main: call_docker_build_main:
name: Call Docker Build Workflow for Langflow name: Call Docker Build Workflow for Langflow
if: inputs.build_docker_main == true if: inputs.build_docker_main == true
needs: release-main needs: [release-main]
uses: ./.github/workflows/docker-build.yml uses: ./.github/workflows/docker-build.yml
with: with:
version: ${{ needs.release-main.outputs.version }} main_version: ${{ needs.release-main.outputs.version }}
release_type: main release_type: main
pre_release: ${{ inputs.pre_release }} pre_release: ${{ inputs.pre_release }}
secrets: inherit secrets: inherit
@ -211,9 +212,10 @@ jobs:
call_docker_build_main_ep: call_docker_build_main_ep:
name: Call Docker Build Workflow for Langflow with Entrypoint name: Call Docker Build Workflow for Langflow with Entrypoint
if: inputs.build_docker_ep == true if: inputs.build_docker_ep == true
needs: [release-main]
uses: ./.github/workflows/docker-build.yml uses: ./.github/workflows/docker-build.yml
with: with:
version: needs.release-main.outputs.version main_version: ${{ needs.release-main.outputs.version }}
release_type: main-ep release_type: main-ep
pre_release: False pre_release: False
secrets: inherit secrets: inherit

29
.github/workflows/release_nightly.yml vendored
View file

@ -207,9 +207,9 @@ jobs:
needs: [release-nightly-base, release-nightly-main] needs: [release-nightly-base, release-nightly-main]
uses: ./.github/workflows/docker-build.yml uses: ./.github/workflows/docker-build.yml
with: with:
release_type: base release_type: nightly-base
nightly_tag_base: ${{ inputs.nightly_tag_base }} base_version: ${{ inputs.nightly_tag_base }}
nightly_tag_main: ${{ inputs.nightly_tag_main }} main_version: ${{ inputs.nightly_tag_main }}
secrets: inherit secrets: inherit
call_docker_build_main: call_docker_build_main:
@ -218,17 +218,16 @@ jobs:
needs: [release-nightly-main] needs: [release-nightly-main]
uses: ./.github/workflows/docker-build.yml uses: ./.github/workflows/docker-build.yml
with: with:
release_type: main release_type: nightly-main
nightly_tag_main: ${{ inputs.nightly_tag_main }} main_version: ${{ inputs.nightly_tag_main }}
secrets: inherit secrets: inherit
# Not currently supported, let's add it later call_docker_build_main_ep:
# call_docker_build_main_ep: name: Call Docker Build Workflow for Langflow with Entrypoint
# name: Call Docker Build Workflow for Langflow with Entrypoint if: always() && ${{ inputs.build_docker_ep == 'true' }}
# if: always() && ${{ inputs.build_docker_ep == 'true' }} needs: [release-nightly-main]
# needs: [release-nightly-main] uses: ./.github/workflows/docker-build.yml
# uses: ./.github/workflows/docker-build.yml with:
# with: release_type: main-ep
# release_type: main-ep main_version: ${{ inputs.nightly_tag_main }}
# nightly_tag_main: ${{ inputs.nightly_tag_main }} secrets: inherit
# secrets: inherit