ci: refactor release workflow and Docker build process (#3245)

* feat: update docker-build.yml to conditionally retrieve version and adjust tagging logic for Docker images in workflows

* Refactor release workflow to separate base and main package handling

- Split `release_package` input into `release_package_base` and `release_package_main`
- Add new inputs for building Docker images: `build_docker_base` and `build_docker_main`
- Update conditional checks and job dependencies to reflect new inputs
- Separate Docker build workflows for base and main packages

* Refactor release.yml to introduce separate inputs for base and main packages, enhancing workflow flexibility and clarity

* chore: update release.yml to set default pre-release option to false, reflecting new workflow strategy

* chore: add pre-release check to release.yml to validate version format before proceeding with the workflow

* chore: remove deprecated pre-release workflows, consolidating configuration for cleaner CI/CD process

* chore: modify pre-release check in release.yml to use poetry version for validation, enhancing version format accuracy

* chore: refine pre-release version check in release.yml for improved regex validation, ensuring accurate version detection

.github/workflows/docker-build.yml

@@ -3,7 +3,7 @@ on:
   workflow_call:
     inputs:
       version:
-        required: true
+        required: false
         type: string
       release_type:
         required: true
@@ -33,8 +33,33 @@ env:
   TEST_TAG: "langflowai/langflow:test"
 
 jobs:
+  get-version:
+    name: Get Version
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.get-version-input.outputs.version || steps.get-version-base.outputs.version || steps.get-version-main.outputs.version }}
+    steps:
+      - name: Get Version from Input
+        if : ${{ inputs.version != '' }}
+        id: get-version-input
+        run: |
+          version=${{ inputs.version }}
+          echo version=$version >> $GITHUB_OUTPUT
+      - name: Get Version Main
+        if : ${{ inputs.version == '' && inputs.release_type == 'base' }}
+        id: get-version-base
+        run: |
+          version=$(poetry version --short)
+          echo version=$version >> $GITHUB_OUTPUT
+      - name: Get Version Base
+        if : ${{ inputs.version == '' && inputs.release_type == 'main' }}
+        id: get-version-main
+        run: |
+          version=$(cd src/backend/base && poetry version --short)
+          echo version=$version >> $GITHUB_OUTPUT
   setup:
     runs-on: ubuntu-latest
+    needs: get-version
     outputs:
       tags: ${{ steps.set-vars.outputs.tags }}
       file: ${{ steps.set-vars.outputs.file }}
@@ -44,13 +69,13 @@ jobs:
         id: set-vars
         run: |
           if [[ "${{ inputs.release_type }}" == "base" ]]; then
-            echo "tags=langflowai/langflow:base-${{ inputs.version }},langflowai/langflow:base-latest" >> $GITHUB_OUTPUT
+            echo "tags=langflowai/langflow:base-${{ needs.get-version.outputs.version }},langflowai/langflow:base-latest" >> $GITHUB_OUTPUT
             echo "file=./docker/build_and_push_base.Dockerfile" >> $GITHUB_OUTPUT
           else
             if [[ "${{ inputs.pre_release }}" == "true" ]]; then
-              echo "tags=langflowai/langflow:${{ inputs.version }}" >> $GITHUB_OUTPUT
+              echo "tags=langflowai/langflow:${{ needs.get-version.outputs.version }}" >> $GITHUB_OUTPUT
             else
-              echo "tags=langflowai/langflow:${{ inputs.version }},langflowai/langflow:latest" >> $GITHUB_OUTPUT
+              echo "tags=langflowai/langflow:${{ needs.get-version.outputs.version }},langflowai/langflow:latest" >> $GITHUB_OUTPUT
             fi
             echo "file=./docker/build_and_push.Dockerfile" >> $GITHUB_OUTPUT
           fi
@@ -79,17 +104,17 @@ jobs:
   build_components:
     if: ${{ inputs.release_type == 'main' }}
     runs-on: ubuntu-latest
-    needs: build
+    needs: [build, get-version]
     strategy:
       matrix:
         component: [backend, frontend]
         include:
           - component: backend
             dockerfile: ./docker/build_and_push_backend.Dockerfile
-            tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-backend:{0}', inputs.version) || format('langflowai/langflow-backend:{0},langflowai/langflow-backend:latest', inputs.version) }}
+            tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-backend:{0}', needs.get-version.outputs.version) || format('langflowai/langflow-backend:{0},langflowai/langflow-backend:latest', needs.get-version.outputs.version) }}
           - component: frontend
             dockerfile: ./docker/frontend/build_and_push_frontend.Dockerfile
-            tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-frontend:{0}', inputs.version) || format('langflowai/langflow-frontend:{0},langflowai/langflow-frontend:latest', inputs.version) }}
+            tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-frontend:{0}', needs.get-version.outputs.version) || format('langflowai/langflow-frontend:{0},langflowai/langflow-frontend:latest', needs.get-version.outputs.version) }}
     steps:
       - uses: actions/checkout@v4
       - name: Set up Docker Buildx
@@ -107,7 +132,7 @@ jobs:
           context: .
           push: true
           build-args: |
-            LANGFLOW_IMAGE=langflowai/langflow:${{ inputs.version }}
+            LANGFLOW_IMAGE=langflowai/langflow:${{ needs.get-version.outputs.version }}
           file: ${{ matrix.dockerfile }}
           tags: ${{ matrix.tags }}
           # provenance: false will result in a single manifest for all platforms which makes the image pullable from arm64 machines via the emulation (e.g. Apple Silicon machines)

.github/workflows/pre-release-base.yml

@@ -1,80 +0,0 @@
-name: Langflow Base Pre-release
-run-name: Langflow Base Pre-release by @${{ github.actor }}
-on:
-  workflow_dispatch:
-    inputs:
-      release_package:
-        description: "Release package"
-        required: true
-        type: boolean
-        default: false
-
-env:
-  POETRY_VERSION: "1.8.2"
-
-jobs:
-  release:
-    name: Release Langflow Base
-    if: inputs.release_package == true
-    runs-on: ubuntu-latest
-    outputs:
-      version: ${{ steps.check-version.outputs.version }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install poetry
-        run: pipx install poetry==${{ env.POETRY_VERSION }}
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-          cache: "poetry"
-      - name: Check Version
-        id: check-version
-        # In this step, we should check the version of the package
-        # and see if it is a version that is already released
-        # echo version=$(cd src/backend/base && poetry version --short) >> $GITHUB_OUTPUT
-        # cd src/backend/base && poetry version --short should
-        # be different than the last release version in pypi
-        # which we can get from curl -s "https://pypi.org/pypi/langflow/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1
-        run: |
-          version=$(cd src/backend/base && poetry version --short)
-          last_released_version=$(curl -s "https://pypi.org/pypi/langflow-base/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1)
-          if [ "$version" = "$last_released_version" ]; then
-            echo "Version $version is already released. Skipping release."
-            exit 1
-          else
-            echo version=$version >> $GITHUB_OUTPUT
-          fi
-      - name: Build project for distribution
-        run: make build base=true
-      - name: Publish to PyPI
-        env:
-          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
-        run: |
-          make publish base=true
-  docker_build:
-    name: Build Docker Image
-    runs-on: ubuntu-latest
-    needs: release
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        id: qemu
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          push: true
-          file: ./docker/build_and_push_base.Dockerfile
-          tags: |
-            langflowai/langflow:base-${{ needs.release.outputs.version }}
-          # provenance: false will result in a single manifest for all platforms which makes the image pullable from arm64 machines via the emulation (e.g. Apple Silicon machines)
-          provenance: false

.github/workflows/pre-release-langflow.yml

@@ -1,133 +0,0 @@
-name: Langflow Pre-release
-run-name: Langflow Pre-release by @${{ github.actor }}
-on:
-  workflow_dispatch:
-    inputs:
-      release_package:
-        description: "Release package"
-        required: true
-        type: boolean
-        default: false
-  workflow_run:
-    workflows: ["pre-release-base"]
-    types: [completed]
-    branches: [dev]
-
-env:
-  POETRY_VERSION: "1.8.2"
-
-jobs:
-  release:
-    name: Release Langflow
-    if: inputs.release_package == true
-    runs-on: ubuntu-latest
-    outputs:
-      version: ${{ steps.check-version.outputs.version }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install poetry
-        run: pipx install poetry==${{ env.POETRY_VERSION }}
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-          cache: "poetry"
-      - name: Check Version
-        id: check-version
-        run: |
-          version=$(poetry version --short)
-          last_released_version=$(curl -s "https://pypi.org/pypi/langflow/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1)
-          if [ "$version" = "$last_released_version" ]; then
-            echo "Version $version is already released. Skipping release."
-            exit 1
-          else
-            echo version=$version >> $GITHUB_OUTPUT
-          fi
-      - name: Build project for distribution
-        run: make build main=true
-      - name: Display pyproject.toml langflow-base Version
-        run: cat pyproject.toml | grep langflow-base
-      - name: Publish to PyPI
-        env:
-          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
-        run: |
-          make publish main=true
-      - name: Upload Artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: dist
-          path: dist
-
-  docker_build:
-    name: Build Docker Image
-    runs-on: ubuntu-latest
-    needs: release
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        id: qemu
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          push: true
-          file: ./docker/build_and_push.Dockerfile
-          # provenance: false will result in a single manifest for all platforms which makes the image pullable from arm64 machines via the emulation (e.g. Apple Silicon machines)
-          provenance: false
-          tags: |
-            langflowai/langflow:${{ needs.release.outputs.version }}
-            langflowai/langflow:1.0-alpha
-      - name: Build and push (frontend)
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          push: true
-          file: ./docker/frontend/build_and_push_frontend.Dockerfile
-          # provenance: false will result in a single manifest for all platforms which makes the image pullable from arm64 machines via the emulation (e.g. Apple Silicon machines)
-          provenance: false
-          tags: |
-            langflowai/langflow-frontend:${{ needs.release.outputs.version }}
-            langflowai/langflow-frontend:1.0-alpha
-      - name: Wait for Docker Hub to propagate
-        run: sleep 120
-      - name: Build and push (backend)
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          push: true
-          file: ./docker/build_and_push_backend.Dockerfile
-          # provenance: false will result in a single manifest for all platforms which makes the image pullable from arm64 machines via the emulation (e.g. Apple Silicon machines)
-          provenance: false
-          build-args: |
-            LANGFLOW_IMAGE=langflowai/langflow:${{ needs.release.outputs.version }}
-          tags: |
-            langflowai/langflow-backend:${{ needs.release.outputs.version }}
-            langflowai/langflow-backend:1.0-alpha
-
-  create_release:
-    name: Create Release
-    runs-on: ubuntu-latest
-    needs: [docker_build, release]
-    steps:
-      - uses: actions/download-artifact@v4
-        with:
-          name: dist
-          path: dist
-      - name: Create Release
-        uses: ncipollo/release-action@v1
-        with:
-          artifacts: "dist/*"
-          token: ${{ secrets.GITHUB_TOKEN }}
-          draft: false
-          generateReleaseNotes: true
-          prerelease: true
-          tag: v${{ needs.release.outputs.version }}
-          commit: dev

.github/workflows/pre-release.yml

@@ -1,129 +0,0 @@
-name: Langflow Pre-release (Unified)
-run-name: Langflow (${{inputs.release_type}}) Pre-release by @${{ github.actor }}
-on:
-  workflow_dispatch:
-    inputs:
-      release_package:
-        description: "Release package"
-        required: true
-        type: boolean
-        default: false
-      release_type:
-        description: "Type of release (base or main)"
-        required: true
-        type: choice
-        options:
-          - base
-          - main
-
-env:
-  POETRY_VERSION: "1.8.2"
-
-jobs:
-  release:
-    name: Release Langflow
-    if: inputs.release_package == true
-    runs-on: ubuntu-latest
-    outputs:
-      version: ${{ steps.check-version.outputs.version }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install poetry
-        run: pipx install poetry==${{ env.POETRY_VERSION }}
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-          cache: "poetry"
-      - name: Set up Nodejs 20
-        uses: actions/setup-node@v4
-        with:
-          node-version: "20"
-      - name: Check Version
-        id: check-version
-        run: |
-          if [ "${{ inputs.release_type }}" == "base" ]; then
-            version=$(cd src/backend/base && poetry version --short)
-            last_released_version=$(curl -s "https://pypi.org/pypi/langflow-base/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1)
-          else
-            version=$(poetry version --short)
-            last_released_version=$(curl -s "https://pypi.org/pypi/langflow/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1)
-          fi
-          if [ "$version" = "$last_released_version" ]; then
-            echo "Version $version is already released. Skipping release."
-            exit 1
-          else
-            echo version=$version >> $GITHUB_OUTPUT
-          fi
-      - name: Build project for distribution
-        run: |
-          if [ "${{ inputs.release_type }}" == "base" ]; then
-            make build base=true
-          else
-            make build main=true
-          fi
-      - name: Test CLI
-        run: |
-          if [ "${{ inputs.release_type }}" == "base" ]; then
-            python -m pip install src/backend/base/dist/*.whl
-          else
-            python -m pip install dist/*.whl
-          fi
-          python -m langflow run --host 127.0.0.1 --port 7860 &
-          SERVER_PID=$!
-          # Wait for the server to start
-          timeout 120 bash -c 'until curl -f http://127.0.0.1:7860/health; do sleep 2; done' || (echo "Server did not start in time" && kill $SERVER_PID && exit 1)
-          # Terminate the server
-          kill $SERVER_PID || (echo "Failed to terminate the server" && exit 1)
-          sleep 10 # give the server some time to terminate
-          # Check if the server is still running
-          if kill -0 $SERVER_PID 2>/dev/null; then
-            echo "Failed to terminate the server"
-            exit 1
-          else
-            echo "Server terminated successfully"
-          fi
-      - name: Publish to PyPI
-        env:
-          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
-        run: |
-          if [ "${{ inputs.release_type }}" == "base" ]; then
-            make publish base=true
-          else
-            make publish main=true
-          fi
-      - name: Upload Artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: dist${{ inputs.release_type }}
-          path: ${{ inputs.release_type == 'base' && 'src/backend/base/dist' || 'dist' }}
-
-  call_docker_build:
-    name: Call Docker Build Workflow
-    needs: release
-    uses: langflow-ai/langflow/.github/workflows/docker-build.yml@dev
-    with:
-      version: ${{ needs.release.outputs.version }}
-      release_type: ${{ inputs.release_type }}
-    secrets: inherit
-
-  create_release:
-    name: Create Release
-    runs-on: ubuntu-latest
-    needs: [release]
-    if: ${{ inputs.release_type == 'main' }}
-    steps:
-      - uses: actions/download-artifact@v4
-        with:
-          name: dist${{ inputs.release_type }}
-          path: dist
-      - name: Create Release
-        uses: ncipollo/release-action@v1
-        with:
-          artifacts: "dist/*"
-          token: ${{ secrets.GITHUB_TOKEN }}
-          draft: false
-          generateReleaseNotes: true
-          prerelease: true
-          tag: v${{ needs.release.outputs.version }}
-          commit: dev

.github/workflows/release.yml

@@ -4,8 +4,23 @@ run-name: Langflow Release by @${{ github.actor }}
 on:
   workflow_dispatch:
     inputs:
-      release_package:
+      release_package_base:
-        description: "Release package"
+        description: "Release Langflow Base"
+        required: true
+        type: boolean
+        default: false
+      release_package_main:
+        description: "Release Langflow"
+        required: true
+        type: boolean
+        default: false
+      build_docker_base:
+        description: "Build Docker Image for Langflow Base"
+        required: true
+        type: boolean
+        default: false
+      build_docker_main:
+        description: "Build Docker Image for Langflow"
         required: true
         type: boolean
         default: false
@@ -13,7 +28,7 @@ on:
         description: "Pre-release"
         required: false
         type: boolean
-        default: true
+        default: false
 
 
 env:
@@ -27,7 +42,7 @@ jobs:
   release-base:
     name: Release Langflow Base
     needs: [ci]
-    if: inputs.release_package == true
+    if: inputs.release_package_base == true
     runs-on: ubuntu-latest
     outputs:
       version: ${{ steps.check-version.outputs.version }}
@@ -93,7 +108,7 @@ jobs:
 
   release-main:
     name: Release Langflow Main
-    if: inputs.release_package == true
+    if: inputs.release_package_main == true
     needs: [release-base]
     runs-on: ubuntu-latest
     outputs:
@@ -111,6 +126,18 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: "20"
+      # If pre-release is true, we need to check if  ["a", "b", "rc", "dev", "post"] is in the version string
+      # if the version string is incorrect, we need to exit the workflow
+      - name: Check if pre-release
+        if: inputs.pre_release == true
+        run: |
+          version=$(poetry version --short)
+          if [[ "${version}" =~ ^([0-9]+\.)?([0-9]+\.)?[0-9]+((a|b|rc|dev|post)([0-9]+))$ ]]; then
+            echo "Pre-release version detected. Continuing with the release."
+          else
+            echo "Invalid pre-release version detected. Exiting the workflow."
+            exit 1
+          fi
       - name: Check Version
         id: check-version
         run: |
@@ -155,19 +182,34 @@ jobs:
           name: dist-main
           path: dist
 
-  call_docker_build:
+  call_docker_build_base:
-    name: Call Docker Build Workflow
+    name: Call Docker Build Workflow for Langflow Base
-    needs: [release-base, release-main]
+    if : inputs.build_docker_base == true
-    uses: langflow-ai/langflow/.github/workflows/docker-build.yml@main
+    uses: ./.github/workflows/docker-build.yml
     strategy:
       matrix:
         release_type:
           - base
+    with:
+      # version should be needs.release-base.outputs.version  if release_type is base
+      # version should be needs.release-main.outputs.version  if release_type is main
+      version: ''
+      release_type: ${{ matrix.release_type }}
+      pre_release: ${{ inputs.pre_release }}
+    secrets: inherit
+
+  call_docker_build_main:
+    name: Call Docker Build Workflow for Langflow
+    if : inputs.build_docker_main == true
+    uses: ./.github/workflows/docker-build.yml
+    strategy:
+      matrix:
+        release_type:
           - main
     with:
       # version should be needs.release-base.outputs.version  if release_type is base
       # version should be needs.release-main.outputs.version  if release_type is main
-      version: ${{ matrix.release_type == 'base' && needs.release-base.outputs.version || matrix.release_type == 'main' && needs.release-main.outputs.version }}
+      version: ''
       release_type: ${{ matrix.release_type }}
       pre_release: ${{ inputs.pre_release }}
     secrets: inherit