From c1097d2dde7e97517cefe3907effa490ada60d02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Est=C3=A9vez?= Date: Thu, 31 Oct 2024 07:16:26 -0400 Subject: [PATCH] chore: update dockerfiles and docker-build workflows to fix releases (#4317) * bash -x * bash verbose * combine bash calls in uv run * combined python script * relative dir * pass args correctly * pass args correctly * function name * fix arg order * merge base and main * remove nightly -ep docker image for now * set up uv in the get-version job in docker-build.yml * v prefix for version in build job * use inputs.nightly_tag_main for checkout actions * mount root project metadata for base build * fix comment * continued dockerfile fixes for workspaces setup * fix path * ruff check fix --- .github/workflows/docker-build.yml | 18 ++++++--- .github/workflows/nightly_build.yml | 15 ++----- .github/workflows/release_nightly.yml | 23 +++++------ docker/build_and_push_base.Dockerfile | 16 +++++++- scripts/ci/update_pyproject_combined.py | 52 +++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 31 deletions(-) create mode 100755 scripts/ci/update_pyproject_combined.py diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 7f221509a..181b0929f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -103,11 +103,14 @@ jobs: ref: ${{ inputs.nightly_tag_main || github.ref }} persist-credentials: true + - name: "Setup Environment" + uses: ./.github/actions/setup-uv + - name: Get Version from Input if: ${{ inputs.version != '' }} id: get-version-input run: | - # Base version cannot have a version input, since you have to specify a valid main tag to checkout. + # Base version cannot have a version input, since you have to specify a valid main tag to checkout. if [[ "${{ inputs.release_type }}" == "base" && "${{ inputs.version }}" != '' ]]; then echo "Cannot specify version for base release." echo "Build type is 'base' and version is non-empty. Exiting the workflow." @@ -150,6 +153,10 @@ jobs: id: get-version-base run: | version=$(uv tree | grep 'langflow-base' | awk '{print $3}' | sed 's/^v//') + if [ -z "$version" ]; then + echo "Failed to extract version from uv tree output" + exit 1 + fi echo version=$version echo version=$version >> $GITHUB_OUTPUT - name: Get Version Main @@ -196,7 +203,7 @@ jobs: echo "docker_tags=langflowai/langflow${nightly_suffix}:${{ needs.get-version.outputs.version }},langflowai/langflow${nightly_suffix}:latest" >> $GITHUB_OUTPUT echo "ghcr_tags=ghcr.io/langflow-ai/langflow${nightly_suffix}:${{ needs.get-version.outputs.version }},ghcr.io/langflow-ai/langflow${nightly_suffix}:latest" >> $GITHUB_OUTPUT echo "file=./docker/build_and_push.Dockerfile" >> $GITHUB_OUTPUT - else + else echo "Invalid release type. Exiting the workflow." exit 1 fi @@ -208,7 +215,8 @@ jobs: - name: Check out the code at a specific ref uses: actions/checkout@v4 with: - ref: ${{ needs.get-version.outputs.version || github.ref }} + ref: ${{ inputs.nightly_tag_main || github.ref }} + #ref: ${{ needs.get-version.outputs.version && format('v{0}', needs.get-version.outputs.version) || github.ref }} persist-credentials: true - name: "Setup Environment" uses: ./.github/actions/setup-uv @@ -284,7 +292,7 @@ jobs: - name: Check out the code at a specific ref uses: actions/checkout@v4 with: - ref: ${{ needs.get-version.outputs.version || github.ref }} + ref: ${{ inputs.nightly_tag_main || github.ref }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -332,7 +340,7 @@ jobs: - name: Check out the code at a specific ref uses: actions/checkout@v4 with: - ref: ${{ needs.get-version.outputs.version || github.ref }} + ref: ${{ inputs.nightly_tag_main || github.ref }} - name: "Setup Environment" uses: ./.github/actions/setup-uv diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index 6514b9c04..9890ee3e4 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - shell: bash + shell: bash -ex -o pipefail {0} permissions: # Required to create tag contents: write @@ -69,17 +69,10 @@ jobs: git config --global user.email "bot-nightly-builds@langflow.org" git config --global user.name "Langflow Bot" - BASE_TAG="${{ steps.generate_base_tag.outputs.base_tag }}" - echo "Updating base project version to $BASE_TAG" - uv run ./scripts/ci/update_pyproject_name.py langflow-base-nightly base - uv run ./scripts/ci/update_pyproject_version.py $BASE_TAG base - - # Use the main tag created earlier MAIN_TAG="${{ steps.generate_main_tag.outputs.main_tag }}" - echo "Updating main project version to $MAIN_TAG" - uv run ./scripts/ci/update_pyproject_version.py $MAIN_TAG main - uv run ./scripts/ci/update_pyproject_name.py langflow-nightly main - uv run ./scripts/ci/update_uv_dependency.py $BASE_TAG + BASE_TAG="${{ steps.generate_base_tag.outputs.base_tag }}" + echo "Updating base project version to $BASE_TAG and updating main project version to $MAIN_TAG" + uv run ./scripts/ci/update_pyproject_combined.py main $MAIN_TAG $BASE_TAG uv lock cd src/backend/base && uv lock && cd ../../.. diff --git a/.github/workflows/release_nightly.yml b/.github/workflows/release_nightly.yml index 74ecdf744..585354b8e 100644 --- a/.github/workflows/release_nightly.yml +++ b/.github/workflows/release_nightly.yml @@ -222,16 +222,13 @@ jobs: nightly_tag_main: ${{ inputs.nightly_tag_main }} secrets: inherit - call_docker_build_main_ep: - name: Call Docker Build Workflow for Langflow with Entrypoint - if: always() && ${{ inputs.build_docker_ep == 'true' }} - needs: [release-nightly-main] - uses: ./.github/workflows/docker-build.yml - with: - release_type: main-ep - nightly_tag_main: ${{ inputs.nightly_tag_main }} - secrets: inherit - - - - +# Not currently supported, let's add it later +# call_docker_build_main_ep: +# name: Call Docker Build Workflow for Langflow with Entrypoint +# if: always() && ${{ inputs.build_docker_ep == 'true' }} +# needs: [release-nightly-main] +# uses: ./.github/workflows/docker-build.yml +# with: +# release_type: main-ep +# nightly_tag_main: ${{ inputs.nightly_tag_main }} +# secrets: inherit \ No newline at end of file diff --git a/docker/build_and_push_base.Dockerfile b/docker/build_and_push_base.Dockerfile index 7c5cb9f1e..c20f8b6bd 100644 --- a/docker/build_and_push_base.Dockerfile +++ b/docker/build_and_push_base.Dockerfile @@ -33,10 +33,14 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* # Install the project's dependencies using the lockfile and settings +# We need to mount the root uv.lock and pyproject.toml to build the base with uv because we're still using uv workspaces RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,source=src/backend/base/README.md,target=src/backend/base/README.md \ --mount=type=bind,source=src/backend/base/uv.lock,target=src/backend/base/uv.lock \ --mount=type=bind,source=src/backend/base/pyproject.toml,target=src/backend/base/pyproject.toml \ + --mount=type=bind,source=uv.lock,target=uv.lock \ + --mount=type=bind,source=README.md,target=README.md \ + --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ cd src/backend/base && uv sync --frozen --no-install-project --no-dev --no-editable ADD ./src /app/src @@ -48,7 +52,14 @@ RUN npm install \ && cp -r build /app/src/backend/base/langflow/frontend \ && rm -rf /tmp/src/frontend +ADD ./src/backend/base /app/src/backend/base WORKDIR /app/src/backend/base +# again we need these because of workspaces +ADD ./pyproject.toml /app/pyproject.toml +ADD ./uv.lock /app/uv.lock +ADD ./src/backend/base/pyproject.toml /app/src/backend/base/pyproject.toml +ADD ./src/backend/base/uv.lock /app/src/backend/base/uv.lock +ADD ./src/backend/base/README.md /app/src/backend/base/README.md RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --frozen --no-dev --no-editable @@ -59,10 +70,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \ FROM python:3.12.3-slim AS runtime RUN useradd user -u 1000 -g 0 --no-create-home --home-dir /app/data -COPY --from=builder --chown=1000 /app/src/backend/base/.venv /app/src/backend/base/.venv +# and we use the venv at the root because workspaces +COPY --from=builder --chown=1000 /app/.venv /app/.venv # Place executables in the environment at the front of the path -ENV PATH="/app/src/backend/base/.venv/bin:$PATH" +ENV PATH="/app/.venv/bin:$PATH" LABEL org.opencontainers.image.title=langflow LABEL org.opencontainers.image.authors=['Langflow'] diff --git a/scripts/ci/update_pyproject_combined.py b/scripts/ci/update_pyproject_combined.py new file mode 100755 index 000000000..20c6e3ded --- /dev/null +++ b/scripts/ci/update_pyproject_combined.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# scripts/ci/update_pyproject_combined.py +import sys +from pathlib import Path + +from update_pyproject_name import update_pyproject_name +from update_pyproject_name import update_uv_dep as update_name_uv_dep +from update_pyproject_version import update_pyproject_version +from update_uv_dependency import update_uv_dep as update_version_uv_dep + +# Add the current directory to the path so we can import the other scripts +current_dir = Path(__file__).resolve().parent +sys.path.append(str(current_dir)) + + +def main(): + """Universal update script that handles both base and main updates in a single run. + + Usage: + update_pyproject_combined.py main + """ + arg_count = 4 + if len(sys.argv) != arg_count: + print("Usage:") + print(" update_pyproject_combined.py main ") + sys.exit(1) + + mode = sys.argv[1] + if mode != "main": + print("Only 'main' mode is supported") + print("Usage: update_pyproject_combined.py main ") + sys.exit(1) + + main_tag = sys.argv[2] + base_tag = sys.argv[3] + + # First handle base package updates + update_pyproject_name("src/backend/base/pyproject.toml", "langflow-base-nightly") + update_name_uv_dep("pyproject.toml", "langflow-base-nightly") + update_pyproject_version("src/backend/base/pyproject.toml", base_tag) + + # Then handle main package updates + update_pyproject_name("pyproject.toml", "langflow-nightly") + update_name_uv_dep("pyproject.toml", "langflow-nightly") + update_pyproject_version("pyproject.toml", main_tag) + # Update dependency version (strip 'v' prefix if present) + base_version = base_tag.lstrip("v") + update_version_uv_dep(base_version) + + +if __name__ == "__main__": + main()