From ba59a9f449df383940b26690f10da9834bc4170f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 5 Jun 2024 13:46:14 +0200 Subject: [PATCH] docker: improve image layout and backend-only/frontend-only images (#2071) * docker: improve image layout and backend-only image * add tests * add tests * add frontend * add frontend * label * fix --- .github/workflows/docker-build.yml | 22 +++++ .github/workflows/docker_test.yml | 61 ++++++++++++++ .github/workflows/pre-release-langflow.yml | 22 +++++ .github/workflows/release.yml | 22 +++++ docker/build_and_push.Dockerfile | 81 ++++++++----------- docker/build_and_push_backend.Dockerfile | 8 ++ .../build_and_push_frontend.Dockerfile | 27 +++++++ docker/frontend/nginx.conf | 22 +++++ docker/frontend/start-nginx.sh | 16 ++++ 9 files changed, 233 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/docker_test.yml create mode 100644 docker/build_and_push_backend.Dockerfile create mode 100644 docker/frontend/build_and_push_frontend.Dockerfile create mode 100644 docker/frontend/nginx.conf create mode 100644 docker/frontend/start-nginx.sh diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 8cb6d0a8d..cdab4c526 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -54,6 +54,28 @@ jobs: tags: ${{ env.TAGS }} - name: Wait for Docker Hub to propagate run: sleep 120 + - name: Build and push (backend) + if: ${{ inputs.release_type == 'main' }} + uses: docker/build-push-action@v5 + with: + context: . + push: true + file: ./docker/build_and_push_backend.Dockerfile + build-args: | + LANGFLOW_IMAGE=langflowai/langflow:${{ inputs.version }} + tags: | + langflowai/langflow-backend:${{ inputs.version }} + langflowai/langflow-backend:1.0-alpha + - name: Build and push (frontend) + if: ${{ inputs.release_type == 'main' }} + uses: docker/build-push-action@v5 + with: + context: . + push: true + file: ./docker/frontend/build_and_push_frontend.Dockerfile + tags: | + langflowai/langflow-frontend:${{ inputs.version }} + langflowai/langflow-frontend:1.0-alpha restart-space: runs-on: ubuntu-latest diff --git a/.github/workflows/docker_test.yml b/.github/workflows/docker_test.yml new file mode 100644 index 000000000..f46010358 --- /dev/null +++ b/.github/workflows/docker_test.yml @@ -0,0 +1,61 @@ +name: Test Docker images + +on: + push: + branches: [main] + paths: + - "docker/**" + - "poetry.lock" + - "pyproject.toml" + - "src/backend/**" + pull_request: + branches: [dev] + paths: + - "docker/**" + - "poetry.lock" + - "pyproject.toml" + - "src/**" + +env: + POETRY_VERSION: "1.8.2" + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build image + run: | + docker build -t langflowai/langflow:latest-dev \ + -f docker/build_and_push.Dockerfile \ + . + - name: Test image + run: | + expected_version=$(cat pyproject.toml | grep version | head -n 1 | cut -d '"' -f 2) + version=$(docker run --rm --entrypoint bash langflowai/langflow:latest-dev -c 'python -c "from langflow.version import __version__ as langflow_version; print(langflow_version)"') + if [ "$expected_version" != "$version" ]; then + echo "Expected version: $expected_version" + echo "Actual version: $version" + exit 1 + fi + + - name: Build backend image + run: | + docker build -t langflowai/langflow-backend:latest-dev \ + --build-arg LANGFLOW_IMAGE=langflowai/langflow:latest-dev \ + -f docker/build_and_push_backend.Dockerfile \ + . + - name: Test backend image + run: | + expected_version=$(cat pyproject.toml | grep version | head -n 1 | cut -d '"' -f 2) + version=$(docker run --rm --entrypoint bash langflowai/langflow-backend:latest-dev -c 'python -c "from langflow.version import __version__ as langflow_version; print(langflow_version)"') + if [ "$expected_version" != "$version" ]; then + echo "Expected version: $expected_version" + echo "Actual version: $version" + exit 1 + fi + - name: Build frontend image + run: | + docker build -t langflowai/langflow-frontend:latest-dev \ + -f docker/frontend/build_and_push_frontend.Dockerfile \ + . diff --git a/.github/workflows/pre-release-langflow.yml b/.github/workflows/pre-release-langflow.yml index 82cb580f3..15726385e 100644 --- a/.github/workflows/pre-release-langflow.yml +++ b/.github/workflows/pre-release-langflow.yml @@ -82,6 +82,28 @@ jobs: tags: | langflowai/langflow:${{ needs.release.outputs.version }} langflowai/langflow:1.0-alpha + - name: Build and push (frontend) + uses: docker/build-push-action@v5 + with: + context: . + push: true + file: ./docker/frontend/build_and_push_frontend.Dockerfile + tags: | + langflowai/langflow-frontend:${{ needs.release.outputs.version }} + langflowai/langflow-frontend:1.0-alpha + - name: Wait for Docker Hub to propagate + run: sleep 120 + - name: Build and push (backend) + uses: docker/build-push-action@v5 + with: + context: . + push: true + file: ./docker/build_and_push_backend.Dockerfile + build-args: | + LANGFLOW_IMAGE=langflowai/langflow:${{ needs.release.outputs.version }} + tags: | + langflowai/langflow-backend:${{ needs.release.outputs.version }} + langflowai/langflow-backend:1.0-alpha create_release: name: Create Release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 06df72e9f..3cd8fc2f0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,6 +54,28 @@ jobs: tags: | langflowai/langflow:${{ steps.check-version.outputs.version }} langflowai/langflow:latest + - name: Wait for Docker Hub to propagate + run: sleep 120 + - name: Build and push (backend) + uses: docker/build-push-action@v5 + with: + context: . + push: true + file: ./docker/build_and_push_backend.Dockerfile + build-args: | + LANGFLOW_IMAGE=langflowai/langflow:${{ steps.check-version.outputs.version }} + tags: | + langflowai/langflow-backend:${{ steps.check-version.outputs.version }} + langflowai/langflow-backend:latest + - name: Build and push (frontend) + uses: docker/build-push-action@v5 + with: + context: . + push: true + file: ./docker/frontend/build_and_push_frontend.Dockerfile + tags: | + langflowai/langflow-frontend:${{ steps.check-version.outputs.version }} + langflowai/langflow-frontend:latest - name: Create Release uses: ncipollo/release-action@v1 with: diff --git a/docker/build_and_push.Dockerfile b/docker/build_and_push.Dockerfile index 3a34db188..cabc1a753 100644 --- a/docker/build_and_push.Dockerfile +++ b/docker/build_and_push.Dockerfile @@ -1,21 +1,13 @@ - - # syntax=docker/dockerfile:1 # Keep this syntax directive! It's used to enable Docker BuildKit -# Based on https://github.com/python-poetry/poetry/discussions/1879?sort=top#discussioncomment-216865 -# but I try to keep it updated (see history) - ################################ -# PYTHON-BASE -# Sets up all our shared environment variables +# BUILDER-BASE +# Used to build deps + create our virtual environment ################################ -FROM python:3.12-slim as python-base +FROM python:3.12-slim as builder-base -# python -ENV PYTHONUNBUFFERED=1 \ - # prevents python creating .pyc files - PYTHONDONTWRITEBYTECODE=1 \ +ENV PYTHONDONTWRITEBYTECODE=1 \ \ # pip PIP_DISABLE_PIP_VERSION_CHECK=on \ @@ -37,56 +29,49 @@ ENV PYTHONUNBUFFERED=1 \ PYSETUP_PATH="/opt/pysetup" \ VENV_PATH="/opt/pysetup/.venv" - -# prepend poetry and venv to path -ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH" - - -################################ -# BUILDER-BASE -# Used to build deps + create our virtual environment -################################ -FROM python-base as builder-base - RUN apt-get update \ && apt-get install --no-install-recommends -y \ # deps for installing poetry curl \ # deps for building python deps - build-essential \ - # npm - npm \ + build-essential npm \ # gcc gcc \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* - - -# Now we need to copy the entire project into the image -WORKDIR /app -COPY pyproject.toml poetry.lock ./ -COPY src ./src -COPY scripts ./scripts -COPY Makefile ./ -COPY README.md ./ RUN --mount=type=cache,target=/root/.cache \ curl -sSL https://install.python-poetry.org | python3 - -RUN useradd -m -u 1000 user && \ - mkdir -p /app/langflow && \ - chown -R user:user /app && \ - chmod -R u+w /app/langflow -# Update PATH with home/user/.local/bin -ENV PATH="/home/user/.local/bin:${PATH}" -RUN python -m pip install requests && cd ./scripts && python update_dependencies.py -RUN $POETRY_HOME/bin/poetry lock -RUN $POETRY_HOME/bin/poetry build +WORKDIR /app +COPY pyproject.toml poetry.lock README.md ./ +COPY src/ ./src +COPY scripts/ ./scripts + +RUN python -m pip install requests --user && cd ./scripts && python update_dependencies.py +RUN $POETRY_HOME/bin/poetry lock --no-update \ + && $POETRY_HOME/bin/poetry install --no-interaction --no-ansi -E deploy \ + && $POETRY_HOME/bin/poetry build -f wheel \ + && $POETRY_HOME/bin/poetry run pip install dist/*.whl + +################################ +# RUNTIME +# Setup user, utilities and copy the virtual environment only +################################ +FROM python:3.12-slim as runtime + +LABEL org.opencontainers.image.title=langflow +LABEL org.opencontainers.image.authors=['Langflow'] +LABEL org.opencontainers.image.licenses=MIT +LABEL org.opencontainers.image.url=https://github.com/langflow-ai/langflow +LABEL org.opencontainers.image.source=https://github.com/langflow-ai/langflow + +RUN useradd user -u 1000 -g 0 --no-create-home --home-dir /app/data +COPY --from=builder-base --chown=1000 /app/.venv /app/.venv +ENV PATH="/app/.venv/bin:${PATH}" -# Copy virtual environment and built .tar.gz from builder base USER user -# Install the package from the .tar.gz -RUN python -m pip install /app/dist/*.tar.gz --user +WORKDIR /app ENTRYPOINT ["python", "-m", "langflow", "run"] -CMD ["--host", "0.0.0.0", "--port", "7860"] +CMD ["--host", "0.0.0.0", "--port", "7860"] \ No newline at end of file diff --git a/docker/build_and_push_backend.Dockerfile b/docker/build_and_push_backend.Dockerfile new file mode 100644 index 000000000..8b82da524 --- /dev/null +++ b/docker/build_and_push_backend.Dockerfile @@ -0,0 +1,8 @@ +# syntax=docker/dockerfile:1 +# Keep this syntax directive! It's used to enable Docker BuildKit + +ARG LANGFLOW_IMAGE +FROM $LANGFLOW_IMAGE + +RUN rm -rf /app/.venv/langflow/frontend +CMD ["--host", "0.0.0.0", "--port", "7860", "--backend-only"] diff --git a/docker/frontend/build_and_push_frontend.Dockerfile b/docker/frontend/build_and_push_frontend.Dockerfile new file mode 100644 index 000000000..e954a801e --- /dev/null +++ b/docker/frontend/build_and_push_frontend.Dockerfile @@ -0,0 +1,27 @@ +# syntax=docker/dockerfile:1 +# Keep this syntax directive! It's used to enable Docker BuildKit + +################################ +# BUILDER-BASE +################################ +FROM node:lts-bookworm-slim as builder-base +COPY src/frontend /frontend + +RUN cd /frontend && npm install && npm run build + +################################ +# RUNTIME +################################ +FROM nginxinc/nginx-unprivileged:stable-bookworm-perl as runtime + +LABEL org.opencontainers.image.title=langflow-frontend +LABEL org.opencontainers.image.authors=['Langflow'] +LABEL org.opencontainers.image.licenses=MIT +LABEL org.opencontainers.image.url=https://github.com/langflow-ai/langflow +LABEL org.opencontainers.image.source=https://github.com/langflow-ai/langflow + +COPY --from=builder-base --chown=nginx /frontend/build /usr/share/nginx/html +COPY --chown=nginx ./docker/frontend/nginx.conf /etc/nginx/conf.d/default.conf +COPY --chown=nginx ./docker/frontend/start-nginx.sh /start-nginx.sh +RUN chmod +x /start-nginx.sh +ENTRYPOINT ["/start-nginx.sh"] \ No newline at end of file diff --git a/docker/frontend/nginx.conf b/docker/frontend/nginx.conf new file mode 100644 index 000000000..d5ecfce43 --- /dev/null +++ b/docker/frontend/nginx.conf @@ -0,0 +1,22 @@ +server { + gzip on; + gzip_comp_level 2; + gzip_min_length 1000; + gzip_types text/xml text/css; + gzip_http_version 1.1; + gzip_vary on; + gzip_disable "MSIE [4-6] \."; + + listen 80; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + try_files $uri $uri/ /index.html =404; + } + location /api { + proxy_pass __BACKEND_URL__; + } + + include /etc/nginx/extra-conf.d/*.conf; +} diff --git a/docker/frontend/start-nginx.sh b/docker/frontend/start-nginx.sh new file mode 100644 index 000000000..3607adf7d --- /dev/null +++ b/docker/frontend/start-nginx.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -e +trap 'kill -TERM $PID' TERM INT +if [ -z "$BACKEND_URL" ]; then + BACKEND_URL="$1" +fi +if [ -z "$BACKEND_URL" ]; then + echo "BACKEND_URL must be set as an environment variable or as first parameter. (e.g. http://localhost:7860)" + exit 1 +fi +sed -i "s|__BACKEND_URL__|$BACKEND_URL|g" /etc/nginx/conf.d/default.conf +cat /etc/nginx/conf.d/default.conf + + +# Start nginx +exec nginx -g 'daemon off;'