Merge dev

.github/workflows/docker-build.yml

@@ -0,0 +1,54 @@
+name: Docker Build and Push
+on:
+  workflow_call:
+    inputs:
+      version:
+        required: true
+        type: string
+      release_type:
+        required: true
+        type: string
+  workflow_dispatch:
+    inputs:
+      version:
+        required: true
+        type: string
+      release_type:
+        required: true
+        type: choice
+        options:
+          - base
+          - main
+
+jobs:
+  docker_build:
+    name: Build Docker Image
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          file: |
+            if [ "${{ inputs.release_type }}" == "base" ]; then
+              ./docker/build_and_push_base.Dockerfile
+            else
+              ./docker/build_and_push.Dockerfile
+            fi
+          tags: |
+            if [ "${{ inputs.release_type }}" == "base" ]; then
+              langflowai/langflow:base-${{ inputs.version }}
+            else
+              langflowai/langflow:${{ inputs.version }}
+              langflowai/langflow:1.0-alpha

.github/workflows/pre-release-base.yml

@@ -72,6 +72,6 @@ jobs:
         with:
           context: .
           push: true
-          file: ./build_and_push_base.Dockerfile
+          file: ./docker/build_and_push_base.Dockerfile
           tags: |
             langflowai/langflow:base-${{ needs.release.outputs.version }}

.github/workflows/pre-release-langflow.yml

@@ -78,7 +78,7 @@ jobs:
         with:
           context: .
           push: true
-          file: ./build_and_push.Dockerfile
+          file: ./docker/build_and_push.Dockerfile
           tags: |
             langflowai/langflow:${{ needs.release.outputs.version }}
             langflowai/langflow:1.0-alpha

.github/workflows/pre-release.yml

@@ -0,0 +1,101 @@
+name: Langflow Pre-release (Unified)
+run-name: Langflow (${{inputs.release_type}}) Pre-release by @${{ github.actor }}
+on:
+  workflow_dispatch:
+    inputs:
+      release_package:
+        description: "Release package"
+        required: true
+        type: boolean
+        default: false
+      release_type:
+        description: "Type of release (base or main)"
+        required: true
+        type: choice
+        options:
+          - base
+          - main
+
+env:
+  POETRY_VERSION: "1.8.2"
+
+jobs:
+  release:
+    name: Release Langflow
+    if: inputs.release_package == true
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.check-version.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install poetry
+        run: pipx install poetry==$POETRY_VERSION
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "poetry"
+      - name: Check Version
+        id: check-version
+        run: |
+          if [ "${{ inputs.release_type }}" == "base" ]; then
+            version=$(cd src/backend/base && poetry version --short)
+            last_released_version=$(curl -s "https://pypi.org/pypi/langflow-base/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1)
+          else
+            version=$(poetry version --short)
+            last_released_version=$(curl -s "https://pypi.org/pypi/langflow/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1)
+          fi
+          if [ "$version" = "$last_released_version" ]; then
+            echo "Version $version is already released. Skipping release."
+            exit 1
+          else
+            echo version=$version >> $GITHUB_OUTPUT
+          fi
+      - name: Build project for distribution
+        run: |
+          if [ "${{ inputs.release_type }}" == "base" ]; then
+            make build base=true
+          else
+            make build main=true
+          fi
+      - name: Publish to PyPI
+        env:
+          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          if [ "${{ inputs.release_type }}" == "base" ]; then
+            make publish base=true
+          else
+            make publish main=true
+          fi
+
+  call_docker_build:
+    name: Call Docker Build Workflow
+    runs-on: ubuntu-latest
+    needs: release
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/workflows/docker-build.yml
+        with:
+          version: ${{ needs.release.outputs.version }}
+          release_type: ${{ inputs.release_type }}
+
+  create_release:
+    name: Create Release
+    runs-on: ubuntu-latest
+    needs: [call_docker_build, release]
+    if: ${{ inputs.release_type == 'main' }}
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist
+      - name: Create Release
+        uses: ncipollo/release-action@v1
+        with:
+          artifacts: "dist/*"
+          token: ${{ secrets.GITHUB_TOKEN }}
+          draft: false
+          generateReleaseNotes: true
+          prerelease: true
+          tag: v${{ needs.release.outputs.version }}
+          commit: dev

.github/workflows/release.yml

@@ -50,7 +50,7 @@ jobs:
         with:
           context: .
           push: true
-          file: ./build_and_push.Dockerfile
+          file: ./docker/build_and_push.Dockerfile
           tags: |
             langflowai/langflow:${{ steps.check-version.outputs.version }}
             langflowai/langflow:latest

.github/workflows/typescript_test.yml

@@ -19,8 +19,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        shardIndex: [1]
-        shardTotal: [1]
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -30,7 +30,15 @@ jobs:
         id: setup-node
         with:
           node-version: ${{ env.NODE_VERSION }}
-          cache: "npm"
+
+      - name: Cache Node.js dependencies
+        uses: actions/cache@v4
+        id: npm-cache
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('src/frontend/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
 
       - name: Install Node.js dependencies
         run: |
@@ -80,7 +88,7 @@ jobs:
       - name: Run Playwright Tests
         run: |
           cd src/frontend
-          npx playwright test
+          npx playwright test --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --workers 2
 
       - name: Upload blob report to GitHub Actions Artifacts
         if: always()

.pre-commit-config.yaml

@@ -18,7 +18,10 @@ repos:
     hooks:
       - id: check-case-conflict
       - id: end-of-file-fixer
+        # python, js and ts only
+        files: \.(py|js|ts)$
       - id: mixed-line-ending
+        files: \.(py|js|ts)$
         args:
           - --fix=lf
       - id: trailing-whitespace

.readthedocs.yaml

@@ -1,31 +0,0 @@
-# Read the Docs configuration file for Sphinx projects
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-# Required
-version: 2
-
-# Set the OS, Python version and other tools you might need
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.11"
-    # You can also specify other tool versions:
-    # nodejs: "19"
-    # rust: "1.64"
-    # golang: "1.19"
-
-# Build documentation in the "docs/" directory with Sphinx
-sphinx:
-   configuration: docs/conf.py
-
-# Optionally build your docs in additional formats such as PDF and ePub
-# formats:
-#    - pdf
-#    - epub
-
-# Optional but recommended, declare the Python requirements required
-# to build your documentation
-# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-# python:
-#    install:
-#    - requirements: docs/requirements.txt

README.md

@@ -65,7 +65,6 @@ Each option is detailed below:
 - `--workers`: Sets the number of worker processes. Can be set using the `LANGFLOW_WORKERS` environment variable. The default is `1`.
 - `--timeout`: Sets the worker timeout in seconds. The default is `60`.
 - `--port`: Sets the port to listen on. Can be set using the `LANGFLOW_PORT` environment variable. The default is `7860`.
-- `--config`: Defines the path to the configuration file. The default is `config.yaml`.
 - `--env-file`: Specifies the path to the .env file containing environment variables. The default is `.env`.
 - `--log-level`: Defines the logging level. Can be set using the `LANGFLOW_LOG_LEVEL` environment variable. The default is `critical`.
 - `--components-path`: Specifies the path to the directory containing custom components. Can be set using the `LANGFLOW_COMPONENTS_PATH` environment variable. The default is `langflow/components`.

docker/.dockerignore

@@ -0,0 +1,7 @@
+.venv/
+**/aws
+node_modules
+**/node_modules/
+dist/
+**/build/
+src/backend/langflow/frontend

docker/build_and_push.Dockerfile

@@ -10,7 +10,7 @@
 # PYTHON-BASE
 # Sets up all our shared environment variables
 ################################
-FROM python:3.10-slim as python-base
+FROM python:3.12-slim as python-base
 
 # python
 ENV PYTHONUNBUFFERED=1 \
@@ -47,53 +47,41 @@ ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
 # Used to build deps + create our virtual environment
 ################################
 FROM python-base as builder-base
+
 RUN apt-get update \
     && apt-get install --no-install-recommends -y \
     # deps for installing poetry
     curl \
     # deps for building python deps
-    build-essential
+    build-essential \
+    # npm
+    npm \
+    # gcc
+    gcc \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 
-# install poetry - respects $POETRY_VERSION & $POETRY_HOME
-# The --mount will mount the buildx cache directory to where
-# Poetry and Pip store their cache so that they can reuse it
+
+# Now we need to copy the entire project into the image
+WORKDIR /app
+COPY pyproject.toml poetry.lock ./
+COPY src ./src
+COPY scripts ./scripts
+COPY Makefile ./
+COPY README.md ./
 RUN --mount=type=cache,target=/root/.cache \
     curl -sSL https://install.python-poetry.org | python3 -
+RUN python -m pip install requests && cd ./scripts && python update_dependencies.py
+RUN $POETRY_HOME/bin/poetry lock
+RUN $POETRY_HOME/bin/poetry build
 
-# copy project requirement files here to ensure they will be cached.
-WORKDIR $PYSETUP_PATH
-# Copy just one file to avoid rebuilding the whole image
-COPY poetry.lock pyproject.toml ./
-COPY ./src/backend/langflow ./src/backend/langflow
-COPY ./src/backend/base/pyproject.toml ./src/backend/base/pyproject.toml
-# Copy README.md to the build context
-COPY README.md .
-# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
-RUN --mount=type=cache,target=/root/.cache \
-    poetry install --without dev --extras deploy
-
-
-################################
-# DEVELOPMENT
-# Image used during development / testing
-################################
-FROM python-base as development
-WORKDIR $PYSETUP_PATH
-
-# copy in our built poetry + venv
-COPY --from=builder-base $POETRY_HOME $POETRY_HOME
-COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
-
-# Copy just one file to avoid rebuilding the whole image
-COPY ./src/backend/langflow ./src/backend/langflow
-# quicker install as runtime deps are already installed
-RUN --mount=type=cache,target=/root/.cache \
-    poetry install --with=dev --extras deploy
-
-# copy in our app code
-COPY ./src/backend ./src/backend
-RUN --mount=type=cache,target=/root/.cache \
-    poetry install --with=dev --extras deploy
-COPY ./tests ./tests=
+# Copy virtual environment and built .tar.gz from builder base
+RUN useradd -m -u 1000 user
+RUN chown -R user:user /app
+USER user
+# Install the package from the .tar.gz
+RUN python -m pip install /app/dist/*.tar.gz
 
+ENTRYPOINT ["python", "-m", "langflow", "run"]
+CMD ["--host", "0.0.0.0", "--port", "7860"]

docker/build_and_push_base.Dockerfile

@@ -0,0 +1,92 @@
+
+
+# syntax=docker/dockerfile:1
+# Keep this syntax directive! It's used to enable Docker BuildKit
+
+# Based on https://github.com/python-poetry/poetry/discussions/1879?sort=top#discussioncomment-216865
+# but I try to keep it updated (see history)
+
+################################
+# PYTHON-BASE
+# Sets up all our shared environment variables
+################################
+FROM python:3.12-slim as python-base
+
+# python
+ENV PYTHONUNBUFFERED=1 \
+    # prevents python creating .pyc files
+    PYTHONDONTWRITEBYTECODE=1 \
+    \
+    # pip
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=100 \
+    \
+    # poetry
+    # https://python-poetry.org/docs/configuration/#using-environment-variables
+    POETRY_VERSION=1.8.2 \
+    # make poetry install to this location
+    POETRY_HOME="/opt/poetry" \
+    # make poetry create the virtual environment in the project's root
+    # it gets named `.venv`
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    # do not ask any interactive question
+    POETRY_NO_INTERACTION=1 \
+    \
+    # paths
+    # this is where our requirements + virtual environment will live
+    PYSETUP_PATH="/opt/pysetup" \
+    VENV_PATH="/opt/pysetup/.venv"
+
+
+# prepend poetry and venv to path
+ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
+
+
+################################
+# BUILDER-BASE
+# Used to build deps + create our virtual environment
+################################
+FROM python-base as builder-base
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+    # deps for installing poetry
+    curl \
+    # deps for building python deps
+    build-essential \
+    # npm
+    npm \
+    # gcc
+    gcc \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN --mount=type=cache,target=/root/.cache \
+    curl -sSL https://install.python-poetry.org | python3 -
+
+# Now we need to copy the entire project into the image
+COPY pyproject.toml poetry.lock ./
+COPY src/frontend/package.json /tmp/package.json
+RUN cd /tmp && npm install
+WORKDIR /app
+COPY src/frontend ./src/frontend
+RUN rm -rf src/frontend/node_modules
+RUN cp -a /tmp/node_modules /app/src/frontend
+COPY scripts ./scripts
+COPY Makefile ./
+COPY README.md ./
+RUN cd src/frontend && npm run build
+COPY src/backend ./src/backend
+RUN cp -r src/frontend/build src/backend/base/langflow/frontend
+RUN rm -rf src/backend/base/dist
+RUN cd src/backend/base && $POETRY_HOME/bin/poetry build
+
+# Copy virtual environment and built .tar.gz from builder base
+RUN useradd -m -u 1000 user
+RUN chown -R user:user /app
+USER user
+# Install the package from the .tar.gz
+RUN python -m pip install /app/src/backend/base/dist/*.tar.gz
+
+
+ENTRYPOINT ["python", "-m", "langflow", "run"]
+CMD ["--host", "0.0.0.0", "--port", "7860"]

docker/cdk-docker-compose.yml

@@ -1,33 +1,37 @@
-version: "3.4"
+version: "3"
+networks:
+  langflow:
 
 services:
   backend:
-    volumes:
-      - ./:/app
     build:
       context: ./
       dockerfile: ./dev.Dockerfile
-    command:
-      [
-        "sh",
-        "-c",
-        "pip install debugpy -t /tmp && python /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 -m uvicorn --factory langflow.main:create_app --host 0.0.0.0 --port 7860 --reload --loop asyncio",
-      ]
+    env_file:
+      - .env
     ports:
-      - 7860:7860
-      - 5678:5678
-    restart: on-failure
-
+      - "7860:7860"
+    volumes:
+      - ./:/app
+    command: bash -c "uvicorn --factory langflow.main:create_app --host 0.0.0.0 --port 7860 --reload --loop asyncio"
+    networks:
+      - langflow
   frontend:
     build:
       context: ./src/frontend
-      dockerfile: ./dev.Dockerfile
+      dockerfile: ./cdk.Dockerfile
       args:
         - BACKEND_URL=http://backend:7860
+    depends_on:
+      - backend
+    environment:
+      - VITE_PROXY_TARGET=http://backend:7860
     ports:
-      - "3000:3000"
+      - "8080:3000"
     volumes:
       - ./src/frontend/public:/home/node/app/public
       - ./src/frontend/src:/home/node/app/src
       - ./src/frontend/package.json:/home/node/app/package.json
     restart: on-failure
+    networks:
+      - langflow

docker/cdk.Dockerfile

@@ -0,0 +1,21 @@
+FROM --platform=linux/amd64 python:3.10-slim
+
+WORKDIR /app
+
+# Install Poetry
+RUN apt-get update && apt-get install gcc g++ curl build-essential postgresql-server-dev-all -y
+RUN curl -sSL https://install.python-poetry.org | python3 -
+# # Add Poetry to PATH
+ENV PATH="${PATH}:/root/.local/bin"
+# # Copy the pyproject.toml and poetry.lock files
+COPY poetry.lock pyproject.toml ./
+# Copy the rest of the application codes
+COPY ./ ./
+
+# Install dependencies
+RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi
+
+RUN poetry add botocore
+RUN poetry add pymysql
+
+CMD ["sh", "./container-cmd-cdk.sh"]

docker/container-cmd-cdk.sh

@@ -0,0 +1,5 @@
+export LANGFLOW_DATABASE_URL="mysql+pymysql://${username}:${password}@${host}:3306/${dbname}"
+# echo $LANGFLOW_DATABASE_URL
+uvicorn --factory langflow.main:create_app --host 0.0.0.0 --port 7860 --reload --log-level debug --loop asyncio
+
+# python -m langflow run --host 0.0.0.0 --port 7860

docker/dev.Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+# Install Poetry
+RUN apt-get update && apt-get install gcc g++ curl build-essential postgresql-server-dev-all -y
+RUN curl -sSL https://install.python-poetry.org | python3 -
+# # Add Poetry to PATH
+ENV PATH="${PATH}:/root/.local/bin"
+# # Copy the pyproject.toml and poetry.lock files
+COPY poetry.lock pyproject.toml ./
+# Copy the rest of the application codes
+COPY ./ ./
+
+# Install dependencies
+RUN poetry config virtualenvs.create false && poetry install --no-interaction --no-ansi
+
+CMD ["uvicorn", "--factory", "langflow.main:create_app", "--host", "0.0.0.0", "--port", "7860", "--reload", "--log-level", "debug", "--loop", "asyncio"]

docker/render.Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.10-slim
+
+RUN apt-get update && apt-get install gcc g++ git make -y && apt-get clean \
+	&& rm -rf /var/lib/apt/lists/*
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+COPY --chown=user . $HOME/app
+
+RUN pip install langflow>==0.5.0 -U --user
+CMD ["python", "-m", "langflow", "run", "--host", "0.0.0.0", "--port", "7860"]

docker_example/pre.docker-compose.yml

@@ -10,9 +10,9 @@ services:
     environment:
       - LANGFLOW_DATABASE_URL=postgresql://langflow:langflow@postgres:5432/langflow
       # This variable defines where the logs, file storage, monitor data and secret keys are stored.
-      - LANGFLOW_CONFIG_DIR=/var/lib/langflow
+      - LANGFLOW_CONFIG_DIR=app/langflow
     volumes:
-      - langflow-data:/var/lib/langflow
+      - langflow-data:app/langflow
 
   postgres:
     image: postgres:16