ci: create a nightly build workflow (#3553)

* test poetry install

* Add nightly builds workflow

* remove old comments and fix poetry

* remove old debug statement

* [autofix.ci] apply automated fixes

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

.github/workflows/docker-build.yml

@@ -12,22 +12,31 @@ on:
         required: false
         type: boolean
         default: false
-
+      nightly_build:
+        required: false
+        type: boolean
+        default: false
   workflow_dispatch:
     inputs:
       version:
-        required: true
+        description: "Version to build"
+        required: false
         type: string
       release_type:
+        description: "Type of release"
         required: true
-        type: choice
+        type: string
-        options:
-          - base
-          - main
       pre_release:
+        description: "Pre-release"
         required: false
         type: boolean
-        default: true
+        default: false
+      nightly_build:
+        description: "Build nightly build"
+        required: false
+        type: boolean
+        default: false
+
 env:
   POETRY_VERSION: "1.8.2"
   TEST_TAG: "langflowai/langflow:test"
@@ -48,19 +57,19 @@ jobs:
           poetry-version: ${{ env.POETRY_VERSION }}
           cache-key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}
       - name: Get Version from Input
-        if : ${{ inputs.version != '' }}
+        if: ${{ inputs.version != '' }}
         id: get-version-input
         run: |
           version=${{ inputs.version }}
           echo version=$version >> $GITHUB_OUTPUT
-      - name: Get Version Main
+      - name: Get Version Base
-        if : ${{ inputs.version == '' && inputs.release_type == 'base' }}
+        if: ${{ inputs.version == '' && inputs.release_type == 'base' }}
         id: get-version-base
         run: |
           version=$(cd src/backend/base && poetry version --short)
           echo version=$version >> $GITHUB_OUTPUT
-      - name: Get Version Base
+      - name: Get Version Main
-        if : ${{ inputs.version == '' && inputs.release_type == 'main' }}
+        if: ${{ inputs.version == '' && inputs.release_type == 'main' }}
         id: get-version-main
         run: |
           version=$(poetry version --short)
@@ -76,14 +85,19 @@ jobs:
       - name: Set Dockerfile and Tags
         id: set-vars
         run: |
+          nightly_suffix=""
+          if [[ "${{ inputs.nightly_build }}" == "true" ]]; then
+            nightly_suffix="-nightly"
+          fi
+
           if [[ "${{ inputs.release_type }}" == "base" ]]; then
-            echo "tags=langflowai/langflow:base-${{ needs.get-version.outputs.version }},langflowai/langflow:base-latest" >> $GITHUB_OUTPUT
+            echo "tags=langflowai/langflow${nightly_suffix}:base-${{ needs.get-version.outputs.version }},langflowai/langflow${nightly_suffix}:base-latest" >> $GITHUB_OUTPUT
             echo "file=./docker/build_and_push_base.Dockerfile" >> $GITHUB_OUTPUT
           else
             if [[ "${{ inputs.pre_release }}" == "true" ]]; then
-              echo "tags=langflowai/langflow:${{ needs.get-version.outputs.version }}" >> $GITHUB_OUTPUT
+              echo "tags=langflowai/langflow${nightly_suffix}:${{ needs.get-version.outputs.version }}" >> $GITHUB_OUTPUT
             else
-              echo "tags=langflowai/langflow:${{ needs.get-version.outputs.version }},langflowai/langflow:latest" >> $GITHUB_OUTPUT
+              echo "tags=langflowai/langflow${nightly_suffix}:${{ needs.get-version.outputs.version }},langflowai/langflow${nightly_suffix}:latest" >> $GITHUB_OUTPUT
             fi
             echo "file=./docker/build_and_push.Dockerfile" >> $GITHUB_OUTPUT
           fi
@@ -119,10 +133,10 @@ jobs:
         include:
           - component: backend
             dockerfile: ./docker/build_and_push_backend.Dockerfile
-            tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-backend:{0}', needs.get-version.outputs.version) || format('langflowai/langflow-backend:{0},langflowai/langflow-backend:latest', needs.get-version.outputs.version) }}
+            tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-backend{0}:{1}', inputs.nightly_build && '-nightly' || '', needs.get-version.outputs.version) || format('langflowai/langflow-backend{0}:{1},langflowai/langflow-backend{0}:latest{1}',  inputs.nightly_build && '-nightly' || '', needs.get-version.outputs.version) }}
           - component: frontend
             dockerfile: ./docker/frontend/build_and_push_frontend.Dockerfile
-            tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-frontend:{0}', needs.get-version.outputs.version) || format('langflowai/langflow-frontend:{0},langflowai/langflow-frontend:latest', needs.get-version.outputs.version) }}
+            tags: ${{ inputs.pre_release == 'true' && format('langflowai/langflow-frontend{0}:{1}', inputs.nightly_build && '-nightly' || '', needs.get-version.outputs.version) || format('langflowai/langflow-frontend{0}:{1},langflowai/langflow-frontend{0}:latest{1}',  inputs.nightly_build && '-nightly' || '', needs.get-version.outputs.version) }}
     steps:
       - uses: actions/checkout@v4
       - name: Set up Docker Buildx
@@ -140,7 +154,7 @@ jobs:
           context: .
           push: true
           build-args: |
-            LANGFLOW_IMAGE=langflowai/langflow:${{ needs.get-version.outputs.version }}
+            LANGFLOW_IMAGE=langflowai/langflow${{ inputs.nightly_build && '-nightly' || '' }}:${{ needs.get-version.outputs.version }}
           file: ${{ matrix.dockerfile }}
           tags: ${{ matrix.tags }}
           # provenance: false will result in a single manifest for all platforms which makes the image pullable from arm64 machines via the emulation (e.g. Apple Silicon machines)
@@ -148,7 +162,7 @@ jobs:
 
   restart-space:
     name: Restart HuggingFace Spaces
-    if: ${{ inputs.release_type == 'main' }}
+    if: ${{ inputs.release_type == 'main' && inputs.nightly_build == 'false' }}
     runs-on: ubuntu-latest
     needs: build
     strategy:

.github/workflows/integration_tests.yml

@@ -1,20 +1,29 @@
-name: Integration tests
+name: Integration Tests
 
 on:
   workflow_dispatch:
     inputs:
-      branch:
+      ref:
-        description: "(Optional) Branch to checkout"
+        description: "(Optional) ref to checkout"
+        required: false
+        type: string
+  workflow_call:
+    inputs:
+      python-versions:
+        description: "(Optional) Python versions to test"
+        required: true
+        type: string
+        default: "['3.10', '3.11', '3.12']"
+      ref:
+        description: "(Optional) ref to checkout"
         required: false
         type: string
-  schedule:
-    - cron: "0 0 */2 * *" # Run every 2 days
 
 env:
   POETRY_VERSION: "1.8.2"
 
 jobs:
-  test-integration:
+  integration-tests:
     name: Run Integration Tests
     runs-on: ubuntu-latest
     strategy:
@@ -32,7 +41,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.branch || github.ref }}
+          ref: ${{ inputs.ref || github.ref }}
       - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
         uses: "./.github/actions/poetry_caching"
         with:

.github/workflows/nightly_build.yml

@@ -3,15 +3,142 @@ name: Nightly Build
 on:
   workflow_dispatch:
   schedule:
-    - cron: "0 0 * * *" # Run every day at midnight (UTC)
+    # Run job at 6:30 UTC, 10.30pm PST, or 11.30pm PDT
+    - cron: "30 6 * * *"
 
 env:
-  POETRY_VERSION: "1.8.2"
+  POETRY_VERSION: "1.8.3"
+  PYTHON_VERSION: "3.12"
 
 jobs:
-  hello-world:
+  create-nightly-tag:
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    permissions:
+      # Required to create tag
+      contents: write
+    outputs:
+      main_tag: ${{ steps.create_tag.outputs.main_tag }}
+      base_tag: ${{ steps.create_tag.outputs.base_tag }}
     steps:
-      - name: Run hello world
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: true
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_caching"
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          cache-key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
         run: |
-          echo "Hello, world!"
+          poetry env use ${{ env.PYTHON_VERSION }}
+          poetry install
+
+      - name: Create tag
+        id: create_tag
+        run: |
+          git config --global user.email "bot-nightly-builds@langflow.org"
+          git config --global user.name "Langflow Bot"
+
+          # WARNING: These scripts must be run in this order.
+          # Poetry will use a different cached virtual environment once the main pyproject.toml
+          # project-name is updated, which does not have dependencies installed.
+          BASE_TAG="$(poetry run python ./scripts/ci/pypi_nightly_tag.py base)"
+          echo "base_tag=$BASE_TAG" >> $GITHUB_OUTPUT
+          poetry run python ./scripts/ci/update_pyproject_name.py langflow-base-nightly base
+          poetry run python ./scripts/ci/update_pyproject_version.py $BASE_TAG base
+
+          # This updates the dependency of langflow-base to langflow-base-nightly in {project_root}/pyproject.toml
+          poetry run python ./scripts/ci/update_lf_base_dependency.py $BASE_TAG
+
+          MAIN_TAG="$(poetry run python ./scripts/ci/pypi_nightly_tag.py main)"
+          echo "main_tag=$MAIN_TAG" >> $GITHUB_OUTPUT
+          poetry run python ./scripts/ci/update_pyproject_version.py $MAIN_TAG main
+          poetry run python ./scripts/ci/update_pyproject_name.py langflow-nightly main
+
+          git add pyproject.toml src/backend/base/pyproject.toml
+          git commit -m "Update version and project name in files"
+
+          git tag -a $MAIN_TAG -m "Langflow nightly $MAIN_TAG"
+          git push origin $MAIN_TAG || echo "Tag push failed. Check if the tag already exists."
+          # TODO: notify on failure
+
+  frontend-tests:
+    name: Run Frontend Tests
+    needs: create-nightly-tag
+    uses: ./.github/workflows/typescript_test.yml
+    with:
+      tests_folder: "tests/end-to-end"
+      ref: ${{ needs.create-nightly-tag.outputs.tag }}
+    secrets:
+      OPENAI_API_KEY: "${{ secrets.OPENAI_API_KEY }}"
+      STORE_API_KEY: "${{ secrets.STORE_API_KEY }}"
+
+  backend-unit-tests:
+    name: Run Backend Unit Tests
+    needs: create-nightly-tag
+    uses: ./.github/workflows/python_test.yml
+    with:
+      python-versions: '["3.10", "3.11", "3.12"]'
+      ref: ${{ needs.create-nightly-tag.outputs.tag }}
+
+  # Not making nightly builds dependent on integration test success
+  # due to inherent flakiness of 3rd party integrations
+  # Revisit when https://github.com/langflow-ai/langflow/pull/3607 is merged.
+  # backend-integration-tests:
+  #   name: Run Backend Integration Tests
+  #   needs: create-nightly-tag
+  #   uses: ./.github/workflows/integration_tests.yml
+  #   with:
+  #     python-versions: '["3.10", "3.11", "3.12"]'
+  #     ref: ${{ needs.create-nightly-tag.outputs.tag }}
+
+  release-nightly-build:
+    name: Run Nightly Langflow Build
+    needs: [frontend-tests, backend-unit-tests, create-nightly-tag]
+    uses: ./.github/workflows/release_nightly.yml
+    with:
+      build_docker_base: false # TODO: Docker builds
+      build_docker_main: false
+      nightly_tag_main: ${{ needs.create-nightly-tag.outputs.main_tag }}
+      nightly_tag_base: ${{ needs.create-nightly-tag.outputs.base_tag }}
+    secrets: inherit
+
+  # slack-notification:
+  #   name: Send Slack Notification
+  #   needs: run-nightly-build
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Send success notification to Slack
+  #       if: success()
+  #       uses: slackapi/slack-github-action@v1.26.0
+  #       with:
+  #         payload: |
+  #           {
+  #             "channel": "#langflow-nightly-builds",
+  #             "username": "GitHub Actions",
+  #             "text": "Nightly Build Successful :white_check_mark:",
+  #             "icon_emoji": ":rocket:"
+  #           }
+  #       env:
+  #         SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+
+  #     - name: Send failure notification to Slack
+  #       if: failure()
+  #       uses: slackapi/slack-github-action@v1.26.0
+  #       with:
+  #         payload: |
+  #           {
+  #             "channel": "#langflow-nightly-builds",
+  #             "username": "GitHub Actions",
+  #             "text": "Nightly Build Failed :x:",
+  #             "icon_emoji": ":warning:"
+  #           }
+  #       env:
+  #         SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/python_test.yml

@@ -8,12 +8,17 @@ on:
         required: true
         type: string
         default: "['3.10', '3.11', '3.12']"
-  workflow_dispatch:
+      ref:
-    inputs:
+        description: "(Optional) ref to checkout"
-      branch:
-        description: "(Optional) Branch to checkout"
         required: false
         type: string
+      nightly:
+        description: "Whether run is from the nightly build"
+        required: false
+        type: boolean
+        default: false
+  workflow_dispatch:
+    inputs:
       python-versions:
         description: "(Optional) Python versions to test"
         required: true
@@ -35,7 +40,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.branch || github.ref }}
+          ref: ${{ inputs.ref || github.ref }}
       - name: Setup Node.js
         uses: actions/setup-node@v4
         id: setup-node
@@ -88,6 +93,9 @@ jobs:
         python-version: ${{ fromJson(inputs.python-versions || '["3.10", "3.11", "3.12"]') }}
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
       - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
         uses: "./.github/actions/poetry_caching"
         with:
@@ -98,7 +106,12 @@ jobs:
         id: check-version
         run: |
           version=$(cd src/backend/base && poetry version --short)
-          last_released_version=$(curl -s "https://pypi.org/pypi/langflow-base/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1)
+          url="https://pypi.org/pypi/langflow-base/json"
+          if [ ${{ inputs.nightly }} == true ]; then
+            url="https://pypi.org/pypi/langflow-base-nightly/json"
+          fi
+
+          last_released_version=$(curl -s $url | jq -r '.releases | keys | .[]' | sort -V | tail -n 1)
           if [ "$version" != "$last_released_version" ]; then
             echo "Version $version has not been released yet. Skipping the rest of the job."
             echo skipped=true >> $GITHUB_OUTPUT

.github/workflows/release.yml

@@ -29,7 +29,11 @@ on:
         required: false
         type: boolean
         default: false
-
+      create_release:
+        description: "Whether to create a gh release"
+        required: false
+        type: boolean
+        default: true
 
 env:
   POETRY_VERSION: "1.8.2"
@@ -155,7 +159,7 @@ jobs:
           fi
       - name: Wait for PyPI Propagation
         if: needs.release-base.outputs.skipped == 'false'
-        run: sleep 300  # wait for 5 minutes to ensure PyPI propagation
+        run: sleep 300 # wait for 5 minutes to ensure PyPI propagation
 
       - name: Build project for distribution
         run: make build main=true
@@ -188,7 +192,7 @@ jobs:
 
   call_docker_build_base:
     name: Call Docker Build Workflow for Langflow Base
-    if : inputs.build_docker_base == true
+    if: inputs.build_docker_base == true
     uses: ./.github/workflows/docker-build.yml
     strategy:
       matrix:
@@ -197,14 +201,14 @@ jobs:
     with:
       # version should be needs.release-base.outputs.version  if release_type is base
       # version should be needs.release-main.outputs.version  if release_type is main
-      version: ''
+      version: ""
       release_type: ${{ matrix.release_type }}
       pre_release: ${{ inputs.pre_release }}
     secrets: inherit
 
   call_docker_build_main:
     name: Call Docker Build Workflow for Langflow
-    if : inputs.build_docker_main == true
+    if: inputs.build_docker_main == true
     uses: ./.github/workflows/docker-build.yml
     strategy:
       matrix:
@@ -213,7 +217,7 @@ jobs:
     with:
       # version should be needs.release-base.outputs.version  if release_type is base
       # version should be needs.release-main.outputs.version  if release_type is main
-      version: ''
+      version: ""
       release_type: ${{ matrix.release_type }}
       pre_release: ${{ inputs.pre_release }}
     secrets: inherit
@@ -236,4 +240,4 @@ jobs:
           generateReleaseNotes: true
           prerelease: ${{ inputs.pre_release }}
           tag: v${{ needs.release-main.outputs.version }}
-          commit: ${{ github.ref }}
+          commit: ${{ github.ref }}

.github/workflows/release_nightly.yml

@@ -0,0 +1,225 @@
+name: Langflow Nightly Build
+run-name: Langflow Nightly Release by @${{ github.actor }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_docker_base:
+        description: "Build Docker Image for Langflow Nightly Base"
+        required: true
+        type: boolean
+        default: false
+      build_docker_main:
+        description: "Build Docker Image for Langflow Nightly"
+        required: true
+        type: boolean
+        default: false
+      nightly_tag_main:
+        description: "Tag for the nightly main build"
+        required: true
+        type: string
+      nightly_tag_base:
+        description: "Tag for the nightly base build"
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      build_docker_base:
+        description: "Build Docker Image for Langflow Nightly Base"
+        required: true
+        type: boolean
+        default: false
+      build_docker_main:
+        description: "Build Docker Image for Langflow Nightly"
+        required: true
+        type: boolean
+        default: false
+      nightly_tag_main:
+        description: "Tag for the nightly main build"
+        required: true
+        type: string
+      nightly_tag_base:
+        description: "Tag for the nightly base build"
+        required: true
+        type: string
+
+env:
+  POETRY_VERSION: "1.8.3"
+  PYTHON_VERSION: "3.12"
+
+jobs:
+  release-nightly-base:
+    name: Release Langflow Nightly Base
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Check out the code at a specific ref (e.g., nightly tag)
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.nightly_tag_main }}
+          persist-credentials: true
+      - name: Install poetry
+        run: |
+          pipx install poetry==${{ env.POETRY_VERSION }}
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "poetry"
+      - name: Set up Nodejs 20
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Verify Nightly Name and Version
+        working-directory: src/backend/base
+        run: |
+          name=$(poetry version | cut -d' ' -f1)
+          version=$(poetry version --short)
+          if [ "$name" != "langflow-base-nightly" ]; then
+            echo "Name $name does not match langflow-base-nightly. Exiting the workflow."
+            exit 1
+          fi
+          if [ "$version" != "${{ inputs.nightly_tag_base }}" ]; then
+            echo "Version $version does not match nightly tag ${{ inputs.nightly_tag_base }}. Exiting the workflow."
+            exit 1
+          fi
+
+      - name: Build project for distribution
+        run: make build base=true
+
+      - name: Test CLI
+        run: |
+          python -m pip install src/backend/base/dist/*.whl
+          python -m langflow run --host 127.0.0.1 --port 7860 &
+          SERVER_PID=$!
+          # Wait for the server to start
+          timeout 120 bash -c 'until curl -f http://127.0.0.1:7860/api/v1/auto_login; do sleep 2; done' || (echo "Server did not start in time" && kill $SERVER_PID && exit 1)
+          # Terminate the server
+          kill $SERVER_PID || (echo "Failed to terminate the server" && exit 1)
+          sleep 10 # give the server some time to terminate
+          # Check if the server is still running
+          if kill -0 $SERVER_PID 2>/dev/null; then
+            echo "Failed to terminate the server"
+            exit 1
+          else
+            echo "Server terminated successfully"
+          fi
+
+      - name: Publish to PyPI
+        env:
+          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          make publish base=true
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-base
+          path: src/backend/base/dist
+
+  release-nightly-main:
+    name: Release Langflow Nightly Main
+    needs: [release-nightly-base]
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Check out the code at a specific ref (e.g., nightly tag)
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.nightly_tag_main }}
+
+      - name: Install poetry
+        run: pipx install poetry==${{ env.POETRY_VERSION }}
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "poetry"
+      - name: Set up Nodejs 20
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+      - name: Verify Nightly Name and Version
+        run: |
+          name=$(poetry version | cut -d' ' -f1)
+          version=$(poetry version --short)
+          if [ "$name" != "langflow-nightly" ]; then
+            echo "Name $name does not match langflow-nightly. Exiting the workflow."
+            exit 1
+          fi
+          if [ "$version" != "${{ inputs.nightly_tag_main }}" ]; then
+            echo "Version $version does not match nightly tag ${{ inputs.nightly_tag_main }}. Exiting the workflow."
+            exit 1
+          fi
+
+      - name: Wait for PyPI Propagation
+        run: sleep 300 # wait for 5 minutes to ensure PyPI propagation of base
+
+      - name: Build project for distribution
+        run: make build main=true
+      - name: Test CLI
+        run: |
+          python -m pip install dist/*.whl
+          python -m langflow run --host 127.0.0.1 --port 7860 --backend-only &
+          SERVER_PID=$!
+          # Wait for the server to start
+          timeout 120 bash -c 'until curl -f http://127.0.0.1:7860/health_check; do sleep 2; done' || (echo "Server did not start in time" && kill $SERVER_PID && exit 1)
+          # Terminate the server
+          kill $SERVER_PID || (echo "Failed to terminate the server" && exit 1)
+          sleep 10 # give the server some time to terminate
+          # Check if the server is still running
+          if kill -0 $SERVER_PID 2>/dev/null; then
+            echo "Failed to terminate the server"
+            exit 1
+          else
+            echo "Server terminated successfully"
+          fi
+      - name: Publish to PyPI
+        env:
+          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          make publish main=true
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-main
+          path: dist
+
+  call_docker_build_base:
+    name: Call Docker Build Workflow for Langflow Base
+    if: ${{ inputs.build_docker_base == 'true' }}
+    needs: [release-nightly-base]
+    uses: ./.github/workflows/docker-build.yml
+    strategy:
+      matrix:
+        release_type:
+          - base
+    with:
+      # version should be needs.release-base.outputs.version  if release_type is base
+      # version should be needs.release-main.outputs.version  if release_type is main
+      version: ""
+      release_type: ${{ matrix.release_type }}
+      nightly_build: true
+    secrets: inherit
+
+  call_docker_build_main:
+    name: Call Docker Build Workflow for Langflow
+    if: ${{ inputs.build_docker_main == 'true' }}
+    needs: [release-nightly-main]
+    uses: ./.github/workflows/docker-build.yml
+    strategy:
+      matrix:
+        release_type:
+          - main
+    with:
+      # version should be needs.release-base.outputs.version  if release_type is base
+      # version should be needs.release-main.outputs.version  if release_type is main
+      version: ""
+      release_type: ${{ matrix.release_type }}
+      nightly_build: true
+    secrets: inherit

.github/workflows/typescript_test.yml

@@ -13,12 +13,12 @@ on:
         required: false
         type: string
         default: "tests"
-  workflow_dispatch:
+      ref:
-    inputs:
+        description: "(Optional) ref to checkout"
-      branch:
-        description: "(Optional) Branch to checkout"
         required: false
         type: string
+  workflow_dispatch:
+    inputs:
       tests_folder:
         description: "(Optional) Tests to run"
         required: false
@@ -54,9 +54,9 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          # If branch is passed as input, checkout that branch
+          # If ref is passed as input, checkout that ref
-          # else checkout the default branch
+          # else checkout the default ref
-          ref: ${{ github.event.inputs.branch || github.ref }}
+          ref: ${{ inputs.ref || github.ref }}
 
       - name: Setup Node.js
         uses: actions/setup-node@v4

Makefile

@@ -409,6 +409,12 @@ publish_base:
 publish_langflow:
 	poetry publish
 
+publish_base_testpypi:
+	cd src/backend/base && poetry publish --skip-existing -r test-pypi
+
+publish_langflow_testpypi:
+	poetry publish -r test-pypi
+
 publish: ## build the frontend static files and package the project and publish it to PyPI
 	@echo 'Publishing the project'
 ifdef base
@@ -418,3 +424,17 @@ endif
 ifdef main
 	make publish_langflow
 endif
+
+publish_testpypi: ## build the frontend static files and package the project and publish it to PyPI
+	@echo 'Publishing the project'
+
+ifdef base
+	poetry config repositories.test-pypi https://test.pypi.org/legacy/
+	make publish_base_testpypi
+endif
+
+ifdef main
+	poetry config repositories.test-pypi https://test.pypi.org/legacy/
+	make publish_langflow_testpypi
+endif
+

pyproject.toml

@@ -142,6 +142,7 @@ dictdiffer = "^0.9.0"
 pytest-split = "^0.9.0"
 pytest-flakefinder = "^1.1.0"
 types-markdown = "^3.7.0.20240822"
+packaging = ">=23.2,<24.0"
 
 [tool.poetry.extras]
 deploy = ["celery", "redis", "flower"]

scripts/ci/pypi_nightly_tag.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+"""
+Idea from https://github.com/streamlit/streamlit/blob/4841cf91f1c820a392441092390c4c04907f9944/scripts/pypi_nightly_create_tag.py
+"""
+
+import sys
+
+import packaging.version
+from packaging.version import Version
+
+PYPI_LANGFLOW_URL = "https://pypi.org/pypi/langflow/json"
+PYPI_LANGFLOW_NIGHTLY_URL = "https://pypi.org/pypi/langflow-nightly/json"
+
+PYPI_LANGFLOW_BASE_URL = "https://pypi.org/pypi/langflow-base/json"
+PYPI_LANGFLOW_BASE_NIGHTLY_URL = "https://pypi.org/pypi/langflow-base-nightly/json"
+
+
+def get_latest_published_version(build_type: str, is_nightly: bool) -> Version:
+    import requests
+
+    url = ""
+    if build_type == "base":
+        url = PYPI_LANGFLOW_BASE_NIGHTLY_URL if is_nightly else PYPI_LANGFLOW_BASE_URL
+    elif build_type == "main":
+        url = PYPI_LANGFLOW_NIGHTLY_URL if is_nightly else PYPI_LANGFLOW_URL
+    else:
+        raise ValueError(f"Invalid build type: {build_type}")
+
+    res = requests.get(url)
+    try:
+        version_str = res.json()["info"]["version"]
+    except Exception as e:
+        raise RuntimeError("Got unexpected response from PyPI", e)
+    return Version(version_str)
+
+
+def create_tag(build_type: str):
+    current_version = get_latest_published_version(build_type, is_nightly=False)
+    current_nightly_version = get_latest_published_version(build_type, is_nightly=True)
+
+    build_number = "0"
+    latest_base_version = current_version.base_version
+    nightly_base_version = current_nightly_version.base_version
+
+    if latest_base_version == nightly_base_version:
+        # If the latest version is the same as the nightly version, increment the build number
+        build_number = str(current_nightly_version.dev + 1)
+
+    new_nightly_version = latest_base_version + ".dev" + build_number
+
+    # X.Y.Z.dev.YYYYMMDD
+    # This takes the base version of the current version and appends the
+    # current date. If the last release was on the same day, we exit, as
+    # pypi does not allow for overwriting the same version.
+    #
+    # We could use a different versioning scheme, such as just incrementing
+    # an integer.
+    # version_with_date = (
+    #     ".".join([str(x) for x in current_version.release])
+    #     + ".dev"
+    #     + "0"
+    #     + datetime.now(pytz.timezone("UTC")).strftime("%Y%m%d")
+    # )
+
+    # Verify if version is PEP440 compliant.
+    packaging.version.Version(new_nightly_version)
+
+    return new_nightly_version
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        raise Exception("Specify base or main")
+
+    build_type = sys.argv[1]
+    tag = create_tag(build_type)
+    print(tag)

scripts/ci/update_lf_base_dependency.py

@@ -0,0 +1,62 @@
+import os
+import sys
+import re
+
+import packaging.version
+
+BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
+
+
+def update_base_dep(pyproject_path: str, new_version: str) -> None:
+    """Update the langflow-base dependency in pyproject.toml."""
+    filepath = os.path.join(BASE_DIR, pyproject_path)
+    with open(filepath, "r") as file:
+        content = file.read()
+
+    # Regex to match the langflow-base dep line under [tool.poetry]
+    # NOTE: this functions similarly to `update_dependencies.py`.
+    # The order of operations for these scripts is more complex than it needs to be;
+    # this is a reminder to revisit this process.
+    # Currently, the process is as follows:
+    # 1. nightly-build workflow updates all names, version, and the langflow-base
+    #    dependency in pyproject.toml, and commits the changes and creates a tag.
+    # 2. release-nightly workflow runs, which calls update_dependencies.py to update
+    #   the langflow-base dependency in pyproject.toml _again_. This is redundant, but
+    #   necessary because the release workflow relies on that script to update the base
+    #   dependency.
+    pattern = re.compile(r'langflow-base = \{ path = "\./src/backend/base", develop = true \}')
+
+    if not pattern.search(content):
+        raise Exception(f'langflow-base dependency not found in "{filepath}"')
+
+    replacement = f'langflow-base-nightly = "^{new_version}"'
+    content = pattern.sub(replacement, content)
+
+    with open(filepath, "w") as file:
+        file.write(content)
+
+
+def verify_pep440(version):
+    """
+    Verify if version is PEP440 compliant.
+
+    https://github.com/pypa/packaging/blob/16.7/packaging/version.py#L191
+    """
+
+    try:
+        return packaging.version.Version(version)
+    except packaging.version.InvalidVersion as e:
+        raise e
+
+
+def main() -> None:
+    if len(sys.argv) != 2:
+        raise Exception("New version not specified")
+    base_version = sys.argv[1]
+
+    verify_pep440(base_version)
+    update_base_dep("pyproject.toml", base_version)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/update_pyproject_name.py

@@ -0,0 +1,41 @@
+import os
+import sys
+import re
+
+BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
+
+
+def update_pyproject_name(pyproject_path: str, new_project_name: str) -> None:
+    """Update the project name in pyproject.toml."""
+    filepath = os.path.join(BASE_DIR, pyproject_path)
+    with open(filepath, "r") as file:
+        content = file.read()
+
+    # Regex to match the version line under [tool.poetry]
+    pattern = re.compile(r'(?<=^name = ")[^"]+(?=")', re.MULTILINE)
+
+    if not pattern.search(content):
+        raise Exception(f'Project name not found in "{filepath}"')
+
+    content = pattern.sub(new_project_name, content)
+
+    with open(filepath, "w") as file:
+        file.write(content)
+
+
+def main() -> None:
+    if len(sys.argv) != 3:
+        raise Exception("Must specify project name and build type, e.g. langflow-nightly base")
+    new_project_name = sys.argv[1]
+    build_type = sys.argv[2]
+
+    if build_type == "base":
+        update_pyproject_name("src/backend/base/pyproject.toml", new_project_name)
+    elif build_type == "main":
+        update_pyproject_name("pyproject.toml", new_project_name)
+    else:
+        raise ValueError(f"Invalid build type: {build_type}")
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/update_pyproject_version.py

@@ -0,0 +1,58 @@
+import os
+import sys
+import re
+
+import packaging.version
+
+BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
+
+
+def update_pyproject_version(pyproject_path: str, new_version: str) -> None:
+    """Update the version in pyproject.toml."""
+    filepath = os.path.join(BASE_DIR, pyproject_path)
+    with open(filepath, "r") as file:
+        content = file.read()
+
+    # Regex to match the version line under [tool.poetry]
+    pattern = re.compile(r'(?<=^version = ")[^"]+(?=")', re.MULTILINE)
+
+    if not pattern.search(content):
+        raise Exception(f'Project version not found in "{filepath}"')
+
+    content = pattern.sub(new_version, content)
+
+    with open(filepath, "w") as file:
+        file.write(content)
+
+
+def verify_pep440(version):
+    """
+    Verify if version is PEP440 compliant.
+
+    https://github.com/pypa/packaging/blob/16.7/packaging/version.py#L191
+    """
+
+    try:
+        return packaging.version.Version(version)
+    except packaging.version.InvalidVersion as e:
+        raise e
+
+
+def main() -> None:
+    if len(sys.argv) != 3:
+        raise Exception("New version not specified")
+    new_version = sys.argv[1]
+    build_type = sys.argv[2]
+
+    verify_pep440(new_version)
+
+    if build_type == "base":
+        update_pyproject_version("src/backend/base/pyproject.toml", new_version)
+    elif build_type == "main":
+        update_pyproject_version("pyproject.toml", new_version)
+    else:
+        raise ValueError(f"Invalid build type: {build_type}")
+
+
+if __name__ == "__main__":
+    main()

scripts/update_dependencies.py

@@ -31,9 +31,22 @@ def get_version_from_pypi(package_name):
     return None
 
 
-def update_pyproject_dependency(pyproject_path, version):
+def is_development_release(version):
+    """
+    Determines if the version is a development version based on PEP 440.
+
+    We consider a development version (.devN) as our nightly versions
+    """
+    return "dev" in version
+
+
+def update_pyproject_dependency(pyproject_path, version, is_nightly):
     pattern = re.compile(r'langflow-base = \{ path = "\./src/backend/base", develop = true \}')
-    replacement = f'langflow-base = "^{version}"'
+    if is_nightly:
+        # NOTE: This process can be simplified; see the note in update_lf_base_dependency.py
+        replacement = f'langflow-base-nightly = "{version}"'
+    else:
+        replacement = f'langflow-base = "^{version}"'
     with open(pyproject_path, "r") as file:
         content = file.read()
     content = pattern.sub(replacement, content)
@@ -57,6 +70,9 @@ if __name__ == "__main__":
     langflow_base_path = Path(__file__).resolve().parent / "../src/backend/base/pyproject.toml"
     version = read_version_from_pyproject(langflow_base_path)
     if version:
-        update_pyproject_dependency(pyproject_path, version)
+        # Nightly versions contain "dev"
+        # WARNING: This will cause issues if we release `.dev` versions to `langflow` or `langflow-base`
+        is_nightly = is_development_release(version)
+        update_pyproject_dependency(pyproject_path, version, is_nightly)
     else:
         print("Error: Version not found.")

src/backend/base/langflow/__main__.py

@@ -8,6 +8,8 @@ from typing import Optional
 
 import click
 import httpx
+from langflow.utils.version import get_version_info, fetch_latest_version
+from langflow.utils.version import is_pre_release as langflow_is_pre_release
 import typer
 from dotenv import load_dotenv
 from multiprocess import cpu_count  # type: ignore
@@ -22,7 +24,9 @@ from sqlmodel import select
 
 from langflow.logging.logger import configure, logger
 from langflow.main import setup_app
-from langflow.services.database.models.folder.utils import create_default_folder_if_it_doesnt_exist
+from langflow.services.database.models.folder.utils import (
+    create_default_folder_if_it_doesnt_exist,
+)
 from langflow.services.database.utils import session_getter
 from langflow.services.deps import get_db_service, get_settings_service, session_scope
 from langflow.services.settings.constants import DEFAULT_SUPERUSER
@@ -259,13 +263,6 @@ def get_free_port(port):
     return port
 
 
-def version_is_prerelease(version: str):
-    """
-    Check if a version is a pre-release version.
-    """
-    return "a" in version or "b" in version or "rc" in version
-
-
 def get_letter_from_version(version: str):
     """
     Get the letter from a pre-release version.
@@ -279,30 +276,10 @@ def get_letter_from_version(version: str):
     return None
 
 
-def is_prerelease(version: str) -> bool:
-    return "a" in version or "b" in version or "rc" in version
-
-
-def fetch_latest_version(package_name: str, include_prerelease: bool) -> Optional[str]:
-    valid_versions = []
-    try:
-        response = httpx.get(f"https://pypi.org/pypi/{package_name}/json")
-        versions = response.json()["releases"].keys()
-        valid_versions = [v for v in versions if include_prerelease or not is_prerelease(v)]
-
-    except Exception as e:
-        logger.exception(e)
-
-    finally:
-        if not valid_versions:
-            return None  # Handle case where no valid versions are found
-        return max(valid_versions, key=lambda v: pkg_version.parse(v))
-
-
 def build_version_notice(current_version: str, package_name: str) -> str:
-    latest_version = fetch_latest_version(package_name, is_prerelease(current_version))
+    latest_version = fetch_latest_version(package_name, langflow_is_pre_release(current_version))
     if latest_version and pkg_version.parse(current_version) < pkg_version.parse(latest_version):
-        release_type = "pre-release" if is_prerelease(latest_version) else "version"
+        release_type = "pre-release" if langflow_is_pre_release(latest_version) else "version"
         return f"A new {release_type} of {package_name} is available: {latest_version}"
     return ""
 
@@ -331,35 +308,17 @@ def print_banner(host: str, port: int):
     is_pre_release = False  # Track if any package is a pre-release
     package_name = ""
 
-    try:
+    # Use langflow.utils.version to get the version info
-        from langflow.version import __version__ as langflow_version  # type: ignore
+    version_info = get_version_info()
+    langflow_version = version_info["version"]
+    package_name = version_info["package"]
+    is_pre_release |= langflow_is_pre_release(langflow_version)  # Update pre-release status
 
-        is_pre_release |= is_prerelease(langflow_version)  # Update pre-release status
+    notice = build_version_notice(langflow_version, package_name)
-        notice = build_version_notice(langflow_version, "langflow")
+    notice = stylize_text(notice, package_name, is_pre_release)
-        notice = stylize_text(notice, "langflow", is_pre_release)
+    if notice:
-        if notice:
+        notices.append(notice)
-            notices.append(notice)
+    package_names.append(package_name)
-        package_names.append("langflow")
-        package_name = "Langflow"
-    except ImportError:
-        langflow_version = None
-
-    # Attempt to handle langflow-base similarly
-    if langflow_version is None:  # This means langflow.version was not imported
-        try:
-            from importlib import metadata
-
-            langflow_base_version = metadata.version("langflow-base")
-            is_pre_release |= is_prerelease(langflow_base_version)  # Update pre-release status
-            notice = build_version_notice(langflow_base_version, "langflow-base")
-            notice = stylize_text(notice, "langflow-base", is_pre_release)
-            if notice:
-                notices.append(notice)
-            package_names.append("langflow-base")
-            package_name = "Langflow Base"
-        except ImportError as e:
-            logger.exception(e)
-            raise e
 
     # Generate pip command based on the collected data
     pip_command = generate_pip_command(package_names, is_pre_release)
@@ -528,7 +487,10 @@ def api_key(
             typer.echo("Default superuser not found. This command requires a superuser and AUTO_LOGIN to be enabled.")
             return
         from langflow.services.database.models.api_key import ApiKey, ApiKeyCreate
-        from langflow.services.database.models.api_key.crud import create_api_key, delete_api_key
+        from langflow.services.database.models.api_key.crud import (
+            create_api_key,
+            delete_api_key,
+        )
 
         api_key = session.exec(select(ApiKey).where(ApiKey.user_id == superuser.id)).first()
         if api_key:

src/backend/base/langflow/api/utils.py

@@ -85,10 +85,12 @@ def get_is_component_from_data(data: dict):
 
 
 async def check_langflow_version(component: StoreComponentCreate):
-    from langflow.version.version import __version__ as current_version  # type: ignore
+    from langflow.version import get_version
+
+    __version__ = get_version()
 
     if not component.last_tested_version:
-        component.last_tested_version = current_version
+        component.last_tested_version = __version__
 
     langflow_version = get_lf_version_from_pypi()
     if langflow_version is None:

src/backend/base/langflow/main.py

@@ -111,12 +111,9 @@ def get_lifespan(fix_migration=False, socketio_server=None, version=None):
 
 def create_app():
     """Create the FastAPI app and include the router."""
-    try:
+    from langflow.utils.version import get_version_info
-        from langflow.version import __version__  # type: ignore
-    except ImportError:
-        from importlib.metadata import version
 
-        __version__ = version("langflow-base")
+    __version__ = get_version_info()["version"]
 
     configure()
     lifespan = get_lifespan(version=__version__)

src/backend/base/langflow/services/settings/base.py

@@ -220,13 +220,12 @@ class Settings(BaseSettings):
                 # if there is a database in that location
                 if not info.data["config_dir"]:
                     raise ValueError("config_dir not set, please set it or provide a database_url")
-                try:
-                    from langflow.version import is_pre_release  # type: ignore
-                except ImportError:
-                    from importlib import metadata
 
-                    version = metadata.version("langflow-base")
+                from langflow.utils.version import get_version_info
-                    is_pre_release = "a" in version or "b" in version or "rc" in version
+                from langflow.utils.version import is_pre_release as langflow_is_pre_release
+
+                version = get_version_info()["version"]
+                is_pre_release = langflow_is_pre_release(version)
 
                 if info.data["save_db_in_config_dir"]:
                     database_dir = info.data["config_dir"]

src/backend/base/langflow/utils/version.py

@@ -1,3 +1,9 @@
+import httpx
+
+from typing import Optional
+from langflow.logging.logger import logger
+
+
 def _compute_non_prerelease_version(prerelease_version: str) -> str:
     prerelease_keywords = ["a", "b", "rc", "dev", "post"]
     for keyword in prerelease_keywords:
@@ -7,23 +13,80 @@ def _compute_non_prerelease_version(prerelease_version: str) -> str:
 
 
 def _get_version_info():
-    try:
+    """
-        from langflow.version import __version__  # type: ignore
+    Retrieves the version of the package from a possible list of package names.
+    This accounts for after package names are updated for -nightly builds.
 
-        prerelease_version = __version__
+    Returns:
-        version = _compute_non_prerelease_version(prerelease_version)
+        str: The version of the package
-        package = "Langflow"
-    except ImportError:
-        from importlib import metadata
 
-        prerelease_version = metadata.version("langflow-base")
+    Raises:
-        version = _compute_non_prerelease_version(prerelease_version)
+        ValueError: If the package is not found from the list of package names.
-        package = "Langflow Base"
+    """
-    return {"version": prerelease_version, "main_version": version, "package": package}
+    from importlib import metadata
+
+    package_options = [
+        ("langflow", "Langflow"),
+        ("langflow-base", "Langflow Base"),
+        ("langflow-nightly", "Langflow Nightly"),
+        ("langflow-base-nightly", "Langflow Base Nightly"),
+    ]
+    __version__ = None
+    for pkg_name, display_name in package_options:
+        try:
+            __version__ = metadata.version(pkg_name)
+            prerelease_version = __version__
+            version = _compute_non_prerelease_version(prerelease_version)
+
+            return {
+                "version": prerelease_version,
+                "main_version": version,
+                "package": display_name,
+            }
+        except (ImportError, metadata.PackageNotFoundError):
+            pass
+
+    if __version__ is None:
+        raise ValueError(f"Package not found from options {package_options}")
 
 
 VERSION_INFO = _get_version_info()
 
 
+def is_pre_release(v: str) -> bool:
+    """
+    Returns a boolean indicating whether the version is a pre-release version,
+    as per the definition of a pre-release segment from PEP 440.
+    """
+    return any(label in v for label in ["a", "b", "rc"])
+
+
+def is_nightly(v: str) -> bool:
+    """
+    Returns a boolean indicating whether the version is a dev (nightly) version,
+    as per the definition of a dev segment from PEP 440.
+    """
+    return "dev" in v
+
+
+def fetch_latest_version(package_name: str, include_prerelease: bool) -> Optional[str]:
+    from packaging import version as pkg_version
+
+    package_name = package_name.replace(" ", "-").lower()
+    valid_versions = []
+    try:
+        response = httpx.get(f"https://pypi.org/pypi/{package_name}/json")
+        versions = response.json()["releases"].keys()
+        valid_versions = [v for v in versions if include_prerelease or not is_pre_release(v)]
+
+    except Exception as e:
+        logger.exception(e)
+
+    finally:
+        if not valid_versions:
+            return None  # Handle case where no valid versions are found
+        return max(valid_versions, key=lambda v: pkg_version.parse(v))
+
+
 def get_version_info():
     return VERSION_INFO

src/backend/langflow/version/__init__.py

@@ -1 +0,0 @@
-from .version import __version__, is_pre_release  # noqa: F401

src/backend/langflow/version/version.py

@@ -1,10 +1,38 @@
-from importlib import metadata
+def get_version() -> str:
+    """
+    Retrieves the version of the package from a possible list of package names.
+    This accounts for after package names are updated for -nightly builds.
 
-try:
+    Returns:
-    __version__ = metadata.version("langflow")
+        str: The version of the package
-    # Check if the version is a pre-release version
+
-    is_pre_release = any(label in __version__ for label in ["a", "b", "rc", "dev", "post"])
+    Raises:
-except metadata.PackageNotFoundError:
+        ValueError: If the package is not found from the list of package names.
-    __version__ = ""
+    """
-    is_pre_release = False
+    from importlib import metadata
-del metadata
+
+    pkg_names = [
+        "langflow",
+        "langflow-base",
+        "langflow-nightly",
+        "langflow-base-nightly",
+    ]
+    _version = None
+    for pkg_name in pkg_names:
+        try:
+            _version = metadata.version(pkg_name)
+        except (ImportError, metadata.PackageNotFoundError):
+            pass
+
+    if _version is None:
+        raise ValueError(f"Package not found from options {pkg_names}")
+
+    return _version
+
+
+def is_pre_release(v: str) -> bool:
+    """
+    Returns a boolean indicating whether the version is a pre-release version,
+    as per the definition of a pre-release segment from PEP 440.
+    """
+    return any(label in v for label in ["a", "b", "rc"])