ci: create a nightly build workflow (#3553)

* test poetry install

* Add nightly builds workflow

* remove old comments and fix poetry

* remove old debug statement

* [autofix.ci] apply automated fixes

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

scripts/ci/pypi_nightly_tag.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+"""
+Idea from https://github.com/streamlit/streamlit/blob/4841cf91f1c820a392441092390c4c04907f9944/scripts/pypi_nightly_create_tag.py
+"""
+
+import sys
+
+import packaging.version
+from packaging.version import Version
+
+PYPI_LANGFLOW_URL = "https://pypi.org/pypi/langflow/json"
+PYPI_LANGFLOW_NIGHTLY_URL = "https://pypi.org/pypi/langflow-nightly/json"
+
+PYPI_LANGFLOW_BASE_URL = "https://pypi.org/pypi/langflow-base/json"
+PYPI_LANGFLOW_BASE_NIGHTLY_URL = "https://pypi.org/pypi/langflow-base-nightly/json"
+
+
+def get_latest_published_version(build_type: str, is_nightly: bool) -> Version:
+    import requests
+
+    url = ""
+    if build_type == "base":
+        url = PYPI_LANGFLOW_BASE_NIGHTLY_URL if is_nightly else PYPI_LANGFLOW_BASE_URL
+    elif build_type == "main":
+        url = PYPI_LANGFLOW_NIGHTLY_URL if is_nightly else PYPI_LANGFLOW_URL
+    else:
+        raise ValueError(f"Invalid build type: {build_type}")
+
+    res = requests.get(url)
+    try:
+        version_str = res.json()["info"]["version"]
+    except Exception as e:
+        raise RuntimeError("Got unexpected response from PyPI", e)
+    return Version(version_str)
+
+
+def create_tag(build_type: str):
+    current_version = get_latest_published_version(build_type, is_nightly=False)
+    current_nightly_version = get_latest_published_version(build_type, is_nightly=True)
+
+    build_number = "0"
+    latest_base_version = current_version.base_version
+    nightly_base_version = current_nightly_version.base_version
+
+    if latest_base_version == nightly_base_version:
+        # If the latest version is the same as the nightly version, increment the build number
+        build_number = str(current_nightly_version.dev + 1)
+
+    new_nightly_version = latest_base_version + ".dev" + build_number
+
+    # X.Y.Z.dev.YYYYMMDD
+    # This takes the base version of the current version and appends the
+    # current date. If the last release was on the same day, we exit, as
+    # pypi does not allow for overwriting the same version.
+    #
+    # We could use a different versioning scheme, such as just incrementing
+    # an integer.
+    # version_with_date = (
+    #     ".".join([str(x) for x in current_version.release])
+    #     + ".dev"
+    #     + "0"
+    #     + datetime.now(pytz.timezone("UTC")).strftime("%Y%m%d")
+    # )
+
+    # Verify if version is PEP440 compliant.
+    packaging.version.Version(new_nightly_version)
+
+    return new_nightly_version
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        raise Exception("Specify base or main")
+
+    build_type = sys.argv[1]
+    tag = create_tag(build_type)
+    print(tag)

scripts/ci/update_lf_base_dependency.py

@@ -0,0 +1,62 @@
+import os
+import sys
+import re
+
+import packaging.version
+
+BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
+
+
+def update_base_dep(pyproject_path: str, new_version: str) -> None:
+    """Update the langflow-base dependency in pyproject.toml."""
+    filepath = os.path.join(BASE_DIR, pyproject_path)
+    with open(filepath, "r") as file:
+        content = file.read()
+
+    # Regex to match the langflow-base dep line under [tool.poetry]
+    # NOTE: this functions similarly to `update_dependencies.py`.
+    # The order of operations for these scripts is more complex than it needs to be;
+    # this is a reminder to revisit this process.
+    # Currently, the process is as follows:
+    # 1. nightly-build workflow updates all names, version, and the langflow-base
+    #    dependency in pyproject.toml, and commits the changes and creates a tag.
+    # 2. release-nightly workflow runs, which calls update_dependencies.py to update
+    #   the langflow-base dependency in pyproject.toml _again_. This is redundant, but
+    #   necessary because the release workflow relies on that script to update the base
+    #   dependency.
+    pattern = re.compile(r'langflow-base = \{ path = "\./src/backend/base", develop = true \}')
+
+    if not pattern.search(content):
+        raise Exception(f'langflow-base dependency not found in "{filepath}"')
+
+    replacement = f'langflow-base-nightly = "^{new_version}"'
+    content = pattern.sub(replacement, content)
+
+    with open(filepath, "w") as file:
+        file.write(content)
+
+
+def verify_pep440(version):
+    """
+    Verify if version is PEP440 compliant.
+
+    https://github.com/pypa/packaging/blob/16.7/packaging/version.py#L191
+    """
+
+    try:
+        return packaging.version.Version(version)
+    except packaging.version.InvalidVersion as e:
+        raise e
+
+
+def main() -> None:
+    if len(sys.argv) != 2:
+        raise Exception("New version not specified")
+    base_version = sys.argv[1]
+
+    verify_pep440(base_version)
+    update_base_dep("pyproject.toml", base_version)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/update_pyproject_name.py

@@ -0,0 +1,41 @@
+import os
+import sys
+import re
+
+BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
+
+
+def update_pyproject_name(pyproject_path: str, new_project_name: str) -> None:
+    """Update the project name in pyproject.toml."""
+    filepath = os.path.join(BASE_DIR, pyproject_path)
+    with open(filepath, "r") as file:
+        content = file.read()
+
+    # Regex to match the version line under [tool.poetry]
+    pattern = re.compile(r'(?<=^name = ")[^"]+(?=")', re.MULTILINE)
+
+    if not pattern.search(content):
+        raise Exception(f'Project name not found in "{filepath}"')
+
+    content = pattern.sub(new_project_name, content)
+
+    with open(filepath, "w") as file:
+        file.write(content)
+
+
+def main() -> None:
+    if len(sys.argv) != 3:
+        raise Exception("Must specify project name and build type, e.g. langflow-nightly base")
+    new_project_name = sys.argv[1]
+    build_type = sys.argv[2]
+
+    if build_type == "base":
+        update_pyproject_name("src/backend/base/pyproject.toml", new_project_name)
+    elif build_type == "main":
+        update_pyproject_name("pyproject.toml", new_project_name)
+    else:
+        raise ValueError(f"Invalid build type: {build_type}")
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/update_pyproject_version.py

@@ -0,0 +1,58 @@
+import os
+import sys
+import re
+
+import packaging.version
+
+BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
+
+
+def update_pyproject_version(pyproject_path: str, new_version: str) -> None:
+    """Update the version in pyproject.toml."""
+    filepath = os.path.join(BASE_DIR, pyproject_path)
+    with open(filepath, "r") as file:
+        content = file.read()
+
+    # Regex to match the version line under [tool.poetry]
+    pattern = re.compile(r'(?<=^version = ")[^"]+(?=")', re.MULTILINE)
+
+    if not pattern.search(content):
+        raise Exception(f'Project version not found in "{filepath}"')
+
+    content = pattern.sub(new_version, content)
+
+    with open(filepath, "w") as file:
+        file.write(content)
+
+
+def verify_pep440(version):
+    """
+    Verify if version is PEP440 compliant.
+
+    https://github.com/pypa/packaging/blob/16.7/packaging/version.py#L191
+    """
+
+    try:
+        return packaging.version.Version(version)
+    except packaging.version.InvalidVersion as e:
+        raise e
+
+
+def main() -> None:
+    if len(sys.argv) != 3:
+        raise Exception("New version not specified")
+    new_version = sys.argv[1]
+    build_type = sys.argv[2]
+
+    verify_pep440(new_version)
+
+    if build_type == "base":
+        update_pyproject_version("src/backend/base/pyproject.toml", new_version)
+    elif build_type == "main":
+        update_pyproject_version("pyproject.toml", new_version)
+    else:
+        raise ValueError(f"Invalid build type: {build_type}")
+
+
+if __name__ == "__main__":
+    main()

scripts/update_dependencies.py

@@ -31,9 +31,22 @@ def get_version_from_pypi(package_name):
     return None
 
 
-def update_pyproject_dependency(pyproject_path, version):
+def is_development_release(version):
+    """
+    Determines if the version is a development version based on PEP 440.
+
+    We consider a development version (.devN) as our nightly versions
+    """
+    return "dev" in version
+
+
+def update_pyproject_dependency(pyproject_path, version, is_nightly):
     pattern = re.compile(r'langflow-base = \{ path = "\./src/backend/base", develop = true \}')
-    replacement = f'langflow-base = "^{version}"'
+    if is_nightly:
+        # NOTE: This process can be simplified; see the note in update_lf_base_dependency.py
+        replacement = f'langflow-base-nightly = "{version}"'
+    else:
+        replacement = f'langflow-base = "^{version}"'
     with open(pyproject_path, "r") as file:
         content = file.read()
     content = pattern.sub(replacement, content)
@@ -57,6 +70,9 @@ if __name__ == "__main__":
     langflow_base_path = Path(__file__).resolve().parent / "../src/backend/base/pyproject.toml"
     version = read_version_from_pyproject(langflow_base_path)
     if version:
-        update_pyproject_dependency(pyproject_path, version)
+        # Nightly versions contain "dev"
+        # WARNING: This will cause issues if we release `.dev` versions to `langflow` or `langflow-base`
+        is_nightly = is_development_release(version)
+        update_pyproject_dependency(pyproject_path, version, is_nightly)
     else:
         print("Error: Version not found.")