jyapayne/langflow
Watch
1
0
Fork
You've already forked langflow
 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/dev' into NGNMergeDev

Browse source
This commit is contained in:
anovazzi1 2023-10-06 15:41:42 -03:00
commit 88d91c48d8
274 changed files with 11673 additions and 3904 deletions
Download patch file Download diff file Expand all files Collapse all files

18
.env.example
View file

@ -45,3 +45,21 @@ LANGFLOW_OPEN_BROWSER=
# Values: true, false
# Example: LANGFLOW_REMOVE_API_KEYS=false
LANGFLOW_REMOVE_API_KEYS=
# Whether to use RedisCache or InMemoryCache
# Values: memory, redis
# Example: LANGFLOW_CACHE_TYPE=memory
# If you want to use redis then the following environment variables must be set:
# LANGFLOW_REDIS_HOST (default: localhost)
# LANGFLOW_REDIS_PORT (default: 6379)
# LANGFLOW_REDIS_DB (default: 0)
# LANGFLOW_REDIS_CACHE_EXPIRE (default: 3600)
LANGFLOW_CACHE_TYPE=
# Superuser username
# Example: LANGFLOW_SUPERUSER=admin
LANGFLOW_SUPERUSER=
# Superuser password
# Example: LANGFLOW_SUPERUSER_PASSWORD=123456
LANGFLOW_SUPERUSER_PASSWORD=

44
.github/workflows/ci.yml vendored Normal file
View file

@ -0,0 +1,44 @@
name: "Async API tests"
on:
push:
branches:
- dev
pull_request:
branches:
- dev
- main
jobs:
build-and-test:
runs-on: ubuntu-latest
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Set up Docker
run: docker --version && docker-compose --version
- name: "Create env file"
working-directory: ./deploy
run: |
echo "${{ secrets.ENV_FILE }}" > .env
- name: Build and start services
working-directory: ./deploy
run: docker compose up --exit-code-from tests tests result_backend broker celeryworker db --build
continue-on-error: true
- name: Stop services
run: docker compose down

1
.github/workflows/pre-release.yml vendored
View file

@ -38,6 +38,7 @@ jobs:
token: ${{ secrets.GITHUB_TOKEN }}
draft: false
generateReleaseNotes: true
prerelease: true
tag: v${{ steps.check-version.outputs.version }}
commit: main
- name: Publish to PyPI

8
.github/workflows/release.yml vendored
View file

@ -45,11 +45,3 @@ jobs:
POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
run: |
poetry publish
- name: Trigger build and push on langchain-serve
uses: peter-evans/repository-dispatch@v2
with:
token: ${{ secrets.SERVE_GITHUB_TOKEN }}
repository: jina-ai/langchain-serve
event-type: langflow-push
client-payload: '{"push_token": "${{ secrets.LCSERVE_PUSH_TOKEN }}", "branch": "main"}'

17
.github/workflows/test-lcserve-push.yml vendored
View file

@ -1,17 +0,0 @@
name: Trigger build and push on langchain-serve
on:
workflow_dispatch:
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Trigger build and push on langchain-serve
uses: peter-evans/repository-dispatch@v2
with:
token: ${{ secrets.SERVE_GITHUB_TOKEN }}
repository: jina-ai/langchain-serve
event-type: langflow-push
client-payload: '{"push_token": "${{ secrets.LCSERVE_PUSH_TOKEN }}", "branch": "dev"}'

4
.github/workflows/test.yml vendored
View file

@ -7,7 +7,7 @@ on:
branches: [dev]
env:
POETRY_VERSION: "1.4.0"
POETRY_VERSION: "1.5.0"
jobs:
build:
@ -16,6 +16,8 @@ jobs:
matrix:
python-version:
- "3.10"
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
steps:
- uses: actions/checkout@v3
- name: Install poetry

1
.gitignore vendored
View file

@ -254,3 +254,4 @@ langflow.db
/tmp/*
src/backend/langflow/frontend/
.docker

22
Makefile
View file

@ -19,7 +19,8 @@ coverage:
--cov-report term-missing:skip-covered
tests:
poetry run pytest tests -n auto
@make install_backend
poetry run pytest tests
format:
poetry run black .
@ -41,10 +42,10 @@ run_frontend:
cd src/frontend && npm start
run_cli:
poetry run langflow --path src/frontend/build
poetry run langflow run --path src/frontend/build
run_cli_debug:
poetry run langflow --path src/frontend/build --log-level debug
poetry run langflow run --path src/frontend/build --log-level debug
setup_devcontainer:
make init
@ -60,7 +61,7 @@ frontendc:
make run_frontend
install_backend:
poetry install
poetry install --extras deploy
backend:
make install_backend
@ -69,7 +70,7 @@ backend:
build_and_run:
echo 'Removing dist folder'
rm -rf dist
make build && poetry run pip install dist/*.tar.gz && poetry run langflow
make build && poetry run pip install dist/*.tar.gz && poetry run langflow run
build_and_install:
echo 'Removing dist folder'
@ -86,17 +87,6 @@ build:
poetry build --format sdist
rm -rf src/backend/langflow/frontend
lcserve_push:
make build_frontend
@version=$$(poetry version --short); \
lc-serve push --app langflow.lcserve:app --app-dir . \
--image-name langflow --image-tag $${version} --verbose --public
lcserve_deploy:
@:$(if $(uses),,$(error `uses` is not set. Please run `make uses=... lcserve_deploy`))
lc-serve deploy jcloud --app langflow.lcserve:app --app-dir . \
--uses $(uses) --config src/backend/langflow/jcloud.yml --verbose
dev:
make install_frontend
ifeq ($(build),1)

118
README.md
View file

@ -36,8 +36,6 @@
- [Environment Variables](#environment-variables)
- [Deployment](#deployment)
- [Deploy Langflow on Google Cloud Platform](#deploy-langflow-on-google-cloud-platform)
- [Deploy Langflow on Jina AI Cloud](#deploy-langflow-on-jina-ai-cloud)
- [API Usage](#api-usage)
- [Deploy on Railway](#deploy-on-railway)
- [Deploy on Render](#deploy-on-render)
- [🎨 Creating Flows](#-creating-flows)
@ -78,7 +76,7 @@ python -m langflow
or
```shell
langflow # or langflow --help
langflow run # or langflow --help
```
### HuggingFace Spaces
@ -94,7 +92,7 @@ Langflow provides a command-line interface (CLI) for easy management and configu
You can run the Langflow using the following command:
```shell
langflow [OPTIONS]
langflow run [OPTIONS]
```
Each option is detailed below:
@ -110,7 +108,6 @@ Each option is detailed below:
- `--components-path`: Specifies the path to the directory containing custom components. Can be set using the `LANGFLOW_COMPONENTS_PATH` environment variable. The default is `langflow/components`.
- `--log-file`: Specifies the path to the log file. Can be set using the `LANGFLOW_LOG_FILE` environment variable. The default is `logs/langflow.log`.
- `--cache`: Selects the type of cache to use. Options are `InMemoryCache` and `SQLiteCache`. Can be set using the `LANGFLOW_LANGCHAIN_CACHE` environment variable. The default is `SQLiteCache`.
- `--jcloud/--no-jcloud`: Toggles the option to deploy on Jina AI Cloud. The default is `no-jcloud`.
- `--dev/--no-dev`: Toggles the development mode. The default is `no-dev`.
- `--path`: Specifies the path to the frontend directory containing build files. This option is for development purposes only. Can be set using the `LANGFLOW_FRONTEND_PATH` environment variable.
- `--open-browser/--no-open-browser`: Toggles the option to open the browser after starting the server. Can be set using the `LANGFLOW_OPEN_BROWSER` environment variable. The default is `open-browser`.
@ -134,118 +131,9 @@ Alternatively, click the **"Open in Cloud Shell"** button below to launch Google
[![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.svg)](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/logspace-ai/langflow&working_dir=scripts&shellonly=true&tutorial=walkthroughtutorial_spot.md)
## Deploy Langflow on [Jina AI Cloud](https://github.com/jina-ai/langchain-serve)
Langflow integrates with langchain-serve to provide a one-command deployment to Jina AI Cloud.
Start by installing `langchain-serve` with
```bash
pip install langflow[deploy]
# or
pip install -U langchain-serve
```
Then, run:
```bash
langflow --jcloud
```
```text
🎉 Langflow server successfully deployed on Jina AI Cloud 🎉
🔗 Click on the link to open the server (please allow ~1-2 minutes for the server to startup): https://<your-app>.wolf.jina.ai/
📖 Read more about managing the server: https://github.com/jina-ai/langchain-serve
```
<details>
<summary>Show complete (example) output</summary>
```text
🚀 Deploying Langflow server on Jina AI Cloud
╭───────────────────────── 🎉 Flow is available! ──────────────────────────╮
│ │
│ ID langflow-e3dd8820ec │
│ Gateway (Websocket) wss://langflow-e3dd8820ec.wolf.jina.ai │
│ Dashboard https://dashboard.wolf.jina.ai/flow/e3dd8820ec │
│ │
╰──────────────────────────────────────────────────────────────────────────╯
╭──────────────┬──────────────────────────────────────────────────────────────────────────────╮
│ App ID │ langflow-e3dd8820ec │
├──────────────┼──────────────────────────────────────────────────────────────────────────────┤
│ Phase │ Serving │
├──────────────┼──────────────────────────────────────────────────────────────────────────────┤
│ Endpoint │ wss://langflow-e3dd8820ec.wolf.jina.ai │
├──────────────┼──────────────────────────────────────────────────────────────────────────────┤
│ App logs │ dashboards.wolf.jina.ai │
├──────────────┼──────────────────────────────────────────────────────────────────────────────┤
│ Swagger UI │ https://langflow-e3dd8820ec.wolf.jina.ai/docs │
├──────────────┼──────────────────────────────────────────────────────────────────────────────┤
│ OpenAPI JSON │ https://langflow-e3dd8820ec.wolf.jina.ai/openapi.json │
╰──────────────┴──────────────────────────────────────────────────────────────────────────────╯
🎉 Langflow server successfully deployed on Jina AI Cloud 🎉
🔗 Click on the link to open the server (please allow ~1-2 minutes for the server to startup): https://langflow-e3dd8820ec.wolf.jina.ai/
📖 Read more about managing the server: https://github.com/jina-ai/langchain-serve
```
</details>
#### API Usage
You can use Langflow directly on your browser, or use the API endpoints on Jina AI Cloud to interact with the server.
<details>
<summary>Show API usage (with python)</summary>
```python
import requests
BASE_API_URL = "https://langflow-e3dd8820ec.wolf.jina.ai/api/v1/predict"
FLOW_ID = "864c4f98-2e59-468b-8e13-79cd8da07468"
# You can tweak the flow by adding a tweaks dictionary
# e.g {"OpenAI-XXXXX": {"model_name": "gpt-4"}}
TWEAKS = {
"ChatOpenAI-g4jEr": {},
"ConversationChain-UidfJ": {}
}
def run_flow(message: str, flow_id: str, tweaks: dict = None) -> dict:
"""
Run a flow with a given message and optional tweaks.
:param message: The message to send to the flow
:param flow_id: The ID of the flow to run
:param tweaks: Optional tweaks to customize the flow
:return: The JSON response from the flow
"""
api_url = f"{BASE_API_URL}/{flow_id}"
payload = {"message": message}
if tweaks:
payload["tweaks"] = tweaks
response = requests.post(api_url, json=payload)
return response.json()
# Setup any tweaks you want to apply to the flow
print(run_flow("Your message", flow_id=FLOW_ID, tweaks=TWEAKS))
```
```json
{
"result": "Great choice! Bangalore in the 1920s was a vibrant city with a rich cultural and political scene. Here are some suggestions for things to see and do:\n\n1. Visit the Bangalore Palace - built in 1887, this stunning palace is a perfect example of Tudor-style architecture. It was home to the Maharaja of Mysore and is now open to the public.\n\n2. Attend a performance at the Ravindra Kalakshetra - this cultural center was built in the 1920s and is still a popular venue for music and dance performances.\n\n3. Explore the neighborhoods of Basavanagudi and Malleswaram - both of these areas have retained much of their old-world charm and are great places to walk around and soak up the atmosphere.\n\n4. Check out the Bangalore Club - founded in 1868, this exclusive social club was a favorite haunt of the British expat community in the 1920s.\n\n5. Attend a meeting of the Indian National Congress - founded in 1885, the INC was a major force in the Indian independence movement and held many meetings and rallies in Bangalore in the 1920s.\n\nHope you enjoy your trip to 1920s Bangalore!"
}
```
</details>
> Read more about resource customization, cost, and management of Langflow apps on Jina AI Cloud in the **[langchain-serve](https://github.com/jina-ai/langchain-serve)** repository.
## Deploy on Railway
[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/Emy2sU?referralCode=MnPSdg)
[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/JMXEWp?referralCode=MnPSdg)
## Deploy on Render

97
base.Dockerfile Normal file
View file

@ -0,0 +1,97 @@
# syntax=docker/dockerfile:1
# Keep this syntax directive! It's used to enable Docker BuildKit
# Based on https://github.com/python-poetry/poetry/discussions/1879?sort=top#discussioncomment-216865
# but I try to keep it updated (see history)
################################
# PYTHON-BASE
# Sets up all our shared environment variables
################################
FROM python:3.10-slim as python-base
# python
ENV PYTHONUNBUFFERED=1 \
# prevents python creating .pyc files
PYTHONDONTWRITEBYTECODE=1 \
\
# pip
PIP_DISABLE_PIP_VERSION_CHECK=on \
PIP_DEFAULT_TIMEOUT=100 \
\
# poetry
# https://python-poetry.org/docs/configuration/#using-environment-variables
POETRY_VERSION=1.5.1 \
# make poetry install to this location
POETRY_HOME="/opt/poetry" \
# make poetry create the virtual environment in the project's root
# it gets named `.venv`
POETRY_VIRTUALENVS_IN_PROJECT=true \
# do not ask any interactive question
POETRY_NO_INTERACTION=1 \
\
# paths
# this is where our requirements + virtual environment will live
PYSETUP_PATH="/opt/pysetup" \
VENV_PATH="/opt/pysetup/.venv"
# prepend poetry and venv to path
ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
################################
# BUILDER-BASE
# Used to build deps + create our virtual environment
################################
FROM python-base as builder-base
RUN apt-get update \
&& apt-get install --no-install-recommends -y \
# deps for installing poetry
curl \
# deps for building python deps
build-essential
# install poetry - respects $POETRY_VERSION & $POETRY_HOME
# The --mount will mount the buildx cache directory to where
# Poetry and Pip store their cache so that they can re-use it
RUN --mount=type=cache,target=/root/.cache \
curl -sSL https://install.python-poetry.org | python3 -
# copy project requirement files here to ensure they will be cached.
WORKDIR $PYSETUP_PATH
COPY poetry.lock pyproject.toml ./
COPY ./src/backend/langflow/main.py ./src/backend/langflow/main.py
# Copy README.md to the build context
COPY README.md .
# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
RUN --mount=type=cache,target=/root/.cache \
poetry install --without dev --extras deploy
################################
# DEVELOPMENT
# Image used during development / testing
################################
FROM python-base as development
WORKDIR $PYSETUP_PATH
# copy in our built poetry + venv
COPY --from=builder-base $POETRY_HOME $POETRY_HOME
COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
# Copy just one file to avoid rebuilding the whole image
COPY ./src/backend/langflow/__init__.py ./src/backend/langflow/__init__.py
# quicker install as runtime deps are already installed
RUN --mount=type=cache,target=/root/.cache \
poetry install --with=dev --extras deploy
# copy in our app code
COPY ./src/backend ./src/backend
RUN --mount=type=cache,target=/root/.cache \
poetry install --with=dev --extras deploy
COPY ./tests ./tests=

57
deploy/.env.example Normal file
View file

@ -0,0 +1,57 @@
DOMAIN=localhost
STACK_NAME=langflow-stack
ENVIRONMENT=development
TRAEFIK_PUBLIC_NETWORK=traefik-public
TRAEFIK_TAG=langflow-traefik
TRAEFIK_PUBLIC_TAG=traefik-public
# RabbitMQ configuration
RABBITMQ_DEFAULT_USER=langflow
RABBITMQ_DEFAULT_PASS=langflow
# Database configuration
DB_USER=langflow
DB_PASSWORD=langflow
DB_HOST=db
DB_PORT=5432
DB_NAME=langflow
# Logging configuration
LOG_LEVEL=debug
# DB configuration
POSTGRES_USER=langflow
POSTGRES_PASSWORD=langflow
POSTGRES_DB=langflow
POSTGRES_PORT=5432
# Flower configuration
LANGFLOW_CACHE_TYPE=redis
LANGFLOW_REDIS_HOST=result_backend
LANGFLOW_REDIS_PORT=6379
LANGFLOW_REDIS_DB=0
LANGFLOW_REDIS_EXPIRE=3600
LANGFLOW_REDIS_PASSWORD=
FLOWER_UNAUTHENTICATED_API=True
BROKER_URL=amqp://langflow:langflow@broker:5672
RESULT_BACKEND=redis://result_backend:6379/0
C_FORCE_ROOT="true"
# Frontend configuration
VITE_PROXY_TARGET=http://backend:7860/api/
BACKEND_URL=http://backend:7860
# PGAdmin configuration
PGADMIN_DEFAULT_EMAIL=admin@admin.com
PGADMIN_DEFAULT_PASSWORD=admin
# OpenAI configuration (for testing purposes)
OPENAI_API_KEY=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
# Superuser configuration
LANGFLOW_SUPERUSER=superuser
LANGFLOW_SUPERUSER_PASSWORD=superuser
# New user configuration
LANGFLOW_NEW_USER_IS_ACTIVE=False

1
deploy/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
pgadmin

92
deploy/base.Dockerfile Normal file
View file

@ -0,0 +1,92 @@
# syntax=docker/dockerfile:1
# Keep this syntax directive! It's used to enable Docker BuildKit
# Based on https://github.com/python-poetry/poetry/discussions/1879?sort=top#discussioncomment-216865
# but I try to keep it updated (see history)
################################
# PYTHON-BASE
# Sets up all our shared environment variables
################################
FROM python:3.10-slim as python-base
# python
ENV PYTHONUNBUFFERED=1 \
# prevents python creating .pyc files
PYTHONDONTWRITEBYTECODE=1 \
\
# pip
PIP_DISABLE_PIP_VERSION_CHECK=on \
PIP_DEFAULT_TIMEOUT=100 \
\
# poetry
# https://python-poetry.org/docs/configuration/#using-environment-variables
POETRY_VERSION=1.5.1 \
# make poetry install to this location
POETRY_HOME="/opt/poetry" \
# make poetry create the virtual environment in the project's root
# it gets named `.venv`
POETRY_VIRTUALENVS_IN_PROJECT=true \
# do not ask any interactive question
POETRY_NO_INTERACTION=1 \
\
# paths
# this is where our requirements + virtual environment will live
PYSETUP_PATH="/opt/pysetup" \
VENV_PATH="/opt/pysetup/.venv"
# prepend poetry and venv to path
ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
################################
# BUILDER-BASE
# Used to build deps + create our virtual environment
################################
FROM python-base as builder-base
RUN apt-get update \
&& apt-get install --no-install-recommends -y \
# deps for installing poetry
curl \
# deps for building python deps
build-essential
# install poetry - respects $POETRY_VERSION & $POETRY_HOME
# The --mount will mount the buildx cache directory to where
# Poetry and Pip store their cache so that they can re-use it
RUN --mount=type=cache,target=/root/.cache \
curl -sSL https://install.python-poetry.org | python3 -
# copy project requirement files here to ensure they will be cached.
WORKDIR $PYSETUP_PATH
COPY ./poetry.lock ./pyproject.toml ./
# Copy README.md to the build context
COPY ./README.md ./
# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
RUN --mount=type=cache,target=/root/.cache \
poetry install --without dev --extras deploy
################################
# DEVELOPMENT
# Image used during development / testing
################################
FROM python-base as development
WORKDIR $PYSETUP_PATH
# copy in our built poetry + venv
COPY --from=builder-base $POETRY_HOME $POETRY_HOME
COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
# Copy just one file to avoid rebuilding the whole image
COPY ./src/backend/langflow/__init__.py ./src/backend/langflow/__init__.py
# quicker install as runtime deps are already installed
RUN --mount=type=cache,target=/root/.cache \
poetry install --with=dev --extras deploy
# copy in our app code
COPY ./src/backend ./src/backend
COPY ./tests ./tests

67
deploy/docker-compose.override.yml Normal file
View file

@ -0,0 +1,67 @@
version: "3.8"
services:
proxy:
ports:
- "80:80"
- "8090:8080"
command:
# Enable Docker in Traefik, so that it reads labels from Docker services
- --providers.docker
# Add a constraint to only use services with the label for this stack
# from the env var TRAEFIK_TAG
- --providers.docker.constraints=Label(`traefik.constraint-label-stack`, `${TRAEFIK_TAG?Variable not set}`)
# Do not expose all Docker services, only the ones explicitly exposed
- --providers.docker.exposedbydefault=false
# Disable Docker Swarm mode for local development
# - --providers.docker.swarmmode
# Enable the access log, with HTTP requests
- --accesslog
# Enable the Traefik log, for configurations and errors
- --log
# Enable the Dashboard and API
- --api
# Enable the Dashboard and API in insecure mode for local development
- --api.insecure=true
labels:
- traefik.enable=true
- traefik.http.routers.${STACK_NAME?Variable not set}-traefik-public-http.rule=Host(`${DOMAIN?Variable not set}`)
- traefik.http.services.${STACK_NAME?Variable not set}-traefik-public.loadbalancer.server.port=80
result_backend:
ports:
- "6379:6379"
pgadmin:
ports:
- "5050:5050"
flower:
ports:
- "5555:5555"
backend:
labels:
- traefik.enable=true
- traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-backend-http.rule=PathPrefix(`/api/v1`) || PathPrefix(`/docs`) || PathPrefix(`/health`)
- traefik.http.services.${STACK_NAME?Variable not set}-backend.loadbalancer.server.port=7860
frontend:
labels:
- traefik.enable=true
- traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-frontend-http.rule=PathPrefix(`/`)
- traefik.http.services.${STACK_NAME?Variable not set}-frontend.loadbalancer.server.port=80
celeryworker:
labels:
- traefik.enable=true
- traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-celeryworker-http.rule=PathPrefix(`/api/v1`) || PathPrefix(`/docs`) || PathPrefix(`/health`)
- traefik.http.services.${STACK_NAME?Variable not set}-celeryworker.loadbalancer.server.port=7860
networks:
traefik-public:
# For local dev, don't expect an external Traefik network
external: false

277
deploy/docker-compose.with_tests.yml Normal file
View file

@ -0,0 +1,277 @@
version: "3.8"
services:
proxy:
image: traefik:v3.0
env_file:
- .env
networks:
- ${TRAEFIK_PUBLIC_NETWORK?Variable not set}
- default
volumes:
- /var/run/docker.sock:/var/run/docker.sock
command:
# Enable Docker in Traefik, so that it reads labels from Docker services
- --providers.docker
# Add a constraint to only use services with the label for this stack
# from the env var TRAEFIK_TAG
- --providers.docker.constraints=Label(`traefik.constraint-label-stack`, `${TRAEFIK_TAG?Variable not set}`)
# Do not expose all Docker services, only the ones explicitly exposed
- --providers.docker.exposedbydefault=false
# Enable the access log, with HTTP requests
- --accesslog
# Enable the Traefik log, for configurations and errors
- --log
# Enable the Dashboard and API
- --api
deploy:
placement:
constraints:
- node.role == manager
labels:
# Enable Traefik for this service, to make it available in the public network
- traefik.enable=true
# Use the traefik-public network (declared below)
- traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK?Variable not set}
# Use the custom label "traefik.constraint-label=traefik-public"
# This public Traefik will only use services with this label
- traefik.constraint-label=${TRAEFIK_PUBLIC_TAG?Variable not set}
# traefik-http set up only to use the middleware to redirect to https
- traefik.http.middlewares.${STACK_NAME?Variable not set}-https-redirect.redirectscheme.scheme=https
- traefik.http.middlewares.${STACK_NAME?Variable not set}-https-redirect.redirectscheme.permanent=true
# Handle host with and without "www" to redirect to only one of them
# Uses environment variable DOMAIN
# To disable www redirection remove the Host() you want to discard, here and
# below for HTTPS
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-http.rule=Host(`${DOMAIN?Variable not set}`) || Host(`www.${DOMAIN?Variable not set}`)
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-http.entrypoints=http
# traefik-https the actual router using HTTPS
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.rule=Host(`${DOMAIN?Variable not set}`) || Host(`www.${DOMAIN?Variable not set}`)
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.entrypoints=https
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.tls=true
# Use the "le" (Let's Encrypt) resolver created below
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.tls.certresolver=le
# Define the port inside of the Docker service to use
- traefik.http.services.${STACK_NAME?Variable not set}-proxy.loadbalancer.server.port=80
# Handle domain with and without "www" to redirect to only one
# To disable www redirection remove the next line
- traefik.http.middlewares.${STACK_NAME?Variable not set}-www-redirect.redirectregex.regex=^https?://(www.)?(${DOMAIN?Variable not set})/(.*)
# Redirect a domain with www to non-www
# To disable it remove the next line
- traefik.http.middlewares.${STACK_NAME?Variable not set}-www-redirect.redirectregex.replacement=https://${DOMAIN?Variable not set}/$${3}
# Redirect a domain without www to www
# To enable it remove the previous line and uncomment the next
# - traefik.http.middlewares.${STACK_NAME}-www-redirect.redirectregex.replacement=https://www.${DOMAIN}/$${3}
# Middleware to redirect www, to disable it remove the next line
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.middlewares=${STACK_NAME?Variable not set}-www-redirect
# Middleware to redirect www, and redirect HTTP to HTTPS
# to disable www redirection remove the section: ${STACK_NAME?Variable not set}-www-redirect,
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-http.middlewares=${STACK_NAME?Variable not set}-www-redirect,${STACK_NAME?Variable not set}-https-redirect
backend: &backend
image: "ogabrielluiz/langflow:latest"
build:
context: ../
dockerfile: base.Dockerfile
depends_on:
- db
- broker
- result_backend
env_file:
- .env
volumes:
- ../:/app
- ./startup-backend.sh:/startup-backend.sh # Ensure the paths match
command: /startup-backend.sh # Fixed the path
healthcheck:
test: "exit 0"
deploy:
labels:
- traefik.enable=true
- traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-backend-http.rule=PathPrefix(`/api/v1`) || PathPrefix(`/docs`) || PathPrefix(`/health`)
- traefik.http.services.${STACK_NAME?Variable not set}-backend.loadbalancer.server.port=7860
db:
image: postgres:15.4
volumes:
- app-db-data:/var/lib/postgresql/data/pgdata
environment:
- PGDATA=/var/lib/postgresql/data/pgdata
deploy:
placement:
constraints:
- node.labels.app-db-data == true
healthcheck:
test: "exit 0"
env_file:
- .env
pgadmin:
image: dpage/pgadmin4
networks:
- ${TRAEFIK_PUBLIC_NETWORK?Variable not set}
- default
volumes:
- pgadmin-data:/var/lib/pgadmin
env_file:
- .env
deploy:
labels:
- traefik.enable=true
- traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK?Variable not set}
- traefik.constraint-label=${TRAEFIK_PUBLIC_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-http.rule=Host(`pgadmin.${DOMAIN?Variable not set}`)
- traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-http.entrypoints=http
- traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-http.middlewares=${STACK_NAME?Variable not set}-https-redirect
- traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-https.rule=Host(`pgadmin.${DOMAIN?Variable not set}`)
- traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-https.entrypoints=https
- traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-https.tls=true
- traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-https.tls.certresolver=le
- traefik.http.services.${STACK_NAME?Variable not set}-pgadmin.loadbalancer.server.port=5050
result_backend:
image: redis:6.2.5
env_file:
- .env
# ports:
# - 6379:6379
healthcheck:
test: "exit 0"
celeryworker:
<<: *backend
env_file:
- .env
build:
context: ../
dockerfile: base.Dockerfile
command: celery -A langflow.worker.celery_app worker --loglevel=INFO --concurrency=1 -n lf-worker@%h
healthcheck:
test: "exit 0"
deploy:
replicas: 1
flower:
<<: *backend
env_file:
- .env
networks:
- default
build:
context: ../
dockerfile: base.Dockerfile
environment:
- FLOWER_PORT=5555
command: /bin/sh -c "celery -A langflow.worker.celery_app --broker=${BROKER_URL?Variable not set} flower --port=5555"
deploy:
labels:
- traefik.enable=true
- traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK?Variable not set}
- traefik.constraint-label=${TRAEFIK_PUBLIC_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-flower-http.rule=Host(`flower.${DOMAIN?Variable not set}`)
- traefik.http.routers.${STACK_NAME?Variable not set}-flower-http.entrypoints=http
- traefik.http.routers.${STACK_NAME?Variable not set}-flower-http.middlewares=${STACK_NAME?Variable not set}-https-redirect
- traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.rule=Host(`flower.${DOMAIN?Variable not set}`)
- traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.entrypoints=https
- traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.tls=true
- traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.tls.certresolver=le
- traefik.http.services.${STACK_NAME?Variable not set}-flower.loadbalancer.server.port=5555
frontend:
image: "ogabrielluiz/langflow_frontend:latest"
env_file:
- .env
# user: your-non-root-user
build:
context: ../src/frontend
dockerfile: Dockerfile
args:
- BACKEND_URL=http://backend:7860
restart: on-failure
deploy:
labels:
- traefik.enable=true
- traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-frontend-http.rule=PathPrefix(`/`)
- traefik.http.services.${STACK_NAME?Variable not set}-frontend.loadbalancer.server.port=80
broker:
# RabbitMQ management console
image: rabbitmq:3-management
environment:
- RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER:-admin}
- RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS:-admin}
volumes:
- rabbitmq_data:/etc/rabbitmq/
- rabbitmq_data:/var/lib/rabbitmq/
- rabbitmq_log:/var/log/rabbitmq/
ports:
- 5672:5672
- 15672:15672
prometheus:
image: prom/prometheus:v2.37.9
env_file:
- .env
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
command:
- "--config.file=/etc/prometheus/prometheus.yml"
# ports:
# - 9090:9090
healthcheck:
test: "exit 0"
deploy:
labels:
- traefik.enable=true
- traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-prometheus-http.rule=PathPrefix(`/metrics`)
- traefik.http.services.${STACK_NAME?Variable not set}-prometheus.loadbalancer.server.port=9090
grafana:
image: grafana/grafana:8.2.6
env_file:
- .env
# ports:
# - 3000:3000
volumes:
- grafana_data:/var/lib/grafana
deploy:
labels:
- traefik.enable=true
- traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-grafana-http.rule=PathPrefix(`/grafana`)
- traefik.http.services.${STACK_NAME?Variable not set}-grafana.loadbalancer.server.port=3000
tests:
extends:
file: docker-compose.yml
service: backend
env_file:
- .env
build:
context: ../
dockerfile: base.Dockerfile
command: pytest -vv
healthcheck:
test: "exit 0"
# override deploy labels to avoid conflicts with the backend service
labels:
- traefik.enable=true
- traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-tests-http.rule=PathPrefix(`/api/v1`) || PathPrefix(`/docs`) || PathPrefix(`/health`)
- traefik.http.services.${STACK_NAME?Variable not set}-tests.loadbalancer.server.port=7861
volumes:
grafana_data:
app-db-data:
rabbitmq_data:
rabbitmq_log:
pgadmin-data:
networks:
traefik-public:
# Allow setting it to false for testing
external: false # ${TRAEFIK_PUBLIC_NETWORK_IS_EXTERNAL-true}

258
deploy/docker-compose.yml Normal file
View file

@ -0,0 +1,258 @@
version: "3.8"
services:
proxy:
image: traefik:v3.0
env_file:
- .env
networks:
- ${TRAEFIK_PUBLIC_NETWORK?Variable not set}
- default
volumes:
- /var/run/docker.sock:/var/run/docker.sock
command:
# Enable Docker in Traefik, so that it reads labels from Docker services
- --providers.docker
# Add a constraint to only use services with the label for this stack
# from the env var TRAEFIK_TAG
- --providers.docker.constraints=Label(`traefik.constraint-label-stack`, `${TRAEFIK_TAG?Variable not set}`)
# Do not expose all Docker services, only the ones explicitly exposed
- --providers.docker.exposedbydefault=false
# Enable the access log, with HTTP requests
- --accesslog
# Enable the Traefik log, for configurations and errors
- --log
# Enable the Dashboard and API
- --api
deploy:
placement:
constraints:
- node.role == manager
labels:
# Enable Traefik for this service, to make it available in the public network
- traefik.enable=true
# Use the traefik-public network (declared below)
- traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK?Variable not set}
# Use the custom label "traefik.constraint-label=traefik-public"
# This public Traefik will only use services with this label
- traefik.constraint-label=${TRAEFIK_PUBLIC_TAG?Variable not set}
# traefik-http set up only to use the middleware to redirect to https
- traefik.http.middlewares.${STACK_NAME?Variable not set}-https-redirect.redirectscheme.scheme=https
- traefik.http.middlewares.${STACK_NAME?Variable not set}-https-redirect.redirectscheme.permanent=true
# Handle host with and without "www" to redirect to only one of them
# Uses environment variable DOMAIN
# To disable www redirection remove the Host() you want to discard, here and
# below for HTTPS
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-http.rule=Host(`${DOMAIN?Variable not set}`) || Host(`www.${DOMAIN?Variable not set}`)
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-http.entrypoints=http
# traefik-https the actual router using HTTPS
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.rule=Host(`${DOMAIN?Variable not set}`) || Host(`www.${DOMAIN?Variable not set}`)
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.entrypoints=https
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.tls=true
# Use the "le" (Let's Encrypt) resolver created below
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.tls.certresolver=le
# Define the port inside of the Docker service to use
- traefik.http.services.${STACK_NAME?Variable not set}-proxy.loadbalancer.server.port=80
# Handle domain with and without "www" to redirect to only one
# To disable www redirection remove the next line
- traefik.http.middlewares.${STACK_NAME?Variable not set}-www-redirect.redirectregex.regex=^https?://(www.)?(${DOMAIN?Variable not set})/(.*)
# Redirect a domain with www to non-www
# To disable it remove the next line
- traefik.http.middlewares.${STACK_NAME?Variable not set}-www-redirect.redirectregex.replacement=https://${DOMAIN?Variable not set}/$${3}
# Redirect a domain without www to www
# To enable it remove the previous line and uncomment the next
# - traefik.http.middlewares.${STACK_NAME}-www-redirect.redirectregex.replacement=https://www.${DOMAIN}/$${3}
# Middleware to redirect www, to disable it remove the next line
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.middlewares=${STACK_NAME?Variable not set}-www-redirect
# Middleware to redirect www, and redirect HTTP to HTTPS
# to disable www redirection remove the section: ${STACK_NAME?Variable not set}-www-redirect,
- traefik.http.routers.${STACK_NAME?Variable not set}-proxy-http.middlewares=${STACK_NAME?Variable not set}-www-redirect,${STACK_NAME?Variable not set}-https-redirect
backend: &backend
image: "ogabrielluiz/langflow:latest"
build:
context: ../
dockerfile: base.Dockerfile
depends_on:
- db
- broker
- result_backend
env_file:
- .env
volumes:
- ../:/app
- ./startup-backend.sh:/startup-backend.sh # Ensure the paths match
command: /startup-backend.sh # Fixed the path
healthcheck:
test: "exit 0"
deploy:
labels:
- traefik.enable=true
- traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-backend-http.rule=PathPrefix(`/api/v1`) || PathPrefix(`/docs`) || PathPrefix(`/health`)
- traefik.http.services.${STACK_NAME?Variable not set}-backend.loadbalancer.server.port=7860
db:
image: postgres:15.4
volumes:
- app-db-data:/var/lib/postgresql/data/pgdata
environment:
- PGDATA=/var/lib/postgresql/data/pgdata
deploy:
placement:
constraints:
- node.labels.app-db-data == true
healthcheck:
test: "exit 0"
env_file:
- .env
pgadmin:
image: dpage/pgadmin4
networks:
- ${TRAEFIK_PUBLIC_NETWORK?Variable not set}
- default
volumes:
- pgadmin-data:/var/lib/pgadmin
env_file:
- .env
deploy:
labels:
- traefik.enable=true
- traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK?Variable not set}
- traefik.constraint-label=${TRAEFIK_PUBLIC_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-http.rule=Host(`pgadmin.${DOMAIN?Variable not set}`)
- traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-http.entrypoints=http
- traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-http.middlewares=${STACK_NAME?Variable not set}-https-redirect
- traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-https.rule=Host(`pgadmin.${DOMAIN?Variable not set}`)
- traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-https.entrypoints=https
- traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-https.tls=true
- traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-https.tls.certresolver=le
- traefik.http.services.${STACK_NAME?Variable not set}-pgadmin.loadbalancer.server.port=5050
result_backend:
image: redis:6.2.5
env_file:
- .env
# ports:
# - 6379:6379
healthcheck:
test: "exit 0"
celeryworker:
<<: *backend
env_file:
- .env
build:
context: ../
dockerfile: base.Dockerfile
command: celery -A langflow.worker.celery_app worker --loglevel=INFO --concurrency=1 -n lf-worker@%h
healthcheck:
test: "exit 0"
deploy:
replicas: 1
flower:
<<: *backend
env_file:
- .env
networks:
- default
build:
context: ../
dockerfile: base.Dockerfile
environment:
- FLOWER_PORT=5555
command: /bin/sh -c "celery -A langflow.worker.celery_app --broker=${BROKER_URL?Variable not set} flower --port=5555"
deploy:
labels:
- traefik.enable=true
- traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK?Variable not set}
- traefik.constraint-label=${TRAEFIK_PUBLIC_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-flower-http.rule=Host(`flower.${DOMAIN?Variable not set}`)
- traefik.http.routers.${STACK_NAME?Variable not set}-flower-http.entrypoints=http
- traefik.http.routers.${STACK_NAME?Variable not set}-flower-http.middlewares=${STACK_NAME?Variable not set}-https-redirect
- traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.rule=Host(`flower.${DOMAIN?Variable not set}`)
- traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.entrypoints=https
- traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.tls=true
- traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.tls.certresolver=le
- traefik.http.services.${STACK_NAME?Variable not set}-flower.loadbalancer.server.port=5555
frontend:
image: "ogabrielluiz/langflow_frontend:latest"
env_file:
- .env
# user: your-non-root-user
build:
context: ../src/frontend
dockerfile: Dockerfile
args:
- BACKEND_URL=http://backend:7860
restart: on-failure
deploy:
labels:
- traefik.enable=true
- traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-frontend-http.rule=PathPrefix(`/`)
- traefik.http.services.${STACK_NAME?Variable not set}-frontend.loadbalancer.server.port=80
broker:
# RabbitMQ management console
image: rabbitmq:3-management
environment:
- RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER:-admin}
- RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS:-admin}
volumes:
- rabbitmq_data:/etc/rabbitmq/
- rabbitmq_data:/var/lib/rabbitmq/
- rabbitmq_log:/var/log/rabbitmq/
ports:
- 5672:5672
- 15672:15672
prometheus:
image: prom/prometheus:v2.37.9
env_file:
- .env
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
command:
- "--config.file=/etc/prometheus/prometheus.yml"
# ports:
# - 9090:9090
healthcheck:
test: "exit 0"
deploy:
labels:
- traefik.enable=true
- traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-prometheus-http.rule=PathPrefix(`/metrics`)
- traefik.http.services.${STACK_NAME?Variable not set}-prometheus.loadbalancer.server.port=9090
grafana:
image: grafana/grafana:8.2.6
env_file:
- .env
# ports:
# - 3000:3000
volumes:
- grafana_data:/var/lib/grafana
deploy:
labels:
- traefik.enable=true
- traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set}
- traefik.http.routers.${STACK_NAME?Variable not set}-grafana-http.rule=PathPrefix(`/grafana`)
- traefik.http.services.${STACK_NAME?Variable not set}-grafana.loadbalancer.server.port=3000
volumes:
grafana_data:
app-db-data:
rabbitmq_data:
rabbitmq_log:
pgadmin-data:
networks:
traefik-public:
# Allow setting it to false for testing
external: false # ${TRAEFIK_PUBLIC_NETWORK_IS_EXTERNAL-true}

11
deploy/prometheus.yml Normal file
View file

@ -0,0 +1,11 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
scrape_configs:
- job_name: prometheus
static_configs:
- targets: ["prometheus:9090"]
- job_name: flower
static_configs:
- targets: ["flower:5555"]

17
deploy/startup-backend.sh Executable file
View file

@ -0,0 +1,17 @@
#!/bin/bash
export LANGFLOW_DATABASE_URL="postgresql://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}"
# Your command to start the backend
# If the ENVIRONMENT variable is set to "development", then start the backend in development mode
# else start the backend in production mode with guvicorn
if [ "$ENVIRONMENT" = "development" ]; then
echo "Starting backend in development mode"
exec python -m uvicorn --factory langflow.main:create_app --host 0.0.0.0 --port 7860 --log-level ${LOG_LEVEL:-info} --workers 2 --reload
else
echo "Starting backend in production mode"
exec langflow run --host 0.0.0.0 --port 7860 --log-level ${LOG_LEVEL:-info} --workers -1 --backend-only
fi

3
docs/docs/components/custom.mdx
View file

@ -33,6 +33,7 @@ The CustomComponent class serves as the foundation for creating custom component
| Supported Types |
| --------------------------------------------------------- |
| _`str`_, _`int`_, _`float`_, _`bool`_, _`list`_, _`dict`_ |
| _`langflow.field_typing.NestedDict`_ |
| _`langchain.chains.base.Chain`_ |
| _`langchain.PromptTemplate`_ |
| _`langchain.llms.base.BaseLLM`_ |
@ -44,6 +45,8 @@ The CustomComponent class serves as the foundation for creating custom component
| _`langchain.embeddings.base.Embeddings`_ |
| _`langchain.schema.BaseRetriever`_ |
The difference between _`dict`_ and _`langflow.field_typing.NestedDict`_ is that one adds a simple key-value pair field, while the other opens a more robust dictionary editor.
<Admonition type="info">
Unlike Langchain types, base Python types do not add a
[handle](../guidelines/components) to the field by default. To add handles,

32
docs/docs/components/text-splitters.mdx
View file

@ -1,11 +1,13 @@
import Admonition from '@theme/Admonition';
import Admonition from "@theme/Admonition";
# Text Splitters
<Admonition type="caution" icon="🚧" title="ZONE UNDER CONSTRUCTION">
<p>
We appreciate your understanding as we polish our documentation – it may contain some rough edges. Share your feedback or report issues to help us improve! 🛠️📝
</p>
<p>
We appreciate your understanding as we polish our documentation – it may
contain some rough edges. Share your feedback or report issues to help us
improve! 🛠️📝
</p>
</Admonition>
A text splitter is a tool that divides a document or text into smaller chunks or segments. It is used to break down large texts into more manageable pieces for analysis or processing.
@ -22,13 +24,13 @@ The `CharacterTextSplitter` is used to split a long text into smaller chunks bas
- **chunk_overlap:** Determines the number of characters that overlap between consecutive chunks when splitting text. It specifies how much of the previous chunk should be included in the next chunk.
For example, if the `chunk_overlap` is set to 20 and the `chunk_size` is set to 100, the splitter will create chunks of 100 characters each, but the last 20 characters of each chunk will overlap with the first 20 characters of the next chunk. This allows for a smoother transition between chunks and ensures that no information is lost – defaults to `200`.
For example, if the `chunk_overlap` is set to 20 and the `chunk_size` is set to 100, the splitter will create chunks of 100 characters each, but the last 20 characters of each chunk will overlap with the first 20 characters of the next chunk. This allows for a smoother transition between chunks and ensures that no information is lost – defaults to `200`.
- **chunk_size:** Determines the maximum number of characters in each chunk when splitting a text. It specifies the size or length of each chunk.
For example, if the chunk_size is set to 100, the splitter will create chunks of 100 characters each. If the text is longer than 100 characters, it will be divided into multiple chunks of equal size, except for the last chunk, which may be smaller if there are remaining characters –defaults to `1000`.
For example, if the chunk_size is set to 100, the splitter will create chunks of 100 characters each. If the text is longer than 100 characters, it will be divided into multiple chunks of equal size, except for the last chunk, which may be smaller if there are remaining characters –defaults to `1000`.
- **separator:** Specifies the character that will be used to split the text into chunks – defaults to `.`
- **separator:** Specifies the character that will be used to split the text into chunks – defaults to `.`
---
@ -44,6 +46,18 @@ The `RecursiveCharacterTextSplitter` splits the text by trying to keep paragra
- **chunk_size:** Determines the maximum number of characters in each chunk when splitting a text. It specifies the size or length of each chunk.
- **separator_type:** The parameter allows the user to split the code with multiple language support. It supports various languages such as Text, Ruby, Python, Solidity, Java, and more. Defaults to `Text`.
- **separators:** The `separators` in RecursiveCharacterTextSplitter are the characters used to split the text into chunks. The text splitter tries to create chunks based on splitting on the first character in the list of `separators`. If any chunks are too large, it moves on to the next character in the list and continues splitting. Defaults to ["\n\n", "\n", " ", ""].
- **separators:** The `separators` in RecursiveCharacterTextSplitter are the characters used to split the text into chunks. The text splitter tries to create chunks based on splitting on the first character in the list of `separators`. If any chunks are too large, it moves on to the next character in the list and continues splitting. Defaults to `.`
### LanguageRecursiveTextSplitter
The `LanguageRecursiveTextSplitter` is a text splitter that splits the text into smaller chunks based on the (programming) language of the text.
**Params**
- **Documents:** Input documents to split.
- **chunk_overlap:** Determines the number of characters that overlap between consecutive chunks when splitting text. It specifies how much of the previous chunk should be included in the next chunk.
- **chunk_size:** Determines the maximum number of characters in each chunk when splitting a text. It specifies the size or length of each chunk.
- **separator_type:** The parameter allows the user to split the code with multiple language support. It supports various languages such as Ruby, Python, Solidity, Java, and more. Defaults to `Python`.

Some files were not shown because too many files have changed in this diff Show more