From 6ffb994e53f8414103c9f907c02174a7be0e6c37 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Sun, 20 Aug 2023 23:39:19 -0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=93=A6=20chore(docker-compose.override.ym?= =?UTF-8?q?l):=20add=20docker-compose.override.yml=20file=20to=20configure?= =?UTF-8?q?=20Traefik=20proxy=20and=20services=20for=20local=20development?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds a new file `docker-compose.override.yml` to the `deploy` directory. The file contains configuration for the Traefik proxy and services used for local development. The configuration includes: - Enabling Docker in Traefik to read labels from Docker services - Adding a constraint to only use services with a specific label for this stack - Disabling the exposure of all Docker services by default - Enabling the access log and Traefik log - Enabling the Traefik Dashboard and API in insecure mode for local development The file also includes configuration for the following services: - `proxy`: Configured to listen on ports 80 and 8090, with labels for routing and load balancing - `pgadmin`: Configured to listen on port 5050 - `flower`: Configured to listen on port 5555 - `backend`: Configured with labels for routing and load balancing on paths `/api/v1`, `/docs`, and `/health` - `frontend`: Configured with labels for routing and load balancing on the root path The `traefik-public` network is also defined for use by the Traefik proxy. This file is intended to be used as an override for the main `docker-compose.yml` file during local development. 🐳 chore(docker-compose.yml): add Traefik reverse proxy configuration for services 🔧 fix(docker-compose.yml): fix path to startup-backend.sh script 🚀 feat(docker-compose.yml): add support for deploying backend, celery worker, flower, frontend, prometheus, and grafana services 🔧 fix(docker-compose.yml): fix volume configuration for db service 🔧 fix(docker-compose.yml): fix network configuration for pgadmin service 🔧 fix(docker-compose.yml): fix network configuration for traefik proxy service 🔧 fix(docker-compose.yml): fix network configuration for flower service 🔧 fix(docker-compose.yml): fix network configuration for frontend service 🔧 fix(docker-compose.yml): fix network configuration for prometheus service 🔧 fix(docker-compose.yml): fix network configuration for grafana service --- deploy/docker-compose.override.yml | 55 +++++++++++ deploy/docker-compose.yml | 153 ++++++++++++++++++++++++++--- 2 files changed, 194 insertions(+), 14 deletions(-) create mode 100644 deploy/docker-compose.override.yml diff --git a/deploy/docker-compose.override.yml b/deploy/docker-compose.override.yml new file mode 100644 index 000000000..eda492a24 --- /dev/null +++ b/deploy/docker-compose.override.yml @@ -0,0 +1,55 @@ +version: "3.3" +services: + proxy: + ports: + - "80:80" + - "8090:8080" + command: + # Enable Docker in Traefik, so that it reads labels from Docker services + - --providers.docker + # Add a constraint to only use services with the label for this stack + # from the env var TRAEFIK_TAG + - --providers.docker.constraints=Label(`traefik.constraint-label-stack`, `${TRAEFIK_TAG?Variable not set}`) + # Do not expose all Docker services, only the ones explicitly exposed + - --providers.docker.exposedbydefault=false + # Disable Docker Swarm mode for local development + # - --providers.docker.swarmmode + # Enable the access log, with HTTP requests + - --accesslog + # Enable the Traefik log, for configurations and errors + - --log + # Enable the Dashboard and API + - --api + # Enable the Dashboard and API in insecure mode for local development + - --api.insecure=true + labels: + - traefik.enable=true + - traefik.http.routers.${STACK_NAME?Variable not set}-traefik-public-http.rule=Host(`${DOMAIN?Variable not set}`) + - traefik.http.services.${STACK_NAME?Variable not set}-traefik-public.loadbalancer.server.port=80 + + pgadmin: + ports: + - "5050:5050" + + flower: + ports: + - "5555:5555" + + backend: + labels: + - traefik.enable=true + - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} + - traefik.http.routers.${STACK_NAME?Variable not set}-backend-http.rule=PathPrefix(`/api/v1`) || PathPrefix(`/docs`) || PathPrefix(`/health`) + - traefik.http.services.${STACK_NAME?Variable not set}-backend.loadbalancer.server.port=7860 + + frontend: + labels: + - traefik.enable=true + - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} + - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-http.rule=PathPrefix(`/`) + - traefik.http.services.${STACK_NAME?Variable not set}-frontend.loadbalancer.server.port=80 + +networks: + traefik-public: + # For local dev, don't expect an external Traefik network + external: false diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 09c994f78..e35424c03 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -1,7 +1,73 @@ version: "3.3" services: + proxy: + image: traefik:v2.2 + networks: + - ${TRAEFIK_PUBLIC_NETWORK?Variable not set} + - default + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: + # Enable Docker in Traefik, so that it reads labels from Docker services + - --providers.docker + # Add a constraint to only use services with the label for this stack + # from the env var TRAEFIK_TAG + - --providers.docker.constraints=Label(`traefik.constraint-label-stack`, `${TRAEFIK_TAG?Variable not set}`) + # Do not expose all Docker services, only the ones explicitly exposed + - --providers.docker.exposedbydefault=false + # Enable the access log, with HTTP requests + - --accesslog + # Enable the Traefik log, for configurations and errors + - --log + # Enable the Dashboard and API + - --api + deploy: + placement: + constraints: + - node.role == manager + labels: + # Enable Traefik for this service, to make it available in the public network + - traefik.enable=true + # Use the traefik-public network (declared below) + - traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK?Variable not set} + # Use the custom label "traefik.constraint-label=traefik-public" + # This public Traefik will only use services with this label + - traefik.constraint-label=${TRAEFIK_PUBLIC_TAG?Variable not set} + # traefik-http set up only to use the middleware to redirect to https + - traefik.http.middlewares.${STACK_NAME?Variable not set}-https-redirect.redirectscheme.scheme=https + - traefik.http.middlewares.${STACK_NAME?Variable not set}-https-redirect.redirectscheme.permanent=true + # Handle host with and without "www" to redirect to only one of them + # Uses environment variable DOMAIN + # To disable www redirection remove the Host() you want to discard, here and + # below for HTTPS + - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-http.rule=Host(`${DOMAIN?Variable not set}`) || Host(`www.${DOMAIN?Variable not set}`) + - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-http.entrypoints=http + # traefik-https the actual router using HTTPS + - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.rule=Host(`${DOMAIN?Variable not set}`) || Host(`www.${DOMAIN?Variable not set}`) + - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.entrypoints=https + - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.tls=true + # Use the "le" (Let's Encrypt) resolver created below + - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.tls.certresolver=le + # Define the port inside of the Docker service to use + - traefik.http.services.${STACK_NAME?Variable not set}-proxy.loadbalancer.server.port=80 + # Handle domain with and without "www" to redirect to only one + # To disable www redirection remove the next line + - traefik.http.middlewares.${STACK_NAME?Variable not set}-www-redirect.redirectregex.regex=^https?://(www.)?(${DOMAIN?Variable not set})/(.*) + # Redirect a domain with www to non-www + # To disable it remove the next line + - traefik.http.middlewares.${STACK_NAME?Variable not set}-www-redirect.redirectregex.replacement=https://${DOMAIN?Variable not set}/$${3} + # Redirect a domain without www to www + # To enable it remove the previous line and uncomment the next + # - traefik.http.middlewares.${STACK_NAME}-www-redirect.redirectregex.replacement=https://www.${DOMAIN}/$${3} + # Middleware to redirect www, to disable it remove the next line + - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-https.middlewares=${STACK_NAME?Variable not set}-www-redirect + # Middleware to redirect www, and redirect HTTP to HTTPS + # to disable www redirection remove the section: ${STACK_NAME?Variable not set}-www-redirect, + - traefik.http.routers.${STACK_NAME?Variable not set}-proxy-http.middlewares=${STACK_NAME?Variable not set}-www-redirect,${STACK_NAME?Variable not set}-https-redirect + backend: + image: "langflow:${TAG-latest}" build: context: ../ dockerfile: base.Dockerfile @@ -19,24 +85,50 @@ services: command: /startup-backend.sh # Fixed the path healthcheck: test: "exit 0" + deploy: + labels: + - traefik.enable=true + - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} + - traefik.http.routers.${STACK_NAME?Variable not set}-backend-http.rule=PathPrefix(`/api/v1`) || PathPrefix(`/docs`) || PathPrefix(`/health`) + - traefik.http.services.${STACK_NAME?Variable not set}-backend.loadbalancer.server.port=7860 db: image: postgres:15.4 + volumes: + - app-db-data:/var/lib/postgresql/data/pgdata env_file: - - ./db.env - ports: - - "5432:5432" + - db.env + environment: + - PGDATA=/var/lib/postgresql/data/pgdata + deploy: + placement: + constraints: + - node.labels.${STACK_NAME?Variable not set}.app-db-data == true pgadmin: image: dpage/pgadmin4 + networks: + - ${TRAEFIK_PUBLIC_NETWORK?Variable not set} + - default env_file: - ./pgadmin.env - ports: - - "5050:80" depends_on: - db volumes: - ./pgadmin:/var/lib/pgadmin + deploy: + labels: + - traefik.enable=true + - traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK?Variable not set} + - traefik.constraint-label=${TRAEFIK_PUBLIC_TAG?Variable not set} + - traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-http.rule=Host(`pgadmin.${DOMAIN?Variable not set}`) + - traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-http.entrypoints=http + - traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-http.middlewares=${STACK_NAME?Variable not set}-https-redirect + - traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-https.rule=Host(`pgadmin.${DOMAIN?Variable not set}`) + - traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-https.entrypoints=https + - traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-https.tls=true + - traefik.http.routers.${STACK_NAME?Variable not set}-pgadmin-https.tls.certresolver=le + - traefik.http.services.${STACK_NAME?Variable not set}-pgadmin.loadbalancer.server.port=5050 queue: image: redis:6.2.5 @@ -46,7 +138,7 @@ services: test: "exit 0" celeryworker: - # user: your-non-root-user + image: "langflow:${TAG-latest}" depends_on: queue: condition: service_healthy @@ -55,14 +147,14 @@ services: build: context: ../ dockerfile: base.Dockerfile - command: celery -A langflow.worker.celery_app worker --loglevel=INFO + command: celery -A langflow.worker.celery_app worker --loglevel=INFO --concurrency=1 -n lf-worker@%h healthcheck: test: "exit 0" deploy: replicas: 1 flower: - # user: your-non-root-user + image: mher/flower networks: - default depends_on: @@ -70,14 +162,25 @@ services: condition: service_healthy celeryworker: condition: service_healthy - build: - context: ../ - dockerfile: base.Dockerfile env_file: - ./flower.env - command: /bin/sh -c "celery -A langflow.worker.celery_app --broker=redis://$${LANGFLOW_REDIS_PASSWORD:+default:$${LANGFLOW_REDIS_PASSWORD}@}$${LANGFLOW_REDIS_HOST}:$${LANGFLOW_REDIS_PORT}/$${LANGFLOW_REDIS_DB} flower --port=5555" + environment: + - CELERY_BROKER_URL='redis://$${LANGFLOW_REDIS_PASSWORD:+default:$${LANGFLOW_REDIS_PASSWORD}@}$${LANGFLOW_REDIS_HOST}:$${LANGFLOW_REDIS_PORT}/$${LANGFLOW_REDIS_DB}' ports: - "5555:5555" + deploy: + labels: + - traefik.enable=true + - traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK?Variable not set} + - traefik.constraint-label=${TRAEFIK_PUBLIC_TAG?Variable not set} + - traefik.http.routers.${STACK_NAME?Variable not set}-flower-http.rule=Host(`flower.${DOMAIN?Variable not set}`) + - traefik.http.routers.${STACK_NAME?Variable not set}-flower-http.entrypoints=http + - traefik.http.routers.${STACK_NAME?Variable not set}-flower-http.middlewares=${STACK_NAME?Variable not set}-https-redirect + - traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.rule=Host(`flower.${DOMAIN?Variable not set}`) + - traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.entrypoints=https + - traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.tls=true + - traefik.http.routers.${STACK_NAME?Variable not set}-flower-https.tls.certresolver=le + - traefik.http.services.${STACK_NAME?Variable not set}-flower.loadbalancer.server.port=5555 frontend: # user: your-non-root-user @@ -91,9 +194,13 @@ services: condition: service_healthy env_file: - ./frontend.env - ports: - - "80:80" restart: on-failure + deploy: + labels: + - traefik.enable=true + - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} + - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-http.rule=PathPrefix(`/`) + - traefik.http.services.${STACK_NAME?Variable not set}-frontend.loadbalancer.server.port=80 prometheus: image: prom/prometheus:v2.37.9 @@ -105,6 +212,12 @@ services: - "9090:9090" healthcheck: test: "exit 0" + deploy: + labels: + - traefik.enable=true + - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} + - traefik.http.routers.${STACK_NAME?Variable not set}-prometheus-http.rule=PathPrefix(`/metrics`) + - traefik.http.services.${STACK_NAME?Variable not set}-prometheus.loadbalancer.server.port=9090 grafana: image: grafana/grafana:8.2.6 @@ -115,6 +228,18 @@ services: - "3000:3000" volumes: - grafana_data:/var/lib/grafana + deploy: + labels: + - traefik.enable=true + - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} + - traefik.http.routers.${STACK_NAME?Variable not set}-grafana-http.rule=PathPrefix(`/grafana`) + - traefik.http.services.${STACK_NAME?Variable not set}-grafana.loadbalancer.server.port=3000 volumes: grafana_data: + app-db-data: + +networks: + traefik-public: + # Allow setting it to false for testing + external: false # ${TRAEFIK_PUBLIC_NETWORK_IS_EXTERNAL-true}