Airflow instance runtime
Launch the Airflow webserver UI in Celery Executor mode to visualise and interact with dashboard:
Start
You can access your local Airflow webserver console via https://localhost:8443.
Note
It is safe to ignore the certificate error exceptions raised by your browser as the stack is using a self-signed certificate.
Info
Check the credentials management for instructions on Airflow webserver authentication.
Stop
Docker files
Flowz Apache Airflow in CeleryExecutor mode is delivered as a containerised service. Use Docker compose to standup the services.
Configuration
The environment variables are fed into the docker-compose.yml:
Apache Airflow CeleryExecutor Docker compose file.
---
x-airflow-common: &airflow-common
AIRFLOW_HOME: ${FLOWZ_AIRFLOW_HOME:-/opt/airflow/airflow}
AIRFLOW_CUSTOM_ENV: ${FLOWZ_AIRFLOW_CUSTOM_ENV:-dev}
AIRFLOW_CUSTOM_CONFIG:
${FLOWZ_CUSTOM_CONFIG:-/home/airflow/.local/lib/python3.12/site-packages/flowz/config}
AIRFLOW__CORE__EXECUTOR: CeleryExecutor
AIRFLOW__CORE__DAGS_FOLDER:
${AIRFLOW__CORE__DAGS_FOLDER:-/home/airflow/.local/lib/python3.12/site-packages/flowz/dags}
AIRFLOW__CORE__PLUGINS_FOLDER:
${AIRFLOW__CORE__PLUGINS_FOLDER:-/home/airflow/.local/lib/python3.12/site-packages/flowz/plugins}
AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES:-False}
AIRFLOW__CORE__UNIT_TEST_MODE: ${AIRFLOW__CORE__UNIT_TEST_MODE:-False}
AIRFLOW__CORE__STORE_SERIALIZED_DAGS: ${AIRFLOW__CORE__STORE_SERIALIZED_DAGS:-True}
AIRFLOW__CORE__MIN_SERIALIZED_DAG_UPDATE_INTERVAL:
${AIRFLOW__CORE__MIN_SERIALIZED_DAG_UPDATE_INTERVAL:-30}
AIRFLOW__CORE__FERNET_KEY:
${AIRFLOW__CORE__FERNET_KEY:-"LFKF4PSrAOG-kbxOouoLj8Du2QCnsp9qw7G21-WPsLU="}
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN:
${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN:-postgresql://airflow:airflow@postgres:5432/airflow?}
AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS: ${AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS:-False}
AIRFLOW__DATABASE__CHECK_MIGRATIONS: ${AIRFLOW__DATABASE__CHECK_MIGRATIONS:-False}
AIRFLOW__WEBSERVER__DAG_DEFAULT_VIEW: ${AIRFLOW__WEBSERVER__DAG_DEFAULT_VIEW:-graph}
AIRFLOW__WEBSERVER__WEB_SERVER_PORT: ${AIRFLOW__WEBSERVER__WEB_SERVER_PORT:-8443}
AIRFLOW__WEBSERVER__WORKERS: ${AIRFLOW__WEBSERVER__WORKERS:-1}
AIRFLOW__WEBSERVER__WORKER_REFRESH_BATCH_SIZE:
${AIRFLOW__WEBSERVER__WORKER_REFRESH_BATCH_SIZE:-1}
AIRFLOW__WEBSERVER__WEB_SERVER_SSL_CERT:
${AIRFLOW__WEBSERVER__WEB_SERVER_SSL_CERT:-/etc/ssl/certs/airflow-selfsigned.crt}
AIRFLOW__WEBSERVER__WEB_SERVER_SSL_KEY:
${AIRFLOW__WEBSERVER__WEB_SERVER_SSL_KEY:-/home/airflow/ssl/private/airflow-selfsigned.key}
AIRFLOW__SCHEDULER__PRINT_STATS_INTERVAL: ${AIRFLOW__SCHEDULER__PRINT_STATS_INTERVAL:-300}
AIRFLOW__SCHEDULER__SCHEDULER_IDLE_SLEEP_TIME: ${AIRFLOW__SCHEDULER__SCHEDULER_IDLE_SLEEP_TIME:-5}
AIRFLOW__LOGGING__BASE_LOG_FOLDER: ${AIRFLOW__LOGGING__BASE_LOG_FOLDER:-/var/log/airflow}
AIRFLOW__LOGGING__REMOTE_LOGGING: ${AIRFLOW__LOGGING__REMOTE_LOGGING:-False}
services:
redis:
image: redis:7.2-alpine
container_name: airflow-redis
entrypoint: ["redis-server", "--protected-mode", "no"]
environment:
REDIS_PASSWORD: ${REDIS_PASSWORD:-redispass}
REDIS_PROTO: ${REDIS_PROTO:-"redis://"}
REDIS_HOST: ${REDIS_HOST:-redis}
REDIS_PORT: ${REDIS_PORT:-6379}
REDIS_DBNUM: ${REDIS_DBNUM:-1}
AIRFLOW__CELERY__BROKER_URL: ${AIRFLOW__CELERY__BROKER_URL:-"redis://redis:6379/1"}
AIRFLOW__CELERY__RESULT_BACKEND:
${AIRFLOW__CELERY__RESULT_BACKEND:-db+postgresql://${POSTGRES_USER:-airflow}:${POSTGRES_USER:-airflow}@postgres:5432/airflow?}
healthcheck:
test: ["CMD-SHELL", "redis-cli", "--raw", "incr", "ping"]
interval: 2s
timeout: 5s
retries: 2
postgres:
image: postgres:15.4-alpine
container_name: airflow-postgres
environment:
POSTGRES_USER: ${POSTGRES_USER:-airflow}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-airflow}
POSTGRES_HOST: ${POSTGRES_HOST:-postgres}
POSTGRES_PORT: ${POSTGRES_PORT:-5432}
POSTGRES_DB: ${POSTGRES_DB:-airflow}
POSTGRES_EXTRAS: ${POSTGRES_EXTRAS:-}
PGDATA: ${PGDATA:-/var/lib/postgresql/data/pgdata/db-files}
volumes:
- airflow-postgres-vol:/var/lib/postgresql/data/pgdata
healthcheck:
test: [
"CMD-SHELL",
"pg_isready -d $${POSTGRES_DB:-airflow} -U $${POSTGRES_USER:-airflow}"
]
interval: 2s
timeout: 5s
retries: 2
init-db:
image: loum/flowz:latest
container_name: airflow-initdb
depends_on:
postgres:
condition: service_healthy
environment:
<<: *airflow-common
restart: "no"
command: db migrate
healthcheck:
test: ["CMD", "db check"]
interval: 2s
timeout: 5s
retries: 2
scheduler:
image: loum/flowz:latest
container_name: airflow-scheduler
hostname: scheduler
depends_on:
init-db:
condition: service_completed_successfully
redis:
condition: service_started
environment:
<<: *airflow-common
volumes:
- airflow-logs-vol:/var/log/airflow
restart: always
command: scheduler
webserver:
image: loum/flowz:latest
container_name: airflow-webserver
hostname: webserver
depends_on:
init-db:
condition: service_completed_successfully
environment:
<<: *airflow-common
volumes:
- airflow-logs-vol:/var/log/airflow
ports:
- 8443:8443
restart: always
command: webserver
worker:
image: loum/flowz:latest
container_name: airflow-worker
hostname: worker
extra_hosts:
host.docker.internal: host-gateway
depends_on:
init-db:
condition: service_completed_successfully
redis:
condition: service_started
environment:
<<: *airflow-common
FLOWZ_AIRFLOW_ADMIN_USER: ${FLOWZ_AIRFLOW_ADMIN_USER:-airflow}
FLOWZ_AIRFLOW_ADMIN_PASSWORD: ${FLOWZ_AIRFLOW_ADMIN_PASSWORD:-airflow}
volumes:
- airflow-worker-vol:/opt/airflow/data
- airflow-logs-vol:/var/log/airflow
restart: always
command: celery worker
minio:
image: quay.io/minio/minio
container_name: airflow-s3
hostname: minio
volumes:
- airflow-s3-vol:/data
ports:
- 9000:9000
- 9001:9001
command: server /data --console-address ":9001"
volumes:
airflow-logs-vol:
airflow-worker-vol:
airflow-postgres-vol:
airflow-s3-vol:
Flowz default container image
Use the following shell snippet if you are only interested in sampling the default Flowz container image:
curl -s https://raw.githubusercontent.com/loum/flowz/main/docker/docker-compose.yml |\
docker compose -f - up
For this simple demo usecase, the default Airflow webserver login credentials are airflow:airflow.
This blocking variant of the docker compose can be stopped by hitting Ctrl-C.