108 lines
No EOL
3.1 KiB
YAML
108 lines
No EOL
3.1 KiB
YAML
x-airflow-common: &airflow-common
|
|
image: apache/airflow:2.7.2
|
|
extra_hosts:
|
|
- "host.docker.internal:host-gateway"
|
|
environment:
|
|
&airflow-common-env
|
|
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
|
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow_password@postgres:5432/fraud_db
|
|
AIRFLOW__CORE__FERNET_KEY: ''
|
|
AIRFLOW__CORE__LOAD_EXAMPLES: 'False'
|
|
_PIP_ADDITIONAL_REQUIREMENTS: 'apache-airflow-providers-amazon apache-airflow-providers-postgres pandas pyarrow requests'
|
|
AIRFLOW__WEBSERVER__SECRET_KEY: 'this_is_a_very_secure_secret_key'
|
|
AIRFLOW_CONN_MINIO_CONN: 'aws://minio_admin:minio_password@/?endpoint_url=http%3A%2F%2Fminio%3A9000'
|
|
env_file:
|
|
- .env
|
|
volumes:
|
|
- ./dags:/opt/airflow/dags
|
|
- airflow_logs:/opt/airflow/logs
|
|
user: "50000:0"
|
|
services:
|
|
postgres:
|
|
image: postgres:15
|
|
container_name: postgres_db
|
|
environment:
|
|
POSTGRES_USER: airflow
|
|
POSTGRES_PASSWORD: airflow_password
|
|
POSTGRES_DB: fraud_db
|
|
ports:
|
|
- "5432:5432"
|
|
volumes:
|
|
- ./dags/sql/init.sql:/docker-entrypoint-initdb.d/init.sql
|
|
- postgres_data:/var/lib/postgresql/data
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U airflow -d fraud_db"]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 5
|
|
|
|
minio:
|
|
image: quay.io/minio/minio
|
|
container_name: minio_storage
|
|
command: server /data --console-address ":9001"
|
|
environment:
|
|
MINIO_ROOT_USER: minio_admin
|
|
MINIO_ROOT_PASSWORD: minio_password
|
|
ports:
|
|
- "9000:9000"
|
|
- "9001:9001"
|
|
volumes:
|
|
- minio_data:/data
|
|
|
|
minio-setup:
|
|
image: quay.io/minio/mc
|
|
container_name: minio_setup
|
|
depends_on:
|
|
- minio
|
|
entrypoint: >
|
|
/bin/sh -c "
|
|
sleep 5;
|
|
mc alias set myminio http://minio:9000 minio_admin minio_password;
|
|
mc mb myminio/fraud-features --ignore-existing;
|
|
mc policy set public myminio/fraud-features;
|
|
exit 0;
|
|
"
|
|
|
|
airflow-init:
|
|
<<: *airflow-common
|
|
container_name: airflow_init
|
|
command: version
|
|
environment:
|
|
<<: *airflow-common-env
|
|
_AIRFLOW_DB_MIGRATE: 'true'
|
|
_AIRFLOW_WWW_USER_CREATE: 'true'
|
|
_AIRFLOW_WWW_USER_USERNAME: airflow
|
|
_AIRFLOW_WWW_USER_PASSWORD: airflow_password
|
|
|
|
airflow-webserver:
|
|
<<: *airflow-common
|
|
container_name: airflow_webserver
|
|
command: webserver
|
|
ports:
|
|
- "8080:8080"
|
|
healthcheck:
|
|
test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
|
|
interval: 10s
|
|
timeout: 10s
|
|
retries: 5
|
|
restart: always
|
|
depends_on:
|
|
airflow-init:
|
|
condition: service_completed_successfully
|
|
|
|
airflow-scheduler:
|
|
<<: *airflow-common
|
|
container_name: airflow_scheduler
|
|
command: scheduler
|
|
restart: always
|
|
depends_on:
|
|
airflow-init:
|
|
condition: service_completed_successfully
|
|
environment:
|
|
<<: *airflow-common-env # <-- Add this line to merge the common variables
|
|
AIRFLOW_CONN_POSTGRES_CONN: "postgresql://airflow:airflow_password@postgres_db:5432/fraud_db"
|
|
|
|
volumes:
|
|
postgres_data:
|
|
minio_data:
|
|
airflow_logs: # <-- Add this line |