fix(compose): Create a proper example compose

.github/workflows/deploy.yml

@@ -45,5 +45,5 @@ jobs:
           cat src/airflow_dags/.airflowignore
 
       - name: Deploy to S3 dagbag
-        run: aws s3 sync src/airflow_dags s3://$BUCKET_NAME/dags/airflow_dags --delete
+        run: aws s3 sync src/airflow_dags s3://$BUCKET_NAME/dags/airflow_dags
 

docker-compose.yml

@@ -1,23 +1,58 @@
 name: airflow
 
+x-airflow-variables: &airflow-variables
+  AIRFLOW__CORE__EXECUTOR: "SequentialExecutor" # Use LocalExecutor when running with postgres
+  AIRFLOW__CORE__LOAD_EXAMPLES: "False"
+  AIRFLOW__CORE__DAG_IGNORE_FILE_SYNTAX: "glob"
+  AIRFLOW__CORE__DAGS_FOLDER: "/opt/airflow/dags"
+  AIRFLOW__LOGGING__BASE_LOG_FOLDER: "/opt/airflow/logs"
+  AIRFLOW__LOGGING__LOGGING_LEVEL: WARNING
+  AIRFLOW_CONN_SLACK_API_DEFAULT: not-a-slack-conn
+  AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "True"
+
+
+x-common-variables: &common-variables
+  SECRET_KEY: ${SECRET_KEY}
+  ECS_SUBNET: $ECS_SUBNET
+  ECS_SECURITY_GROUP: $ECS_SECURITY_GROUP
+  AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
+  ENVIRONMENT: $ENVIRONMENT
+
 services:
 
-  airflow:
-    container_name: airflow-dags
-    build: 
-      context: .
-      dockerfile: Dockerfile
-    post_start:
-      - command: |
-          /usr/bin/dumb-init -- /entrypoint db init && \
-          /usr/bin/dumb-init -- /entrypoint users create \
-            --role Admin --username admin --password admin \
-            --firstname admin --lastname admin --email [email protected]
-    command: ["webserver", "-w", "4"]
+  scheduler:
+    image: apache/airflow:2.10.5
+    container_name: airflow-scheduler
+    command: scheduler
+    restart: always
+    ports:
+      - "8793:8793"
     environment:
-      _AIRFLOW_WWW_USER_CREATE: "true"
-      _AIRFLOW_WWW_USER_USERNAME: airflow
-      _AIRFLOW_WWW_USER_PASSWORD: airflow
+      <<: [*airflow-variables, *common-variables]
+    user: "${AIRFLOW_UID:-50000}:0"
+    volumes:
+      - data:/opt/airflow
+
+  webserver:
+    image: apache/airflow:2.10.5
+    container_name: airflow-webserver
+    command: webserver -w 4
     ports:
-      - "4040:8080"
+      - 4040:8080
+    restart: always
+    post_start:
+      - command: | 
+          /usr/bin/dumb-init -- /entrypoint db migrate
+    environment:
+      <<: [*airflow-variables, *common-variables]
+      # AIRFLOW__WEBSERVER__SECRET_KEY: ${SECRET_KEY}
+      # FORWARDED_ALLOW_IPS: "*"
+      AIRFLOW__WEBSERVER__WORKER_CLASS: "gevent"
+      _AIRFLOW_PATCH_GEVENT: "1"
+    user: "${AIRFLOW_UID:-50000}:0"
+    volumes:
+      - data:/opt/airflow
+
+volumes:
+  data:
 

src/airflow_dags/plugins/operators/ecs_run_task_operator.py

@@ -157,7 +157,7 @@ def run_task_operator(
             launch_type="FARGATE",
             network_configuration=networks_dict,
             awslogs_group=f"/aws/ecs/{self.name}",
-            awslogs_stream_prefix=f"streaming/{airflow_task_id}",
+            awslogs_stream_prefix=f"streaming/{self.name}",
             awslogs_region=region,
             trigger_rule=trigger_rule,
             on_failure_callback=on_failure_callback,