Airflow demo: Clean up scripts and more clearly describe actions (airbytehq#4639)

* Airflow demo: Script cleanup

* Correct docker compose name for airflow file

* Final fixes

* Clean up airbyte destination

Co-authored-by: Abhi Vaidyanatha <[email protected]>

Author: avaidyanatha

resources/examples/airflow/docker-compose.yaml resources/examples/airflow/docker-compose-airflow.yaml

@@ -2,5 +2,6 @@
 cd ../../..
 docker-compose down -v
 cd resources/examples/airflow || exit
-docker-compose -f docker-compose-superset.yaml down -v
-docker-compose -f docker-compose.yaml down -v
+docker-compose -f docker-compose-airflow.yaml down -v
+docker-compose -f superset/docker-compose-superset.yaml down -v
+docker stop airbyte-destination

resources/examples/airflow/down.sh

@@ -33,15 +33,15 @@ services:
       - redis:/data
 
   db:
-    env_file: docker/.env
+    env_file: ../docker/.env
     image: postgres:10
     container_name: superset_db
     restart: unless-stopped
     volumes:
       - db_home:/var/lib/postgresql/data
 
   superset:
-    env_file: docker/.env-non-dev
+    env_file: ../docker/.env-non-dev
     image: *superset-image
     container_name: superset_app
     command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
@@ -56,7 +56,7 @@ services:
     image: *superset-image
     container_name: superset_init
     command: ["/app/docker/docker-init.sh"]
-    env_file: docker/.env-non-dev
+    env_file: ../docker/.env-non-dev
     depends_on: *superset-depends-on
     user: "root"
     volumes: *superset-volumes
@@ -65,7 +65,7 @@ services:
     image: *superset-image
     container_name: superset_worker
     command: ["/app/docker/docker-bootstrap.sh", "worker"]
-    env_file: docker/.env-non-dev
+    env_file: ../docker/.env-non-dev
     restart: unless-stopped
     depends_on: *superset-depends-on
     user: "root"
@@ -75,7 +75,7 @@ services:
     image: *superset-image
     container_name: superset_worker_beat
     command: ["/app/docker/docker-bootstrap.sh", "beat"]
-    env_file: docker/.env-non-dev
+    env_file: ../docker/.env-non-dev
     restart: unless-stopped
     depends_on: *superset-depends-on
     user: "root"

resources/examples/airflow/docker-compose-superset.yaml resources/examples/airflow/superset/docker-compose-superset.yaml

@@ -1,10 +1,12 @@
 #!/usr/bin/env bash
 cd ../../..
+echo "Attempting to remove previous Airbyte installation..."
 docker-compose down -v
 docker-compose up -d
 cd resources/examples/airflow || exit
-docker-compose down -v
-docker-compose up -d
+echo "Attempting to remove previous Airflow installation..."
+docker-compose -f docker-compose-airflow.yaml down -v
+docker-compose -f docker-compose-airflow.yaml up -d
 # Create Postgres Database to replicate to.
 docker run --rm --name airbyte-destination -e POSTGRES_PASSWORD=password -p 2000:5432 -d postgres
 echo "Access Airbyte at http://localhost:8000 and set up a connection."
@@ -14,6 +16,6 @@ read connection_id
 docker exec -ti airflow_webserver airflow variables set 'AIRBYTE_CONNECTION_ID' "$connection_id"
 docker exec -ti airflow_webserver airflow connections add 'airbyte_example' --conn-uri 'airbyte://host.docker.internal:8000'
 echo "Access Airflow at http://localhost:8085 to kick off your Airbyte sync DAG."
-# Create Superset containers.
-docker-compose -f docker-compose-superset.yaml down -v
-docker-compose -f docker-compose-superset.yaml up -d
+echo "Attempting to remove previous Superset installation."
+docker-compose -f superset/docker-compose-superset.yaml down -v
+docker-compose -f superset/docker-compose-superset.yaml up -d