Allow multiple devstack instances on one machine (openedx-unsupported#532)

* Update many `docker` usages to be `docker-compose` to allow
  multi-devstack support via setting of `COMPOSE_PROJECT_NAME`
  connfiguration option in `options.local.mk`.
* Add network aliases for backwards-compatibility.
* Add several new Makefile targets:
    + dev.print-container.<service>
    + dev.ps
    + dev.stop[.<services>]
    + dev.kill[.<services>]
    + dev.down[.<services>]
    + dev.restart[.<services>]
    + dev.rm-stopped
* Using new targets, fix features that would break when using an
  alternative `COMPOSE_PROJECT_NAME`.
* Utilize `COMPOSE_FILE` docker-compose environment variable
   to set Docker Compose file list for all Makefile rules and scripts
   used through the Makefile.

Co-authored-by: Adolfo R. Brandes <[email protected]>
Co-authored-by: Kyle McCormick <[email protected]>

Author: 3 people

Makefile

@@ -23,8 +23,8 @@ dev.provision.whitelabel:
 # The containers must be started with the 'dev.up.e2e_wl_tests' target.
 # AND the test must be setup using the 'dev.provision.whitelabel' target.
 whitelabel-tests:
-	docker run -d --name=devstack.whitelabel --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e
-	docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh devstack.whitelabel:/tmp/run_whitelabel_tests.sh
+	docker run -d --name=devstack.whitelabel --network=${COMPOSE_PROJECT_NAME:-devstack}_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e
+	docker cp ${DEVSTACK_WORKSPACE}/edx-themes/edx-platform/run_whitelabel_tests.sh $(make --silent dev.print-container.devstack.whitelabel)":/tmp/run_whitelabel_tests.sh
 	docker exec -t devstack.whitelabel env TEST_ENV=devstack TERM=$(TERM) bash /tmp/run_whitelabel_tests.sh
 
 whitelabel-cleanup:

Makefile.edx

@@ -405,11 +405,11 @@ already being correcty provisioned.
 So, when in doubt, it may still be best to run the full ``make dev.provision``.
 
 Sometimes you may need to restart a particular application server. To do so,
-simply use the ``docker-compose restart`` command:
+simply use the ``make dev.restart.%`` command:
 
 .. code:: sh
 
-    docker-compose restart <service>
+    make dev.restart.<service>
 
 In all the above commands, ``<service>`` should be replaced with one of the following:
 
@@ -681,7 +681,7 @@ vary depending on the database. For all of the options, see ``provision.sql``.
 - Password: ``password``
 
 If you have trouble connecting, ensure the port was mapped successfully by
-running ``docker-compose ps`` and looking for a line like this:
+running ``make dev.ps`` and looking for a line like this:
 ``edx.devstack.mysql docker-entrypoint.sh mysql ... Up 0.0.0.0:3506→3306/tcp``.
 
 Switching branches
@@ -774,20 +774,20 @@ Set a PDB breakpoint anywhere in the code using:
 
 and your attached session will offer an interactive PDB prompt when the breakpoint is hit.
 
-To detach from the container, you'll need to stop the container with:
+You may be able to detach from the container with the ``Ctrl-P, Ctrl-Q`` key sequence.
+If that doesn't work, you will have either close your terminal window,
+stop the container with:
 
 .. code:: sh
 
-    make stop
+    make dev.stop.<service>
 
-or a manual Docker command to bring down the container:
+or kill the container with:
 
 .. code:: sh
 
-   docker kill $(docker ps -a -q --filter="name=edx.devstack.<container name>")
+   make dev.kill.<service>
 
-Alternatively, some terminals allow detachment from a running container with the
-``Ctrl-P, Ctrl-Q`` key sequence.
 
 Running LMS and Studio Tests
 ----------------------------
@@ -872,7 +872,7 @@ Check the logs
 
 If a container stops unexpectedly, you can look at its logs for clues::
 
-    docker-compose logs lms
+    make <service>-logs
 
 Update the code and images
 ~~~~~~~~~~~~~~~~~~~~~~~~~~

README.rst

@@ -15,14 +15,14 @@ for arg in "$@"; do
             studio=true
         fi
     elif [ $arg == "--ecommerce" ]; then
-        if [ ! "$(docker exec -t edx.devstack.ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then
+        if [ ! "$(docker-compose exec ecommerce bash -c 'echo "Course will be created for ecommerce"; exit $?')" ]; then
             echo "Issue with ecommerce container"
             container_error=true
         else
             ecommerce=true
         fi
     elif [ $arg == "--marketing" ]; then
-        if [ ! "$(docker exec -t edx.devstack.marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then
+        if [ ! "$(docker-compose exec marketing bash -c 'echo "Course will be created for marketing"; exit $?')" ]; then
             echo "Issue with marketing container. Course creation will proceed without marketing container."
         else
             marketing=true
@@ -54,10 +54,10 @@ fi
 
 if $ecommerce ; then
 	echo "Creating courses on ecommerce."
-	docker exec -t edx.devstack.ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'"
+	docker-compose exec ecommerce bash -c "source /edx/app/ecommerce/ecommerce_env && python /edx/app/ecommerce/ecommerce/manage.py generate_courses '$course_json'"
 fi
 
 if $marketing ; then
 	echo "Creating courses on marketing."
-	docker exec -t edx.devstack.marketing bash -c "drush generate_courses '$course_json'"
+	docker-compose exec marketing bash -c "drush generate_courses '$course_json'"
 fi

course-generator/create-courses.sh

@@ -5,5 +5,5 @@ set -e
 read -p "This will delete all data in your devstack. Would you like to proceed? [y/n] " -r
 if [[ $REPLY =~ ^[Yy]$ ]]
 then
-    docker-compose -f docker-compose.yml -f docker-compose-watchers.yml -f docker-compose-host.yml -f docker-compose-themes.yml -f docker-compose-analytics-pipeline.yml down -v
+    docker-compose down -v
 fi

destroy.sh

@@ -3,10 +3,14 @@ version: "2.1"
 services:
   namenode:
     image: edxops/analytics_pipeline_hadoop_namenode:${OPENEDX_RELEASE:-latest}
-    container_name: edx.devstack.analytics_pipeline.namenode
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.namenode"
     hostname: namenode
     environment:
       - CLUSTER_NAME=devstack
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline.namenode
     ports:
       - 127.0.0.1:50070:50070
     command: ["/run.sh"]
@@ -15,12 +19,16 @@ services:
 
   datanode:
     image: edxops/analytics_pipeline_hadoop_datanode:${OPENEDX_RELEASE:-latest}
-    container_name: edx.devstack.analytics_pipeline.datanode
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.datanode"
     hostname: datanode
     environment:
       CORE_CONF_fs_defaultFS: "hdfs://namenode:8020"
     depends_on:
       - namenode
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline.datanode
     ports:
       - 127.0.0.1:50075:50075
     command: ["/run.sh"]
@@ -29,7 +37,7 @@ services:
 
   resourcemanager:
     image: edxops/analytics_pipeline_hadoop_resourcemanager:${OPENEDX_RELEASE:-latest}
-    container_name: edx.devstack.analytics_pipeline.resourcemanager
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.resourcemanager"
     hostname: resourcemanager
     environment:
       CORE_CONF_fs_defaultFS: "hdfs://namenode:8020"
@@ -40,13 +48,17 @@ services:
     depends_on:
       - namenode
       - datanode
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline.resourcemanager
     ports:
       - 127.0.0.1:8088:8088      # resource manager web ui
     command: ["/run.sh"]
 
   nodemanager:
     image: edxops/analytics_pipeline_hadoop_nodemanager:${OPENEDX_RELEASE:-latest}
-    container_name: edx.devstack.analytics_pipeline.nodemanager
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.nodemanager"
     hostname: nodemanager
     environment:
       CORE_CONF_fs_defaultFS: "hdfs://namenode:8020"
@@ -60,15 +72,23 @@ services:
       - resourcemanager
       - namenode
       - datanode
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline.nodemanager
     ports:
       - 127.0.0.1:8042:8042      # node manager web ui
       - 127.0.0.1:19888:19888    # node manager job history server ui
     command: ["/run.sh"]
 
   sparkmaster:
     image: edxops/analytics_pipeline_spark_master:${OPENEDX_RELEASE:-latest}
-    container_name: edx.devstack.analytics_pipeline.sparkmaster
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkmaster"
     hostname: sparkmaster
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline.sparkmaster
     ports:
       - 127.0.0.1:8080:8080
       - 127.0.0.1:7077:7077       # spark master port
@@ -77,25 +97,33 @@ services:
 
   sparkworker:
     image: edxops/analytics_pipeline_spark_worker:${OPENEDX_RELEASE:-latest}
-    container_name: edx.devstack.analytics_pipeline.sparkworker
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.sparkworker"
     hostname: sparkworker
     depends_on:
       - sparkmaster
     environment:
       - SPARK_MASTER=spark://sparkmaster:7077
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline.sparkworker
     ports:
       - 127.0.0.1:8081:8081       # spark worker UI
 
   vertica:
     image: iamamr/vertica:9.1.0-0
     hostname: vertica
-    container_name: edx.devstack.analytics_pipeline.vertica
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline.vertica"
     volumes:
       - vertica_data:/home/dbadmin/docker
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline.vertica
 
   analyticspipeline:
     image: edxops/analytics_pipeline:${OPENEDX_RELEASE:-latest}
-    container_name: edx.devstack.analytics_pipeline
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.analytics_pipeline"
     hostname: analyticspipeline
     volumes:
       - ${DEVSTACK_WORKSPACE}/edx-analytics-pipeline:/edx/app/analytics_pipeline/analytics_pipeline
@@ -109,6 +137,10 @@ services:
       - sparkworker
       - elasticsearch
       - vertica
+    networks:
+      default:
+        aliases:
+          - edx.devstack.analytics_pipeline
     ports:
       - 127.0.0.1:4040:4040     # spark web UI
     environment:

docker-compose-analytics-pipeline.yml

@@ -7,7 +7,7 @@ services:
       - MARKETING_SITE_ROOT="http://localhost:8080"
 
   marketing:
-    container_name: edx.devstack.marketing
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.marketing"
     depends_on:
       - mysql
       - memcached
@@ -26,5 +26,9 @@ services:
       # IP address of your machine to enable debugging (IP_ADDRESS set in .env file)
       - XDEBUG_CONFIG=remote_host=${XDEBUG_IP_ADDRESS:-127.0.0.1}
     image: edxops/edx-mktg:${OPENEDX_RELEASE:-latest}
+    networks:
+      default:
+        aliases:
+          - edx.devstack.marketing
     ports:
       - "8080:80"

docker-compose-marketing-site.yml

@@ -3,7 +3,7 @@ version: "2.1"
 services:
   lms_watcher:
     command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done'
-    container_name: edx.devstack.lms_watcher
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms_watcher"
     environment:
       BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher
       ASSET_WATCHER_TIMEOUT: 12
@@ -14,10 +14,14 @@ services:
       - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules
       - src-nfs:/edx/src
       - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached
+    networks:
+      default:
+        aliases:
+          - edx.devstack.lms_watcher
 
   studio_watcher:
     command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done'
-    container_name: edx.devstack.studio_watcher
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio_watcher"
     environment:
       BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher
       ASSET_WATCHER_TIMEOUT: 12
@@ -28,6 +32,11 @@ services:
       - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules
       - src-nfs:/edx/src:cached
       - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached
+    networks:
+      default:
+        aliases:
+          - edx.devstack.studio_watcher
+
 volumes:
   edxapp_lms_assets:
   edxapp_studio_assets:

docker-compose-watchers-nfs.yml

@@ -3,7 +3,7 @@ version: "2.1"
 services:
   lms_watcher:
     command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done'
-    container_name: edx.devstack.lms_watcher
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms_watcher"
     environment:
       BOK_CHOY_HOSTNAME: edx.devstack.lms_watcher
       ASSET_WATCHER_TIMEOUT: 12
@@ -14,10 +14,14 @@ services:
       - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules
       - ${DEVSTACK_WORKSPACE}/src:/edx/src:cached
       - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached
+    networks:
+      default:
+        aliases:
+          - edx.devstack.lms_watcher
 
   studio_watcher:
     command: bash -c 'cd /edx/app/edxapp/edx-platform && source ../edxapp_env && while true; do paver watch_assets --w=$$ASSET_WATCHER_TIMEOUT; sleep 2; done'
-    container_name: edx.devstack.studio_watcher
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.studio_watcher"
     environment:
       BOK_CHOY_HOSTNAME: edx.devstack.studio_watcher
       ASSET_WATCHER_TIMEOUT: 12
@@ -28,6 +32,10 @@ services:
       - edxapp_node_modules:/edx/app/edxapp/edx-platform/node_modules
       - ${DEVSTACK_WORKSPACE}/src:/edx/src:cached
       - ${DEVSTACK_WORKSPACE}/edx-themes:/edx/app/edx-themes:cached
+    networks:
+      default:
+        aliases:
+          - edx.devstack.studio_watcher
 
 volumes:
   edxapp_lms_assets:

docker-compose-watchers.yml

@@ -2,19 +2,27 @@ version: "2.1"
 
 services:
   xqueue:
-    container_name: edx.devstack.xqueue
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue"
     image: edxops/xqueue:${OPENEDX_RELEASE:-latest}
     command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py runserver 0.0.0.0:18040 ; sleep 2; done'
     volumes:
       - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached
     # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file
+    networks:
+      default:
+        aliases:
+          - edx.devstack.xqueue
     ports:
       - 18040:18040
 
   xqueue_consumer:
-    container_name: edx.devstack.xqueue_consumer
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue_consumer"
     image: edxops/xqueue:${OPENEDX_RELEASE:-latest}
     command: bash -c 'source /edx/app/xqueue/xqueue_env && while true; do python /edx/app/xqueue/xqueue/manage.py run_consumer ; sleep 2; done'
     volumes:
       - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue:cached
+    networks:
+      default:
+        aliases:
+          - edx.devstack.xqueue_consumer
     # depends_on: even though we need mysql, we can't refer to it because it's started in the other compose file