Simplify handling of Analytics Pipeline and Marketing

* Allow analytics pipeline to be provisioned and run
  just like other services.
* Test marketing site in its own shard.
* Allow marketing site to be provisioned and run just
  like other services.
* Clean up redundant Makefile rules a bit.
* Fix quality issues in provision.sh with
  shellcheck==0.7.0.

Author: kdmccormick

.travis.yml

@@ -10,12 +10,13 @@ branches:
     - master
 
 env:
-  - DEVSTACK_WORKSPACE=/tmp SHALLOW_CLONE=1 DEVSTACK='lms'
-  - DEVSTACK_WORKSPACE=/tmp SHALLOW_CLONE=1 DEVSTACK='registrar'
-  - DEVSTACK_WORKSPACE=/tmp SHALLOW_CLONE=1 DEVSTACK='ecommerce'
-  - DEVSTACK_WORKSPACE=/tmp SHALLOW_CLONE=1 DEVSTACK='edx_notes_api'
-  - DEVSTACK_WORKSPACE=/tmp SHALLOW_CLONE=1 DEVSTACK='credentials'
-  - DEVSTACK_WORKSPACE=/tmp SHALLOW_CLONE=1 DEVSTACK='analytics_pipeline'
+  - DEVSTACK_WORKSPACE=/tmp SHALLOW_CLONE=1 SERVICES='lms+discovery+forum'
+  - DEVSTACK_WORKSPACE=/tmp SHALLOW_CLONE=1 SERVICES='lms+registrar'
+  - DEVSTACK_WORKSPACE=/tmp SHALLOW_CLONE=1 SERVICES='ecommerce'
+  - DEVSTACK_WORKSPACE=/tmp SHALLOW_CLONE=1 SERVICES='edx_notes_api'
+  - DEVSTACK_WORKSPACE=/tmp SHALLOW_CLONE=1 SERVICES='credentials'
+  - DEVSTACK_WORKSPACE=/tmp SHALLOW_CLONE=1 SERVICES='analyticspipeline'
+  - DEVSTACK_WORKSPACE=/tmp SHALLOW_CLONE=1 SERVICES='marketing'
 
 services:
   - docker
@@ -27,15 +28,11 @@ before_install:
   - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
   - docker version
   - docker-compose --version
-
   - make requirements
   - make dev.clone
-  - |
-    if [[ $DEVSTACK == 'analytics_pipeline' ]]; then
-        make dev.up.analytics_pipeline
-    else
-        make dev.pull.${DEVSTACK}
-    fi
+  - make dev.pull."$SERVICES"
 
 script:
-  - "./.travis/run.sh"
+  - make dev.provision.services."$SERVICES"
+  - make dev.up."$SERVICES"
+  - make dev.check."$SERVICES"

.travis/run.sh

@@ -6,22 +6,23 @@
 ########################################################################################################################
 .DEFAULT_GOAL := help
 
-.PHONY: analytics-pipeline-devstack-test analytics-pipeline-shell backup \
-        build-courses check-memory create-test-course credentials-shell \
-        destroy dev.cache-programs dev.checkout dev.clone dev.clone.ssh \
-        dev.nfs.provision dev.nfs.provision.services dev.nfs.setup dev.nfs.up \
-        dev.nfs.up.all dev.nfs.up.watchers devpi-password dev.provision \
-        dev.provision.analytics_pipeline dev.provision.analytics_pipeline.run \
-        dev.provision.services dev.provision.xqueue dev.provision.xqueue.run \
-        dev.pull dev.repo.reset dev.reset dev.status dev.sync.daemon.start \
-        dev.sync.provision dev.sync.requirements dev.sync.up dev.up dev.up.all \
+.PHONY: analytics-pipeline-devstack-test analytics-pipeline-shell \
+        analyticspipeline-shell backup build-courses check-memory \
+        create-test-course credentials-shell destroy dev.cache-programs \
+        dev.checkout dev.clone dev.clone.ssh dev.nfs.provision \
+        dev.nfs.provision.services dev.nfs.setup dev.nfs.up dev.nfs.up.all \
+        dev.nfs.up.watchers devpi-password dev.provision \
+        dev.provision.analytics_pipeline dev.provision.services \
+        dev.provision.xqueue dev.provision.xqueue.run dev.pull dev.repo.reset \
+        dev.reset dev.status dev.sync.daemon.start dev.sync.provision \
+        dev.sync.requirements dev.sync.up dev.up dev.up.all \
         dev.up.analytics_pipeline dev.up.watchers dev.up.with-programs \
         dev.up.xqueue discovery-shell down e2e-shell e2e-tests ecommerce-shell \
         feature-toggle-state forum-restart-devserver healthchecks help lms-restart \
         lms-shell lms-static lms-update-db lms-watcher-shell logs mongo-shell \
         mysql-shell mysql-shell-edxapp provision pull pull.analytics_pipeline \
-        pull.xqueue registrar-shell requirements restore static stats stop \
-        stop.all stop.analytics_pipeline stop.watchers stop.xqueue \
+        pull.xqueue registrar-shell requirements restore selfcheck static \
+        stats stop stop.all stop.analytics_pipeline stop.watchers stop.xqueue \
         studio-restart studio-shell studio-static studio-update-db \
         studio-watcher-shell update-db upgrade upgrade validate \
         validate-lms-volume vnc-passwords xqueue_consumer-logs \
@@ -113,7 +114,7 @@ dev.provision.services: ## Provision all services with local mounted directories
 	DOCKER_COMPOSE_FILES="$(STANDARD_COMPOSE_SUITE)" $(WINPTY) bash ./provision.sh
 
 dev.provision.services.%: ## Provision specified services with local mounted directories, separated by plus signs
-	DOCKER_COMPOSE_FILES="$(STANDARD_COMPOSE_SUITE)" $(WINPTY) bash ./provision.sh $*
+	DOCKER_COMPOSE_FILES="$(FULL_COMPOSE_SUITE)" $(WINPTY) bash ./provision.sh $*
 
 dev.provision: | check-memory dev.clone.ssh dev.provision.services stop ## Provision dev environment with all services stopped
 
@@ -149,7 +150,7 @@ ifeq ($(ALWAYS_CACHE_PROGRAMS),true)
 endif
 
 dev.up.%: | check-memory ## Bring up specific services (separated by plus-signs) and their dependencies with host volumes.
-	bash -c 'docker-compose $(STANDARD_COMPOSE_SUITE) up -d $$(echo $* | tr + " ")'
+	bash -c 'docker-compose $(FULL_COMPOSE_SUITE) up -d $$(echo $* | tr + " ")'
 ifeq ($(ALWAYS_CACHE_PROGRAMS),true)
 	make dev.cache-programs
 endif
@@ -198,6 +199,9 @@ dev.sync.requirements: ## Install requirements
 dev.sync.up: dev.sync.daemon.start ## Bring up all services with docker-sync enabled
 	docker-compose $(STANDARD_COMPOSE_SUITE_FOR_SYNC) up -d
 
+dev.check.%:  # Run checks for a given service or set of services (separated by plus-signs).
+	$(WINPTY) bash ./check.sh $*
+
 provision: | dev.provision ## This command will be deprecated in a future release, use dev.provision
 	echo "\033[0;31mThis command will be deprecated in a future release, use dev.provision\033[0m"
 
@@ -288,6 +292,9 @@ restore: dev.up.mysql+mongo+elasticsearch ## Restore all data volumes from the h
 %-shell: ## Run a shell on the specified service container
 	docker exec -it edx.devstack.$* /bin/bash
 
+analyticspipeline-shell: ## Run a shell on the analytics pipeline container
+	docker exec -it edx.devstack.analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open
+
 credentials-shell: ## Run a shell on the credentials container
 	docker exec -it edx.devstack.credentials env TERM=$(TERM) bash -c 'source /edx/app/credentials/credentials_env && cd /edx/app/credentials/credentials && /bin/bash'
 
@@ -361,11 +368,9 @@ studio-static: ## Rebuild static assets for the Studio container
 
 static: | credentials-static discovery-static ecommerce-static lms-static studio-static ## Rebuild static assets for all service containers
 
-healthchecks: ## Run a curl against all services' healthcheck endpoints to make sure they are up. This will eventually be parameterized
-	$(WINPTY) bash ./check.sh all
+healthchecks: dev.check.registrar+lms+ecommerce+discovery+forum+edx_notes_api+credentials
 
-healthchecks.%:
-	$(WINPTY) bash ./check.sh $*
+healthchecks.%: dev.check.%
 
 e2e-tests: ## Run the end-to-end tests against the service containers
 	docker run -t --network=devstack_default -v ${DEVSTACK_WORKSPACE}/edx-e2e-tests:/edx-e2e-tests -v ${DEVSTACK_WORKSPACE}/edx-platform:/edx-e2e-tests/lib/edx-platform --env-file ${DEVSTACK_WORKSPACE}/edx-e2e-tests/devstack_env edxops/e2e env TERM=$(TERM)  bash -c 'paver e2e_test --exclude="whitelabel\|enterprise"'
@@ -391,23 +396,13 @@ mysql-shell-edxapp: ## Run a mysql shell on the edxapp database
 mongo-shell: ## Run a shell on the mongo container
 	docker-compose exec mongo bash
 
-### analytics pipeline commands
+dev.provision.analytics_pipeline: dev.provision.services.analyticspipeline
 
+analytics-pipeline-shell: analyticspipeline-shell
 
-dev.provision.analytics_pipeline: | check-memory dev.provision.analytics_pipeline.run stop.analytics_pipeline stop ## Provision analyticstack dev environment with all services stopped
-	echo "Ran dev.provision.analytics_pipeline"
-
-dev.provision.analytics_pipeline.run:
-	DOCKER_COMPOSE_FILES="$(ANALYTICS_COMPOSE_SUITE)" ./provision-analytics-pipeline.sh
-
-analytics-pipeline-shell: ## Run a shell on the analytics pipeline container
-	docker exec -it edx.devstack.analytics_pipeline env TERM=$(TERM) /edx/app/analytics_pipeline/devstack.sh open
-
-dev.up.analytics_pipeline: | check-memory ## Bring up analytics pipeline services
-	bash -c 'docker-compose $(ANALYTICS_COMPOSE_SUITE) up -d analyticspipeline'
+dev.up.analytics_pipeline: dev.up.analyticspipeline ## Bring up analytics pipeline services
 
-pull.analytics_pipeline: ## Update analytics pipeline docker images
-	docker-compose $(ANALYTICS_COMPOSE_SUITE) pull
+pull.analytics_pipeline: dev.pull.analyticspipeline ## Update analytics pipeline docker images
 
 analytics-pipeline-devstack-test: ## Run analytics pipeline tests in travis build
 	docker exec -u hadoop -i edx.devstack.analytics_pipeline bash -c 'sudo chown -R hadoop:hadoop /edx/app/analytics_pipeline && source /edx/app/hadoop/.bashrc && make develop-local && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_internal_reporting_database && make docker-test-acceptance-local ONLY_TESTS=edx.analytics.tasks.tests.acceptance.test_user_activity'

Makefile

@@ -105,6 +105,12 @@ if should_check credentials; then
         "curl --fail -L http://localhost:18150/health"
 fi
 
+if should_check marketing; then
+    echo "Seeing if we can curl root of Marketing site: "
+    run_check marketing_curl marketing \
+        "curl http://localhost:8080"
+fi
+
 echo "Successful checks:${succeeded:- NONE}"
 echo "Failed checks:${failed:- NONE}"
 if [[ "$succeeded" ]]; then

check.sh

@@ -16,20 +16,8 @@ elif [ ! -d "$DEVSTACK_WORKSPACE" ]; then
     exit 1
 fi
 
-# Bring the mysql & pipeline containers online.
-docker-compose $DOCKER_COMPOSE_FILES up -d mysql analyticspipeline
-
-# Ensure the MySQL server is online and usable
-echo "Waiting for MySQL"
-until docker exec -i edx.devstack.mysql mysql -uroot -se "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')" &> /dev/null
-do
-  printf "."
-  sleep 1
-done
-
-# In the event of a fresh MySQL container, wait a few seconds for the server to restart.
-# This can be removed once https://github.com/docker-library/mysql/issues/245 is resolved.
-sleep 20
+# Bring the pipeline containers online.
+docker-compose $DOCKER_COMPOSE_FILES up -d analyticspipeline
 
 # Analytics pipeline has dependency on lms but we only need its db schema & not full lms. So we'll just load their db
 # schemas as part of analytics pipeline provisioning. If there is a need of a fully fledged LMS, then provision lms
@@ -69,7 +57,3 @@ sleep 10 # for datanode & other services to activate
 echo -e "${GREEN}Namenode is ready!${NC}"
 
 docker exec -u hadoop -i edx.devstack.analytics_pipeline bash -c 'sudo /edx/app/hadoop/hadoop/bin/hdfs dfs -chown -R hadoop:hadoop hdfs://namenode:8020/; hdfs dfs -mkdir -p hdfs://namenode:8020/edx-analytics-pipeline/{warehouse,marker,manifest,packages} hdfs://namenode:8020/{spark-warehouse,data} hdfs://namenode:8020/tmp/spark-events;hdfs dfs -copyFromLocal -f /edx/app/hadoop/lib/edx-analytics-hadoop-util.jar hdfs://namenode:8020/edx-analytics-pipeline/packages/;'
-
-docker image prune -f
-
-echo -e "${GREEN}Analytics pipeline provisioning complete!${NC}"

provision-analytics-pipeline.sh renamed to provision-analyticspipeline.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+set -x
+
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+docker-compose $DOCKER_COMPOSE_FILES up -d marketing
+
+set +x
+echo -e "${YELLOW}edX Marketing Site is not fully provisioned yet.${NC}"
+echo -e "${YELLOW}For full setup, see:${NC}"
+echo -e "${YELLOW}    https://openedx.atlassian.net/wiki/spaces/ENG/pages/159162183/Marketing+Site${NC}"
+set -x

provision-marketing.sh

@@ -28,6 +28,7 @@
 
 set -e
 set -o pipefail
+set -u
 set -x
 
 RED='\033[0;31m'
@@ -37,13 +38,24 @@ NC='\033[0m' # No Color
 
 # All provisionable services.
 # Note: leading and trailing space are necessary for if-checks.
-ALL_SERVICES=" lms ecommerce discovery credentials e2e forum notes registrar "
+ALL_SERVICES=" \
+lms \
+ecommerce \
+discovery \
+credentials \
+e2e \
+forum \
+notes \
+registrar \
+analyticspipeline \
+marketing \
+ "
 
 # What should we provision?
 if [[ $# -eq 0 ]]; then
 	requested_services=$ALL_SERVICES
 else
-	arg_string=" $@ "
+	arg_string=" $* "
 	# Replace plus signs with spaces in order to allow plus-sign-separated
 	# services in addition to space-separated services.
 	requested_services="${arg_string//+/ }"
@@ -53,7 +65,7 @@ fi
 is_substring() {
 	local str="$1"
 	local substr="$2"
-	if [[ "$1" == *" ${2} "* ]]; then
+	if [[ "$str" == *" ${substr} "* ]]; then
 		return 0  # Note that '0' means 'success' (i.e., true) in bash.
 	else
 		return 1
@@ -145,7 +157,7 @@ fi
 # Run the service-specific provisioning script(s)
 for service in $to_provision; do
 	echo -e "${GREEN} Provisioning ${service}...${NC}"
-	./provision-${service}.sh
+	./provision-"$service".sh
 	echo -e "${GREEN} Provisioned ${service}.${NC}"
 done