Merge branch 'compose' into 'staging'

Compose

See merge request !350

.env

@@ -1 +1 @@
-MIST_TAG=staging
+MIST_TAG=compose

.gitlab-ci.yml

@@ -30,87 +30,115 @@ discover-submodules:
 
 #################### BUILD STAGE ####################
 
-.docker_wait_img_template: &docker_wait_img_template
+# Wait for images we depend on to become available.
+
+.docker_wait_submodule_img_template: &docker_wait_submodule_img_template
   stage: build
   variables:
     GIT_STRATEGY: none
     NUM: "60"
     SLEEP: "10"
-    RETAG: "1"
   script:
-    - SHA=$(grep ^$REPO, all-submodules.csv | cut -d, -f2)
-    - echo SHA is $SHA
-    - IMGTAG="$IMG:$SHA"
+    - SUBMODULE_SHA=$(grep ^$REPO, all-submodules.csv | cut -d, -f2)
+    - echo SUBMODULE_SHA is $SUBMODULE_SHA
+    - GCR_IMG="gcr.io/mist-ops/$IMG"
+    - DOCKERHUB_IMG="mist/$IMG"
+    - IMGTAG="$GCR_IMG:$SUBMODULE_SHA"
     - echo IMTAG is $IMGTAG
-    - DOCKERHUB_IMG=mist/$(echo $IMG | cut -d/ -f3)
-    - echo DOCKERHUB_IMG is $DOCKERHUB_IMG
 
     - echo "Will pull docker image $IMGTAG"
     - for i in $(seq 1 $NUM); do docker pull $IMGTAG && OK=1 && break || sleep $SLEEP; done
     - if [ -z "$OK" ]; then echo "ERROR Couldn't pull $IMGTAG!" && exit 1; fi
-    - if [ -z "$RETAG" ]; then exit 0; fi
-
-    - IMGTAG2="$IMG:io-$CI_COMMIT_SHA"
-    - echo "Retagging using mist.io sha"
-    - echo "$IMGTAG -> $IMGTAG2"
-    - docker tag $IMGTAG $IMGTAG2
-    - docker push $IMGTAG2
-
-    - IMGTAG2="$IMG:io-$CI_COMMIT_REF_SLUG"
-    - echo "Retagging using mist.io branch"
-    - echo "$IMGTAG -> $IMGTAG2"
-    - docker tag $IMGTAG $IMGTAG2
-    - docker push $IMGTAG2
-
-    - IMGTAG2="$DOCKERHUB_IMG:io-$CI_COMMIT_SHA"
-    - echo "Retagging using mist.io sha"
-    - echo "$IMGTAG -> $IMGTAG2"
-    - docker tag $IMGTAG $IMGTAG2
-    - docker push $IMGTAG2
-
-    - IMGTAG2="$DOCKERHUB_IMG:io-$CI_COMMIT_REF_SLUG"
-    - echo "Retagging to push to dockerhub"
-    - echo "$IMGTAG -> $IMGTAG2"
-    - docker tag $IMGTAG $IMGTAG2
-    - docker push $IMGTAG2
+
+    - |
+        for img_name in $GCR_IMG $DOCKERHUB_IMG; do
+          for ref in $CI_COMMIT_SHA $CI_COMMIT_REF_SLUG; do
+            IMGTAG2=$img_name:io-$ref
+            echo "Will tag and push $IMGTAG2"
+            docker tag $IMGTAG $IMGTAG2
+            docker push $IMGTAG2
+          done
+        done
   tags:
     - shell-docker-light
   dependencies:
     - discover-submodules
 
 wait-api-image:
-  <<: *docker_wait_img_template
+  <<: *docker_wait_submodule_img_template
   before_script:
     - REPO=mistio/mist.api
-    - IMG=gcr.io/mist-ops/mist
+    - IMG=mist
 
 wait-ui-image:
-  <<: *docker_wait_img_template
+  <<: *docker_wait_submodule_img_template
   before_script:
     - REPO=mistio/mist.ui
-    - IMG=gcr.io/mist-ops/ui
+    - IMG=ui
 
 wait-landing-image:
-  <<: *docker_wait_img_template
+  <<: *docker_wait_submodule_img_template
   before_script:
     - REPO=mistio/mist.landing
-    - IMG=gcr.io/mist-ops/landing
+    - IMG=landing
 
 wait-tests-image:
-  <<: *docker_wait_img_template
+  <<: *docker_wait_submodule_img_template
   before_script:
     - REPO=mistio/mist.tests
-    - IMG=gcr.io/mist-ops/tests_base
+    - IMG=tests_base
 
 
+# Build secondary images needed for deploying. Tag with SHA and branch name.
+
 .docker_build_img_template: &docker_build_img_template
+  stage: build
+  script:
+    - GCR_IMG="gcr.io/mist-ops/$IMG"
+    - DOCKERHUB_IMG="mist/$IMG"
+    - IMGTAG="$GCR_IMG:$CI_COMMIT_SHA"
+    - echo "Will build $IMGTAG"
+    - cd docker/$IMG
+    - docker build --pull -t $IMGTAG .
+    - |
+        for img_name in $GCR_IMG $DOCKERHUB_IMG; do
+          for ref in $CI_COMMIT_SHA $CI_COMMIT_REF_SLUG; do
+            IMGTAG2=$img_name:$ref
+            echo "Will tag and push $IMGTAG2"
+            docker tag $IMGTAG $IMGTAG2
+            docker push $IMGTAG2
+          done
+        done
+  tags:
+    - builder
+  dependencies: []
+
+build-nginx:
+  <<: *docker_build_img_template
+  variables:
+    IMG: nginx
+
+build-logstash:
+  <<: *docker_build_img_template
+  variables:
+    IMG: logstash
+
+build-elasticsearch-manage:
+  <<: *docker_build_img_template
+  variables:
+    IMG: elasticsearch-manage
+
+
+# Manually build images, tag is SHA/branch independant
+
+.docker_manual_build_img_template: &docker_manual_build_img_template
   stage: build
   when: manual
   script:
     - export TAG="${TAG:-latest}"
     - echo "Will build img $IMG with tag $TAG."
     - cd docker/$IMG
-    - docker build -t gcr.io/mist-ops/$IMG:$TAG .
+    - docker build --pull -t gcr.io/mist-ops/$IMG:$TAG .
     - docker tag gcr.io/mist-ops/$IMG:$TAG mist/$IMG:$TAG
     - docker push gcr.io/mist-ops/$IMG:$TAG
     - docker push mist/$IMG:$TAG
@@ -119,23 +147,23 @@ wait-tests-image:
   dependencies: []
 
 build-alpine-manual:
-  <<: *docker_build_img_template
+  <<: *docker_manual_build_img_template
   variables:
     IMG: alpine
     TAG: "3.4"
 
 build-socat-manual:
-  <<: *docker_build_img_template
+  <<: *docker_manual_build_img_template
   variables:
     IMG: docker-socat
 
 build-mailmock-manual:
-  <<: *docker_build_img_template
+  <<: *docker_manual_build_img_template
   variables:
     IMG: mailmock
 
 build-debugger-manual:
-  <<: *docker_build_img_template
+  <<: *docker_manual_build_img_template
   variables:
     IMG: debugger
 

README.md

@@ -1,8 +1,65 @@
 # mist.io
 
 Mist.io helps you operate, monitor and govern your computing infrastructure,
-across clouds and platforms. The code is provided under the GNU AGPL v3.0 
+across clouds and platforms. The code is provided under the GNU AGPL v3.0
 License.
 
 An enterprise version that includes Role Based Access, VPN tunnels and
 Insights for cost optimization is available as a service at https://mist.io
+
+
+## Installation
+
+Mist.io is a large application split into microservices which are packages in
+docker containers. The easiest way to run it is by using `docker-compose`. So,
+in order to run it, one needs to install a recent version of `docker` and
+`docker-compose`.
+
+There are two ways to run this application with docker-compose:
+
+
+### Single file deployment
+
+Simply download the `docker-compose.yml` file from this repository and put it
+in a directory. The directory name is used by `docker-compose` as the `project`
+name, so use something descriptive and unique, like `mist.io`.
+
+
+### Development deployment
+
+Clone this git repo and all its submodules with something like:
+
+    git clone --recursive https://github.com/mistio/mist.io.git
+    cd mist.io
+
+This may take some time.
+
+This setup will mount the checked out code into the containers. By cloning the
+directory, now there's also a `docker-compose.override.yml` file in the current
+directory in addition to `docker-compose.yml` and is used to modify the
+configuration for development mode.
+
+
+## Running mist.io
+
+Switch to the directory containing the `docker-compose.yml` file and run
+
+    docker-compose up -d
+
+This will start all the mist.io docker containers in the background.
+
+To create a user for the first time, first run
+
+    docker-compose exec api sh
+
+This should drop you in a shell into one of the mist.io containers. In there,
+run
+
+    ./bin/adduser --admin --docker-cloud [email protected]
+
+Replace the email address with yours. Try running `./bin/adduser -h` for more
+options.
+
+Visit http://localhost and login with the email and password specified above.
+
+Welcome to mist.io! Enjoy!

docker-compose.override.yml

@@ -0,0 +1,62 @@
+# Docker compose overrides for running mist.io in dev mode. This mounts local
+# code in the running containers, defines some extra containers needed for
+# development, etc. This is not a standalone file. It extends
+# docker-compose.yml and requires a local recursive clone of the mist.io git
+# repository.
+
+version: '2.0'
+
+services:
+
+
+  elasticsearch:
+    ports:
+      - 9200:9200
+
+  logstash:
+    volumes:
+      - ./docker/logstash/config:/config-dir:rw
+
+  kibana:
+    ports:
+      - 5601:5601
+
+  elasticsearch-manage:
+    image: mist/elasticsearch-manage:$MIST_TAG
+    volumes:
+      - ./docker/elasticsearch-manage:/opt/elasticsearch-manage
+
+
+  debugger:
+    image: mist/debugger
+    volumes:
+      - ./keys:/keys
+
+
+  celery: &backend
+    image: mist/mist:io-$MIST_TAG
+    volumes:
+      - ./api:/mist.api:rw
+      - ./keys:/keys
+      - ./elasticsearch:/elasticsearch
+  scheduler: *backend
+  poller: *backend
+  hubshell: *backend
+  api: *backend
+  sockjs: *backend
+
+
+  ui:
+    image: mist/ui:io-$MIST_TAG
+    volumes:
+      - ./ui:/ui:rw
+
+  landing:
+    image: mist/landing:io-$MIST_TAG
+    volumes:
+      - ./landing:/landing:rw
+
+
+  nginx:
+    volumes:
+      - ./docker/nginx/nginx.conf:/etc/nginx/nginx.conf:rw