Merge branch '84-external-artifact-storage' into 'master'

Upload CLI artifacts to external storage:
- create a GCP_SERVICE_KEY
- upload CLI binaries to GCP Buckets
- improve make commands
- add new target architectures

See merge request postgres-ai/database-lab!98

Author: agneum

.gitlab-ci.yml

@@ -15,6 +15,27 @@ lint:
   script:
     - make lint
 
+.only_var_template: &only_tag_release
+  only:
+    variables:
+      - $CI_COMMIT_TAG =~ /^[0-9.]+$/
+
+.only_var_template: &only_tag_rc
+  only:
+    variables:
+      - $CI_COMMIT_TAG =~ /^[0-9.]+[\-_]*[a-zA-Z]+[a-zA-Z0-9.\-_]*[a-zA-Z0-9]+$/
+
+.only_var_template: &only_master
+  only:
+    - master
+
+.only_var_template: &only_feature
+  only:
+    refs:
+      - branches
+    variables:
+      - $CI_COMMIT_REF_SLUG != "master"
+
 build-binary-alpine:
   image: golang:1.13-alpine
   stage: build-binary
@@ -26,20 +47,35 @@ build-binary-alpine:
     paths:
       - bin
   script:
-    - go build -o bin/dblab-server-alpine ./cmd/database-lab/main.go
-    - go build -o bin/dblab-alpine ./cmd/cli/main.go
+    - apk add --update --no-cache make
+    - make build
 
-build-binary-generic:
+build-binary-client-linux:
+  <<: *only_feature
   stage: build-binary
-  only:
-    refs:
-      - branches
-      - tags
   artifacts:
     paths:
       - bin
   script:
-    - make build-generic
+    - go build -o bin/dblab-linux-amd64 ./cmd/cli/main.go
+
+build-binary-client:
+  <<: *only_tag_release
+  stage: build-binary
+  script:
+    - make build-client
+
+    # Install google-cloud-sdk.
+    - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+    - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+    - apt-get update && apt-get install -y google-cloud-sdk
+
+    # Authenticate.
+    - echo $GCP_SERVICE_KEY | gcloud auth activate-service-account --key-file=-
+
+    # Upload artifacts.
+    - gsutil -m cp -r bin/cli/* gs://database-lab-cli/${CI_COMMIT_TAG}/
+    - gsutil -m cp -r bin/cli/* gs://database-lab-cli/latest/
 
 .job_template: &build_image_definition
   image: docker:19
@@ -53,27 +89,6 @@ build-binary-generic:
     - apk update && apk upgrade && apk add --no-cache bash # TODO(anatoly): Remove dependency.
     - bash ./scripts/ci_docker_build_push.sh
 
-.only_var_template: &only_tag_release
-  only:
-    variables:
-      - $CI_COMMIT_TAG =~ /^[0-9.]+$/
-
-.only_var_template: &only_tag_rc
-  only:
-    variables:
-      - $CI_COMMIT_TAG =~ /^[0-9.]+[\-_]*[a-zA-Z]+[a-zA-Z0-9.\-_]*[a-zA-Z0-9]+$/
-
-.only_var_template: &only_master
-  only:
-    - master
-
-.only_var_template: &only_feature
-  only:
-    refs:
-      - branches
-    variables:
-      - $CI_COMMIT_REF_SLUG != "master"
-
 build-image-feature-server:
   <<: *build_image_definition
   <<: *only_feature
@@ -84,8 +99,6 @@ build-image-feature-server:
     DOCKER_FILE: "Dockerfile.dblab-server"
     DOCKER_NAME: "registry.gitlab.com/postgres-ai/database-lab/dblab-server"
     TAGS: "${DOCKER_NAME}:${CI_COMMIT_REF_SLUG}"
-  before_script:
-    - cp ./bin/dblab-server-alpine ./bin/dblab-server
 
 build-image-feature-client:
   <<: *build_image_definition
@@ -97,8 +110,6 @@ build-image-feature-client:
     DOCKER_FILE: "Dockerfile.dblab"
     DOCKER_NAME: "registry.gitlab.com/postgres-ai/database-lab/dblab"
     TAGS: "${DOCKER_NAME}:${CI_COMMIT_REF_SLUG}"
-  before_script:
-    - cp ./bin/dblab-alpine ./bin/dblab
 
 build-image-feature-client-extended:
   <<: *build_image_definition
@@ -120,8 +131,6 @@ build-image-master-server:
     DOCKER_FILE: "Dockerfile.dblab-server"
     DOCKER_NAME: "registry.gitlab.com/postgres-ai/database-lab/dblab-server"
     TAGS: "${DOCKER_NAME}:master,${DOCKER_NAME}:master-${CI_COMMIT_SHORT_SHA}"
-  before_script:
-    - cp ./bin/dblab-server-alpine ./bin/dblab-server
 
 build-image-master-client:
   <<: *build_image_definition
@@ -130,8 +139,6 @@ build-image-master-client:
     DOCKER_FILE: "Dockerfile.dblab"
     DOCKER_NAME: "registry.gitlab.com/postgres-ai/database-lab/dblab"
     TAGS: "${DOCKER_NAME}:master,${DOCKER_NAME}:master-${CI_COMMIT_SHORT_SHA}"
-  before_script:
-    - cp ./bin/dblab-alpine ./bin/dblab
 
 build-image-latest-server:
   <<: *build_image_definition
@@ -143,8 +150,6 @@ build-image-latest-server:
     DOCKER_FILE: "Dockerfile.dblab-server"
     DOCKER_NAME: "postgresai/dblab-server"
     TAGS: "${DOCKER_NAME}:latest,${DOCKER_NAME}:${CI_COMMIT_TAG}"
-  before_script:
-    - cp ./bin/dblab-server-alpine ./bin/dblab-server
 
 build-image-latest-client:
   <<: *build_image_definition
@@ -156,8 +161,6 @@ build-image-latest-client:
     DOCKER_FILE: "Dockerfile.dblab"
     DOCKER_NAME: "postgresai/dblab"
     TAGS: "${DOCKER_NAME}:latest,${DOCKER_NAME}:${CI_COMMIT_TAG}"
-  before_script:
-    - cp ./bin/dblab-alpine ./bin/dblab
 
 build-image-rc-server:
   <<: *build_image_definition
@@ -169,8 +172,6 @@ build-image-rc-server:
     DOCKER_FILE: "Dockerfile.dblab-server"
     DOCKER_NAME: "postgresai/dblab-server"
     TAGS: "${DOCKER_NAME}:${CI_COMMIT_TAG}"
-  before_script:
-    - cp ./bin/dblab-server-alpine ./bin/dblab-server
 
 build-image-rc-client:
   <<: *build_image_definition
@@ -182,8 +183,6 @@ build-image-rc-client:
     DOCKER_FILE: "Dockerfile.dblab"
     DOCKER_NAME: "postgresai/dblab"
     TAGS: "${DOCKER_NAME}:${CI_COMMIT_TAG}"
-  before_script:
-    - cp ./bin/dblab-alpine ./bin/dblab
 
 build-image-swagger-latest:
   <<: *build_image_definition

Makefile

@@ -24,8 +24,8 @@ GOBUILD = GO111MODULE=on GOARCH=${GOARCH} go build ${LDFLAGS}
 GOTEST = GO111MODULE=on go test -race 
 GORUN = GO111MODULE=on go run ${LDFLAGS}
 
-PLATFORMS=darwin linux
-ARCHITECTURES=386 amd64
+CLIENT_PLATFORMS=darwin linux freebsd windows
+ARCHITECTURES=amd64
 
 # Build the project
 all: clean build
@@ -43,14 +43,13 @@ build:
 	${GOBUILD} -o bin/${SERVER_BINARY} ./cmd/database-lab/main.go
 	${GOBUILD} -o bin/${CLI_BINARY} ./cmd/cli/main.go
 
-build-generic:
-	$(foreach GOOS, $(PLATFORMS),\
+build-client:
+	$(foreach GOOS, $(CLIENT_PLATFORMS),\
 		$(foreach GOARCH, $(ARCHITECTURES), \
 		$(shell \
 			export GOOS=$(GOOS); \
 			export GOARCH=$(GOARCH); \
-			go build -o bin/$(SERVER_BINARY)-$(GOOS)-$(GOARCH) ./cmd/database-lab/main.go; \
-			go build -o bin/$(CLI_BINARY)-$(GOOS)-$(GOARCH) ./cmd/cli/main.go)))
+			go build -o bin/cli/$(CLI_BINARY)-$(GOOS)-$(GOARCH) ./cmd/cli/main.go)))
 
 test:
 	${GOTEST} ./...

scripts/cli_install.sh

@@ -4,8 +4,16 @@
 
 mkdir -p ~/.dblab
 
+uname_os() {
+  os=$(uname -s | tr '[:upper:]' '[:lower:]')
+  case "$os" in
+    cygwin_nt*|mingw*|msys_nt*|nt*|win*) os="windows" ;;
+  esac
+  echo "$os"
+}
+
 curl --location --fail --output ~/.dblab/dblab \
-  https://gitlab.com/postgres-ai/database-lab/-/jobs/artifacts/master/raw/bin/dblab-linux-amd64?job=build-binary-generic \
+  https://storage.googleapis.com/database-lab-cli/latest/dblab-$(uname_os)-amd64 \
   && chmod a+x ~/.dblab/dblab
 
 {