feat(backup): honor s3 endpoint config value

Author: rwos

.travis.yml

@@ -2,7 +2,7 @@ sudo: required
 language: go
 services:
 - docker
-script: make docker-build && make test
+script: DEV_REGISTRY=quay.io make docker-build && make test
 notifications:
   slack:
     secure: pWDCV3od8gxvzxh9DrOTvBL54XoCfWYhZZlwd2ZbyyOz6SS12Psg/ZuCT2253p4yMfF/LPlsz76mr7NgcCrMI0ReveTa/rnt3XBZtyY+1rlsQsy2oxgdAzbO587ENCQeMw2F/OWHaixMT8NDqxEqQd6xafK9Zmg6BeBjwgs7XfXKcR3WzNIuCO0ZG05+Yd0FIxmd/8Xm5tGiFEYr05+Ix6MLdF9MSCXZUPeu1EsYXhDljokLq49w63W1UMU10tm4t7VCEdaO+X9w6EJ5Ov8HDxb6L6IviUYY6+IGTZ01nwIoM6OrGQqfEAytYqgTKdehgQzQnAbLI6TW2wJ0twqEsLrlbTa4NW4j0KkazQJkN5kqcKYQvaeKJJhvJIG44Gi/u78pW3S6W7NU5DhrlE6bbxdIBHJW1vJBimkqu2oBNrO5ZoBB9MS9zflBsU5g/pQpVeHWMnWE8fcYDGa1PqAcr7q6wtdPsrVZhnHmmARN3PwZzIVVVsXbaIQG8VLC5grLGnwMf1Y1fz2nK3sVpCftvrYZT3G6CNAASo+eLOwYdZdiJ9jIS7WNLN1GtpIEvjeDt3QRqsDyH8YoAKUvY5h/v8IWPP/BaSwQbJwep4+Dj7xkpXX5/4wm4jEnVFV1p4xE0lD1AXvEMAVHtPhhggvscNhF9j6oeoPju6eTPcxG+5o=

Makefile

@@ -3,7 +3,7 @@
 # - Docker image name
 # - Kubernetes service, rc, pod, secret, volume names
 SHORT_NAME := postgres
-DEIS_REGISTY ?= ${DEV_REGISTRY}/
+DEIS_REGISTRY ?= ${DEV_REGISTRY}/
 IMAGE_PREFIX ?= deis
 
 include versioning.mk

charts/database/templates/database-deployment.yaml

@@ -41,10 +41,6 @@ spec:
               value: "{{.Values.global.storage}}"
             - name: PGCTLTIMEOUT
               value: "{{.Values.postgres.timeout}}"
-{{- if eq .Values.global.storage "s3" }}
-            - name: S3_SSE
-              value: "{{.Values.s3.use_sse}}"
-{{- end}}
           lifecycle:
             preStop:
               exec:

rootfs/bin/create_bucket

@@ -12,6 +12,7 @@ from oauth2client.service_account import ServiceAccountCredentials
 from gcloud.storage.client import Client
 from gcloud import exceptions
 from azure.storage.blob import BlobService
+from urllib.parse import urlparse
 
 def bucket_exists(conn, name):
     bucket = conn.lookup(name)
@@ -23,25 +24,22 @@ bucket_name = os.getenv('BUCKET_NAME')
 region = os.getenv('S3_REGION')
 
 if os.getenv('DATABASE_STORAGE') == "s3":
-    conn = boto.s3.connect_to_region(region)
+    if os.getenv('S3_ENDPOINT'):
+        endpoint = urlparse(os.getenv('S3_ENDPOINT'))
+        conn = boto.s3.connect_to_region(region,
+            host=endpoint.hostname,
+            port=endpoint.port,
+            path=endpoint.path,
+            calling_format=boto.s3.connection.OrdinaryCallingFormat())
+    else:
+        conn = boto.s3.connect_to_region(region)
+
     if not bucket_exists(conn, bucket_name):
-        try:
-            if region == "us-east-1":
-                # use "US Standard" region. workaround for https://github.com/boto/boto3/issues/125
-                conn.create_bucket(bucket_name)
-            else:
-                conn.create_bucket(bucket_name, location=region)
-        # NOTE(bacongobbler): for versions prior to v2.9.0, the bucket is created in the default region.
-        # if we got here, we need to propagate "us-east-1" into WALE_S3_ENDPOINT because the bucket
-        # exists in a different region and we cannot find it.
-        # TODO(bacongobbler): deprecate this once we drop support for v2.8.0 and lower
-        except S3CreateError as err:
-            if region != 'us-east-1':
-                print('Failed to create bucket in {}. We are now assuming that the bucket was created in us-east-1.'.format(region))
-                with open(os.path.join(os.environ['WALE_ENVDIR'], "WALE_S3_ENDPOINT"), "w+") as file:
-                    file.write('https+path://s3.amazonaws.com:443')
-            else:
-                raise
+        if region == "us-east-1":
+            # use "US Standard" region. workaround for https://github.com/boto/boto3/issues/125
+            conn.create_bucket(bucket_name)
+        else:
+            conn.create_bucket(bucket_name, location=region)
 
 elif os.getenv('DATABASE_STORAGE') == "gcs":
     scopes = ['https://www.googleapis.com/auth/devstorage.full_control']

rootfs/docker-entrypoint-initdb.d/001_setup_envdir.sh

@@ -6,22 +6,25 @@ if [[ "$DATABASE_STORAGE" == "s3" || "$DATABASE_STORAGE" == "minio" ]]; then
   AWS_ACCESS_KEY_ID=$(cat /var/run/secrets/deis/objectstore/creds/accesskey)
   AWS_SECRET_ACCESS_KEY=$(cat /var/run/secrets/deis/objectstore/creds/secretkey)
   if [[ "$DATABASE_STORAGE" == "s3" ]]; then
+    USE_SSE=$(cat /var/run/secrets/deis/objectstore/creds/use-sse)
     AWS_REGION=$(cat /var/run/secrets/deis/objectstore/creds/region)
+    S3_ENDPOINT=$(cat /var/run/secrets/deis/objectstore/creds/endpoint)
     BUCKET_NAME=$(cat /var/run/secrets/deis/objectstore/creds/database-bucket)
-    # Convert $AWS_REGION into $WALE_S3_ENDPOINT to avoid "Connection reset by peer" from
-    # regions other than us-standard.
-    # See https://github.com/wal-e/wal-e/issues/167
-    # See https://github.com/boto/boto/issues/2207
-    if [[ "$AWS_REGION" == "us-east-1" ]]; then
-      echo "https+path://s3.amazonaws.com:443" > WALE_S3_ENDPOINT
+    if [[ "$S3_ENDPOINT" == "" ]]; then
+        # Convert $AWS_REGION into $WALE_S3_ENDPOINT to avoid "Connection reset by peer" from
+        # regions other than us-standard.
+        # See https://github.com/wal-e/wal-e/issues/167
+        # See https://github.com/boto/boto/issues/2207
+        if [[ "$AWS_REGION" == "us-east-1" ]]; then
+          echo "https+path://s3.amazonaws.com:443" > WALE_S3_ENDPOINT
+        else
+          echo "https+path://s3-${AWS_REGION}.amazonaws.com:443" > WALE_S3_ENDPOINT
+        fi
     else
-      echo "https+path://s3-${AWS_REGION}.amazonaws.com:443" > WALE_S3_ENDPOINT
-    fi
-    if [[ $S3_SSE ]]; then
-      echo $S3_SSE > WALE_S3_SSE
-    else
-      echo "false" > WALE_S3_SSE
+        echo "$S3_ENDPOINT" > S3_ENDPOINT
+        echo "$S3_ENDPOINT" | sed -E -e 's!http(s?)://!http\1+path://!' -e 's!/$!!' > WALE_S3_ENDPOINT
     fi
+    echo $USE_SSE > WALE_S3_SSE
   else
     AWS_REGION="us-east-1"
     BUCKET_NAME="dbwal"