feat(minio): support the latest minio

Author: duanhongyi

Dockerfile

@@ -36,8 +36,12 @@ RUN apt-get purge -y --auto-remove $BUILD_DEPS && \
 COPY rootfs /
 ENV WALE_ENVDIR=/etc/wal-e.d/env
 RUN mkdir -p $WALE_ENVDIR
-RUN python3 /patcher-script.py /bin/create_bucket
-RUN python3 /patcher-script.py /usr/local/bin/wal-e
+
+ARG PATCH_CMD="python3 /patcher-script.py"
+RUN $PATCH_CMD file /bin/create_bucket /patcher-script.d/patch_boto_s3.py
+RUN $PATCH_CMD file /usr/local/bin/wal-e /patcher-script.d/patch_boto_s3.py
+RUN $PATCH_CMD module wal_e.worker.worker_util /patcher-script.d/patch_wal_e_s3.py
+
 
 CMD ["/docker-entrypoint.sh", "postgres"]
 EXPOSE 5432

rootfs/bin/create_bucket

@@ -76,9 +76,11 @@ elif os.getenv('DATABASE_STORAGE') == "swift":
     conn.put_container(os.getenv('BUCKET_NAME'))
 
 else:
-    botoconfig.add_section('s3')
+    if not botoconfig.has_section("s3"):
+        botoconfig.add_section('s3')
     botoconfig.set('s3', 'use-sigv4', 'True')
-    botoconfig.add_section('Boto')
+    if not botoconfig.has_section("Boto"):
+        botoconfig.add_section('Boto')
     botoconfig.set('Boto', 'is_secure', 'False')
     conn = S3Connection(
         host=os.getenv('S3_HOST'),

rootfs/patcher-script.d/patch_boto_s3.py

@@ -1,12 +1,9 @@
-def patch_wal_e_hmac_auth_v4_handler():
+def patch_boto_s3_hmac_auth_v4_handler():
     import os
     from boto.auth import HmacAuthV4Handler
     _init = HmacAuthV4Handler.__init__
     def wrap_init(self, *args, **kwargs):
         _init(self, *args, **kwargs)
         self.region_name = os.getenv('S3_REGION', self.region_name)
     HmacAuthV4Handler.__init__ = wrap_init
-
-
-if __name__ == '__main__':
-    patch_wal_e_hmac_auth_v4_handler()
+patch_boto_s3_hmac_auth_v4_handler()

rootfs/patcher-script.d/patch_wal_e_s3.py

@@ -0,0 +1,18 @@
+def patch_uri_put_file():
+    import os
+    from wal_e.blobstore import s3
+    from wal_e.blobstore.s3 import s3_util
+    def wrap_uri_put_file(creds, uri, fp, content_type=None, conn=None):
+        assert fp.tell() == 0
+        k = s3_util._uri_to_key(creds, uri, conn=conn)
+        if content_type is not None:
+            k.content_type = content_type
+        if os.getenv('DATABASE_STORAGE') == 's3':
+            encrypt_key=True
+        else:
+            encrypt_key=False
+        k.set_contents_from_file(fp, encrypt_key=encrypt_key)
+        return k
+    s3.uri_put_file = wrap_uri_put_file
+    s3_util.uri_put_file = wrap_uri_put_file
+patch_uri_put_file()

rootfs/patcher-script.py

@@ -1,34 +1,41 @@
 import sys
 
-patch_script = """
+patch_script_tmp = """
 
 def run_patch_scripts(patch_script_path):
-    import os
-    for patch in os.listdir(patch_script_path):
-        full_patch_file = os.path.join(patch_script_path, patch)
-        if full_patch_file.endswith('.py') and os.path.isfile(full_patch_file):
-            with open(full_patch_file, 'r') as f:
-                try:
-                    exec(f.read())
-                except:
-                    pass
-run_patch_scripts('/patcher-script.d')
+    with open(patch_script_path, 'r') as f:
+        try:
+            exec(f.read())
+        except:
+            pass
+run_patch_scripts("%s")
 
 """
 
 
-def main(patch_file):
+def main(patch_file, patch_script_file):
     result_list = []
+    patch_script = patch_script_tmp % patch_script_file
     with open(patch_file, "r") as f:
         has_patched = False
         for line in f:
-            if not has_patched and line.startswith('import'):
+            if (line.startswith('import') or line.startswith('from')) \
+                    and not has_patched:
                 result_list.append(patch_script)
                 has_patched = True
             result_list.append(line)
+        if not has_patched: result_list.append(patch_script)
     with open(patch_file, "w") as f:
         for line in result_list:
             f.write(line)
 
 if __name__ == '__main__':
-    main(sys.argv[1])
+    patch_type = sys.argv[1]
+    if patch_type == 'file':
+        patch_file = sys.argv[2]
+    elif patch_type == 'module':
+        module = __import__(sys.argv[2], fromlist=True)
+        patch_file = module.__file__
+    patch_script_file = sys.argv[3]
+    main(patch_file, patch_script_file)
+