diff --git a/.ci.s3cfg b/.ci.s3cfg index 91797afe..d2283daa 100644 --- a/.ci.s3cfg +++ b/.ci.s3cfg @@ -6,7 +6,7 @@ add_headers = bucket_location = us-east-1 ca_certs_file = cache_file = -check_ssl_certificate = True +check_ssl_certificate = False check_ssl_hostname = True cloudfront_host = cloudfront.amazonaws.com default_mime_type = binary/octet-stream @@ -67,9 +67,9 @@ stop_on_error = False storage_class = urlencoding_mode = normal use_http_expect = False -use_https = False +use_https = True use_mime_magic = True verbosity = WARNING -website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/ +website_endpoint = https://%(bucket)s.s3-website-%(location)s.amazonaws.com/ website_error = website_index = index.html diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e5e5db1b..47a4e0ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,17 +28,33 @@ jobs: with: path: ~/cache key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.cache-revision }} + - name: Cache certgen + id: cache-certgen + uses: actions/cache@v2 + env: + cache-name: cache-certgen + with: + path: ~/.minio/certs + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.cache-revision }} - name: Download minio on cache miss if: steps.cache-minio.outputs.cache-hit != 'true' run: | mkdir -p ~/cache test ! -e ~/cache/minio && wget -O ~/cache/minio https://dl.minio.io/server/minio/release/linux-amd64/minio || echo "Minio already in cache" + - name: Download certgen on cache miss + if: steps.cache-certgen.outputs.cache-hit != 'true' + run: | + mkdir -p ~/.minio/certs + test ! -e ~/.minio/certs/certgen && wget -O ~/.minio/certs/certgen https://github.com/minio/certgen/releases/download/v0.0.2/certgen-linux-amd64 || echo "Certgen already in cache" + chmod +x ~/.minio/certs/certgen + cd ~/.minio/certs + ~/.minio/certs/certgen -ca -host "localhost" - name: Start a local instance of minio run: | export AWS_ACCESS_KEY_ID=Q3AM3UQ867SPQQA43P2F export AWS_SECRET_ACCESS_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG - export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F - export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG + export MINIO_ROOT_USER=Q3AM3UQ867SPQQA43P2F + export MINIO_ROOT_PASSWORD=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG chmod +x ~/cache/minio mkdir -p ~/minio_tmp ~/cache/minio server ~/minio_tmp & diff --git a/S3/Config.py b/S3/Config.py index 68c14174..2854236a 100644 --- a/S3/Config.py +++ b/S3/Config.py @@ -132,6 +132,8 @@ class Config(object): extra_headers = SortedDict(ignore_case = True) force = False server_side_encryption = False + sse_customer_key = "" + sse_copy_source_customer_key = "" enable = None get_continue = False put_continue = False @@ -293,6 +295,10 @@ def __init__(self, configfile = None, access_key=None, secret_key=None, access_t warning('Cannot have server_side_encryption (S3 SSE) and KMS_key set (S3 KMS). KMS encryption will be used. Please set server_side_encryption to False') if self.kms_key and self.signature_v2 == True: raise Exception('KMS encryption requires signature v4. Please set signature_v2 to False') + if self.sse_customer_key and len(self.sse_customer_key) != 32: + raise Exception('sse-customer-key must be 32 characters') + if self.sse_copy_source_customer_key and len(self.sse_copy_source_customer_key) != 32: + raise Exception('sse_copy_source_customer_key must be 32 characters') def role_config(self): """ diff --git a/S3/FileDict.py b/S3/FileDict.py index 3890248f..aac4e31f 100644 --- a/S3/FileDict.py +++ b/S3/FileDict.py @@ -43,7 +43,7 @@ def get_md5(self, relative_file): if 'md5' in self[relative_file]: return self[relative_file]['md5'] md5 = self.get_hardlink_md5(relative_file) - if md5 is None and 'md5' in cfg.sync_checks: + if md5 is None and 'md5' in cfg.preserve_attrs_list: logging.debug(u"doing file I/O to read md5 of %s" % relative_file) md5 = Utils.hash_file_md5(self[relative_file]['full_name']) self.record_md5(relative_file, md5) diff --git a/S3/FileLists.py b/S3/FileLists.py index d2d70bec..111f9d65 100644 --- a/S3/FileLists.py +++ b/S3/FileLists.py @@ -552,6 +552,23 @@ def _compare(src_list, dst_lst, src_remote, dst_remote, file): attribs_match = False debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5)) + # Check mtime. This compares local mtime to the upload time of remote file + compare_mtime = 'mtime' in cfg.sync_checks + if attribs_match and compare_mtime: + try: + src_mtime = src_list[file]['mtime'] + dst_mtime = dst_list[file]['timestamp'] + except (IOError,OSError): + # mtime sum verification failed - ignore that file altogether + debug(u"IGNR: %s (disappeared)" % (file)) + warning(u"%s: file disappeared, ignoring." % (file)) + raise + + if src_mtime > dst_mtime: + ## checksums are different. + attribs_match = False + debug(u"XFER: %s (mtime newer than last upload: src=%s dst=%s)" % (file, src_mtime, dst_mtime)) + return attribs_match # we don't support local->local sync, use 'rsync' or something like that instead ;-) diff --git a/S3/S3.py b/S3/S3.py index 04e06a63..55aa8414 100644 --- a/S3/S3.py +++ b/S3/S3.py @@ -8,17 +8,18 @@ from __future__ import absolute_import, division -import sys -import os -import time import errno -import mimetypes import io +import mimetypes +import os import pprint -from xml.sax import saxutils +import sys +import time +from logging import debug, error, info, warning from socket import timeout as SocketTimeoutException -from logging import debug, info, warning, error from stat import ST_SIZE +from xml.sax import saxutils + try: # python 3 support from urlparse import urlparse @@ -26,10 +27,11 @@ from urllib.parse import urlparse try: # Python 2 support - from base64 import encodestring + from base64 import encodestring, b64encode except ImportError: # Python 3.9.0+ support from base64 import encodebytes as encodestring + from base64 import b64encode import select @@ -38,26 +40,27 @@ except ImportError: from md5 import md5 -from .BaseUtils import (getListFromXml, getTextFromXml, getRootTagName, - decode_from_s3, encode_to_s3, s3_quote) -from .Utils import (convertHeaderTupleListToDict, hash_file_md5, unicodise, - deunicodise, check_bucket_name, - check_bucket_name_dns_support, getHostnameFromBucket, - calculateChecksum) -from .SortedDict import SortedDict from .AccessLog import AccessLog from .ACL import ACL, GranteeLogDelivery +from .BaseUtils import (decode_from_s3, encode_to_s3, getListFromXml, + getRootTagName, getTextFromXml, s3_quote) from .BidirMap import BidirMap from .Config import Config +from .ConnMan import ConnMan +from .Crypto import (checksum_sha256_buffer, checksum_sha256_file, + format_param_str, sign_request_v2, sign_request_v4) from .Exceptions import * from .MultiPart import MultiPartUpload from .S3Uri import S3Uri -from .ConnMan import ConnMan -from .Crypto import (sign_request_v2, sign_request_v4, checksum_sha256_file, - checksum_sha256_buffer, format_param_str) +from .SortedDict import SortedDict +from .Utils import (calculateChecksum, check_bucket_name, + check_bucket_name_dns_support, + convertHeaderTupleListToDict, deunicodise, + getHostnameFromBucket, hash_file_md5, unicodise) try: from ctypes import ArgumentError + import magic try: ## https://github.com/ahupp/python-magic @@ -701,6 +704,14 @@ def object_put(self, filename, uri, extra_headers = None, extra_label = ""): headers['x-amz-server-side-encryption'] = 'aws:kms' headers['x-amz-server-side-encryption-aws-kms-key-id'] = self.config.kms_key + if self.config.sse_customer_key: + sse_customer_key = encode_to_s3(self.config.sse_customer_key) + key_encoded = b64encode(sse_customer_key) + md5_encoded = b64encode(md5(sse_customer_key).digest()) + headers["x-amz-server-side-encryption-customer-algorithm"] = "AES256" + headers["x-amz-server-side-encryption-customer-key"] = decode_from_s3(key_encoded) + headers["x-amz-server-side-encryption-customer-key-md5"] = decode_from_s3(md5_encoded) + ## MIME-type handling headers["content-type"] = self.content_type(filename=filename) @@ -755,10 +766,30 @@ def object_put(self, filename, uri, extra_headers = None, extra_label = ""): response = self.send_file(request, src_stream, labels) return response - def object_get(self, uri, stream, dest_name, start_position = 0, extra_label = ""): + def object_get(self, uri, stream, dest_name, extra_headers, start_position = 0, extra_label = ""): if uri.type != "s3": raise ValueError("Expected URI type 's3', got '%s'" % uri.type) - request = self.create_request("OBJECT_GET", uri = uri) + headers = SortedDict(ignore_case=True) + if extra_headers: + headers.update(extra_headers) + ## Set server side encryption + if self.config.server_side_encryption: + headers["x-amz-server-side-encryption"] = "AES256" + + ## Set kms headers + if self.config.kms_key: + headers['x-amz-server-side-encryption'] = 'aws:kms' + headers['x-amz-server-side-encryption-aws-kms-key-id'] = self.config.kms_key + + if self.config.sse_customer_key: + sse_customer_key = encode_to_s3(self.config.sse_customer_key) + key_encoded = b64encode(sse_customer_key) + md5_encoded = b64encode(md5(sse_customer_key).digest()) + headers["x-amz-server-side-encryption-customer-algorithm"] = "AES256" + headers["x-amz-server-side-encryption-customer-key"] = decode_from_s3(key_encoded) + headers["x-amz-server-side-encryption-customer-key-md5"] = decode_from_s3(md5_encoded) + + request = self.create_request("OBJECT_GET", uri = uri, headers=headers) labels = { 'source' : uri.uri(), 'destination' : dest_name, 'extra' : extra_label } response = self.recv_file(request, stream, labels, start_position) return response @@ -954,6 +985,14 @@ def object_copy(self, src_uri, dst_uri, extra_headers=None, headers['x-amz-server-side-encryption-aws-kms-key-id'] = \ self.config.kms_key + if self.config.sse_copy_source_customer_key: + sse_copy_source_customer_key = encode_to_s3(self.config.sse_copy_source_customer_key) + key_encoded = b64encode(sse_copy_source_customer_key) + md5_encoded = b64encode(md5(sse_copy_source_customer_key).digest()) + headers["x-amz-copy-source-server-side-encryption-customer-algorithm"] = "AES256" + headers["x-amz-copy-source-server-side-encryption-customer-key"] = decode_from_s3(key_encoded) + headers["x-amz-copy-source-server-side-encryption-customer-key-md5"] = decode_from_s3(md5_encoded) + # Following meta data are not updated in simple COPY by aws. if extra_headers: headers.update(extra_headers) @@ -1828,19 +1867,32 @@ def send_file(self, request, stream, labels, buffer = '', throttle = 0, ## Non-recoverable error raise S3Error(response) - debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"].get('etag', '').strip('"\''))) - ## when using KMS encryption, MD5 etag value will not match - md5_from_s3 = response["headers"].get("etag", "").strip('"\'') - if ('-' not in md5_from_s3) and (md5_from_s3 != md5_hash.hexdigest()) and response["headers"].get("x-amz-server-side-encryption") != 'aws:kms': - warning("MD5 Sums don't match!") - if retries: - warning("Retrying upload of %s" % (filename)) - return self.send_file(request, stream, labels, buffer, throttle, - retries - 1, offset, chunk_size, use_expect_continue) + if self.config.sse_customer_key: + if response["headers"]["x-amz-server-side-encryption-customer-key-md5"] != \ + request.headers["x-amz-server-side-encryption-customer-key-md5"]: + warning("MD5 of customer key don't match!") + if retries: + warning("Retrying upload of %s" % (filename)) + return self.send_file(request, stream, labels, buffer, throttle, retries - 1, offset, chunk_size) + else: + warning("Too many failures. Giving up on '%s'" % (filename)) + raise S3UploadError else: - warning("Too many failures. Giving up on '%s'" % (filename)) - raise S3UploadError("Too many failures. Giving up on '%s'" - % filename) + debug("Match of x-amz-server-side-encryption-customer-key-md5") + else: + debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"].get('etag', '').strip('"\''))) + ## when using KMS encryption, MD5 etag value will not match + md5_from_s3 = response["headers"].get("etag", "").strip('"\'') + if ('-' not in md5_from_s3) and (md5_from_s3 != md5_hash.hexdigest()) and response["headers"].get("x-amz-server-side-encryption") != 'aws:kms': + warning("MD5 Sums don't match!") + if retries: + warning("Retrying upload of %s" % (filename)) + return self.send_file(request, stream, labels, buffer, throttle, + retries - 1, offset, chunk_size, use_expect_continue) + else: + warning("Too many failures. Giving up on '%s'" % (filename)) + raise S3UploadError("Too many failures. Giving up on '%s'" + % filename) return response diff --git a/run-tests-minio.py b/run-tests-minio.py index c493a814..458dac48 100755 --- a/run-tests-minio.py +++ b/run-tests-minio.py @@ -743,6 +743,24 @@ def pbucket(tail): test_s3cmd("Simple delete with rm", ['rm', '%s/xyz/test_rm/TypeRa.ttf' % pbucket(1)], must_find = [ "delete: '%s/xyz/test_rm/TypeRa.ttf'" % pbucket(1) ]) +## ====== Check SSE-C encrypted object upload +test_s3cmd("Put server-side encrypted object", ['put', 'testsuite/demo/some-file.xml', 's3://%s/xyz/demo/some-file.xml' % bucket(1), '--sse-customer-key=12345678901234567890123456789012'], + retcode=0, + must_find=["upload: 'testsuite/demo/some-file.xml' -> '%s/xyz/demo/some-file.xml'" % pbucket(1)]) + +## ====== Check SSE-C encrypted object wrong passphrase +test_s3cmd("Get server-side encrypted object with wrong passphrase", ['get', u'%s/xyz/demo/some-file.xml' % pbucket(1), 'testsuite-out', '--sse-customer-key=11111111111111111111111111111111'], + retcode = EX_ACCESSDENIED, + must_find = [ "Access Denied." ]) + +## ====== Check SSE-C encrypted object download +test_s3cmd("Get server-side encrypted object", ['get', u'%s/xyz/demo/some-file.xml' % pbucket(1), 'testsuite-out', '--sse-customer-key=12345678901234567890123456789012'], + retcode = 0, + must_find = [ "-> 'testsuite-out/some-file.xml'" ]) + +## ====== Clean up local destination dir +test_flushdir("Clean testsuite-out/", "testsuite-out") + ## ====== Create expiration rule with days and prefix # Minio: disabled #test_s3cmd("Create expiration rule with days and prefix", ['expire', pbucket(1), '--expiry-days=365', '--expiry-prefix=log/'], diff --git a/run-tests.py b/run-tests.py index 3145ba8a..b218881b 100755 --- a/run-tests.py +++ b/run-tests.py @@ -733,6 +733,24 @@ def pbucket(tail): test_s3cmd("Simple delete with rm", ['rm', '%s/xyz/test_rm/TypeRa.ttf' % pbucket(1)], must_find = [ "delete: '%s/xyz/test_rm/TypeRa.ttf'" % pbucket(1) ]) +## ====== Check SSE-C encrypted object upload +test_s3cmd("Put server-side encrypted object", ['put', 'testsuite/demo/some-file.xml', 's3://%s/xyz/demo/some-file.xml' % bucket(1), '--sse-customer-key=12345678901234567890123456789012'], + retcode=0, + must_find=["upload: 'testsuite/demo/some-file.xml' -> '%s/xyz/demo/some-file.xml'" % pbucket(1)]) + +## ====== Check SSE-C encrypted object wrong passphrase +test_s3cmd("Get server-side encrypted object with wrong passphrase", ['get', u'%s/xyz/demo/some-file.xml' % pbucket(1), 'testsuite-out', '--sse-customer-key=11111111111111111111111111111111'], + retcode = EX_SERVERERROR, + must_find = [ "The calculated MD5 hash of the key did not match" ]) + +## ====== Check SSE-C encrypted object download +test_s3cmd("Get server-side encrypted object", ['get', u'%s/xyz/demo/some-file.xml' % pbucket(1), 'testsuite-out', '--sse-customer-key=12345678901234567890123456789012'], + retcode = 0, + must_find = [ "-> 'testsuite-out/some-file.xml'" ]) + +## ====== Clean up local destination dir +test_flushdir("Clean testsuite-out/", "testsuite-out") + ## ====== Create expiration rule with days and prefix test_s3cmd("Create expiration rule with days and prefix", ['expire', pbucket(1), '--expiry-days=365', '--expiry-prefix=log/'], must_find = [ "Bucket '%s/': expiration configuration is set." % pbucket(1)]) diff --git a/s3cmd b/s3cmd index f1c99895..b0d5743d 100755 --- a/s3cmd +++ b/s3cmd @@ -531,6 +531,8 @@ def cmd_object_get(args): remote_count = len(remote_list) + extra_headers = copy(cfg.extra_headers) + info(u"Summary: %d remote files to download" % remote_count) if remote_count > 0: @@ -618,7 +620,7 @@ def cmd_object_get(args): try: try: - response = s3.object_get(uri, dst_stream, destination, start_position = start_position, extra_label = seq_label) + response = s3.object_get(uri, dst_stream, destination, extra_headers, start_position = start_position, extra_label = seq_label) finally: dst_stream.close() except S3DownloadError as e: @@ -1341,6 +1343,7 @@ def cmd_sync_remote2local(args): def _download(remote_list, seq, total, total_size, dir_cache): original_umask = os.umask(0) os.umask(original_umask) + extra_headers = copy(cfg.extra_headers) file_list = remote_list.keys() file_list.sort() ret = EX_OK @@ -1375,7 +1378,7 @@ def cmd_sync_remote2local(args): with io.open(chkptfd, mode='wb') as dst_stream: dst_stream.stream_name = unicodise(chkptfname_b) debug(u"created chkptfname=%s" % dst_stream.stream_name) - response = s3.object_get(uri, dst_stream, dst_file, extra_label = seq_label) + response = s3.object_get(uri, dst_stream, dst_file, extra_headers, extra_label = seq_label) # download completed, rename the file to destination if os.name == "nt": @@ -1920,6 +1923,15 @@ def cmd_sync_local2remote(args): error(u"or disable encryption with --no-encrypt parameter.") sys.exit(EX_USAGE) + # Disable md5 checks if using SSE-C. Add mtime check + if cfg.sse_customer_key or cfg.sse_copy_source_customer_key: + try: + cfg.sync_checks.remove("md5") + except Exception: + pass + if "mtime" not in cfg.sync_checks: + cfg.sync_checks.append("mtime") + for arg in args[:-1]: if not os.path.exists(deunicodise(arg)): raise ParameterError("Invalid source: '%s' is not an existing file or directory" % arg) @@ -2365,6 +2377,8 @@ def run_configure(config_file, args): ("gpg_passphrase", "Encryption password", "Encryption password is used to protect your files from reading\nby unauthorized persons while in transfer to S3"), ("gpg_command", "Path to GPG program"), ("use_https", "Use HTTPS protocol", "When using secure HTTPS protocol all communication with Amazon S3\nservers is protected from 3rd party eavesdropping. This method is\nslower than plain HTTP, and can only be proxied with Python 2.7 or newer"), + ("sse_customer_key", "Encryption key for server-side-encryption with customer key.\nMust be 32 characters"), + ("sse_copy_source_customer_key", "Encryption key for server-side-encryption with customer key.\nMust be 32 characters"), ("proxy_host", "HTTP Proxy server name", "On some networks all internet access must go through a HTTP proxy.\nTry setting it here if you can't connect to S3 directly"), ("proxy_port", "HTTP Proxy server port"), ] @@ -2804,6 +2818,8 @@ def main(): optparser.add_option( "--server-side-encryption", dest="server_side_encryption", action="store_true", help="Specifies that server-side encryption will be used when putting objects. [put, sync, cp, modify]") optparser.add_option( "--server-side-encryption-kms-id", dest="kms_key", action="store", help="Specifies the key id used for server-side encryption with AWS KMS-Managed Keys (SSE-KMS) when putting objects. [put, sync, cp, modify]") + optparser.add_option( "--sse-customer-key", dest="sse_customer_key", action="store", metavar="12345678901234567890123456789012", help="Specifies a customer provided key for server-side encryption. Must be 32 character string.") + optparser.add_option( "--sse-copy-source-customer-key", dest="sse_copy_source_customer_key", action="store", metavar="12345678901234567890123456789012", help="Specifies the encryption key for copying or moving objects with a customer provided key for server-side encryption.") optparser.add_option( "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % autodetected_encoding) optparser.add_option( "--add-encoding-exts", dest="add_encoding_exts", metavar="EXTENSIONs", help="Add encoding to these comma delimited extensions i.e. (css,js,html) when uploading to S3 )") @@ -3081,6 +3097,12 @@ def main(): if options.content_type: cfg.content_type = options.content_type + if options.sse_customer_key: + cfg.sse_customer_key = options.sse_customer_key + + if options.sse_copy_source_customer_key: + cfg.sse_copy_source_customer_key = options.sse_copy_source_customer_key + if len(args) < 1: optparser.print_help() sys.exit(EX_USAGE) diff --git a/s3cmd.1 b/s3cmd.1 index 224fa037..0e1ec58d 100644 --- a/s3cmd.1 +++ b/s3cmd.1 @@ -417,6 +417,14 @@ Specifies the key id used for server\-side encryption with AWS KMS\-Managed Keys (SSE\-KMS) when putting objects. [put, sync, cp, modify] .TP +\fB\-\-sse\-customer\-key\fR=12345678901234567890123456789012 +Specifies a customer key for server-side encryption, to be used +when putting objects. Must be 32 characters. +.TP +\fB\-\-sse\-copy\-source\-customer\-key\fR=12345678901234567890123456789012 +Specifies the key for copying objects with server-side +encryption customer key. Must be 32 characters. +.TP \fB\-\-encoding\fR=ENCODING Override autodetected terminal and filesystem encoding (character set). Autodetected: UTF\-8 diff --git a/test.yml b/test.yml new file mode 100644 index 00000000..e69de29b