From f0b540d63fb8b0701c9210b4042cf35fa89abffd Mon Sep 17 00:00:00 2001 From: Nate Prewitt Date: Thu, 21 Dec 2023 10:21:54 -0700 Subject: [PATCH] Enable s3express support in the transfer manager when CRT is enabled --- s3transfer/crt.py | 21 ++++++++++++++++++--- tests/functional/test_crt.py | 3 +++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/s3transfer/crt.py b/s3transfer/crt.py index 367f1f6f..d1f8366a 100644 --- a/s3transfer/crt.py +++ b/s3transfer/crt.py @@ -18,7 +18,12 @@ import awscrt.s3 import botocore.awsrequest import botocore.session -from awscrt.auth import AwsCredentials, AwsCredentialsProvider +from awscrt.auth import ( + AwsCredentials, + AwsCredentialsProvider, + AwsSigningAlgorithm, + AwsSigningConfig, +) from awscrt.io import ( ClientBootstrap, ClientTlsContext, @@ -35,7 +40,12 @@ from s3transfer.constants import MB from s3transfer.exceptions import TransferNotDoneError from s3transfer.futures import BaseTransferFuture, BaseTransferMeta -from s3transfer.utils import CallArgs, OSUtils, get_callbacks +from s3transfer.utils import ( + CallArgs, + OSUtils, + get_callbacks, + is_s3express_bucket, +) logger = logging.getLogger(__name__) @@ -807,7 +817,7 @@ def _default_get_make_request_args( on_done_before_calls, on_done_after_calls, ): - return { + make_request_args = { 'request': self._request_serializer.serialize_http_request( request_type, future ), @@ -819,6 +829,11 @@ def _default_get_make_request_args( ), 'on_progress': self.get_crt_callback(future, 'progress'), } + if is_s3express_bucket(call_args.bucket): + make_request_args['signing_config'] = AwsSigningConfig( + algorithm=AwsSigningAlgorithm.V4_S3EXPRESS + ) + return make_request_args class RenameTempFileHandler: diff --git a/tests/functional/test_crt.py b/tests/functional/test_crt.py index c56ea301..0ad27487 100644 --- a/tests/functional/test_crt.py +++ b/tests/functional/test_crt.py @@ -68,6 +68,7 @@ class TestCRTTransferManager(unittest.TestCase): def setUp(self): self.region = 'us-west-2' self.bucket = "test_bucket" + self.s3express_bucket = 's3expressbucket--usw2-az5--x-s3' self.key = "test_key" self.expected_content = b'my content' self.expected_download_content = b'new content' @@ -77,6 +78,8 @@ def setUp(self): ) self.expected_path = "/" + self.bucket + "/" + self.key self.expected_host = "s3.%s.amazonaws.com" % (self.region) + self.expected_s3express_host = f'{self.s3express_bucket}.s3express-usw2-az5.us-west-2.amazonaws.com' + self.expected_s3express_path = f'/{self.key}' self.s3_request = mock.Mock(awscrt.s3.S3Request) self.s3_crt_client = mock.Mock(awscrt.s3.S3Client) self.s3_crt_client.make_request.side_effect = (