From 4ff551c89969a2f7ac159601b955411917e84d84 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Tue, 25 Mar 2025 11:16:28 -0400 Subject: [PATCH 01/46] add test configs for all streaming type --- buildspecs/s3-checksums-tests.yml | 21 + .../awssdk/core/async/AsyncRequestBody.java | 4 +- pom.xml | 46 ++ .../checksum/ChecksumIntegrationTesting.java | 428 ++++++++++++++++-- test/s3-tests/src/it/resources/log4j2.xml | 1 + 5 files changed, 456 insertions(+), 44 deletions(-) create mode 100644 buildspecs/s3-checksums-tests.yml diff --git a/buildspecs/s3-checksums-tests.yml b/buildspecs/s3-checksums-tests.yml new file mode 100644 index 000000000000..fb9dee3bf06e --- /dev/null +++ b/buildspecs/s3-checksums-tests.yml @@ -0,0 +1,21 @@ +version: 0.2 + +phases: + install: + runtime-versions: + java: "$JAVA_RUNTIME" + + build: + commands: + - | + - mvn clean install -Dskip.unit.tests -P s3-checksums-tests -pl :s3-tests -Dfindbugs.skip -Dcheckstyle.skip -T1C $MAVEN_OPTIONS + - JAVA_VERSION=$(java -version 2>&1 | grep -i version | cut -d'"' -f2 | cut -d'.' -f1-1) + - echo $JAVA_VERSION + - echo $MAVEN_OPTIONS + finally: + - mkdir -p codebuild-test-reports + - find ./ -name 'TEST-*.xml' -type f -exec cp {} codebuild-test-reports/ \; +reports: + IntegTests: + files: + - 'codebuild-test-reports/**/*' diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncRequestBody.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncRequestBody.java index b39f7b568a11..1ad90667a657 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncRequestBody.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/async/AsyncRequestBody.java @@ -393,8 +393,8 @@ static AsyncRequestBody fromInputStream(Consumer**/*IntegrationTests.java **/*IntegTest.java **/RunCucumberTest.java + **/ChecksumIntegrationTesting.java **/SimpleMethodsIntegrationTest.java @@ -876,6 +877,51 @@ + + s3-checksums-tests + + + doRelease + + + + + true + true + true + true + true + true + + + + + org.apache.maven.plugins + maven-failsafe-plugin + ${maven-failsafe-plugin.version} + + + integration-test + + integration-test + verify + + + + ${argLine} + + **/s3/checksum/ChecksumIntegrationTesting.java + + false + + + + + + + + + endpoint-tests diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index ab927aa9eda0..1cc38cef6c73 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -39,6 +39,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.function.Consumer; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assumptions; @@ -54,12 +55,16 @@ import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm; import software.amazon.awssdk.checksums.SdkChecksum; import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.BlockingInputStreamAsyncRequestBody; +import software.amazon.awssdk.core.async.BlockingOutputStreamAsyncRequestBody; import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; import software.amazon.awssdk.core.interceptor.Context; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.http.SdkHttpMethod; import software.amazon.awssdk.http.SdkHttpRequest; import software.amazon.awssdk.regions.Region; @@ -90,10 +95,10 @@ import software.amazon.awssdk.services.s3control.model.S3ControlException; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.utils.BinaryUtils; +import software.amazon.awssdk.utils.CancellableOutputStream; import software.amazon.awssdk.utils.CompletableFutureUtils; import software.amazon.awssdk.utils.FunctionalUtils; import software.amazon.awssdk.utils.Logger; -import software.amazon.awssdk.crt.Log; public class ChecksumIntegrationTesting { private static final String BUCKET_NAME_PREFIX = "do-not-delete-checksums-"; @@ -126,7 +131,8 @@ public class ChecksumIntegrationTesting { private static S3Client s3; private static StsClient sts; - private static Path testFile; + private static Path testFileSmall; + private static Path testFileLarge; private Map> bucketCleanup = new HashMap<>(); @@ -158,7 +164,8 @@ static void setup() throws InterruptedException, IOException { apArn = createAccessPoint(); - testFile = createRandomFile(); + testFileSmall = createRandomFile16KB(); + testFileLarge = createRandomFile200MB(); } @AfterEach @@ -201,6 +208,7 @@ private void assumeNotAccelerateWithEoz(TestConfig config) { @ParameterizedTest @MethodSource("testConfigs") void deleteObject(TestConfig config) throws Exception { + LOG.debug(() -> "Running deleteObject with config: " + config.toString()); assumeNotAccessPointWithPathStyle(config); assumeNotAccelerateWithPathStyle(config); assumeNotAccelerateWithArnType(config); @@ -208,7 +216,7 @@ void deleteObject(TestConfig config) throws Exception { String bucket = bucketForType(config.getBucketType()); String key = putRandomObject(config.getBucketType()); - TestCallable callable = null; + TestCallable callable = null; try { DeleteObjectsRequest req = DeleteObjectsRequest.builder() .bucket(bucket) @@ -232,6 +240,7 @@ void deleteObject(TestConfig config) throws Exception { @ParameterizedTest @MethodSource("testConfigs") void restoreObject(TestConfig config) throws Exception { + LOG.debug(() -> "Running restoreObject with config: " + config.toString()); assumeNotAccessPointWithPathStyle(config); assumeNotAccelerateWithPathStyle(config); assumeNotAccelerateWithArnType(config); @@ -263,9 +272,17 @@ void restoreObject(TestConfig config) throws Exception { } } + @ParameterizedTest + @MethodSource("") + void getObject(TestConfig config) { + LOG.debug(() -> "Running putObject with config: " + config.toString()); + + } + @ParameterizedTest @MethodSource("uploadConfigs") void putObject(UploadConfig config) throws Exception { + LOG.debug(() -> "Running putObject with config: " + config.toString()); assumeNotAccelerateWithPathStyle(config.getBaseConfig()); assumeNotAccessPointWithPathStyle(config.getBaseConfig()); assumeNotAccelerateWithArnType(config.getBaseConfig()); @@ -278,7 +295,9 @@ void putObject(UploadConfig config) throws Exception { "No way to create AsyncRequestBody by giving both an Publisher and the content length"); // Payload signing doesn't work correctly for async java based - Assumptions.assumeFalse(config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED + Assumptions.assumeFalse( + (config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED + || config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED_MULTI) && (config.getBaseConfig().isPayloadSigning() // MRAP requires body signing || config.getBaseConfig().getBucketType() == BucketType.MRAP), @@ -286,7 +305,9 @@ void putObject(UploadConfig config) throws Exception { // For testing purposes, ContentProvider is Publisher for async clients // Async java based clients don't currently support unknown content-length bodies - Assumptions.assumeFalse(config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED + Assumptions.assumeFalse( + (config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED + || config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED_MULTI) && config.getBodyType() == BodyType.CONTENT_PROVIDER_NO_LENGTH, "Async Java based support unknown content length"); @@ -319,13 +340,13 @@ void putObject(UploadConfig config) throws Exception { String actualCrc32; if (!config.getBaseConfig().getFlavor().isAsync()) { - TestRequestBody body = getRequestBody(config.getBodyType()); + TestRequestBody body = getRequestBody(config.getBodyType(), config.getContentSize()); callable = callPutObject(request, body, config.getBaseConfig(), overrideConfiguration.build()); actualContentLength = body.getActualContentLength(); requestBodyHasContentLength = body.optionalContentLength().isPresent(); actualCrc32 = body.getChecksum(); } else { - TestAsyncBody body = getAsyncRequestBody(config.getBodyType()); + TestAsyncBody body = getAsyncRequestBody(config.getBodyType(), config.contentSize); callable = callPutObject(request, body, config.getBaseConfig(), overrideConfiguration.build()); actualContentLength = body.getActualContentLength(); requestBodyHasContentLength = body.getAsyncRequestBody().contentLength().isPresent(); @@ -436,7 +457,19 @@ private TestCallable callPutObject(PutObjectRequest request, S3AsyncClient s3Client = makeAsyncClient(config, overrideConfiguration); Callable callable = () -> { try { - CompletableFuture future = s3Client.putObject(request, requestBody.getAsyncRequestBody()); + AsyncRequestBody asyncRequestBody = requestBody.getAsyncRequestBody(); + CompletableFuture future = s3Client.putObject(request, asyncRequestBody); + if (requestBody.bodyType == BodyType.BLOCKING_INPUT_STREAM) { + BlockingInputStreamAsyncRequestBody body = (BlockingInputStreamAsyncRequestBody) requestBody.asyncRequestBody; + InputStream inputStream = ((TestAsyncBodyForBlockingInputStream) requestBody).inputStream; + body.writeInputStream(inputStream); + } + if (requestBody.bodyType == BodyType.BLOCKING_OUTPUT_STREAM) { + TestAsyncBodyForBlockingOutputStream body = (TestAsyncBodyForBlockingOutputStream) requestBody; + CancellableOutputStream outputStream = + ((BlockingOutputStreamAsyncRequestBody) body.getAsyncRequestBody()).outputStream(); + body.bodyWrite.accept(outputStream); + } return CompletableFutureUtils.joinLikeSync(future); } catch (Exception e) { throw new RuntimeException(e); @@ -482,6 +515,16 @@ private S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideConfigura .accelerate(config.isAccelerateEnabled()) .overrideConfiguration(overrideConfiguration) .build(); + case ASYNC_JAVA_BASED_MULTI: + return S3AsyncClient.builder() + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .accelerate(config.isAccelerateEnabled()) + .overrideConfiguration(overrideConfiguration) + .multipartEnabled(true) + .build(); case ASYNC_CRT: { if (overrideConfiguration != null) { LOG.warn(() -> "Override configuration cannot be set for Async S3 CRT!"); @@ -523,7 +566,7 @@ enum BucketType { EOZ(false), ; - private boolean arnType; + private final boolean arnType; private BucketType(boolean arnType) { this.arnType = arnType; @@ -537,11 +580,12 @@ public boolean isArnType() { enum S3ClientFlavor { JAVA_BASED(false), ASYNC_JAVA_BASED(true), + ASYNC_JAVA_BASED_MULTI(true), ASYNC_CRT(true) ; - private boolean async; + private final boolean async; private S3ClientFlavor(boolean async) { this.async = async; @@ -555,6 +599,7 @@ public boolean isAsync() { static class UploadConfig { private TestConfig baseConfig; private BodyType bodyType; + private ContentSize contentSize; public TestConfig getBaseConfig() { return baseConfig; @@ -572,13 +617,68 @@ public void setBodyType(BodyType bodyType) { this.bodyType = bodyType; } + public void setContentSize(ContentSize contentSize) { + this.contentSize = contentSize; + } + + public ContentSize getContentSize() { + return this.contentSize; + } + @Override public String toString() { return "UploadConfig{" + "baseConfig=" + baseConfig + ", bodyType=" + bodyType + + ", contentSize=" + contentSize + '}'; } + + } + + enum ResponseTransformerType { + FILE, + BYTES, + INPUT_STREAM, + OUTPUT_STREAM, + UNMANAGED, + PUBLISHER + } + + static class DownloadConfig { + private TestConfig testConfig; + private ResponseTransformerType responseTransformerType; + private ContentSize contentSize; + + public DownloadConfig(TestConfig testConfig, ResponseTransformerType responseTransformerType, ContentSize contentSize) { + this.testConfig = testConfig; + this.responseTransformerType = responseTransformerType; + this.contentSize = contentSize; + } + + public TestConfig getTestConfig() { + return this.testConfig; + } + + public void setTestConfig(TestConfig testConfig) { + this.testConfig = testConfig; + } + + public ResponseTransformerType getResponseTransformerType() { + return responseTransformerType; + } + + public void setResponseTransformerType(ResponseTransformerType responseTransformerType) { + this.responseTransformerType = responseTransformerType; + } + + public ContentSize getContentSize() { + return contentSize; + } + + public void setContentSize(ContentSize contentSize) { + this.contentSize = contentSize; + } } static class TestRequestBody extends RequestBody { @@ -604,11 +704,13 @@ private static class TestAsyncBody { private final AsyncRequestBody asyncRequestBody; private final long actualContentLength; private final String checksum; + private final BodyType bodyType; - private TestAsyncBody(AsyncRequestBody asyncRequestBody, long actualContentLength, String checksum) { + private TestAsyncBody(AsyncRequestBody asyncRequestBody, long actualContentLength, String checksum, BodyType bodyType) { this.asyncRequestBody = asyncRequestBody; this.actualContentLength = actualContentLength; this.checksum = checksum; + this.bodyType = bodyType; } public AsyncRequestBody getAsyncRequestBody() { @@ -622,7 +724,30 @@ public long getActualContentLength() { public String getChecksum() { return checksum; } + } + + private static class TestAsyncBodyForBlockingOutputStream extends TestAsyncBody { + private final Consumer bodyWrite; + private TestAsyncBodyForBlockingOutputStream(AsyncRequestBody asyncRequestBody, + Consumer bodyWrite, + long actualContentLength, + String checksum, + BodyType bodyType) { + super(asyncRequestBody, actualContentLength, checksum, bodyType); + this.bodyWrite = bodyWrite; + } + } + private static class TestAsyncBodyForBlockingInputStream extends TestAsyncBody { + private final InputStream inputStream; + private TestAsyncBodyForBlockingInputStream(AsyncRequestBody asyncRequestBody, + InputStream inputStream, + long actualContentLength, + String checksum, + BodyType bodyType) { + super(asyncRequestBody, actualContentLength, checksum, bodyType); + this.inputStream = inputStream; + } } static class TestConfig { @@ -736,7 +861,47 @@ enum BodyType { CONTENT_PROVIDER_WITH_LENGTH, - CONTENT_PROVIDER_NO_LENGTH + CONTENT_PROVIDER_NO_LENGTH, + + BYTES, + BYTE_BUFFER, + REMAINING_BYTE_BUFFER, + + BYTES_UNSAFE, + BYTE_BUFFER_UNSAFE, + REMAINING_BYTE_BUFFER_UNSAFE, + + BUFFERS, + BUFFERS_REMAINING, + BUFFERS_UNSAFE, + BUFFERS_REMAINING_UNSAFE, + + BLOCKING_INPUT_STREAM, + BLOCKING_OUTPUT_STREAM + } + + enum ContentSize { + SMALL, + LARGE; // 200 MiB + + static final byte[] smallContent = "Hello world".getBytes(StandardCharsets.UTF_8); + static final byte[] largeContent = largeContent(); + + byte[] content() { + switch (this) { + case SMALL: return smallContent; + case LARGE: return largeContent; + default: throw new IllegalArgumentException("not supported ContentSize " + this); + } + } + } + + private static byte[] largeContent() { + // 200 MiB + Random r = new Random(); + byte[] b = new byte[200 * 1024 * 1024]; + r.nextBytes(b); + return b; } private static List uploadConfigs() { @@ -744,10 +909,26 @@ private static List uploadConfigs() { for (BodyType bodyType : BodyType.values()) { for (TestConfig baseConfig : testConfigs()) { - UploadConfig config = new UploadConfig(); - config.setBaseConfig(baseConfig); - config.setBodyType(bodyType); - configs.add(config); + for (ContentSize size : ContentSize.values()) { + UploadConfig config = new UploadConfig(); + config.setBaseConfig(baseConfig); + config.setBodyType(bodyType); + config.setContentSize(size); + configs.add(config); + } + } + } + return configs; + } + + private static List downloadConfigs() { + List configs = new ArrayList<>(); + for (ResponseTransformerType responseTransformerType : ResponseTransformerType.values()) { + for (TestConfig baseConfig : testConfigs()) { + for (ContentSize contentSize : ContentSize.values()) { + DownloadConfig config = new DownloadConfig(baseConfig, responseTransformerType, contentSize); + configs.add(config); + } } } return configs; @@ -770,28 +951,28 @@ private String putRandomArchivedObject(BucketType bucketType) { return key; } - private TestRequestBody getRequestBody(BodyType bodyType) throws IOException { + private TestRequestBody getRequestBody(BodyType bodyType, ContentSize contentSize) throws IOException { switch (bodyType) { case STRING: { - String content = "Hello world"; - long contentLength = content.getBytes(StandardCharsets.UTF_8).length; + byte[] content = contentSize.content(); + long contentLength = content.length; return new TestRequestBody(RequestBody.fromString("Hello world"), contentLength, crc32(content)); } case FILE: - return new TestRequestBody(RequestBody.fromFile(testFile), Files.size(testFile), crc32(testFile)); + return new TestRequestBody(RequestBody.fromFile(testFileSmall), Files.size(testFileSmall), crc32(testFileSmall)); case CONTENT_PROVIDER_NO_LENGTH: { RequestBody wrapped = - RequestBody.fromContentProvider(() -> FunctionalUtils.invokeSafely(() -> Files.newInputStream(testFile)), + RequestBody.fromContentProvider(() -> FunctionalUtils.invokeSafely(() -> Files.newInputStream(testFileSmall)), "application/octet-stream"); - return new TestRequestBody(wrapped, Files.size(testFile), crc32(testFile)); + return new TestRequestBody(wrapped, Files.size(testFileSmall), crc32(testFileSmall)); } case CONTENT_PROVIDER_WITH_LENGTH: { - long contentLength = Files.size(testFile); - RequestBody wrapped = RequestBody.fromContentProvider(() -> FunctionalUtils.invokeSafely(() -> Files.newInputStream(testFile)), - Files.size(testFile), + long contentLength = Files.size(testFileSmall); + RequestBody wrapped = RequestBody.fromContentProvider(() -> FunctionalUtils.invokeSafely(() -> Files.newInputStream(testFileSmall)), + Files.size(testFileSmall), "application/octet-stream"); - return new TestRequestBody(wrapped, contentLength, crc32(testFile)); + return new TestRequestBody(wrapped, contentLength, crc32(testFileSmall)); } case INPUTSTREAM_RESETABLE: { byte[] content = "Hello world".getBytes(StandardCharsets.UTF_8); @@ -803,41 +984,192 @@ private TestRequestBody getRequestBody(BodyType bodyType) throws IOException { RequestBody wrapped = RequestBody.fromInputStream(new NonResettableByteStream(content), content.length); return new TestRequestBody(wrapped, content.length, crc32(content)); } + case BYTES: { + byte[] content = contentSize.content(); + RequestBody wrapped = RequestBody.fromBytes(content); + return new TestRequestBody(wrapped, content.length, crc32(content)); + } + case BYTE_BUFFER: { + byte[] content = contentSize.content(); + RequestBody wrapped = RequestBody.fromByteBuffer(ByteBuffer.wrap(content)); + return new TestRequestBody(wrapped, content.length, crc32(content)); + } + case REMAINING_BYTE_BUFFER: { + byte[] content = "Hello world, Hello world".getBytes(StandardCharsets.UTF_8); + ByteBuffer buff = ByteBuffer.wrap(content); + int offset = 2; + buff.position(offset); + RequestBody asyncRequestBody = RequestBody.fromRemainingByteBuffer(buff); + byte[] crcArray = new byte[content.length - offset]; + System.arraycopy(crcArray, 2, crcArray, 0, crcArray.length); + return new TestRequestBody(asyncRequestBody, content.length, crc32(crcArray)); + } + case BUFFERS: + case BUFFERS_REMAINING: + case BUFFERS_UNSAFE: + case BUFFERS_REMAINING_UNSAFE: + case BYTES_UNSAFE: + case BYTE_BUFFER_UNSAFE: + case REMAINING_BYTE_BUFFER_UNSAFE: + case BLOCKING_INPUT_STREAM: + case BLOCKING_OUTPUT_STREAM: + Assumptions.abort("Test BodyType not supported for sync client: " + bodyType); default: throw new RuntimeException("Unsupported body type: " + bodyType); } } - private TestAsyncBody getAsyncRequestBody(BodyType bodyType) throws IOException { + private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize contentSize) throws IOException { switch (bodyType) { case STRING: { - String content = "Hello world"; - long contentLength = content.getBytes(StandardCharsets.UTF_8).length; - return new TestAsyncBody(AsyncRequestBody.fromString(content), contentLength, crc32(content)); + byte[] content = contentSize.content(); + long contentLength = content.length; + return new TestAsyncBody(AsyncRequestBody.fromString(new String(content)), contentLength, crc32(content), bodyType); } case FILE: { - long contentLength = Files.size(testFile); - return new TestAsyncBody(AsyncRequestBody.fromFile(testFile), contentLength, crc32(testFile)); + long contentLength = Files.size(testFileSmall); + return new TestAsyncBody(AsyncRequestBody.fromFile(testFileSmall), contentLength, crc32(testFileSmall), bodyType); } case INPUTSTREAM_RESETABLE: { - byte[] content = "Hello world".getBytes(StandardCharsets.UTF_8); + byte[] content = contentSize.content(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromInputStream(new ByteArrayInputStream(content), (long) content.length, ASYNC_REQUEST_BODY_EXECUTOR); - return new TestAsyncBody(asyncRequestBody, content.length, crc32(content)); + return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); } case INPUTSTREAM_NOT_RESETABLE: { - byte[] content = "Hello world".getBytes(StandardCharsets.UTF_8); + byte[] content = contentSize.content(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromInputStream(new NonResettableByteStream(content), (long) content.length, ASYNC_REQUEST_BODY_EXECUTOR); - return new TestAsyncBody(asyncRequestBody, content.length, crc32(content)); + return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); } case CONTENT_PROVIDER_NO_LENGTH: { - byte[] content = "Hello world".getBytes(StandardCharsets.UTF_8); + byte[] content = contentSize.content(); Flowable publisher = Flowable.just(ByteBuffer.wrap(content)); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromPublisher(publisher); - return new TestAsyncBody(asyncRequestBody, content.length, crc32(content)); + return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); + } + + case BYTES: { + byte[] content = contentSize.content(); + AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromBytes(content); + return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); + } + case BYTE_BUFFER: { + byte[] content = contentSize.content(); + AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromByteBuffer(ByteBuffer.wrap(content)); + return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); + } + case REMAINING_BYTE_BUFFER: { + byte[] content = "Hello world, Hello world".getBytes(StandardCharsets.UTF_8); + ByteBuffer buff = ByteBuffer.wrap(content); + int offset = 2; + buff.position(offset); + AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromRemainingByteBuffer(buff); + byte[] crcArray = new byte[content.length - offset]; + System.arraycopy(crcArray, 2, crcArray, 0, crcArray.length); + return new TestAsyncBody(asyncRequestBody, content.length, crc32(crcArray), bodyType); + } + case BYTES_UNSAFE:{ + byte[] content = contentSize.content(); + AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromBytesUnsafe(content); + return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); + } + case BYTE_BUFFER_UNSAFE: { + byte[] content = contentSize.content(); + AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromByteBufferUnsafe(ByteBuffer.wrap(content)); + return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); + } + case REMAINING_BYTE_BUFFER_UNSAFE: { + byte[] content = "Hello world, Hello world".getBytes(StandardCharsets.UTF_8); + ByteBuffer buff = ByteBuffer.wrap(content); + int offset = 2; + buff.position(offset); + AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromRemainingByteBufferUnsafe(buff); + byte[] crcArray = new byte[content.length - offset]; + System.arraycopy(crcArray, 2, crcArray, 0, crcArray.length); + return new TestAsyncBody(asyncRequestBody, content.length, crc32(crcArray), bodyType); + } + case BUFFERS: { + byte[] content1 = contentSize.content(); + byte[] content2 = contentSize.content(); + AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromByteBuffers(ByteBuffer.wrap(content1), + ByteBuffer.wrap(content2)); + byte[] crcArray = new byte[content2.length + content2.length]; + System.arraycopy(content1, 0, crcArray, 0, content1.length); + System.arraycopy(content2, 0, crcArray, content1.length, content2.length); + return new TestAsyncBody(asyncRequestBody, + content1.length + content2.length, + crc32(crcArray), + bodyType); + } + case BUFFERS_REMAINING: { + byte[] content1 = contentSize.content(); + byte[] content2 = contentSize.content(); + AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromRemainingByteBuffers(ByteBuffer.wrap(content1), + ByteBuffer.wrap(content2)); + byte[] crcArray = new byte[content2.length + content2.length]; + System.arraycopy(content1, 0, crcArray, 0, content1.length); + System.arraycopy(content2, 0, crcArray, content1.length, content2.length); + return new TestAsyncBody(asyncRequestBody, + content1.length + content2.length, + crc32(crcArray), + bodyType); + } + case BUFFERS_UNSAFE: { + byte[] content1 = contentSize.content(); + byte[] content2 = contentSize.content(); + AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromByteBuffersUnsafe(ByteBuffer.wrap(content1), + ByteBuffer.wrap(content2)); + byte[] crcArray = new byte[content2.length + content2.length]; + System.arraycopy(content1, 0, crcArray, 0, content1.length); + System.arraycopy(content2, 0, crcArray, content1.length, content2.length); + return new TestAsyncBody(asyncRequestBody, + content1.length + content2.length, + crc32(crcArray), + bodyType); + + } + case BUFFERS_REMAINING_UNSAFE: { + byte[] content1 = contentSize.content(); + byte[] content2 = contentSize.content(); + AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromRemainingByteBuffersUnsafe(ByteBuffer.wrap(content1), + ByteBuffer.wrap(content2)); + byte[] crcArray = new byte[content2.length + content2.length]; + System.arraycopy(content1, 0, crcArray, 0, content1.length); + System.arraycopy(content2, 0, crcArray, content1.length, content2.length); + return new TestAsyncBody(asyncRequestBody, + content1.length + content2.length, + crc32(crcArray), + bodyType); + } + case BLOCKING_INPUT_STREAM: { + byte[] content = contentSize.content(); + long streamToSendLength = content.length; + BlockingInputStreamAsyncRequestBody body = AsyncRequestBody.forBlockingInputStream(streamToSendLength); + return new TestAsyncBodyForBlockingInputStream(body, + new ByteArrayInputStream(content), + content.length, + crc32(content), + bodyType); + } + case BLOCKING_OUTPUT_STREAM: { + byte[] content = contentSize.content(); + long streamToSendLength = content.length; + BlockingOutputStreamAsyncRequestBody body = AsyncRequestBody.forBlockingOutputStream(streamToSendLength); + Consumer bodyWrite = outputStream -> { + try { + outputStream.write(content); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + }; + return new TestAsyncBodyForBlockingOutputStream(body, + bodyWrite, + content.length, + crc32(content), + bodyType); } default: throw new RuntimeException("Unsupported async body type: " + bodyType); @@ -960,9 +1292,9 @@ private static void createBucket(String bucketName, int retryCount) { .build()); } catch (S3Exception e) { LOG.debug(() -> "Error attempting to create bucket: " + bucketName); - if (e.awsErrorDetails().errorCode().equals("BucketAlreadyOwnedByYou")) { + if ("BucketAlreadyOwnedByYou".equals(e.awsErrorDetails().errorCode())) { LOG.debug(() -> String.format("%s bucket already exists, likely leaked by a previous run%n", bucketName)); - } else if (e.awsErrorDetails().errorCode().equals("TooManyBuckets")) { + } else if ("TooManyBuckets".equals(e.awsErrorDetails().errorCode())) { LOG.debug(() -> "Printing all buckets for debug:"); s3.listBuckets().buckets().forEach(l -> LOG.debug(l::toString)); if (retryCount < 2) { @@ -979,7 +1311,7 @@ private static void createBucket(String bucketName, int retryCount) { s3.waiter().waitUntilBucketExists(r -> r.bucket(bucketName)); } - private static Path createRandomFile() throws IOException { + private static Path createRandomFile16KB() throws IOException { Path tmp = Files.createTempFile(null, null); byte[] randomBytes = new byte[1024]; new Random().nextBytes(randomBytes); @@ -991,6 +1323,18 @@ private static Path createRandomFile() throws IOException { return tmp; } + private static Path createRandomFile200MB() throws IOException { + Path tmp = Files.createTempFile(null, null); + byte[] randomBytes = new byte[1024 * 1024]; + new Random().nextBytes(randomBytes); + try (OutputStream os = Files.newOutputStream(tmp)) { + for (int i = 0; i < 200; ++i) { + os.write(randomBytes); + } + } + return tmp; + } + private static class NonResettableByteStream extends ByteArrayInputStream { public NonResettableByteStream(byte[] buf) { super(buf); diff --git a/test/s3-tests/src/it/resources/log4j2.xml b/test/s3-tests/src/it/resources/log4j2.xml index 0ae8e7c45738..22261c126476 100644 --- a/test/s3-tests/src/it/resources/log4j2.xml +++ b/test/s3-tests/src/it/resources/log4j2.xml @@ -24,6 +24,7 @@ + From c80568e7325e614ad8658018db7be3a10ab39105 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Tue, 25 Mar 2025 13:05:25 -0400 Subject: [PATCH 02/46] buildspec fix --- buildspecs/s3-checksums-tests.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/buildspecs/s3-checksums-tests.yml b/buildspecs/s3-checksums-tests.yml index fb9dee3bf06e..5b38b9e37dbb 100644 --- a/buildspecs/s3-checksums-tests.yml +++ b/buildspecs/s3-checksums-tests.yml @@ -7,15 +7,12 @@ phases: build: commands: - - | - - mvn clean install -Dskip.unit.tests -P s3-checksums-tests -pl :s3-tests -Dfindbugs.skip -Dcheckstyle.skip -T1C $MAVEN_OPTIONS - - JAVA_VERSION=$(java -version 2>&1 | grep -i version | cut -d'"' -f2 | cut -d'.' -f1-1) - - echo $JAVA_VERSION + - mvn clean install -Dskip.unit.tests -P s3-checksums-tests -pl :s3-tests -am -Dfindbugs.skip -Dcheckstyle.skip -T1C $MAVEN_OPTIONS - echo $MAVEN_OPTIONS finally: - mkdir -p codebuild-test-reports - find ./ -name 'TEST-*.xml' -type f -exec cp {} codebuild-test-reports/ \; reports: - IntegTests: + ChecksumsTests: files: - 'codebuild-test-reports/**/*' From 53b55b3dc2ac06a985119cd3a0eac8b707f2f7e7 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Tue, 25 Mar 2025 14:18:09 -0400 Subject: [PATCH 03/46] buildspec fix --- buildspecs/s3-checksums-tests.yml | 6 +----- .../s3/checksum/ChecksumIntegrationTesting.java | 16 ++++++++-------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/buildspecs/s3-checksums-tests.yml b/buildspecs/s3-checksums-tests.yml index 5b38b9e37dbb..56bec6d6e59f 100644 --- a/buildspecs/s3-checksums-tests.yml +++ b/buildspecs/s3-checksums-tests.yml @@ -1,13 +1,9 @@ version: 0.2 phases: - install: - runtime-versions: - java: "$JAVA_RUNTIME" - build: commands: - - mvn clean install -Dskip.unit.tests -P s3-checksums-tests -pl :s3-tests -am -Dfindbugs.skip -Dcheckstyle.skip -T1C $MAVEN_OPTIONS + - mvn clean install -P s3-checksums-tests -pl :s3-tests -am -T1C $MAVEN_OPTIONS - echo $MAVEN_OPTIONS finally: - mkdir -p codebuild-test-reports diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index 1cc38cef6c73..ffadf67d5636 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -110,6 +110,8 @@ public class ChecksumIntegrationTesting { private static final Region REGION = Region.US_WEST_2; private static final String TEST_CREDENTIALS_PROFILE_NAME = "aws-test-account"; + static final byte[] smallContent = "Hello world".getBytes(StandardCharsets.UTF_8); + static final byte[] largeContent = largeContent(); public static final AwsCredentialsProviderChain CREDENTIALS_PROVIDER_CHAIN = AwsCredentialsProviderChain.of(ProfileCredentialsProvider.builder() @@ -272,12 +274,12 @@ void restoreObject(TestConfig config) throws Exception { } } - @ParameterizedTest - @MethodSource("") - void getObject(TestConfig config) { - LOG.debug(() -> "Running putObject with config: " + config.toString()); - - } + // @ParameterizedTest + // @MethodSource("") + // void getObject(TestConfig config) { + // LOG.debug(() -> "Running putObject with config: " + config.toString()); + // + // } @ParameterizedTest @MethodSource("uploadConfigs") @@ -884,8 +886,6 @@ enum ContentSize { SMALL, LARGE; // 200 MiB - static final byte[] smallContent = "Hello world".getBytes(StandardCharsets.UTF_8); - static final byte[] largeContent = largeContent(); byte[] content() { switch (this) { From d892595f725f5d3cd6d2f4eb0ddc960f17ab7a5c Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Wed, 26 Mar 2025 09:19:36 -0400 Subject: [PATCH 04/46] check if tests hand without multipart --- .../checksum/ChecksumIntegrationTesting.java | 41 +++++++++++++------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index ffadf67d5636..0e2c3e73898b 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -100,6 +100,19 @@ import software.amazon.awssdk.utils.FunctionalUtils; import software.amazon.awssdk.utils.Logger; +// Running putObject with config: +// UploadConfig{ +// baseConfig=[ +// flavor=ASYNC_JAVA_BASED_MULTI, +// bucketType=STANDARD_BUCKET, +// forcePathStyle=true, +// requestChecksumValidation=WHEN_REQUIRED, +// accelerateEnabled=false, +// payloadSigning=false +// ], +// bodyType=BLOCKING_OUTPUT_STREAM, +// contentSize=SMALL +// } public class ChecksumIntegrationTesting { private static final String BUCKET_NAME_PREFIX = "do-not-delete-checksums-"; private static final String MRAP_NAME = "do-not-delete-checksum-testing"; @@ -299,7 +312,8 @@ void putObject(UploadConfig config) throws Exception { // Payload signing doesn't work correctly for async java based Assumptions.assumeFalse( (config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED - || config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED_MULTI) + // || config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED_MULTI + ) && (config.getBaseConfig().isPayloadSigning() // MRAP requires body signing || config.getBaseConfig().getBucketType() == BucketType.MRAP), @@ -309,7 +323,8 @@ void putObject(UploadConfig config) throws Exception { // Async java based clients don't currently support unknown content-length bodies Assumptions.assumeFalse( (config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED - || config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED_MULTI) + // || config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED_MULTI + ) && config.getBodyType() == BodyType.CONTENT_PROVIDER_NO_LENGTH, "Async Java based support unknown content length"); @@ -517,16 +532,16 @@ private S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideConfigura .accelerate(config.isAccelerateEnabled()) .overrideConfiguration(overrideConfiguration) .build(); - case ASYNC_JAVA_BASED_MULTI: - return S3AsyncClient.builder() - .forcePathStyle(config.isForcePathStyle()) - .requestChecksumCalculation(config.getRequestChecksumValidation()) - .region(REGION) - .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - .accelerate(config.isAccelerateEnabled()) - .overrideConfiguration(overrideConfiguration) - .multipartEnabled(true) - .build(); + // case ASYNC_JAVA_BASED_MULTI: + // return S3AsyncClient.builder() + // .forcePathStyle(config.isForcePathStyle()) + // .requestChecksumCalculation(config.getRequestChecksumValidation()) + // .region(REGION) + // .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + // .accelerate(config.isAccelerateEnabled()) + // .overrideConfiguration(overrideConfiguration) + // .multipartEnabled(true) + // .build(); case ASYNC_CRT: { if (overrideConfiguration != null) { LOG.warn(() -> "Override configuration cannot be set for Async S3 CRT!"); @@ -582,7 +597,7 @@ public boolean isArnType() { enum S3ClientFlavor { JAVA_BASED(false), ASYNC_JAVA_BASED(true), - ASYNC_JAVA_BASED_MULTI(true), + // ASYNC_JAVA_BASED_MULTI(true), ASYNC_CRT(true) ; From bd21f3a28f1be5db8653f4a50fd8d198834ce304 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Wed, 26 Mar 2025 10:06:56 -0400 Subject: [PATCH 05/46] add http-auth-aws-crt for crt sigv4a tests --- test/s3-tests/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/s3-tests/pom.xml b/test/s3-tests/pom.xml index 3f452d65df14..a70daa046cb2 100644 --- a/test/s3-tests/pom.xml +++ b/test/s3-tests/pom.xml @@ -152,6 +152,12 @@ ${awsjavasdk.version} test + + software.amazon.awssdk + http-auth-aws-crt + ${awsjavasdk.version} + test + From bbb5d1e60060ae4f8c4eb8d4579b5cd7afc800f0 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Wed, 26 Mar 2025 10:13:56 -0400 Subject: [PATCH 06/46] fix request body methods with 'remaining' --- .../checksum/ChecksumIntegrationTesting.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index 0e2c3e73898b..42f0fe96f37a 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -971,7 +971,9 @@ private TestRequestBody getRequestBody(BodyType bodyType, ContentSize contentSiz case STRING: { byte[] content = contentSize.content(); long contentLength = content.length; - return new TestRequestBody(RequestBody.fromString("Hello world"), contentLength, crc32(content)); + return new TestRequestBody(RequestBody.fromString(new String(content, StandardCharsets.UTF_8)), + contentLength, + crc32(content)); } case FILE: return new TestRequestBody(RequestBody.fromFile(testFileSmall), Files.size(testFileSmall), crc32(testFileSmall)); @@ -990,12 +992,12 @@ private TestRequestBody getRequestBody(BodyType bodyType, ContentSize contentSiz return new TestRequestBody(wrapped, contentLength, crc32(testFileSmall)); } case INPUTSTREAM_RESETABLE: { - byte[] content = "Hello world".getBytes(StandardCharsets.UTF_8); + byte[] content = contentSize.content(); RequestBody wrapped = RequestBody.fromInputStream(new ByteArrayInputStream(content), content.length); return new TestRequestBody(wrapped, content.length, crc32(content)); } case INPUTSTREAM_NOT_RESETABLE: { - byte[] content = "Hello world".getBytes(StandardCharsets.UTF_8); + byte[] content = contentSize.content(); RequestBody wrapped = RequestBody.fromInputStream(new NonResettableByteStream(content), content.length); return new TestRequestBody(wrapped, content.length, crc32(content)); } @@ -1010,13 +1012,13 @@ private TestRequestBody getRequestBody(BodyType bodyType, ContentSize contentSiz return new TestRequestBody(wrapped, content.length, crc32(content)); } case REMAINING_BYTE_BUFFER: { - byte[] content = "Hello world, Hello world".getBytes(StandardCharsets.UTF_8); + byte[] content = contentSize.content(); ByteBuffer buff = ByteBuffer.wrap(content); int offset = 2; buff.position(offset); RequestBody asyncRequestBody = RequestBody.fromRemainingByteBuffer(buff); byte[] crcArray = new byte[content.length - offset]; - System.arraycopy(crcArray, 2, crcArray, 0, crcArray.length); + System.arraycopy(content, offset, crcArray, 0, crcArray.length); return new TestRequestBody(asyncRequestBody, content.length, crc32(crcArray)); } case BUFFERS: @@ -1077,13 +1079,13 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); } case REMAINING_BYTE_BUFFER: { - byte[] content = "Hello world, Hello world".getBytes(StandardCharsets.UTF_8); + byte[] content = contentSize.content(); ByteBuffer buff = ByteBuffer.wrap(content); int offset = 2; buff.position(offset); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromRemainingByteBuffer(buff); byte[] crcArray = new byte[content.length - offset]; - System.arraycopy(crcArray, 2, crcArray, 0, crcArray.length); + System.arraycopy(content, offset, crcArray, 0, crcArray.length); return new TestAsyncBody(asyncRequestBody, content.length, crc32(crcArray), bodyType); } case BYTES_UNSAFE:{ @@ -1097,13 +1099,13 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); } case REMAINING_BYTE_BUFFER_UNSAFE: { - byte[] content = "Hello world, Hello world".getBytes(StandardCharsets.UTF_8); + byte[] content = contentSize.content(); ByteBuffer buff = ByteBuffer.wrap(content); int offset = 2; buff.position(offset); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromRemainingByteBufferUnsafe(buff); byte[] crcArray = new byte[content.length - offset]; - System.arraycopy(crcArray, 2, crcArray, 0, crcArray.length); + System.arraycopy(content, 2, crcArray, 0, crcArray.length); return new TestAsyncBody(asyncRequestBody, content.length, crc32(crcArray), bodyType); } case BUFFERS: { From c889f6ebf275ba41c4f5902d37c61bd938283370 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Wed, 26 Mar 2025 11:09:03 -0400 Subject: [PATCH 07/46] test 2MiB instead of 200MiB --- .../checksum/ChecksumIntegrationTesting.java | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index 42f0fe96f37a..b76e6ffa6e10 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -382,27 +382,30 @@ void putObject(UploadConfig config) throws Exception { } // We can't set an execution interceptor when using CRT - if (config.getBaseConfig().getFlavor() != S3ClientFlavor.ASYNC_CRT) { - assertThat(recorder.getRequests()).isNotEmpty(); + if (config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_CRT) { + return; + } - for (SdkHttpRequest httpRequest : recorder.getRequests()) { - // skip any non-PUT requests, e.g. GetSession for EOZ requests - if (httpRequest.method() != SdkHttpMethod.PUT) { - continue; - } + assertThat(recorder.getRequests()).isNotEmpty(); - String payloadSha = httpRequest.firstMatchingHeader("x-amz-content-sha256").get(); - if (payloadSha.startsWith("STREAMING")) { - String decodedContentLength = httpRequest.firstMatchingHeader("x-amz-decoded-content-length").get(); - assertThat(Long.parseLong(decodedContentLength)).isEqualTo(actualContentLength); - } else { - Optional contentLength = httpRequest.firstMatchingHeader("Content-Length"); - if (requestBodyHasContentLength) { - assertThat(Long.parseLong(contentLength.get())).isEqualTo(actualContentLength); - } + for (SdkHttpRequest httpRequest : recorder.getRequests()) { + // skip any non-PUT requests, e.g. GetSession for EOZ requests + if (httpRequest.method() != SdkHttpMethod.PUT) { + continue; + } + + String payloadSha = httpRequest.firstMatchingHeader("x-amz-content-sha256").get(); + if (payloadSha.startsWith("STREAMING")) { + String decodedContentLength = httpRequest.firstMatchingHeader("x-amz-decoded-content-length").get(); + assertThat(Long.parseLong(decodedContentLength)).isEqualTo(actualContentLength); + } else { + Optional contentLength = httpRequest.firstMatchingHeader("Content-Length"); + if (requestBodyHasContentLength) { + assertThat(Long.parseLong(contentLength.get())).isEqualTo(actualContentLength); } } } + } finally { if (callable != null) { callable.client.close(); @@ -480,12 +483,14 @@ private TestCallable callPutObject(PutObjectRequest request, BlockingInputStreamAsyncRequestBody body = (BlockingInputStreamAsyncRequestBody) requestBody.asyncRequestBody; InputStream inputStream = ((TestAsyncBodyForBlockingInputStream) requestBody).inputStream; body.writeInputStream(inputStream); + inputStream.close(); } if (requestBody.bodyType == BodyType.BLOCKING_OUTPUT_STREAM) { TestAsyncBodyForBlockingOutputStream body = (TestAsyncBodyForBlockingOutputStream) requestBody; CancellableOutputStream outputStream = ((BlockingOutputStreamAsyncRequestBody) body.getAsyncRequestBody()).outputStream(); body.bodyWrite.accept(outputStream); + outputStream.close(); } return CompletableFutureUtils.joinLikeSync(future); } catch (Exception e) { @@ -914,7 +919,8 @@ byte[] content() { private static byte[] largeContent() { // 200 MiB Random r = new Random(); - byte[] b = new byte[200 * 1024 * 1024]; + // byte[] b = new byte[200 * 1024 * 1024]; + byte[] b = new byte[2 * 1024 * 1024]; r.nextBytes(b); return b; } From 89c5edc273a23ddfaaf7d9319c875525f2d383f9 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Wed, 26 Mar 2025 12:05:24 -0400 Subject: [PATCH 08/46] fix for 'remaining' test type --- .../s3/checksum/ChecksumIntegrationTesting.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index b76e6ffa6e10..c504c6514f78 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -29,6 +29,7 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -1023,9 +1024,8 @@ private TestRequestBody getRequestBody(BodyType bodyType, ContentSize contentSiz int offset = 2; buff.position(offset); RequestBody asyncRequestBody = RequestBody.fromRemainingByteBuffer(buff); - byte[] crcArray = new byte[content.length - offset]; - System.arraycopy(content, offset, crcArray, 0, crcArray.length); - return new TestRequestBody(asyncRequestBody, content.length, crc32(crcArray)); + byte[] crcArray = Arrays.copyOfRange(content, offset, content.length); + return new TestRequestBody(asyncRequestBody, content.length - offset, crc32(crcArray)); } case BUFFERS: case BUFFERS_REMAINING: @@ -1090,9 +1090,8 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content int offset = 2; buff.position(offset); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromRemainingByteBuffer(buff); - byte[] crcArray = new byte[content.length - offset]; - System.arraycopy(content, offset, crcArray, 0, crcArray.length); - return new TestAsyncBody(asyncRequestBody, content.length, crc32(crcArray), bodyType); + byte[] crcArray = Arrays.copyOfRange(content, offset, content.length); + return new TestAsyncBody(asyncRequestBody, content.length - offset, crc32(crcArray), bodyType); } case BYTES_UNSAFE:{ byte[] content = contentSize.content(); @@ -1110,9 +1109,8 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content int offset = 2; buff.position(offset); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromRemainingByteBufferUnsafe(buff); - byte[] crcArray = new byte[content.length - offset]; - System.arraycopy(content, 2, crcArray, 0, crcArray.length); - return new TestAsyncBody(asyncRequestBody, content.length, crc32(crcArray), bodyType); + byte[] crcArray = Arrays.copyOfRange(content, offset, content.length); + return new TestAsyncBody(asyncRequestBody, content.length - offset, crc32(crcArray), bodyType); } case BUFFERS: { byte[] content1 = contentSize.content(); @@ -1152,7 +1150,6 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content content1.length + content2.length, crc32(crcArray), bodyType); - } case BUFFERS_REMAINING_UNSAFE: { byte[] content1 = contentSize.content(); From 3737d0a073072077d39aff8cbaf6113848422641 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Wed, 26 Mar 2025 12:34:10 -0400 Subject: [PATCH 09/46] fix for fromString, reduce logs --- .../checksum/ChecksumIntegrationTesting.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index c504c6514f78..abebd21a2eb6 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -224,12 +224,13 @@ private void assumeNotAccelerateWithEoz(TestConfig config) { @ParameterizedTest @MethodSource("testConfigs") void deleteObject(TestConfig config) throws Exception { - LOG.debug(() -> "Running deleteObject with config: " + config.toString()); assumeNotAccessPointWithPathStyle(config); assumeNotAccelerateWithPathStyle(config); assumeNotAccelerateWithArnType(config); assumeNotAccelerateWithEoz(config); + LOG.debug(() -> "Running deleteObject with config: " + config.toString()); + String bucket = bucketForType(config.getBucketType()); String key = putRandomObject(config.getBucketType()); TestCallable callable = null; @@ -256,7 +257,6 @@ void deleteObject(TestConfig config) throws Exception { @ParameterizedTest @MethodSource("testConfigs") void restoreObject(TestConfig config) throws Exception { - LOG.debug(() -> "Running restoreObject with config: " + config.toString()); assumeNotAccessPointWithPathStyle(config); assumeNotAccelerateWithPathStyle(config); assumeNotAccelerateWithArnType(config); @@ -264,6 +264,8 @@ void restoreObject(TestConfig config) throws Exception { Assumptions.assumeFalse(config.getBucketType() == BucketType.EOZ, "Restore is not supported for S3 Express"); + LOG.debug(() -> "Running restoreObject with config: " + config.toString()); + String bucket = bucketForType(config.getBucketType()); String key = putRandomArchivedObject(config.getBucketType()); TestCallable callable = null; @@ -298,12 +300,12 @@ void restoreObject(TestConfig config) throws Exception { @ParameterizedTest @MethodSource("uploadConfigs") void putObject(UploadConfig config) throws Exception { - LOG.debug(() -> "Running putObject with config: " + config.toString()); assumeNotAccelerateWithPathStyle(config.getBaseConfig()); assumeNotAccessPointWithPathStyle(config.getBaseConfig()); assumeNotAccelerateWithArnType(config.getBaseConfig()); assumeNotAccelerateWithEoz(config.getBaseConfig()); + // For testing purposes, ContentProvider is Publisher for async clients // There is no way to create AsyncRequestBody with a Publisher and also provide the content length Assumptions.assumeFalse(config.getBodyType() == BodyType.CONTENT_PROVIDER_WITH_LENGTH @@ -329,6 +331,8 @@ void putObject(UploadConfig config) throws Exception { && config.getBodyType() == BodyType.CONTENT_PROVIDER_NO_LENGTH, "Async Java based support unknown content length"); + LOG.debug(() -> "Running putObject with config: " + config.toString()); + BucketType bucketType = config.getBaseConfig().getBucketType(); String bucket = bucketForType(bucketType); @@ -549,9 +553,9 @@ private S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideConfigura // .multipartEnabled(true) // .build(); case ASYNC_CRT: { - if (overrideConfiguration != null) { - LOG.warn(() -> "Override configuration cannot be set for Async S3 CRT!"); - } + // if (overrideConfiguration != null) { + // LOG.warn(() -> "Override configuration cannot be set for Async S3 CRT!"); + // } return S3AsyncClient.crtBuilder() .forcePathStyle(config.isForcePathStyle()) .requestChecksumCalculation(config.getRequestChecksumValidation()) @@ -978,7 +982,7 @@ private TestRequestBody getRequestBody(BodyType bodyType, ContentSize contentSiz case STRING: { byte[] content = contentSize.content(); long contentLength = content.length; - return new TestRequestBody(RequestBody.fromString(new String(content, StandardCharsets.UTF_8)), + return new TestRequestBody(RequestBody.fromString(new String(content)), contentLength, crc32(content)); } From 633e9166f6452f91a0e0ba364dbf3ce6acd58a96 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Wed, 26 Mar 2025 13:38:54 -0400 Subject: [PATCH 10/46] fix for string content length --- .../checksum/ChecksumIntegrationTesting.java | 95 +++++++++++-------- 1 file changed, 54 insertions(+), 41 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index abebd21a2eb6..c0fa2da6b4cb 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -56,7 +56,6 @@ import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm; import software.amazon.awssdk.checksums.SdkChecksum; import software.amazon.awssdk.core.async.AsyncRequestBody; -import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.async.BlockingInputStreamAsyncRequestBody; import software.amazon.awssdk.core.async.BlockingOutputStreamAsyncRequestBody; import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; @@ -65,7 +64,6 @@ import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.http.SdkHttpMethod; import software.amazon.awssdk.http.SdkHttpRequest; import software.amazon.awssdk.regions.Region; @@ -912,13 +910,29 @@ enum ContentSize { LARGE; // 200 MiB - byte[] content() { + byte[] byteContent() { switch (this) { case SMALL: return smallContent; case LARGE: return largeContent; default: throw new IllegalArgumentException("not supported ContentSize " + this); } } + + String stringContent() { + switch (this) { + case SMALL: return "Hello World!"; + case LARGE: return new String(largeContent(), StandardCharsets.UTF_8); + default: throw new IllegalArgumentException("not supported ContentSize " + this); + } + } + + Path fileContent() { + switch (this) { + case SMALL: return testFileSmall; + case LARGE: return testFileLarge; + default: throw new IllegalArgumentException("not supported ContentSize " + this); + } + } } private static byte[] largeContent() { @@ -980,50 +994,49 @@ private String putRandomArchivedObject(BucketType bucketType) { private TestRequestBody getRequestBody(BodyType bodyType, ContentSize contentSize) throws IOException { switch (bodyType) { case STRING: { - byte[] content = contentSize.content(); - long contentLength = content.length; - return new TestRequestBody(RequestBody.fromString(new String(content)), - contentLength, + String content = contentSize.stringContent(); + return new TestRequestBody(RequestBody.fromString(content), + content.getBytes().length, crc32(content)); } case FILE: - return new TestRequestBody(RequestBody.fromFile(testFileSmall), Files.size(testFileSmall), crc32(testFileSmall)); + return new TestRequestBody(RequestBody.fromFile(contentSize.fileContent()), Files.size(contentSize.fileContent()), crc32(contentSize.fileContent())); case CONTENT_PROVIDER_NO_LENGTH: { RequestBody wrapped = - RequestBody.fromContentProvider(() -> FunctionalUtils.invokeSafely(() -> Files.newInputStream(testFileSmall)), + RequestBody.fromContentProvider(() -> FunctionalUtils.invokeSafely(() -> Files.newInputStream(contentSize.fileContent())), "application/octet-stream"); - return new TestRequestBody(wrapped, Files.size(testFileSmall), crc32(testFileSmall)); + return new TestRequestBody(wrapped, Files.size(contentSize.fileContent()), crc32(contentSize.fileContent())); } case CONTENT_PROVIDER_WITH_LENGTH: { - long contentLength = Files.size(testFileSmall); - RequestBody wrapped = RequestBody.fromContentProvider(() -> FunctionalUtils.invokeSafely(() -> Files.newInputStream(testFileSmall)), - Files.size(testFileSmall), + long contentLength = Files.size(contentSize.fileContent()); + RequestBody wrapped = RequestBody.fromContentProvider(() -> FunctionalUtils.invokeSafely(() -> Files.newInputStream(contentSize.fileContent())), + Files.size(contentSize.fileContent()), "application/octet-stream"); - return new TestRequestBody(wrapped, contentLength, crc32(testFileSmall)); + return new TestRequestBody(wrapped, contentLength, crc32(contentSize.fileContent())); } case INPUTSTREAM_RESETABLE: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); RequestBody wrapped = RequestBody.fromInputStream(new ByteArrayInputStream(content), content.length); return new TestRequestBody(wrapped, content.length, crc32(content)); } case INPUTSTREAM_NOT_RESETABLE: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); RequestBody wrapped = RequestBody.fromInputStream(new NonResettableByteStream(content), content.length); return new TestRequestBody(wrapped, content.length, crc32(content)); } case BYTES: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); RequestBody wrapped = RequestBody.fromBytes(content); return new TestRequestBody(wrapped, content.length, crc32(content)); } case BYTE_BUFFER: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); RequestBody wrapped = RequestBody.fromByteBuffer(ByteBuffer.wrap(content)); return new TestRequestBody(wrapped, content.length, crc32(content)); } case REMAINING_BYTE_BUFFER: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); ByteBuffer buff = ByteBuffer.wrap(content); int offset = 2; buff.position(offset); @@ -1049,47 +1062,47 @@ private TestRequestBody getRequestBody(BodyType bodyType, ContentSize contentSiz private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize contentSize) throws IOException { switch (bodyType) { case STRING: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); long contentLength = content.length; return new TestAsyncBody(AsyncRequestBody.fromString(new String(content)), contentLength, crc32(content), bodyType); } case FILE: { - long contentLength = Files.size(testFileSmall); - return new TestAsyncBody(AsyncRequestBody.fromFile(testFileSmall), contentLength, crc32(testFileSmall), bodyType); + long contentLength = Files.size(contentSize.fileContent()); + return new TestAsyncBody(AsyncRequestBody.fromFile(contentSize.fileContent()), contentLength, crc32(contentSize.fileContent()), bodyType); } case INPUTSTREAM_RESETABLE: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromInputStream(new ByteArrayInputStream(content), (long) content.length, ASYNC_REQUEST_BODY_EXECUTOR); return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); } case INPUTSTREAM_NOT_RESETABLE: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromInputStream(new NonResettableByteStream(content), (long) content.length, ASYNC_REQUEST_BODY_EXECUTOR); return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); } case CONTENT_PROVIDER_NO_LENGTH: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); Flowable publisher = Flowable.just(ByteBuffer.wrap(content)); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromPublisher(publisher); return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); } case BYTES: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromBytes(content); return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); } case BYTE_BUFFER: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromByteBuffer(ByteBuffer.wrap(content)); return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); } case REMAINING_BYTE_BUFFER: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); ByteBuffer buff = ByteBuffer.wrap(content); int offset = 2; buff.position(offset); @@ -1098,17 +1111,17 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content return new TestAsyncBody(asyncRequestBody, content.length - offset, crc32(crcArray), bodyType); } case BYTES_UNSAFE:{ - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromBytesUnsafe(content); return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); } case BYTE_BUFFER_UNSAFE: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromByteBufferUnsafe(ByteBuffer.wrap(content)); return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); } case REMAINING_BYTE_BUFFER_UNSAFE: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); ByteBuffer buff = ByteBuffer.wrap(content); int offset = 2; buff.position(offset); @@ -1117,8 +1130,8 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content return new TestAsyncBody(asyncRequestBody, content.length - offset, crc32(crcArray), bodyType); } case BUFFERS: { - byte[] content1 = contentSize.content(); - byte[] content2 = contentSize.content(); + byte[] content1 = contentSize.byteContent(); + byte[] content2 = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromByteBuffers(ByteBuffer.wrap(content1), ByteBuffer.wrap(content2)); byte[] crcArray = new byte[content2.length + content2.length]; @@ -1130,8 +1143,8 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content bodyType); } case BUFFERS_REMAINING: { - byte[] content1 = contentSize.content(); - byte[] content2 = contentSize.content(); + byte[] content1 = contentSize.byteContent(); + byte[] content2 = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromRemainingByteBuffers(ByteBuffer.wrap(content1), ByteBuffer.wrap(content2)); byte[] crcArray = new byte[content2.length + content2.length]; @@ -1143,8 +1156,8 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content bodyType); } case BUFFERS_UNSAFE: { - byte[] content1 = contentSize.content(); - byte[] content2 = contentSize.content(); + byte[] content1 = contentSize.byteContent(); + byte[] content2 = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromByteBuffersUnsafe(ByteBuffer.wrap(content1), ByteBuffer.wrap(content2)); byte[] crcArray = new byte[content2.length + content2.length]; @@ -1156,8 +1169,8 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content bodyType); } case BUFFERS_REMAINING_UNSAFE: { - byte[] content1 = contentSize.content(); - byte[] content2 = contentSize.content(); + byte[] content1 = contentSize.byteContent(); + byte[] content2 = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromRemainingByteBuffersUnsafe(ByteBuffer.wrap(content1), ByteBuffer.wrap(content2)); byte[] crcArray = new byte[content2.length + content2.length]; @@ -1169,7 +1182,7 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content bodyType); } case BLOCKING_INPUT_STREAM: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); long streamToSendLength = content.length; BlockingInputStreamAsyncRequestBody body = AsyncRequestBody.forBlockingInputStream(streamToSendLength); return new TestAsyncBodyForBlockingInputStream(body, @@ -1179,7 +1192,7 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content bodyType); } case BLOCKING_OUTPUT_STREAM: { - byte[] content = contentSize.content(); + byte[] content = contentSize.byteContent(); long streamToSendLength = content.length; BlockingOutputStreamAsyncRequestBody body = AsyncRequestBody.forBlockingOutputStream(streamToSendLength); Consumer bodyWrite = outputStream -> { From 203b0ed3861a8c925b0e46b22582c01b1aa8ea9a Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Thu, 27 Mar 2025 09:50:33 -0400 Subject: [PATCH 11/46] fix for string content length --- .../checksum/ChecksumIntegrationTesting.java | 24 +++---------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index c0fa2da6b4cb..804f8d1f61fd 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -288,13 +288,6 @@ void restoreObject(TestConfig config) throws Exception { } } - // @ParameterizedTest - // @MethodSource("") - // void getObject(TestConfig config) { - // LOG.debug(() -> "Running putObject with config: " + config.toString()); - // - // } - @ParameterizedTest @MethodSource("uploadConfigs") void putObject(UploadConfig config) throws Exception { @@ -540,16 +533,6 @@ private S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideConfigura .accelerate(config.isAccelerateEnabled()) .overrideConfiguration(overrideConfiguration) .build(); - // case ASYNC_JAVA_BASED_MULTI: - // return S3AsyncClient.builder() - // .forcePathStyle(config.isForcePathStyle()) - // .requestChecksumCalculation(config.getRequestChecksumValidation()) - // .region(REGION) - // .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - // .accelerate(config.isAccelerateEnabled()) - // .overrideConfiguration(overrideConfiguration) - // .multipartEnabled(true) - // .build(); case ASYNC_CRT: { // if (overrideConfiguration != null) { // LOG.warn(() -> "Override configuration cannot be set for Async S3 CRT!"); @@ -996,7 +979,7 @@ private TestRequestBody getRequestBody(BodyType bodyType, ContentSize contentSiz case STRING: { String content = contentSize.stringContent(); return new TestRequestBody(RequestBody.fromString(content), - content.getBytes().length, + content.getBytes(StandardCharsets.UTF_8).length, crc32(content)); } case FILE: @@ -1062,9 +1045,8 @@ private TestRequestBody getRequestBody(BodyType bodyType, ContentSize contentSiz private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize contentSize) throws IOException { switch (bodyType) { case STRING: { - byte[] content = contentSize.byteContent(); - long contentLength = content.length; - return new TestAsyncBody(AsyncRequestBody.fromString(new String(content)), contentLength, crc32(content), bodyType); + String content = contentSize.stringContent(); + return new TestAsyncBody(AsyncRequestBody.fromString(content), content.getBytes(StandardCharsets.UTF_8).length, crc32(content), bodyType); } case FILE: { long contentLength = Files.size(contentSize.fileContent()); From babbbdd7c8ab4cb25a329f3d08c91afa7fee10a3 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Thu, 27 Mar 2025 10:22:19 -0400 Subject: [PATCH 12/46] test 200MiB large size --- .../awssdk/services/s3/checksum/ChecksumIntegrationTesting.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index 804f8d1f61fd..24e695463232 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -922,7 +922,7 @@ private static byte[] largeContent() { // 200 MiB Random r = new Random(); // byte[] b = new byte[200 * 1024 * 1024]; - byte[] b = new byte[2 * 1024 * 1024]; + byte[] b = new byte[200 * 1024 * 1024]; r.nextBytes(b); return b; } From 422c858fce236f43b87ed38620ccd7163ce2f87e Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Thu, 27 Mar 2025 10:40:18 -0400 Subject: [PATCH 13/46] use 80MiB for large files --- .../checksum/ChecksumIntegrationTesting.java | 10 +++++----- .../checksum/StreamingIntegrationTesting.java | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/StreamingIntegrationTesting.java diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index 24e695463232..7abb7e9db4e2 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -179,7 +179,7 @@ static void setup() throws InterruptedException, IOException { apArn = createAccessPoint(); testFileSmall = createRandomFile16KB(); - testFileLarge = createRandomFile200MB(); + testFileLarge = createRandomFile80MB(); } @AfterEach @@ -919,10 +919,10 @@ Path fileContent() { } private static byte[] largeContent() { - // 200 MiB + // 80 MiB Random r = new Random(); // byte[] b = new byte[200 * 1024 * 1024]; - byte[] b = new byte[200 * 1024 * 1024]; + byte[] b = new byte[80 * 1024 * 1024]; r.nextBytes(b); return b; } @@ -1342,12 +1342,12 @@ private static Path createRandomFile16KB() throws IOException { return tmp; } - private static Path createRandomFile200MB() throws IOException { + private static Path createRandomFile80MB() throws IOException { Path tmp = Files.createTempFile(null, null); byte[] randomBytes = new byte[1024 * 1024]; new Random().nextBytes(randomBytes); try (OutputStream os = Files.newOutputStream(tmp)) { - for (int i = 0; i < 200; ++i) { + for (int i = 0; i < 80; ++i) { os.write(randomBytes); } } diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/StreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/StreamingIntegrationTesting.java new file mode 100644 index 000000000000..f3b23823ffce --- /dev/null +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/StreamingIntegrationTesting.java @@ -0,0 +1,20 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.checksum; + +public class StreamingIntegrationTesting { + +} From 25e3666b6e5c082058c725fd0fd6adcf4e8c9e88 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Thu, 27 Mar 2025 11:54:00 -0400 Subject: [PATCH 14/46] Fix for TM_JAVA --- test/s3-tests/pom.xml | 5 + .../amazon/awssdk/services/s3/BodyType.java | 44 +++++++ .../checksum/ChecksumIntegrationTesting.java | 118 ++++++++++-------- .../GetObjectChecksumIntegrationTesting.java | 19 +++ .../StreamingIntegrationTesting.java | 6 +- 5 files changed, 137 insertions(+), 55 deletions(-) create mode 100644 test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/BodyType.java create mode 100644 test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/GetObjectChecksumIntegrationTesting.java rename test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/{checksum => streaming}/StreamingIntegrationTesting.java (75%) diff --git a/test/s3-tests/pom.xml b/test/s3-tests/pom.xml index a70daa046cb2..bb14a1f14ce0 100644 --- a/test/s3-tests/pom.xml +++ b/test/s3-tests/pom.xml @@ -158,6 +158,11 @@ ${awsjavasdk.version} test + + software.amazon.awssdk + s3-transfer-manager + ${awsjavasdk.version} + diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/BodyType.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/BodyType.java new file mode 100644 index 000000000000..3f4b8dea34c2 --- /dev/null +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/BodyType.java @@ -0,0 +1,44 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3; + +public enum BodyType { + INPUTSTREAM_RESETABLE, + INPUTSTREAM_NOT_RESETABLE, + + STRING, + + FILE, + + CONTENT_PROVIDER_WITH_LENGTH, + CONTENT_PROVIDER_NO_LENGTH, + + BYTES, + BYTE_BUFFER, + REMAINING_BYTE_BUFFER, + + BYTES_UNSAFE, + BYTE_BUFFER_UNSAFE, + REMAINING_BYTE_BUFFER_UNSAFE, + + BUFFERS, + BUFFERS_REMAINING, + BUFFERS_UNSAFE, + BUFFERS_REMAINING_UNSAFE, + + BLOCKING_INPUT_STREAM, + BLOCKING_OUTPUT_STREAM +} diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index 7abb7e9db4e2..bc6ae23889e4 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -19,6 +19,7 @@ import io.reactivex.Flowable; import java.io.ByteArrayInputStream; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -67,6 +68,7 @@ import software.amazon.awssdk.http.SdkHttpMethod; import software.amazon.awssdk.http.SdkHttpRequest; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.BodyType; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.BucketAccelerateStatus; @@ -93,11 +95,15 @@ import software.amazon.awssdk.services.s3control.model.MultiRegionAccessPointStatus; import software.amazon.awssdk.services.s3control.model.S3ControlException; import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.CompletedUpload; +import software.amazon.awssdk.transfer.s3.model.Upload; import software.amazon.awssdk.utils.BinaryUtils; import software.amazon.awssdk.utils.CancellableOutputStream; import software.amazon.awssdk.utils.CompletableFutureUtils; import software.amazon.awssdk.utils.FunctionalUtils; import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.SdkAutoCloseable; // Running putObject with config: // UploadConfig{ @@ -178,7 +184,7 @@ static void setup() throws InterruptedException, IOException { apArn = createAccessPoint(); - testFileSmall = createRandomFile16KB(); + testFileSmall = createRandomFile2KB(); testFileLarge = createRandomFile80MB(); } @@ -358,6 +364,12 @@ void putObject(UploadConfig config) throws Exception { actualContentLength = body.getActualContentLength(); requestBodyHasContentLength = body.optionalContentLength().isPresent(); actualCrc32 = body.getChecksum(); + } else if (config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) { + TestAsyncBody body = getAsyncRequestBody(config.getBodyType(), config.getContentSize()); + callable = callTmUpload(request, body, config.getBaseConfig(), overrideConfiguration.build()); + actualContentLength = body.getActualContentLength(); + requestBodyHasContentLength = body.getAsyncRequestBody().contentLength().isPresent(); + actualCrc32 = body.getChecksum(); } else { TestAsyncBody body = getAsyncRequestBody(config.getBodyType(), config.contentSize); callable = callPutObject(request, body, config.getBaseConfig(), overrideConfiguration.build()); @@ -475,19 +487,7 @@ private TestCallable callPutObject(PutObjectRequest request, try { AsyncRequestBody asyncRequestBody = requestBody.getAsyncRequestBody(); CompletableFuture future = s3Client.putObject(request, asyncRequestBody); - if (requestBody.bodyType == BodyType.BLOCKING_INPUT_STREAM) { - BlockingInputStreamAsyncRequestBody body = (BlockingInputStreamAsyncRequestBody) requestBody.asyncRequestBody; - InputStream inputStream = ((TestAsyncBodyForBlockingInputStream) requestBody).inputStream; - body.writeInputStream(inputStream); - inputStream.close(); - } - if (requestBody.bodyType == BodyType.BLOCKING_OUTPUT_STREAM) { - TestAsyncBodyForBlockingOutputStream body = (TestAsyncBodyForBlockingOutputStream) requestBody; - CancellableOutputStream outputStream = - ((BlockingOutputStreamAsyncRequestBody) body.getAsyncRequestBody()).outputStream(); - body.bodyWrite.accept(outputStream); - outputStream.close(); - } + performUploadIfRequired(requestBody.bodyType, requestBody); return CompletableFutureUtils.joinLikeSync(future); } catch (Exception e) { throw new RuntimeException(e); @@ -496,11 +496,44 @@ private TestCallable callPutObject(PutObjectRequest request, return new TestCallable<>(s3Client, callable); } + private TestCallable callTmUpload(PutObjectRequest request, + TestAsyncBody requestBody, + TestConfig config, + ClientOverrideConfiguration overrideConfiguration) { + S3TransferManager tm = S3TransferManager.builder() + .s3Client(makeAsyncClient(config, overrideConfiguration)) + .build(); + Callable callable = () -> { + Upload upload = tm.upload(r -> r.requestBody(requestBody.getAsyncRequestBody()).putObjectRequest(request)); + CompletableFuture future = upload.completionFuture(); + performUploadIfRequired(requestBody.bodyType, requestBody); + CompletedUpload completedUpload = CompletableFutureUtils.joinLikeSync(future); + return completedUpload.response(); + }; + return new TestCallable<>(tm, callable); + } + + private void performUploadIfRequired(BodyType bodyType, TestAsyncBody requestBody) throws IOException { + if (bodyType == BodyType.BLOCKING_INPUT_STREAM) { + BlockingInputStreamAsyncRequestBody body = (BlockingInputStreamAsyncRequestBody) requestBody.asyncRequestBody; + InputStream inputStream = ((TestAsyncBodyForBlockingInputStream) requestBody).inputStream; + body.writeInputStream(inputStream); + inputStream.close(); + } + if (bodyType == BodyType.BLOCKING_OUTPUT_STREAM) { + TestAsyncBodyForBlockingOutputStream body = (TestAsyncBodyForBlockingOutputStream) requestBody; + CancellableOutputStream outputStream = + ((BlockingOutputStreamAsyncRequestBody) body.getAsyncRequestBody()).outputStream(); + body.bodyWrite.accept(outputStream); + outputStream.close(); + } + } + private static class TestCallable { - private AwsClient client; + private SdkAutoCloseable client; private Callable runnable; - TestCallable(AwsClient client, Callable runnable) { + TestCallable(SdkAutoCloseable client, Callable runnable) { this.client = client; this.runnable = runnable; } @@ -533,10 +566,17 @@ private S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideConfigura .accelerate(config.isAccelerateEnabled()) .overrideConfiguration(overrideConfiguration) .build(); + case TM_JAVA: + return S3AsyncClient.builder() + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .accelerate(config.isAccelerateEnabled()) + .overrideConfiguration(overrideConfiguration) + .multipartEnabled(true) + .build(); case ASYNC_CRT: { - // if (overrideConfiguration != null) { - // LOG.warn(() -> "Override configuration cannot be set for Async S3 CRT!"); - // } return S3AsyncClient.crtBuilder() .forcePathStyle(config.isForcePathStyle()) .requestChecksumCalculation(config.getRequestChecksumValidation()) @@ -588,7 +628,7 @@ public boolean isArnType() { enum S3ClientFlavor { JAVA_BASED(false), ASYNC_JAVA_BASED(true), - // ASYNC_JAVA_BASED_MULTI(true), + TM_JAVA(true), ASYNC_CRT(true) ; @@ -837,7 +877,7 @@ static List testConfigs() { boolean[] payloadSigningEnabled = {true, false}; for (boolean pathStyle : forcePathStyle) { for (RequestChecksumCalculation checksumValidation : checksumValidations) { - for (S3ClientFlavor flavor : S3ClientFlavor.values()) { + for (S3ClientFlavor flavor : new S3ClientFlavor[]{ S3ClientFlavor.TM_JAVA }) { for (BucketType bucketType : BucketType.values()) { for (boolean accelerate : accelerateEnabled) { for (boolean payloadSigning : payloadSigningEnabled) { @@ -859,39 +899,9 @@ static List testConfigs() { return configs; } - enum BodyType { - INPUTSTREAM_RESETABLE, - INPUTSTREAM_NOT_RESETABLE, - - STRING, - - FILE, - - CONTENT_PROVIDER_WITH_LENGTH, - - CONTENT_PROVIDER_NO_LENGTH, - - BYTES, - BYTE_BUFFER, - REMAINING_BYTE_BUFFER, - - BYTES_UNSAFE, - BYTE_BUFFER_UNSAFE, - REMAINING_BYTE_BUFFER_UNSAFE, - - BUFFERS, - BUFFERS_REMAINING, - BUFFERS_UNSAFE, - BUFFERS_REMAINING_UNSAFE, - - BLOCKING_INPUT_STREAM, - BLOCKING_OUTPUT_STREAM - } - enum ContentSize { SMALL, - LARGE; // 200 MiB - + LARGE; // 80 MiB byte[] byteContent() { switch (this) { @@ -1330,12 +1340,12 @@ private static void createBucket(String bucketName, int retryCount) { s3.waiter().waitUntilBucketExists(r -> r.bucket(bucketName)); } - private static Path createRandomFile16KB() throws IOException { + private static Path createRandomFile2KB() throws IOException { Path tmp = Files.createTempFile(null, null); byte[] randomBytes = new byte[1024]; new Random().nextBytes(randomBytes); try (OutputStream os = Files.newOutputStream(tmp)) { - for (int i = 0; i < 16; ++i) { + for (int i = 0; i < 2; ++i) { os.write(randomBytes); } } diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/GetObjectChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/GetObjectChecksumIntegrationTesting.java new file mode 100644 index 000000000000..58f505739c73 --- /dev/null +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/GetObjectChecksumIntegrationTesting.java @@ -0,0 +1,19 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.checksum; + +public class GetObjectChecksumIntegrationTesting { +} diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/StreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/streaming/StreamingIntegrationTesting.java similarity index 75% rename from test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/StreamingIntegrationTesting.java rename to test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/streaming/StreamingIntegrationTesting.java index f3b23823ffce..e830b0653c3b 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/StreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/streaming/StreamingIntegrationTesting.java @@ -13,7 +13,11 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.services.s3.checksum; +package software.amazon.awssdk.services.s3.streaming; + +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ChecksumMode; public class StreamingIntegrationTesting { From 67842a349db841b567bec3b0dc5286f44538ae4b Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Thu, 27 Mar 2025 12:00:36 -0400 Subject: [PATCH 15/46] Fix for TM_JAVA --- .../GetObjectChecksumIntegrationTesting.java | 19 --------------- .../StreamingIntegrationTesting.java | 24 ------------------- 2 files changed, 43 deletions(-) delete mode 100644 test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/GetObjectChecksumIntegrationTesting.java delete mode 100644 test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/streaming/StreamingIntegrationTesting.java diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/GetObjectChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/GetObjectChecksumIntegrationTesting.java deleted file mode 100644 index 58f505739c73..000000000000 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/GetObjectChecksumIntegrationTesting.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.services.s3.checksum; - -public class GetObjectChecksumIntegrationTesting { -} diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/streaming/StreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/streaming/StreamingIntegrationTesting.java deleted file mode 100644 index e830b0653c3b..000000000000 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/streaming/StreamingIntegrationTesting.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.services.s3.streaming; - -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.ChecksumMode; - -public class StreamingIntegrationTesting { - -} From 764629a0b37288033b0ab78cd78a4260fcd680c4 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Thu, 27 Mar 2025 12:54:19 -0400 Subject: [PATCH 16/46] Fix for TM_JAVA --- test/s3-tests/pom.xml | 6 ++ .../checksum/ChecksumIntegrationTesting.java | 98 ++++++++++++++----- .../checksum/StreamingIntegrationTesting.java | 20 ---- 3 files changed, 77 insertions(+), 47 deletions(-) delete mode 100644 test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/StreamingIntegrationTesting.java diff --git a/test/s3-tests/pom.xml b/test/s3-tests/pom.xml index a70daa046cb2..5b106776ca86 100644 --- a/test/s3-tests/pom.xml +++ b/test/s3-tests/pom.xml @@ -158,6 +158,12 @@ ${awsjavasdk.version} test + + software.amazon.awssdk + s3-transfer-manager + ${awsjavasdk.version} + test + diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index 7abb7e9db4e2..2a3f2599ceba 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -93,11 +93,15 @@ import software.amazon.awssdk.services.s3control.model.MultiRegionAccessPointStatus; import software.amazon.awssdk.services.s3control.model.S3ControlException; import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.CompletedUpload; +import software.amazon.awssdk.transfer.s3.model.Upload; import software.amazon.awssdk.utils.BinaryUtils; import software.amazon.awssdk.utils.CancellableOutputStream; import software.amazon.awssdk.utils.CompletableFutureUtils; import software.amazon.awssdk.utils.FunctionalUtils; import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.SdkAutoCloseable; // Running putObject with config: // UploadConfig{ @@ -262,7 +266,7 @@ void restoreObject(TestConfig config) throws Exception { Assumptions.assumeFalse(config.getBucketType() == BucketType.EOZ, "Restore is not supported for S3 Express"); - LOG.debug(() -> "Running restoreObject with config: " + config.toString()); + LOG.debug(() -> "Running restoreObject with config: " + config); String bucket = bucketForType(config.getBucketType()); String key = putRandomArchivedObject(config.getBucketType()); @@ -305,9 +309,8 @@ void putObject(UploadConfig config) throws Exception { // Payload signing doesn't work correctly for async java based Assumptions.assumeFalse( - (config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED - // || config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED_MULTI - ) + (config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED || + config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) && (config.getBaseConfig().isPayloadSigning() // MRAP requires body signing || config.getBaseConfig().getBucketType() == BucketType.MRAP), @@ -316,13 +319,12 @@ void putObject(UploadConfig config) throws Exception { // For testing purposes, ContentProvider is Publisher for async clients // Async java based clients don't currently support unknown content-length bodies Assumptions.assumeFalse( - (config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED - // || config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED_MULTI - ) + (config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED || + config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) && config.getBodyType() == BodyType.CONTENT_PROVIDER_NO_LENGTH, "Async Java based support unknown content length"); - LOG.debug(() -> "Running putObject with config: " + config.toString()); + LOG.debug(() -> "Running putObject with config: " + config); BucketType bucketType = config.getBaseConfig().getBucketType(); @@ -358,6 +360,12 @@ void putObject(UploadConfig config) throws Exception { actualContentLength = body.getActualContentLength(); requestBodyHasContentLength = body.optionalContentLength().isPresent(); actualCrc32 = body.getChecksum(); + } else if (config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) { + TestAsyncBody body = getAsyncRequestBody(config.getBodyType(), config.contentSize); + callable = callTmUpload(request, body, config.getBaseConfig(), overrideConfiguration.build()); + actualContentLength = body.getActualContentLength(); + requestBodyHasContentLength = body.getAsyncRequestBody().contentLength().isPresent(); + actualCrc32 = body.getChecksum(); } else { TestAsyncBody body = getAsyncRequestBody(config.getBodyType(), config.contentSize); callable = callPutObject(request, body, config.getBaseConfig(), overrideConfiguration.build()); @@ -409,6 +417,7 @@ void putObject(UploadConfig config) throws Exception { } } + private TestCallable callDeleteObjects(DeleteObjectsRequest request, TestConfig config) { AwsClient toClose; Callable runnable = null; @@ -456,7 +465,7 @@ private TestCallable callRestoreObject(RestoreObjectRequest request, TestC } private TestCallable callPutObject(PutObjectRequest request, TestRequestBody requestBody, TestConfig config, - ClientOverrideConfiguration overrideConfiguration) throws IOException { + ClientOverrideConfiguration overrideConfiguration) { S3Client s3Client = makeSyncClient(config, overrideConfiguration); Callable callable = () -> { try { @@ -469,25 +478,13 @@ private TestCallable callPutObject(PutObjectRequest request, } private TestCallable callPutObject(PutObjectRequest request, TestAsyncBody requestBody, TestConfig config, - ClientOverrideConfiguration overrideConfiguration) throws IOException { + ClientOverrideConfiguration overrideConfiguration) { S3AsyncClient s3Client = makeAsyncClient(config, overrideConfiguration); Callable callable = () -> { try { AsyncRequestBody asyncRequestBody = requestBody.getAsyncRequestBody(); CompletableFuture future = s3Client.putObject(request, asyncRequestBody); - if (requestBody.bodyType == BodyType.BLOCKING_INPUT_STREAM) { - BlockingInputStreamAsyncRequestBody body = (BlockingInputStreamAsyncRequestBody) requestBody.asyncRequestBody; - InputStream inputStream = ((TestAsyncBodyForBlockingInputStream) requestBody).inputStream; - body.writeInputStream(inputStream); - inputStream.close(); - } - if (requestBody.bodyType == BodyType.BLOCKING_OUTPUT_STREAM) { - TestAsyncBodyForBlockingOutputStream body = (TestAsyncBodyForBlockingOutputStream) requestBody; - CancellableOutputStream outputStream = - ((BlockingOutputStreamAsyncRequestBody) body.getAsyncRequestBody()).outputStream(); - body.bodyWrite.accept(outputStream); - outputStream.close(); - } + performWriteIfNeeded(requestBody); return CompletableFutureUtils.joinLikeSync(future); } catch (Exception e) { throw new RuntimeException(e); @@ -496,11 +493,44 @@ private TestCallable callPutObject(PutObjectRequest request, return new TestCallable<>(s3Client, callable); } + private TestCallable callTmUpload(PutObjectRequest request, TestAsyncBody requestBody, TestConfig config, + ClientOverrideConfiguration overrideConfiguration) { + S3TransferManager transferManager = makeTm(config, overrideConfiguration); + Callable callable = () -> { + try { + Upload upload = transferManager.upload( + r -> r.requestBody(requestBody.getAsyncRequestBody()).putObjectRequest(request)); + performWriteIfNeeded(requestBody); + CompletedUpload completedUpload = CompletableFutureUtils.joinLikeSync(upload.completionFuture()); + return completedUpload.response(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + return new TestCallable<>(transferManager, callable); + } + + void performWriteIfNeeded(TestAsyncBody requestBody) throws IOException { + if (requestBody.bodyType == BodyType.BLOCKING_INPUT_STREAM) { + BlockingInputStreamAsyncRequestBody body = (BlockingInputStreamAsyncRequestBody) requestBody.asyncRequestBody; + InputStream inputStream = ((TestAsyncBodyForBlockingInputStream) requestBody).inputStream; + body.writeInputStream(inputStream); + inputStream.close(); + } + if (requestBody.bodyType == BodyType.BLOCKING_OUTPUT_STREAM) { + TestAsyncBodyForBlockingOutputStream body = (TestAsyncBodyForBlockingOutputStream) requestBody; + CancellableOutputStream outputStream = + ((BlockingOutputStreamAsyncRequestBody) body.getAsyncRequestBody()).outputStream(); + body.bodyWrite.accept(outputStream); + outputStream.close(); + } + } + private static class TestCallable { - private AwsClient client; + private SdkAutoCloseable client; private Callable runnable; - TestCallable(AwsClient client, Callable runnable) { + TestCallable(SdkAutoCloseable client, Callable runnable) { this.client = client; this.runnable = runnable; } @@ -533,6 +563,16 @@ private S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideConfigura .accelerate(config.isAccelerateEnabled()) .overrideConfiguration(overrideConfiguration) .build(); + case TM_JAVA: + return S3AsyncClient.builder() + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .accelerate(config.isAccelerateEnabled()) + .overrideConfiguration(overrideConfiguration) + .multipartEnabled(true) + .build(); case ASYNC_CRT: { // if (overrideConfiguration != null) { // LOG.warn(() -> "Override configuration cannot be set for Async S3 CRT!"); @@ -550,6 +590,11 @@ private S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideConfigura } } + private S3TransferManager makeTm(TestConfig config, ClientOverrideConfiguration overrideConfiguration) { + S3AsyncClient s3AsyncClient = makeAsyncClient(config, overrideConfiguration); + return S3TransferManager.builder().s3Client(s3AsyncClient).build(); + } + private static String bucketForType(BucketType type) { switch (type) { case STANDARD_BUCKET: @@ -588,7 +633,7 @@ public boolean isArnType() { enum S3ClientFlavor { JAVA_BASED(false), ASYNC_JAVA_BASED(true), - // ASYNC_JAVA_BASED_MULTI(true), + TM_JAVA(true), ASYNC_CRT(true) ; @@ -921,7 +966,6 @@ Path fileContent() { private static byte[] largeContent() { // 80 MiB Random r = new Random(); - // byte[] b = new byte[200 * 1024 * 1024]; byte[] b = new byte[80 * 1024 * 1024]; r.nextBytes(b); return b; diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/StreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/StreamingIntegrationTesting.java deleted file mode 100644 index f3b23823ffce..000000000000 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/StreamingIntegrationTesting.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.services.s3.checksum; - -public class StreamingIntegrationTesting { - -} From 2f01bc1475eceabbff1c95f83a1a9d3e6cf79001 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Thu, 27 Mar 2025 13:36:59 -0400 Subject: [PATCH 17/46] need more memory --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 91da20273825..35afe154c6ec 100644 --- a/pom.xml +++ b/pom.xml @@ -908,7 +908,7 @@ - ${argLine} + -Xmx8g -Xms1024m -XX:+AllowRedefinitionToAddDeleteMethods **/s3/checksum/ChecksumIntegrationTesting.java From 3ba77d28897208956d5d63bae189be063e62695a Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Thu, 27 Mar 2025 14:45:10 -0400 Subject: [PATCH 18/46] more fix for TM --- .../services/s3/checksum/ChecksumIntegrationTesting.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index 2a3f2599ceba..e76a7a70966e 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -378,6 +378,11 @@ void putObject(UploadConfig config) throws Exception { recordObjectToCleanup(bucketType, key); + // mpu not supported + if (config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) { + return; + } + // We only validate when configured to WHEN_SUPPORTED since checksums are optional for PutObject if (config.getBaseConfig().getRequestChecksumValidation() == RequestChecksumCalculation.WHEN_SUPPORTED // CRT switches to MPU under the hood which doesn't support checksums From c07dab303dde0ee0175893f44c37c883c50d3e7b Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 31 Mar 2025 10:51:49 -0400 Subject: [PATCH 19/46] some more dl stuff --- pom.xml | 2 + test/s3-tests/pom.xml | 4 + .../services/s3/checksum/BucketType.java | 36 ++ .../checksum/ChecksumIntegrationTesting.java | 417 +------------ .../DownloadStreamingIntegrationTesting.java | 589 ++++++++++++++++++ .../s3/checksum/S3ChecksumsTestUtils.java | 231 +++++++ .../services/s3/checksum/S3ClientFlavor.java | 34 + .../services/s3/checksum/TestConfig.java | 122 ++++ 8 files changed, 1036 insertions(+), 399 deletions(-) create mode 100644 test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/BucketType.java create mode 100644 test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java create mode 100644 test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/S3ChecksumsTestUtils.java create mode 100644 test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/S3ClientFlavor.java create mode 100644 test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/TestConfig.java diff --git a/pom.xml b/pom.xml index 35afe154c6ec..d569ffa24181 100644 --- a/pom.xml +++ b/pom.xml @@ -908,9 +908,11 @@ + -Xmx8g -Xms1024m -XX:+AllowRedefinitionToAddDeleteMethods **/s3/checksum/ChecksumIntegrationTesting.java + **/s3/checksum/DownloadStreamingIntegrationTesting.java false diff --git a/test/s3-tests/pom.xml b/test/s3-tests/pom.xml index 5b106776ca86..c5464c6ac77f 100644 --- a/test/s3-tests/pom.xml +++ b/test/s3-tests/pom.xml @@ -164,6 +164,10 @@ ${awsjavasdk.version} test + + software.amazon.awssdk + test-utils + diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/BucketType.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/BucketType.java new file mode 100644 index 000000000000..8ffdb5cfb6dc --- /dev/null +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/BucketType.java @@ -0,0 +1,36 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.checksum; + +public enum BucketType { + STANDARD_BUCKET(false), + ACCESS_POINT(true), + // Multi-region access point + MRAP(true), + // Express one zone/S3 express + EOZ(false), + ; + + private final boolean arnType; + + BucketType(boolean arnType) { + this.arnType = arnType; + } + + public boolean isArnType() { + return arnType; + } +} diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index e76a7a70966e..76693eb11e23 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -16,6 +16,11 @@ package software.amazon.awssdk.services.s3.checksum; import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccelerateWithArnType; +import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccelerateWithEoz; +import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccelerateWithPathStyle; +import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccessPointWithPathStyle; +import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.crc32; import io.reactivex.Flowable; import java.io.ByteArrayInputStream; @@ -103,19 +108,6 @@ import software.amazon.awssdk.utils.Logger; import software.amazon.awssdk.utils.SdkAutoCloseable; -// Running putObject with config: -// UploadConfig{ -// baseConfig=[ -// flavor=ASYNC_JAVA_BASED_MULTI, -// bucketType=STANDARD_BUCKET, -// forcePathStyle=true, -// requestChecksumValidation=WHEN_REQUIRED, -// accelerateEnabled=false, -// payloadSigning=false -// ], -// bodyType=BLOCKING_OUTPUT_STREAM, -// contentSize=SMALL -// } public class ChecksumIntegrationTesting { private static final String BUCKET_NAME_PREFIX = "do-not-delete-checksums-"; private static final String MRAP_NAME = "do-not-delete-checksum-testing"; @@ -135,8 +127,6 @@ public class ChecksumIntegrationTesting { .build(), DefaultCredentialsProvider.create()); - private static final SdkChecksum CRC32 = SdkChecksum.forAlgorithm(DefaultChecksumAlgorithm.CRC32); - private static final ExecutorService ASYNC_REQUEST_BODY_EXECUTOR = Executors.newSingleThreadExecutor(); private static String accountId; @@ -172,15 +162,15 @@ static void setup() throws InterruptedException, IOException { .region(REGION) .build(); - accountId = getAccountId(); + accountId = S3ChecksumsTestUtils.getAccountId(sts); - bucketName = createBucket(); + bucketName = S3ChecksumsTestUtils.createBucket(s3, getBucketName(), LOG); - mrapArn = createMrap(); + mrapArn = S3ChecksumsTestUtils.createMrap(s3Control, accountId, MRAP_NAME, bucketName, LOG); - eozBucket = createEozBucket(); + eozBucket = S3ChecksumsTestUtils.createEozBucket(s3, getBucketName() + EOZ_SUFFIX, LOG); - apArn = createAccessPoint(); + apArn = S3ChecksumsTestUtils.createAccessPoint(s3Control, accountId, AP_NAME, bucketName); testFileSmall = createRandomFile16KB(); testFileLarge = createRandomFile80MB(); @@ -201,26 +191,6 @@ public static void cleanup() { ASYNC_REQUEST_BODY_EXECUTOR.shutdownNow(); } - private void assumeNotAccessPointWithPathStyle(TestConfig config) { - BucketType bucketType = config.getBucketType(); - Assumptions.assumeFalse(config.isForcePathStyle() && bucketType.isArnType(), - "Path style doesn't work with ARN type buckets"); - } - - private void assumeNotAccelerateWithPathStyle(TestConfig config) { - Assumptions.assumeFalse(config.isForcePathStyle() && config.isAccelerateEnabled(), - "Path style doesn't work with Accelerate"); - } - - private void assumeNotAccelerateWithArnType(TestConfig config) { - Assumptions.assumeFalse(config.isAccelerateEnabled() && config.getBucketType().isArnType(), - "Accelerate doesn't work with ARN buckets"); - } - - private void assumeNotAccelerateWithEoz(TestConfig config) { - Assumptions.assumeFalse(config.isAccelerateEnabled() && config.getBucketType() == BucketType.EOZ, - "Accelerate is not supported with Express One Zone"); - } // Request checksum required @ParameterizedTest @@ -300,7 +270,6 @@ void putObject(UploadConfig config) throws Exception { assumeNotAccelerateWithArnType(config.getBaseConfig()); assumeNotAccelerateWithEoz(config.getBaseConfig()); - // For testing purposes, ContentProvider is Publisher for async clients // There is no way to create AsyncRequestBody with a Publisher and also provide the content length Assumptions.assumeFalse(config.getBodyType() == BodyType.CONTENT_PROVIDER_WITH_LENGTH @@ -561,13 +530,13 @@ private S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideConfigura switch (config.getFlavor()) { case ASYNC_JAVA_BASED: return S3AsyncClient.builder() - .forcePathStyle(config.isForcePathStyle()) - .requestChecksumCalculation(config.getRequestChecksumValidation()) - .region(REGION) - .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - .accelerate(config.isAccelerateEnabled()) - .overrideConfiguration(overrideConfiguration) - .build(); + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .accelerate(config.isAccelerateEnabled()) + .overrideConfiguration(overrideConfiguration) + .build(); case TM_JAVA: return S3AsyncClient.builder() .forcePathStyle(config.isForcePathStyle()) @@ -579,9 +548,6 @@ private S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideConfigura .multipartEnabled(true) .build(); case ASYNC_CRT: { - // if (overrideConfiguration != null) { - // LOG.warn(() -> "Override configuration cannot be set for Async S3 CRT!"); - // } return S3AsyncClient.crtBuilder() .forcePathStyle(config.isForcePathStyle()) .requestChecksumCalculation(config.getRequestChecksumValidation()) @@ -615,45 +581,6 @@ private static String bucketForType(BucketType type) { } } - enum BucketType { - STANDARD_BUCKET(false), - ACCESS_POINT(true), - // Multi-region access point - MRAP(true), - // Express one zone/S3 express - EOZ(false), - ; - - private final boolean arnType; - - private BucketType(boolean arnType) { - this.arnType = arnType; - } - - public boolean isArnType() { - return arnType; - } - } - - enum S3ClientFlavor { - JAVA_BASED(false), - ASYNC_JAVA_BASED(true), - TM_JAVA(true), - - ASYNC_CRT(true) - ; - - private final boolean async; - - private S3ClientFlavor(boolean async) { - this.async = async; - } - - public boolean isAsync() { - return async; - } - } - static class UploadConfig { private TestConfig baseConfig; private BodyType bodyType; @@ -694,51 +621,6 @@ public String toString() { } - enum ResponseTransformerType { - FILE, - BYTES, - INPUT_STREAM, - OUTPUT_STREAM, - UNMANAGED, - PUBLISHER - } - - static class DownloadConfig { - private TestConfig testConfig; - private ResponseTransformerType responseTransformerType; - private ContentSize contentSize; - - public DownloadConfig(TestConfig testConfig, ResponseTransformerType responseTransformerType, ContentSize contentSize) { - this.testConfig = testConfig; - this.responseTransformerType = responseTransformerType; - this.contentSize = contentSize; - } - - public TestConfig getTestConfig() { - return this.testConfig; - } - - public void setTestConfig(TestConfig testConfig) { - this.testConfig = testConfig; - } - - public ResponseTransformerType getResponseTransformerType() { - return responseTransformerType; - } - - public void setResponseTransformerType(ResponseTransformerType responseTransformerType) { - this.responseTransformerType = responseTransformerType; - } - - public ContentSize getContentSize() { - return contentSize; - } - - public void setContentSize(ContentSize contentSize) { - this.contentSize = contentSize; - } - } - static class TestRequestBody extends RequestBody { private final long contentLength; private final String checksum; @@ -808,105 +690,8 @@ private TestAsyncBodyForBlockingInputStream(AsyncRequestBody asyncRequestBody, } } - static class TestConfig { - private S3ClientFlavor flavor; - private BucketType bucketType; - private boolean forcePathStyle; - private RequestChecksumCalculation requestChecksumValidation; - private boolean accelerateEnabled; - private boolean payloadSigning; - - public S3ClientFlavor getFlavor() { - return flavor; - } - - public void setFlavor(S3ClientFlavor flavor) { - this.flavor = flavor; - } - - public BucketType getBucketType() { - return bucketType; - } - - public void setBucketType(BucketType bucketType) { - this.bucketType = bucketType; - } - - public boolean isForcePathStyle() { - return forcePathStyle; - } - - public void setForcePathStyle(boolean forcePathStyle) { - this.forcePathStyle = forcePathStyle; - } - - public RequestChecksumCalculation getRequestChecksumValidation() { - return requestChecksumValidation; - } - - public void setRequestChecksumValidation(RequestChecksumCalculation requestChecksumValidation) { - this.requestChecksumValidation = requestChecksumValidation; - } - - public boolean isAccelerateEnabled() { - return accelerateEnabled; - } - - public void setAccelerateEnabled(boolean accelerateEnabled) { - this.accelerateEnabled = accelerateEnabled; - } - - public boolean isPayloadSigning() { - return payloadSigning; - } - - public void setPayloadSigning(boolean payloadSigning) { - this.payloadSigning = payloadSigning; - } - - @Override - public String toString() { - return "[" + - "flavor=" + flavor + - ", bucketType=" + bucketType + - ", forcePathStyle=" + forcePathStyle + - ", requestChecksumValidation=" + requestChecksumValidation + - ", accelerateEnabled=" + accelerateEnabled + - ", payloadSigning=" + payloadSigning + - ']'; - } - } - static List testConfigs() { - List configs = new ArrayList<>(); - - boolean[] forcePathStyle = {true, false}; - RequestChecksumCalculation[] checksumValidations = {RequestChecksumCalculation.WHEN_REQUIRED, - RequestChecksumCalculation.WHEN_SUPPORTED}; - boolean[] accelerateEnabled = {true, false}; - boolean[] payloadSigningEnabled = {true, false}; - for (boolean pathStyle : forcePathStyle) { - for (RequestChecksumCalculation checksumValidation : checksumValidations) { - for (S3ClientFlavor flavor : S3ClientFlavor.values()) { - for (BucketType bucketType : BucketType.values()) { - for (boolean accelerate : accelerateEnabled) { - for (boolean payloadSigning : payloadSigningEnabled) { - TestConfig testConfig = new TestConfig(); - testConfig.setFlavor(flavor); - testConfig.setBucketType(bucketType); - testConfig.setForcePathStyle(pathStyle); - testConfig.setRequestChecksumValidation(checksumValidation); - testConfig.setAccelerateEnabled(accelerate); - testConfig.setPayloadSigning(payloadSigning); - configs.add(testConfig); - } - } - } - } - } - } - - return configs; + return TestConfig.testConfigs(); } enum BodyType { @@ -942,7 +727,6 @@ enum ContentSize { SMALL, LARGE; // 200 MiB - byte[] byteContent() { switch (this) { case SMALL: return smallContent; @@ -993,19 +777,6 @@ private static List uploadConfigs() { return configs; } - private static List downloadConfigs() { - List configs = new ArrayList<>(); - for (ResponseTransformerType responseTransformerType : ResponseTransformerType.values()) { - for (TestConfig baseConfig : testConfigs()) { - for (ContentSize contentSize : ContentSize.values()) { - DownloadConfig config = new DownloadConfig(baseConfig, responseTransformerType, contentSize); - configs.add(config); - } - } - } - return configs; - } - private String putRandomObject(BucketType bucketType) { String key = randomKey(); String bucketName = bucketForType(bucketType); @@ -1014,7 +785,6 @@ private String putRandomObject(BucketType bucketType) { return key; } - private String putRandomArchivedObject(BucketType bucketType) { String key = randomKey(); String bucketName = bucketForType(bucketType); @@ -1248,137 +1018,14 @@ private String randomKey() { return BinaryUtils.toHex(UUID.randomUUID().toString().getBytes()); } - private static String getAccountId() { - return sts.getCallerIdentity().account(); - } - private static String getBucketName() { return BUCKET_NAME_PREFIX + accountId; } - private static String createAccessPoint() { - try { - s3Control.getAccessPoint(r -> r.accountId(accountId).name(AP_NAME)); - } catch (S3ControlException e) { - if (e.awsErrorDetails().sdkHttpResponse().statusCode() != 404) { - throw e; - } - - s3Control.createAccessPoint(r -> r.bucket(bucketName).name(AP_NAME).accountId(accountId)); - } - - return waitForApToBeReady(); - } - - private static String createMrap() throws InterruptedException { - try { - s3Control.getMultiRegionAccessPoint(r -> r.accountId(accountId).name(MRAP_NAME)); - } catch (S3ControlException e) { - if (e.awsErrorDetails().sdkHttpResponse().statusCode() != 404) { - throw e; - } - - CreateMultiRegionAccessPointRequest createMrap = - CreateMultiRegionAccessPointRequest.builder() - .accountId(accountId) - .details(d -> d.name(MRAP_NAME) - .regions(software.amazon.awssdk.services.s3control.model.Region.builder() - .bucket(bucketName) - .build())) - .build(); - - s3Control.createMultiRegionAccessPoint(createMrap); - } - - return waitForMrapToBeReady(); - } - - private static String createBucket() { - String name = getBucketName(); - LOG.debug(() -> "Creating bucket: " + name); - createBucket(name, 3); - s3.putBucketAccelerateConfiguration(r -> r.bucket(name) - .accelerateConfiguration(c -> c.status(BucketAccelerateStatus.ENABLED))); - return name; - } - - private static String createEozBucket() { - String eozBucketName = getBucketName() + EOZ_SUFFIX; - LOG.debug(() -> "Creating EOZ bucket: " + eozBucketName); - CreateBucketConfiguration cfg = CreateBucketConfiguration.builder() - .bucket(info -> info.dataRedundancy(DataRedundancy.SINGLE_AVAILABILITY_ZONE) - .type(software.amazon.awssdk.services.s3.model.BucketType.DIRECTORY)) - .location(LocationInfo.builder() - .name("usw2-az3") - .type(LocationType.AVAILABILITY_ZONE) - .build()) - .build(); - - try { - s3.createBucket(r -> r.bucket(eozBucketName).createBucketConfiguration(cfg)); - } catch (S3Exception e) { - AwsErrorDetails awsErrorDetails = e.awsErrorDetails(); - if (!"BucketAlreadyOwnedByYou".equals(awsErrorDetails.errorCode())) { - throw e; - } - } - return eozBucketName; - } - - private static String waitForMrapToBeReady() throws InterruptedException { - GetMultiRegionAccessPointResponse getMrapResponse = null; - - Instant waitStart = Instant.now(); - boolean initial = true; - do { - if (!initial) { - Thread.sleep(Duration.ofSeconds(10).toMillis()); - initial = true; - } - GetMultiRegionAccessPointResponse response = s3Control.getMultiRegionAccessPoint(r -> r.accountId(accountId).name(MRAP_NAME)); - LOG.debug(() -> "Wait response: " + response); - getMrapResponse = response; - } while (MultiRegionAccessPointStatus.READY != getMrapResponse.accessPoint().status() - && Duration.between(Instant.now(), waitStart).compareTo(Duration.ofMinutes(5)) < 0); - - return "arn:aws:s3::" + accountId + ":accesspoint/" + getMrapResponse.accessPoint().alias(); - } - private static String waitForApToBeReady() { return s3Control.getAccessPoint(r -> r.accountId(accountId).name(AP_NAME)).accessPointArn(); } - private static void createBucket(String bucketName, int retryCount) { - try { - s3.createBucket( - CreateBucketRequest.builder() - .bucket(bucketName) - .createBucketConfiguration( - CreateBucketConfiguration.builder() - .locationConstraint(BucketLocationConstraint.US_WEST_2) - .build()) - .build()); - } catch (S3Exception e) { - LOG.debug(() -> "Error attempting to create bucket: " + bucketName); - if ("BucketAlreadyOwnedByYou".equals(e.awsErrorDetails().errorCode())) { - LOG.debug(() -> String.format("%s bucket already exists, likely leaked by a previous run%n", bucketName)); - } else if ("TooManyBuckets".equals(e.awsErrorDetails().errorCode())) { - LOG.debug(() -> "Printing all buckets for debug:"); - s3.listBuckets().buckets().forEach(l -> LOG.debug(l::toString)); - if (retryCount < 2) { - LOG.debug(() -> "Retrying..."); - createBucket(bucketName, retryCount + 1); - } else { - throw e; - } - } else { - throw e; - } - } - - s3.waiter().waitUntilBucketExists(r -> r.bucket(bucketName)); - } - private static Path createRandomFile16KB() throws IOException { Path tmp = Files.createTempFile(null, null); byte[] randomBytes = new byte[1024]; @@ -1419,34 +1066,6 @@ public synchronized void reset() { } } - private static String crc32(String s) { - return crc32(s.getBytes(StandardCharsets.UTF_8)); - } - - private static String crc32(byte[] bytes) { - CRC32.reset(); - CRC32.update(bytes); - return BinaryUtils.toBase64(CRC32.getChecksumBytes()); - } - - private static String crc32(Path p) throws IOException { - CRC32.reset(); - - byte[] buff = new byte[4096]; - int read; - try (InputStream is = Files.newInputStream(p)) { - while (true) { - read = is.read(buff); - if (read == -1) { - break; - } - CRC32.update(buff, 0, read); - } - } - - return BinaryUtils.toBase64(CRC32.getChecksumBytes()); - } - private void recordObjectToCleanup(BucketType type, String key) { bucketCleanup.computeIfAbsent(type, k -> new ArrayList<>()).add(key); } diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java new file mode 100644 index 000000000000..8c3bc6ca7a16 --- /dev/null +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -0,0 +1,589 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.checksum; + +import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.awssdk.services.s3.checksum.ChecksumIntegrationTesting.testConfigs; +import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccelerateWithArnType; +import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccelerateWithEoz; +import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccelerateWithPathStyle; +import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccessPointWithPathStyle; +import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.crc32; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.ResponsePublisher; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.core.sync.ResponseTransformer; +import software.amazon.awssdk.http.AbortableInputStream; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3control.S3ControlClient; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.testutils.InputStreamUtils; +import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.ToString; + +public class DownloadStreamingIntegrationTesting { + private static final Logger LOG = Logger.loggerFor(DownloadStreamingIntegrationTesting.class); + + private static final String BUCKET_NAME_PREFIX = "do-not-delete-dl-streaming-"; + private static final String MRAP_NAME = "do-not-delete-dl-streaming-testing"; + private static final String AP_NAME = "do-not-delete-dl-streaming-testing-ap"; + private static final String EOZ_SUFFIX = "--usw2-az3--x-s3"; + + private static final Region REGION = Region.US_WEST_2; + private static final String TEST_CREDENTIALS_PROFILE_NAME = "aws-test-account"; + + public static final AwsCredentialsProviderChain CREDENTIALS_PROVIDER_CHAIN = + AwsCredentialsProviderChain.of(ProfileCredentialsProvider.builder() + .profileName(TEST_CREDENTIALS_PROFILE_NAME) + .build(), + DefaultCredentialsProvider.create()); + + static ObjectWithCRC smallObject; + static ObjectWithCRC largeObject; + static ObjectWithCRC largeObjectMulti; + + private static String accountId; + private static String bucketName; + private static String mrapArn; + private static String eozBucket; + private static String apArn; + + private static S3ControlClient s3Control; + private static S3Client s3; + private static StsClient sts; + + private static Path tempDirPath; + + private List pathsToDelete; + + @BeforeAll + static void init() throws Exception { + + s3 = S3Client.builder() + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .region(REGION) + .build(); + + s3Control = S3ControlClient.builder() + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .region(REGION) + .build(); + + sts = StsClient.builder().credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .region(REGION) + .build(); + + accountId = S3ChecksumsTestUtils.getAccountId(sts); + bucketName = S3ChecksumsTestUtils.createBucket(s3, getBucketName(), LOG); + mrapArn = S3ChecksumsTestUtils.createMrap(s3Control, accountId, MRAP_NAME, bucketName, LOG); + eozBucket = S3ChecksumsTestUtils.createEozBucket(s3, getBucketName() + EOZ_SUFFIX, LOG); + apArn = S3ChecksumsTestUtils.createAccessPoint(s3Control, accountId, AP_NAME, bucketName); + + tempDirPath = createTempDir("DownloadStreamingIntegrationTesting"); + + smallObject = uploadObjectSmall(); // 16 KiB + largeObject = uploadObjectLarge(); // 200MiB + largeObjectMulti = uploadMultiPartObject(); // 200 MiB, default multipart config + } + + @AfterAll + static void cleanup() { + + } + + @BeforeEach + void setup() { + pathsToDelete = new ArrayList<>(); + } + + @AfterEach + void methodCleanup() { + pathsToDelete.forEach(p -> { + try { + Files.delete(p); + } catch (Exception e) { + LOG.error(() -> String.format("Unable to delete file %s", p.toString()), e); + } + }); + } + + @ParameterizedTest + @MethodSource("downloadConfigs") + void downloadObject(DownloadConfig config) throws Exception { + assumeNotAccelerateWithPathStyle(config.getBaseConfig()); + assumeNotAccessPointWithPathStyle(config.getBaseConfig()); + assumeNotAccelerateWithArnType(config.getBaseConfig()); + assumeNotAccelerateWithEoz(config.getBaseConfig()); + + LOG.debug(() -> "Running downloadObject with config: " + config); + + String key = config.getContentSize() == ContentSize.SMALL ? smallObject.key() : largeObject.key(); + GetObjectRequest request = GetObjectRequest.builder() + .bucket(bucketForType(config.getBaseConfig().getBucketType())) + .key(key) + .build(); + + byte[] content; + switch (config.getBaseConfig().getFlavor()) { + case JAVA_BASED: { + content = callSyncGetObject(config, request); + break; + } + case ASYNC_JAVA_BASED: + case TM_JAVA: + case ASYNC_CRT: { + content = callAsyncGetObject(request, config); + break; + } + default: + throw new RuntimeException("Unsupported java client flavor: " + config.getBaseConfig().getFlavor()); + } + + String receivedContentCRC32 = S3ChecksumsTestUtils.crc32(content); + String expectedCRC32 = objectForConfig(config).crc32; + assertThat(receivedContentCRC32).isEqualTo(expectedCRC32); + } + + // 16 KiB + static ObjectWithCRC uploadObjectSmall() throws IOException { + String name = String.format("%s-%s.dat", System.currentTimeMillis(), UUID.randomUUID()); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + byte[] rand = new byte[1024]; + for (int i = 0; i < 16; i++) { + new Random().nextBytes(rand); + os.write(rand); + } + byte[] fullContent = os.toByteArray(); + for (BucketType bucketType : BucketType.values()) { + String bucket = bucketForType(bucketType); + PutObjectRequest req = PutObjectRequest.builder() + .bucket(bucket) + .key(name) + .build(); + + s3.putObject(req, RequestBody.fromBytes(fullContent)); + } + return new ObjectWithCRC(name, crc32(fullContent)); + } + + // 80 MiB + static ObjectWithCRC uploadObjectLarge() throws IOException { + String name = String.format("%s-%s.dat", System.currentTimeMillis(), UUID.randomUUID()); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + byte[] rand = new byte[1024 * 1024]; + for (int i = 0; i < 80; i++) { + new Random().nextBytes(rand); + os.write(rand); + } + byte[] fullContent = os.toByteArray(); + for (BucketType bucketType : BucketType.values()) { + String bucket = bucketForType(bucketType); + PutObjectRequest req = PutObjectRequest.builder() + .bucket(bucket) + .key(name) + .build(); + + s3.putObject(req, RequestBody.fromBytes(fullContent)); + } + return new ObjectWithCRC(name, crc32(fullContent)); + } + + // 80MiB, multipart default config + static ObjectWithCRC uploadMultiPartObject() throws Exception { + S3AsyncClient s3AsyncClient = S3AsyncClient.builder() + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .multipartEnabled(true) + .region(REGION) + .build(); + String name = String.format("%s-%s.dat", System.currentTimeMillis(), UUID.randomUUID()); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + byte[] rand = new byte[1024 * 1024]; + for (int i = 0; i < 80; i++) { + new Random().nextBytes(rand); + os.write(rand); + } + byte[] fullContent = os.toByteArray(); + for (BucketType bucketType : BucketType.values()) { + String bucket = bucketForType(bucketType); + PutObjectRequest req = PutObjectRequest.builder() + .bucket(bucket) + .key(name) + .build(); + + s3AsyncClient.putObject(req, AsyncRequestBody.fromBytes(fullContent)) + .get(5, TimeUnit.MINUTES); + } + return new ObjectWithCRC(name, S3ChecksumsTestUtils.crc32(fullContent)); + } + + private static List downloadConfigs() { + List configs = new ArrayList<>(); + for (ResponseTransformerType responseTransformerType : ResponseTransformerType.values()) { + for (TestConfig baseConfig : testConfigs()) { + for (ContentSize contentSize : ContentSize.values()) { + DownloadConfig config = new DownloadConfig(baseConfig, responseTransformerType, contentSize); + configs.add(config); + } + } + } + return configs; + } + + byte[] callSyncGetObject(DownloadConfig config, GetObjectRequest request) throws IOException { + S3Client s3Client = makeSyncClient(config.getBaseConfig()); + byte[] content; + switch (config.getResponseTransformerType()) { + case FILE: { + String filename = request.key(); + Path filePath = Paths.get(tempDirPath.toString(), filename); + pathsToDelete.add(filePath); + s3Client.getObject(request, ResponseTransformer.toFile(filePath)); + content = Files.readAllBytes(filePath); + break; + } + + case BYTES: { + ResponseBytes res = s3Client.getObject(request, ResponseTransformer.toBytes()); + content = res.asByteArray(); + break; + } + + case INPUT_STREAM: { + ResponseInputStream res = s3Client.getObject(request, ResponseTransformer.toInputStream()); + content = InputStreamUtils.drainInputStream(res); + break; + } + + case OUTPUT_STREAM: { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + s3Client.getObject(request, ResponseTransformer.toOutputStream(os)); + content = os.toByteArray(); + break; + } + + case UNMANAGED: { + UnmanagedResponseTransformer tr = new UnmanagedResponseTransformer(); + s3Client.getObject(request, tr); + content = tr.content; + break; + } + + case PUBLISHER: + Assumptions.abort("Skipping 'publisher' transformer type for sync client: " + config); + content = null; + break; + default: + throw new UnsupportedOperationException("unsupported response transformer type: " + config.getResponseTransformerType()); + + } + return content; + } + + byte[] callAsyncGetObject(GetObjectRequest request, DownloadConfig config) throws Exception { + S3AsyncClient s3AsyncClient = makeAsyncClient(config.getBaseConfig()); + byte[] content; + switch (config.getResponseTransformerType()) { + case FILE: { + String filename = randomFileName(); + Path filePath = Paths.get(tempDirPath.toString(), filename); + s3AsyncClient.getObject(request, AsyncResponseTransformer.toFile(filePath)) + .get(5, TimeUnit.MINUTES); + content = Files.readAllBytes(filePath); + break; + } + + case BYTES: { + ResponseBytes res = s3AsyncClient.getObject(request, AsyncResponseTransformer.toBytes()) + .get(5, TimeUnit.MINUTES); + content = res.asByteArray(); + break; + } + + case INPUT_STREAM: { + ResponseInputStream res = s3AsyncClient.getObject(request, + AsyncResponseTransformer.toBlockingInputStream()) + .get(5, TimeUnit.MINUTES); + content = InputStreamUtils.drainInputStream(res); + break; + } + + case PUBLISHER: { + ResponsePublisher res = s3AsyncClient.getObject(request, + AsyncResponseTransformer.toPublisher()) + .get(5, TimeUnit.MINUTES); + ContentConsumer consumer = new ContentConsumer(); + CompletableFuture fut = res.subscribe(consumer); + fut.get(5, TimeUnit.MINUTES); + content = consumer.getFullContent(); + break; + } + + case OUTPUT_STREAM: + case UNMANAGED: + Assumptions.abort(String.format("Skipping '%s' transformer type for async client: %s", + config.getResponseTransformerType(), config)); + content = null; + break; + default: + throw new UnsupportedOperationException("unsupported response transformer type: " + config.getResponseTransformerType()); + } + + return content; + } + + private static String getBucketName() { + return BUCKET_NAME_PREFIX + accountId; + } + + enum ResponseTransformerType { + FILE, + BYTES, + INPUT_STREAM, + OUTPUT_STREAM, + UNMANAGED, + PUBLISHER + } + + private String randomFileName() { + return String.format("%s-%S", System.currentTimeMillis(), UUID.randomUUID()); + } + + static ObjectWithCRC objectForConfig(DownloadConfig config) { + if (config.getContentSize() == ContentSize.SMALL) { + return smallObject; + } + + if (config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) { + return largeObjectMulti; + } + return largeObject; + + } + + static class DownloadConfig { + private TestConfig baseConfig; + private ResponseTransformerType responseTransformerType; + private ContentSize contentSize; + + public DownloadConfig(TestConfig baseConfig, ResponseTransformerType responseTransformerType, + ContentSize contentSize) { + this.baseConfig = baseConfig; + this.responseTransformerType = responseTransformerType; + this.contentSize = contentSize; + } + + public TestConfig getBaseConfig() { + return this.baseConfig; + } + + public void setBaseConfig(TestConfig baseConfig) { + this.baseConfig = baseConfig; + } + + public ResponseTransformerType getResponseTransformerType() { + return responseTransformerType; + } + + public void setResponseTransformerType(ResponseTransformerType responseTransformerType) { + this.responseTransformerType = responseTransformerType; + } + + public ContentSize getContentSize() { + return contentSize; + } + + public void setContentSize(ContentSize contentSize) { + this.contentSize = contentSize; + } + + @Override + public String toString() { + return ToString.builder("DownloadConfig") + .add("baseConfig", baseConfig) + .add("responseTransformerType", responseTransformerType) + .add("contentSize", contentSize) + .build(); + } + } + + private static Path createTempDir(String path) { + try { + return Files.createDirectories(Paths.get(path)); + } catch (Exception e) { + LOG.error(() -> "Unable to create directory", e); + throw new RuntimeException(e); + } + } + + private S3Client makeSyncClient(TestConfig config) { + switch (config.getFlavor()) { + case JAVA_BASED: + return S3Client.builder() + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .accelerate(config.isAccelerateEnabled()) + .build(); + default: + throw new RuntimeException("Unsupported sync flavor: " + config.getFlavor()); + } + } + + private S3AsyncClient makeAsyncClient(TestConfig config) { + switch (config.getFlavor()) { + case ASYNC_JAVA_BASED: + return S3AsyncClient.builder() + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .accelerate(config.isAccelerateEnabled()) + .build(); + case TM_JAVA: + return S3AsyncClient.builder() + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .accelerate(config.isAccelerateEnabled()) + .multipartEnabled(true) + .build(); + case ASYNC_CRT: { + return S3AsyncClient.crtBuilder() + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .accelerate(config.isAccelerateEnabled()) + .build(); + } + default: + throw new RuntimeException("Unsupported async flavor: " + config.getFlavor()); + } + } + + private static String bucketForType(BucketType type) { + switch (type) { + case STANDARD_BUCKET: + return bucketName; + case MRAP: + return mrapArn; + case EOZ: + return eozBucket; + case ACCESS_POINT: + return apArn; + default: + throw new RuntimeException("Unknown bucket type: " + type); + } + } + + enum ContentSize { + SMALL, + LARGE + } + + private static class ObjectWithCRC { + private String key; + private String crc32; + + public ObjectWithCRC(String key, String crc32) { + this.key = key; + this.crc32 = crc32; + } + + public String key() { + return key; + } + + public void key(String key) { + this.key = key; + } + + public String crc32() { + return crc32; + } + + public void crc32(String crc32) { + this.crc32 = crc32; + } + } + + private static class UnmanagedResponseTransformer implements ResponseTransformer { + byte[] content; + + @Override + public byte[] transform(GetObjectResponse response, AbortableInputStream inputStream) throws Exception { + this.content = InputStreamUtils.drainInputStream(inputStream); // stream will be closed + return content; + } + } + + private static class ContentConsumer implements Consumer { + private List buffs = new ArrayList<>(); + + @Override + public void accept(ByteBuffer byteBuffer) { + buffs.add(byteBuffer); + } + + byte[] getFullContent() { + int totalSize = buffs.stream() + .mapToInt(ByteBuffer::remaining) + .sum(); + byte[] result = new byte[totalSize]; + int offset = 0; + for (ByteBuffer buff : buffs) { + int length = buff.remaining(); + buff.get(result, offset, length); + offset += length; + } + return result; + } + } +} diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/S3ChecksumsTestUtils.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/S3ChecksumsTestUtils.java new file mode 100644 index 000000000000..dc437bff07e6 --- /dev/null +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/S3ChecksumsTestUtils.java @@ -0,0 +1,231 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.checksum; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import org.junit.jupiter.api.Assumptions; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm; +import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.BucketAccelerateStatus; +import software.amazon.awssdk.services.s3.model.BucketLocationConstraint; +import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.DataRedundancy; +import software.amazon.awssdk.services.s3.model.LocationInfo; +import software.amazon.awssdk.services.s3.model.LocationType; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3control.S3ControlClient; +import software.amazon.awssdk.services.s3control.model.CreateMultiRegionAccessPointRequest; +import software.amazon.awssdk.services.s3control.model.GetMultiRegionAccessPointResponse; +import software.amazon.awssdk.services.s3control.model.MultiRegionAccessPointStatus; +import software.amazon.awssdk.services.s3control.model.S3ControlException; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.utils.BinaryUtils; +import software.amazon.awssdk.utils.Logger; + +public final class S3ChecksumsTestUtils { + + private static final SdkChecksum CRC32 = SdkChecksum.forAlgorithm(DefaultChecksumAlgorithm.CRC32); + + private S3ChecksumsTestUtils() { + } + + public static String createBucket(S3Client s3, String name, Logger log) { + log.debug(() -> "Creating bucket: " + name); + createBucket(s3, name, 3, log); + s3.putBucketAccelerateConfiguration(r -> r.bucket(name) + .accelerateConfiguration(c -> c.status(BucketAccelerateStatus.ENABLED))); + return name; + } + + public static void createBucket(S3Client s3, String bucketName, int retryCount, Logger log) { + try { + s3.createBucket( + CreateBucketRequest.builder() + .bucket(bucketName) + .createBucketConfiguration( + CreateBucketConfiguration.builder() + .locationConstraint(BucketLocationConstraint.US_WEST_2) + .build()) + .build()); + } catch (S3Exception e) { + log.debug(() -> "Error attempting to create bucket: " + bucketName); + if ("BucketAlreadyOwnedByYou".equals(e.awsErrorDetails().errorCode())) { + log.debug(() -> String.format("%s bucket already exists, likely leaked by a previous run%n", bucketName)); + } else if ("TooManyBuckets".equals(e.awsErrorDetails().errorCode())) { + log.debug(() -> "Printing all buckets for debug:"); + s3.listBuckets().buckets().forEach(l -> log.debug(l::toString)); + if (retryCount < 2) { + log.debug(() -> "Retrying..."); + createBucket(s3, bucketName, retryCount + 1, log); + } else { + throw e; + } + } else { + throw e; + } + } + + s3.waiter().waitUntilBucketExists(r -> r.bucket(bucketName)); + } + + public static String createEozBucket(S3Client s3, String bucketName, Logger log) { + String eozBucketName = bucketName; + log.debug(() -> "Creating EOZ bucket: " + eozBucketName); + CreateBucketConfiguration cfg = + CreateBucketConfiguration.builder() + .bucket(info -> info.dataRedundancy(DataRedundancy.SINGLE_AVAILABILITY_ZONE) + .type(software.amazon.awssdk.services.s3.model.BucketType.DIRECTORY)) + .location(LocationInfo.builder() + .name("usw2-az3") + .type(LocationType.AVAILABILITY_ZONE) + .build()) + .build(); + + try { + s3.createBucket(r -> r.bucket(eozBucketName).createBucketConfiguration(cfg)); + } catch (S3Exception e) { + AwsErrorDetails awsErrorDetails = e.awsErrorDetails(); + if (!"BucketAlreadyOwnedByYou".equals(awsErrorDetails.errorCode())) { + throw e; + } + } + return eozBucketName; + } + + public static String createMrap(S3ControlClient s3Control, String accountId, String mrapName, String bucketName, Logger log) + throws InterruptedException { + try { + s3Control.getMultiRegionAccessPoint(r -> r.accountId(accountId).name(mrapName)); + } catch (S3ControlException e) { + if (e.awsErrorDetails().sdkHttpResponse().statusCode() != 404) { + throw e; + } + + CreateMultiRegionAccessPointRequest createMrap = + CreateMultiRegionAccessPointRequest.builder() + .accountId(accountId) + .details(d -> d.name(mrapName) + .regions(software.amazon.awssdk.services.s3control.model.Region.builder() + .bucket(bucketName) + .build())) + .build(); + + s3Control.createMultiRegionAccessPoint(createMrap); + } + + return waitForMrapToBeReady(s3Control, accountId, mrapName, log); + } + + private static String waitForMrapToBeReady(S3ControlClient s3Control, String accountId, String mrapName, Logger log) + throws InterruptedException { + GetMultiRegionAccessPointResponse getMrapResponse = null; + + Instant waitStart = Instant.now(); + boolean initial = true; + do { + if (!initial) { + Thread.sleep(Duration.ofSeconds(10).toMillis()); + initial = true; + } + GetMultiRegionAccessPointResponse response = s3Control.getMultiRegionAccessPoint(r -> r.accountId(accountId).name(mrapName)); + log.debug(() -> "Wait response: " + response); + getMrapResponse = response; + } while (MultiRegionAccessPointStatus.READY != getMrapResponse.accessPoint().status() + && Duration.between(Instant.now(), waitStart).compareTo(Duration.ofMinutes(5)) < 0); + + return "arn:aws:s3::" + accountId + ":accesspoint/" + getMrapResponse.accessPoint().alias(); + } + + public static String getAccountId(StsClient sts) { + return sts.getCallerIdentity().account(); + } + + public static String createAccessPoint(S3ControlClient s3Control, String accountId, String apName, String bucketName) { + try { + s3Control.getAccessPoint(r -> r.accountId(accountId).name(apName)); + } catch (S3ControlException e) { + if (e.awsErrorDetails().sdkHttpResponse().statusCode() != 404) { + throw e; + } + + s3Control.createAccessPoint(r -> r.bucket(bucketName).name(apName).accountId(accountId)); + } + + // wait for AP to be ready + return s3Control.getAccessPoint(r -> r.accountId(accountId).name(apName)).accessPointArn(); + } + + + public static void assumeNotAccessPointWithPathStyle(TestConfig config) { + BucketType bucketType = config.getBucketType(); + Assumptions.assumeFalse(config.isForcePathStyle() && bucketType.isArnType(), + "Path style doesn't work with ARN type buckets"); + } + + public static void assumeNotAccelerateWithPathStyle(TestConfig config) { + Assumptions.assumeFalse(config.isForcePathStyle() && config.isAccelerateEnabled(), + "Path style doesn't work with Accelerate"); + } + + public static void assumeNotAccelerateWithArnType(TestConfig config) { + Assumptions.assumeFalse(config.isAccelerateEnabled() && config.getBucketType().isArnType(), + "Accelerate doesn't work with ARN buckets"); + } + + public static void assumeNotAccelerateWithEoz(TestConfig config) { + Assumptions.assumeFalse(config.isAccelerateEnabled() && config.getBucketType() == BucketType.EOZ, + "Accelerate is not supported with Express One Zone"); + } + + + public static String crc32(String s) { + return crc32(s.getBytes(StandardCharsets.UTF_8)); + } + + public static String crc32(byte[] bytes) { + CRC32.reset(); + CRC32.update(bytes); + return BinaryUtils.toBase64(CRC32.getChecksumBytes()); + } + + public static String crc32(Path p) throws IOException { + CRC32.reset(); + + byte[] buff = new byte[4096]; + int read; + try (InputStream is = Files.newInputStream(p)) { + while (true) { + read = is.read(buff); + if (read == -1) { + break; + } + CRC32.update(buff, 0, read); + } + } + + return BinaryUtils.toBase64(CRC32.getChecksumBytes()); + } + +} diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/S3ClientFlavor.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/S3ClientFlavor.java new file mode 100644 index 000000000000..0b0174979bdf --- /dev/null +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/S3ClientFlavor.java @@ -0,0 +1,34 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.checksum; + +public enum S3ClientFlavor { + JAVA_BASED(false), + ASYNC_JAVA_BASED(true), + TM_JAVA(true), + ASYNC_CRT(true) + ; + + private final boolean async; + + private S3ClientFlavor(boolean async) { + this.async = async; + } + + public boolean isAsync() { + return async; + } +} diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/TestConfig.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/TestConfig.java new file mode 100644 index 000000000000..1dee2d4f2016 --- /dev/null +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/TestConfig.java @@ -0,0 +1,122 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.checksum; + +import java.util.ArrayList; +import java.util.List; +import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; + +public class TestConfig { + private S3ClientFlavor flavor; + private BucketType bucketType; + private boolean forcePathStyle; + private RequestChecksumCalculation requestChecksumValidation; + private boolean accelerateEnabled; + private boolean payloadSigning; + + public S3ClientFlavor getFlavor() { + return flavor; + } + + public void setFlavor(S3ClientFlavor flavor) { + this.flavor = flavor; + } + + public BucketType getBucketType() { + return bucketType; + } + + public void setBucketType(BucketType bucketType) { + this.bucketType = bucketType; + } + + public boolean isForcePathStyle() { + return forcePathStyle; + } + + public void setForcePathStyle(boolean forcePathStyle) { + this.forcePathStyle = forcePathStyle; + } + + public RequestChecksumCalculation getRequestChecksumValidation() { + return requestChecksumValidation; + } + + public void setRequestChecksumValidation(RequestChecksumCalculation requestChecksumValidation) { + this.requestChecksumValidation = requestChecksumValidation; + } + + public boolean isAccelerateEnabled() { + return accelerateEnabled; + } + + public void setAccelerateEnabled(boolean accelerateEnabled) { + this.accelerateEnabled = accelerateEnabled; + } + + public boolean isPayloadSigning() { + return payloadSigning; + } + + public void setPayloadSigning(boolean payloadSigning) { + this.payloadSigning = payloadSigning; + } + + @Override + public String toString() { + return "[" + + "flavor=" + flavor + + ", bucketType=" + bucketType + + ", forcePathStyle=" + forcePathStyle + + ", requestChecksumValidation=" + requestChecksumValidation + + ", accelerateEnabled=" + accelerateEnabled + + ", payloadSigning=" + payloadSigning + + ']'; + } + + public static List testConfigs() { + List configs = new ArrayList<>(); + + boolean[] forcePathStyle = {true, false}; + RequestChecksumCalculation[] checksumValidations = {RequestChecksumCalculation.WHEN_REQUIRED, + RequestChecksumCalculation.WHEN_SUPPORTED}; + boolean[] accelerateEnabled = {true, false}; + boolean[] payloadSigningEnabled = {true, false}; + for (boolean pathStyle : forcePathStyle) { + for (RequestChecksumCalculation checksumValidation : checksumValidations) { + for (S3ClientFlavor flavor : S3ClientFlavor.values()) { + for (BucketType bucketType : BucketType.values()) { + for (boolean accelerate : accelerateEnabled) { + for (boolean payloadSigning : payloadSigningEnabled) { + TestConfig testConfig = new TestConfig(); + testConfig.setFlavor(flavor); + testConfig.setBucketType(bucketType); + testConfig.setForcePathStyle(pathStyle); + testConfig.setRequestChecksumValidation(checksumValidation); + testConfig.setAccelerateEnabled(accelerate); + testConfig.setPayloadSigning(payloadSigning); + configs.add(testConfig); + } + } + } + } + } + } + + return configs; + } + +} From 14c46decd75be7f6a4a3783e78e79595149447ff Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 31 Mar 2025 11:28:45 -0400 Subject: [PATCH 20/46] logs --- pom.xml | 2 +- .../s3/checksum/ChecksumIntegrationTesting.java | 17 ----------------- test/s3-tests/src/it/resources/log4j2.xml | 2 +- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/pom.xml b/pom.xml index d569ffa24181..16fc2b324813 100644 --- a/pom.xml +++ b/pom.xml @@ -911,7 +911,7 @@ -Xmx8g -Xms1024m -XX:+AllowRedefinitionToAddDeleteMethods - **/s3/checksum/ChecksumIntegrationTesting.java + **/s3/checksum/DownloadStreamingIntegrationTesting.java false diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index 76693eb11e23..f936f35d8562 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -31,8 +31,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Duration; -import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -57,9 +55,6 @@ import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; import software.amazon.awssdk.auth.signer.S3SignerExecutionAttribute; import software.amazon.awssdk.awscore.AwsClient; -import software.amazon.awssdk.awscore.exception.AwsErrorDetails; -import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm; -import software.amazon.awssdk.checksums.SdkChecksum; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.BlockingInputStreamAsyncRequestBody; import software.amazon.awssdk.core.async.BlockingOutputStreamAsyncRequestBody; @@ -74,29 +69,17 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.BucketAccelerateStatus; -import software.amazon.awssdk.services.s3.model.BucketLocationConstraint; -import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration; -import software.amazon.awssdk.services.s3.model.CreateBucketRequest; -import software.amazon.awssdk.services.s3.model.DataRedundancy; import software.amazon.awssdk.services.s3.model.Delete; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; import software.amazon.awssdk.services.s3.model.GlacierJobParameters; -import software.amazon.awssdk.services.s3.model.LocationInfo; -import software.amazon.awssdk.services.s3.model.LocationType; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.RestoreObjectRequest; import software.amazon.awssdk.services.s3.model.RestoreRequest; -import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.model.StorageClass; import software.amazon.awssdk.services.s3.model.Tier; import software.amazon.awssdk.services.s3control.S3ControlClient; -import software.amazon.awssdk.services.s3control.model.CreateMultiRegionAccessPointRequest; -import software.amazon.awssdk.services.s3control.model.GetMultiRegionAccessPointResponse; -import software.amazon.awssdk.services.s3control.model.MultiRegionAccessPointStatus; -import software.amazon.awssdk.services.s3control.model.S3ControlException; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.transfer.s3.S3TransferManager; import software.amazon.awssdk.transfer.s3.model.CompletedUpload; diff --git a/test/s3-tests/src/it/resources/log4j2.xml b/test/s3-tests/src/it/resources/log4j2.xml index 22261c126476..549c9fc988e7 100644 --- a/test/s3-tests/src/it/resources/log4j2.xml +++ b/test/s3-tests/src/it/resources/log4j2.xml @@ -24,7 +24,7 @@ - + From efb4dc8da126c4064cc830a7f7288eca28a820e5 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 31 Mar 2025 11:58:42 -0400 Subject: [PATCH 21/46] fix multi --- .../DownloadStreamingIntegrationTesting.java | 66 ++++++++++++++----- 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index 8c3bc6ca7a16..1bc052fe58db 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -29,7 +29,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Random; import java.util.UUID; @@ -53,13 +55,19 @@ import software.amazon.awssdk.core.async.ResponsePublisher; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.core.sync.ResponseTransformer; +import software.amazon.awssdk.core.waiters.Waiter; import software.amazon.awssdk.http.AbortableInputStream; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; +import software.amazon.awssdk.services.s3.waiters.S3Waiter; import software.amazon.awssdk.services.s3control.S3ControlClient; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.testutils.InputStreamUtils; @@ -184,13 +192,14 @@ void downloadObject(DownloadConfig config) throws Exception { throw new RuntimeException("Unsupported java client flavor: " + config.getBaseConfig().getFlavor()); } - String receivedContentCRC32 = S3ChecksumsTestUtils.crc32(content); + String receivedContentCRC32 = crc32(content); String expectedCRC32 = objectForConfig(config).crc32; assertThat(receivedContentCRC32).isEqualTo(expectedCRC32); } // 16 KiB static ObjectWithCRC uploadObjectSmall() throws IOException { + LOG.debug(() -> "test setup - uploading small test object"); String name = String.format("%s-%s.dat", System.currentTimeMillis(), UUID.randomUUID()); ByteArrayOutputStream os = new ByteArrayOutputStream(); byte[] rand = new byte[1024]; @@ -213,6 +222,7 @@ static ObjectWithCRC uploadObjectSmall() throws IOException { // 80 MiB static ObjectWithCRC uploadObjectLarge() throws IOException { + LOG.debug(() -> "test setup - uploading large test object"); String name = String.format("%s-%s.dat", System.currentTimeMillis(), UUID.randomUUID()); ByteArrayOutputStream os = new ByteArrayOutputStream(); byte[] rand = new byte[1024 * 1024]; @@ -235,30 +245,52 @@ static ObjectWithCRC uploadObjectLarge() throws IOException { // 80MiB, multipart default config static ObjectWithCRC uploadMultiPartObject() throws Exception { - S3AsyncClient s3AsyncClient = S3AsyncClient.builder() - .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - .multipartEnabled(true) - .region(REGION) - .build(); + LOG.debug(() -> "test setup - uploading large test object - multipart"); + String name = String.format("%s-%s.dat", System.currentTimeMillis(), UUID.randomUUID()); ByteArrayOutputStream os = new ByteArrayOutputStream(); - byte[] rand = new byte[1024 * 1024]; - for (int i = 0; i < 80; i++) { + byte[] rand = new byte[8 * 1024 * 1024]; + for (int i = 0; i < 10; i++) { new Random().nextBytes(rand); os.write(rand); } byte[] fullContent = os.toByteArray(); for (BucketType bucketType : BucketType.values()) { - String bucket = bucketForType(bucketType); - PutObjectRequest req = PutObjectRequest.builder() - .bucket(bucket) - .key(name) - .build(); - - s3AsyncClient.putObject(req, AsyncRequestBody.fromBytes(fullContent)) - .get(5, TimeUnit.MINUTES); + doMultipartUpload(bucketType, name, fullContent); } - return new ObjectWithCRC(name, S3ChecksumsTestUtils.crc32(fullContent)); + return new ObjectWithCRC(name, crc32(fullContent)); + } + + static void doMultipartUpload(BucketType bucketType, String objectName, byte[] content) { + String bucket = bucketForType(bucketType); + CreateMultipartUploadRequest createMulti = CreateMultipartUploadRequest.builder() + .bucket(bucket) + .key(objectName) + .build(); + + CreateMultipartUploadResponse res = s3.createMultipartUpload(createMulti); + String uploadId = res.uploadId(); + + List completedParts = new ArrayList<>(); + int partNumber = 10; + int partSize = 8 * 1024 * 1024; + for (int i = 0; i < partNumber; i++) { + final int part = i; + int startIndex = partSize * part; + int endIndex = startIndex + partSize; + byte[] partContent = Arrays.copyOfRange(content, startIndex, endIndex); + UploadPartResponse partResponse = s3.uploadPart(req -> req.partNumber(part).uploadId(uploadId), + RequestBody.fromBytes(partContent)); + completedParts.add(CompletedPart.builder() + .eTag(partResponse.eTag()) + .partNumber(partNumber) + .checksumCRC32(crc32(partContent)) + .build()); + } + + s3.completeMultipartUpload(req -> req.multipartUpload(u -> u.parts(completedParts))); + s3.waiter().waitUntilObjectExists(r -> r.bucket(bucket).key(objectName), + c -> c.waitTimeout(Duration.ofMinutes(5))); } private static List downloadConfigs() { From 51950db9b6cfed81f4090f8891829fbb962a028a Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Thu, 27 Mar 2025 12:54:19 -0400 Subject: [PATCH 22/46] fix multi bucket --- .../s3/checksum/DownloadStreamingIntegrationTesting.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index 1bc052fe58db..81db8db2af12 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -279,7 +279,9 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c int startIndex = partSize * part; int endIndex = startIndex + partSize; byte[] partContent = Arrays.copyOfRange(content, startIndex, endIndex); - UploadPartResponse partResponse = s3.uploadPart(req -> req.partNumber(part).uploadId(uploadId), + UploadPartResponse partResponse = s3.uploadPart(req -> req.partNumber(part) + .uploadId(uploadId) + .bucket(bucket), RequestBody.fromBytes(partContent)); completedParts.add(CompletedPart.builder() .eTag(partResponse.eTag()) From 692fbe0ad97e694e5c549c979ff4e58efc56925e Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 31 Mar 2025 12:29:26 -0400 Subject: [PATCH 23/46] fix multi bucket --- .../s3/checksum/DownloadStreamingIntegrationTesting.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index 81db8db2af12..dbeaba897407 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -281,6 +281,7 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c byte[] partContent = Arrays.copyOfRange(content, startIndex, endIndex); UploadPartResponse partResponse = s3.uploadPart(req -> req.partNumber(part) .uploadId(uploadId) + .key(objectName) .bucket(bucket), RequestBody.fromBytes(partContent)); completedParts.add(CompletedPart.builder() From 3af2ee0b6e3207eed4d0ad745b8e833db78fc8b3 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 31 Mar 2025 12:38:04 -0400 Subject: [PATCH 24/46] part index starts at 1 --- .../s3/checksum/DownloadStreamingIntegrationTesting.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index dbeaba897407..e1ca34390b64 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -279,14 +279,14 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c int startIndex = partSize * part; int endIndex = startIndex + partSize; byte[] partContent = Arrays.copyOfRange(content, startIndex, endIndex); - UploadPartResponse partResponse = s3.uploadPart(req -> req.partNumber(part) + UploadPartResponse partResponse = s3.uploadPart(req -> req.partNumber(part + 1) .uploadId(uploadId) .key(objectName) .bucket(bucket), RequestBody.fromBytes(partContent)); completedParts.add(CompletedPart.builder() .eTag(partResponse.eTag()) - .partNumber(partNumber) + .partNumber(partNumber + 1) .checksumCRC32(crc32(partContent)) .build()); } From 29136b4225a3374497e5bdb35db2797364f86782 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 31 Mar 2025 12:50:18 -0400 Subject: [PATCH 25/46] complete multipart fix --- .../s3/checksum/DownloadStreamingIntegrationTesting.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index e1ca34390b64..23597101e641 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -291,7 +291,10 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c .build()); } - s3.completeMultipartUpload(req -> req.multipartUpload(u -> u.parts(completedParts))); + s3.completeMultipartUpload(req -> req.multipartUpload(u -> u.parts(completedParts)) + .bucket(bucket) + .checksumCRC32(crc32(content)) + .uploadId(uploadId)); s3.waiter().waitUntilObjectExists(r -> r.bucket(bucket).key(objectName), c -> c.waitTimeout(Duration.ofMinutes(5))); } From 994bbc56af6592f0f9c487e8a7efb9a7a0d6621b Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 31 Mar 2025 12:54:40 -0400 Subject: [PATCH 26/46] complete multipart fix --- .../s3/checksum/DownloadStreamingIntegrationTesting.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index 23597101e641..09ed067c262e 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -293,6 +293,7 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c s3.completeMultipartUpload(req -> req.multipartUpload(u -> u.parts(completedParts)) .bucket(bucket) + .key(objectName) .checksumCRC32(crc32(content)) .uploadId(uploadId)); s3.waiter().waitUntilObjectExists(r -> r.bucket(bucket).key(objectName), From 7d2f2fbda913cc7f1c2b60b359670569fcf69a71 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 31 Mar 2025 13:01:36 -0400 Subject: [PATCH 27/46] complete multipart fix --- .../s3/checksum/DownloadStreamingIntegrationTesting.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index 09ed067c262e..c18188617152 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -286,7 +286,7 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c RequestBody.fromBytes(partContent)); completedParts.add(CompletedPart.builder() .eTag(partResponse.eTag()) - .partNumber(partNumber + 1) + .partNumber(part) .checksumCRC32(crc32(partContent)) .build()); } From bf8dd35e80860ec247c7f866bf8df61f7962cf60 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 31 Mar 2025 13:02:29 -0400 Subject: [PATCH 28/46] complete multipart fix --- .../s3/checksum/DownloadStreamingIntegrationTesting.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index c18188617152..1cc9a8f6ebcc 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -286,7 +286,7 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c RequestBody.fromBytes(partContent)); completedParts.add(CompletedPart.builder() .eTag(partResponse.eTag()) - .partNumber(part) + .partNumber(part + 1) .checksumCRC32(crc32(partContent)) .build()); } From 50324d1c8eab5cc87e4949589b5a4c9370c4a4ab Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Tue, 1 Apr 2025 09:48:18 -0400 Subject: [PATCH 29/46] fix for parts --- .../DownloadStreamingIntegrationTesting.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index 1cc9a8f6ebcc..bb8fd90a2216 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -272,21 +272,21 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c String uploadId = res.uploadId(); List completedParts = new ArrayList<>(); - int partNumber = 10; + int partAmount = 10; int partSize = 8 * 1024 * 1024; - for (int i = 0; i < partNumber; i++) { - final int part = i; - int startIndex = partSize * part; + for (int i = 0; i < partAmount; i++) { + final int partNumber = i + 1; + int startIndex = partSize * i; int endIndex = startIndex + partSize; byte[] partContent = Arrays.copyOfRange(content, startIndex, endIndex); - UploadPartResponse partResponse = s3.uploadPart(req -> req.partNumber(part + 1) + UploadPartResponse partResponse = s3.uploadPart(req -> req.partNumber(partNumber) .uploadId(uploadId) .key(objectName) .bucket(bucket), RequestBody.fromBytes(partContent)); completedParts.add(CompletedPart.builder() .eTag(partResponse.eTag()) - .partNumber(part + 1) + .partNumber(partNumber) .checksumCRC32(crc32(partContent)) .build()); } From 2f70826d8806c0987716f6ccac1891462b2aad11 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Tue, 1 Apr 2025 10:06:05 -0400 Subject: [PATCH 30/46] adding logs --- .../s3/checksum/DownloadStreamingIntegrationTesting.java | 5 +++++ test/s3-tests/src/it/resources/log4j2.xml | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index bb8fd90a2216..9636546bce1a 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -263,6 +263,8 @@ static ObjectWithCRC uploadMultiPartObject() throws Exception { static void doMultipartUpload(BucketType bucketType, String objectName, byte[] content) { String bucket = bucketForType(bucketType); + LOG.debug(() -> String.format("Uploading multipart object for bucket type: %s - %s", bucketType, bucket) + ); CreateMultipartUploadRequest createMulti = CreateMultipartUploadRequest.builder() .bucket(bucket) .key(objectName) @@ -279,6 +281,7 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c int startIndex = partSize * i; int endIndex = startIndex + partSize; byte[] partContent = Arrays.copyOfRange(content, startIndex, endIndex); + LOG.debug(() -> "Uploading part: " + partNumber); UploadPartResponse partResponse = s3.uploadPart(req -> req.partNumber(partNumber) .uploadId(uploadId) .key(objectName) @@ -291,6 +294,8 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c .build()); } + LOG.debug(() -> "Finishing MPU, completed parts: " + completedParts); + s3.completeMultipartUpload(req -> req.multipartUpload(u -> u.parts(completedParts)) .bucket(bucket) .key(objectName) diff --git a/test/s3-tests/src/it/resources/log4j2.xml b/test/s3-tests/src/it/resources/log4j2.xml index 549c9fc988e7..2ce221416eef 100644 --- a/test/s3-tests/src/it/resources/log4j2.xml +++ b/test/s3-tests/src/it/resources/log4j2.xml @@ -24,7 +24,8 @@ - + + From cde3548dce52a2fb5927e4ac39048af9edb69ece Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Tue, 1 Apr 2025 10:13:24 -0400 Subject: [PATCH 31/46] more logs, no crc --- .../s3/checksum/DownloadStreamingIntegrationTesting.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index 9636546bce1a..bb5a9c511a4c 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -290,8 +290,8 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c completedParts.add(CompletedPart.builder() .eTag(partResponse.eTag()) .partNumber(partNumber) - .checksumCRC32(crc32(partContent)) .build()); + LOG.debug(() -> String.format("done part %s - etag: %s: ", partNumber, partResponse.eTag())); } LOG.debug(() -> "Finishing MPU, completed parts: " + completedParts); @@ -299,7 +299,6 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c s3.completeMultipartUpload(req -> req.multipartUpload(u -> u.parts(completedParts)) .bucket(bucket) .key(objectName) - .checksumCRC32(crc32(content)) .uploadId(uploadId)); s3.waiter().waitUntilObjectExists(r -> r.bucket(bucket).key(objectName), c -> c.waitTimeout(Duration.ofMinutes(5))); From 59c705b0a00141d22a19c4dd3c48118d4e025406 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Tue, 1 Apr 2025 10:35:21 -0400 Subject: [PATCH 32/46] improved message for failed assertion --- .../s3/checksum/DownloadStreamingIntegrationTesting.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index bb5a9c511a4c..ec16a54de551 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -67,7 +67,6 @@ import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.UploadPartResponse; -import software.amazon.awssdk.services.s3.waiters.S3Waiter; import software.amazon.awssdk.services.s3control.S3ControlClient; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.testutils.InputStreamUtils; @@ -193,8 +192,10 @@ void downloadObject(DownloadConfig config) throws Exception { } String receivedContentCRC32 = crc32(content); - String expectedCRC32 = objectForConfig(config).crc32; - assertThat(receivedContentCRC32).isEqualTo(expectedCRC32); + String expectedCRC32 = objectForConfig(config).crc32(); + assertThat(receivedContentCRC32) + .withFailMessage("Mismatch crc for config " + config) + .isEqualTo(expectedCRC32); } // 16 KiB @@ -374,6 +375,7 @@ byte[] callAsyncGetObject(GetObjectRequest request, DownloadConfig config) throw case FILE: { String filename = randomFileName(); Path filePath = Paths.get(tempDirPath.toString(), filename); + pathsToDelete.add(filePath); s3AsyncClient.getObject(request, AsyncResponseTransformer.toFile(filePath)) .get(5, TimeUnit.MINUTES); content = Files.readAllBytes(filePath); From eba7670ae1f341bf4c1b20c618e64bffd56fb8c2 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Tue, 1 Apr 2025 11:09:58 -0400 Subject: [PATCH 33/46] fix crc check for multi and cleanup --- .../DownloadStreamingIntegrationTesting.java | 42 +++++++++---------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index ec16a54de551..8cd495f48975 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -140,7 +140,12 @@ static void init() throws Exception { @AfterAll static void cleanup() { - + for (BucketType bucketType : BucketType.values()) { + String bucket = bucketForType(bucketType); + s3.deleteObject(req -> req.bucket(bucket).key(smallObject.key())); + s3.deleteObject(req -> req.bucket(bucket).key(largeObject.key())); + s3.deleteObject(req -> req.bucket(bucket).key(largeObjectMulti.key())); + } } @BeforeEach @@ -192,7 +197,7 @@ void downloadObject(DownloadConfig config) throws Exception { } String receivedContentCRC32 = crc32(content); - String expectedCRC32 = objectForConfig(config).crc32(); + String expectedCRC32 = config.getContentSize().s3Object().crc32(); assertThat(receivedContentCRC32) .withFailMessage("Mismatch crc for config " + config) .isEqualTo(expectedCRC32); @@ -438,18 +443,6 @@ private String randomFileName() { return String.format("%s-%S", System.currentTimeMillis(), UUID.randomUUID()); } - static ObjectWithCRC objectForConfig(DownloadConfig config) { - if (config.getContentSize() == ContentSize.SMALL) { - return smallObject; - } - - if (config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) { - return largeObjectMulti; - } - return largeObject; - - } - static class DownloadConfig { private TestConfig baseConfig; private ResponseTransformerType responseTransformerType; @@ -570,7 +563,18 @@ private static String bucketForType(BucketType type) { enum ContentSize { SMALL, - LARGE + LARGE, + LARGE_MULTI; + + ObjectWithCRC s3Object() { + switch (this) { + case SMALL: return smallObject; + case LARGE: return largeObject; + case LARGE_MULTI: return largeObjectMulti; + default: + throw new IllegalArgumentException("Unknown ContentSize " + this); + } + } } private static class ObjectWithCRC { @@ -586,17 +590,9 @@ public String key() { return key; } - public void key(String key) { - this.key = key; - } - public String crc32() { return crc32; } - - public void crc32(String crc32) { - this.crc32 = crc32; - } } private static class UnmanagedResponseTransformer implements ResponseTransformer { From 9ec5867c9e5bf38dab6151a219c42a8d0864dcc3 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Tue, 1 Apr 2025 11:26:06 -0400 Subject: [PATCH 34/46] fix memory issues --- pom.xml | 4 ++-- .../s3/checksum/DownloadStreamingIntegrationTesting.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index a6fe735b0d73..d5e0f3532e6a 100644 --- a/pom.xml +++ b/pom.xml @@ -909,9 +909,9 @@ - -Xmx8g -Xms1024m -XX:+AllowRedefinitionToAddDeleteMethods + -Xmx12g -Xms4g -XX:+AllowRedefinitionToAddDeleteMethods - + **/s3/checksum/ChecksumIntegrationTesting.java **/s3/checksum/DownloadStreamingIntegrationTesting.java false diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index 8cd495f48975..15bba6cc0365 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -370,6 +370,7 @@ byte[] callSyncGetObject(DownloadConfig config, GetObjectRequest request) throws throw new UnsupportedOperationException("unsupported response transformer type: " + config.getResponseTransformerType()); } + s3Client.close(); return content; } @@ -422,7 +423,7 @@ byte[] callAsyncGetObject(GetObjectRequest request, DownloadConfig config) throw default: throw new UnsupportedOperationException("unsupported response transformer type: " + config.getResponseTransformerType()); } - + s3AsyncClient.close(); return content; } From 29c32ed2af03f3e7616d1a551dfc096d8c445673 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Tue, 1 Apr 2025 17:15:24 -0400 Subject: [PATCH 35/46] add s3 checksum validation --- .../DownloadStreamingIntegrationTesting.java | 181 ++++++++++++------ 1 file changed, 120 insertions(+), 61 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index 15bba6cc0365..a78ddaf2b25a 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -50,16 +50,17 @@ import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; -import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.async.ResponsePublisher; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.core.sync.ResponseTransformer; -import software.amazon.awssdk.core.waiters.Waiter; import software.amazon.awssdk.http.AbortableInputStream; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; +import software.amazon.awssdk.services.s3.model.ChecksumMode; +import software.amazon.awssdk.services.s3.model.ChecksumType; import software.amazon.awssdk.services.s3.model.CompletedPart; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; @@ -134,8 +135,8 @@ static void init() throws Exception { tempDirPath = createTempDir("DownloadStreamingIntegrationTesting"); smallObject = uploadObjectSmall(); // 16 KiB - largeObject = uploadObjectLarge(); // 200MiB - largeObjectMulti = uploadMultiPartObject(); // 200 MiB, default multipart config + largeObject = uploadObjectLarge(); // 80 MiB + largeObjectMulti = uploadMultiPartObject(); // 80 MiB, default multipart config } @AfterAll @@ -167,39 +168,49 @@ void methodCleanup() { @ParameterizedTest @MethodSource("downloadConfigs") void downloadObject(DownloadConfig config) throws Exception { - assumeNotAccelerateWithPathStyle(config.getBaseConfig()); - assumeNotAccessPointWithPathStyle(config.getBaseConfig()); - assumeNotAccelerateWithArnType(config.getBaseConfig()); - assumeNotAccelerateWithEoz(config.getBaseConfig()); + assumeNotAccelerateWithPathStyle(config.baseConfig()); + assumeNotAccessPointWithPathStyle(config.baseConfig()); + assumeNotAccelerateWithArnType(config.baseConfig()); + assumeNotAccelerateWithEoz(config.baseConfig()); LOG.debug(() -> "Running downloadObject with config: " + config); - String key = config.getContentSize() == ContentSize.SMALL ? smallObject.key() : largeObject.key(); - GetObjectRequest request = GetObjectRequest.builder() - .bucket(bucketForType(config.getBaseConfig().getBucketType())) - .key(key) - .build(); + String key = config.contentSize().s3Object().key(); + GetObjectRequest.Builder b = GetObjectRequest.builder() + .bucket(bucketForType(config.baseConfig().getBucketType())) + .key(key); + if (config.checksumModeEnabled()) { + b.checksumMode(ChecksumMode.ENABLED); + } - byte[] content; - switch (config.getBaseConfig().getFlavor()) { + GetObjectRequest request = b.build(); + + CallResponse response; + switch (config.baseConfig().getFlavor()) { case JAVA_BASED: { - content = callSyncGetObject(config, request); + response = callSyncGetObject(config, request); break; } case ASYNC_JAVA_BASED: case TM_JAVA: case ASYNC_CRT: { - content = callAsyncGetObject(request, config); + response = callAsyncGetObject(request, config); break; } default: - throw new RuntimeException("Unsupported java client flavor: " + config.getBaseConfig().getFlavor()); + throw new RuntimeException("Unsupported java client flavor: " + config.baseConfig().getFlavor()); } - String receivedContentCRC32 = crc32(content); - String expectedCRC32 = config.getContentSize().s3Object().crc32(); + String receivedContentCRC32 = crc32(response.content()); + if (config.checksumModeEnabled()) { + String s3Crc32 = response.crc32(); + assertThat(receivedContentCRC32) + .withFailMessage("Mismatch with s3 crc32 for config " + config) + .isEqualTo(s3Crc32); + } + String expectedCRC32 = config.contentSize().s3Object().crc32(); assertThat(receivedContentCRC32) - .withFailMessage("Mismatch crc for config " + config) + .withFailMessage("Mismatch with calculated crc32 for config " + config) .isEqualTo(expectedCRC32); } @@ -214,16 +225,19 @@ static ObjectWithCRC uploadObjectSmall() throws IOException { os.write(rand); } byte[] fullContent = os.toByteArray(); + String crc32 = crc32(fullContent); for (BucketType bucketType : BucketType.values()) { String bucket = bucketForType(bucketType); PutObjectRequest req = PutObjectRequest.builder() .bucket(bucket) .key(name) + .checksumAlgorithm(ChecksumAlgorithm.CRC32) + .checksumCRC32(crc32) .build(); s3.putObject(req, RequestBody.fromBytes(fullContent)); } - return new ObjectWithCRC(name, crc32(fullContent)); + return new ObjectWithCRC(name, crc32); } // 80 MiB @@ -237,16 +251,19 @@ static ObjectWithCRC uploadObjectLarge() throws IOException { os.write(rand); } byte[] fullContent = os.toByteArray(); + String crc32 = crc32(fullContent); for (BucketType bucketType : BucketType.values()) { String bucket = bucketForType(bucketType); PutObjectRequest req = PutObjectRequest.builder() .bucket(bucket) + .checksumAlgorithm(ChecksumAlgorithm.CRC32) + .checksumCRC32(crc32) .key(name) .build(); s3.putObject(req, RequestBody.fromBytes(fullContent)); } - return new ObjectWithCRC(name, crc32(fullContent)); + return new ObjectWithCRC(name, crc32); } // 80MiB, multipart default config @@ -261,17 +278,20 @@ static ObjectWithCRC uploadMultiPartObject() throws Exception { os.write(rand); } byte[] fullContent = os.toByteArray(); + String crc32 = crc32(fullContent); for (BucketType bucketType : BucketType.values()) { - doMultipartUpload(bucketType, name, fullContent); + doMultipartUpload(bucketType, name, fullContent, crc32); } - return new ObjectWithCRC(name, crc32(fullContent)); + return new ObjectWithCRC(name, crc32); } - static void doMultipartUpload(BucketType bucketType, String objectName, byte[] content) { + static void doMultipartUpload(BucketType bucketType, String objectName, byte[] content, String fullContentCRC32) { String bucket = bucketForType(bucketType); LOG.debug(() -> String.format("Uploading multipart object for bucket type: %s - %s", bucketType, bucket) ); CreateMultipartUploadRequest createMulti = CreateMultipartUploadRequest.builder() + .checksumAlgorithm(ChecksumAlgorithm.CRC32) + .checksumType(ChecksumType.FULL_OBJECT) .bucket(bucket) .key(objectName) .build(); @@ -303,6 +323,7 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c LOG.debug(() -> "Finishing MPU, completed parts: " + completedParts); s3.completeMultipartUpload(req -> req.multipartUpload(u -> u.parts(completedParts)) + .checksumCRC32(fullContentCRC32) .bucket(bucket) .key(objectName) .uploadId(uploadId)); @@ -315,23 +336,30 @@ private static List downloadConfigs() { for (ResponseTransformerType responseTransformerType : ResponseTransformerType.values()) { for (TestConfig baseConfig : testConfigs()) { for (ContentSize contentSize : ContentSize.values()) { - DownloadConfig config = new DownloadConfig(baseConfig, responseTransformerType, contentSize); - configs.add(config); + DownloadConfig checksumEnabled = + new DownloadConfig(baseConfig, responseTransformerType, contentSize, true); + DownloadConfig checksumDisabled = + new DownloadConfig(baseConfig, responseTransformerType, contentSize, false); + configs.add(checksumEnabled); + configs.add(checksumDisabled); } } } return configs; } - byte[] callSyncGetObject(DownloadConfig config, GetObjectRequest request) throws IOException { - S3Client s3Client = makeSyncClient(config.getBaseConfig()); + CallResponse callSyncGetObject(DownloadConfig config, GetObjectRequest request) throws IOException { + S3Client s3Client = makeSyncClient(config.baseConfig()); + byte[] content; - switch (config.getResponseTransformerType()) { + String s3Crc32 = null; + switch (config.responseTransformerType()) { case FILE: { String filename = request.key(); Path filePath = Paths.get(tempDirPath.toString(), filename); pathsToDelete.add(filePath); - s3Client.getObject(request, ResponseTransformer.toFile(filePath)); + GetObjectResponse res = s3Client.getObject(request, ResponseTransformer.toFile(filePath)); + s3Crc32 = res.checksumCRC32(); content = Files.readAllBytes(filePath); break; } @@ -339,26 +367,30 @@ byte[] callSyncGetObject(DownloadConfig config, GetObjectRequest request) throws case BYTES: { ResponseBytes res = s3Client.getObject(request, ResponseTransformer.toBytes()); content = res.asByteArray(); + s3Crc32 = res.response().checksumCRC32(); break; } case INPUT_STREAM: { ResponseInputStream res = s3Client.getObject(request, ResponseTransformer.toInputStream()); content = InputStreamUtils.drainInputStream(res); + s3Crc32 = res.response().checksumCRC32(); break; } case OUTPUT_STREAM: { ByteArrayOutputStream os = new ByteArrayOutputStream(); - s3Client.getObject(request, ResponseTransformer.toOutputStream(os)); + GetObjectResponse res = s3Client.getObject(request, ResponseTransformer.toOutputStream(os)); content = os.toByteArray(); + s3Crc32 = res.checksumCRC32(); break; } case UNMANAGED: { UnmanagedResponseTransformer tr = new UnmanagedResponseTransformer(); - s3Client.getObject(request, tr); + s3Client.getObject(request, ResponseTransformer.unmanaged(tr)); content = tr.content; + s3Crc32 = tr.response().checksumCRC32(); break; } @@ -366,25 +398,29 @@ byte[] callSyncGetObject(DownloadConfig config, GetObjectRequest request) throws Assumptions.abort("Skipping 'publisher' transformer type for sync client: " + config); content = null; break; + default: - throw new UnsupportedOperationException("unsupported response transformer type: " + config.getResponseTransformerType()); + throw new UnsupportedOperationException("unsupported response transformer type: " + config.responseTransformerType()); } s3Client.close(); - return content; + return new CallResponse(content, s3Crc32); } - byte[] callAsyncGetObject(GetObjectRequest request, DownloadConfig config) throws Exception { - S3AsyncClient s3AsyncClient = makeAsyncClient(config.getBaseConfig()); + CallResponse callAsyncGetObject(GetObjectRequest request, DownloadConfig config) throws Exception { + S3AsyncClient s3AsyncClient = makeAsyncClient(config.baseConfig()); + byte[] content; - switch (config.getResponseTransformerType()) { + String s3crc32 = null; + switch (config.responseTransformerType()) { case FILE: { String filename = randomFileName(); Path filePath = Paths.get(tempDirPath.toString(), filename); pathsToDelete.add(filePath); - s3AsyncClient.getObject(request, AsyncResponseTransformer.toFile(filePath)) - .get(5, TimeUnit.MINUTES); + GetObjectResponse res = s3AsyncClient.getObject(request, AsyncResponseTransformer.toFile(filePath)) + .get(5, TimeUnit.MINUTES); content = Files.readAllBytes(filePath); + s3crc32 = res.checksumCRC32(); break; } @@ -392,6 +428,7 @@ byte[] callAsyncGetObject(GetObjectRequest request, DownloadConfig config) throw ResponseBytes res = s3AsyncClient.getObject(request, AsyncResponseTransformer.toBytes()) .get(5, TimeUnit.MINUTES); content = res.asByteArray(); + s3crc32 = res.response().checksumCRC32(); break; } @@ -400,6 +437,7 @@ byte[] callAsyncGetObject(GetObjectRequest request, DownloadConfig config) throw AsyncResponseTransformer.toBlockingInputStream()) .get(5, TimeUnit.MINUTES); content = InputStreamUtils.drainInputStream(res); + s3crc32 = res.response().checksumCRC32(); break; } @@ -411,20 +449,39 @@ byte[] callAsyncGetObject(GetObjectRequest request, DownloadConfig config) throw CompletableFuture fut = res.subscribe(consumer); fut.get(5, TimeUnit.MINUTES); content = consumer.getFullContent(); + s3crc32 = res.response().checksumCRC32(); break; } case OUTPUT_STREAM: case UNMANAGED: Assumptions.abort(String.format("Skipping '%s' transformer type for async client: %s", - config.getResponseTransformerType(), config)); + config.responseTransformerType(), config)); content = null; break; default: - throw new UnsupportedOperationException("unsupported response transformer type: " + config.getResponseTransformerType()); + throw new UnsupportedOperationException("unsupported response transformer type: " + config.responseTransformerType()); } s3AsyncClient.close(); - return content; + return new CallResponse(content, s3crc32); + } + + private static class CallResponse { + byte[] content; + String crc32; + + public CallResponse(byte[] content, String crc32) { + this.content = content; + this.crc32 = crc32; + } + + public byte[] content() { + return content; + } + + public String crc32() { + return crc32; + } } private static String getBucketName() { @@ -448,36 +505,30 @@ static class DownloadConfig { private TestConfig baseConfig; private ResponseTransformerType responseTransformerType; private ContentSize contentSize; + private boolean checksumModeEnabled; public DownloadConfig(TestConfig baseConfig, ResponseTransformerType responseTransformerType, - ContentSize contentSize) { + ContentSize contentSize, boolean checksumModeEnabled) { this.baseConfig = baseConfig; this.responseTransformerType = responseTransformerType; this.contentSize = contentSize; + this.checksumModeEnabled = checksumModeEnabled; } - public TestConfig getBaseConfig() { + public TestConfig baseConfig() { return this.baseConfig; } - public void setBaseConfig(TestConfig baseConfig) { - this.baseConfig = baseConfig; - } - - public ResponseTransformerType getResponseTransformerType() { + public ResponseTransformerType responseTransformerType() { return responseTransformerType; } - public void setResponseTransformerType(ResponseTransformerType responseTransformerType) { - this.responseTransformerType = responseTransformerType; - } - - public ContentSize getContentSize() { + public ContentSize contentSize() { return contentSize; } - public void setContentSize(ContentSize contentSize) { - this.contentSize = contentSize; + private boolean checksumModeEnabled() { + return this.checksumModeEnabled; } @Override @@ -569,9 +620,12 @@ enum ContentSize { ObjectWithCRC s3Object() { switch (this) { - case SMALL: return smallObject; - case LARGE: return largeObject; - case LARGE_MULTI: return largeObjectMulti; + case SMALL: + return smallObject; + case LARGE: + return largeObject; + case LARGE_MULTI: + return largeObjectMulti; default: throw new IllegalArgumentException("Unknown ContentSize " + this); } @@ -598,12 +652,17 @@ public String crc32() { private static class UnmanagedResponseTransformer implements ResponseTransformer { byte[] content; + GetObjectResponse response; @Override public byte[] transform(GetObjectResponse response, AbortableInputStream inputStream) throws Exception { this.content = InputStreamUtils.drainInputStream(inputStream); // stream will be closed return content; } + + public GetObjectResponse response() { + return this.response; + } } private static class ContentConsumer implements Consumer { From 5c06fe3628a5c57b8d7dfc7331e9cd77778e248b Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Tue, 1 Apr 2025 17:53:18 -0400 Subject: [PATCH 36/46] avoid using AwsChunkedV4aPayloadSigner --- pom.xml | 2 +- .../DownloadStreamingIntegrationTesting.java | 30 +++++++++++-------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/pom.xml b/pom.xml index 75263fa7ad26..b3df8407330d 100644 --- a/pom.xml +++ b/pom.xml @@ -911,7 +911,7 @@ -Xmx12g -Xms4g -XX:+AllowRedefinitionToAddDeleteMethods - **/s3/checksum/ChecksumIntegrationTesting.java + **/s3/checksum/DownloadStreamingIntegrationTesting.java false diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index a78ddaf2b25a..d1bb0254874c 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -48,6 +48,7 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.auth.signer.S3SignerExecutionAttribute; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.async.AsyncResponseTransformer; @@ -228,13 +229,14 @@ static ObjectWithCRC uploadObjectSmall() throws IOException { String crc32 = crc32(fullContent); for (BucketType bucketType : BucketType.values()) { String bucket = bucketForType(bucketType); - PutObjectRequest req = PutObjectRequest.builder() - .bucket(bucket) - .key(name) - .checksumAlgorithm(ChecksumAlgorithm.CRC32) - .checksumCRC32(crc32) - .build(); - + PutObjectRequest req = PutObjectRequest + .builder() + .overrideConfiguration(c -> c.putExecutionAttribute(S3SignerExecutionAttribute.ENABLE_PAYLOAD_SIGNING, false)) + .bucket(bucket) + .key(name) + .checksumAlgorithm(ChecksumAlgorithm.CRC32) + .checksumCRC32(crc32) + .build(); s3.putObject(req, RequestBody.fromBytes(fullContent)); } return new ObjectWithCRC(name, crc32); @@ -254,12 +256,14 @@ static ObjectWithCRC uploadObjectLarge() throws IOException { String crc32 = crc32(fullContent); for (BucketType bucketType : BucketType.values()) { String bucket = bucketForType(bucketType); - PutObjectRequest req = PutObjectRequest.builder() - .bucket(bucket) - .checksumAlgorithm(ChecksumAlgorithm.CRC32) - .checksumCRC32(crc32) - .key(name) - .build(); + PutObjectRequest req = PutObjectRequest + .builder() + .overrideConfiguration(c -> c.putExecutionAttribute(S3SignerExecutionAttribute.ENABLE_PAYLOAD_SIGNING, false)) + .bucket(bucket) + .checksumAlgorithm(ChecksumAlgorithm.CRC32) + .checksumCRC32(crc32) + .key(name) + .build(); s3.putObject(req, RequestBody.fromBytes(fullContent)); } From 92ea909d6aab199ab6c6fd4cfa06000570b846f9 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Wed, 2 Apr 2025 10:08:33 -0400 Subject: [PATCH 37/46] avoid using AwsChunkedV4aPayloadSigner --- .../DownloadStreamingIntegrationTesting.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index d1bb0254874c..08305c8e0108 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -231,11 +231,8 @@ static ObjectWithCRC uploadObjectSmall() throws IOException { String bucket = bucketForType(bucketType); PutObjectRequest req = PutObjectRequest .builder() - .overrideConfiguration(c -> c.putExecutionAttribute(S3SignerExecutionAttribute.ENABLE_PAYLOAD_SIGNING, false)) .bucket(bucket) .key(name) - .checksumAlgorithm(ChecksumAlgorithm.CRC32) - .checksumCRC32(crc32) .build(); s3.putObject(req, RequestBody.fromBytes(fullContent)); } @@ -258,10 +255,7 @@ static ObjectWithCRC uploadObjectLarge() throws IOException { String bucket = bucketForType(bucketType); PutObjectRequest req = PutObjectRequest .builder() - .overrideConfiguration(c -> c.putExecutionAttribute(S3SignerExecutionAttribute.ENABLE_PAYLOAD_SIGNING, false)) .bucket(bucket) - .checksumAlgorithm(ChecksumAlgorithm.CRC32) - .checksumCRC32(crc32) .key(name) .build(); @@ -294,8 +288,8 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c LOG.debug(() -> String.format("Uploading multipart object for bucket type: %s - %s", bucketType, bucket) ); CreateMultipartUploadRequest createMulti = CreateMultipartUploadRequest.builder() - .checksumAlgorithm(ChecksumAlgorithm.CRC32) - .checksumType(ChecksumType.FULL_OBJECT) + // .checksumAlgorithm(ChecksumAlgorithm.CRC32) + // .checksumType(ChecksumType.FULL_OBJECT) .bucket(bucket) .key(objectName) .build(); @@ -327,7 +321,7 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c LOG.debug(() -> "Finishing MPU, completed parts: " + completedParts); s3.completeMultipartUpload(req -> req.multipartUpload(u -> u.parts(completedParts)) - .checksumCRC32(fullContentCRC32) + // .checksumCRC32(fullContentCRC32) .bucket(bucket) .key(objectName) .uploadId(uploadId)); From b0b984e18c82ec5d02111d0fc2d0c704a80a6a6f Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Wed, 2 Apr 2025 10:14:47 -0400 Subject: [PATCH 38/46] DownloadConfig toString --- .../s3/checksum/DownloadStreamingIntegrationTesting.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index 08305c8e0108..bf6a7967271a 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -535,6 +535,7 @@ public String toString() { .add("baseConfig", baseConfig) .add("responseTransformerType", responseTransformerType) .add("contentSize", contentSize) + .add("checksumModeEnabled", checksumModeEnabled) .build(); } } From 10c337e6767e9ba3d0385ed4c7889bd879e8bd99 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Wed, 2 Apr 2025 11:23:08 -0400 Subject: [PATCH 39/46] fix Unmanaged --- .../s3/checksum/DownloadStreamingIntegrationTesting.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index bf6a7967271a..a14e0c7e4179 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -205,6 +205,7 @@ void downloadObject(DownloadConfig config) throws Exception { String receivedContentCRC32 = crc32(response.content()); if (config.checksumModeEnabled()) { String s3Crc32 = response.crc32(); + assertThat(s3Crc32).isNotEmpty(); assertThat(receivedContentCRC32) .withFailMessage("Mismatch with s3 crc32 for config " + config) .isEqualTo(s3Crc32); @@ -655,6 +656,7 @@ private static class UnmanagedResponseTransformer implements ResponseTransformer @Override public byte[] transform(GetObjectResponse response, AbortableInputStream inputStream) throws Exception { + this.response = response; this.content = InputStreamUtils.drainInputStream(inputStream); // stream will be closed return content; } From 78e6b5bf37bef7055bd85e1d4ff4b260c07786f3 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Thu, 3 Apr 2025 10:50:44 -0400 Subject: [PATCH 40/46] fix for crc32 --- .../checksum/ChecksumIntegrationTesting.java | 27 +++++++++++++----- .../DownloadStreamingIntegrationTesting.java | 16 ++++------- .../services/s3/checksum/TestConfig.java | 28 +++++-------------- 3 files changed, 33 insertions(+), 38 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java index f936f35d8562..4b14d4c7e03f 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java @@ -263,7 +263,7 @@ void putObject(UploadConfig config) throws Exception { Assumptions.assumeFalse( (config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED || config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) - && (config.getBaseConfig().isPayloadSigning() + && (config.payloadSigning() // MRAP requires body signing || config.getBaseConfig().getBucketType() == BucketType.MRAP), "Async payload signing doesn't work with Java based clients"); @@ -295,7 +295,7 @@ void putObject(UploadConfig config) throws Exception { ClientOverrideConfiguration.builder() .addExecutionInterceptor(recorder); - if (config.getBaseConfig().isPayloadSigning()) { + if (config.payloadSigning()) { overrideConfiguration.addExecutionInterceptor(new EnablePayloadSigningInterceptor()); } @@ -568,6 +568,15 @@ static class UploadConfig { private TestConfig baseConfig; private BodyType bodyType; private ContentSize contentSize; + private boolean payloadSigning; + + public void setPayloadSigning(boolean payloadSigning) { + this.payloadSigning = payloadSigning; + } + + public boolean payloadSigning() { + return payloadSigning; + } public TestConfig getBaseConfig() { return baseConfig; @@ -746,14 +755,18 @@ private static byte[] largeContent() { private static List uploadConfigs() { List configs = new ArrayList<>(); + boolean[] payloadSigningEnabled = {true, false}; for (BodyType bodyType : BodyType.values()) { for (TestConfig baseConfig : testConfigs()) { for (ContentSize size : ContentSize.values()) { - UploadConfig config = new UploadConfig(); - config.setBaseConfig(baseConfig); - config.setBodyType(bodyType); - config.setContentSize(size); - configs.add(config); + for(boolean payloadSigning : payloadSigningEnabled) { + UploadConfig config = new UploadConfig(); + config.setPayloadSigning(payloadSigning); + config.setBaseConfig(baseConfig); + config.setBodyType(bodyType); + config.setContentSize(size); + configs.add(config); + } } } } diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java index a14e0c7e4179..4ca2e7a72fb6 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java @@ -48,7 +48,6 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; -import software.amazon.awssdk.auth.signer.S3SignerExecutionAttribute; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.async.AsyncResponseTransformer; @@ -59,9 +58,7 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; import software.amazon.awssdk.services.s3.model.ChecksumMode; -import software.amazon.awssdk.services.s3.model.ChecksumType; import software.amazon.awssdk.services.s3.model.CompletedPart; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; @@ -73,6 +70,7 @@ import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.testutils.InputStreamUtils; import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.StringUtils; import software.amazon.awssdk.utils.ToString; public class DownloadStreamingIntegrationTesting { @@ -203,9 +201,8 @@ void downloadObject(DownloadConfig config) throws Exception { } String receivedContentCRC32 = crc32(response.content()); - if (config.checksumModeEnabled()) { - String s3Crc32 = response.crc32(); - assertThat(s3Crc32).isNotEmpty(); + String s3Crc32 = response.crc32(); + if (config.checksumModeEnabled() && StringUtils.isNotBlank(s3Crc32)) { assertThat(receivedContentCRC32) .withFailMessage("Mismatch with s3 crc32 for config " + config) .isEqualTo(s3Crc32); @@ -218,8 +215,8 @@ void downloadObject(DownloadConfig config) throws Exception { // 16 KiB static ObjectWithCRC uploadObjectSmall() throws IOException { - LOG.debug(() -> "test setup - uploading small test object"); String name = String.format("%s-%s.dat", System.currentTimeMillis(), UUID.randomUUID()); + LOG.debug(() -> "test setup - uploading small test object: " + name); ByteArrayOutputStream os = new ByteArrayOutputStream(); byte[] rand = new byte[1024]; for (int i = 0; i < 16; i++) { @@ -242,8 +239,8 @@ static ObjectWithCRC uploadObjectSmall() throws IOException { // 80 MiB static ObjectWithCRC uploadObjectLarge() throws IOException { - LOG.debug(() -> "test setup - uploading large test object"); String name = String.format("%s-%s.dat", System.currentTimeMillis(), UUID.randomUUID()); + LOG.debug(() -> "test setup - uploading large test object: " + name); ByteArrayOutputStream os = new ByteArrayOutputStream(); byte[] rand = new byte[1024 * 1024]; for (int i = 0; i < 80; i++) { @@ -267,9 +264,8 @@ static ObjectWithCRC uploadObjectLarge() throws IOException { // 80MiB, multipart default config static ObjectWithCRC uploadMultiPartObject() throws Exception { - LOG.debug(() -> "test setup - uploading large test object - multipart"); - String name = String.format("%s-%s.dat", System.currentTimeMillis(), UUID.randomUUID()); + LOG.debug(() -> "test setup - uploading large test object - multipart: " + name); ByteArrayOutputStream os = new ByteArrayOutputStream(); byte[] rand = new byte[8 * 1024 * 1024]; for (int i = 0; i < 10; i++) { diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/TestConfig.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/TestConfig.java index 1dee2d4f2016..fa2c22f0d8d6 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/TestConfig.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/TestConfig.java @@ -25,7 +25,6 @@ public class TestConfig { private boolean forcePathStyle; private RequestChecksumCalculation requestChecksumValidation; private boolean accelerateEnabled; - private boolean payloadSigning; public S3ClientFlavor getFlavor() { return flavor; @@ -67,14 +66,6 @@ public void setAccelerateEnabled(boolean accelerateEnabled) { this.accelerateEnabled = accelerateEnabled; } - public boolean isPayloadSigning() { - return payloadSigning; - } - - public void setPayloadSigning(boolean payloadSigning) { - this.payloadSigning = payloadSigning; - } - @Override public String toString() { return "[" + @@ -83,7 +74,6 @@ public String toString() { ", forcePathStyle=" + forcePathStyle + ", requestChecksumValidation=" + requestChecksumValidation + ", accelerateEnabled=" + accelerateEnabled + - ", payloadSigning=" + payloadSigning + ']'; } @@ -94,22 +84,18 @@ public static List testConfigs() { RequestChecksumCalculation[] checksumValidations = {RequestChecksumCalculation.WHEN_REQUIRED, RequestChecksumCalculation.WHEN_SUPPORTED}; boolean[] accelerateEnabled = {true, false}; - boolean[] payloadSigningEnabled = {true, false}; for (boolean pathStyle : forcePathStyle) { for (RequestChecksumCalculation checksumValidation : checksumValidations) { for (S3ClientFlavor flavor : S3ClientFlavor.values()) { for (BucketType bucketType : BucketType.values()) { for (boolean accelerate : accelerateEnabled) { - for (boolean payloadSigning : payloadSigningEnabled) { - TestConfig testConfig = new TestConfig(); - testConfig.setFlavor(flavor); - testConfig.setBucketType(bucketType); - testConfig.setForcePathStyle(pathStyle); - testConfig.setRequestChecksumValidation(checksumValidation); - testConfig.setAccelerateEnabled(accelerate); - testConfig.setPayloadSigning(payloadSigning); - configs.add(testConfig); - } + TestConfig testConfig = new TestConfig(); + testConfig.setFlavor(flavor); + testConfig.setBucketType(bucketType); + testConfig.setForcePathStyle(pathStyle); + testConfig.setRequestChecksumValidation(checksumValidation); + testConfig.setAccelerateEnabled(accelerate); + configs.add(testConfig); } } } From 0084b1f6588b19ba58e3c3b89c0468ae2c9ba084 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Thu, 3 Apr 2025 12:26:03 -0400 Subject: [PATCH 41/46] rename tp regression testing seperate data-place, upload and download test avoid running useless tests cases --- ...sums-tests.yml => s3-regression-tests.yml} | 2 +- pom.xml | 6 +- .../s3/regression/BaseS3RegressionTest.java | 121 +++ .../{checksum => regression}/BucketType.java | 2 +- .../DataplaneOperationRegressionTesting.java | 187 ++++ .../DownloadStreamingRegressionTesting.java} | 93 +- .../S3ChecksumsTestUtils.java | 148 ++- .../S3ClientFlavor.java | 2 +- .../services/s3/regression/TestCallable.java | 37 + .../{checksum => regression}/TestConfig.java | 2 +- .../UploadStreamingRegressionTesting.java} | 895 ++++++------------ test/s3-tests/src/it/resources/log4j2.xml | 4 +- 12 files changed, 790 insertions(+), 709 deletions(-) rename buildspecs/{s3-checksums-tests.yml => s3-regression-tests.yml} (76%) create mode 100644 test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/BaseS3RegressionTest.java rename test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/{checksum => regression}/BucketType.java (94%) create mode 100644 test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DataplaneOperationRegressionTesting.java rename test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/{checksum/DownloadStreamingIntegrationTesting.java => regression/DownloadStreamingRegressionTesting.java} (86%) rename test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/{checksum => regression}/S3ChecksumsTestUtils.java (57%) rename test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/{checksum => regression}/S3ClientFlavor.java (94%) create mode 100644 test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/TestCallable.java rename test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/{checksum => regression}/TestConfig.java (98%) rename test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/{checksum/ChecksumIntegrationTesting.java => regression/UploadStreamingRegressionTesting.java} (67%) diff --git a/buildspecs/s3-checksums-tests.yml b/buildspecs/s3-regression-tests.yml similarity index 76% rename from buildspecs/s3-checksums-tests.yml rename to buildspecs/s3-regression-tests.yml index 56bec6d6e59f..51ec28445c8e 100644 --- a/buildspecs/s3-checksums-tests.yml +++ b/buildspecs/s3-regression-tests.yml @@ -3,7 +3,7 @@ version: 0.2 phases: build: commands: - - mvn clean install -P s3-checksums-tests -pl :s3-tests -am -T1C $MAVEN_OPTIONS + - mvn clean install -P s3-regression-tests -pl :s3-tests -am -T1C $MAVEN_OPTIONS - echo $MAVEN_OPTIONS finally: - mkdir -p codebuild-test-reports diff --git a/pom.xml b/pom.xml index b3df8407330d..2f74faecdf9e 100644 --- a/pom.xml +++ b/pom.xml @@ -862,7 +862,6 @@ **/*IntegrationTests.java **/*IntegTest.java **/RunCucumberTest.java - **/ChecksumIntegrationTesting.java **/SimpleMethodsIntegrationTest.java @@ -878,7 +877,7 @@ - s3-checksums-tests + s3-regression-tests doRelease @@ -911,8 +910,7 @@ -Xmx12g -Xms4g -XX:+AllowRedefinitionToAddDeleteMethods - - **/s3/checksum/DownloadStreamingIntegrationTesting.java + **/*RegressionTesting.java false diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/BaseS3RegressionTest.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/BaseS3RegressionTest.java new file mode 100644 index 000000000000..dceb91781f32 --- /dev/null +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/BaseS3RegressionTest.java @@ -0,0 +1,121 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.regression; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3control.S3ControlClient; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.utils.Logger; + +public abstract class BaseS3RegressionTest { + private static final Logger LOG = Logger.loggerFor(BaseS3RegressionTest.class); + + private static final String BUCKET_NAME_PREFIX = "do-not-delete-checksums-"; + private static final String MRAP_NAME = "do-not-delete-checksum-testing"; + private static final String AP_NAME = "do-not-delete-checksum-testing-ap"; + private static final String EOZ_SUFFIX = "--usw2-az3--x-s3"; + protected static final Region REGION = Region.US_WEST_2; + + protected static final String TEST_CREDENTIALS_PROFILE_NAME = "aws-test-account"; + protected static final AwsCredentialsProviderChain CREDENTIALS_PROVIDER_CHAIN = + AwsCredentialsProviderChain.of(ProfileCredentialsProvider.builder() + .profileName(TEST_CREDENTIALS_PROFILE_NAME) + .build(), + DefaultCredentialsProvider.create()); + + protected static String accountId; + protected static String bucketName; + protected static String mrapArn; + protected static String eozBucket; + protected static String apArn; + + protected static S3ControlClient s3Control; + protected static S3Client s3; + protected static StsClient sts; + + private Map> bucketCleanup = new HashMap<>(); + + @BeforeAll + static void setup() throws InterruptedException, IOException { + // Log.initLoggingToStdout(Log.LogLevel.Trace); + + s3 = S3Client.builder() + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .region(REGION) + .build(); + + s3Control = S3ControlClient.builder() + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .region(REGION) + .build(); + + sts = StsClient.builder().credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .region(REGION) + .build(); + + accountId = S3ChecksumsTestUtils.getAccountId(sts); + bucketName = S3ChecksumsTestUtils.createBucket(s3, getBucketName(), LOG); + mrapArn = S3ChecksumsTestUtils.createMrap(s3Control, accountId, MRAP_NAME, bucketName, LOG); + eozBucket = S3ChecksumsTestUtils.createEozBucket(s3, getBucketName() + EOZ_SUFFIX, LOG); + apArn = S3ChecksumsTestUtils.createAccessPoint(s3Control, accountId, AP_NAME, bucketName); + + } + + @AfterEach + public void methodCleanup() { + bucketCleanup.forEach((bt, keys) -> { + String bucket = bucketForType(bt); + keys.forEach(k -> s3.deleteObject(r -> r.bucket(bucket).key(k))); + }); + + bucketCleanup.clear(); + } + + protected void recordObjectToCleanup(BucketType type, String key) { + bucketCleanup.computeIfAbsent(type, k -> new ArrayList<>()).add(key); + } + + protected static String getBucketName() { + return BUCKET_NAME_PREFIX + accountId; + } + + protected static String bucketForType(BucketType type) { + switch (type) { + case STANDARD_BUCKET: + return bucketName; + case MRAP: + return mrapArn; + case EOZ: + return eozBucket; + case ACCESS_POINT: + return apArn; + default: + throw new RuntimeException("Unknown bucket type: " + type); + } + } + +} diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/BucketType.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/BucketType.java similarity index 94% rename from test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/BucketType.java rename to test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/BucketType.java index 8ffdb5cfb6dc..e7f2ebe36276 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/BucketType.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/BucketType.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.services.s3.checksum; +package software.amazon.awssdk.services.s3.regression; public enum BucketType { STANDARD_BUCKET(false), diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DataplaneOperationRegressionTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DataplaneOperationRegressionTesting.java new file mode 100644 index 000000000000..d2779ea73f60 --- /dev/null +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DataplaneOperationRegressionTesting.java @@ -0,0 +1,187 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.regression; + +import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithArnType; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithEoz; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithPathStyle; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccessPointWithPathStyle; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.crc32; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.makeAsyncClient; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.makeSyncClient; + +import java.util.List; +import java.util.concurrent.Callable; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.awscore.AwsClient; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.Delete; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.GlacierJobParameters; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.RestoreObjectRequest; +import software.amazon.awssdk.services.s3.model.RestoreRequest; +import software.amazon.awssdk.services.s3.model.StorageClass; +import software.amazon.awssdk.services.s3.model.Tier; +import software.amazon.awssdk.utils.CompletableFutureUtils; +import software.amazon.awssdk.utils.Logger; + +public class DataplaneOperationRegressionTesting extends BaseS3RegressionTest { + private static final Logger LOG = Logger.loggerFor(DataplaneOperationRegressionTesting.class); + + // Request checksum required + @ParameterizedTest + @MethodSource("testConfigs") + void deleteObject(TestConfig config) throws Exception { + assumeNotAccessPointWithPathStyle(config); + assumeNotAccelerateWithPathStyle(config); + assumeNotAccelerateWithArnType(config); + assumeNotAccelerateWithEoz(config); + + LOG.debug(() -> "Running deleteObject with config: " + config.toString()); + + String bucket = bucketForType(config.getBucketType()); + String key = putRandomObject(config.getBucketType()); + TestCallable callable = null; + try { + DeleteObjectsRequest req = DeleteObjectsRequest.builder() + .bucket(bucket) + .delete(Delete.builder() + .objects(ObjectIdentifier.builder() + .key(key) + .build()) + .build()) + .build(); + + callable = callDeleteObjects(req, config); + callable.runnable().call(); + } finally { + if (callable != null) { + callable.client().close(); + } + } + } + + // Request checksum optional + @ParameterizedTest + @MethodSource("testConfigs") + void restoreObject(TestConfig config) throws Exception { + assumeNotAccessPointWithPathStyle(config); + assumeNotAccelerateWithPathStyle(config); + assumeNotAccelerateWithArnType(config); + + Assumptions.assumeFalse(config.getBucketType() == BucketType.EOZ, + "Restore is not supported for S3 Express"); + + LOG.debug(() -> "Running restoreObject with config: " + config); + + String bucket = bucketForType(config.getBucketType()); + String key = putRandomArchivedObject(config.getBucketType()); + TestCallable callable = null; + try { + RestoreObjectRequest request = RestoreObjectRequest.builder() + .bucket(bucket) + .key(key) + .restoreRequest(RestoreRequest.builder() + .days(5) + .glacierJobParameters(GlacierJobParameters.builder() + .tier(Tier.STANDARD) + .build()) + .build()) + .build(); + + callable = callRestoreObject(request, config); + callable.runnable().call(); + } finally { + if (callable != null) { + callable.client().close(); + } + } + } + + private TestCallable callDeleteObjects(DeleteObjectsRequest request, TestConfig config) { + AwsClient toClose; + Callable runnable = null; + + if (config.getFlavor().isAsync()) { + S3AsyncClient s3Async = makeAsyncClient(config, REGION, CREDENTIALS_PROVIDER_CHAIN); + toClose = s3Async; + runnable = () -> { + CompletableFutureUtils.joinLikeSync(s3Async.deleteObjects(request)); + return null; + }; + } else { + S3Client s3 = makeSyncClient(config, REGION, CREDENTIALS_PROVIDER_CHAIN); + toClose = s3; + runnable = () -> { + s3.deleteObjects(request); + return null; + }; + } + + return new TestCallable<>(toClose, runnable); + } + + private TestCallable callRestoreObject(RestoreObjectRequest request, TestConfig config) { + AwsClient toClose; + Callable callable = null; + + if (config.getFlavor().isAsync()) { + S3AsyncClient s3Async = makeAsyncClient(config, REGION, CREDENTIALS_PROVIDER_CHAIN); + toClose = s3Async; + callable = () -> { + s3Async.restoreObject(request).join(); + return null; + }; + } else { + S3Client s3 = makeSyncClient(config, REGION, CREDENTIALS_PROVIDER_CHAIN); + toClose = s3; + callable = () -> { + s3.restoreObject(request); + return null; + }; + } + + return new TestCallable<>(toClose, callable); + } + + static List testConfigs() { + return TestConfig.testConfigs(); + } + + private String putRandomObject(BucketType bucketType) { + String key = S3ChecksumsTestUtils.randomKey(); + String bucketName = bucketForType(bucketType); + s3.putObject(r -> r.bucket(bucketName).key(key), RequestBody.fromString("hello")); + recordObjectToCleanup(bucketType, key); + return key; + } + + private String putRandomArchivedObject(BucketType bucketType) { + String key = S3ChecksumsTestUtils.randomKey(); + String bucketName = bucketForType(bucketType); + s3.putObject(r -> r.bucket(bucketName).key(key).storageClass(StorageClass.GLACIER), RequestBody.fromString("hello")); + recordObjectToCleanup(bucketType, key); + return key; + } + + +} diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DownloadStreamingRegressionTesting.java similarity index 86% rename from test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java rename to test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DownloadStreamingRegressionTesting.java index 4ca2e7a72fb6..c16f0447dc63 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/DownloadStreamingIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DownloadStreamingRegressionTesting.java @@ -13,15 +13,15 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.services.s3.checksum; +package software.amazon.awssdk.services.s3.regression; import static org.assertj.core.api.Assertions.assertThat; -import static software.amazon.awssdk.services.s3.checksum.ChecksumIntegrationTesting.testConfigs; -import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccelerateWithArnType; -import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccelerateWithEoz; -import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccelerateWithPathStyle; -import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccessPointWithPathStyle; -import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.crc32; +import static software.amazon.awssdk.services.s3.regression.DataplaneOperationRegressionTesting.testConfigs; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithArnType; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithEoz; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithPathStyle; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccessPointWithPathStyle; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.crc32; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -45,9 +45,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; -import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; -import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.async.AsyncResponseTransformer; @@ -55,7 +52,6 @@ import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.http.AbortableInputStream; -import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.ChecksumMode; @@ -66,73 +62,25 @@ import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.UploadPartResponse; -import software.amazon.awssdk.services.s3control.S3ControlClient; -import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.testutils.InputStreamUtils; import software.amazon.awssdk.utils.Logger; import software.amazon.awssdk.utils.StringUtils; import software.amazon.awssdk.utils.ToString; -public class DownloadStreamingIntegrationTesting { - private static final Logger LOG = Logger.loggerFor(DownloadStreamingIntegrationTesting.class); - - private static final String BUCKET_NAME_PREFIX = "do-not-delete-dl-streaming-"; - private static final String MRAP_NAME = "do-not-delete-dl-streaming-testing"; - private static final String AP_NAME = "do-not-delete-dl-streaming-testing-ap"; - private static final String EOZ_SUFFIX = "--usw2-az3--x-s3"; - - private static final Region REGION = Region.US_WEST_2; - private static final String TEST_CREDENTIALS_PROFILE_NAME = "aws-test-account"; - - public static final AwsCredentialsProviderChain CREDENTIALS_PROVIDER_CHAIN = - AwsCredentialsProviderChain.of(ProfileCredentialsProvider.builder() - .profileName(TEST_CREDENTIALS_PROFILE_NAME) - .build(), - DefaultCredentialsProvider.create()); +public class DownloadStreamingRegressionTesting extends BaseS3RegressionTest { + private static final Logger LOG = Logger.loggerFor(DownloadStreamingRegressionTesting.class); static ObjectWithCRC smallObject; static ObjectWithCRC largeObject; static ObjectWithCRC largeObjectMulti; - private static String accountId; - private static String bucketName; - private static String mrapArn; - private static String eozBucket; - private static String apArn; - - private static S3ControlClient s3Control; - private static S3Client s3; - private static StsClient sts; - private static Path tempDirPath; private List pathsToDelete; @BeforeAll static void init() throws Exception { - - s3 = S3Client.builder() - .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - .region(REGION) - .build(); - - s3Control = S3ControlClient.builder() - .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - .region(REGION) - .build(); - - sts = StsClient.builder().credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - .region(REGION) - .build(); - - accountId = S3ChecksumsTestUtils.getAccountId(sts); - bucketName = S3ChecksumsTestUtils.createBucket(s3, getBucketName(), LOG); - mrapArn = S3ChecksumsTestUtils.createMrap(s3Control, accountId, MRAP_NAME, bucketName, LOG); - eozBucket = S3ChecksumsTestUtils.createEozBucket(s3, getBucketName() + EOZ_SUFFIX, LOG); - apArn = S3ChecksumsTestUtils.createAccessPoint(s3Control, accountId, AP_NAME, bucketName); - tempDirPath = createTempDir("DownloadStreamingIntegrationTesting"); - smallObject = uploadObjectSmall(); // 16 KiB largeObject = uploadObjectLarge(); // 80 MiB largeObjectMulti = uploadMultiPartObject(); // 80 MiB, default multipart config @@ -149,12 +97,12 @@ static void cleanup() { } @BeforeEach - void setup() { + void setupMethod() { pathsToDelete = new ArrayList<>(); } @AfterEach - void methodCleanup() { + void testCleanup() { pathsToDelete.forEach(p -> { try { Files.delete(p); @@ -479,10 +427,6 @@ public String crc32() { } } - private static String getBucketName() { - return BUCKET_NAME_PREFIX + accountId; - } - enum ResponseTransformerType { FILE, BYTES, @@ -594,21 +538,6 @@ private S3AsyncClient makeAsyncClient(TestConfig config) { } } - private static String bucketForType(BucketType type) { - switch (type) { - case STANDARD_BUCKET: - return bucketName; - case MRAP: - return mrapArn; - case EOZ: - return eozBucket; - case ACCESS_POINT: - return apArn; - default: - throw new RuntimeException("Unknown bucket type: " + type); - } - } - enum ContentSize { SMALL, LARGE, diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/S3ChecksumsTestUtils.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ChecksumsTestUtils.java similarity index 57% rename from test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/S3ChecksumsTestUtils.java rename to test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ChecksumsTestUtils.java index dc437bff07e6..920eaa9b00ac 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/S3ChecksumsTestUtils.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ChecksumsTestUtils.java @@ -13,19 +13,26 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.services.s3.checksum; +package software.amazon.awssdk.services.s3.regression; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.time.Instant; +import java.util.Random; +import java.util.UUID; import org.junit.jupiter.api.Assumptions; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.awscore.exception.AwsErrorDetails; import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm; import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.BucketAccelerateStatus; import software.amazon.awssdk.services.s3.model.BucketLocationConstraint; @@ -41,6 +48,7 @@ import software.amazon.awssdk.services.s3control.model.MultiRegionAccessPointStatus; import software.amazon.awssdk.services.s3control.model.S3ControlException; import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.transfer.s3.S3TransferManager; import software.amazon.awssdk.utils.BinaryUtils; import software.amazon.awssdk.utils.Logger; @@ -149,7 +157,8 @@ private static String waitForMrapToBeReady(S3ControlClient s3Control, String acc Thread.sleep(Duration.ofSeconds(10).toMillis()); initial = true; } - GetMultiRegionAccessPointResponse response = s3Control.getMultiRegionAccessPoint(r -> r.accountId(accountId).name(mrapName)); + GetMultiRegionAccessPointResponse response = + s3Control.getMultiRegionAccessPoint(r -> r.accountId(accountId).name(mrapName)); log.debug(() -> "Wait response: " + response); getMrapResponse = response; } while (MultiRegionAccessPointStatus.READY != getMrapResponse.accessPoint().status() @@ -228,4 +237,139 @@ public static String crc32(Path p) throws IOException { return BinaryUtils.toBase64(CRC32.getChecksumBytes()); } + public static S3Client makeSyncClient(TestConfig config, Region region, AwsCredentialsProvider provider) { + switch (config.getFlavor()) { + case JAVA_BASED: + return S3Client.builder() + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(region) + .credentialsProvider(provider) + .accelerate(config.isAccelerateEnabled()) + .build(); + default: + throw new RuntimeException("Unsupported sync flavor: " + config.getFlavor()); + } + } + + public static S3AsyncClient makeAsyncClient(TestConfig config, Region region, AwsCredentialsProvider provider) { + switch (config.getFlavor()) { + case ASYNC_JAVA_BASED: + return S3AsyncClient.builder() + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(region) + .credentialsProvider(provider) + .accelerate(config.isAccelerateEnabled()) + .build(); + case TM_JAVA: + return S3AsyncClient.builder() + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(region) + .credentialsProvider(provider) + .accelerate(config.isAccelerateEnabled()) + .multipartEnabled(true) + .build(); + case ASYNC_CRT: { + return S3AsyncClient.crtBuilder() + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(region) + .credentialsProvider(provider) + .accelerate(config.isAccelerateEnabled()) + .build(); + } + default: + throw new RuntimeException("Unsupported async flavor: " + config.getFlavor()); + } + } + + public static S3Client makeSyncClient(TestConfig config, ClientOverrideConfiguration overrideConfiguration, + Region region, AwsCredentialsProvider provider) { + switch (config.getFlavor()) { + case JAVA_BASED: + return S3Client.builder() + .overrideConfiguration(overrideConfiguration) + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(region) + .credentialsProvider(provider) + .accelerate(config.isAccelerateEnabled()) + .build(); + default: + throw new RuntimeException("Unsupported sync flavor: " + config.getFlavor()); + } + } + + public static S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideConfiguration overrideConfiguration, + Region region, AwsCredentialsProvider provider) { + switch (config.getFlavor()) { + case ASYNC_JAVA_BASED: + return S3AsyncClient.builder() + .overrideConfiguration(overrideConfiguration) + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(region) + .credentialsProvider(provider) + .accelerate(config.isAccelerateEnabled()) + .build(); + case TM_JAVA: + return S3AsyncClient.builder() + .overrideConfiguration(overrideConfiguration) + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(region) + .credentialsProvider(provider) + .accelerate(config.isAccelerateEnabled()) + .multipartEnabled(true) + .build(); + case ASYNC_CRT: { + return S3AsyncClient.crtBuilder() + .forcePathStyle(config.isForcePathStyle()) + .requestChecksumCalculation(config.getRequestChecksumValidation()) + .region(region) + .credentialsProvider(provider) + .accelerate(config.isAccelerateEnabled()) + .build(); + } + default: + throw new RuntimeException("Unsupported async flavor: " + config.getFlavor()); + } + } + + public static S3TransferManager makeTm(TestConfig config, ClientOverrideConfiguration overrideConfiguration, + Region region, AwsCredentialsProvider provider) { + S3AsyncClient s3AsyncClient = makeAsyncClient(config, overrideConfiguration, region, provider); + return S3TransferManager.builder().s3Client(s3AsyncClient).build(); + } + + public static Path createRandomFile16KB() throws IOException { + Path tmp = Files.createTempFile(null, null); + byte[] randomBytes = new byte[1024]; + new Random().nextBytes(randomBytes); + try (OutputStream os = Files.newOutputStream(tmp)) { + for (int i = 0; i < 16; ++i) { + os.write(randomBytes); + } + } + return tmp; + } + + public static Path createRandomFile80MB() throws IOException { + Path tmp = Files.createTempFile(null, null); + byte[] randomBytes = new byte[1024 * 1024]; + new Random().nextBytes(randomBytes); + try (OutputStream os = Files.newOutputStream(tmp)) { + for (int i = 0; i < 80; ++i) { + os.write(randomBytes); + } + } + return tmp; + } + + public static String randomKey() { + return BinaryUtils.toHex(UUID.randomUUID().toString().getBytes()); + } + } diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/S3ClientFlavor.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ClientFlavor.java similarity index 94% rename from test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/S3ClientFlavor.java rename to test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ClientFlavor.java index 0b0174979bdf..2abbc0423b70 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/S3ClientFlavor.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ClientFlavor.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.services.s3.checksum; +package software.amazon.awssdk.services.s3.regression; public enum S3ClientFlavor { JAVA_BASED(false), diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/TestCallable.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/TestCallable.java new file mode 100644 index 000000000000..03e3629e7dfd --- /dev/null +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/TestCallable.java @@ -0,0 +1,37 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.regression; + +import java.util.concurrent.Callable; +import software.amazon.awssdk.utils.SdkAutoCloseable; + +public class TestCallable { + private SdkAutoCloseable client; + private Callable runnable; + + TestCallable(SdkAutoCloseable client, Callable runnable) { + this.client = client; + this.runnable = runnable; + } + + public SdkAutoCloseable client() { + return client; + } + + public Callable runnable() { + return runnable; + } +} diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/TestConfig.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/TestConfig.java similarity index 98% rename from test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/TestConfig.java rename to test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/TestConfig.java index fa2c22f0d8d6..3203faa3213a 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/TestConfig.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/TestConfig.java @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.services.s3.checksum; +package software.amazon.awssdk.services.s3.regression; import java.util.ArrayList; import java.util.List; diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java similarity index 67% rename from test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java rename to test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java index 4b14d4c7e03f..4a2923be401a 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/checksum/ChecksumIntegrationTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java @@ -13,48 +13,42 @@ * permissions and limitations under the License. */ -package software.amazon.awssdk.services.s3.checksum; +package software.amazon.awssdk.services.s3.regression; import static org.assertj.core.api.Assertions.assertThat; -import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccelerateWithArnType; -import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccelerateWithEoz; -import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccelerateWithPathStyle; -import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.assumeNotAccessPointWithPathStyle; -import static software.amazon.awssdk.services.s3.checksum.S3ChecksumsTestUtils.crc32; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithArnType; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithEoz; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithPathStyle; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccessPointWithPathStyle; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.crc32; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.makeAsyncClient; +import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.makeSyncClient; +import static software.amazon.awssdk.services.s3.regression.TestConfig.testConfigs; import io.reactivex.Flowable; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Random; -import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; -import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; -import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; import software.amazon.awssdk.auth.signer.S3SignerExecutionAttribute; -import software.amazon.awssdk.awscore.AwsClient; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.BlockingInputStreamAsyncRequestBody; import software.amazon.awssdk.core.async.BlockingOutputStreamAsyncRequestBody; @@ -66,107 +60,35 @@ import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.http.SdkHttpMethod; import software.amazon.awssdk.http.SdkHttpRequest; -import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.Delete; -import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; -import software.amazon.awssdk.services.s3.model.GlacierJobParameters; -import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; -import software.amazon.awssdk.services.s3.model.RestoreObjectRequest; -import software.amazon.awssdk.services.s3.model.RestoreRequest; -import software.amazon.awssdk.services.s3.model.StorageClass; -import software.amazon.awssdk.services.s3.model.Tier; -import software.amazon.awssdk.services.s3control.S3ControlClient; -import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.transfer.s3.S3TransferManager; import software.amazon.awssdk.transfer.s3.model.CompletedUpload; import software.amazon.awssdk.transfer.s3.model.Upload; -import software.amazon.awssdk.utils.BinaryUtils; import software.amazon.awssdk.utils.CancellableOutputStream; import software.amazon.awssdk.utils.CompletableFutureUtils; import software.amazon.awssdk.utils.FunctionalUtils; import software.amazon.awssdk.utils.Logger; -import software.amazon.awssdk.utils.SdkAutoCloseable; -public class ChecksumIntegrationTesting { - private static final String BUCKET_NAME_PREFIX = "do-not-delete-checksums-"; - private static final String MRAP_NAME = "do-not-delete-checksum-testing"; - private static final String AP_NAME = "do-not-delete-checksum-testing-ap"; - private static final String EOZ_SUFFIX = "--usw2-az3--x-s3"; +public class UploadStreamingRegressionTesting extends BaseS3RegressionTest { + private static final Logger LOG = Logger.loggerFor(UploadStreamingRegressionTesting.class); - private static final Logger LOG = Logger.loggerFor(ChecksumIntegrationTesting.class); - private static final Region REGION = Region.US_WEST_2; - private static final String TEST_CREDENTIALS_PROFILE_NAME = "aws-test-account"; + private static final ExecutorService ASYNC_REQUEST_BODY_EXECUTOR = Executors.newSingleThreadExecutor(); static final byte[] smallContent = "Hello world".getBytes(StandardCharsets.UTF_8); static final byte[] largeContent = largeContent(); - public static final AwsCredentialsProviderChain CREDENTIALS_PROVIDER_CHAIN = - AwsCredentialsProviderChain.of(ProfileCredentialsProvider.builder() - .profileName(TEST_CREDENTIALS_PROFILE_NAME) - .build(), - DefaultCredentialsProvider.create()); - - private static final ExecutorService ASYNC_REQUEST_BODY_EXECUTOR = Executors.newSingleThreadExecutor(); - - private static String accountId; - private static String bucketName; - private static String mrapArn; - private static String eozBucket; - private static String apArn; - - private static S3ControlClient s3Control; - private static S3Client s3; - private static StsClient sts; - private static Path testFileSmall; private static Path testFileLarge; - private Map> bucketCleanup = new HashMap<>(); - @BeforeAll - static void setup() throws InterruptedException, IOException { + static void setupClass() throws InterruptedException, IOException { // Log.initLoggingToStdout(Log.LogLevel.Trace); - s3 = S3Client.builder() - .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - .region(REGION) - .build(); - - s3Control = S3ControlClient.builder() - .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - .region(REGION) - .build(); - - sts = StsClient.builder().credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - .region(REGION) - .build(); - - accountId = S3ChecksumsTestUtils.getAccountId(sts); - - bucketName = S3ChecksumsTestUtils.createBucket(s3, getBucketName(), LOG); - - mrapArn = S3ChecksumsTestUtils.createMrap(s3Control, accountId, MRAP_NAME, bucketName, LOG); - - eozBucket = S3ChecksumsTestUtils.createEozBucket(s3, getBucketName() + EOZ_SUFFIX, LOG); - - apArn = S3ChecksumsTestUtils.createAccessPoint(s3Control, accountId, AP_NAME, bucketName); - - testFileSmall = createRandomFile16KB(); - testFileLarge = createRandomFile80MB(); - } - - @AfterEach - public void methodCleanup() { - bucketCleanup.forEach((bt, keys) -> { - String bucket = bucketForType(bt); - keys.forEach(k -> s3.deleteObject(r -> r.bucket(bucket).key(k))); - }); - - bucketCleanup.clear(); + testFileSmall = S3ChecksumsTestUtils.createRandomFile16KB(); + testFileLarge = S3ChecksumsTestUtils.createRandomFile80MB(); } @AfterAll @@ -174,77 +96,6 @@ public static void cleanup() { ASYNC_REQUEST_BODY_EXECUTOR.shutdownNow(); } - - // Request checksum required - @ParameterizedTest - @MethodSource("testConfigs") - void deleteObject(TestConfig config) throws Exception { - assumeNotAccessPointWithPathStyle(config); - assumeNotAccelerateWithPathStyle(config); - assumeNotAccelerateWithArnType(config); - assumeNotAccelerateWithEoz(config); - - LOG.debug(() -> "Running deleteObject with config: " + config.toString()); - - String bucket = bucketForType(config.getBucketType()); - String key = putRandomObject(config.getBucketType()); - TestCallable callable = null; - try { - DeleteObjectsRequest req = DeleteObjectsRequest.builder() - .bucket(bucket) - .delete(Delete.builder() - .objects(ObjectIdentifier.builder() - .key(key) - .build()) - .build()) - .build(); - - callable = callDeleteObjects(req, config); - callable.runnable.call(); - } finally { - if (callable != null) { - callable.client.close(); - } - } - } - - // Request checksum optional - @ParameterizedTest - @MethodSource("testConfigs") - void restoreObject(TestConfig config) throws Exception { - assumeNotAccessPointWithPathStyle(config); - assumeNotAccelerateWithPathStyle(config); - assumeNotAccelerateWithArnType(config); - - Assumptions.assumeFalse(config.getBucketType() == BucketType.EOZ, - "Restore is not supported for S3 Express"); - - LOG.debug(() -> "Running restoreObject with config: " + config); - - String bucket = bucketForType(config.getBucketType()); - String key = putRandomArchivedObject(config.getBucketType()); - TestCallable callable = null; - try { - RestoreObjectRequest request = RestoreObjectRequest.builder() - .bucket(bucket) - .key(key) - .restoreRequest(RestoreRequest.builder() - .days(5) - .glacierJobParameters(GlacierJobParameters.builder() - .tier(Tier.STANDARD) - .build()) - .build()) - .build(); - - callable = callRestoreObject(request, config); - callable.runnable.call(); - } finally { - if (callable != null) { - callable.client.close(); - } - } - } - @ParameterizedTest @MethodSource("uploadConfigs") void putObject(UploadConfig config) throws Exception { @@ -262,26 +113,26 @@ void putObject(UploadConfig config) throws Exception { // Payload signing doesn't work correctly for async java based Assumptions.assumeFalse( (config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED || - config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) - && (config.payloadSigning() - // MRAP requires body signing - || config.getBaseConfig().getBucketType() == BucketType.MRAP), - "Async payload signing doesn't work with Java based clients"); + config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) + && (config.payloadSigning() + // MRAP requires body signing + || config.getBaseConfig().getBucketType() == BucketType.MRAP), + "Async payload signing doesn't work with Java based clients"); // For testing purposes, ContentProvider is Publisher for async clients // Async java based clients don't currently support unknown content-length bodies Assumptions.assumeFalse( (config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED || - config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) - && config.getBodyType() == BodyType.CONTENT_PROVIDER_NO_LENGTH, - "Async Java based support unknown content length"); + config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) + && config.getBodyType() == BodyType.CONTENT_PROVIDER_NO_LENGTH, + "Async Java based support unknown content length"); LOG.debug(() -> "Running putObject with config: " + config); BucketType bucketType = config.getBaseConfig().getBucketType(); String bucket = bucketForType(bucketType); - String key = randomKey(); + String key = S3ChecksumsTestUtils.randomKey(); PutObjectRequest request = PutObjectRequest.builder() .bucket(bucket) @@ -326,7 +177,7 @@ void putObject(UploadConfig config) throws Exception { actualCrc32 = body.getChecksum(); } - PutObjectResponse response = callable.runnable.call(); + PutObjectResponse response = callable.runnable().call(); recordObjectToCleanup(bucketType, key); @@ -369,424 +220,56 @@ void putObject(UploadConfig config) throws Exception { } finally { if (callable != null) { - callable.client.close(); + callable.client().close(); } } } - - private TestCallable callDeleteObjects(DeleteObjectsRequest request, TestConfig config) { - AwsClient toClose; - Callable runnable = null; - - if (config.getFlavor().isAsync()) { - S3AsyncClient s3Async = makeAsyncClient(config, null); - toClose = s3Async; - runnable = () -> { - CompletableFutureUtils.joinLikeSync(s3Async.deleteObjects(request)); - return null; - }; - } else { - S3Client s3 = makeSyncClient(config, null); - toClose = s3; - runnable = () -> { - s3.deleteObjects(request); - return null; - }; - } - - return new TestCallable<>(toClose, runnable); - } - - private TestCallable callRestoreObject(RestoreObjectRequest request, TestConfig config) { - AwsClient toClose; - Callable callable = null; - - if (config.getFlavor().isAsync()) { - S3AsyncClient s3Async = makeAsyncClient(config, null); - toClose = s3Async; - callable = () -> { - s3Async.restoreObject(request).join(); - return null; - }; - } else { - S3Client s3 = makeSyncClient(config, null); - toClose = s3; - callable = () -> { - s3.restoreObject(request); - return null; - }; - } - - return new TestCallable<>(toClose, callable); - } - private TestCallable callPutObject(PutObjectRequest request, TestRequestBody requestBody, TestConfig config, - ClientOverrideConfiguration overrideConfiguration) { - S3Client s3Client = makeSyncClient(config, overrideConfiguration); + ClientOverrideConfiguration overrideConfiguration) { + S3Client s3Client = makeSyncClient(config, overrideConfiguration, REGION, CREDENTIALS_PROVIDER_CHAIN); Callable callable = () -> { try { return s3Client.putObject(request, requestBody); } catch (Exception e) { throw new RuntimeException(e); - } - }; - return new TestCallable<>(s3Client, callable); - } - - private TestCallable callPutObject(PutObjectRequest request, TestAsyncBody requestBody, TestConfig config, - ClientOverrideConfiguration overrideConfiguration) { - S3AsyncClient s3Client = makeAsyncClient(config, overrideConfiguration); - Callable callable = () -> { - try { - AsyncRequestBody asyncRequestBody = requestBody.getAsyncRequestBody(); - CompletableFuture future = s3Client.putObject(request, asyncRequestBody); - performWriteIfNeeded(requestBody); - return CompletableFutureUtils.joinLikeSync(future); - } catch (Exception e) { - throw new RuntimeException(e); - } - }; - return new TestCallable<>(s3Client, callable); - } - - private TestCallable callTmUpload(PutObjectRequest request, TestAsyncBody requestBody, TestConfig config, - ClientOverrideConfiguration overrideConfiguration) { - S3TransferManager transferManager = makeTm(config, overrideConfiguration); - Callable callable = () -> { - try { - Upload upload = transferManager.upload( - r -> r.requestBody(requestBody.getAsyncRequestBody()).putObjectRequest(request)); - performWriteIfNeeded(requestBody); - CompletedUpload completedUpload = CompletableFutureUtils.joinLikeSync(upload.completionFuture()); - return completedUpload.response(); - } catch (Exception e) { - throw new RuntimeException(e); - } - }; - return new TestCallable<>(transferManager, callable); - } - - void performWriteIfNeeded(TestAsyncBody requestBody) throws IOException { - if (requestBody.bodyType == BodyType.BLOCKING_INPUT_STREAM) { - BlockingInputStreamAsyncRequestBody body = (BlockingInputStreamAsyncRequestBody) requestBody.asyncRequestBody; - InputStream inputStream = ((TestAsyncBodyForBlockingInputStream) requestBody).inputStream; - body.writeInputStream(inputStream); - inputStream.close(); - } - if (requestBody.bodyType == BodyType.BLOCKING_OUTPUT_STREAM) { - TestAsyncBodyForBlockingOutputStream body = (TestAsyncBodyForBlockingOutputStream) requestBody; - CancellableOutputStream outputStream = - ((BlockingOutputStreamAsyncRequestBody) body.getAsyncRequestBody()).outputStream(); - body.bodyWrite.accept(outputStream); - outputStream.close(); - } - } - - private static class TestCallable { - private SdkAutoCloseable client; - private Callable runnable; - - TestCallable(SdkAutoCloseable client, Callable runnable) { - this.client = client; - this.runnable = runnable; - } - } - - private S3Client makeSyncClient(TestConfig config, ClientOverrideConfiguration overrideConfiguration) { - switch (config.getFlavor()) { - case JAVA_BASED: - return S3Client.builder() - .forcePathStyle(config.isForcePathStyle()) - .requestChecksumCalculation(config.getRequestChecksumValidation()) - .region(REGION) - .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - .accelerate(config.isAccelerateEnabled()) - .overrideConfiguration(overrideConfiguration) - .build(); - default: - throw new RuntimeException("Unsupported sync flavor: " + config.getFlavor()); - } - } - - private S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideConfiguration overrideConfiguration) { - switch (config.getFlavor()) { - case ASYNC_JAVA_BASED: - return S3AsyncClient.builder() - .forcePathStyle(config.isForcePathStyle()) - .requestChecksumCalculation(config.getRequestChecksumValidation()) - .region(REGION) - .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - .accelerate(config.isAccelerateEnabled()) - .overrideConfiguration(overrideConfiguration) - .build(); - case TM_JAVA: - return S3AsyncClient.builder() - .forcePathStyle(config.isForcePathStyle()) - .requestChecksumCalculation(config.getRequestChecksumValidation()) - .region(REGION) - .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - .accelerate(config.isAccelerateEnabled()) - .overrideConfiguration(overrideConfiguration) - .multipartEnabled(true) - .build(); - case ASYNC_CRT: { - return S3AsyncClient.crtBuilder() - .forcePathStyle(config.isForcePathStyle()) - .requestChecksumCalculation(config.getRequestChecksumValidation()) - .region(REGION) - .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - .accelerate(config.isAccelerateEnabled()) - .build(); - } - default: - throw new RuntimeException("Unsupported async flavor: " + config.getFlavor()); - } - } - - private S3TransferManager makeTm(TestConfig config, ClientOverrideConfiguration overrideConfiguration) { - S3AsyncClient s3AsyncClient = makeAsyncClient(config, overrideConfiguration); - return S3TransferManager.builder().s3Client(s3AsyncClient).build(); - } - - private static String bucketForType(BucketType type) { - switch (type) { - case STANDARD_BUCKET: - return bucketName; - case MRAP: - return mrapArn; - case EOZ: - return eozBucket; - case ACCESS_POINT: - return apArn; - default: - throw new RuntimeException("Unknown bucket type: " + type); - } - } - - static class UploadConfig { - private TestConfig baseConfig; - private BodyType bodyType; - private ContentSize contentSize; - private boolean payloadSigning; - - public void setPayloadSigning(boolean payloadSigning) { - this.payloadSigning = payloadSigning; - } - - public boolean payloadSigning() { - return payloadSigning; - } - - public TestConfig getBaseConfig() { - return baseConfig; - } - - public void setBaseConfig(TestConfig baseConfig) { - this.baseConfig = baseConfig; - } - - public BodyType getBodyType() { - return bodyType; - } - - public void setBodyType(BodyType bodyType) { - this.bodyType = bodyType; - } - - public void setContentSize(ContentSize contentSize) { - this.contentSize = contentSize; - } - - public ContentSize getContentSize() { - return this.contentSize; - } - - @Override - public String toString() { - return "UploadConfig{" + - "baseConfig=" + baseConfig + - ", bodyType=" + bodyType + - ", contentSize=" + contentSize + - '}'; - } - - } - - static class TestRequestBody extends RequestBody { - private final long contentLength; - private final String checksum; - - protected TestRequestBody(RequestBody wrapped, long contentLength, String checksum) { - super(wrapped.contentStreamProvider(), wrapped.optionalContentLength().orElse(null), wrapped.contentType()); - this.contentLength = contentLength; - this.checksum = checksum; - } - - public long getActualContentLength() { - return contentLength; - } - - public String getChecksum() { - return checksum; - } - } - - private static class TestAsyncBody { - private final AsyncRequestBody asyncRequestBody; - private final long actualContentLength; - private final String checksum; - private final BodyType bodyType; - - private TestAsyncBody(AsyncRequestBody asyncRequestBody, long actualContentLength, String checksum, BodyType bodyType) { - this.asyncRequestBody = asyncRequestBody; - this.actualContentLength = actualContentLength; - this.checksum = checksum; - this.bodyType = bodyType; - } - - public AsyncRequestBody getAsyncRequestBody() { - return asyncRequestBody; - } - - public long getActualContentLength() { - return actualContentLength; - } - - public String getChecksum() { - return checksum; - } - } - - private static class TestAsyncBodyForBlockingOutputStream extends TestAsyncBody { - private final Consumer bodyWrite; - private TestAsyncBodyForBlockingOutputStream(AsyncRequestBody asyncRequestBody, - Consumer bodyWrite, - long actualContentLength, - String checksum, - BodyType bodyType) { - super(asyncRequestBody, actualContentLength, checksum, bodyType); - this.bodyWrite = bodyWrite; - } - } - - private static class TestAsyncBodyForBlockingInputStream extends TestAsyncBody { - private final InputStream inputStream; - private TestAsyncBodyForBlockingInputStream(AsyncRequestBody asyncRequestBody, - InputStream inputStream, - long actualContentLength, - String checksum, - BodyType bodyType) { - super(asyncRequestBody, actualContentLength, checksum, bodyType); - this.inputStream = inputStream; - } - } - - static List testConfigs() { - return TestConfig.testConfigs(); - } - - enum BodyType { - INPUTSTREAM_RESETABLE, - INPUTSTREAM_NOT_RESETABLE, - - STRING, - - FILE, - - CONTENT_PROVIDER_WITH_LENGTH, - - CONTENT_PROVIDER_NO_LENGTH, - - BYTES, - BYTE_BUFFER, - REMAINING_BYTE_BUFFER, - - BYTES_UNSAFE, - BYTE_BUFFER_UNSAFE, - REMAINING_BYTE_BUFFER_UNSAFE, - - BUFFERS, - BUFFERS_REMAINING, - BUFFERS_UNSAFE, - BUFFERS_REMAINING_UNSAFE, - - BLOCKING_INPUT_STREAM, - BLOCKING_OUTPUT_STREAM - } - - enum ContentSize { - SMALL, - LARGE; // 200 MiB - - byte[] byteContent() { - switch (this) { - case SMALL: return smallContent; - case LARGE: return largeContent; - default: throw new IllegalArgumentException("not supported ContentSize " + this); - } - } - - String stringContent() { - switch (this) { - case SMALL: return "Hello World!"; - case LARGE: return new String(largeContent(), StandardCharsets.UTF_8); - default: throw new IllegalArgumentException("not supported ContentSize " + this); - } - } - - Path fileContent() { - switch (this) { - case SMALL: return testFileSmall; - case LARGE: return testFileLarge; - default: throw new IllegalArgumentException("not supported ContentSize " + this); - } - } - } - - private static byte[] largeContent() { - // 80 MiB - Random r = new Random(); - byte[] b = new byte[80 * 1024 * 1024]; - r.nextBytes(b); - return b; - } - - private static List uploadConfigs() { - List configs = new ArrayList<>(); - - boolean[] payloadSigningEnabled = {true, false}; - for (BodyType bodyType : BodyType.values()) { - for (TestConfig baseConfig : testConfigs()) { - for (ContentSize size : ContentSize.values()) { - for(boolean payloadSigning : payloadSigningEnabled) { - UploadConfig config = new UploadConfig(); - config.setPayloadSigning(payloadSigning); - config.setBaseConfig(baseConfig); - config.setBodyType(bodyType); - config.setContentSize(size); - configs.add(config); - } - } - } - } - return configs; + } + }; + return new TestCallable<>(s3Client, callable); } - private String putRandomObject(BucketType bucketType) { - String key = randomKey(); - String bucketName = bucketForType(bucketType); - s3.putObject(r -> r.bucket(bucketName).key(key), RequestBody.fromString("hello")); - recordObjectToCleanup(bucketType, key); - return key; + private TestCallable callPutObject(PutObjectRequest request, TestAsyncBody requestBody, TestConfig config, + ClientOverrideConfiguration overrideConfiguration) { + S3AsyncClient s3Client = makeAsyncClient(config, overrideConfiguration, REGION, CREDENTIALS_PROVIDER_CHAIN); + Callable callable = () -> { + try { + AsyncRequestBody asyncRequestBody = requestBody.getAsyncRequestBody(); + CompletableFuture future = s3Client.putObject(request, asyncRequestBody); + performWriteIfNeeded(requestBody); + return CompletableFutureUtils.joinLikeSync(future); + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + return new TestCallable<>(s3Client, callable); } - private String putRandomArchivedObject(BucketType bucketType) { - String key = randomKey(); - String bucketName = bucketForType(bucketType); - s3.putObject(r -> r.bucket(bucketName).key(key).storageClass(StorageClass.GLACIER), RequestBody.fromString("hello")); - recordObjectToCleanup(bucketType, key); - return key; + private TestCallable callTmUpload(PutObjectRequest request, TestAsyncBody requestBody, TestConfig config, + ClientOverrideConfiguration overrideConfiguration) { + S3TransferManager transferManager = S3ChecksumsTestUtils.makeTm(config, overrideConfiguration, + REGION, CREDENTIALS_PROVIDER_CHAIN); + Callable callable = () -> { + try { + Upload upload = transferManager.upload( + r -> r.requestBody(requestBody.getAsyncRequestBody()).putObjectRequest(request)); + performWriteIfNeeded(requestBody); + CompletedUpload completedUpload = CompletableFutureUtils.joinLikeSync(upload.completionFuture()); + return completedUpload.response(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + return new TestCallable<>(transferManager, callable); } private TestRequestBody getRequestBody(BodyType bodyType, ContentSize contentSize) throws IOException { @@ -809,8 +292,8 @@ private TestRequestBody getRequestBody(BodyType bodyType, ContentSize contentSiz case CONTENT_PROVIDER_WITH_LENGTH: { long contentLength = Files.size(contentSize.fileContent()); RequestBody wrapped = RequestBody.fromContentProvider(() -> FunctionalUtils.invokeSafely(() -> Files.newInputStream(contentSize.fileContent())), - Files.size(contentSize.fileContent()), - "application/octet-stream"); + Files.size(contentSize.fileContent()), + "application/octet-stream"); return new TestRequestBody(wrapped, contentLength, crc32(contentSize.fileContent())); } case INPUTSTREAM_RESETABLE: { @@ -943,7 +426,7 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content byte[] content1 = contentSize.byteContent(); byte[] content2 = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromRemainingByteBuffers(ByteBuffer.wrap(content1), - ByteBuffer.wrap(content2)); + ByteBuffer.wrap(content2)); byte[] crcArray = new byte[content2.length + content2.length]; System.arraycopy(content1, 0, crcArray, 0, content1.length); System.arraycopy(content2, 0, crcArray, content1.length, content2.length); @@ -1010,60 +493,225 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content } } - private String randomKey() { - return BinaryUtils.toHex(UUID.randomUUID().toString().getBytes()); + void performWriteIfNeeded(TestAsyncBody requestBody) throws IOException { + if (requestBody.bodyType == BodyType.BLOCKING_INPUT_STREAM) { + BlockingInputStreamAsyncRequestBody body = (BlockingInputStreamAsyncRequestBody) requestBody.asyncRequestBody; + InputStream inputStream = ((TestAsyncBodyForBlockingInputStream) requestBody).inputStream; + body.writeInputStream(inputStream); + inputStream.close(); + } + if (requestBody.bodyType == BodyType.BLOCKING_OUTPUT_STREAM) { + TestAsyncBodyForBlockingOutputStream body = (TestAsyncBodyForBlockingOutputStream) requestBody; + CancellableOutputStream outputStream = + ((BlockingOutputStreamAsyncRequestBody) body.getAsyncRequestBody()).outputStream(); + body.bodyWrite.accept(outputStream); + outputStream.close(); + } + } + + private static List uploadConfigs() { + List configs = new ArrayList<>(); + + boolean[] payloadSigningEnabled = {true, false}; + for (BodyType bodyType : BodyType.values()) { + for (TestConfig baseConfig : testConfigs()) { + for (ContentSize size : ContentSize.values()) { + for(boolean payloadSigning : payloadSigningEnabled) { + UploadConfig config = new UploadConfig(); + config.setPayloadSigning(payloadSigning); + config.setBaseConfig(baseConfig); + config.setBodyType(bodyType); + config.setContentSize(size); + configs.add(config); + } + } + } + } + return configs; } - private static String getBucketName() { - return BUCKET_NAME_PREFIX + accountId; + static class UploadConfig { + private TestConfig baseConfig; + private BodyType bodyType; + private ContentSize contentSize; + private boolean payloadSigning; + + public void setPayloadSigning(boolean payloadSigning) { + this.payloadSigning = payloadSigning; + } + + public boolean payloadSigning() { + return payloadSigning; + } + + public TestConfig getBaseConfig() { + return baseConfig; + } + + public void setBaseConfig(TestConfig baseConfig) { + this.baseConfig = baseConfig; + } + + public BodyType getBodyType() { + return bodyType; + } + + public void setBodyType(BodyType bodyType) { + this.bodyType = bodyType; + } + + public void setContentSize(ContentSize contentSize) { + this.contentSize = contentSize; + } + + public ContentSize getContentSize() { + return this.contentSize; + } + + @Override + public String toString() { + return "UploadConfig{" + + "baseConfig=" + baseConfig + + ", bodyType=" + bodyType + + ", contentSize=" + contentSize + + '}'; + } + } - private static String waitForApToBeReady() { - return s3Control.getAccessPoint(r -> r.accountId(accountId).name(AP_NAME)).accessPointArn(); + enum BodyType { + INPUTSTREAM_RESETABLE, + INPUTSTREAM_NOT_RESETABLE, + + STRING, + + FILE, + + CONTENT_PROVIDER_WITH_LENGTH, + + CONTENT_PROVIDER_NO_LENGTH, + + BYTES, + BYTE_BUFFER, + REMAINING_BYTE_BUFFER, + + BYTES_UNSAFE, + BYTE_BUFFER_UNSAFE, + REMAINING_BYTE_BUFFER_UNSAFE, + + BUFFERS, + BUFFERS_REMAINING, + BUFFERS_UNSAFE, + BUFFERS_REMAINING_UNSAFE, + + BLOCKING_INPUT_STREAM, + BLOCKING_OUTPUT_STREAM } - private static Path createRandomFile16KB() throws IOException { - Path tmp = Files.createTempFile(null, null); - byte[] randomBytes = new byte[1024]; - new Random().nextBytes(randomBytes); - try (OutputStream os = Files.newOutputStream(tmp)) { - for (int i = 0; i < 16; ++i) { - os.write(randomBytes); + enum ContentSize { + SMALL, + LARGE; // 200 MiB + + byte[] byteContent() { + switch (this) { + case SMALL: return smallContent; + case LARGE: return largeContent; + default: throw new IllegalArgumentException("not supported ContentSize " + this); + } + } + + String stringContent() { + switch (this) { + case SMALL: return "Hello World!"; + case LARGE: return new String(largeContent(), StandardCharsets.UTF_8); + default: throw new IllegalArgumentException("not supported ContentSize " + this); } } - return tmp; - } - private static Path createRandomFile80MB() throws IOException { - Path tmp = Files.createTempFile(null, null); - byte[] randomBytes = new byte[1024 * 1024]; - new Random().nextBytes(randomBytes); - try (OutputStream os = Files.newOutputStream(tmp)) { - for (int i = 0; i < 80; ++i) { - os.write(randomBytes); + Path fileContent() { + switch (this) { + case SMALL: return testFileSmall; + case LARGE: return testFileLarge; + default: throw new IllegalArgumentException("not supported ContentSize " + this); } } - return tmp; } - private static class NonResettableByteStream extends ByteArrayInputStream { - public NonResettableByteStream(byte[] buf) { - super(buf); + private static byte[] largeContent() { + // 80 MiB + Random r = new Random(); + byte[] b = new byte[80 * 1024 * 1024]; + r.nextBytes(b); + return b; + } + + static class TestRequestBody extends RequestBody { + private final long contentLength; + private final String checksum; + + protected TestRequestBody(RequestBody wrapped, long contentLength, String checksum) { + super(wrapped.contentStreamProvider(), wrapped.optionalContentLength().orElse(null), wrapped.contentType()); + this.contentLength = contentLength; + this.checksum = checksum; } - @Override - public boolean markSupported() { - return false; + public long getActualContentLength() { + return contentLength; } - @Override - public synchronized void reset() { - throw new UnsupportedOperationException(); + public String getChecksum() { + return checksum; + } + } + + private static class TestAsyncBody { + private final AsyncRequestBody asyncRequestBody; + private final long actualContentLength; + private final String checksum; + private final BodyType bodyType; + + private TestAsyncBody(AsyncRequestBody asyncRequestBody, long actualContentLength, String checksum, BodyType bodyType) { + this.asyncRequestBody = asyncRequestBody; + this.actualContentLength = actualContentLength; + this.checksum = checksum; + this.bodyType = bodyType; + } + + public AsyncRequestBody getAsyncRequestBody() { + return asyncRequestBody; + } + + public long getActualContentLength() { + return actualContentLength; + } + + public String getChecksum() { + return checksum; + } + } + + private static class TestAsyncBodyForBlockingOutputStream extends TestAsyncBody { + private final Consumer bodyWrite; + private TestAsyncBodyForBlockingOutputStream(AsyncRequestBody asyncRequestBody, + Consumer bodyWrite, + long actualContentLength, + String checksum, + BodyType bodyType) { + super(asyncRequestBody, actualContentLength, checksum, bodyType); + this.bodyWrite = bodyWrite; } } - private void recordObjectToCleanup(BucketType type, String key) { - bucketCleanup.computeIfAbsent(type, k -> new ArrayList<>()).add(key); + private static class TestAsyncBodyForBlockingInputStream extends TestAsyncBody { + private final InputStream inputStream; + private TestAsyncBodyForBlockingInputStream(AsyncRequestBody asyncRequestBody, + InputStream inputStream, + long actualContentLength, + String checksum, + BodyType bodyType) { + super(asyncRequestBody, actualContentLength, checksum, bodyType); + this.inputStream = inputStream; + } } private static class RequestRecorder implements ExecutionInterceptor { @@ -1085,4 +733,21 @@ public void beforeExecution(Context.BeforeExecution context, ExecutionAttributes ExecutionInterceptor.super.beforeExecution(context, executionAttributes); } } + + private static class NonResettableByteStream extends ByteArrayInputStream { + public NonResettableByteStream(byte[] buf) { + super(buf); + } + + @Override + public boolean markSupported() { + return false; + } + + @Override + public synchronized void reset() { + throw new UnsupportedOperationException(); + } + } + } diff --git a/test/s3-tests/src/it/resources/log4j2.xml b/test/s3-tests/src/it/resources/log4j2.xml index 2ce221416eef..777cb0de6394 100644 --- a/test/s3-tests/src/it/resources/log4j2.xml +++ b/test/s3-tests/src/it/resources/log4j2.xml @@ -24,8 +24,8 @@ - - + + From 880c1c86e4670473aa378c3fc39dee1133037143 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Thu, 3 Apr 2025 12:40:27 -0400 Subject: [PATCH 42/46] fix logs --- .../s3/regression/DataplaneOperationRegressionTesting.java | 2 -- test/s3-tests/src/it/resources/log4j2.xml | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DataplaneOperationRegressionTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DataplaneOperationRegressionTesting.java index d2779ea73f60..33ac66dee78d 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DataplaneOperationRegressionTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DataplaneOperationRegressionTesting.java @@ -15,12 +15,10 @@ package software.amazon.awssdk.services.s3.regression; -import static org.assertj.core.api.Assertions.assertThat; import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithArnType; import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithEoz; import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithPathStyle; import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccessPointWithPathStyle; -import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.crc32; import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.makeAsyncClient; import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.makeSyncClient; diff --git a/test/s3-tests/src/it/resources/log4j2.xml b/test/s3-tests/src/it/resources/log4j2.xml index 777cb0de6394..23536e3e7586 100644 --- a/test/s3-tests/src/it/resources/log4j2.xml +++ b/test/s3-tests/src/it/resources/log4j2.xml @@ -24,8 +24,7 @@ - - + From d99d6c1acbcbb39e25c371e604baa80eb42951b4 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Fri, 4 Apr 2025 11:02:46 -0400 Subject: [PATCH 43/46] added README, verify crc32 of uploaded object with an additional get object, precalculated crc32 when possible, rename S3ClientFlavor fields, --- test/s3-tests/README.md | 31 ++++ test/s3-tests/pom.xml | 1 + ...ntrolPlaneOperationRegressionTesting.java} | 4 +- .../DownloadStreamingRegressionTesting.java | 24 ++- .../s3/regression/S3ChecksumsTestUtils.java | 16 +- .../s3/regression/S3ClientFlavor.java | 8 +- .../UploadStreamingRegressionTesting.java | 174 +++++++++++++----- 7 files changed, 181 insertions(+), 77 deletions(-) create mode 100644 test/s3-tests/README.md rename test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/{DataplaneOperationRegressionTesting.java => ControlPlaneOperationRegressionTesting.java} (97%) diff --git a/test/s3-tests/README.md b/test/s3-tests/README.md new file mode 100644 index 000000000000..c8a458f30372 --- /dev/null +++ b/test/s3-tests/README.md @@ -0,0 +1,31 @@ +# SDK Regression Tests for Amazon S3 + +## Description +This module contains SDK regression tests for Amazon S3 streaming operations with various SDK configurations. + + +## How to run + +### Credentials + +The tests require valid AWS credentials to be available in the default credential file under the `aws-test-account` profile. + +### Run the tests + +- Run from your IDE + +- Run from maven command line + +``` +mvn clean install -P s3-regression-tests -pl :stability-tests +``` + +## Adding New Tests + +- The tests are built using [JUnit 5](https://junit.org/junit5/). Make sure you are using the correct APIs and mixing of + Junit 4 and Junit 5 APIs on the same test can have unexpected results. + +- All tests should have the suffix of `RegressionTesting`, eg: `` + + + diff --git a/test/s3-tests/pom.xml b/test/s3-tests/pom.xml index 3b4d8b96f0df..3df269d4e930 100644 --- a/test/s3-tests/pom.xml +++ b/test/s3-tests/pom.xml @@ -167,6 +167,7 @@ software.amazon.awssdk test-utils + test diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DataplaneOperationRegressionTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/ControlPlaneOperationRegressionTesting.java similarity index 97% rename from test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DataplaneOperationRegressionTesting.java rename to test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/ControlPlaneOperationRegressionTesting.java index 33ac66dee78d..13c402322cd3 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DataplaneOperationRegressionTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/ControlPlaneOperationRegressionTesting.java @@ -42,8 +42,8 @@ import software.amazon.awssdk.utils.CompletableFutureUtils; import software.amazon.awssdk.utils.Logger; -public class DataplaneOperationRegressionTesting extends BaseS3RegressionTest { - private static final Logger LOG = Logger.loggerFor(DataplaneOperationRegressionTesting.class); +public class ControlPlaneOperationRegressionTesting extends BaseS3RegressionTest { + private static final Logger LOG = Logger.loggerFor(ControlPlaneOperationRegressionTesting.class); // Request checksum required @ParameterizedTest diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DownloadStreamingRegressionTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DownloadStreamingRegressionTesting.java index c16f0447dc63..f5f73e26af3e 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DownloadStreamingRegressionTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DownloadStreamingRegressionTesting.java @@ -16,7 +16,7 @@ package software.amazon.awssdk.services.s3.regression; import static org.assertj.core.api.Assertions.assertThat; -import static software.amazon.awssdk.services.s3.regression.DataplaneOperationRegressionTesting.testConfigs; +import static software.amazon.awssdk.services.s3.regression.ControlPlaneOperationRegressionTesting.testConfigs; import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithArnType; import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithEoz; import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.assumeNotAccelerateWithPathStyle; @@ -134,13 +134,13 @@ void downloadObject(DownloadConfig config) throws Exception { CallResponse response; switch (config.baseConfig().getFlavor()) { - case JAVA_BASED: { + case STANDARD_SYNC: { response = callSyncGetObject(config, request); break; } - case ASYNC_JAVA_BASED: - case TM_JAVA: - case ASYNC_CRT: { + case STANDARD_ASYNC: + case MULTIPART_ENABLED: + case CRT_BASED: { response = callAsyncGetObject(request, config); break; } @@ -223,18 +223,16 @@ static ObjectWithCRC uploadMultiPartObject() throws Exception { byte[] fullContent = os.toByteArray(); String crc32 = crc32(fullContent); for (BucketType bucketType : BucketType.values()) { - doMultipartUpload(bucketType, name, fullContent, crc32); + doMultipartUpload(bucketType, name, fullContent); } return new ObjectWithCRC(name, crc32); } - static void doMultipartUpload(BucketType bucketType, String objectName, byte[] content, String fullContentCRC32) { + static void doMultipartUpload(BucketType bucketType, String objectName, byte[] content) { String bucket = bucketForType(bucketType); LOG.debug(() -> String.format("Uploading multipart object for bucket type: %s - %s", bucketType, bucket) ); CreateMultipartUploadRequest createMulti = CreateMultipartUploadRequest.builder() - // .checksumAlgorithm(ChecksumAlgorithm.CRC32) - // .checksumType(ChecksumType.FULL_OBJECT) .bucket(bucket) .key(objectName) .build(); @@ -492,7 +490,7 @@ private static Path createTempDir(String path) { private S3Client makeSyncClient(TestConfig config) { switch (config.getFlavor()) { - case JAVA_BASED: + case STANDARD_SYNC: return S3Client.builder() .forcePathStyle(config.isForcePathStyle()) .requestChecksumCalculation(config.getRequestChecksumValidation()) @@ -507,7 +505,7 @@ private S3Client makeSyncClient(TestConfig config) { private S3AsyncClient makeAsyncClient(TestConfig config) { switch (config.getFlavor()) { - case ASYNC_JAVA_BASED: + case STANDARD_ASYNC: return S3AsyncClient.builder() .forcePathStyle(config.isForcePathStyle()) .requestChecksumCalculation(config.getRequestChecksumValidation()) @@ -515,7 +513,7 @@ private S3AsyncClient makeAsyncClient(TestConfig config) { .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) .accelerate(config.isAccelerateEnabled()) .build(); - case TM_JAVA: + case MULTIPART_ENABLED: return S3AsyncClient.builder() .forcePathStyle(config.isForcePathStyle()) .requestChecksumCalculation(config.getRequestChecksumValidation()) @@ -524,7 +522,7 @@ private S3AsyncClient makeAsyncClient(TestConfig config) { .accelerate(config.isAccelerateEnabled()) .multipartEnabled(true) .build(); - case ASYNC_CRT: { + case CRT_BASED: { return S3AsyncClient.crtBuilder() .forcePathStyle(config.isForcePathStyle()) .requestChecksumCalculation(config.getRequestChecksumValidation()) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ChecksumsTestUtils.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ChecksumsTestUtils.java index 920eaa9b00ac..96d0f1c70b26 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ChecksumsTestUtils.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ChecksumsTestUtils.java @@ -239,7 +239,7 @@ public static String crc32(Path p) throws IOException { public static S3Client makeSyncClient(TestConfig config, Region region, AwsCredentialsProvider provider) { switch (config.getFlavor()) { - case JAVA_BASED: + case STANDARD_SYNC: return S3Client.builder() .forcePathStyle(config.isForcePathStyle()) .requestChecksumCalculation(config.getRequestChecksumValidation()) @@ -254,7 +254,7 @@ public static S3Client makeSyncClient(TestConfig config, Region region, AwsCrede public static S3AsyncClient makeAsyncClient(TestConfig config, Region region, AwsCredentialsProvider provider) { switch (config.getFlavor()) { - case ASYNC_JAVA_BASED: + case STANDARD_ASYNC: return S3AsyncClient.builder() .forcePathStyle(config.isForcePathStyle()) .requestChecksumCalculation(config.getRequestChecksumValidation()) @@ -262,7 +262,7 @@ public static S3AsyncClient makeAsyncClient(TestConfig config, Region region, Aw .credentialsProvider(provider) .accelerate(config.isAccelerateEnabled()) .build(); - case TM_JAVA: + case MULTIPART_ENABLED: return S3AsyncClient.builder() .forcePathStyle(config.isForcePathStyle()) .requestChecksumCalculation(config.getRequestChecksumValidation()) @@ -271,7 +271,7 @@ public static S3AsyncClient makeAsyncClient(TestConfig config, Region region, Aw .accelerate(config.isAccelerateEnabled()) .multipartEnabled(true) .build(); - case ASYNC_CRT: { + case CRT_BASED: { return S3AsyncClient.crtBuilder() .forcePathStyle(config.isForcePathStyle()) .requestChecksumCalculation(config.getRequestChecksumValidation()) @@ -288,7 +288,7 @@ public static S3AsyncClient makeAsyncClient(TestConfig config, Region region, Aw public static S3Client makeSyncClient(TestConfig config, ClientOverrideConfiguration overrideConfiguration, Region region, AwsCredentialsProvider provider) { switch (config.getFlavor()) { - case JAVA_BASED: + case STANDARD_SYNC: return S3Client.builder() .overrideConfiguration(overrideConfiguration) .forcePathStyle(config.isForcePathStyle()) @@ -305,7 +305,7 @@ public static S3Client makeSyncClient(TestConfig config, ClientOverrideConfigura public static S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideConfiguration overrideConfiguration, Region region, AwsCredentialsProvider provider) { switch (config.getFlavor()) { - case ASYNC_JAVA_BASED: + case STANDARD_ASYNC: return S3AsyncClient.builder() .overrideConfiguration(overrideConfiguration) .forcePathStyle(config.isForcePathStyle()) @@ -314,7 +314,7 @@ public static S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideCon .credentialsProvider(provider) .accelerate(config.isAccelerateEnabled()) .build(); - case TM_JAVA: + case MULTIPART_ENABLED: return S3AsyncClient.builder() .overrideConfiguration(overrideConfiguration) .forcePathStyle(config.isForcePathStyle()) @@ -324,7 +324,7 @@ public static S3AsyncClient makeAsyncClient(TestConfig config, ClientOverrideCon .accelerate(config.isAccelerateEnabled()) .multipartEnabled(true) .build(); - case ASYNC_CRT: { + case CRT_BASED: { return S3AsyncClient.crtBuilder() .forcePathStyle(config.isForcePathStyle()) .requestChecksumCalculation(config.getRequestChecksumValidation()) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ClientFlavor.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ClientFlavor.java index 2abbc0423b70..54ddcd87e8e5 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ClientFlavor.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/S3ClientFlavor.java @@ -16,10 +16,10 @@ package software.amazon.awssdk.services.s3.regression; public enum S3ClientFlavor { - JAVA_BASED(false), - ASYNC_JAVA_BASED(true), - TM_JAVA(true), - ASYNC_CRT(true) + STANDARD_SYNC(false), + STANDARD_ASYNC(true), + MULTIPART_ENABLED(true), + CRT_BASED(true) ; private final boolean async; diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java index 4a2923be401a..0dca07e783ae 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java @@ -49,6 +49,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import software.amazon.awssdk.auth.signer.S3SignerExecutionAttribute; +import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.BlockingInputStreamAsyncRequestBody; import software.amazon.awssdk.core.async.BlockingOutputStreamAsyncRequestBody; @@ -62,6 +63,8 @@ import software.amazon.awssdk.http.SdkHttpRequest; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ChecksumMode; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.transfer.s3.S3TransferManager; @@ -79,16 +82,31 @@ public class UploadStreamingRegressionTesting extends BaseS3RegressionTest { static final byte[] smallContent = "Hello world".getBytes(StandardCharsets.UTF_8); static final byte[] largeContent = largeContent(); + static final String smallContentCrc32 = crc32(smallContent); + static final String largeContentCrc32 = crc32(largeContent); + + static String smallContentCRC32ForBuffersAPI; + static String largeContentCRC32ForBuffersAPI; private static Path testFileSmall; private static Path testFileLarge; @BeforeAll - static void setupClass() throws InterruptedException, IOException { - // Log.initLoggingToStdout(Log.LogLevel.Trace); - + static void setupClass() throws IOException { testFileSmall = S3ChecksumsTestUtils.createRandomFile16KB(); testFileLarge = S3ChecksumsTestUtils.createRandomFile80MB(); + + // used in RequestBody.*buffers(...) API + // we calculate crc32 once to try to accelerate test execution + byte[] crcArraySmallContentForBuffersApi = new byte[smallContent.length + smallContent.length]; + System.arraycopy(smallContent, 0, crcArraySmallContentForBuffersApi, 0, smallContent.length); + System.arraycopy(smallContent, 0, crcArraySmallContentForBuffersApi, smallContent.length, smallContent.length); + smallContentCRC32ForBuffersAPI = crc32(crcArraySmallContentForBuffersApi); + + byte[] crcArrayLargeContentForBuffersApi = new byte[largeContent.length + largeContent.length]; + System.arraycopy(largeContent, 0, crcArrayLargeContentForBuffersApi, 0, largeContent.length); + System.arraycopy(largeContent, 0, crcArrayLargeContentForBuffersApi, largeContent.length, largeContent.length); + largeContentCRC32ForBuffersAPI = crc32(largeContentCRC32ForBuffersAPI); } @AfterAll @@ -111,9 +129,10 @@ void putObject(UploadConfig config) throws Exception { "No way to create AsyncRequestBody by giving both an Publisher and the content length"); // Payload signing doesn't work correctly for async java based + // TODO(sra-identity-auth) remove when chunked encoding support is added in async code path Assumptions.assumeFalse( - (config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED || - config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) + (config.getBaseConfig().getFlavor() == S3ClientFlavor.STANDARD_ASYNC || + config.getBaseConfig().getFlavor() == S3ClientFlavor.MULTIPART_ENABLED) && (config.payloadSigning() // MRAP requires body signing || config.getBaseConfig().getBucketType() == BucketType.MRAP), @@ -122,8 +141,8 @@ void putObject(UploadConfig config) throws Exception { // For testing purposes, ContentProvider is Publisher for async clients // Async java based clients don't currently support unknown content-length bodies Assumptions.assumeFalse( - (config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_JAVA_BASED || - config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) + (config.getBaseConfig().getFlavor() == S3ClientFlavor.STANDARD_ASYNC || + config.getBaseConfig().getFlavor() == S3ClientFlavor.MULTIPART_ENABLED) && config.getBodyType() == BodyType.CONTENT_PROVIDER_NO_LENGTH, "Async Java based support unknown content length"); @@ -163,7 +182,7 @@ void putObject(UploadConfig config) throws Exception { actualContentLength = body.getActualContentLength(); requestBodyHasContentLength = body.optionalContentLength().isPresent(); actualCrc32 = body.getChecksum(); - } else if (config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) { + } else if (config.getBaseConfig().getFlavor() == S3ClientFlavor.MULTIPART_ENABLED) { TestAsyncBody body = getAsyncRequestBody(config.getBodyType(), config.contentSize); callable = callTmUpload(request, body, config.getBaseConfig(), overrideConfiguration.build()); actualContentLength = body.getActualContentLength(); @@ -182,19 +201,19 @@ void putObject(UploadConfig config) throws Exception { recordObjectToCleanup(bucketType, key); // mpu not supported - if (config.getBaseConfig().getFlavor() == S3ClientFlavor.TM_JAVA) { + if (config.getBaseConfig().getFlavor() == S3ClientFlavor.MULTIPART_ENABLED) { return; } // We only validate when configured to WHEN_SUPPORTED since checksums are optional for PutObject if (config.getBaseConfig().getRequestChecksumValidation() == RequestChecksumCalculation.WHEN_SUPPORTED // CRT switches to MPU under the hood which doesn't support checksums - && config.getBaseConfig().getFlavor() != S3ClientFlavor.ASYNC_CRT) { + && config.getBaseConfig().getFlavor() != S3ClientFlavor.CRT_BASED) { assertThat(response.checksumCRC32()).isEqualTo(actualCrc32); } // We can't set an execution interceptor when using CRT - if (config.getBaseConfig().getFlavor() == S3ClientFlavor.ASYNC_CRT) { + if (config.getBaseConfig().getFlavor() == S3ClientFlavor.CRT_BASED) { return; } @@ -210,6 +229,7 @@ void putObject(UploadConfig config) throws Exception { if (payloadSha.startsWith("STREAMING")) { String decodedContentLength = httpRequest.firstMatchingHeader("x-amz-decoded-content-length").get(); assertThat(Long.parseLong(decodedContentLength)).isEqualTo(actualContentLength); + verifyChecksumResponsePayload(config, key, actualCrc32); } else { Optional contentLength = httpRequest.firstMatchingHeader("Content-Length"); if (requestBodyHasContentLength) { @@ -225,7 +245,17 @@ void putObject(UploadConfig config) throws Exception { } } - private TestCallable callPutObject(PutObjectRequest request, TestRequestBody requestBody, TestConfig config, + private void verifyChecksumResponsePayload(UploadConfig config, String key, String expectedCRC32) { + String bucket = bucketForType(config.getBaseConfig().getBucketType()); + ResponseInputStream response = s3.getObject(req -> req.checksumMode(ChecksumMode.ENABLED) + .key(key) + .bucket(bucket)); + assertThat(response.response().checksumCRC32()).isEqualTo(expectedCRC32); + + } + + private TestCallable callPutObject(PutObjectRequest request, TestRequestBody requestBody, + TestConfig config, ClientOverrideConfiguration overrideConfiguration) { S3Client s3Client = makeSyncClient(config, overrideConfiguration, REGION, CREDENTIALS_PROVIDER_CHAIN); Callable callable = () -> { @@ -281,7 +311,8 @@ private TestRequestBody getRequestBody(BodyType bodyType, ContentSize contentSiz crc32(content)); } case FILE: - return new TestRequestBody(RequestBody.fromFile(contentSize.fileContent()), Files.size(contentSize.fileContent()), crc32(contentSize.fileContent())); + return new TestRequestBody(RequestBody.fromFile(contentSize.fileContent()), + Files.size(contentSize.fileContent()), crc32(contentSize.fileContent())); case CONTENT_PROVIDER_NO_LENGTH: { RequestBody wrapped = RequestBody.fromContentProvider(() -> FunctionalUtils.invokeSafely(() -> Files.newInputStream(contentSize.fileContent())), @@ -291,30 +322,31 @@ private TestRequestBody getRequestBody(BodyType bodyType, ContentSize contentSiz } case CONTENT_PROVIDER_WITH_LENGTH: { long contentLength = Files.size(contentSize.fileContent()); - RequestBody wrapped = RequestBody.fromContentProvider(() -> FunctionalUtils.invokeSafely(() -> Files.newInputStream(contentSize.fileContent())), - Files.size(contentSize.fileContent()), - "application/octet-stream"); + RequestBody wrapped = + RequestBody.fromContentProvider(() -> FunctionalUtils.invokeSafely(() -> Files.newInputStream(contentSize.fileContent())), + Files.size(contentSize.fileContent()), + "application/octet-stream"); return new TestRequestBody(wrapped, contentLength, crc32(contentSize.fileContent())); } case INPUTSTREAM_RESETABLE: { byte[] content = contentSize.byteContent(); RequestBody wrapped = RequestBody.fromInputStream(new ByteArrayInputStream(content), content.length); - return new TestRequestBody(wrapped, content.length, crc32(content)); + return new TestRequestBody(wrapped, content.length, contentSize.precalculatedCrc32()); } case INPUTSTREAM_NOT_RESETABLE: { byte[] content = contentSize.byteContent(); RequestBody wrapped = RequestBody.fromInputStream(new NonResettableByteStream(content), content.length); - return new TestRequestBody(wrapped, content.length, crc32(content)); + return new TestRequestBody(wrapped, content.length, contentSize.precalculatedCrc32()); } case BYTES: { byte[] content = contentSize.byteContent(); RequestBody wrapped = RequestBody.fromBytes(content); - return new TestRequestBody(wrapped, content.length, crc32(content)); + return new TestRequestBody(wrapped, content.length, contentSize.precalculatedCrc32()); } case BYTE_BUFFER: { byte[] content = contentSize.byteContent(); RequestBody wrapped = RequestBody.fromByteBuffer(ByteBuffer.wrap(content)); - return new TestRequestBody(wrapped, content.length, crc32(content)); + return new TestRequestBody(wrapped, content.length, contentSize.precalculatedCrc32()); } case REMAINING_BYTE_BUFFER: { byte[] content = contentSize.byteContent(); @@ -334,6 +366,7 @@ private TestRequestBody getRequestBody(BodyType bodyType, ContentSize contentSiz case REMAINING_BYTE_BUFFER_UNSAFE: case BLOCKING_INPUT_STREAM: case BLOCKING_OUTPUT_STREAM: + case INPUTSTREAM_NO_LENGTH: Assumptions.abort("Test BodyType not supported for sync client: " + bodyType); default: throw new RuntimeException("Unsupported body type: " + bodyType); @@ -344,42 +377,50 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content switch (bodyType) { case STRING: { String content = contentSize.stringContent(); - return new TestAsyncBody(AsyncRequestBody.fromString(content), content.getBytes(StandardCharsets.UTF_8).length, crc32(content), bodyType); + return new TestAsyncBody(AsyncRequestBody.fromString(content), content.getBytes(StandardCharsets.UTF_8).length, + crc32(content), bodyType); } case FILE: { long contentLength = Files.size(contentSize.fileContent()); - return new TestAsyncBody(AsyncRequestBody.fromFile(contentSize.fileContent()), contentLength, crc32(contentSize.fileContent()), bodyType); + return new TestAsyncBody(AsyncRequestBody.fromFile(contentSize.fileContent()), contentLength, + crc32(contentSize.fileContent()), bodyType); } case INPUTSTREAM_RESETABLE: { byte[] content = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromInputStream(new ByteArrayInputStream(content), (long) content.length, ASYNC_REQUEST_BODY_EXECUTOR); - return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); + return new TestAsyncBody(asyncRequestBody, content.length, contentSize.precalculatedCrc32(), bodyType); } case INPUTSTREAM_NOT_RESETABLE: { byte[] content = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromInputStream(new NonResettableByteStream(content), (long) content.length, ASYNC_REQUEST_BODY_EXECUTOR); - return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); + return new TestAsyncBody(asyncRequestBody, content.length, contentSize.precalculatedCrc32(), bodyType); + } + case INPUTSTREAM_NO_LENGTH: { + byte[] content = contentSize.byteContent(); + AsyncRequestBody asyncRequestBody = AsyncRequestBody + .fromInputStream(conf -> conf.inputStream(new ByteArrayInputStream(content)) + .executor(ASYNC_REQUEST_BODY_EXECUTOR)); + return new TestAsyncBody(asyncRequestBody, content.length, contentSize.precalculatedCrc32(), bodyType); } case CONTENT_PROVIDER_NO_LENGTH: { byte[] content = contentSize.byteContent(); Flowable publisher = Flowable.just(ByteBuffer.wrap(content)); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromPublisher(publisher); - return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); + return new TestAsyncBody(asyncRequestBody, content.length, contentSize.precalculatedCrc32(), bodyType); } - case BYTES: { byte[] content = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromBytes(content); - return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); + return new TestAsyncBody(asyncRequestBody, content.length, contentSize.precalculatedCrc32(), bodyType); } case BYTE_BUFFER: { byte[] content = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromByteBuffer(ByteBuffer.wrap(content)); - return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); + return new TestAsyncBody(asyncRequestBody, content.length, contentSize.precalculatedCrc32(), bodyType); } case REMAINING_BYTE_BUFFER: { byte[] content = contentSize.byteContent(); @@ -390,15 +431,15 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content byte[] crcArray = Arrays.copyOfRange(content, offset, content.length); return new TestAsyncBody(asyncRequestBody, content.length - offset, crc32(crcArray), bodyType); } - case BYTES_UNSAFE:{ + case BYTES_UNSAFE: { byte[] content = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromBytesUnsafe(content); - return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); + return new TestAsyncBody(asyncRequestBody, content.length, contentSize.precalculatedCrc32(), bodyType); } case BYTE_BUFFER_UNSAFE: { byte[] content = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromByteBufferUnsafe(ByteBuffer.wrap(content)); - return new TestAsyncBody(asyncRequestBody, content.length, crc32(content), bodyType); + return new TestAsyncBody(asyncRequestBody, content.length, contentSize.precalculatedCrc32(), bodyType); } case REMAINING_BYTE_BUFFER_UNSAFE: { byte[] content = contentSize.byteContent(); @@ -414,12 +455,9 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content byte[] content2 = contentSize.byteContent(); AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromByteBuffers(ByteBuffer.wrap(content1), ByteBuffer.wrap(content2)); - byte[] crcArray = new byte[content2.length + content2.length]; - System.arraycopy(content1, 0, crcArray, 0, content1.length); - System.arraycopy(content2, 0, crcArray, content1.length, content2.length); return new TestAsyncBody(asyncRequestBody, content1.length + content2.length, - crc32(crcArray), + contentSize.precalculatedCrc32forBuffersAPI(), bodyType); } case BUFFERS_REMAINING: { @@ -432,7 +470,7 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content System.arraycopy(content2, 0, crcArray, content1.length, content2.length); return new TestAsyncBody(asyncRequestBody, content1.length + content2.length, - crc32(crcArray), + contentSize.precalculatedCrc32forBuffersAPI(), bodyType); } case BUFFERS_UNSAFE: { @@ -445,7 +483,7 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content System.arraycopy(content2, 0, crcArray, content1.length, content2.length); return new TestAsyncBody(asyncRequestBody, content1.length + content2.length, - crc32(crcArray), + contentSize.precalculatedCrc32forBuffersAPI(), bodyType); } case BUFFERS_REMAINING_UNSAFE: { @@ -458,7 +496,7 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content System.arraycopy(content2, 0, crcArray, content1.length, content2.length); return new TestAsyncBody(asyncRequestBody, content1.length + content2.length, - crc32(crcArray), + contentSize.precalculatedCrc32forBuffersAPI(), bodyType); } case BLOCKING_INPUT_STREAM: { @@ -468,7 +506,7 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content return new TestAsyncBodyForBlockingInputStream(body, new ByteArrayInputStream(content), content.length, - crc32(content), + contentSize.precalculatedCrc32(), bodyType); } case BLOCKING_OUTPUT_STREAM: { @@ -485,7 +523,7 @@ private TestAsyncBody getAsyncRequestBody(BodyType bodyType, ContentSize content return new TestAsyncBodyForBlockingOutputStream(body, bodyWrite, content.length, - crc32(content), + contentSize.precalculatedCrc32(), bodyType); } default: @@ -516,7 +554,7 @@ private static List uploadConfigs() { for (BodyType bodyType : BodyType.values()) { for (TestConfig baseConfig : testConfigs()) { for (ContentSize size : ContentSize.values()) { - for(boolean payloadSigning : payloadSigningEnabled) { + for (boolean payloadSigning : payloadSigningEnabled) { UploadConfig config = new UploadConfig(); config.setPayloadSigning(payloadSigning); config.setBaseConfig(baseConfig); @@ -582,6 +620,7 @@ public String toString() { enum BodyType { INPUTSTREAM_RESETABLE, INPUTSTREAM_NOT_RESETABLE, + INPUTSTREAM_NO_LENGTH, STRING, @@ -614,27 +653,59 @@ enum ContentSize { byte[] byteContent() { switch (this) { - case SMALL: return smallContent; - case LARGE: return largeContent; - default: throw new IllegalArgumentException("not supported ContentSize " + this); + case SMALL: + return smallContent; + case LARGE: + return largeContent; + default: + throw new IllegalArgumentException("not supported ContentSize " + this); } } String stringContent() { switch (this) { - case SMALL: return "Hello World!"; - case LARGE: return new String(largeContent(), StandardCharsets.UTF_8); - default: throw new IllegalArgumentException("not supported ContentSize " + this); + case SMALL: + return "Hello World!"; + case LARGE: + return new String(largeContent(), StandardCharsets.UTF_8); + default: + throw new IllegalArgumentException("not supported ContentSize " + this); } } Path fileContent() { switch (this) { - case SMALL: return testFileSmall; - case LARGE: return testFileLarge; - default: throw new IllegalArgumentException("not supported ContentSize " + this); + case SMALL: + return testFileSmall; + case LARGE: + return testFileLarge; + default: + throw new IllegalArgumentException("not supported ContentSize " + this); } } + + String precalculatedCrc32() { + switch (this) { + case SMALL: + return smallContentCrc32; + case LARGE: + return largeContentCrc32; + default: + throw new IllegalArgumentException("not supported ContentSize " + this); + } + } + + String precalculatedCrc32forBuffersAPI() { + switch (this) { + case SMALL: + return smallContentCRC32ForBuffersAPI; + case LARGE: + return largeContentCRC32ForBuffersAPI; + default: + throw new IllegalArgumentException("not supported ContentSize " + this); + } + } + } private static byte[] largeContent() { @@ -692,6 +763,7 @@ public String getChecksum() { private static class TestAsyncBodyForBlockingOutputStream extends TestAsyncBody { private final Consumer bodyWrite; + private TestAsyncBodyForBlockingOutputStream(AsyncRequestBody asyncRequestBody, Consumer bodyWrite, long actualContentLength, @@ -704,6 +776,7 @@ private TestAsyncBodyForBlockingOutputStream(AsyncRequestBody asyncRequestBody, private static class TestAsyncBodyForBlockingInputStream extends TestAsyncBody { private final InputStream inputStream; + private TestAsyncBodyForBlockingInputStream(AsyncRequestBody asyncRequestBody, InputStream inputStream, long actualContentLength, @@ -716,6 +789,7 @@ private TestAsyncBodyForBlockingInputStream(AsyncRequestBody asyncRequestBody, private static class RequestRecorder implements ExecutionInterceptor { private final List requests = new ArrayList<>(); + @Override public void beforeTransmission(Context.BeforeTransmission context, ExecutionAttributes executionAttributes) { requests.add(context.httpRequest()); From ea4b87fc099c777fe065e8590cc40e0785601a6b Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Fri, 4 Apr 2025 11:54:38 -0400 Subject: [PATCH 44/46] fix npe --- .../s3/regression/UploadStreamingRegressionTesting.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java index 0dca07e783ae..4ab17cd40a07 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java @@ -106,7 +106,7 @@ static void setupClass() throws IOException { byte[] crcArrayLargeContentForBuffersApi = new byte[largeContent.length + largeContent.length]; System.arraycopy(largeContent, 0, crcArrayLargeContentForBuffersApi, 0, largeContent.length); System.arraycopy(largeContent, 0, crcArrayLargeContentForBuffersApi, largeContent.length, largeContent.length); - largeContentCRC32ForBuffersAPI = crc32(largeContentCRC32ForBuffersAPI); + largeContentCRC32ForBuffersAPI = crc32(crcArrayLargeContentForBuffersApi); } @AfterAll From 3bc6cdfcecabee006a1025d72cef4d87e80b190c Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Fri, 4 Apr 2025 14:39:44 -0400 Subject: [PATCH 45/46] junit parallel execution --- .../DownloadStreamingRegressionTesting.java | 4 +--- .../it/resources/junit-platform.properties | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 test/s3-tests/src/it/resources/junit-platform.properties diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DownloadStreamingRegressionTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DownloadStreamingRegressionTesting.java index f5f73e26af3e..6f74393c49d4 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DownloadStreamingRegressionTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/DownloadStreamingRegressionTesting.java @@ -230,8 +230,7 @@ static ObjectWithCRC uploadMultiPartObject() throws Exception { static void doMultipartUpload(BucketType bucketType, String objectName, byte[] content) { String bucket = bucketForType(bucketType); - LOG.debug(() -> String.format("Uploading multipart object for bucket type: %s - %s", bucketType, bucket) - ); + LOG.debug(() -> String.format("Uploading multipart object for bucket type: %s - %s", bucketType, bucket)); CreateMultipartUploadRequest createMulti = CreateMultipartUploadRequest.builder() .bucket(bucket) .key(objectName) @@ -264,7 +263,6 @@ static void doMultipartUpload(BucketType bucketType, String objectName, byte[] c LOG.debug(() -> "Finishing MPU, completed parts: " + completedParts); s3.completeMultipartUpload(req -> req.multipartUpload(u -> u.parts(completedParts)) - // .checksumCRC32(fullContentCRC32) .bucket(bucket) .key(objectName) .uploadId(uploadId)); diff --git a/test/s3-tests/src/it/resources/junit-platform.properties b/test/s3-tests/src/it/resources/junit-platform.properties new file mode 100644 index 000000000000..2c0aa9e64b6a --- /dev/null +++ b/test/s3-tests/src/it/resources/junit-platform.properties @@ -0,0 +1,20 @@ +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. +# + +junit.jupiter.execution.parallel.enabled = true +junit.jupiter.execution.parallel.config.dynamic.factor = 8 + +junit.jupiter.execution.parallel.mode.default = same_thread +junit.jupiter.execution.parallel.mode.classes.default = concurrent \ No newline at end of file From 11b0bcb3dac9345858e4c8ccd9c5207d6e2deed3 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 7 Apr 2025 09:55:47 -0400 Subject: [PATCH 46/46] multipart with no-length --- .../UploadStreamingRegressionTesting.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java index 4ab17cd40a07..85f532d5a140 100644 --- a/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java +++ b/test/s3-tests/src/it/java/software/amazon/awssdk/services/s3/regression/UploadStreamingRegressionTesting.java @@ -23,6 +23,8 @@ import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.crc32; import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.makeAsyncClient; import static software.amazon.awssdk.services.s3.regression.S3ChecksumsTestUtils.makeSyncClient; +import static software.amazon.awssdk.services.s3.regression.S3ClientFlavor.MULTIPART_ENABLED; +import static software.amazon.awssdk.services.s3.regression.S3ClientFlavor.STANDARD_ASYNC; import static software.amazon.awssdk.services.s3.regression.TestConfig.testConfigs; import io.reactivex.Flowable; @@ -124,15 +126,15 @@ void putObject(UploadConfig config) throws Exception { // For testing purposes, ContentProvider is Publisher for async clients // There is no way to create AsyncRequestBody with a Publisher and also provide the content length + S3ClientFlavor flavor = config.getBaseConfig().getFlavor(); Assumptions.assumeFalse(config.getBodyType() == BodyType.CONTENT_PROVIDER_WITH_LENGTH - && config.getBaseConfig().getFlavor().isAsync(), + && flavor.isAsync(), "No way to create AsyncRequestBody by giving both an Publisher and the content length"); // Payload signing doesn't work correctly for async java based // TODO(sra-identity-auth) remove when chunked encoding support is added in async code path Assumptions.assumeFalse( - (config.getBaseConfig().getFlavor() == S3ClientFlavor.STANDARD_ASYNC || - config.getBaseConfig().getFlavor() == S3ClientFlavor.MULTIPART_ENABLED) + (flavor == STANDARD_ASYNC || flavor == MULTIPART_ENABLED) && (config.payloadSigning() // MRAP requires body signing || config.getBaseConfig().getBucketType() == BucketType.MRAP), @@ -141,8 +143,7 @@ void putObject(UploadConfig config) throws Exception { // For testing purposes, ContentProvider is Publisher for async clients // Async java based clients don't currently support unknown content-length bodies Assumptions.assumeFalse( - (config.getBaseConfig().getFlavor() == S3ClientFlavor.STANDARD_ASYNC || - config.getBaseConfig().getFlavor() == S3ClientFlavor.MULTIPART_ENABLED) + flavor == STANDARD_ASYNC && config.getBodyType() == BodyType.CONTENT_PROVIDER_NO_LENGTH, "Async Java based support unknown content length"); @@ -176,13 +177,13 @@ void putObject(UploadConfig config) throws Exception { boolean requestBodyHasContentLength = false; String actualCrc32; - if (!config.getBaseConfig().getFlavor().isAsync()) { + if (!flavor.isAsync()) { TestRequestBody body = getRequestBody(config.getBodyType(), config.getContentSize()); callable = callPutObject(request, body, config.getBaseConfig(), overrideConfiguration.build()); actualContentLength = body.getActualContentLength(); requestBodyHasContentLength = body.optionalContentLength().isPresent(); actualCrc32 = body.getChecksum(); - } else if (config.getBaseConfig().getFlavor() == S3ClientFlavor.MULTIPART_ENABLED) { + } else if (flavor == MULTIPART_ENABLED) { TestAsyncBody body = getAsyncRequestBody(config.getBodyType(), config.contentSize); callable = callTmUpload(request, body, config.getBaseConfig(), overrideConfiguration.build()); actualContentLength = body.getActualContentLength(); @@ -201,19 +202,19 @@ void putObject(UploadConfig config) throws Exception { recordObjectToCleanup(bucketType, key); // mpu not supported - if (config.getBaseConfig().getFlavor() == S3ClientFlavor.MULTIPART_ENABLED) { + if (flavor == MULTIPART_ENABLED) { return; } // We only validate when configured to WHEN_SUPPORTED since checksums are optional for PutObject if (config.getBaseConfig().getRequestChecksumValidation() == RequestChecksumCalculation.WHEN_SUPPORTED // CRT switches to MPU under the hood which doesn't support checksums - && config.getBaseConfig().getFlavor() != S3ClientFlavor.CRT_BASED) { + && flavor != S3ClientFlavor.CRT_BASED) { assertThat(response.checksumCRC32()).isEqualTo(actualCrc32); } // We can't set an execution interceptor when using CRT - if (config.getBaseConfig().getFlavor() == S3ClientFlavor.CRT_BASED) { + if (flavor == S3ClientFlavor.CRT_BASED) { return; }