diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java index 9719f1213834..54b33ca562a2 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkInternalExecutionAttribute.java @@ -17,6 +17,7 @@ import software.amazon.awssdk.annotations.SdkProtectedApi; import software.amazon.awssdk.core.interceptor.trait.HttpChecksumRequired; +import software.amazon.awssdk.http.SdkHttpExecutionAttributes; /** * Attributes that can be applied to all sdk requests. Only generated code from the SDK clients should set these values. @@ -47,6 +48,12 @@ public final class SdkInternalExecutionAttribute extends SdkExecutionAttribute { public static final ExecutionAttribute DISABLE_HOST_PREFIX_INJECTION = new ExecutionAttribute<>("DisableHostPrefixInjection"); + /** + * The SDK HTTP attributes that can be passed to the HTTP client + */ + public static final ExecutionAttribute SDK_HTTP_EXECUTION_ATTRIBUTES = + new ExecutionAttribute<>("SdkHttpExecutionAttributes"); + private SdkInternalExecutionAttribute() { } } diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java index 50c6178cd691..1647f92b7189 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/MakeAsyncHttpRequestStage.java @@ -15,6 +15,7 @@ package software.amazon.awssdk.core.internal.http.pipeline.stages; +import static software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute.SDK_HTTP_EXECUTION_ATTRIBUTES; import static software.amazon.awssdk.core.internal.http.timers.TimerUtils.resolveTimeoutInMillis; import static software.amazon.awssdk.http.Header.CONTENT_LENGTH; @@ -180,15 +181,19 @@ private CompletableFuture> executeHttpRequest(SdkHttpFullReque MetricCollector httpMetricCollector = MetricUtils.createHttpMetricsCollector(context); - AsyncExecuteRequest executeRequest = AsyncExecuteRequest.builder() + AsyncExecuteRequest.Builder executeRequestBuilder = AsyncExecuteRequest.builder() .request(requestWithContentLength) .requestContentPublisher(requestProvider) .responseHandler(wrappedResponseHandler) .fullDuplex(isFullDuplex(context.executionAttributes())) - .metricCollector(httpMetricCollector) - .build(); + .metricCollector(httpMetricCollector); + if (context.executionAttributes().getAttribute(SDK_HTTP_EXECUTION_ATTRIBUTES) != null) { + executeRequestBuilder.httpExecutionAttributes( + context.executionAttributes() + .getAttribute(SDK_HTTP_EXECUTION_ATTRIBUTES)); + } - CompletableFuture httpClientFuture = doExecuteHttpRequest(context, executeRequest); + CompletableFuture httpClientFuture = doExecuteHttpRequest(context, executeRequestBuilder.build()); TimeoutTracker timeoutTracker = setupAttemptTimer(responseFuture, context); context.apiCallAttemptTimeoutTracker(timeoutTracker); diff --git a/http-client-spi/pom.xml b/http-client-spi/pom.xml index dcf4eca7f60f..4a2c74bf7042 100644 --- a/http-client-spi/pom.xml +++ b/http-client-spi/pom.xml @@ -80,6 +80,11 @@ reactive-streams-tck test + + nl.jqno.equalsverifier + equalsverifier + test + diff --git a/http-client-spi/src/main/java/software/amazon/awssdk/http/SdkHttpExecutionAttribute.java b/http-client-spi/src/main/java/software/amazon/awssdk/http/SdkHttpExecutionAttribute.java new file mode 100644 index 000000000000..9c2251423bb2 --- /dev/null +++ b/http-client-spi/src/main/java/software/amazon/awssdk/http/SdkHttpExecutionAttribute.java @@ -0,0 +1,40 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http; + +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.http.async.AsyncExecuteRequest; +import software.amazon.awssdk.utils.AttributeMap; + +/** + * An attribute attached to a particular HTTP request execution, stored in {@link SdkHttpExecutionAttributes}. It can be + * configured on an {@link AsyncExecuteRequest} via + * {@link AsyncExecuteRequest.Builder#putHttpExecutionAttribute(SdkHttpExecutionAttribute, + * Object)} + * + * @param The type of data associated with this attribute. + */ +@SdkPublicApi +public abstract class SdkHttpExecutionAttribute extends AttributeMap.Key { + + protected SdkHttpExecutionAttribute(Class valueType) { + super(valueType); + } + + protected SdkHttpExecutionAttribute(UnsafeValueType unsafeValueType) { + super(unsafeValueType); + } +} diff --git a/http-client-spi/src/main/java/software/amazon/awssdk/http/SdkHttpExecutionAttributes.java b/http-client-spi/src/main/java/software/amazon/awssdk/http/SdkHttpExecutionAttributes.java new file mode 100644 index 000000000000..571de51ec5fe --- /dev/null +++ b/http-client-spi/src/main/java/software/amazon/awssdk/http/SdkHttpExecutionAttributes.java @@ -0,0 +1,109 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http; + +import java.util.Map; +import java.util.Objects; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.http.async.AsyncExecuteRequest; +import software.amazon.awssdk.utils.AttributeMap; +import software.amazon.awssdk.utils.Validate; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * An immutable collection of {@link SdkHttpExecutionAttribute}s that can be configured on an {@link AsyncExecuteRequest} via + * {@link AsyncExecuteRequest.Builder#httpExecutionAttributes(SdkHttpExecutionAttributes)} + */ +@SdkPublicApi +public final class SdkHttpExecutionAttributes implements ToCopyableBuilder { + private final AttributeMap attributes; + + private SdkHttpExecutionAttributes(Builder builder) { + this.attributes = builder.sdkHttpExecutionAttributes.build(); + } + + /** + * Retrieve the current value of the provided attribute in this collection of attributes. This will return null if the value + * is not set. + */ + public T getAttribute(SdkHttpExecutionAttribute attribute) { + return attributes.get(attribute); + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public Builder toBuilder() { + return new Builder(attributes); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + SdkHttpExecutionAttributes that = (SdkHttpExecutionAttributes) o; + + return Objects.equals(attributes, that.attributes); + } + + @Override + public int hashCode() { + return attributes.hashCode(); + } + + public static final class Builder implements CopyableBuilder { + private AttributeMap.Builder sdkHttpExecutionAttributes = AttributeMap.builder(); + + private Builder(AttributeMap attributes) { + sdkHttpExecutionAttributes = attributes.toBuilder(); + } + + private Builder() { + } + + /** + * Add a mapping between the provided key and value. + */ + public SdkHttpExecutionAttributes.Builder put(SdkHttpExecutionAttribute key, T value) { + Validate.notNull(key, "Key to set must not be null."); + sdkHttpExecutionAttributes.put(key, value); + return this; + } + + /** + * Adds all the attributes from the map provided. + */ + public SdkHttpExecutionAttributes.Builder putAll(Map, ?> attributes) { + sdkHttpExecutionAttributes.putAll(attributes); + return this; + } + + @Override + public SdkHttpExecutionAttributes build() { + return new SdkHttpExecutionAttributes(this); + } + } +} \ No newline at end of file diff --git a/http-client-spi/src/main/java/software/amazon/awssdk/http/async/AsyncExecuteRequest.java b/http-client-spi/src/main/java/software/amazon/awssdk/http/async/AsyncExecuteRequest.java index debeaf794d55..1043becee193 100644 --- a/http-client-spi/src/main/java/software/amazon/awssdk/http/async/AsyncExecuteRequest.java +++ b/http-client-spi/src/main/java/software/amazon/awssdk/http/async/AsyncExecuteRequest.java @@ -17,8 +17,11 @@ import java.util.Optional; import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.http.SdkHttpExecutionAttribute; +import software.amazon.awssdk.http.SdkHttpExecutionAttributes; import software.amazon.awssdk.http.SdkHttpRequest; import software.amazon.awssdk.metrics.MetricCollector; +import software.amazon.awssdk.utils.Validate; /** * Request object containing the parameters necessary to make an asynchronous HTTP request. @@ -32,6 +35,7 @@ public final class AsyncExecuteRequest { private final SdkAsyncHttpResponseHandler responseHandler; private final MetricCollector metricCollector; private final boolean isFullDuplex; + private final SdkHttpExecutionAttributes sdkHttpExecutionAttributes; private AsyncExecuteRequest(BuilderImpl builder) { this.request = builder.request; @@ -39,6 +43,7 @@ private AsyncExecuteRequest(BuilderImpl builder) { this.responseHandler = builder.responseHandler; this.metricCollector = builder.metricCollector; this.isFullDuplex = builder.isFullDuplex; + this.sdkHttpExecutionAttributes = builder.executionAttributesBuilder.build(); } /** @@ -76,6 +81,13 @@ public boolean fullDuplex() { return isFullDuplex; } + /** + * @return the SDK HTTP execution attributes associated with this request + */ + public SdkHttpExecutionAttributes httpExecutionAttributes() { + return sdkHttpExecutionAttributes; + } + public static Builder builder() { return new BuilderImpl(); } @@ -125,6 +137,25 @@ public interface Builder { */ Builder fullDuplex(boolean fullDuplex); + /** + * Put an HTTP execution attribute into to the collection of HTTP execution attributes for this request + * + * @param attribute The execution attribute object + * @param value The value of the execution attribute. + */ + Builder putHttpExecutionAttribute(SdkHttpExecutionAttribute attribute, T value); + + /** + * Sets the additional HTTP execution attributes collection for this request. + *

+ * This will override the attributes configured through + * {@link #putHttpExecutionAttribute(SdkHttpExecutionAttribute, Object)} + * + * @param executionAttributes Execution attributes map for this request. + * @return This object for method chaining. + */ + Builder httpExecutionAttributes(SdkHttpExecutionAttributes executionAttributes); + AsyncExecuteRequest build(); } @@ -134,6 +165,7 @@ private static class BuilderImpl implements Builder { private SdkAsyncHttpResponseHandler responseHandler; private MetricCollector metricCollector; private boolean isFullDuplex; + private SdkHttpExecutionAttributes.Builder executionAttributesBuilder = SdkHttpExecutionAttributes.builder(); @Override public Builder request(SdkHttpRequest request) { @@ -165,6 +197,20 @@ public Builder fullDuplex(boolean fullDuplex) { return this; } + @Override + public Builder putHttpExecutionAttribute(SdkHttpExecutionAttribute attribute, T value) { + this.executionAttributesBuilder.put(attribute, value); + return this; + } + + @Override + public Builder httpExecutionAttributes(SdkHttpExecutionAttributes executionAttributes) { + Validate.paramNotNull(executionAttributes, "executionAttributes"); + this.executionAttributesBuilder = executionAttributes.toBuilder(); + return this; + } + + @Override public AsyncExecuteRequest build() { return new AsyncExecuteRequest(this); diff --git a/http-client-spi/src/test/java/software/amazon/awssdk/http/SdkHttpExecutionAttributesTest.java b/http-client-spi/src/test/java/software/amazon/awssdk/http/SdkHttpExecutionAttributesTest.java new file mode 100644 index 000000000000..d71c98511b26 --- /dev/null +++ b/http-client-spi/src/test/java/software/amazon/awssdk/http/SdkHttpExecutionAttributesTest.java @@ -0,0 +1,49 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.http; + + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +import nl.jqno.equalsverifier.EqualsVerifier; +import org.junit.jupiter.api.Test; + +class SdkHttpExecutionAttributesTest { + + @Test + void equalsAndHashcode() { + EqualsVerifier.forClass(SdkHttpExecutionAttributes.class) + .withNonnullFields("attributes") + .verify(); + } + + @Test + void getAttribute_shouldReturnCorrectValue() { + SdkHttpExecutionAttributes attributes = SdkHttpExecutionAttributes.builder() + .put(TestExecutionAttribute.TEST_KEY_FOO, "test") + .build(); + assertThat(attributes.getAttribute(TestExecutionAttribute.TEST_KEY_FOO)).isEqualTo("test"); + } + + private static final class TestExecutionAttribute extends SdkHttpExecutionAttribute { + + private static final TestExecutionAttribute TEST_KEY_FOO = new TestExecutionAttribute<>(String.class); + + private TestExecutionAttribute(Class valueType) { + super(valueType); + } + } +} diff --git a/pom.xml b/pom.xml index 4cda5509b4e8..e715c023509a 100644 --- a/pom.xml +++ b/pom.xml @@ -114,7 +114,7 @@ 2.2.21 1.10 1.29 - 0.15.8 + 0.15.15 5.8.1 diff --git a/services-custom/s3-transfer-manager/pom.xml b/services-custom/s3-transfer-manager/pom.xml index 57f9d5cc3a30..8c0723b99402 100644 --- a/services-custom/s3-transfer-manager/pom.xml +++ b/services-custom/s3-transfer-manager/pom.xml @@ -101,11 +101,6 @@ aws-core ${awsjavasdk.version} - - software.amazon.awssdk - metrics-spi - ${awsjavasdk.version} - software.amazon.awssdk service-test-utils diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3CrtClientPutObjectIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3CrtClientPutObjectIntegrationTest.java index 9e5fa71bb0b5..ed74176dbb32 100644 --- a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3CrtClientPutObjectIntegrationTest.java +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3CrtClientPutObjectIntegrationTest.java @@ -21,25 +21,19 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; -import java.security.NoSuchAlgorithmException; import java.util.List; import java.util.Optional; import java.util.Random; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.stream.Collectors; import java.util.stream.Stream; import org.assertj.core.api.Assertions; -import org.junit.After; import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.reactivestreams.Subscriber; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.async.AsyncRequestBody; -import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.testutils.RandomTempFile; @@ -52,9 +46,7 @@ public class S3CrtClientPutObjectIntegrationTest extends S3IntegrationTestBase { private static final int OBJ_SIZE = 8 * 1024 * 1024; private static RandomTempFile testFile; - private static ExecutorService executorService; - private S3CrtAsyncClient s3Crt; - + private static S3CrtAsyncClient s3Crt; @BeforeClass public static void setup() throws Exception { @@ -62,32 +54,23 @@ public static void setup() throws Exception { S3IntegrationTestBase.createBucket(TEST_BUCKET); testFile = new RandomTempFile(TEST_KEY, OBJ_SIZE); - executorService = Executors.newFixedThreadPool(2); - } - @Before - public void methodSetup() { s3Crt = S3CrtAsyncClient.builder() .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) .region(S3IntegrationTestBase.DEFAULT_REGION) .build(); } - @After - public void methodTeardown() { - s3Crt.close(); - } - @AfterClass public static void teardown() throws IOException { S3IntegrationTestBase.deleteBucketAndAllContents(TEST_BUCKET); Files.delete(testFile.toPath()); - executorService.shutdown(); + s3Crt.close(); S3IntegrationTestBase.cleanUp(); } @Test - public void putObject_fileRequestBody_objectSentCorrectly() throws IOException, NoSuchAlgorithmException { + public void putObject_fileRequestBody_objectSentCorrectly() throws Exception { AsyncRequestBody body = AsyncRequestBody.fromFile(testFile.toPath()); s3Crt.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), body).join(); @@ -100,7 +83,7 @@ public void putObject_fileRequestBody_objectSentCorrectly() throws IOException, } @Test - public void putObject_byteBufferBody_objectSentCorrectly() throws IOException, NoSuchAlgorithmException { + public void putObject_byteBufferBody_objectSentCorrectly() { byte[] data = new byte[16384]; new Random().nextBytes(data); ByteBuffer byteBuffer = ByteBuffer.wrap(data); @@ -118,7 +101,7 @@ public void putObject_byteBufferBody_objectSentCorrectly() throws IOException, N } @Test - public void putObject_customRequestBody_objectSentCorrectly() throws IOException, NoSuchAlgorithmException { + public void putObject_customRequestBody_objectSentCorrectly() throws IOException { Random rng = new Random(); int bufferSize = 16384; int nBuffers = 15; @@ -154,26 +137,4 @@ public void subscribe(Subscriber subscriber) { Assertions.assertThat(ChecksumUtils.computeCheckSum(objContent)).isEqualTo(expectedSum); } - - @Test - public void putObject_customExecutorService_objectSentCorrectly() throws IOException { - AsyncRequestBody body = AsyncRequestBody.fromFile(testFile.toPath()); - try (S3CrtAsyncClient s3Client = - S3CrtAsyncClient.builder() - .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - .region(DEFAULT_REGION) - .asyncConfiguration(b -> b.advancedOption(SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR, - executorService)) - .build()) { - - s3Client.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), body).join(); - ResponseInputStream objContent = - S3IntegrationTestBase.s3.getObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), - ResponseTransformer.toInputStream()); - - byte[] expectedSum = ChecksumUtils.computeCheckSum(Files.newInputStream(testFile.toPath())); - - Assertions.assertThat(ChecksumUtils.computeCheckSum(objContent)).isEqualTo(expectedSum); - } - } } diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3CrtGetObjectIntegrationTest.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3CrtGetObjectIntegrationTest.java index eb80d2ae9058..15b28161a804 100644 --- a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3CrtGetObjectIntegrationTest.java +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3CrtGetObjectIntegrationTest.java @@ -31,7 +31,6 @@ import org.junit.Test; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.core.async.SdkPublisher; -import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; import software.amazon.awssdk.http.async.SimpleSubscriber; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @@ -94,24 +93,6 @@ public void getObject_customResponseTransformer() { } - @Test - public void getObject_customExecutors_fileDownloadCorrectly() throws IOException { - Path path = RandomTempFile.randomUncreatedFile().toPath(); - - try (S3CrtAsyncClient s3Client = - S3CrtAsyncClient.builder() - .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) - .region(DEFAULT_REGION) - .asyncConfiguration(b -> b.advancedOption(SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR, - executorService)) - .build()) { - GetObjectResponse response = - s3Client.getObject(b -> b.bucket(BUCKET).key(KEY), AsyncResponseTransformer.toFile(path)).join(); - - assertThat(Md5Utils.md5AsBase64(path.toFile())).isEqualTo(Md5Utils.md5AsBase64(file)); - } - } - private static final class TestResponseTransformer implements AsyncResponseTransformer { private CompletableFuture future; diff --git a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java index 2dcd74f9bd5a..8e392dbd07ba 100644 --- a/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java +++ b/services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java @@ -18,6 +18,7 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import software.amazon.awssdk.crt.CrtResource; +import software.amazon.awssdk.crt.Log; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; @@ -43,7 +44,7 @@ */ public class S3IntegrationTestBase extends AwsTestBase { - protected static final Region DEFAULT_REGION = Region.US_WEST_2; + protected static final Region DEFAULT_REGION = Region.US_WEST_1; /** * The S3 client for all tests to use. */ @@ -57,6 +58,7 @@ public class S3IntegrationTestBase extends AwsTestBase { */ @BeforeClass public static void setUp() throws Exception { + Log.initLoggingToStdout(Log.LogLevel.Warn); System.setProperty("aws.crt.debugnative", "true"); s3 = s3ClientBuilder().build(); s3Async = s3AsyncClientBuilder().build(); @@ -90,7 +92,7 @@ private static void createBucket(String bucketName, int retryCount) { .bucket(bucketName) .createBucketConfiguration( CreateBucketConfiguration.builder() - .locationConstraint(BucketLocationConstraint.US_WEST_2) + .locationConstraint(BucketLocationConstraint.US_WEST_1) .build()) .build()); } catch (S3Exception e) { diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/ApplyUserAgentInterceptor.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/ApplyUserAgentInterceptor.java new file mode 100644 index 000000000000..2b4c465ec2de --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/ApplyUserAgentInterceptor.java @@ -0,0 +1,54 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; +import software.amazon.awssdk.core.ApiName; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.services.s3.model.S3Request; + +/** + * Apply TM specific user agent to the request + */ +@SdkInternalApi +public final class ApplyUserAgentInterceptor implements ExecutionInterceptor { + private static final ApiName API_NAME = + ApiName.builder().name("ft").version("s3-transfer").build(); + private static final Consumer USER_AGENT_APPLIER = + b -> b.addApiName(API_NAME); + + @Override + public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes executionAttributes) { + assert context.request() instanceof S3Request; + + S3Request request = (S3Request) context.request(); + AwsRequestOverrideConfiguration overrideConfiguration = + request.overrideConfiguration() + .map(c -> c.toBuilder() + .applyMutation(USER_AGENT_APPLIER) + .build()) + .orElseGet(() -> AwsRequestOverrideConfiguration.builder() + .applyMutation(USER_AGENT_APPLIER) + .build()); + + return request.toBuilder().overrideConfiguration(overrideConfiguration).build(); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtErrorHandler.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtErrorHandler.java deleted file mode 100644 index 8b955e0d811b..000000000000 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtErrorHandler.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.transfer.s3.internal; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.awscore.exception.AwsErrorDetails; -import software.amazon.awssdk.core.exception.SdkClientException; -import software.amazon.awssdk.crt.s3.CrtS3RuntimeException; -import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException; -import software.amazon.awssdk.services.s3.model.BucketAlreadyOwnedByYouException; -import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; -import software.amazon.awssdk.services.s3.model.NoSuchBucketException; -import software.amazon.awssdk.services.s3.model.NoSuchKeyException; -import software.amazon.awssdk.services.s3.model.NoSuchUploadException; -import software.amazon.awssdk.services.s3.model.ObjectAlreadyInActiveTierErrorException; -import software.amazon.awssdk.services.s3.model.S3Exception; -import software.amazon.awssdk.utils.StringUtils; - -@SdkInternalApi -public class CrtErrorHandler { - - private final Map s3ExceptionBuilderMap; - - public CrtErrorHandler() { - s3ExceptionBuilderMap = getS3ExceptionBuilderMap(); - } - - /** - * This class transform a crtTRunTimeException to the S3 service Exceptions. - * CrtS3RuntimeException are the exceptions generated due to failures in CRTClient due to S3 Service errors. - * - * @param crtRuntimeException Exception that is thrown by CrtClient. - * @return - */ - public Exception transformException(Exception crtRuntimeException) { - Optional crtS3RuntimeExceptionOptional = getCrtS3RuntimeException(crtRuntimeException); - Exception exception = crtS3RuntimeExceptionOptional - .filter(CrtErrorHandler::isErrorDetailsAvailable) - .map(e -> getServiceSideException(e)) - .orElseGet(() -> SdkClientException.create(crtRuntimeException.getMessage(), crtRuntimeException)); - return exception; - } - - private Exception getServiceSideException(CrtS3RuntimeException e) { - if (s3ExceptionBuilderMap.get(e.getAwsErrorCode()) != null) { - return s3ExceptionBuilderMap.get(e.getAwsErrorCode()) - .awsErrorDetails( - AwsErrorDetails.builder().errorCode(e.getAwsErrorCode()) - .errorMessage(e.getAwsErrorMessage()).build()) - .cause(e) - .message(e.getMessage()) - .statusCode(e.getStatusCode()) - .build(); - } - return S3Exception.builder().statusCode(e.getStatusCode()).message(e.getMessage()).cause(e).build(); - } - - /** - * This method checks if the exception has the required details to transform to S3 Exception. - * @param crtS3RuntimeException the exception that needs to be checked - * @return true if exception has the required details. - */ - private static boolean isErrorDetailsAvailable(CrtS3RuntimeException crtS3RuntimeException) { - return StringUtils.isNotBlank(crtS3RuntimeException.getAwsErrorCode()); - } - - /** - * Checks if the Exception or its cause is of CrtS3RuntimeException. - * The S3 Service related exception are in the form of CrtS3RuntimeException. - * @param crtRuntimeException - * @return CrtS3RuntimeException else return empty, - */ - private Optional getCrtS3RuntimeException(Exception crtRuntimeException) { - if (crtRuntimeException instanceof CrtS3RuntimeException) { - return Optional.of((CrtS3RuntimeException) crtRuntimeException); - } - Throwable cause = crtRuntimeException.getCause(); - if (cause instanceof CrtS3RuntimeException) { - return Optional.of((CrtS3RuntimeException) cause); - } - return Optional.empty(); - } - - - /** - * Gets a Mapping of AWSErrorCode to its corresponding S3 Exception Builders. - * - * @return - */ - private Map getS3ExceptionBuilderMap() { - Map s3ExceptionBuilderMap = new HashMap<>(); - s3ExceptionBuilderMap.put("ObjectAlreadyInActiveTierError", ObjectAlreadyInActiveTierErrorException.builder()); - s3ExceptionBuilderMap.put("NoSuchUpload", NoSuchUploadException.builder()); - s3ExceptionBuilderMap.put("BucketAlreadyExists", BucketAlreadyExistsException.builder()); - s3ExceptionBuilderMap.put("BucketAlreadyOwnedByYou", BucketAlreadyOwnedByYouException.builder()); - s3ExceptionBuilderMap.put("InvalidObjectState", InvalidObjectStateException.builder()); - s3ExceptionBuilderMap.put("NoSuchBucket", NoSuchBucketException.builder()); - s3ExceptionBuilderMap.put("NoSuchKey", NoSuchKeyException.builder()); - return s3ExceptionBuilderMap; - } -} \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtResponseDataConsumerAdapter.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtResponseDataConsumerAdapter.java deleted file mode 100644 index 2c6632d7d0c0..000000000000 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/CrtResponseDataConsumerAdapter.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.transfer.s3.internal; - -import com.amazonaws.s3.ResponseDataConsumer; -import com.amazonaws.s3.model.GetObjectOutput; -import java.nio.ByteBuffer; -import java.util.concurrent.CompletableFuture; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.core.async.AsyncResponseTransformer; -import software.amazon.awssdk.crt.CrtRuntimeException; -import software.amazon.awssdk.crt.http.HttpHeader; -import software.amazon.awssdk.http.SdkHttpResponse; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.utils.Logger; - -/** - * Adapt the SDK API {@link AsyncResponseTransformer} to the CRT API {@link ResponseDataConsumer}. - */ -@SdkInternalApi -public class CrtResponseDataConsumerAdapter implements ResponseDataConsumer { - - private static final Logger log = Logger.loggerFor(CrtResponseDataConsumerAdapter.class); - private final AsyncResponseTransformer transformer; - private final CompletableFuture future; - private final S3CrtDataPublisher publisher; - private final ResponseHeadersHandler headerHandler; - private final CrtErrorHandler errorHandler; - - - public CrtResponseDataConsumerAdapter(AsyncResponseTransformer transformer) { - this(transformer, new S3CrtDataPublisher(), new ResponseHeadersHandler()); - } - - @SdkInternalApi - CrtResponseDataConsumerAdapter(AsyncResponseTransformer transformer, - S3CrtDataPublisher s3CrtDataPublisher, - ResponseHeadersHandler headersHandler) { - this.transformer = transformer; - this.future = transformer.prepare(); - this.publisher = s3CrtDataPublisher; - this.headerHandler = headersHandler; - this.errorHandler = new CrtErrorHandler(); - } - - public CompletableFuture transformerFuture() { - return future; - } - - @Override - public void onResponseHeaders(int statusCode, HttpHeader[] headers) { - headerHandler.onResponseHeaders(statusCode, headers); - } - - @Override - public void onResponse(GetObjectOutput output) { - // Passing empty SdkHttpResponse if it's not available - SdkHttpResponse sdkHttpResponse = headerHandler.sdkHttpResponseFuture() - .getNow(SdkHttpResponse.builder().build()); - - GetObjectResponse response = S3CrtPojoConversion.fromCrtGetObjectOutput(output, - sdkHttpResponse); - transformer.onResponse(response); - transformer.onStream(publisher); - } - - @Override - public void onResponseData(ByteBuffer byteBuffer) { - log.trace(() -> "Received data of size " + byteBuffer.remaining()); - publisher.deliverData(byteBuffer); - } - - @Override - public void onException(CrtRuntimeException e) { - log.debug(() -> "An error occurred ", e); - Exception transformException = errorHandler.transformException(e); - transformer.exceptionOccurred(transformException); - publisher.notifyError(transformException); - } - - @Override - public void onFinished() { - log.debug(() -> "Finished streaming "); - publisher.notifyStreamingFinished(); - } -} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClient.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClient.java index 5bc122bcb5d0..01b6b6d4fbd6 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClient.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClient.java @@ -15,144 +15,112 @@ package software.amazon.awssdk.transfer.s3.internal; -import com.amazonaws.s3.S3NativeClient; -import com.amazonaws.s3.model.GetObjectOutput; -import com.amazonaws.s3.model.PutObjectOutput; +import static software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute.SDK_HTTP_EXECUTION_ATTRIBUTES; +import static software.amazon.awssdk.transfer.s3.internal.S3InternalSdkHttpExecutionAttribute.OPERATION_NAME; + import java.util.concurrent.CompletableFuture; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.annotations.SdkTestInternalApi; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.awscore.AwsRequest; +import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.AsyncResponseTransformer; -import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration; -import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.core.interceptor.SdkExecutionAttribute; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.core.signer.NoOpSigner; +import software.amazon.awssdk.http.SdkHttpExecutionAttributes; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Configuration; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; -import software.amazon.awssdk.utils.CompletableFutureUtils; @SdkInternalApi public final class DefaultS3CrtAsyncClient implements S3CrtAsyncClient { - private final S3NativeClient s3NativeClient; - private final S3NativeClientConfiguration configuration; - private final CrtErrorHandler crtErrorHandler; - - public DefaultS3CrtAsyncClient(DefaultS3CrtClientBuilder builder) { - S3NativeClientConfiguration.Builder configBuilder = - S3NativeClientConfiguration.builder() - .targetThroughputInGbps(builder.targetThroughputInGbps()) - .partSizeInBytes(builder.minimumPartSizeInBytes()) - .maxConcurrency(builder.maxConcurrency) - .credentialsProvider(builder.credentialsProvider) - .asyncConfiguration(builder.asyncConfiguration); - if (builder.region() != null) { - configBuilder.signingRegion(builder.region().id()); - } - - configuration = configBuilder.build(); - - this.s3NativeClient = new S3NativeClient(configuration.signingRegion(), - configuration.clientBootstrap(), - configuration.credentialsProvider(), - configuration.partSizeBytes(), - configuration.targetThroughputInGbps(), - configuration.maxConcurrency()); - this.crtErrorHandler = new CrtErrorHandler(); + private final SdkAsyncHttpClient s3CrtAsyncHttpClient; + private final S3AsyncClient s3AsyncClient; + + private DefaultS3CrtAsyncClient(DefaultS3CrtClientBuilder builder) { + this.s3CrtAsyncHttpClient = S3CrtAsyncHttpClient.builder() + .targetThroughputInGbps(builder.targetThroughputInGbps()) + .minimumPartSizeInBytes(builder.minimumPartSizeInBytes()) + .maxConcurrency(builder.maxConcurrency) + .region(builder.region) + .credentialsProvider(builder.credentialsProvider) + .build(); + + this.s3AsyncClient = initializeS3AsyncClient(builder); } @SdkTestInternalApi - DefaultS3CrtAsyncClient(S3NativeClientConfiguration configuration, - S3NativeClient nativeClient) { - this.configuration = configuration; - this.s3NativeClient = nativeClient; - this.crtErrorHandler = new CrtErrorHandler(); + DefaultS3CrtAsyncClient(SdkAsyncHttpClient s3CrtAsyncHttpClient, + S3AsyncClient s3AsyncClient) { + this.s3CrtAsyncHttpClient = s3CrtAsyncHttpClient; + this.s3AsyncClient = s3AsyncClient; + } + + private S3AsyncClient initializeS3AsyncClient(DefaultS3CrtClientBuilder builder) { + return S3AsyncClient.builder() + // Disable checksum, retry policy and signer because they are handled in crt + .serviceConfiguration(S3Configuration.builder() + .checksumValidationEnabled(false) + .build()) + .region(builder.region) + .credentialsProvider(builder.credentialsProvider) + .overrideConfiguration(o -> o.putAdvancedOption(SdkAdvancedClientOption.SIGNER, + new NoOpSigner()) + .retryPolicy(RetryPolicy.none()) + .addExecutionInterceptor(new AttachHttpAttributesExecutionInterceptor())) + .httpClient(s3CrtAsyncHttpClient) + .build(); } @Override public CompletableFuture getObject( GetObjectRequest getObjectRequest, AsyncResponseTransformer asyncResponseTransformer) { - - CompletableFuture returnFuture = new CompletableFuture<>(); - com.amazonaws.s3.model.GetObjectRequest crtGetObjectRequest = S3CrtPojoConversion.toCrtGetObjectRequest(getObjectRequest); - CrtResponseDataConsumerAdapter adapter = new CrtResponseDataConsumerAdapter<>(asyncResponseTransformer); - - CompletableFuture adapterFuture = adapter.transformerFuture(); - - CompletableFuture crtFuture = s3NativeClient.getObject(crtGetObjectRequest, adapter); - - // Forward the cancellation to crtFuture to cancel the request - CompletableFutureUtils.forwardExceptionTo(returnFuture, crtFuture); - - - // Forward the exception from the CRT future to the return future in case - // the adapter callback didn't get it - CompletableFutureUtils.forwardTransformedExceptionTo(crtFuture, returnFuture, - t -> t instanceof Exception ? crtErrorHandler.transformException((Exception) t) : t); - - returnFuture.whenComplete((r, t) -> { - if (t == null) { - returnFuture.complete(r); - } else { - returnFuture.completeExceptionally(t instanceof Exception - ? crtErrorHandler.transformException((Exception) t) : t); - } - }); - - CompletableFutureUtils.forwardResultTo(adapterFuture, returnFuture, configuration.futureCompletionExecutor()); - - return CompletableFutureUtils.forwardExceptionTo(returnFuture, adapterFuture); + validateOverrideConfiguration(getObjectRequest); + return s3AsyncClient.getObject(getObjectRequest, asyncResponseTransformer); } @Override public CompletableFuture putObject(PutObjectRequest putObjectRequest, AsyncRequestBody requestBody) { - CompletableFuture returnFuture = new CompletableFuture<>(); - - com.amazonaws.s3.model.PutObjectRequest adaptedRequest = S3CrtPojoConversion.toCrtPutObjectRequest(putObjectRequest); - - if (adaptedRequest.contentLength() == null && requestBody.contentLength().isPresent()) { - adaptedRequest = adaptedRequest.toBuilder() - .contentLength(requestBody.contentLength().get()) - .build(); - } - - RequestDataSupplierAdapter requestDataSupplier = new RequestDataSupplierAdapter(requestBody); - CompletableFuture crtFuture = s3NativeClient.putObject(adaptedRequest, - requestDataSupplier); - // Forward the cancellation to crtFuture to cancel the request - CompletableFutureUtils.forwardExceptionTo(returnFuture, crtFuture); - - CompletableFuture httpResponseFuture = requestDataSupplier.sdkHttpResponseFuture(); - CompletableFuture executeFuture = - // If the header is not available, passing empty SDK HTTP response - crtFuture.thenApply(putObjectOutput -> S3CrtPojoConversion.fromCrtPutObjectOutput( - putObjectOutput, httpResponseFuture.getNow(SdkHttpResponse.builder().build()))); - - executeFuture.whenComplete((r, t) -> { - if (t == null) { - returnFuture.complete(r); - } else { - returnFuture.completeExceptionally(t instanceof Exception - ? crtErrorHandler.transformException((Exception) t) : t); - } - }); - - CompletableFutureUtils.forwardResultTo(executeFuture, returnFuture, configuration.futureCompletionExecutor()); - - return CompletableFutureUtils.forwardExceptionTo(returnFuture, executeFuture); + validateOverrideConfiguration(putObjectRequest); + return s3AsyncClient.putObject(putObjectRequest, requestBody); } @Override public String serviceName() { - return "s3"; + return SERVICE_NAME; } @Override public void close() { - s3NativeClient.close(); - configuration.close(); + s3CrtAsyncHttpClient.close(); + s3AsyncClient.close(); + } + + private static void validateOverrideConfiguration(AwsRequest request) { + + if (request.overrideConfiguration().isPresent()) { + AwsRequestOverrideConfiguration overrideConfiguration = request.overrideConfiguration().get(); + if (overrideConfiguration.signer().isPresent()) { + throw new UnsupportedOperationException("Request-level signer override is not supported"); + } + + // TODO: support request-level credential override + if (overrideConfiguration.credentialsProvider().isPresent()) { + throw new UnsupportedOperationException("Request-level credentials override is not supported"); + } + } } public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientBuilder { @@ -161,7 +129,6 @@ public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientB private Long minimalPartSizeInBytes; private Double targetThroughputInGbps; private Integer maxConcurrency; - private ClientAsyncConfiguration asyncConfiguration; public AwsCredentialsProvider credentialsProvider() { return credentialsProvider; @@ -214,14 +181,23 @@ public S3CrtAsyncClientBuilder maxConcurrency(Integer maxConcurrency) { } @Override - public S3CrtAsyncClientBuilder asyncConfiguration(ClientAsyncConfiguration configuration) { - this.asyncConfiguration = configuration; - return this; + public S3CrtAsyncClient build() { + return new DefaultS3CrtAsyncClient(this); } + } + private static final class AttachHttpAttributesExecutionInterceptor implements ExecutionInterceptor { @Override - public S3CrtAsyncClient build() { - return new DefaultS3CrtAsyncClient(this); + public void afterMarshalling(Context.AfterMarshalling context, + ExecutionAttributes executionAttributes) { + SdkHttpExecutionAttributes attributes = + SdkHttpExecutionAttributes.builder() + .put(OPERATION_NAME, + executionAttributes.getAttribute(SdkExecutionAttribute.OPERATION_NAME)) + .build(); + + executionAttributes.putAttribute(SDK_HTTP_EXECUTION_ATTRIBUTES, + attributes); } } } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java index a0c08ff6951d..69fa4809b867 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java @@ -21,8 +21,6 @@ import software.amazon.awssdk.arns.Arn; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.AsyncResponseTransformer; -import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration; -import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; import software.amazon.awssdk.services.s3.internal.resource.S3AccessPointResource; import software.amazon.awssdk.services.s3.internal.resource.S3ArnConverter; import software.amazon.awssdk.services.s3.internal.resource.S3Resource; @@ -70,7 +68,7 @@ public DefaultS3TransferManager(DefaultBuilder tmBuilder) { this.uploadDirectoryManager = uploadDirectoryManager; } - private TransferManagerConfiguration resolveTransferManagerConfiguration(DefaultBuilder tmBuilder) { + private static TransferManagerConfiguration resolveTransferManagerConfiguration(DefaultBuilder tmBuilder) { TransferManagerConfiguration.Builder transferConfigBuilder = TransferManagerConfiguration.builder(); tmBuilder.transferManagerConfiguration.uploadDirectoryConfiguration() .ifPresent(transferConfigBuilder::uploadDirectoryConfiguration); @@ -78,19 +76,13 @@ private TransferManagerConfiguration resolveTransferManagerConfiguration(Default return transferConfigBuilder.build(); } - private S3CrtAsyncClient initializeS3CrtClient(DefaultBuilder tmBuilder) { + private static S3CrtAsyncClient initializeS3CrtClient(DefaultBuilder tmBuilder) { S3CrtAsyncClient.S3CrtAsyncClientBuilder clientBuilder = S3CrtAsyncClient.builder(); tmBuilder.s3ClientConfiguration.credentialsProvider().ifPresent(clientBuilder::credentialsProvider); tmBuilder.s3ClientConfiguration.maxConcurrency().ifPresent(clientBuilder::maxConcurrency); tmBuilder.s3ClientConfiguration.minimumPartSizeInBytes().ifPresent(clientBuilder::minimumPartSizeInBytes); tmBuilder.s3ClientConfiguration.region().ifPresent(clientBuilder::region); tmBuilder.s3ClientConfiguration.targetThroughputInGbps().ifPresent(clientBuilder::targetThroughputInGbps); - ClientAsyncConfiguration clientAsyncConfiguration = - ClientAsyncConfiguration.builder() - .advancedOption(SdkAdvancedAsyncClientOption.FUTURE_COMPLETION_EXECUTOR, - transferConfiguration.option(TransferConfigurationOption.EXECUTOR)) - .build(); - clientBuilder.asyncConfiguration(clientAsyncConfiguration); return clientBuilder.build(); } @@ -285,7 +277,7 @@ private static boolean isMrapArn(Arn arn) { return !s3EndpointResource.region().isPresent(); } - private static class DefaultBuilder implements S3TransferManager.Builder { + private static final class DefaultBuilder implements S3TransferManager.Builder { private S3ClientConfiguration s3ClientConfiguration = S3ClientConfiguration.builder().build(); private S3TransferManagerOverrideConfiguration transferManagerConfiguration = S3TransferManagerOverrideConfiguration.builder().build(); diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/ResponseHeadersHandler.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/ResponseHeadersHandler.java deleted file mode 100644 index 5f5d7193bcef..000000000000 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/ResponseHeadersHandler.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.transfer.s3.internal; - -import static software.amazon.awssdk.core.http.HttpResponseHandler.X_AMZ_ID_2_HEADER; - -import com.amazonaws.s3.OperationHandler; -import java.util.concurrent.CompletableFuture; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.core.SdkStandardLogger; -import software.amazon.awssdk.crt.http.HttpHeader; -import software.amazon.awssdk.http.HttpStatusFamily; -import software.amazon.awssdk.http.SdkHttpResponse; - -@SdkInternalApi -public final class ResponseHeadersHandler implements OperationHandler { - private static final String REQUEST_ID = "x-amz-request-id"; - private final SdkHttpResponse.Builder responseBuilder; - private final CompletableFuture responseFuture; - - public ResponseHeadersHandler() { - responseBuilder = SdkHttpResponse.builder(); - responseFuture = new CompletableFuture<>(); - } - - @Override - public void onResponseHeaders(int statusCode, HttpHeader[] headers) { - if (HttpStatusFamily.of(statusCode) == HttpStatusFamily.SUCCESSFUL) { - SdkStandardLogger.REQUEST_LOGGER.debug(() -> "Received successful response: " + statusCode); - } else { - SdkStandardLogger.REQUEST_LOGGER.debug(() -> "Received error response: " + statusCode); - } - - for (HttpHeader header : headers) { - responseBuilder.appendHeader(header.getName(), header.getValue()); - } - responseBuilder.statusCode(statusCode); - SdkStandardLogger.REQUEST_ID_LOGGER.debug(() -> REQUEST_ID + " : " + - responseBuilder.firstMatchingHeader(REQUEST_ID) - .orElse("not available")); - SdkStandardLogger.REQUEST_ID_LOGGER.debug(() -> X_AMZ_ID_2_HEADER + " : " + - responseBuilder.firstMatchingHeader(X_AMZ_ID_2_HEADER) - .orElse("not available")); - responseFuture.complete(responseBuilder.build()); - } - - public CompletableFuture sdkHttpResponseFuture() { - return responseFuture; - } -} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncClient.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncClient.java index 097313ed67c1..9236a47b5995 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncClient.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncClient.java @@ -15,10 +15,8 @@ package software.amazon.awssdk.transfer.s3.internal; -import java.util.function.Consumer; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.utils.builder.SdkBuilder; @@ -41,20 +39,6 @@ interface S3CrtAsyncClientBuilder extends SdkBuilder clientAsyncConfiguration) { - return asyncConfiguration(ClientAsyncConfiguration.builder().applyMutation(clientAsyncConfiguration).build()); - } - @Override S3CrtAsyncClient build(); } diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncHttpClient.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncHttpClient.java new file mode 100644 index 000000000000..225bab02a26d --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncHttpClient.java @@ -0,0 +1,255 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static software.amazon.awssdk.transfer.s3.internal.S3InternalSdkHttpExecutionAttribute.OPERATION_NAME; +import static software.amazon.awssdk.utils.CollectionUtils.isNullOrEmpty; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.SdkTestInternalApi; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.crt.http.HttpHeader; +import software.amazon.awssdk.crt.http.HttpRequest; +import software.amazon.awssdk.crt.s3.S3Client; +import software.amazon.awssdk.crt.s3.S3ClientOptions; +import software.amazon.awssdk.crt.s3.S3MetaRequest; +import software.amazon.awssdk.crt.s3.S3MetaRequestOptions; +import software.amazon.awssdk.http.Header; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.async.AsyncExecuteRequest; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.utils.AttributeMap; +import software.amazon.awssdk.utils.http.SdkHttpUtils; + +/** + * An implementation of {@link SdkAsyncHttpClient} that uses an CRT S3 HTTP client {@link S3Client} to communicate with S3. + * Note that it does not work with other services + */ +@SdkInternalApi +public final class S3CrtAsyncHttpClient implements SdkAsyncHttpClient { + + private final S3Client crtS3Client; + private final S3NativeClientConfiguration s3NativeClientConfiguration; + + private S3CrtAsyncHttpClient(Builder builder) { + s3NativeClientConfiguration = + S3NativeClientConfiguration.builder() + .targetThroughputInGbps(builder.targetThroughputInGbps) + .partSizeInBytes(builder.minimalPartSizeInBytes) + .maxConcurrency(builder.maxConcurrency) + .signingRegion(builder.region == null ? null : builder.region.id()) + .credentialsProvider(builder.credentialsProvider) + .build(); + + S3ClientOptions s3ClientOptions = + new S3ClientOptions().withRegion(s3NativeClientConfiguration.signingRegion()) + .withCredentialsProvider(s3NativeClientConfiguration.credentialsProvider()) + .withClientBootstrap(s3NativeClientConfiguration.clientBootstrap()) + .withPartSize(s3NativeClientConfiguration.partSizeBytes()) + .withThroughputTargetGbps(s3NativeClientConfiguration.targetThroughputInGbps()); + this.crtS3Client = new S3Client(s3ClientOptions); + } + + @SdkTestInternalApi + S3CrtAsyncHttpClient(S3Client crtS3Client, S3NativeClientConfiguration nativeClientConfiguration) { + this.crtS3Client = crtS3Client; + this.s3NativeClientConfiguration = nativeClientConfiguration; + } + + @Override + public CompletableFuture execute(AsyncExecuteRequest request) { + CompletableFuture executeFuture = new CompletableFuture<>(); + HttpRequest httpRequest = toCrtRequest(request); + S3CrtResponseHandlerAdapter responseHandler = new S3CrtResponseHandlerAdapter(executeFuture, request.responseHandler()); + + S3MetaRequestOptions.MetaRequestType requestType = requestType(request); + + S3MetaRequestOptions requestOptions = new S3MetaRequestOptions() + .withHttpRequest(httpRequest) + .withMetaRequestType(requestType) + .withResponseHandler(responseHandler); + + try (S3MetaRequest s3MetaRequest = crtS3Client.makeMetaRequest(requestOptions)) { + closeResourcesWhenComplete(executeFuture, s3MetaRequest); + } + + return executeFuture; + } + + @Override + public String clientName() { + return "s3crt"; + } + + private static S3MetaRequestOptions.MetaRequestType requestType(AsyncExecuteRequest request) { + String operationName = request.httpExecutionAttributes().getAttribute(OPERATION_NAME); + if (operationName != null) { + switch (operationName) { + case "GetObject": + return S3MetaRequestOptions.MetaRequestType.GET_OBJECT; + case "PutObject": + return S3MetaRequestOptions.MetaRequestType.PUT_OBJECT; + default: + return S3MetaRequestOptions.MetaRequestType.DEFAULT; + } + } + return S3MetaRequestOptions.MetaRequestType.DEFAULT; + } + + private static void closeResourcesWhenComplete(CompletableFuture executeFuture, + S3MetaRequest s3MetaRequest) { + executeFuture.whenComplete((r, t) -> { + if (executeFuture.isCancelled()) { + s3MetaRequest.cancel(); + } + + s3MetaRequest.close(); + }); + } + + private static HttpRequest toCrtRequest(AsyncExecuteRequest asyncRequest) { + URI uri = asyncRequest.request().getUri(); + SdkHttpRequest sdkRequest = asyncRequest.request(); + + String method = sdkRequest.method().name(); + String encodedPath = sdkRequest.encodedPath(); + if (encodedPath == null || encodedPath.isEmpty()) { + encodedPath = "/"; + } + + String encodedQueryString = SdkHttpUtils.encodeAndFlattenQueryParameters(sdkRequest.rawQueryParameters()) + .map(value -> "?" + value) + .orElse(""); + + HttpHeader[] crtHeaderArray = createHttpHeaderList(uri, asyncRequest).toArray(new HttpHeader[0]); + + S3CrtRequestBodyStreamAdapter sdkToCrtRequestPublisher = + new S3CrtRequestBodyStreamAdapter(asyncRequest.requestContentPublisher()); + + return new HttpRequest(method, encodedPath + encodedQueryString, crtHeaderArray, sdkToCrtRequestPublisher); + } + + @Override + public void close() { + s3NativeClientConfiguration.close(); + crtS3Client.close(); + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder implements SdkAsyncHttpClient.Builder { + private AwsCredentialsProvider credentialsProvider; + private Region region; + private Long minimalPartSizeInBytes; + private Double targetThroughputInGbps; + private Integer maxConcurrency; + + /** + * Configure the credentials that should be used to authenticate with S3. + */ + public Builder credentialsProvider(AwsCredentialsProvider credentialsProvider) { + this.credentialsProvider = credentialsProvider; + return this; + } + + /** + * Configure the region with which the SDK should communicate. + */ + public Builder region(Region region) { + this.region = region; + return this; + } + + /** + * Sets the minimum part size for transfer parts. Decreasing the minimum part size causes + * multipart transfer to be split into a larger number of smaller parts. Setting this value too low + * has a negative effect on transfer speeds, causing extra latency and network communication for each part. + */ + public Builder minimumPartSizeInBytes(Long partSizeBytes) { + this.minimalPartSizeInBytes = partSizeBytes; + return this; + } + + /** + * The target throughput for transfer requests. Higher value means more S3 connections + * will be opened. Whether the transfer manager can achieve the configured target throughput depends + * on various factors such as the network bandwidth of the environment and the configured {@link #maxConcurrency}. + */ + public Builder targetThroughputInGbps(Double targetThroughputInGbps) { + this.targetThroughputInGbps = targetThroughputInGbps; + return this; + } + + /** + * Specifies the maximum number of S3 connections that should be established during + * a transfer. + * + *

+ * If not provided, the TransferManager will calculate the optional number of connections + * based on {@link #targetThroughputInGbps}. If the value is too low, the S3TransferManager + * might not achieve the specified target throughput. + * + * @param maxConcurrency the max number of concurrent requests + * @return this builder for method chaining. + * @see #targetThroughputInGbps(Double) + */ + public Builder maxConcurrency(Integer maxConcurrency) { + this.maxConcurrency = maxConcurrency; + return this; + } + + @Override + public SdkAsyncHttpClient build() { + return new S3CrtAsyncHttpClient(this); + } + + @Override + public SdkAsyncHttpClient buildWithDefaults(AttributeMap serviceDefaults) { + // Intentionally ignore serviceDefaults + return build(); + } + } + + private static List createHttpHeaderList(URI uri, AsyncExecuteRequest asyncRequest) { + SdkHttpRequest sdkRequest = asyncRequest.request(); + List crtHeaderList = new ArrayList<>(); + + // Set Host Header if needed + if (isNullOrEmpty(sdkRequest.headers().get(Header.HOST))) { + crtHeaderList.add(new HttpHeader(Header.HOST, uri.getHost())); + } + + // Set Content-Length if needed + Optional contentLength = asyncRequest.requestContentPublisher().contentLength(); + if (isNullOrEmpty(sdkRequest.headers().get(Header.CONTENT_LENGTH)) && contentLength.isPresent()) { + crtHeaderList.add(new HttpHeader(Header.CONTENT_LENGTH, Long.toString(contentLength.get()))); + } + + // Add the rest of the Headers + sdkRequest.headers().forEach((key, value) -> value.stream().map(val -> new HttpHeader(key, val)) + .forEach(crtHeaderList::add)); + + return crtHeaderList; + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversion.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversion.java deleted file mode 100644 index 6e2fab6991a5..000000000000 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversion.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.transfer.s3.internal; - -import com.amazonaws.s3.model.GetObjectOutput; -import com.amazonaws.s3.model.ObjectCannedACL; -import com.amazonaws.s3.model.ObjectLockLegalHoldStatus; -import com.amazonaws.s3.model.ObjectLockMode; -import com.amazonaws.s3.model.PutObjectOutput; -import com.amazonaws.s3.model.RequestPayer; -import com.amazonaws.s3.model.ServerSideEncryption; -import com.amazonaws.s3.model.StorageClass; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; -import software.amazon.awssdk.awscore.DefaultAwsResponseMetadata; -import software.amazon.awssdk.core.util.SdkUserAgent; -import software.amazon.awssdk.crt.http.HttpHeader; -import software.amazon.awssdk.http.SdkHttpResponse; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.PutObjectResponse; -import software.amazon.awssdk.services.s3.model.S3ResponseMetadata; -import software.amazon.awssdk.utils.http.SdkHttpUtils; - -/** - * Helper class to convert CRT POJOs to SDK POJOs and vice versa - */ -//TODO: codegen this class in the future -@SdkInternalApi -public final class S3CrtPojoConversion { - private static final String HEADER_USER_AGENT = "User-Agent"; - private static final String USER_AGENT_STRING = SdkUserAgent.create().userAgent() + " ft/s3-transfer"; - - private S3CrtPojoConversion() { - } - - public static com.amazonaws.s3.model.GetObjectRequest toCrtGetObjectRequest(GetObjectRequest request) { - com.amazonaws.s3.model.GetObjectRequest.Builder getObjectBuilder = - com.amazonaws.s3.model.GetObjectRequest.builder() - .bucket(request.bucket()) - .ifMatch(request.ifMatch()) - .ifModifiedSince(request.ifModifiedSince()) - .ifNoneMatch(request.ifNoneMatch()) - .ifUnmodifiedSince(request.ifUnmodifiedSince()) - .key(request.key()) - .range(request.range()) - .responseCacheControl(request.responseCacheControl()) - .responseContentDisposition(request.responseContentDisposition()) - .responseContentEncoding(request.responseContentEncoding()) - .responseContentLanguage(request.responseContentLanguage()) - .responseContentType(request.responseContentType()) - .responseExpires(request.responseExpires()) - .versionId(request.versionId()) - .sSECustomerAlgorithm(request.sseCustomerAlgorithm()) - .sSECustomerKey(request.sseCustomerKey()) - .sSECustomerKeyMD5(request.sseCustomerKeyMD5()) - .requestPayer(RequestPayer.fromValue(request.requestPayerAsString())) - .partNumber(request.partNumber()) - .expectedBucketOwner(request.expectedBucketOwner()); - - processRequestOverrideConfiguration(request.overrideConfiguration().orElse(null), - getObjectBuilder::customQueryParameters); - - addCustomHeaders(request.overrideConfiguration().orElse(null), getObjectBuilder::customHeaders); - - return getObjectBuilder.build(); - - } - - public static GetObjectResponse fromCrtGetObjectOutput(GetObjectOutput response, SdkHttpResponse sdkHttpResponse) { - S3ResponseMetadata s3ResponseMetadata = createS3ResponseMetadata(sdkHttpResponse); - - GetObjectResponse.Builder builder = GetObjectResponse.builder() - .deleteMarker(response.deleteMarker()) - .acceptRanges(response.acceptRanges()) - .expiration(response.expiration()) - .restore(response.restore()) - .lastModified(response.lastModified()) - .contentLength(response.contentLength()) - .eTag(response.eTag()) - .missingMeta(response.missingMeta()) - .versionId(response.versionId()) - .cacheControl(response.cacheControl()) - .contentDisposition(response.contentDisposition()) - .contentEncoding(response.contentEncoding()) - .contentLanguage(response.contentLanguage()) - .contentRange(response.contentRange()) - .contentType(response.contentType()) - .expires(response.expires()) - .websiteRedirectLocation(response.websiteRedirectLocation()) - .metadata(response.metadata()) - .sseCustomerAlgorithm(response.sSECustomerAlgorithm()) - .sseCustomerKeyMD5(response.sSECustomerKeyMD5()) - .ssekmsKeyId(response.sSEKMSKeyId()) - .bucketKeyEnabled(response.bucketKeyEnabled()) - .partsCount(response.partsCount()) - .tagCount(response.tagCount()) - .objectLockRetainUntilDate(response.objectLockRetainUntilDate()); - - if (response.serverSideEncryption() != null) { - builder.serverSideEncryption(response.serverSideEncryption().name()); - } - - if (response.storageClass() != null) { - builder.storageClass(response.storageClass().name()); - } - - if (response.requestCharged() != null) { - builder.requestCharged(response.requestCharged().name()); - } - - if (response.replicationStatus() != null) { - builder.replicationStatus(response.replicationStatus().name()); - } - - if (response.objectLockMode() != null) { - builder.objectLockMode(response.objectLockMode().name()); - } - - if (response.objectLockLegalHoldStatus() != null) { - builder.objectLockLegalHoldStatus(response.objectLockLegalHoldStatus().name()); - } - - return (GetObjectResponse) builder.responseMetadata(s3ResponseMetadata) - .sdkHttpResponse(sdkHttpResponse) - .build(); - - } - - public static com.amazonaws.s3.model.PutObjectRequest toCrtPutObjectRequest(PutObjectRequest sdkPutObject) { - com.amazonaws.s3.model.PutObjectRequest.Builder putObjectBuilder = - com.amazonaws.s3.model.PutObjectRequest.builder() - .contentLength(sdkPutObject.contentLength()) - .bucket(sdkPutObject.bucket()) - .key(sdkPutObject.key()) - .bucketKeyEnabled(sdkPutObject.bucketKeyEnabled()) - .cacheControl(sdkPutObject.cacheControl()) - .contentDisposition(sdkPutObject.contentDisposition()) - .contentEncoding(sdkPutObject.contentEncoding()) - .contentLanguage(sdkPutObject.contentLanguage()) - .contentMD5(sdkPutObject.contentMD5()) - .contentType(sdkPutObject.contentType()) - .expectedBucketOwner(sdkPutObject.expectedBucketOwner()) - .expires(sdkPutObject.expires()) - .grantFullControl(sdkPutObject.grantFullControl()) - .grantRead(sdkPutObject.grantRead()) - .grantReadACP(sdkPutObject.grantReadACP()) - .grantWriteACP(sdkPutObject.grantWriteACP()) - .metadata(sdkPutObject.metadata()) - .objectLockRetainUntilDate(sdkPutObject.objectLockRetainUntilDate()) - .sSECustomerAlgorithm(sdkPutObject.sseCustomerAlgorithm()) - .sSECustomerKey(sdkPutObject.sseCustomerKey()) - .sSECustomerKeyMD5(sdkPutObject.sseCustomerKeyMD5()) - .sSEKMSEncryptionContext(sdkPutObject.ssekmsEncryptionContext()) - .sSEKMSKeyId(sdkPutObject.ssekmsKeyId()) - .tagging(sdkPutObject.tagging()) - .websiteRedirectLocation(sdkPutObject.websiteRedirectLocation()); - - if (sdkPutObject.acl() != null) { - putObjectBuilder.aCL(ObjectCannedACL.fromValue(sdkPutObject.acl().name())); - } - - if (sdkPutObject.objectLockLegalHoldStatus() != null) { - putObjectBuilder.objectLockLegalHoldStatus(ObjectLockLegalHoldStatus.fromValue( - sdkPutObject.objectLockLegalHoldStatus().name())); - } - - if (sdkPutObject.objectLockMode() != null) { - putObjectBuilder.objectLockMode(ObjectLockMode.fromValue( - sdkPutObject.objectLockMode().name())); - } - - if (sdkPutObject.requestPayer() != null) { - putObjectBuilder.requestPayer(RequestPayer.fromValue(sdkPutObject.requestPayer().name())); - } - - if (sdkPutObject.serverSideEncryption() != null) { - putObjectBuilder.serverSideEncryption(ServerSideEncryption.fromValue( - sdkPutObject.serverSideEncryption().name())); - } - - if (sdkPutObject.storageClass() != null) { - putObjectBuilder.storageClass(StorageClass.fromValue( - sdkPutObject.storageClass().name())); - } - - processRequestOverrideConfiguration(sdkPutObject.overrideConfiguration().orElse(null), - putObjectBuilder::customQueryParameters); - - addCustomHeaders(sdkPutObject.overrideConfiguration().orElse(null), putObjectBuilder::customHeaders); - - return putObjectBuilder.build(); - } - - public static PutObjectResponse fromCrtPutObjectOutput(PutObjectOutput crtPutObjectOutput, - SdkHttpResponse sdkHttpResponse) { - S3ResponseMetadata s3ResponseMetadata = createS3ResponseMetadata(sdkHttpResponse); - PutObjectResponse.Builder builder = PutObjectResponse.builder() - .bucketKeyEnabled(crtPutObjectOutput.bucketKeyEnabled()) - .eTag(crtPutObjectOutput.eTag()) - .expiration(crtPutObjectOutput.expiration()) - .sseCustomerAlgorithm(crtPutObjectOutput.sSECustomerAlgorithm()) - .sseCustomerKeyMD5(crtPutObjectOutput.sSECustomerKeyMD5()) - .ssekmsEncryptionContext( - crtPutObjectOutput.sSEKMSEncryptionContext()) - .ssekmsKeyId(crtPutObjectOutput.sSEKMSKeyId()) - .versionId(crtPutObjectOutput.versionId()); - - if (crtPutObjectOutput.requestCharged() != null) { - builder.requestCharged(crtPutObjectOutput.requestCharged().name()); - } - - if (crtPutObjectOutput.serverSideEncryption() != null) { - builder.serverSideEncryption(crtPutObjectOutput.serverSideEncryption().name()); - } - - return (PutObjectResponse) builder.responseMetadata(s3ResponseMetadata) - .sdkHttpResponse(sdkHttpResponse) - .build(); - } - - private static S3ResponseMetadata createS3ResponseMetadata(SdkHttpResponse sdkHttpResponse) { - Map metadata = new HashMap<>(); - sdkHttpResponse.headers().forEach((key, value) -> metadata.put(key, value.get(0))); - return S3ResponseMetadata.create(DefaultAwsResponseMetadata.create(metadata)); - } - - private static void throwExceptionForUnsupportedConfigurations(AwsRequestOverrideConfiguration overrideConfiguration) { - if (!overrideConfiguration.metricPublishers().isEmpty()) { - throw new UnsupportedOperationException("Metric publishers are not supported"); - } - - if (overrideConfiguration.signer().isPresent()) { - throw new UnsupportedOperationException("signer is not supported"); - } - - if (!overrideConfiguration.apiNames().isEmpty()) { - throw new UnsupportedOperationException("apiNames is not supported"); - } - - if (overrideConfiguration.apiCallAttemptTimeout().isPresent()) { - throw new UnsupportedOperationException("apiCallAttemptTimeout is not supported"); - } - - if (overrideConfiguration.apiCallTimeout().isPresent()) { - throw new UnsupportedOperationException("apiCallTimeout is not supported"); - } - - if (overrideConfiguration.credentialsProvider().isPresent()) { - throw new UnsupportedOperationException("credentialsProvider is not supported"); - } - } - - private static void addRequestCustomHeaders(List crtHeaders, Map> headers) { - headers.forEach((key, value) -> { - value.stream().map(val -> new HttpHeader(key, val)).forEach(crtHeaders::add); - }); - } - - private static String encodedQueryString(Map> rawQueryParameters) { - return SdkHttpUtils.encodeAndFlattenQueryParameters(rawQueryParameters) - .orElse(""); - } - - private static void processRequestOverrideConfiguration(AwsRequestOverrideConfiguration requestOverrideConfiguration, - Consumer queryParametersConsumer) { - if (requestOverrideConfiguration != null) { - throwExceptionForUnsupportedConfigurations(requestOverrideConfiguration); - - if (!requestOverrideConfiguration.rawQueryParameters().isEmpty()) { - String encodedQueryString = encodedQueryString(requestOverrideConfiguration.rawQueryParameters()); - queryParametersConsumer.accept(encodedQueryString); - } - } - } - - private static void addCustomHeaders(AwsRequestOverrideConfiguration requestOverrideConfiguration, - Consumer headersConsumer) { - - List crtHeaders = new ArrayList<>(); - crtHeaders.add(new HttpHeader(HEADER_USER_AGENT, USER_AGENT_STRING)); - - if (requestOverrideConfiguration != null && !requestOverrideConfiguration.headers().isEmpty()) { - addRequestCustomHeaders(crtHeaders, requestOverrideConfiguration.headers()); - } - - headersConsumer.accept(crtHeaders.toArray(new HttpHeader[0])); - } -} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapter.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtRequestBodyStreamAdapter.java similarity index 89% rename from services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapter.java rename to services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtRequestBodyStreamAdapter.java index 14cbecb4f29b..e0b938ce2cc9 100644 --- a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapter.java +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtRequestBodyStreamAdapter.java @@ -15,12 +15,10 @@ package software.amazon.awssdk.transfer.s3.internal; -import com.amazonaws.s3.RequestDataSupplier; import java.nio.ByteBuffer; import java.util.Deque; import java.util.concurrent.BlockingDeque; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -30,18 +28,16 @@ import org.reactivestreams.Subscriber; import org.reactivestreams.Subscription; import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.crt.CrtRuntimeException; -import software.amazon.awssdk.crt.http.HttpHeader; -import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.crt.http.HttpRequestBodyStream; import software.amazon.awssdk.utils.Logger; /** - * Adapts an SDK {@link software.amazon.awssdk.core.async.AsyncRequestBody} to CRT's {@link RequestDataSupplier}. + * Adapts an SDK {@link software.amazon.awssdk.core.async.AsyncRequestBody} to CRT's {@link HttpRequestBodyStream}. */ @SdkInternalApi -public final class RequestDataSupplierAdapter implements RequestDataSupplier { +public final class S3CrtRequestBodyStreamAdapter implements HttpRequestBodyStream { static final long DEFAULT_REQUEST_SIZE = 8; - private static final Logger LOG = Logger.loggerFor(RequestDataSupplierAdapter.class); + private static final Logger LOG = Logger.loggerFor(S3CrtRequestBodyStreamAdapter.class); private final AtomicReference subscriptionStatus = new AtomicReference<>(SubscriptionStatus.NOT_SUBSCRIBED); @@ -56,25 +52,14 @@ public final class RequestDataSupplierAdapter implements RequestDataSupplier { // ensure that CRT actually ensures consistency across their threads... private Subscriber subscriber; private long pending = 0; - private final ResponseHeadersHandler headersHandler; - public RequestDataSupplierAdapter(Publisher bodyPublisher) { + public S3CrtRequestBodyStreamAdapter(Publisher bodyPublisher) { this.bodyPublisher = bodyPublisher; this.subscriber = createSubscriber(); - this.headersHandler = new ResponseHeadersHandler(); - } - - public CompletableFuture sdkHttpResponseFuture() { - return headersHandler.sdkHttpResponseFuture(); - } - - @Override - public void onResponseHeaders(final int statusCode, final HttpHeader[] headers) { - headersHandler.onResponseHeaders(statusCode, headers); } @Override - public boolean getRequestBytes(ByteBuffer outBuffer) { + public boolean sendRequestBody(ByteBuffer outBuffer) { LOG.trace(() -> "Getting data to fill buffer of size " + outBuffer.remaining()); // Per the spec, onSubscribe is always called before any other @@ -175,20 +160,6 @@ public boolean resetPosition() { return true; } - @Override - public void onException(CrtRuntimeException e) { - if (subscription != null) { - subscription.cancel(); - } - } - - @Override - public void onFinished() { - if (subscription != null) { - subscription.cancel(); - } - } - private Event takeFirstEvent() { try { return eventBuffer.takeFirst(); diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtResponseHandlerAdapter.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtResponseHandlerAdapter.java new file mode 100644 index 000000000000..5aaf3c92cc0d --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3CrtResponseHandlerAdapter.java @@ -0,0 +1,98 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import java.nio.ByteBuffer; +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.SdkTestInternalApi; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.crt.CRT; +import software.amazon.awssdk.crt.http.HttpHeader; +import software.amazon.awssdk.crt.s3.S3MetaRequestResponseHandler; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.http.async.SdkAsyncHttpResponseHandler; + +/** + * Adapts {@link SdkAsyncHttpResponseHandler} to {@link S3MetaRequestResponseHandler}. + */ +@SdkInternalApi +public class S3CrtResponseHandlerAdapter implements S3MetaRequestResponseHandler { + private final CompletableFuture resultFuture; + private final SdkAsyncHttpResponseHandler responseHandler; + private final S3CrtDataPublisher publisher; + private final SdkHttpResponse.Builder respBuilder = SdkHttpResponse.builder(); + + public S3CrtResponseHandlerAdapter(CompletableFuture executeFuture, SdkAsyncHttpResponseHandler responseHandler) { + this(executeFuture, responseHandler, new S3CrtDataPublisher()); + } + + @SdkTestInternalApi + public S3CrtResponseHandlerAdapter(CompletableFuture executeFuture, + SdkAsyncHttpResponseHandler responseHandler, + S3CrtDataPublisher crtDataPublisher) { + this.resultFuture = executeFuture; + this.responseHandler = responseHandler; + this.publisher = crtDataPublisher; + } + + @Override + public void onResponseHeaders(int statusCode, HttpHeader[] headers) { + for (HttpHeader h : headers) { + respBuilder.appendHeader(h.getName(), h.getValue()); + } + + respBuilder.statusCode(statusCode); + responseHandler.onHeaders(respBuilder.build()); + responseHandler.onStream(publisher); + } + + @Override + public int onResponseBody(ByteBuffer bodyBytesIn, long objectRangeStart, long objectRangeEnd) { + publisher.deliverData(bodyBytesIn); + return 0; + } + + @Override + public void onFinished(int crtCode, int responseStatus, byte[] errorPayload) { + if (crtCode != CRT.AWS_CRT_SUCCESS) { + handleError(crtCode, responseStatus, errorPayload); + } else { + resultFuture.complete(null); + publisher.notifyStreamingFinished(); + } + } + + private void handleError(int crtCode, int responseStatus, byte[] errorPayload) { + if (isErrorResponse(responseStatus) && errorPayload != null) { + publisher.deliverData(ByteBuffer.wrap(errorPayload)); + publisher.notifyStreamingFinished(); + resultFuture.complete(null); + } else { + SdkClientException sdkClientException = + SdkClientException.create(String.format("Failed to send the request. CRT error code: %s", + crtCode)); + resultFuture.completeExceptionally(sdkClientException); + + responseHandler.onError(sdkClientException); + publisher.notifyError(sdkClientException); + } + } + + private static boolean isErrorResponse(int responseStatus) { + return responseStatus != 0; + } +} diff --git a/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3InternalSdkHttpExecutionAttribute.java b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3InternalSdkHttpExecutionAttribute.java new file mode 100644 index 000000000000..24f8b857b983 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/S3InternalSdkHttpExecutionAttribute.java @@ -0,0 +1,33 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.http.SdkHttpExecutionAttribute; + +@SdkInternalApi +public final class S3InternalSdkHttpExecutionAttribute extends SdkHttpExecutionAttribute { + + /** + * The key to indicate the name of the operation + */ + public static final S3InternalSdkHttpExecutionAttribute OPERATION_NAME = + new S3InternalSdkHttpExecutionAttribute<>(String.class); + + private S3InternalSdkHttpExecutionAttribute(Class valueClass) { + super(valueClass); + } +} diff --git a/services-custom/s3-transfer-manager/src/main/resources/software/amazon/awssdk/services/s3/execution.interceptors b/services-custom/s3-transfer-manager/src/main/resources/software/amazon/awssdk/services/s3/execution.interceptors new file mode 100644 index 000000000000..a723ef2c71b9 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/main/resources/software/amazon/awssdk/services/s3/execution.interceptors @@ -0,0 +1 @@ +software.amazon.awssdk.transfer.s3.internal.ApplyUserAgentInterceptor \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/ApplyUserAgentInterceptorTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/ApplyUserAgentInterceptorTest.java new file mode 100644 index 000000000000..53fe60c5166e --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/ApplyUserAgentInterceptorTest.java @@ -0,0 +1,65 @@ +/* + * Copyright 2010-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.RequestOverrideConfiguration; +import software.amazon.awssdk.core.SdkField; +import software.amazon.awssdk.core.SdkRequest; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; + +class ApplyUserAgentInterceptorTest { + + private final ApplyUserAgentInterceptor interceptor = new ApplyUserAgentInterceptor(); + + @Test + void s3Request_shouldModifyRequest() { + GetObjectRequest getItemRequest = GetObjectRequest.builder().build(); + SdkRequest sdkRequest = interceptor.modifyRequest(() -> getItemRequest, new ExecutionAttributes()); + + RequestOverrideConfiguration requestOverrideConfiguration = sdkRequest.overrideConfiguration().get(); + assertThat(requestOverrideConfiguration.apiNames().stream().anyMatch(a -> a.name().equals("ft") && a.version().equals( + "s3-transfer"))).isTrue(); + } + + @Test + void otherRequest_shouldThrowAssertionError() { + SdkRequest someOtherRequest = new SdkRequest() { + @Override + public List> sdkFields() { + return null; + } + + @Override + public Optional overrideConfiguration() { + return Optional.empty(); + } + + @Override + public Builder toBuilder() { + return null; + } + }; + assertThatThrownBy(() -> interceptor.modifyRequest(() -> someOtherRequest, new ExecutionAttributes())) + .isInstanceOf(AssertionError.class); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtErrorHandlerTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtErrorHandlerTest.java deleted file mode 100644 index 9d23be5c9781..000000000000 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtErrorHandlerTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.transfer.s3.internal; - -import org.assertj.core.api.Assertions; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; -import software.amazon.awssdk.core.exception.SdkClientException; -import software.amazon.awssdk.core.exception.SdkServiceException; -import software.amazon.awssdk.crt.CrtRuntimeException; -import software.amazon.awssdk.crt.s3.CrtS3RuntimeException; -import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException; -import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; - -import static org.mockito.Mockito.when; - -@RunWith(MockitoJUnitRunner.class) -public class CrtErrorHandlerTest { - - @Mock - private CrtS3RuntimeException mockCrtS3RuntimeException; - - @Test - public void crtS3ExceptionAreTransformed(){ - CrtErrorHandler crtErrorHandler = new CrtErrorHandler(); - when(mockCrtS3RuntimeException.getAwsErrorCode()).thenReturn("BucketAlreadyExists"); - when(mockCrtS3RuntimeException.getAwsErrorMessage()).thenReturn("Bucket Already Exists"); - when(mockCrtS3RuntimeException.getStatusCode()).thenReturn(404); - Exception transformException = crtErrorHandler.transformException(mockCrtS3RuntimeException); - Assertions.assertThat(transformException).isInstanceOf(BucketAlreadyExistsException.class); - Assertions.assertThat(transformException.getMessage()).contains("Bucket Already Exists"); - } - - @Test - public void nonCrtS3ExceptionAreNotTransformed(){ - CrtErrorHandler crtErrorHandler = new CrtErrorHandler(); - Exception transformException = crtErrorHandler.transformException(new CrtRuntimeException("AWS_ERROR")); - Assertions.assertThat(transformException).isInstanceOf(SdkClientException.class); - } - - - @Test - public void crtS3ExceptionAreTransformedWhenExceptionIsInCause(){ - CrtErrorHandler crtErrorHandler = new CrtErrorHandler(); - when(mockCrtS3RuntimeException.getAwsErrorCode()).thenReturn("InvalidObjectState"); - when(mockCrtS3RuntimeException.getAwsErrorMessage()).thenReturn("Invalid Object State"); - when(mockCrtS3RuntimeException.getStatusCode()).thenReturn(404); - final Exception transformException = crtErrorHandler.transformException(new Exception("Some Exception", mockCrtS3RuntimeException)); - - System.out.println("transformException " +transformException); - - Assertions.assertThat(transformException).isInstanceOf(InvalidObjectStateException.class); - Assertions.assertThat(transformException.getMessage()).contains("Invalid Object State"); - Assertions.assertThat(transformException.getCause()).isInstanceOf(CrtS3RuntimeException.class); - } - - @Test - public void nonCrtS3ExceptionAreNotTransformedWhenExceptionIsInCause(){ - CrtErrorHandler crtErrorHandler = new CrtErrorHandler(); - final Exception crtRuntimeException = new Exception("Some Exception", new CrtRuntimeException("AWS_ERROR")); - Exception transformException = crtErrorHandler.transformException( - crtRuntimeException); - Assertions.assertThat(transformException).isNotInstanceOf(CrtRuntimeException.class); - Assertions.assertThat(transformException).isInstanceOf(SdkClientException.class); - Assertions.assertThat(transformException.getMessage()).isEqualTo("Some Exception"); - Assertions.assertThat(transformException.getCause()).isEqualTo(crtRuntimeException); - } - - @Test - public void crtS3ExceptionWithErrorCodeNodeNotInS3Model() { - CrtErrorHandler crtErrorHandler = new CrtErrorHandler(); - when(mockCrtS3RuntimeException.getAwsErrorCode()).thenReturn("NewS3ExceptionFromCrt"); - when(mockCrtS3RuntimeException.getAwsErrorMessage()).thenReturn("New S3 Exception From Crt"); - when(mockCrtS3RuntimeException.getStatusCode()).thenReturn(404); - Exception transformException = crtErrorHandler.transformException(mockCrtS3RuntimeException); - Assertions.assertThat(transformException).isInstanceOf(SdkServiceException.class); - Assertions.assertThat(transformException.getCause()).isEqualTo(mockCrtS3RuntimeException); - Assertions.assertThat(transformException.getMessage()).isEqualTo(mockCrtS3RuntimeException.getMessage()); - Assertions.assertThat(((SdkServiceException)transformException).statusCode()) - .isEqualTo(mockCrtS3RuntimeException.getStatusCode()); - } - - @Test - public void crtS3ExceptionInCauseWithErrorCodeNodeNotInS3Model() { - CrtErrorHandler crtErrorHandler = new CrtErrorHandler(); - when(mockCrtS3RuntimeException.getAwsErrorCode()).thenReturn("NewS3ExceptionFromCrt"); - when(mockCrtS3RuntimeException.getAwsErrorMessage()).thenReturn("New S3 Exception From Crt"); - when(mockCrtS3RuntimeException.getStatusCode()).thenReturn(404); - final Exception crtRuntimeException = new Exception(mockCrtS3RuntimeException); - Exception transformException = crtErrorHandler.transformException(crtRuntimeException); - Assertions.assertThat(transformException).isInstanceOf(SdkServiceException.class); - Assertions.assertThat(transformException.getCause()).isEqualTo(mockCrtS3RuntimeException); - Assertions.assertThat(transformException.getMessage()).isEqualTo(mockCrtS3RuntimeException.getMessage()); - Assertions.assertThat(((SdkServiceException) transformException).statusCode()).isEqualTo(404); - } -} \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtResponseDataConsumerAdapterTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtResponseDataConsumerAdapterTest.java deleted file mode 100644 index 87d2a1181252..000000000000 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/CrtResponseDataConsumerAdapterTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.transfer.s3.internal; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.verify; - -import com.amazonaws.s3.model.GetObjectOutput; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; -import software.amazon.awssdk.core.async.AsyncResponseTransformer; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; - -@RunWith(MockitoJUnitRunner.class) -public class CrtResponseDataConsumerAdapterTest { - private CrtResponseDataConsumerAdapter adapter; - - @Mock - private S3CrtDataPublisher publisher; - - @Mock - private AsyncResponseTransformer transformer; - - @Before - public void setup() { - ResponseHeadersHandler handler = new ResponseHeadersHandler(); - adapter = new CrtResponseDataConsumerAdapter<>(transformer, publisher, handler); - } - - @Test - public void onResponse_noSdkHttpResponse_shouldCreateEmptySdkHttpResponse() { - adapter.onResponse(GetObjectOutput.builder().build()); - ArgumentCaptor captor = ArgumentCaptor.forClass(GetObjectResponse.class); - verify(transformer).onResponse(captor.capture()); - assertThat(captor.getValue().responseMetadata().requestId()).isEqualTo("UNKNOWN"); - assertThat(captor.getValue().sdkHttpResponse()).isNotNull(); - } -} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClientTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClientTest.java index 9e56594d68c9..ad519ff38e52 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClientTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3CrtAsyncClientTest.java @@ -15,147 +15,49 @@ package software.amazon.awssdk.transfer.s3.internal; -import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.Matchers.any; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import com.amazonaws.s3.RequestDataSupplier; -import com.amazonaws.s3.ResponseDataConsumer; -import com.amazonaws.s3.S3NativeClient; -import com.amazonaws.s3.model.GetObjectOutput; -import com.amazonaws.s3.model.GetObjectRequest; -import com.amazonaws.s3.model.PutObjectOutput; -import com.amazonaws.s3.model.PutObjectRequest; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import org.junit.AfterClass; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; -import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.auth.signer.AwsS3V4Signer; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.AsyncResponseTransformer; -import software.amazon.awssdk.core.exception.SdkClientException; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +import software.amazon.awssdk.services.s3.S3AsyncClient; @RunWith(MockitoJUnitRunner.class) public class DefaultS3CrtAsyncClientTest { @Mock - private S3NativeClient mockS3NativeClient; + private SdkAsyncHttpClient mockHttpClient; @Mock - private S3NativeClientConfiguration mockConfiguration; + private S3AsyncClient mockS3AsyncClient; - private S3CrtAsyncClient s3CrtAsyncClient; - - private static ExecutorService executor; - - @BeforeClass - public static void setUp() { - executor = Executors.newSingleThreadExecutor(); - } + private DefaultS3CrtAsyncClient s3CrtAsyncClient; @Before public void methodSetup() { - s3CrtAsyncClient = new DefaultS3CrtAsyncClient(mockConfiguration, - mockS3NativeClient); - when(mockConfiguration.futureCompletionExecutor()).thenReturn(executor); - } - - @AfterClass - public static void cleanUp() { - executor.shutdown(); - } - - @Test - public void getObject_cancels_shouldForwardCancellation() { - CompletableFuture crtFuture = new CompletableFuture<>(); - when(mockS3NativeClient.getObject(any(GetObjectRequest.class), - any(ResponseDataConsumer.class))) - .thenReturn(crtFuture); - - CompletableFuture> future = - s3CrtAsyncClient.getObject(b -> b.bucket("bucket").key("key"), - AsyncResponseTransformer.toBytes()); - - future.cancel(true); - assertThat(crtFuture).isCancelled(); - } - - @Test - public void putObject_cancels_shouldForwardCancellation() { - CompletableFuture crtFuture = new CompletableFuture<>(); - when(mockS3NativeClient.putObject(any(PutObjectRequest.class), - any(RequestDataSupplier.class))) - .thenReturn(crtFuture); - - CompletableFuture future = - s3CrtAsyncClient.putObject(b -> b.bucket("bucket").key("key"), - AsyncRequestBody.empty()); - - future.cancel(true); - assertThat(crtFuture).isCancelled(); - } - - @Test - public void putObject_crtFutureCompletedExceptionally_shouldFail() { - RuntimeException runtimeException = new RuntimeException("test"); - CompletableFuture crtFuture = new CompletableFuture<>(); - crtFuture.completeExceptionally(runtimeException); - when(mockS3NativeClient.putObject(any(PutObjectRequest.class), - any(RequestDataSupplier.class))) - .thenReturn(crtFuture); - - CompletableFuture future = - s3CrtAsyncClient.putObject(b -> b.bucket("bucket").key("key"), - AsyncRequestBody.empty()); - - assertThatThrownBy(() -> future.join()).hasCause(SdkClientException - .create("java.lang.RuntimeException: test", runtimeException)); + s3CrtAsyncClient = new DefaultS3CrtAsyncClient(mockHttpClient, + mockS3AsyncClient); } @Test - public void getObject_crtFutureCompletedExceptionally_shouldFail() { - RuntimeException runtimeException = new RuntimeException("test"); - CompletableFuture crtFuture = new CompletableFuture<>(); - crtFuture.completeExceptionally(runtimeException); - when(mockS3NativeClient.getObject(any(GetObjectRequest.class), - any(ResponseDataConsumer.class))) - .thenReturn(crtFuture); - - CompletableFuture> future = - s3CrtAsyncClient.getObject(b -> b.bucket("bucket").key("key"), - AsyncResponseTransformer.toBytes()); - - assertThatThrownBy(() -> future.join()).hasCause(SdkClientException.create("test", runtimeException)); - } - - @Test - public void putObject_crtFutureCompletedSuccessfully_shouldSucceed() { - CompletableFuture crtFuture = new CompletableFuture<>(); - crtFuture.complete(PutObjectOutput.builder().build()); - when(mockS3NativeClient.putObject(any(PutObjectRequest.class), - any(RequestDataSupplier.class))) - .thenReturn(crtFuture); - - CompletableFuture future = - s3CrtAsyncClient.putObject(b -> b.bucket("bucket").key("key"), - AsyncRequestBody.empty()); + public void requestSignerOverrideProvided_shouldThrowException() { + assertThatThrownBy(() -> s3CrtAsyncClient.getObject(b -> b.bucket("bucket").key("key").overrideConfiguration(o -> o.signer(AwsS3V4Signer.create())), + AsyncResponseTransformer.toBytes())).isInstanceOf(UnsupportedOperationException.class); - assertThat(future.join().sdkHttpResponse().statusText()).isEmpty(); + assertThatThrownBy(() -> s3CrtAsyncClient.putObject(b -> b.bucket("bucket").key("key").overrideConfiguration(o -> o.signer(AwsS3V4Signer.create())), + AsyncRequestBody.fromString("foobar"))).isInstanceOf(UnsupportedOperationException.class); } @Test public void closeS3Client_shouldCloseUnderlyingResources() { s3CrtAsyncClient.close(); - verify(mockS3NativeClient).close(); - verify(mockConfiguration).close(); + verify(mockHttpClient).close(); + verify(mockS3AsyncClient).close(); } } diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapterTckTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapterTckTest.java index 9f202bde9ad1..e7e9a5418429 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapterTckTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapterTckTest.java @@ -31,7 +31,7 @@ protected RequestDataSupplierAdapterTckTest() { @Override public Subscriber createSubscriber(WhiteboxSubscriberProbe whiteboxSubscriberProbe) { - return new RequestDataSupplierAdapter.SubscriberImpl((s) -> {}, new ArrayDeque<>()) { + return new S3CrtRequestBodyStreamAdapter.SubscriberImpl((s) -> {}, new ArrayDeque<>()) { @Override public void onSubscribe(Subscription subscription) { super.onSubscribe(subscription); diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/ResponseHeadersHandlerTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/ResponseHeadersHandlerTest.java deleted file mode 100644 index fde79558b417..000000000000 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/ResponseHeadersHandlerTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.transfer.s3.internal; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.crt.http.HttpHeader; - -public class ResponseHeadersHandlerTest { - private ResponseHeadersHandler handler; - - @BeforeEach - public void setUp() { - handler = new ResponseHeadersHandler(); - } - - @Test - public void onResponseHeaders_shouldCreateSdkHttpResponse() { - HttpHeader[] headers = new HttpHeader[1]; - headers[0] = new HttpHeader("foo", "bar"); - - handler.onResponseHeaders(400, headers); - assertThat(handler.sdkHttpResponseFuture()).isCompleted(); - Map> actualHeaders - = handler.sdkHttpResponseFuture().join().headers(); - assertThat(actualHeaders).hasSize(1); - assertThat(actualHeaders.get("foo")).containsExactlyInAnyOrder("bar"); - } - - @Test - public void responseNotReady() { - assertThat(handler.sdkHttpResponseFuture()).isNotCompleted(); - } -} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncHttpClientTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncHttpClientTest.java new file mode 100644 index 000000000000..3bc8f16b34d2 --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtAsyncHttpClientTest.java @@ -0,0 +1,164 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static software.amazon.awssdk.http.Header.CONTENT_LENGTH; +import static software.amazon.awssdk.transfer.s3.internal.S3InternalSdkHttpExecutionAttribute.OPERATION_NAME; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; +import software.amazon.awssdk.crt.http.HttpRequest; +import software.amazon.awssdk.crt.s3.S3Client; +import software.amazon.awssdk.crt.s3.S3MetaRequest; +import software.amazon.awssdk.crt.s3.S3MetaRequestOptions; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.async.AsyncExecuteRequest; +import software.amazon.awssdk.http.async.SdkAsyncHttpResponseHandler; +import software.amazon.awssdk.http.async.SdkHttpContentPublisher; + +@RunWith(MockitoJUnitRunner.class) +public class S3CrtAsyncHttpClientTest { + private S3CrtAsyncHttpClient asyncHttpClient; + + @Mock + private S3Client s3Client; + + @Mock + private S3NativeClientConfiguration mockConfiguration; + + @Mock + private SdkAsyncHttpResponseHandler responseHandler; + + @Mock + private SdkHttpContentPublisher contentPublisher; + + @Before + public void methodSetup() { + asyncHttpClient = new S3CrtAsyncHttpClient(s3Client, mockConfiguration); + } + + @Test + public void defaultRequest_shouldSetMetaRequestOptionsCorrectly() { + AsyncExecuteRequest asyncExecuteRequest = getExecuteRequestBuilder().build(); + + ArgumentCaptor s3MetaRequestOptionsArgumentCaptor = + ArgumentCaptor.forClass(S3MetaRequestOptions.class); + + asyncHttpClient.execute(asyncExecuteRequest); + + verify(s3Client).makeMetaRequest(s3MetaRequestOptionsArgumentCaptor.capture()); + + S3MetaRequestOptions actual = s3MetaRequestOptionsArgumentCaptor.getValue(); + assertThat(actual.getMetaRequestType()).isEqualTo(S3MetaRequestOptions.MetaRequestType.DEFAULT); + assertThat(actual.getCredentialsProvider()).isNull(); + + HttpRequest httpRequest = actual.getHttpRequest(); + assertThat(httpRequest.getEncodedPath()).isEqualTo("/key"); + + Map headers = httpRequest.getHeaders() + .stream() + .collect(HashMap::new, (m, h) -> m.put(h.getName(), h.getValue()) + , Map::putAll); + + assertThat(headers).hasSize(4) + .containsEntry("Host", "127.0.0.1") + .containsEntry("custom-header", "foobar") + .containsEntry("amz-sdk-invocation-id", "1234") + .containsEntry("Content-Length", "100"); + } + + @Test + public void getObject_shouldSetMetaRequestTypeCorrectly() { + AsyncExecuteRequest asyncExecuteRequest = getExecuteRequestBuilder().putHttpExecutionAttribute(OPERATION_NAME, + "GetObject").build(); + + ArgumentCaptor s3MetaRequestOptionsArgumentCaptor = + ArgumentCaptor.forClass(S3MetaRequestOptions.class); + + asyncHttpClient.execute(asyncExecuteRequest); + + verify(s3Client).makeMetaRequest(s3MetaRequestOptionsArgumentCaptor.capture()); + + S3MetaRequestOptions actual = s3MetaRequestOptionsArgumentCaptor.getValue(); + assertThat(actual.getMetaRequestType()).isEqualTo(S3MetaRequestOptions.MetaRequestType.GET_OBJECT); + } + + @Test + public void putObject_shouldSetMetaRequestTypeCorrectly() { + AsyncExecuteRequest asyncExecuteRequest = getExecuteRequestBuilder().putHttpExecutionAttribute(OPERATION_NAME, + "PutObject").build(); + + ArgumentCaptor s3MetaRequestOptionsArgumentCaptor = + ArgumentCaptor.forClass(S3MetaRequestOptions.class); + + asyncHttpClient.execute(asyncExecuteRequest); + + verify(s3Client).makeMetaRequest(s3MetaRequestOptionsArgumentCaptor.capture()); + + S3MetaRequestOptions actual = s3MetaRequestOptionsArgumentCaptor.getValue(); + assertThat(actual.getMetaRequestType()).isEqualTo(S3MetaRequestOptions.MetaRequestType.PUT_OBJECT); + } + + @Test + public void cancelRequest_shouldForwardCancellation() { + AsyncExecuteRequest asyncExecuteRequest = getExecuteRequestBuilder().build(); + S3MetaRequest metaRequest = Mockito.mock(S3MetaRequest.class); + when(s3Client.makeMetaRequest(any(S3MetaRequestOptions.class))).thenReturn(metaRequest); + + CompletableFuture future = asyncHttpClient.execute(asyncExecuteRequest); + + future.cancel(false); + + verify(metaRequest).cancel(); + } + + @Test + public void closeHttpClient_shouldCloseUnderlyingResources() { + asyncHttpClient.close(); + verify(s3Client).close(); + verify(mockConfiguration).close(); + } + + private AsyncExecuteRequest.Builder getExecuteRequestBuilder() { + return AsyncExecuteRequest.builder() + .responseHandler(responseHandler) + .requestContentPublisher(contentPublisher) + .request(SdkHttpRequest.builder() + .protocol("https") + .method(SdkHttpMethod.GET) + .host("127.0.0.1") + .port(443) + .encodedPath("/key") + .putHeader(CONTENT_LENGTH, "100") + .putHeader("amz-sdk-invocation-id", + "1234") + .putHeader("custom-header", "foobar") + .build()); + } +} diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversionTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversionTest.java deleted file mode 100644 index c690a74080e9..000000000000 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtPojoConversionTest.java +++ /dev/null @@ -1,467 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.transfer.s3.internal; - - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import com.amazonaws.s3.model.GetObjectOutput; -import com.amazonaws.s3.model.PutObjectOutput; -import com.amazonaws.s3.model.ReplicationStatus; -import java.lang.reflect.Field; -import java.time.Duration; -import java.time.Instant; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; -import org.apache.commons.lang3.RandomStringUtils; -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.signer.AwsS3V4Signer; -import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; -import software.amazon.awssdk.core.ApiName; -import software.amazon.awssdk.core.SdkField; -import software.amazon.awssdk.core.util.SdkUserAgent; -import software.amazon.awssdk.crt.http.HttpHeader; -import software.amazon.awssdk.http.SdkHttpResponse; -import software.amazon.awssdk.metrics.LoggingMetricPublisher; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.ObjectCannedACL; -import software.amazon.awssdk.services.s3.model.ObjectLockLegalHoldStatus; -import software.amazon.awssdk.services.s3.model.ObjectLockMode; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.PutObjectResponse; -import software.amazon.awssdk.services.s3.model.RequestPayer; -import software.amazon.awssdk.services.s3.model.ServerSideEncryption; -import software.amazon.awssdk.services.s3.model.StorageClass; -import software.amazon.awssdk.utils.Logger; - -public class S3CrtPojoConversionTest { - private static final Logger log = Logger.loggerFor(S3CrtPojoConversionTest.class); - private static final Random RNG = new Random(); - - @Test - public void fromCrtPutObjectOutputAllFields_shouldConvert() throws IllegalAccessException { - - PutObjectOutput crtResponse = randomCrtPutObjectOutput(); - SdkHttpResponse sdkHttpResponse = SdkHttpResponse.builder() - .build(); - PutObjectResponse sdkResponse = S3CrtPojoConversion.fromCrtPutObjectOutput(crtResponse, sdkHttpResponse); - - // ignoring fields with different casings and enum fields. - assertThat(sdkResponse).usingRecursiveComparison().ignoringFields( "sseCustomerAlgorithm", - "sseCustomerKeyMD5", - "ssekmsKeyId", - "ssekmsEncryptionContext", - "serverSideEncryption", - "requestCharged", - "responseMetadata", - "sdkHttpResponse").isEqualTo(crtResponse); - assertThat(sdkResponse.serverSideEncryption().name()).isEqualTo(crtResponse.serverSideEncryption().name()); - assertThat(sdkResponse.sseCustomerAlgorithm()).isEqualTo(crtResponse.sSECustomerAlgorithm()); - assertThat(sdkResponse.ssekmsKeyId()).isEqualTo(crtResponse.sSEKMSKeyId()); - assertThat(sdkResponse.sseCustomerKeyMD5()).isEqualTo(crtResponse.sSECustomerKeyMD5()); - assertThat(sdkResponse.ssekmsEncryptionContext()).isEqualTo(crtResponse.sSEKMSEncryptionContext()); - - // TODO: CRT enums dont' have valid values. Uncomment this once it's fixed in CRT. - //assertThat(sdkResponse.requestCharged().name()).isEqualTo(crtResponse.requestCharged().name()); - } - - @Test - public void fromCrtPutObjectOutputAllFields_shouldAddSdkHttpResponse() throws IllegalAccessException { - String expectedRequestId = "123456"; - PutObjectOutput crtResponse = PutObjectOutput.builder().build(); - SdkHttpResponse sdkHttpResponse = SdkHttpResponse.builder() - .statusCode(200) - .appendHeader("x-amz-request-id", expectedRequestId) - .build(); - PutObjectResponse sdkResponse = S3CrtPojoConversion.fromCrtPutObjectOutput(crtResponse, sdkHttpResponse); - - // ignoring fields with different casing and enum fields. - assertThat(sdkResponse).isEqualToIgnoringGivenFields(crtResponse, - "sseCustomerAlgorithm", - "sseCustomerKeyMD5", - "ssekmsKeyId", - "ssekmsEncryptionContext", - "serverSideEncryption", - "requestCharged", - "responseMetadata", - "sdkHttpResponse"); - assertThat(sdkResponse.sdkHttpResponse()).isEqualTo(sdkHttpResponse); - assertThat(sdkResponse.responseMetadata().requestId()).isEqualTo(expectedRequestId); - } - - @Test - public void fromCrtGetObjectOutput_shouldAddSdkHttpResponse() { - String expectedRequestId = "123456"; - GetObjectOutput output = GetObjectOutput.builder().build(); - SdkHttpResponse response = SdkHttpResponse.builder() - .statusCode(200) - .appendHeader("x-amz-request-id", expectedRequestId) - .build(); - - - GetObjectResponse getObjectResponse = S3CrtPojoConversion.fromCrtGetObjectOutput(output, response); - assertThat(output).usingRecursiveComparison().ignoringFields("body", - "sSECustomerAlgorithm", - "sSECustomerKeyMD5", - "sSEKMSKeyId", - "metadata").isEqualTo(getObjectResponse); - - assertThat(getObjectResponse.sdkHttpResponse()).isEqualTo(response); - assertThat(getObjectResponse.responseMetadata().requestId()).isEqualTo(expectedRequestId); - - } - - @Test - public void fromCrtGetObjectOutputAllFields_shouldConvert() throws IllegalAccessException { - GetObjectOutput crtResponse = randomCrtGetObjectOutput(); - GetObjectResponse sdkResponse = S3CrtPojoConversion.fromCrtGetObjectOutput(crtResponse, SdkHttpResponse.builder().build()); - - // ignoring fields with different casings and enum fields. - assertThat(sdkResponse).usingRecursiveComparison().ignoringFields("sseCustomerAlgorithm", - "body", - "sseCustomerKeyMD5", - "ssekmsKeyId", - "ssekmsEncryptionContext", - "serverSideEncryption", - "responseMetadata", - "sdkHttpResponse", - "storageClass", - "requestCharged", - "replicationStatus", - "objectLockMode", - "objectLockLegalHoldStatus").isEqualTo(crtResponse); - assertThat(sdkResponse.serverSideEncryption().name()).isEqualTo(crtResponse.serverSideEncryption().name()); - assertThat(sdkResponse.sseCustomerAlgorithm()).isEqualTo(crtResponse.sSECustomerAlgorithm()); - assertThat(sdkResponse.ssekmsKeyId()).isEqualTo(crtResponse.sSEKMSKeyId()); - assertThat(sdkResponse.sseCustomerKeyMD5()).isEqualTo(crtResponse.sSECustomerKeyMD5()); - assertThat(sdkResponse.storageClass().name()).isEqualTo(crtResponse.storageClass().name()); - assertThat(sdkResponse.replicationStatus().name()).isEqualTo(crtResponse.replicationStatus().name()); - assertThat(sdkResponse.objectLockMode().name()).isEqualTo(crtResponse.objectLockMode().name()); - assertThat(sdkResponse.objectLockLegalHoldStatus().name()).isEqualTo(crtResponse.objectLockLegalHoldStatus().name()); - - // TODO: CRT enums dont' have valid values. Uncomment this once it's fixed in CRT. - // assertThat(sdkResponse.requestCharged().name()).isEqualTo(crtResponse.requestCharged().name()); - } - - @Test - public void toCrtPutObjectRequest_shouldAddUserAgent() { - - PutObjectRequest sdkRequest = PutObjectRequest.builder() - .build(); - - com.amazonaws.s3.model.PutObjectRequest crtRequest = S3CrtPojoConversion.toCrtPutObjectRequest(sdkRequest); - HttpHeader[] headers = crtRequest.customHeaders(); - verifyUserAgent(headers); - } - - @Test - public void toCrtPutObjectRequestAllFields_shouldConvert() { - PutObjectRequest sdkRequest = randomPutObjectRequest(); - - com.amazonaws.s3.model.PutObjectRequest crtRequest = S3CrtPojoConversion.toCrtPutObjectRequest(sdkRequest); - - // ignoring fields with different casings and enum fields. - assertThat(crtRequest).usingRecursiveComparison().ignoringFields("aCL", "body", "sSECustomerAlgorithm", - "sSECustomerKey", "sSECustomerKeyMD5", - "sSEKMSKeyId", "sSEKMSEncryptionContext", - "customHeaders", "customQueryParameters", - "serverSideEncryption", - "storageClass", - "requestPayer", - "objectLockMode", - "objectLockLegalHoldStatus").isEqualTo(sdkRequest); - assertThat(crtRequest.aCL().name()).isEqualTo(sdkRequest.acl().name()); - assertThat(crtRequest.serverSideEncryption().name()).isEqualTo(sdkRequest.serverSideEncryption().name()); - assertThat(crtRequest.storageClass().name()).isEqualTo(sdkRequest.storageClass().name()); - assertThat(crtRequest.requestPayer().name()).isEqualTo(sdkRequest.requestPayer().name()); - assertThat(crtRequest.objectLockMode().name()).isEqualTo(sdkRequest.objectLockMode().name()); - assertThat(crtRequest.objectLockLegalHoldStatus().name()).isEqualTo(sdkRequest.objectLockLegalHoldStatus().name()); - - assertThat(crtRequest.sSECustomerAlgorithm()).isEqualTo(sdkRequest.sseCustomerAlgorithm()); - assertThat(crtRequest.sSECustomerKey()).isEqualTo(sdkRequest.sseCustomerKey()); - assertThat(crtRequest.sSECustomerKeyMD5()).isEqualTo(sdkRequest.sseCustomerKeyMD5()); - assertThat(crtRequest.sSEKMSKeyId()).isEqualTo(sdkRequest.ssekmsKeyId()); - assertThat(crtRequest.sSEKMSEncryptionContext()).isEqualTo(sdkRequest.ssekmsEncryptionContext()); - assertThat(crtRequest.sSECustomerAlgorithm()).isEqualTo(sdkRequest.sseCustomerAlgorithm()); - } - - @Test - public void toCrtPutObjectRequest_withCustomHeaders_shouldAttach() { - - AwsRequestOverrideConfiguration requestOverrideConfiguration = requestOverrideConfigWithCustomHeaders(); - - PutObjectRequest sdkRequest = PutObjectRequest.builder() - .overrideConfiguration(requestOverrideConfiguration) - .build(); - - com.amazonaws.s3.model.PutObjectRequest crtRequest = S3CrtPojoConversion.toCrtPutObjectRequest(sdkRequest); - HttpHeader[] headers = crtRequest.customHeaders(); - verifyHeaders(headers); - assertThat(crtRequest.customQueryParameters()).isEqualTo("hello1=world1&hello2=world2"); - } - - @Test - public void toCrtGetObjectRequest_shouldAddUserAgent() { - GetObjectRequest sdkRequest = GetObjectRequest.builder() - .build(); - - com.amazonaws.s3.model.GetObjectRequest crtRequest = S3CrtPojoConversion.toCrtGetObjectRequest(sdkRequest); - - HttpHeader[] headers = crtRequest.customHeaders(); - verifyUserAgent(headers); - } - - @Test - public void toCrtGetObjectRequestAllFields_shouldConvert() { - GetObjectRequest sdkRequest = randomGetObjectRequest(); - - com.amazonaws.s3.model.GetObjectRequest crtRequest = S3CrtPojoConversion.toCrtGetObjectRequest(sdkRequest); - - // ignoring fields with different casings and enum fields. - assertThat(crtRequest).usingRecursiveComparison().ignoringFields("body", "sSECustomerAlgorithm", - "sSECustomerKey", "sSECustomerKeyMD5", - "customHeaders", "customQueryParameters", - "requestPayer").isEqualTo(sdkRequest); - assertThat(crtRequest.requestPayer().name()).isEqualTo(sdkRequest.requestPayer().name()); - assertThat(crtRequest.sSECustomerAlgorithm()).isEqualTo(sdkRequest.sseCustomerAlgorithm()); - assertThat(crtRequest.sSECustomerKey()).isEqualTo(sdkRequest.sseCustomerKey()); - assertThat(crtRequest.sSECustomerKeyMD5()).isEqualTo(sdkRequest.sseCustomerKeyMD5()); - assertThat(crtRequest.sSECustomerAlgorithm()).isEqualTo(sdkRequest.sseCustomerAlgorithm()); - } - - @Test - public void toCrtGetObjectRequest_withCustomHeaders_shouldAttach() { - AwsRequestOverrideConfiguration requestOverrideConfiguration = requestOverrideConfigWithCustomHeaders(); - - GetObjectRequest sdkRequest = GetObjectRequest.builder() - .overrideConfiguration(requestOverrideConfiguration) - .build(); - - com.amazonaws.s3.model.GetObjectRequest crtRequest = S3CrtPojoConversion.toCrtGetObjectRequest(sdkRequest); - - HttpHeader[] headers = crtRequest.customHeaders(); - verifyHeaders(headers); - assertThat(crtRequest.customQueryParameters()).isEqualTo("hello1=world1&hello2=world2"); - } - - @Test - public void toCrtPutObjectRequest_withUnsupportedConfigs_shouldThrowException() { - assertThatThrownBy(() -> S3CrtPojoConversion.toCrtPutObjectRequest(PutObjectRequest.builder() - .overrideConfiguration(b -> b.apiCallAttemptTimeout(Duration.ofMinutes(1))) - .build())).isExactlyInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> S3CrtPojoConversion.toCrtPutObjectRequest(PutObjectRequest.builder() - .overrideConfiguration(b -> b.apiCallTimeout(Duration.ofMinutes(1))) - .build())).isExactlyInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> S3CrtPojoConversion.toCrtPutObjectRequest(PutObjectRequest.builder() - .overrideConfiguration(b -> b.credentialsProvider(() -> - AwsBasicCredentials.create("", ""))) - .build())).isExactlyInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> S3CrtPojoConversion.toCrtPutObjectRequest(PutObjectRequest.builder() - .overrideConfiguration(b -> b.addApiName(ApiName.builder() - .name("test") - .version("1") - .build())) - .build())).isExactlyInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> S3CrtPojoConversion.toCrtPutObjectRequest(PutObjectRequest.builder() - .overrideConfiguration(b -> b.addMetricPublisher(LoggingMetricPublisher.create())) - .build())).isExactlyInstanceOf(UnsupportedOperationException.class); - - assertThatThrownBy(() -> S3CrtPojoConversion.toCrtPutObjectRequest(PutObjectRequest.builder() - .overrideConfiguration(b -> b.signer(AwsS3V4Signer.create())) - .build())).isExactlyInstanceOf(UnsupportedOperationException.class); - } - - @Test - public void toCrtGetObjectRequest_withUnsupportedConfigs_shouldThrowException() { - assertThatThrownBy(() -> S3CrtPojoConversion.toCrtGetObjectRequest(GetObjectRequest.builder() - .overrideConfiguration(b -> b.apiCallAttemptTimeout(Duration.ofMinutes(1))) - .build())).isExactlyInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> S3CrtPojoConversion.toCrtGetObjectRequest(GetObjectRequest.builder() - .overrideConfiguration(b -> b.apiCallTimeout(Duration.ofMinutes(1))) - .build())).isExactlyInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> S3CrtPojoConversion.toCrtGetObjectRequest(GetObjectRequest.builder() - .overrideConfiguration(b -> b.credentialsProvider(() -> - AwsBasicCredentials.create("", ""))) - .build())).isExactlyInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> S3CrtPojoConversion.toCrtGetObjectRequest(GetObjectRequest.builder() - .overrideConfiguration(b -> b.addApiName(ApiName.builder() - .name("test") - .version("1") - .build())) - .build())).isExactlyInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> S3CrtPojoConversion.toCrtGetObjectRequest(GetObjectRequest.builder() - .overrideConfiguration(b -> b.addMetricPublisher(LoggingMetricPublisher.create())) - .build())).isExactlyInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> S3CrtPojoConversion.toCrtGetObjectRequest(GetObjectRequest.builder() - .overrideConfiguration(b -> b.signer(AwsS3V4Signer.create())) - .build())).isExactlyInstanceOf(UnsupportedOperationException.class); - } - - private AwsRequestOverrideConfiguration requestOverrideConfigWithCustomHeaders() { - return AwsRequestOverrideConfiguration.builder() - .putHeader("foo", "bar") - .putRawQueryParameter("hello1", "world1") - .putRawQueryParameter("hello2", "world2") - .build(); - } - - private void verifyHeaders(HttpHeader[] headers) { - assertThat(headers).hasSize(2); - verifyUserAgent(headers); - assertThat(headers[1].getName()).isEqualTo("foo"); - assertThat(headers[1].getValue()).isEqualTo("bar"); - } - - private void verifyUserAgent(HttpHeader[] headers) { - assertThat(headers[0].getName()).isEqualTo("User-Agent"); - assertThat(headers[0].getValue()).contains("ft/s3-transfer"); - assertThat(headers[0].getValue()).contains(SdkUserAgent.create().userAgent()); - } - - private GetObjectRequest randomGetObjectRequest() { - GetObjectRequest.Builder builder = GetObjectRequest.builder(); - setSdkFieldsToRandomValues(builder.sdkFields(), builder); - return builder.build(); - } - - private PutObjectRequest randomPutObjectRequest() { - PutObjectRequest.Builder builder = PutObjectRequest.builder(); - setSdkFieldsToRandomValues(builder.sdkFields(), builder); - return builder.build(); - } - - - private com.amazonaws.s3.model.GetObjectOutput randomCrtGetObjectOutput() throws IllegalAccessException { - com.amazonaws.s3.model.GetObjectOutput.Builder builder = com.amazonaws.s3.model.GetObjectOutput.builder(); - Class aClass = builder.getClass(); - setFieldsToRandomValues(Arrays.asList(aClass.getDeclaredFields()), builder); - return builder.build(); - } - - private com.amazonaws.s3.model.PutObjectOutput randomCrtPutObjectOutput() throws IllegalAccessException { - com.amazonaws.s3.model.PutObjectOutput.Builder builder = com.amazonaws.s3.model.PutObjectOutput.builder(); - Class aClass = builder.getClass(); - setFieldsToRandomValues(Arrays.asList(aClass.getDeclaredFields()), builder); - return builder.build(); - } - - private void setFieldsToRandomValues(Collection fields, Object builder) throws IllegalAccessException { - for (Field f : fields) { - setFieldToRandomValue(f, builder); - } - } - - private void setFieldToRandomValue(Field field, Object obj) throws IllegalAccessException { - Class targetClass = field.getType(); - field.setAccessible(true); - if (targetClass.equals(String.class)) { - field.set(obj, RandomStringUtils.randomAscii(8)); - } else if (targetClass.equals(Integer.class)) { - field.set(obj, randomInteger()); - } else if (targetClass.equals(Instant.class)) { - field.set(obj, randomInstant()); - } else if (targetClass.equals(Long.class)) { - field.set(obj, RNG.nextLong()); - } else if (targetClass.equals(Map.class)) { - field.set(obj, new HashMap<>()); - } else if (targetClass.equals(Boolean.class)) { - field.set(obj, Boolean.TRUE); - } else if (targetClass.isEnum()) { - if (targetClass.equals(com.amazonaws.s3.model.ServerSideEncryption.class)) { - field.set(obj, com.amazonaws.s3.model.ServerSideEncryption.AES256); - } else if (targetClass.equals(com.amazonaws.s3.model.StorageClass.class)) { - field.set(obj, com.amazonaws.s3.model.StorageClass.GLACIER); - } else if (targetClass.equals(com.amazonaws.s3.model.RequestCharged.class)) { - field.set(obj, com.amazonaws.s3.model.RequestCharged.REQUESTER); - } else if (targetClass.equals(com.amazonaws.s3.model.ReplicationStatus.class)) { - field.set(obj, ReplicationStatus.COMPLETE); - } else if (targetClass.equals(com.amazonaws.s3.model.ObjectLockMode.class)) { - field.set(obj, com.amazonaws.s3.model.ObjectLockMode.GOVERNANCE); - } else if (targetClass.equals(com.amazonaws.s3.model.ObjectLockLegalHoldStatus.class)) { - field.set(obj, com.amazonaws.s3.model.ObjectLockLegalHoldStatus.OFF); - } else { - throw new IllegalArgumentException("Unknown enum: " + field.getName()); - } - } else if (field.getName().equals("body")) { - log.info(() -> "ignore non s3 fields"); - } else if (field.isSynthetic()) { - // ignore jacoco https://github.com/jacoco/jacoco/issues/168 - log.info(() -> "ignore synthetic fields"); - } else { - throw new IllegalArgumentException("Unknown Field type: " + field.getName()); - } - } - - private void setSdkFieldsToRandomValues(Collection> fields, Object builder) { - for (SdkField f : fields) { - setSdkFieldToRandomValue(f, builder); - } - } - - private static void setSdkFieldToRandomValue(SdkField sdkField, Object obj) { - Class targetClass = sdkField.marshallingType().getTargetClass(); - if (targetClass.equals(String.class)) { - switch (sdkField.memberName()) { - case "ACL": - sdkField.set(obj, ObjectCannedACL.PUBLIC_READ.toString()); - break; - case "ServerSideEncryption": - sdkField.set(obj, ServerSideEncryption.AES256.toString()); - break; - case "StorageClass": - sdkField.set(obj, StorageClass.DEEP_ARCHIVE.toString()); - break; - case "RequestPayer": - sdkField.set(obj, RequestPayer.UNKNOWN_TO_SDK_VERSION.toString()); - break; - case "ObjectLockMode": - sdkField.set(obj, ObjectLockMode.COMPLIANCE.toString()); - break; - case "ObjectLockLegalHoldStatus": - sdkField.set(obj, ObjectLockLegalHoldStatus.OFF.toString()); - break; - default: - sdkField.set(obj, RandomStringUtils.random(8)); - } - } else if (targetClass.equals(Integer.class)) { - sdkField.set(obj, randomInteger()); - } else if (targetClass.equals(Instant.class)) { - sdkField.set(obj, randomInstant()); - } else if (targetClass.equals(Long.class)) { - sdkField.set(obj, RNG.nextLong()); - } else if (targetClass.equals(Map.class)) { - sdkField.set(obj, new HashMap<>()); - } else if (targetClass.equals(Boolean.class)) { - sdkField.set(obj, Boolean.TRUE); - } else { - throw new IllegalArgumentException("Unknown SdkField type: " + targetClass); - } - } - - private static Instant randomInstant() { - return Instant.ofEpochMilli(RNG.nextLong()); - } - - private static Integer randomInteger() { - return RNG.nextInt(); - } -} \ No newline at end of file diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapterTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtRequestBodyStreamAdapterTest.java similarity index 52% rename from services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapterTest.java rename to services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtRequestBodyStreamAdapterTest.java index ecd49477593a..fb02c88400e4 100644 --- a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/RequestDataSupplierAdapterTest.java +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtRequestBodyStreamAdapterTest.java @@ -17,46 +17,41 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; import io.reactivex.Flowable; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.reactivestreams.Publisher; import org.reactivestreams.Subscriber; -import org.reactivestreams.Subscription; import software.amazon.awssdk.core.async.AsyncRequestBody; -import software.amazon.awssdk.crt.CrtRuntimeException; -public class RequestDataSupplierAdapterTest { +class S3CrtRequestBodyStreamAdapterTest { @Test - public void getRequestData_fillsInputBuffer_publisherBuffersAreSmaller() { + void getRequestData_fillsInputBuffer_publisherBuffersAreSmaller() { int inputBufferSize = 16; List data = Stream.generate(() -> (byte) 42) - .limit(inputBufferSize) - .map(b -> { - ByteBuffer bb = ByteBuffer.allocate(1); - bb.put(b); - bb.flip(); - return bb; - }) - .collect(Collectors.toList()); + .limit(inputBufferSize) + .map(b -> { + ByteBuffer bb = ByteBuffer.allocate(1); + bb.put(b); + bb.flip(); + return bb; + }) + .collect(Collectors.toList()); AsyncRequestBody requestBody = AsyncRequestBody.fromPublisher(Flowable.fromIterable(data)); - RequestDataSupplierAdapter adapter = new RequestDataSupplierAdapter(requestBody); + S3CrtRequestBodyStreamAdapter adapter = new S3CrtRequestBodyStreamAdapter(requestBody); ByteBuffer inputBuffer = ByteBuffer.allocate(inputBufferSize); - adapter.getRequestBytes(inputBuffer); + adapter.sendRequestBody(inputBuffer); assertThat(inputBuffer.remaining()).isEqualTo(0); } @@ -71,12 +66,12 @@ public void getRequestData_fillsInputBuffer_publisherBuffersAreLarger() { AsyncRequestBody requestBody = AsyncRequestBody.fromPublisher(Flowable.just(data)); - RequestDataSupplierAdapter adapter = new RequestDataSupplierAdapter(requestBody); + S3CrtRequestBodyStreamAdapter adapter = new S3CrtRequestBodyStreamAdapter(requestBody); ByteBuffer inputBuffer = ByteBuffer.allocate(1); for (int i = 0; i < bodySize; ++i) { - adapter.getRequestBytes(inputBuffer); + adapter.sendRequestBody(inputBuffer); assertThat(inputBuffer.remaining()).isEqualTo(0); inputBuffer.flip(); } @@ -87,11 +82,11 @@ public void getRequestData_publisherThrows_surfacesException() { Publisher errorPublisher = Flowable.error(new RuntimeException("Something wrong happened")); AsyncRequestBody requestBody = AsyncRequestBody.fromPublisher(errorPublisher); - RequestDataSupplierAdapter adapter = new RequestDataSupplierAdapter(requestBody); + S3CrtRequestBodyStreamAdapter adapter = new S3CrtRequestBodyStreamAdapter(requestBody); - assertThatThrownBy(() -> adapter.getRequestBytes(ByteBuffer.allocate(16))) - .isInstanceOf(RuntimeException.class) - .hasMessageContaining("Something wrong happened"); + assertThatThrownBy(() -> adapter.sendRequestBody(ByteBuffer.allocate(16))) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("Something wrong happened"); } @Test @@ -99,16 +94,16 @@ public void getRequestData_publisherThrows_wrapsExceptionIfNotRuntimeException() Publisher errorPublisher = Flowable.error(new IOException("Some I/O error happened")); AsyncRequestBody requestBody = AsyncRequestBody.fromPublisher(errorPublisher); - RequestDataSupplierAdapter adapter = new RequestDataSupplierAdapter(requestBody); + S3CrtRequestBodyStreamAdapter adapter = new S3CrtRequestBodyStreamAdapter(requestBody); - assertThatThrownBy(() -> adapter.getRequestBytes(ByteBuffer.allocate(16))) - .isInstanceOf(RuntimeException.class) - .hasCauseInstanceOf(IOException.class); + assertThatThrownBy(() -> adapter.sendRequestBody(ByteBuffer.allocate(16))) + .isInstanceOf(RuntimeException.class) + .hasCauseInstanceOf(IOException.class); } @Test public void resetMidStream_discardsBufferedData() { - long requestSize = RequestDataSupplierAdapter.DEFAULT_REQUEST_SIZE; + long requestSize = S3CrtRequestBodyStreamAdapter.DEFAULT_REQUEST_SIZE; int inputBufferSize = 16; Publisher requestBody = new Publisher() { @@ -119,12 +114,12 @@ public void subscribe(Subscriber subscriber) { byte byteVal = value++; List dataList = Stream.generate(() -> { - byte[] data = new byte[inputBufferSize]; - Arrays.fill(data, byteVal); - return ByteBuffer.wrap(data); - }) - .limit(requestSize) - .collect(Collectors.toList()); + byte[] data = new byte[inputBufferSize]; + Arrays.fill(data, byteVal); + return ByteBuffer.wrap(data); + }) + .limit(requestSize) + .collect(Collectors.toList()); Flowable realPublisher = Flowable.fromIterable(dataList); @@ -132,14 +127,14 @@ public void subscribe(Subscriber subscriber) { } }; - RequestDataSupplierAdapter adapter = new RequestDataSupplierAdapter(requestBody); + S3CrtRequestBodyStreamAdapter adapter = new S3CrtRequestBodyStreamAdapter(requestBody); long resetAfter = requestSize / 2; ByteBuffer inputBuffer = ByteBuffer.allocate(inputBufferSize); for (long l = 0; l < resetAfter; ++l) { - adapter.getRequestBytes(inputBuffer); + adapter.sendRequestBody(inputBuffer); inputBuffer.flip(); } @@ -150,7 +145,7 @@ public void subscribe(Subscriber subscriber) { byte[] readBuffer = new byte[inputBufferSize]; for (int l = 0; l < requestSize; ++l) { - adapter.getRequestBytes(inputBuffer); + adapter.sendRequestBody(inputBuffer); // flip for reading inputBuffer.flip(); inputBuffer.get(readBuffer); @@ -162,55 +157,4 @@ public void subscribe(Subscriber subscriber) { } } - @Test - public void onException_cancelsSubscription() { - Subscription subscription = mock(Subscription.class); - - AsyncRequestBody requestBody = new AsyncRequestBody() { - @Override - public Optional contentLength() { - return Optional.empty(); - } - - @Override - public void subscribe(Subscriber subscriber) { - subscriber.onSubscribe(subscription); - } - }; - - RequestDataSupplierAdapter adapter = new RequestDataSupplierAdapter(requestBody); - - // getRequestBytes() triggers a subscribe() on the publisher - adapter.getRequestBytes(ByteBuffer.allocate(0)); - - adapter.onException(new CrtRuntimeException("error")); - - verify(subscription).cancel(); - } - - @Test - public void onFinished_cancelsSubscription() { - Subscription subscription = mock(Subscription.class); - - AsyncRequestBody requestBody = new AsyncRequestBody() { - @Override - public Optional contentLength() { - return Optional.empty(); - } - - @Override - public void subscribe(Subscriber subscriber) { - subscriber.onSubscribe(subscription); - } - }; - - RequestDataSupplierAdapter adapter = new RequestDataSupplierAdapter(requestBody); - - // getRequestBytes() triggers a subscribe() on the publisher - adapter.getRequestBytes(ByteBuffer.allocate(0)); - - adapter.onFinished(); - - verify(subscription).cancel(); - } } diff --git a/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtResponseHandlerAdapterTest.java b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtResponseHandlerAdapterTest.java new file mode 100644 index 000000000000..d67741d55e5b --- /dev/null +++ b/services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3CrtResponseHandlerAdapterTest.java @@ -0,0 +1,118 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.transfer.s3.internal; + + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CompletableFuture; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.crt.http.HttpHeader; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.http.async.SdkAsyncHttpResponseHandler; + +@RunWith(MockitoJUnitRunner.class) +public class S3CrtResponseHandlerAdapterTest { + private S3CrtResponseHandlerAdapter responseHandlerAdapter; + + @Mock + private SdkAsyncHttpResponseHandler sdkResponseHandler; + + @Mock + private S3CrtDataPublisher crtDataPublisher; + private CompletableFuture future; + + @Before + public void setup() { + future = new CompletableFuture<>(); + responseHandlerAdapter = new S3CrtResponseHandlerAdapter(future, + sdkResponseHandler, + crtDataPublisher); + } + + @Test + public void successfulResponse_shouldCompleteFutureSuccessfully() { + HttpHeader[] httpHeaders = new HttpHeader[2]; + httpHeaders[0] = new HttpHeader("foo", "1"); + httpHeaders[1] = new HttpHeader("bar", "2"); + + int statusCode = 200; + responseHandlerAdapter.onResponseHeaders(statusCode, httpHeaders); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(SdkHttpResponse.class); + verify(sdkResponseHandler).onHeaders(argumentCaptor.capture()); + + SdkHttpResponse actualSdkHttpResponse = argumentCaptor.getValue(); + assertThat(actualSdkHttpResponse.statusCode()).isEqualTo(statusCode); + assertThat(actualSdkHttpResponse.firstMatchingHeader("foo")).contains("1"); + assertThat(actualSdkHttpResponse.firstMatchingHeader("bar")).contains("2"); + + verify(sdkResponseHandler).onStream(crtDataPublisher); + + responseHandlerAdapter.onFinished(0, 0, null); + assertThat(future).isCompleted(); + } + + @Test + public void errorResponse_shouldCompleteFutureSuccessfully() { + int statusCode = 400; + responseHandlerAdapter.onResponseHeaders(statusCode, new HttpHeader[0]); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(SdkHttpResponse.class); + verify(sdkResponseHandler).onHeaders(argumentCaptor.capture()); + + SdkHttpResponse actualSdkHttpResponse = argumentCaptor.getValue(); + assertThat(actualSdkHttpResponse.statusCode()).isEqualTo(400); + assertThat(actualSdkHttpResponse.headers()).isEmpty(); + + verify(sdkResponseHandler).onStream(crtDataPublisher); + + byte[] errorPayload = "errorResponse".getBytes(StandardCharsets.UTF_8); + responseHandlerAdapter.onFinished(1, statusCode, errorPayload); + + ArgumentCaptor byteBufferArgumentCaptor = ArgumentCaptor.forClass(ByteBuffer.class); + verify(crtDataPublisher).deliverData(byteBufferArgumentCaptor.capture()); + + ByteBuffer actualByteBuffer = byteBufferArgumentCaptor.getValue(); + + assertThat(actualByteBuffer).isEqualTo(ByteBuffer.wrap(errorPayload)); + + assertThat(future).isCompleted(); + } + + @Test + public void requestFailed_shouldCompleteFutureExceptionally() { + + responseHandlerAdapter.onFinished(1, 0, null); + + ArgumentCaptor exceptionArgumentCaptor = ArgumentCaptor.forClass(Exception.class); + verify(crtDataPublisher).notifyError(exceptionArgumentCaptor.capture()); + verify(sdkResponseHandler).onError(exceptionArgumentCaptor.capture()); + + Exception actualException = exceptionArgumentCaptor.getValue(); + assertThat(actualException).isInstanceOf(SdkClientException.class); + assertThat(future).isCompletedExceptionally(); + } +}