From 43a1cae1b8fc17c0c28edc3aec4b70aa5d115f46 Mon Sep 17 00:00:00 2001 From: Anna-Karin Salander Date: Fri, 2 Feb 2024 14:01:05 -0800 Subject: [PATCH 1/4] Bump aws-crt version to 0.29.9 --- .../next-release/feature-AWSCRTbasedS3Client-353870f.json | 6 ++++++ pom.xml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/next-release/feature-AWSCRTbasedS3Client-353870f.json diff --git a/.changes/next-release/feature-AWSCRTbasedS3Client-353870f.json b/.changes/next-release/feature-AWSCRTbasedS3Client-353870f.json new file mode 100644 index 000000000000..d27fc271c8ef --- /dev/null +++ b/.changes/next-release/feature-AWSCRTbasedS3Client-353870f.json @@ -0,0 +1,6 @@ +{ + "type": "feature", + "category": "AWS CRT-based S3 Client", + "contributor": "", + "description": "Bump `aws-crt` version to `0.29.9`" +} diff --git a/pom.xml b/pom.xml index 2bcc54a152df..6b9cf0ecbb33 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,7 @@ 2.2.21 1.15 1.29 - 0.29.7 + 0.29.9 5.10.0 From 387853a0bcda409e4991747d2f1f12a922dac9f1 Mon Sep 17 00:00:00 2001 From: Anna-Karin Salander Date: Fri, 2 Feb 2024 14:02:57 -0800 Subject: [PATCH 2/4] Exposes a setting to set the memory limit when making asynchronous calls with the CRT-based S3 client --- .../feature-AmazonS3-c28f9de.json | 6 +++++ .../services/s3/S3CrtAsyncClientBuilder.java | 24 ++++++++++++++++++- .../internal/crt/DefaultS3CrtAsyncClient.java | 10 +++++++- .../s3/internal/crt/S3CrtAsyncHttpClient.java | 1 + .../crt/S3NativeClientConfiguration.java | 14 +++++++++-- .../crt/S3CrtAsyncHttpClientTest.java | 4 ++++ 6 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 .changes/next-release/feature-AmazonS3-c28f9de.json diff --git a/.changes/next-release/feature-AmazonS3-c28f9de.json b/.changes/next-release/feature-AmazonS3-c28f9de.json new file mode 100644 index 000000000000..b31c5ddf51bf --- /dev/null +++ b/.changes/next-release/feature-AmazonS3-c28f9de.json @@ -0,0 +1,6 @@ +{ + "type": "feature", + "category": "Amazon S3", + "contributor": "", + "description": "Exposes a setting to set the memory limit when making asynchronous calls with the CRT-based S3 client" +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java index bd9558e1f87b..1fcbf92734fb 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java @@ -114,12 +114,30 @@ default S3CrtAsyncClientBuilder credentialsProvider(IdentityProvider + * If not provided, the CRT attempts to limit memory usage in an optimal way, based on a number of parameters + * such as target throughput. Therefore, only configure the memory limit explicitly when needed. + *

+ * Supported range: + *

+ * + * @param memoryLimitInBytes the memory limit in bytes + * @return this builder for method chaining. + * @see #targetThroughputInGbps(Double) + */ + S3CrtAsyncClientBuilder memoryLimitInBytes(Long memoryLimitInBytes); + /** * The target throughput for transfer requests. Higher value means more connections will be established with S3. * *

* Whether the transfer manager can achieve the configured target throughput depends on various factors such as the network - * bandwidth of the environment and whether {@link #maxConcurrency} is configured. + * bandwidth of the environment, whether {@link #maxConcurrency} is configured and amount of available memory. * *

* By default, it is 10 gigabits per second. If users want to transfer as fast as possible, it's recommended to set it to the @@ -128,10 +146,14 @@ default S3CrtAsyncClientBuilder credentialsProvider(IdentityProviderAmazon EC2 instance type page. * If you are running into out of file descriptors error, consider using {@link #maxConcurrency(Integer)} to limit the * number of connections. + *

+ * Note: this setting affects the memory usage of CRT; a higher throughput value will result in a larger memory + * usage. Typically, a range of throughput values maps to a discrete memory limit value in CRT, with a maximum upper limit. * * @param targetThroughputInGbps the target throughput in Gbps * @return this builder for method chaining. * @see #maxConcurrency(Integer) + * @see #memoryLimitInBytes(Long) */ S3CrtAsyncClientBuilder targetThroughputInGbps(Double targetThroughputInGbps); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java index 2a8ad361bfb7..c363200dc77c 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java @@ -157,7 +157,8 @@ private static S3CrtAsyncHttpClient.Builder initializeS3CrtAsyncHttpClient(Defau .credentialsProvider(builder.credentialsProvider) .readBufferSizeInBytes(builder.readBufferSizeInBytes) .httpConfiguration(builder.httpConfiguration) - .thresholdInBytes(builder.thresholdInBytes); + .thresholdInBytes(builder.thresholdInBytes) + .memoryLimitInBytes(builder.memoryLimitInBytes); if (builder.retryConfiguration != null) { nativeClientBuilder.standardRetryOptions( @@ -175,6 +176,7 @@ public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientB private Region region; private Long minimalPartSizeInBytes; private Double targetThroughputInGbps; + private Long memoryLimitInBytes; private Integer maxConcurrency; private URI endpointOverride; private Boolean checksumValidationEnabled; @@ -212,6 +214,12 @@ public S3CrtAsyncClientBuilder targetThroughputInGbps(Double targetThroughputInG return this; } + @Override + public S3CrtAsyncClientBuilder memoryLimitInBytes(Long memoryLimitInBytes) { + this.memoryLimitInBytes = memoryLimitInBytes; + return this; + } + @Override public S3CrtAsyncClientBuilder maxConcurrency(Integer maxConcurrency) { this.maxConcurrency = maxConcurrency; diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java index 463e8f629cd3..907da99e0df9 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java @@ -106,6 +106,7 @@ private S3ClientOptions createS3ClientOption() { .withComputeContentMd5(false) .withEnableS3Express(true) .withMaxConnections(s3NativeClientConfiguration.maxConcurrency()) + .withMemoryLimitInBytes(s3NativeClientConfiguration.memoryLimitInBytes()) .withThroughputTargetGbps(s3NativeClientConfiguration.targetThroughputInGbps()) .withInitialReadWindowSize(initialWindowSize) .withReadBackpressureEnabled(true); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java index 96c69deb88d5..001923f8a69e 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java @@ -59,13 +59,12 @@ public class S3NativeClientConfiguration implements SdkAutoCloseable { private final boolean checksumValidationEnabled; private final Long readBufferSizeInBytes; private final TlsContext tlsContext; - private final TlsContextOptions clientTlsContextOptions; private final HttpProxyOptions proxyOptions; private final Duration connectionTimeout; private final HttpMonitoringOptions httpMonitoringOptions; - private final Boolean useEnvironmentVariableProxyOptionsValues; + private final long memoryLimitInBytes; public S3NativeClientConfiguration(Builder builder) { this.signingRegion = builder.signingRegion == null ? DefaultAwsRegionProviderChain.builder().build().getRegion().id() : @@ -98,6 +97,7 @@ public S3NativeClientConfiguration(Builder builder) { // Using 0 so that CRT will calculate it based on targetThroughputGbps this.maxConcurrency = builder.maxConcurrency == null ? 0 : builder.maxConcurrency; + this.memoryLimitInBytes = builder.memoryLimitInBytes == null ? 0 : builder.memoryLimitInBytes; this.endpointOverride = builder.endpointOverride; @@ -177,6 +177,10 @@ public double targetThroughputInGbps() { return targetThroughputInGbps; } + public long memoryLimitInBytes() { + return memoryLimitInBytes; + } + public int maxConcurrency() { return maxConcurrency; } @@ -218,6 +222,7 @@ public static final class Builder { private S3CrtHttpConfiguration httpConfiguration; private StandardRetryOptions standardRetryOptions; private Long thresholdInBytes; + private Long memoryLimitInBytes; private Builder() { } @@ -247,6 +252,11 @@ public Builder maxConcurrency(Integer maxConcurrency) { return this; } + public Builder memoryLimitInBytes(Long memoryLimitInBytes) { + this.memoryLimitInBytes = memoryLimitInBytes; + return this; + } + public Builder endpointOverride(URI endpointOverride) { this.endpointOverride = endpointOverride; return this; diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java index 2af0ee9f65d1..cf6388d8c6f6 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java @@ -379,6 +379,8 @@ void build_shouldPassThroughParameters() { .maxConcurrency(100) .signingRegion(signingRegion) .thresholdInBytes(1024L) + .targetThroughputInGbps(3.5) + .memoryLimitInBytes(5L * 1024 * 1024 * 1024) .standardRetryOptions( new StandardRetryOptions() .withBackoffRetryOptions(new ExponentialBackoffRetryOptions().withMaxRetries(7))) @@ -409,6 +411,8 @@ void build_shouldPassThroughParameters() { assertThat(options.getMinThroughputBytesPerSecond()).isEqualTo(1024); }); assertThat(clientOptions.getMaxConnections()).isEqualTo(100); + assertThat(clientOptions.getThroughputTargetGbps()).isEqualTo(3.5); + assertThat(clientOptions.getMemoryLimitInBytes()).isEqualTo(5L * 1024 * 1024 * 1024); } } From b2a47f25a2c7bfc2d78d7bfc84de0187dfbd7e30 Mon Sep 17 00:00:00 2001 From: Anna-Karin Salander Date: Tue, 6 Feb 2024 14:04:07 -0800 Subject: [PATCH 3/4] Changing parameter name to be more specific --- .../services/s3/S3CrtAsyncClientBuilder.java | 17 ++++++++++------- .../internal/crt/DefaultS3CrtAsyncClient.java | 9 +++++---- .../s3/internal/crt/S3CrtAsyncHttpClient.java | 2 +- .../crt/S3NativeClientConfiguration.java | 14 +++++++------- .../internal/crt/S3CrtAsyncHttpClientTest.java | 2 +- 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java index 1fcbf92734fb..793e807543b3 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java @@ -115,9 +115,9 @@ default S3CrtAsyncClientBuilder credentialsProvider(IdentityProvider - * If not provided, the CRT attempts to limit memory usage in an optimal way, based on a number of parameters + * If not provided, the CRT attempts to limit native memory usage in an optimal way, based on a number of parameters * such as target throughput. Therefore, only configure the memory limit explicitly when needed. *

* Supported range: @@ -126,11 +126,13 @@ default S3CrtAsyncClientBuilder credentialsProvider(IdentityProviderMax: The lowest value of the supplied value and the SIZE_MAX of the system * * - * @param memoryLimitInBytes the memory limit in bytes + * @param maxNativeMemoryLimitInBytes + the native memory limit in bytes * @return this builder for method chaining. * @see #targetThroughputInGbps(Double) */ - S3CrtAsyncClientBuilder memoryLimitInBytes(Long memoryLimitInBytes); + S3CrtAsyncClientBuilder maxNativeMemoryLimitInBytes(Long maxNativeMemoryLimitInBytes +); /** * The target throughput for transfer requests. Higher value means more connections will be established with S3. @@ -147,13 +149,14 @@ default S3CrtAsyncClientBuilder credentialsProvider(IdentityProvider - * Note: this setting affects the memory usage of CRT; a higher throughput value will result in a larger memory - * usage. Typically, a range of throughput values maps to a discrete memory limit value in CRT, with a maximum upper limit. + * Note: This setting affects the native memory usage used by CRT; a higher throughput value will result in a larger + * memory usage. Typically, a range of throughput values maps to a discrete memory limit value in CRT, with a maximum upper + * limit. * * @param targetThroughputInGbps the target throughput in Gbps * @return this builder for method chaining. * @see #maxConcurrency(Integer) - * @see #memoryLimitInBytes(Long) + * @see #maxNativeMemoryLimitInBytes(Long) */ S3CrtAsyncClientBuilder targetThroughputInGbps(Double targetThroughputInGbps); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java index c363200dc77c..2eff64770c47 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java @@ -158,7 +158,7 @@ private static S3CrtAsyncHttpClient.Builder initializeS3CrtAsyncHttpClient(Defau .readBufferSizeInBytes(builder.readBufferSizeInBytes) .httpConfiguration(builder.httpConfiguration) .thresholdInBytes(builder.thresholdInBytes) - .memoryLimitInBytes(builder.memoryLimitInBytes); + .maxNativeMemoryLimitInBytes(builder.maxNativeMemoryLimitInBytes); if (builder.retryConfiguration != null) { nativeClientBuilder.standardRetryOptions( @@ -176,7 +176,8 @@ public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientB private Region region; private Long minimalPartSizeInBytes; private Double targetThroughputInGbps; - private Long memoryLimitInBytes; + private Long maxNativeMemoryLimitInBytes +; private Integer maxConcurrency; private URI endpointOverride; private Boolean checksumValidationEnabled; @@ -215,8 +216,8 @@ public S3CrtAsyncClientBuilder targetThroughputInGbps(Double targetThroughputInG } @Override - public S3CrtAsyncClientBuilder memoryLimitInBytes(Long memoryLimitInBytes) { - this.memoryLimitInBytes = memoryLimitInBytes; + public S3CrtAsyncClientBuilder maxNativeMemoryLimitInBytes(Long maxNativeMemoryLimitInBytes) { + this.maxNativeMemoryLimitInBytes = maxNativeMemoryLimitInBytes; return this; } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java index 907da99e0df9..14a2eb9a1af0 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java @@ -106,7 +106,7 @@ private S3ClientOptions createS3ClientOption() { .withComputeContentMd5(false) .withEnableS3Express(true) .withMaxConnections(s3NativeClientConfiguration.maxConcurrency()) - .withMemoryLimitInBytes(s3NativeClientConfiguration.memoryLimitInBytes()) + .withMemoryLimitInBytes(s3NativeClientConfiguration.maxNativeMemoryLimitInBytes()) .withThroughputTargetGbps(s3NativeClientConfiguration.targetThroughputInGbps()) .withInitialReadWindowSize(initialWindowSize) .withReadBackpressureEnabled(true); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java index 001923f8a69e..fcd5b9ae5db7 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java @@ -64,7 +64,7 @@ public class S3NativeClientConfiguration implements SdkAutoCloseable { private final Duration connectionTimeout; private final HttpMonitoringOptions httpMonitoringOptions; private final Boolean useEnvironmentVariableProxyOptionsValues; - private final long memoryLimitInBytes; + private final long maxNativeMemoryLimitInBytes; public S3NativeClientConfiguration(Builder builder) { this.signingRegion = builder.signingRegion == null ? DefaultAwsRegionProviderChain.builder().build().getRegion().id() : @@ -97,7 +97,7 @@ public S3NativeClientConfiguration(Builder builder) { // Using 0 so that CRT will calculate it based on targetThroughputGbps this.maxConcurrency = builder.maxConcurrency == null ? 0 : builder.maxConcurrency; - this.memoryLimitInBytes = builder.memoryLimitInBytes == null ? 0 : builder.memoryLimitInBytes; + this.maxNativeMemoryLimitInBytes = builder.maxNativeMemoryLimitInBytes == null ? 0 : builder.maxNativeMemoryLimitInBytes; this.endpointOverride = builder.endpointOverride; @@ -177,8 +177,8 @@ public double targetThroughputInGbps() { return targetThroughputInGbps; } - public long memoryLimitInBytes() { - return memoryLimitInBytes; + public long maxNativeMemoryLimitInBytes() { + return maxNativeMemoryLimitInBytes; } public int maxConcurrency() { @@ -222,7 +222,7 @@ public static final class Builder { private S3CrtHttpConfiguration httpConfiguration; private StandardRetryOptions standardRetryOptions; private Long thresholdInBytes; - private Long memoryLimitInBytes; + private Long maxNativeMemoryLimitInBytes; private Builder() { } @@ -252,8 +252,8 @@ public Builder maxConcurrency(Integer maxConcurrency) { return this; } - public Builder memoryLimitInBytes(Long memoryLimitInBytes) { - this.memoryLimitInBytes = memoryLimitInBytes; + public Builder maxNativeMemoryLimitInBytes(Long maxNativeMemoryLimitInBytes) { + this.maxNativeMemoryLimitInBytes = maxNativeMemoryLimitInBytes; return this; } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java index cf6388d8c6f6..72aa189c168c 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java @@ -380,7 +380,7 @@ void build_shouldPassThroughParameters() { .signingRegion(signingRegion) .thresholdInBytes(1024L) .targetThroughputInGbps(3.5) - .memoryLimitInBytes(5L * 1024 * 1024 * 1024) + .maxNativeMemoryLimitInBytes(5L * 1024 * 1024 * 1024) .standardRetryOptions( new StandardRetryOptions() .withBackoffRetryOptions(new ExponentialBackoffRetryOptions().withMaxRetries(7))) From 7791d63a575e4e93d5dc113a87b6df6ace0fbdbe Mon Sep 17 00:00:00 2001 From: Anna-Karin Salander Date: Tue, 6 Feb 2024 16:03:53 -0800 Subject: [PATCH 4/4] Attributing CRT change to the whole SDK, and the new setting to S3 CRT client --- .changes/next-release/feature-AWSCRTbasedS3Client-353870f.json | 2 +- .changes/next-release/feature-AmazonS3-c28f9de.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.changes/next-release/feature-AWSCRTbasedS3Client-353870f.json b/.changes/next-release/feature-AWSCRTbasedS3Client-353870f.json index d27fc271c8ef..033e694991cb 100644 --- a/.changes/next-release/feature-AWSCRTbasedS3Client-353870f.json +++ b/.changes/next-release/feature-AWSCRTbasedS3Client-353870f.json @@ -1,6 +1,6 @@ { "type": "feature", - "category": "AWS CRT-based S3 Client", + "category": "AWS SDK for Java v2", "contributor": "", "description": "Bump `aws-crt` version to `0.29.9`" } diff --git a/.changes/next-release/feature-AmazonS3-c28f9de.json b/.changes/next-release/feature-AmazonS3-c28f9de.json index b31c5ddf51bf..1ed7a8046a47 100644 --- a/.changes/next-release/feature-AmazonS3-c28f9de.json +++ b/.changes/next-release/feature-AmazonS3-c28f9de.json @@ -1,6 +1,6 @@ { "type": "feature", - "category": "Amazon S3", + "category": "AWS CRT-based S3 Client", "contributor": "", "description": "Exposes a setting to set the memory limit when making asynchronous calls with the CRT-based S3 client" }