diff --git a/.changes/next-release/feature-AWSCRTbasedS3Client-353870f.json b/.changes/next-release/feature-AWSCRTbasedS3Client-353870f.json new file mode 100644 index 000000000000..033e694991cb --- /dev/null +++ b/.changes/next-release/feature-AWSCRTbasedS3Client-353870f.json @@ -0,0 +1,6 @@ +{ + "type": "feature", + "category": "AWS SDK for Java v2", + "contributor": "", + "description": "Bump `aws-crt` version to `0.29.9`" +} diff --git a/.changes/next-release/feature-AmazonS3-c28f9de.json b/.changes/next-release/feature-AmazonS3-c28f9de.json new file mode 100644 index 000000000000..1ed7a8046a47 --- /dev/null +++ b/.changes/next-release/feature-AmazonS3-c28f9de.json @@ -0,0 +1,6 @@ +{ + "type": "feature", + "category": "AWS CRT-based S3 Client", + "contributor": "", + "description": "Exposes a setting to set the memory limit when making asynchronous calls with the CRT-based S3 client" +} diff --git a/pom.xml b/pom.xml index 2bcc54a152df..6b9cf0ecbb33 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,7 @@ 2.2.21 1.15 1.29 - 0.29.7 + 0.29.9 5.10.0 diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java index bd9558e1f87b..793e807543b3 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java @@ -114,12 +114,32 @@ default S3CrtAsyncClientBuilder credentialsProvider(IdentityProvider + * If not provided, the CRT attempts to limit native memory usage in an optimal way, based on a number of parameters + * such as target throughput. Therefore, only configure the memory limit explicitly when needed. + *

+ * Supported range: + *

+ * + * @param maxNativeMemoryLimitInBytes + the native memory limit in bytes + * @return this builder for method chaining. + * @see #targetThroughputInGbps(Double) + */ + S3CrtAsyncClientBuilder maxNativeMemoryLimitInBytes(Long maxNativeMemoryLimitInBytes +); + /** * The target throughput for transfer requests. Higher value means more connections will be established with S3. * *

* Whether the transfer manager can achieve the configured target throughput depends on various factors such as the network - * bandwidth of the environment and whether {@link #maxConcurrency} is configured. + * bandwidth of the environment, whether {@link #maxConcurrency} is configured and amount of available memory. * *

* By default, it is 10 gigabits per second. If users want to transfer as fast as possible, it's recommended to set it to the @@ -128,10 +148,15 @@ default S3CrtAsyncClientBuilder credentialsProvider(IdentityProviderAmazon EC2 instance type page. * If you are running into out of file descriptors error, consider using {@link #maxConcurrency(Integer)} to limit the * number of connections. + *

+ * Note: This setting affects the native memory usage used by CRT; a higher throughput value will result in a larger + * memory usage. Typically, a range of throughput values maps to a discrete memory limit value in CRT, with a maximum upper + * limit. * * @param targetThroughputInGbps the target throughput in Gbps * @return this builder for method chaining. * @see #maxConcurrency(Integer) + * @see #maxNativeMemoryLimitInBytes(Long) */ S3CrtAsyncClientBuilder targetThroughputInGbps(Double targetThroughputInGbps); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java index 2a8ad361bfb7..2eff64770c47 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java @@ -157,7 +157,8 @@ private static S3CrtAsyncHttpClient.Builder initializeS3CrtAsyncHttpClient(Defau .credentialsProvider(builder.credentialsProvider) .readBufferSizeInBytes(builder.readBufferSizeInBytes) .httpConfiguration(builder.httpConfiguration) - .thresholdInBytes(builder.thresholdInBytes); + .thresholdInBytes(builder.thresholdInBytes) + .maxNativeMemoryLimitInBytes(builder.maxNativeMemoryLimitInBytes); if (builder.retryConfiguration != null) { nativeClientBuilder.standardRetryOptions( @@ -175,6 +176,8 @@ public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientB private Region region; private Long minimalPartSizeInBytes; private Double targetThroughputInGbps; + private Long maxNativeMemoryLimitInBytes +; private Integer maxConcurrency; private URI endpointOverride; private Boolean checksumValidationEnabled; @@ -212,6 +215,12 @@ public S3CrtAsyncClientBuilder targetThroughputInGbps(Double targetThroughputInG return this; } + @Override + public S3CrtAsyncClientBuilder maxNativeMemoryLimitInBytes(Long maxNativeMemoryLimitInBytes) { + this.maxNativeMemoryLimitInBytes = maxNativeMemoryLimitInBytes; + return this; + } + @Override public S3CrtAsyncClientBuilder maxConcurrency(Integer maxConcurrency) { this.maxConcurrency = maxConcurrency; diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java index 463e8f629cd3..14a2eb9a1af0 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java @@ -106,6 +106,7 @@ private S3ClientOptions createS3ClientOption() { .withComputeContentMd5(false) .withEnableS3Express(true) .withMaxConnections(s3NativeClientConfiguration.maxConcurrency()) + .withMemoryLimitInBytes(s3NativeClientConfiguration.maxNativeMemoryLimitInBytes()) .withThroughputTargetGbps(s3NativeClientConfiguration.targetThroughputInGbps()) .withInitialReadWindowSize(initialWindowSize) .withReadBackpressureEnabled(true); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java index 96c69deb88d5..fcd5b9ae5db7 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java @@ -59,13 +59,12 @@ public class S3NativeClientConfiguration implements SdkAutoCloseable { private final boolean checksumValidationEnabled; private final Long readBufferSizeInBytes; private final TlsContext tlsContext; - private final TlsContextOptions clientTlsContextOptions; private final HttpProxyOptions proxyOptions; private final Duration connectionTimeout; private final HttpMonitoringOptions httpMonitoringOptions; - private final Boolean useEnvironmentVariableProxyOptionsValues; + private final long maxNativeMemoryLimitInBytes; public S3NativeClientConfiguration(Builder builder) { this.signingRegion = builder.signingRegion == null ? DefaultAwsRegionProviderChain.builder().build().getRegion().id() : @@ -98,6 +97,7 @@ public S3NativeClientConfiguration(Builder builder) { // Using 0 so that CRT will calculate it based on targetThroughputGbps this.maxConcurrency = builder.maxConcurrency == null ? 0 : builder.maxConcurrency; + this.maxNativeMemoryLimitInBytes = builder.maxNativeMemoryLimitInBytes == null ? 0 : builder.maxNativeMemoryLimitInBytes; this.endpointOverride = builder.endpointOverride; @@ -177,6 +177,10 @@ public double targetThroughputInGbps() { return targetThroughputInGbps; } + public long maxNativeMemoryLimitInBytes() { + return maxNativeMemoryLimitInBytes; + } + public int maxConcurrency() { return maxConcurrency; } @@ -218,6 +222,7 @@ public static final class Builder { private S3CrtHttpConfiguration httpConfiguration; private StandardRetryOptions standardRetryOptions; private Long thresholdInBytes; + private Long maxNativeMemoryLimitInBytes; private Builder() { } @@ -247,6 +252,11 @@ public Builder maxConcurrency(Integer maxConcurrency) { return this; } + public Builder maxNativeMemoryLimitInBytes(Long maxNativeMemoryLimitInBytes) { + this.maxNativeMemoryLimitInBytes = maxNativeMemoryLimitInBytes; + return this; + } + public Builder endpointOverride(URI endpointOverride) { this.endpointOverride = endpointOverride; return this; diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java index 2af0ee9f65d1..72aa189c168c 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java @@ -379,6 +379,8 @@ void build_shouldPassThroughParameters() { .maxConcurrency(100) .signingRegion(signingRegion) .thresholdInBytes(1024L) + .targetThroughputInGbps(3.5) + .maxNativeMemoryLimitInBytes(5L * 1024 * 1024 * 1024) .standardRetryOptions( new StandardRetryOptions() .withBackoffRetryOptions(new ExponentialBackoffRetryOptions().withMaxRetries(7))) @@ -409,6 +411,8 @@ void build_shouldPassThroughParameters() { assertThat(options.getMinThroughputBytesPerSecond()).isEqualTo(1024); }); assertThat(clientOptions.getMaxConnections()).isEqualTo(100); + assertThat(clientOptions.getThroughputTargetGbps()).isEqualTo(3.5); + assertThat(clientOptions.getMemoryLimitInBytes()).isEqualTo(5L * 1024 * 1024 * 1024); } }