Bump CRT version and expose setting memory limits for S3 calls (#4885)

* Bump aws-crt version to 0.29.9

* Exposes a setting to set the memory limit when making asynchronous calls with the CRT-based S3 client

Author: cenedhryn

.changes/next-release/feature-AWSCRTbasedS3Client-353870f.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "Bump `aws-crt` version to `0.29.9`"
+}

.changes/next-release/feature-AmazonS3-c28f9de.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS CRT-based S3 Client",
+    "contributor": "",
+    "description": "Exposes a setting to set the memory limit when making asynchronous calls with the CRT-based S3 client"
+}

pom.xml

@@ -119,7 +119,7 @@
         <rxjava.version>2.2.21</rxjava.version>
         <commons-codec.verion>1.15</commons-codec.verion>
         <jmh.version>1.29</jmh.version>
-        <awscrt.version>0.29.7</awscrt.version>
+        <awscrt.version>0.29.9</awscrt.version>
 
         <!--Test dependencies -->
         <junit5.version>5.10.0</junit5.version>

services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java

@@ -117,12 +117,32 @@ default S3CrtAsyncClientBuilder credentialsProvider(IdentityProvider<? extends A
      */
     S3CrtAsyncClientBuilder minimumPartSizeInBytes(Long uploadPartSize);
 
+    /**
+     * The amount of native memory that CRT is allowed to use when making requests to S3.
+     * <p>
+     * If not provided, the CRT attempts to limit native memory usage in an optimal way, based on a number of parameters
+     * such as target throughput. Therefore, only configure the memory limit explicitly when needed.
+     * <p>
+     * Supported range:
+     * <ul>
+     *     <li><b>Min: </b>1 GB</li>
+     *     <li><b>Max: </b>The lowest value of the supplied value and the SIZE_MAX of the system</li>
+     * </ul>
+     *
+     * @param maxNativeMemoryLimitInBytes
+ the native memory limit in bytes
+     * @return this builder for method chaining.
+     * @see #targetThroughputInGbps(Double)
+     */
+    S3CrtAsyncClientBuilder maxNativeMemoryLimitInBytes(Long maxNativeMemoryLimitInBytes
+);
+
     /**
      * The target throughput for transfer requests. Higher value means more connections will be established with S3.
      *
      * <p>
      * Whether the transfer manager can achieve the configured target throughput depends on various factors such as the network
-     * bandwidth of the environment and whether {@link #maxConcurrency} is configured.
+     * bandwidth of the environment, whether {@link #maxConcurrency} is configured and amount of available memory.
      *
      * <p>
      * By default, it is 10 gigabits per second. If users want to transfer as fast as possible, it's recommended to set it to the
@@ -131,10 +151,15 @@ default S3CrtAsyncClientBuilder credentialsProvider(IdentityProvider<? extends A
      * instance type in <a href="https://aws.amazon.com/ec2/instance-types/">Amazon EC2 instance type page</a>.
      * If you are running into out of file descriptors error, consider using {@link #maxConcurrency(Integer)} to limit the
      * number of connections.
+     * <p>
+     * <b>Note: </b> This setting affects the native memory usage used by CRT; a higher throughput value will result in a larger
+     * memory usage. Typically, a range of throughput values maps to a discrete memory limit value in CRT, with a maximum upper
+     * limit.
      *
      * @param targetThroughputInGbps the target throughput in Gbps
      * @return this builder for method chaining.
      * @see #maxConcurrency(Integer)
+     * @see #maxNativeMemoryLimitInBytes(Long)
      */
     S3CrtAsyncClientBuilder targetThroughputInGbps(Double targetThroughputInGbps);
 

services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java

@@ -168,7 +168,8 @@ private static S3CrtAsyncHttpClient.Builder initializeS3CrtAsyncHttpClient(Defau
                                        .credentialsProvider(builder.credentialsProvider)
                                        .readBufferSizeInBytes(builder.readBufferSizeInBytes)
                                        .httpConfiguration(builder.httpConfiguration)
-                                       .thresholdInBytes(builder.thresholdInBytes);
+                                       .thresholdInBytes(builder.thresholdInBytes)
+                                       .maxNativeMemoryLimitInBytes(builder.maxNativeMemoryLimitInBytes);
 
         if (builder.retryConfiguration != null) {
             nativeClientBuilder.standardRetryOptions(
@@ -186,6 +187,8 @@ public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientB
         private Region region;
         private Long minimalPartSizeInBytes;
         private Double targetThroughputInGbps;
+        private Long maxNativeMemoryLimitInBytes
+;
         private Integer maxConcurrency;
         private URI endpointOverride;
         private Boolean checksumValidationEnabled;
@@ -224,6 +227,12 @@ public S3CrtAsyncClientBuilder targetThroughputInGbps(Double targetThroughputInG
             return this;
         }
 
+        @Override
+        public S3CrtAsyncClientBuilder maxNativeMemoryLimitInBytes(Long maxNativeMemoryLimitInBytes) {
+            this.maxNativeMemoryLimitInBytes = maxNativeMemoryLimitInBytes;
+            return this;
+        }
+
         @Override
         public S3CrtAsyncClientBuilder maxConcurrency(Integer maxConcurrency) {
             this.maxConcurrency = maxConcurrency;

services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java

@@ -106,6 +106,7 @@ private S3ClientOptions createS3ClientOption() {
                                  .withComputeContentMd5(false)
                                  .withEnableS3Express(true)
                                  .withMaxConnections(s3NativeClientConfiguration.maxConcurrency())
+                                 .withMemoryLimitInBytes(s3NativeClientConfiguration.maxNativeMemoryLimitInBytes())
                                  .withThroughputTargetGbps(s3NativeClientConfiguration.targetThroughputInGbps())
                                  .withInitialReadWindowSize(initialWindowSize)
                                  .withReadBackpressureEnabled(true);

services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java

@@ -59,13 +59,12 @@ public class S3NativeClientConfiguration implements SdkAutoCloseable {
     private final boolean checksumValidationEnabled;
     private final Long readBufferSizeInBytes;
     private final TlsContext tlsContext;
-
     private final TlsContextOptions clientTlsContextOptions;
     private final HttpProxyOptions proxyOptions;
     private final Duration connectionTimeout;
     private final HttpMonitoringOptions httpMonitoringOptions;
-
     private final Boolean useEnvironmentVariableProxyOptionsValues;
+    private final long maxNativeMemoryLimitInBytes;
 
     public S3NativeClientConfiguration(Builder builder) {
         this.signingRegion = builder.signingRegion == null ? DefaultAwsRegionProviderChain.builder().build().getRegion().id() :
@@ -98,6 +97,7 @@ public S3NativeClientConfiguration(Builder builder) {
 
         // Using 0 so that CRT will calculate it based on targetThroughputGbps
         this.maxConcurrency = builder.maxConcurrency == null ? 0 : builder.maxConcurrency;
+        this.maxNativeMemoryLimitInBytes = builder.maxNativeMemoryLimitInBytes == null ? 0 : builder.maxNativeMemoryLimitInBytes;
 
         this.endpointOverride = builder.endpointOverride;
 
@@ -177,6 +177,10 @@ public double targetThroughputInGbps() {
         return targetThroughputInGbps;
     }
 
+    public long maxNativeMemoryLimitInBytes() {
+        return maxNativeMemoryLimitInBytes;
+    }
+
     public int maxConcurrency() {
         return maxConcurrency;
     }
@@ -218,6 +222,7 @@ public static final class Builder {
         private S3CrtHttpConfiguration httpConfiguration;
         private StandardRetryOptions standardRetryOptions;
         private Long thresholdInBytes;
+        private Long maxNativeMemoryLimitInBytes;
 
         private Builder() {
         }
@@ -247,6 +252,11 @@ public Builder maxConcurrency(Integer maxConcurrency) {
             return this;
         }
 
+        public Builder maxNativeMemoryLimitInBytes(Long maxNativeMemoryLimitInBytes) {
+            this.maxNativeMemoryLimitInBytes = maxNativeMemoryLimitInBytes;
+            return this;
+        }
+
         public Builder endpointOverride(URI endpointOverride) {
             this.endpointOverride = endpointOverride;
             return this;

services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java

@@ -379,6 +379,8 @@ void build_shouldPassThroughParameters() {
                                        .maxConcurrency(100)
                                        .signingRegion(signingRegion)
                                        .thresholdInBytes(1024L)
+                                       .targetThroughputInGbps(3.5)
+                                       .maxNativeMemoryLimitInBytes(5L * 1024 * 1024 * 1024)
                                        .standardRetryOptions(
                                            new StandardRetryOptions()
                                                .withBackoffRetryOptions(new ExponentialBackoffRetryOptions().withMaxRetries(7)))
@@ -409,6 +411,8 @@ void build_shouldPassThroughParameters() {
                 assertThat(options.getMinThroughputBytesPerSecond()).isEqualTo(1024);
             });
             assertThat(clientOptions.getMaxConnections()).isEqualTo(100);
+            assertThat(clientOptions.getThroughputTargetGbps()).isEqualTo(3.5);
+            assertThat(clientOptions.getMemoryLimitInBytes()).isEqualTo(5L * 1024 * 1024 * 1024);
         }
     }