HADOOP-18975 AWS SDK v2: support FIPS endpoints

Adds a new option `fs.s3a.endpoint.fips` to switch the SDK client to use
FIPS endpoints, as an alternative to explicitly declaring them.

* Also provides it as a path capability.
* This is not a blocker for FIPS support.
* SDK itself doesn't know that some places (eu) don't have FIPs endpoints
* SDK only fails with endpoint + fips flag as a retried exception; this
  PR fails fast.
* Adds a new "connecting.md" doc; moves existing docs there and restructures.

Change-Id: I0a7cc60022f2ee657c8f18ed9ed18f8f66a15567

Author: steveloughran

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java

@@ -1335,6 +1335,15 @@ private Constants() {
    */
   public static final String AWS_S3_DEFAULT_REGION = "us-east-2";
 
+  /**
+   * Is the endpoint a FIPS endpoint?
+   * Can be queried as a path capability.
+   * Value {@value}.
+   */
+  public static final String ENDPOINT_FIPS = "fs.s3a.endpoint.fips";
+
+  public static final boolean ENDPOINT_FIPS_DEFAULT = false;
+
   /**
    * Require that all S3 access is made through Access Points.
    */

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java

@@ -56,6 +56,7 @@
 import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SECURE_CONNECTIONS;
 import static org.apache.hadoop.fs.s3a.Constants.SECURE_CONNECTIONS;
 import static org.apache.hadoop.fs.s3a.Constants.AWS_SERVICE_IDENTIFIER_S3;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
 
 
 /**
@@ -163,6 +164,8 @@ private <BuilderT extends S3BaseClientBuilder<BuilderT, ClientT>, ClientT> Build
             .pathStyleAccessEnabled(parameters.isPathStyleAccess())
             .build();
 
+    builder.fipsEnabled(parameters.isFipsEnabled());
+
     return builder
         .overrideConfiguration(createClientOverrideConfiguration(parameters, conf))
         .credentialsProvider(parameters.getCredentialSet())
@@ -229,6 +232,7 @@ protected ClientOverrideConfiguration createClientOverrideConfiguration(
    * @param conf  conf configuration object
    * @param <BuilderT> S3 client builder type
    * @param <ClientT> S3 client type
+   * @throws IllegalArgumentException if endpoint is set when FIPS is enabled.
    */
   private <BuilderT extends S3BaseClientBuilder<BuilderT, ClientT>, ClientT> void configureEndpointAndRegion(
       BuilderT builder, S3ClientCreationParameters parameters, Configuration conf) {
@@ -244,7 +248,16 @@ private <BuilderT extends S3BaseClientBuilder<BuilderT, ClientT>, ClientT> void
       region = Region.of(configuredRegion);
     }
 
+    // FIPs? Set it, then reject any attempt to set an endpoint
+    final boolean fipsEnabled = parameters.isFipsEnabled();
+    if (fipsEnabled) {
+      LOG.debug("Enabling FIPS mode");
+      builder.fipsEnabled(true);
+    }
+
     if (endpoint != null) {
+      checkArgument(!fipsEnabled,
+          "A custom endpoint cannot be combined with FIPS: %s", endpoint);
       builder.endpointOverride(endpoint);
       // No region was configured, try to determine it from the endpoint.
       if (region == null) {

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java

@@ -461,6 +461,11 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
    */
   private boolean isMultipartCopyEnabled;
 
+  /**
+   * Is FIPS enabled?
+   */
+  private boolean fipsEnabled;
+
   /**
    * A cache of files that should be deleted when the FileSystem is closed
    * or the JVM is exited.
@@ -614,6 +619,8 @@ public void initialize(URI name, Configuration originalConf)
           ? conf.getTrimmed(AWS_REGION)
           : accessPoint.getRegion();
 
+      fipsEnabled = conf.getBoolean(ENDPOINT_FIPS, ENDPOINT_FIPS_DEFAULT);
+
       // is this an S3Express store?
       s3ExpressStore = isS3ExpressStore(bucket, endpoint);
 
@@ -1045,7 +1052,8 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
         .withMultipartCopyEnabled(isMultipartCopyEnabled)
         .withMultipartThreshold(multiPartThreshold)
         .withTransferManagerExecutor(unboundedThreadPool)
-        .withRegion(configuredRegion);
+        .withRegion(configuredRegion)
+        .withFipsEnabled(fipsEnabled);
 
     S3ClientFactory clientFactory = ReflectionUtils.newInstance(s3ClientFactoryClass, conf);
     s3Client = clientFactory.createS3Client(getUri(), parameters);
@@ -5514,6 +5522,10 @@ public boolean hasPathCapability(final Path path, final String capability)
     case OPTIMIZED_COPY_FROM_LOCAL:
       return optimizedCopyFromLocal;
 
+    // probe for a fips endpoint
+    case ENDPOINT_FIPS:
+      return fipsEnabled;
+
     default:
       return super.hasPathCapability(p, cap);
     }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java

@@ -170,6 +170,10 @@ final class S3ClientCreationParameters {
      */
     private String region;
 
+    /**
+     * Is FIPS enabled?
+     */
+    private boolean fipsEnabled;
 
     /**
      * List of execution interceptors to include in the chain
@@ -422,5 +426,23 @@ public S3ClientCreationParameters withRegion(
     public String getRegion() {
       return region;
     }
+
+    /**
+     * Get the FIPS flag.
+     * @return is fips enabled
+     */
+    public boolean isFipsEnabled() {
+      return fipsEnabled;
+    }
+
+    /**
+     * Set builder value.
+     * @param value new value
+     * @return the builder
+     */
+    public S3ClientCreationParameters withFipsEnabled(final boolean value) {
+      fipsEnabled = value;
+      return this;
+    }
   }
 }