HADOOP-16470. Make last AWS credential provider in default auth chain EC2ContainerCredentialsProviderWrapper.

Contributed by Steve Loughran.

Contains HADOOP-16471. Restore (documented) fs.s3a.SharedInstanceProfileCredentialsProvider.

Change-Id: I06b99b57459cac80bf743c5c54f04e59bb54c2f8

Author: steveloughran

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -1092,8 +1092,8 @@
         configuration of AWS access key ID and secret access key in
         environment variables named AWS_ACCESS_KEY_ID and
         AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
-    * com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
-        of instance profile credentials if running in an EC2 VM.
+    * org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider: picks up
+       IAM credentials of any EC2 VM or AWS container in which the process is running.
   </description>
 </property>
 

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider;
+import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException;
+
+/**
+ * This credential provider has jittered between existing and non-existing,
+ * but it turns up in documentation enough that it has been restored.
+ * It extends {@link IAMInstanceCredentialsProvider} to pick up its
+ * bindings, which are currently to use the
+ * {@code EC2ContainerCredentialsProviderWrapper} class for IAM and container
+ * authentication.
+ * <p>
+ * When it fails to authenticate, it raises a
+ * {@link NoAwsCredentialsException} which can be recognized by retry handlers
+ * as a non-recoverable failure.
+ * <p>
+ * It is implicitly public; marked evolving as we can change its semantics.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class SharedInstanceCredentialProvider extends
+    IAMInstanceCredentialsProvider {
+}

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java

@@ -24,38 +24,44 @@
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * This is going to be an IAM credential provider which performs
- * async refresh for lower-latency on IO calls.
- * Initially it does not do this, simply shares the single IAM instance
- * across all instances. This makes it less expensive to declare.
- *
+ * This is an IAM credential provider which wraps
+ * an {@code EC2ContainerCredentialsProviderWrapper}
+ * to provide credentials when the S3A connector is instantiated on AWS EC2
+ * or the AWS container services.
+ * <p>
+ * When it fails to authenticate, it raises a
+ * {@link NoAwsCredentialsException} which can be recognized by retry handlers
+ * as a non-recoverable failure.
+ * <p>
+ * It is implicitly public; marked evolving as we can change its semantics.
  */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
 public class IAMInstanceCredentialsProvider
     implements AWSCredentialsProvider, Closeable {
 
-  private static final InstanceProfileCredentialsProvider INSTANCE =
-      InstanceProfileCredentialsProvider.getInstance();
+  private final AWSCredentialsProvider provider =
+      new EC2ContainerCredentialsProviderWrapper();
 
   public IAMInstanceCredentialsProvider() {
   }
 
   /**
    * Ask for the credentials.
-   * as it invariably means "you aren't running on EC2"
+   * Failure invariably means "you aren't running in an EC2 VM or AWS container".
    * @return the credentials
+   * @throws NoAwsCredentialsException on auth failure to indicate non-recoverable.
    */
   @Override
   public AWSCredentials getCredentials() {
     try {
-      return INSTANCE.getCredentials();
+      return provider.getCredentials();
     } catch (AmazonClientException e) {
       throw new NoAwsCredentialsException("IAMInstanceCredentialsProvider",
           e.getMessage(),
@@ -65,11 +71,11 @@ public AWSCredentials getCredentials() {
 
   @Override
   public void refresh() {
-    INSTANCE.refresh();
+    provider.refresh();
   }
 
   @Override
   public void close() throws IOException {
-    // until async, no-op.
+    // no-op.
   }
 }

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md

@@ -539,8 +539,8 @@ This means that the default S3A authentication chain can be defined as
         configuration of AWS access key ID and secret access key in
         environment variables named AWS_ACCESS_KEY_ID and
         AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
-    * com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
-        of instance profile credentials if running in an EC2 VM.
+    * org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider: picks up
+       IAM credentials of any EC2 VM or AWS container in which the process is running.
   </description>
 </property>
 ```