HADOOP-19740. use region "external" for any third party endpoint

...and no attempt to fall back to central-1 here.

You no longer need to declare a region for a third party store;
endpoint is enough.

Author: steveloughran

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java

@@ -323,18 +323,20 @@ private <BuilderT extends S3BaseClientBuilder<BuilderT, ClientT>, ClientT> Regio
       DEFAULT_REGION_CHAIN.info(SDK_REGION_CHAIN_IN_USE);
       LOG.debug(SDK_REGION_CHAIN_IN_USE);
     } else {
+
+      // a region has been determined from configuration,
+      // or it is falling back to central region.
+
       final Region region = resolution.getRegion();
       builder.region(requireNonNull(region));
       // s3 cross region access
       if (resolution.isCrossRegionAccessEnabled()) {
         builder.crossRegionAccessEnabled(true);
       }
-      if (!resolution.isUseCentralEndpoint()) {
-        final URI endpointUri = resolution.getEndpointUri();
-        if  (endpointUri != null) {
-          builder.endpointOverride(endpointUri);
-          LOG.debug("Setting endpoint to {}", endpointUri);
-        }
+      final URI endpointUri = resolution.getEndpointUri();
+      if (endpointUri != null && !resolution.isUseCentralEndpoint()) {
+        LOG.debug("Setting endpoint to {}", endpointUri);
+        builder.endpointOverride(endpointUri);
       }
     }
     return resolution;

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RegionResolution.java

@@ -21,10 +21,10 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.Locale;
 import java.util.Optional;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-
 import javax.annotation.Nullable;
 
 import org.slf4j.Logger;
@@ -80,56 +80,67 @@ public class RegionResolution {
   private static final Pattern VPC_ENDPOINT_PATTERN =
       Pattern.compile("^(?:.+\\.)?([a-z0-9-]+)\\.vpce\\.amazonaws\\.(?:com|com\\.cn)$");
 
- /**
-  * Error message when an endpoint is set with FIPS enabled: {@value}.
-  */
- @VisibleForTesting
- public static final String ERROR_ENDPOINT_WITH_FIPS =
-     "Only S3 central endpoint cannot be set when " + FIPS_ENDPOINT + " is true";
+  /**
+   * Error message when an endpoint is set with FIPS enabled: {@value}.
+   */
+  @VisibleForTesting
+  public static final String ERROR_ENDPOINT_WITH_FIPS =
+      "Only S3 central endpoint cannot be set when " + FIPS_ENDPOINT + " is true";
 
   /**
    * Virtual hostnames MUST be used when using the FIPS endpoint.
    */
- public static final String FIPS_PATH_ACCESS_INCOMPATIBLE =
-     "Path style access must be disabled when "+ FIPS_ENDPOINT + " is true";
+  public static final String FIPS_PATH_ACCESS_INCOMPATIBLE =
+      "Path style access must be disabled when " + FIPS_ENDPOINT + " is true";
+
+  /**
+   * String value for external region: {@value}.
+   */
+  public static final String EXTERNAL = "external";
+
+  /**
+   * External region, used for third party endpoints.
+   */
+  public static final Region EXTERNAL_REGION = Region.of(EXTERNAL);
 
   /**
    * How was the region resolved?
    */
- public enum RegionResolutionMechanism {
+  public enum RegionResolutionMechanism {
 
-   CalculatedFromEndpoint("Calculated from endpoint"),
-   FallbackToCentral("Fallback to central endpoint"),
-   ParseVpceEndpoint("Parse VPCE Endpoint"),
-   Ec2Metadata("EC2 Metadata"),
-   Sdk("SDK resolution chain"),
-   Specified("region specified");
+    CalculatedFromEndpoint("Calculated from endpoint"),
+    ExternalEndpoint("External endpoint"),
+    FallbackToCentral("Fallback to central endpoint"),
+    ParseVpceEndpoint("Parse VPCE Endpoint"),
+    Ec2Metadata("EC2 Metadata"),
+    Sdk("SDK resolution chain"),
+    Specified("region specified");
 
-   /**
-    * Text of the mechanism.
-    */
-   private final String mechanism;
+    /**
+     * Text of the mechanism.
+     */
+    private final String mechanism;
 
-   RegionResolutionMechanism(String mechanism) {
-     this.mechanism = mechanism;
-   }
+    RegionResolutionMechanism(String mechanism) {
+      this.mechanism = mechanism;
+    }
 
     /**
      * String value of the resolution mechanism.
      * @return the resolution mechanism.
      */
-   public String getMechanism() {
-     return mechanism;
-   }
-
-   @Override
-   public String toString() {
-     final StringBuilder sb = new StringBuilder("RegionResolutionMechanism{");
-     sb.append("mechanism='").append(mechanism).append('\'');
-     sb.append('}');
-     return sb.toString();
-   }
- }
+    public String getMechanism() {
+      return mechanism;
+    }
+
+    @Override
+    public String toString() {
+      final StringBuilder sb = new StringBuilder("RegionResolutionMechanism{");
+      sb.append("mechanism='").append(mechanism).append('\'');
+      sb.append('}');
+      return sb.toString();
+    }
+  }
 
   /**
    * The resolution of a region and endpoint..
@@ -298,7 +309,6 @@ public String toString() {
 
   /**
    * Given a endpoint string, create the endpoint URI.
-   *
    * @param endpoint possibly null endpoint.
    * @param secureConnections use secure HTTPS connection?
    * @return an endpoint uri or null if the endpoint was passed in was null/empty
@@ -341,7 +351,8 @@ public static Optional<Resolution> getS3RegionFromEndpoint(
       Matcher matcher = VPC_ENDPOINT_PATTERN.matcher(endpoint);
       if (matcher.find()) {
         LOG.debug("Mapping to VPCE");
-        LOG.debug("Endpoint {} is vpc endpoint; parsing region as {}", endpoint, matcher.group(1));
+        LOG.debug("Endpoint {} is VPC endpoint; parsing region as {}",
+            endpoint, matcher.group(1));
         return Optional.of(new Resolution(
             Region.of(matcher.group(1)),
             RegionResolutionMechanism.ParseVpceEndpoint));
@@ -357,6 +368,20 @@ public static Optional<Resolution> getS3RegionFromEndpoint(
     return Optional.empty();
   }
 
+  /**
+   * Is this an AWS endpoint, that is: has an endpoint been set which matches
+   * amazon.
+   * @param endpoint non-null endpoint URL
+   * @return true if this is amazonaws or amazonaws china
+   */
+  public static boolean isAwsEndpoint(final String endpoint) {
+    final String h = endpoint.toLowerCase(Locale.ROOT);
+    // Common AWS partitions: global (.amazonaws.com) and China (.amazonaws.com.cn).
+    return h.endsWith(".amazonaws.com")
+        || h.endsWith(".amazonaws.com.cn");
+  }
+
+
   /**
    * Does the region name refer to an SDK region?
    * @param configuredRegion region in the configuration
@@ -394,6 +419,7 @@ public static Resolution calculateRegion(
 
     // endpoint; may be null
     final String endpointStr = parameters.getEndpoint();
+    boolean endpointDeclared = endpointStr != null && !endpointStr.isEmpty();
     // will be null if endpointStr is null/empty
     final URI endpoint = buildEndpointUri(endpointStr,
         conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS));
@@ -419,8 +445,7 @@ public static Resolution calculateRegion(
 
     // central endpoint if no endpoint has been set, or it is explicitly
     // requested
-    boolean endpointEndsWithCentral = endpointStr == null
-        || endpointStr.isEmpty()
+    boolean endpointEndsWithCentral = !endpointDeclared
         || endpointStr.endsWith(CENTRAL_ENDPOINT);
 
     if (!resolution.isRegionResolved()) {
@@ -452,10 +477,18 @@ public static Resolution calculateRegion(
           FIPS_PATH_ACCESS_INCOMPATIBLE);
     }
 
+
     if (!resolution.isRegionResolved()) {
-      // still failing to resolve the region
-      // fall back to central
-      resolution.withRegion(US_EAST_2, RegionResolutionMechanism.FallbackToCentral);
+      // still not resolved.
+      if (!endpointDeclared || isAwsEndpoint(endpointStr)) {
+        // still failing to resolve the region
+        // fall back to central
+        resolution.withRegion(US_EAST_2, RegionResolutionMechanism.FallbackToCentral);
+      } else {
+        // we are not resolved and not an aws region.
+        // set the region to being "external"
+        resolution.withRegion(EXTERNAL_REGION, RegionResolutionMechanism.ExternalEndpoint);
+      }
     }
 
     // No need to override endpoint with "s3.amazonaws.com".

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRegionResolution.java

@@ -35,6 +35,7 @@
 import static org.apache.hadoop.fs.s3a.Constants.SDK_REGION;
 import static org.apache.hadoop.fs.s3a.impl.RegionResolution.ERROR_ENDPOINT_WITH_FIPS;
 import static org.apache.hadoop.fs.s3a.impl.RegionResolution.calculateRegion;
+import static org.apache.hadoop.fs.s3a.impl.RegionResolution.isEc2Region;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
@@ -140,7 +141,7 @@ public void testWithChinaVPCE() throws IOException {
         resolve(getConfiguration(), CN_VPC_ENDPOINT, null, false,
             CN_NORTHWEST_1, RegionResolution.RegionResolutionMechanism.ParseVpceEndpoint);
     assertEndpoint(r, CN_VPC_ENDPOINT);
-    assertUseCentral(r, false);
+    assertUseCentralValue(r, false);
   }
 
   @Test
@@ -150,7 +151,7 @@ public void testCentralEndpointNoRegion() throws IOException {
             US_EAST_2,
             RegionResolution.RegionResolutionMechanism.FallbackToCentral);
     assertEndpoint(r, null);
-    assertUseCentral(r, true);
+    assertUseCentralValue(r, true);
   }
 
   @Test
@@ -159,7 +160,7 @@ public void testCentralEndpointWithRegion() throws IOException {
         resolve(getConfiguration(), CENTRAL_ENDPOINT, US_WEST_2, false,
             US_WEST_2, RegionResolution.RegionResolutionMechanism.Specified);
     assertEndpoint(r, null);
-    assertUseCentral(r, true);
+    assertUseCentralValue(r, true);
   }
 
   @Test
@@ -169,7 +170,7 @@ public void testConfiguredRegion() throws IOException {
             EU_WEST_2, RegionResolution.RegionResolutionMechanism.Specified);
     // this still uses the central endpoint.
     assertEndpoint(r, null);
-    assertUseCentral(r, true);
+    assertUseCentralValue(r, true);
   }
 
   @Test
@@ -179,17 +180,7 @@ public void testSDKRegion() throws IOException {
             null, RegionResolution.RegionResolutionMechanism.Sdk);
     // SDK handles endpoint logic.
     assertEndpoint(r, null);
-    assertUseCentral(r, true);
-  }
-
-  @Test
-  public void testEC2UpperCaseRegion() throws IOException {
-    final RegionResolution.Resolution r =
-        resolve(getConfiguration(), null, "EC2", false,
-            null, RegionResolution.RegionResolutionMechanism.Sdk);
-    // SDK handles endpoint logic.
-    assertEndpoint(r, null);
-    assertUseCentral(r, true);
+    assertUseCentralValue(r, true);
   }
 
   @Test
@@ -199,7 +190,7 @@ public void testSDKUpperCaseRegion() throws IOException {
             null, RegionResolution.RegionResolutionMechanism.Sdk);
     // SDK handles endpoint logic.
     assertEndpoint(r, null);
-    assertUseCentral(r, true);
+    assertUseCentralValue(r, true);
   }
 
   @Test
@@ -209,7 +200,7 @@ public void testEmptyStringRegion() throws IOException {
             null, RegionResolution.RegionResolutionMechanism.Sdk);
     // SDK handles endpoint logic.
     assertEndpoint(r, null);
-    assertUseCentral(r, true);
+    assertUseCentralValue(r, true);
   }
 
   @Test
@@ -265,7 +256,7 @@ public void testWithChinaEndpoint() throws IOException {
             CN_NORTHWEST_1,
             RegionResolution.RegionResolutionMechanism.CalculatedFromEndpoint);
     assertEndpoint(r, CN_ENDPOINT);
-    assertUseCentral(r, false);
+    assertUseCentralValue(r, false);
   }
 
   @Test
@@ -276,7 +267,7 @@ public void testWithGovCloudEndpoint() throws IOException {
             US_GOV_EAST_1,
             RegionResolution.RegionResolutionMechanism.CalculatedFromEndpoint);
     assertEndpoint(r, GOV_ENDPOINT);
-    assertUseCentral(r, false);
+    assertUseCentralValue(r, false);
   }
 
   @Test
@@ -310,7 +301,28 @@ public void testEC2Region() {
       // expected on anything except EC2
       LOG.info("Expected failure when EC2 IAM is not present", e);
     }
+  }
 
+  @Test
+  public void testEc2RegionCaseLogic() throws Throwable {
+    Assertions.assertThat(isEc2Region("ec2"))
+        .describedAs("lower case ec2").isTrue();
+    Assertions.assertThat(isEc2Region("EC2"))
+        .describedAs("upper case ec2").isTrue();
+  }
+
+  @Test
+  public void testGcsRegion() throws Throwable {
+    resolve(getConfiguration(), "https://storage.googleapis.com", null, false,
+        RegionResolution.EXTERNAL,
+        RegionResolution.RegionResolutionMechanism.ExternalEndpoint);
+  }
+
+  @Test
+  public void testLocalhostRegion() throws Throwable {
+    resolve(getConfiguration(), "127.0.0.1", null, false,
+        RegionResolution.EXTERNAL,
+        RegionResolution.RegionResolutionMechanism.ExternalEndpoint);
   }
 
   /**
@@ -331,7 +343,7 @@ private static void assertEndpoint(final RegionResolution.Resolution r,
    * @param r resolution
    * @param expected expected value.
    */
-  private static void assertUseCentral(final RegionResolution.Resolution r,
+  private static void assertUseCentralValue(final RegionResolution.Resolution r,
       final boolean expected) {
     Assertions.assertThat(r.isUseCentralEndpoint())
         .describedAs("Endpoint of %s", r)