Merge branch 'apache:trunk' into hadoop-trunk

Author: ThinkerLei

NOTICE-binary

@@ -66,7 +66,7 @@ available from http://www.digip.org/jansson/.
 
 
 AWS SDK for Java
-Copyright 2010-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2010-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 
 This product includes software developed by
 Amazon Technologies, Inc (http://www.amazon.com/).

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -1229,7 +1229,7 @@
     com.amazonaws.auth.AWSCredentialsProvider.
 
     When S3A delegation tokens are not enabled, this list will be used
-    to directly authenticate with S3 and DynamoDB services.
+    to directly authenticate with S3 and other AWS services.
     When S3A Delegation tokens are enabled, depending upon the delegation
     token binding it may be used
     to communicate wih the STS endpoint to request session/role
@@ -1686,180 +1686,18 @@
   </description>
 </property>
 
-<property>
-    <name>fs.s3a.metadatastore.authoritative</name>
-    <value>false</value>
-    <description>
-        When true, allow MetadataStore implementations to act as source of
-        truth for getting file status and directory listings.  Even if this
-        is set to true, MetadataStore implementations may choose not to
-        return authoritative results.  If the configured MetadataStore does
-        not support being authoritative, this setting will have no effect.
-    </description>
-</property>
-
-<property>
-    <name>fs.s3a.metadatastore.metadata.ttl</name>
-    <value>15m</value>
-    <description>
-        This value sets how long an entry in a MetadataStore is valid.
-    </description>
-</property>
-
-<property>
-    <name>fs.s3a.metadatastore.impl</name>
-    <value>org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore</value>
-    <description>
-        Fully-qualified name of the class that implements the MetadataStore
-        to be used by s3a.  The default class, NullMetadataStore, has no
-        effect: s3a will continue to treat the backing S3 service as the one
-        and only source of truth for file and directory metadata.
-    </description>
-</property>
-
-<property>
-  <name>fs.s3a.metadatastore.fail.on.write.error</name>
-  <value>true</value>
-  <description>
-    When true (default), FileSystem write operations generate
-    org.apache.hadoop.fs.s3a.MetadataPersistenceException if the metadata
-    cannot be saved to the metadata store.  When false, failures to save to
-    metadata store are logged at ERROR level, but the overall FileSystem
-    write operation succeeds.
-  </description>
-</property>
-
-<property>
-    <name>fs.s3a.s3guard.cli.prune.age</name>
-    <value>86400000</value>
-    <description>
-        Default age (in milliseconds) after which to prune metadata from the
-        metadatastore when the prune command is run.  Can be overridden on the
-        command-line.
-    </description>
-</property>
-
-
 <property>
   <name>fs.s3a.impl</name>
   <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
   <description>The implementation class of the S3A Filesystem</description>
 </property>
 
-<property>
-  <name>fs.s3a.s3guard.ddb.region</name>
-  <value></value>
-  <description>
-    AWS DynamoDB region to connect to. An up-to-date list is
-    provided in the AWS Documentation: regions and endpoints. Without this
-    property, the S3Guard will operate table in the associated S3 bucket region.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.table</name>
-  <value></value>
-  <description>
-    The DynamoDB table name to operate. Without this property, the respective
-    S3 bucket name will be used.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.table.create</name>
-  <value>false</value>
-  <description>
-    If true, the S3A client will create the table if it does not already exist.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.table.capacity.read</name>
-  <value>0</value>
-  <description>
-    Provisioned throughput requirements for read operations in terms of capacity
-    units for the DynamoDB table. This config value will only be used when
-    creating a new DynamoDB table.
-    If set to 0 (the default), new tables are created with "per-request" capacity.
-    If a positive integer is provided for this and the write capacity, then
-    a table with "provisioned capacity" will be created.
-    You can change the capacity of an existing provisioned-capacity table
-    through the "s3guard set-capacity" command.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.table.capacity.write</name>
-  <value>0</value>
-  <description>
-    Provisioned throughput requirements for write operations in terms of
-    capacity units for the DynamoDB table.
-    If set to 0 (the default), new tables are created with "per-request" capacity.
-    Refer to related configuration option fs.s3a.s3guard.ddb.table.capacity.read
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.table.sse.enabled</name>
-  <value>false</value>
-  <description>
-    Whether server-side encryption (SSE) is enabled or disabled on the table.
-    By default it's disabled, meaning SSE is set to AWS owned CMK.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.table.sse.cmk</name>
-  <value/>
-  <description>
-    The KMS Customer Master Key (CMK) used for the KMS encryption on the table.
-    To specify a CMK, this config value can be its key ID, Amazon Resource Name
-    (ARN), alias name, or alias ARN. Users only need to provide this config if
-    the key is different from the default DynamoDB KMS Master Key, which is
-    alias/aws/dynamodb.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.max.retries</name>
-  <value>9</value>
-    <description>
-      Max retries on throttled/incompleted DynamoDB operations
-      before giving up and throwing an IOException.
-      Each retry is delayed with an exponential
-      backoff timer which starts at 100 milliseconds and approximately
-      doubles each time.  The minimum wait before throwing an exception is
-      sum(100, 200, 400, 800, .. 100*2^N-1 ) == 100 * ((2^N)-1)
-    </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.throttle.retry.interval</name>
-  <value>100ms</value>
-    <description>
-      Initial interval to retry after a request is throttled events;
-      the back-off policy is exponential until the number of retries of
-      fs.s3a.s3guard.ddb.max.retries is reached.
-    </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.ddb.background.sleep</name>
-  <value>25ms</value>
-  <description>
-    Length (in milliseconds) of pause between each batch of deletes when
-    pruning metadata.  Prevents prune operations (which can typically be low
-    priority background operations) from overly interfering with other I/O
-    operations.
-  </description>
-</property>
-
 <property>
   <name>fs.s3a.retry.limit</name>
   <value>7</value>
   <description>
     Number of times to retry any repeatable S3 client request on failure,
-    excluding throttling requests and S3Guard inconsistency resolution.
+    excluding throttling requests.
   </description>
 </property>
 
@@ -1868,7 +1706,7 @@
   <value>500ms</value>
   <description>
     Initial retry interval when retrying operations for any reason other
-    than S3 throttle errors and S3Guard inconsistency resolution.
+    than S3 throttle errors.
   </description>
 </property>
 
@@ -1891,27 +1729,6 @@
   </description>
 </property>
 
-<property>
-  <name>fs.s3a.s3guard.consistency.retry.limit</name>
-  <value>7</value>
-  <description>
-    Number of times to retry attempts to read/open/copy files when
-    S3Guard believes a specific version of the file to be available,
-    but the S3 request does not find any version of a file, or a different
-    version.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.s3guard.consistency.retry.interval</name>
-  <value>2s</value>
-  <description>
-    Initial interval between attempts to retry operations while waiting for S3
-    to become consistent with the S3Guard data.
-    An exponential back-off is used here: every failure doubles the delay.
-  </description>
-</property>
-
 <property>
   <name>fs.s3a.committer.name</name>
   <value>file</value>

hadoop-common-project/hadoop-common/src/site/markdown/AdminCompatibilityGuide.md

@@ -137,7 +137,8 @@ internal state stores:
 
 * The internal MapReduce state data will remain compatible across minor releases within the same major version to facilitate rolling upgrades while MapReduce workloads execute.
 * HDFS maintains metadata about the data stored in HDFS in a private, internal format that is versioned. In the event of an incompatible change, the store's version number will be incremented. When upgrading an existing cluster, the metadata store will automatically be upgraded if possible. After the metadata store has been upgraded, it is always possible to reverse the upgrade process.
-* The AWS S3A guard keeps a private, internal metadata store that is versioned. Incompatible changes will cause the version number to be incremented. If an upgrade requires reformatting the store, it will be indicated in the release notes.
+* The AWS S3A guard kept a private, internal metadata store.
+  Now that the feature has been removed, the store is obsolete and can be deleted.
 * The YARN resource manager keeps a private, internal state store of application and scheduler information that is versioned. Incompatible changes will cause the version number to be incremented. If an upgrade requires reformatting the store, it will be indicated in the release notes.
 * The YARN node manager keeps a private, internal state store of application information that is versioned. Incompatible changes will cause the version number to be incremented. If an upgrade requires reformatting the store, it will be indicated in the release notes.
 * The YARN federation service keeps a private, internal state store of application and cluster information that is versioned. Incompatible changes will cause the version number to be incremented. If an upgrade requires reformatting the store, it will be indicated in the release notes.

hadoop-common-project/hadoop-common/src/site/markdown/Compatibility.md

@@ -477,19 +477,12 @@ rolled back to the older layout.
 
 ##### AWS S3A Guard Metadata
 
-For each operation in the Hadoop S3 client (s3a) that reads or modifies
-file metadata, a shadow copy of that file metadata is stored in a separate
-metadata store, which offers HDFS-like consistency for the metadata, and may
-also provide faster lookups for things like file status or directory listings.
-S3A guard tables are created with a version marker which indicates
-compatibility.
+The S3Guard metastore used to store metadata in DynamoDB tables;
+as such it had to maintain a compatibility strategy.
+Now that S3Guard is removed, the tables are not needed.
 
-###### Policy
-
-The S3A guard metadata schema SHALL be considered
-[Private](./InterfaceClassification.html#Private) and
-[Unstable](./InterfaceClassification.html#Unstable). Any incompatible change
-to the schema MUST result in the version number of the schema being incremented.
+Applications configured to use an S3A metadata store other than
+the "null" store will fail.
 
 ##### YARN Resource Manager State Store
 

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md

@@ -343,7 +343,7 @@ stores pretend that they are a FileSystem, a FileSystem with the same
 features and operations as HDFS. This is —ultimately—a pretence:
 they have different characteristics and occasionally the illusion fails.
 
-1. **Consistency**. Object stores are generally *Eventually Consistent*: it
+1. **Consistency**. Object may be *Eventually Consistent*: it
 can take time for changes to objects —creation, deletion and updates—
 to become visible to all callers. Indeed, there is no guarantee a change is
 immediately visible to the client which just made the change. As an example,
@@ -447,10 +447,6 @@ Object stores have an even vaguer view of time, which can be summarized as
  * The timestamp is likely to be in UTC or the TZ of the object store. If the
    client is in a different timezone, the timestamp of objects may be ahead or
    behind that of the client.
- * Object stores with cached metadata databases (for example: AWS S3 with
-   an in-memory or a DynamoDB metadata store) may have timestamps generated
-   from the local system clock, rather than that of the service.
-   This is an optimization to avoid round-trip calls to the object stores.
  + A file's modification time is often the same as its creation time.
  + The `FileSystem.setTimes()` operation to set file timestamps *may* be ignored.
  * `FileSystem.chmod()` may update modification times (example: Azure `wasb://`).

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -1664,9 +1664,16 @@ public BlocksWithLocations getBlocksWithLocations(final DatanodeID datanode,
     if(numBlocks == 0) {
       return new BlocksWithLocations(new BlockWithLocations[0]);
     }
+
+    // skip stale storage
+    DatanodeStorageInfo[] storageInfos = Arrays
+        .stream(node.getStorageInfos())
+        .filter(s -> !s.areBlockContentsStale())
+        .toArray(DatanodeStorageInfo[]::new);
+
     // starting from a random block
     int startBlock = ThreadLocalRandom.current().nextInt(numBlocks);
-    Iterator<BlockInfo> iter = node.getBlockIterator(startBlock);
+    Iterator<BlockInfo> iter = node.getBlockIterator(startBlock, storageInfos);
     List<BlockWithLocations> results = new ArrayList<BlockWithLocations>();
     List<BlockInfo> pending = new ArrayList<BlockInfo>();
     long totalSize = 0;
@@ -1685,8 +1692,8 @@ public BlocksWithLocations getBlocksWithLocations(final DatanodeID datanode,
       }
     }
     if(totalSize<size) {
-      iter = node.getBlockIterator(); // start from the beginning
-      for(int i=0; i<startBlock&&totalSize<size; i++) {
+      iter = node.getBlockIterator(0, storageInfos); // start from the beginning
+      for(int i = 0; i < startBlock && totalSize < size && iter.hasNext(); i++) {
         curBlock = iter.next();
         if(!curBlock.isComplete())  continue;
         if (curBlock.getNumBytes() < minBlockSize) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java

@@ -647,6 +647,17 @@ Iterator<BlockInfo> getBlockIterator(final int startBlock) {
     return new BlockIterator(startBlock, getStorageInfos());
   }
 
+  /**
+   * Get iterator, which starts iterating from the specified block and storages.
+   *
+   * @param startBlock on which blocks are start iterating
+   * @param storageInfos specified storages
+   */
+  Iterator<BlockInfo> getBlockIterator(
+      final int startBlock, final DatanodeStorageInfo[] storageInfos) {
+    return new BlockIterator(startBlock, storageInfos);
+  }
+
   @VisibleForTesting
   public void incrementPendingReplicationWithoutTargets() {
     pendingReplicationWithoutTargets++;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java

@@ -168,6 +168,11 @@ public boolean areBlockContentsStale() {
     return blockContentsStale;
   }
 
+  @VisibleForTesting
+  public void setBlockContentsStale(boolean value) {
+    blockContentsStale = value;
+  }
+
   void markStaleAfterFailover() {
     heartbeatedSinceFailover = false;
     blockContentsStale = true;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java

@@ -1271,6 +1271,7 @@ boolean isOutliersReportDue(long curTime) {
 
     void forceFullBlockReportNow() {
       forceFullBlockReport.set(true);
+      resetBlockReportTime = true;
     }
 
     /**

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java

@@ -113,7 +113,7 @@ public class DNConf {
   final long outliersReportIntervalMs;
   final long ibrInterval;
   final long initialBlockReportDelayMs;
-  final long cacheReportInterval;
+  volatile long cacheReportInterval;
   final long datanodeSlowIoWarningThresholdMs;
 
   final String minimumNameNodeVersion;
@@ -484,4 +484,14 @@ void setBlockReportInterval(long intervalMs) {
   public long getBlockReportInterval() {
     return blockReportInterval;
   }
+
+  void setCacheReportInterval(long intervalMs) {
+    Preconditions.checkArgument(intervalMs > 0,
+        "dfs.cachereport.intervalMsec should be larger than 0");
+    cacheReportInterval = intervalMs;
+  }
+
+  public long getCacheReportInterval() {
+    return cacheReportInterval;
+  }
 }