Merge branch 'trunk' into YARN-11218

Author: slfan1989

hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md

@@ -24,7 +24,7 @@ This filter must be configured in front of all the web application resources tha
 
 The Hadoop Auth and dependent JAR files must be in the web application classpath (commonly the `WEB-INF/lib` directory).
 
-Hadoop Auth uses SLF4J-API for logging. Auth Maven POM dependencies define the SLF4J API dependency but it does not define the dependency on a concrete logging implementation, this must be addded explicitly to the web application. For example, if the web applicationan uses Log4j, the SLF4J-LOG4J12 and LOG4J jar files must be part part of the web application classpath as well as the Log4j configuration file.
+Hadoop Auth uses SLF4J-API for logging. Auth Maven POM dependencies define the SLF4J API dependency but it does not define the dependency on a concrete logging implementation, this must be addded explicitly to the web application. For example, if the web applicationan uses Log4j, the SLF4J-LOG4J12 and LOG4J jar files must be part of the web application classpath as well as the Log4j configuration file.
 
 ### Common Configuration parameters
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/XMLUtils.java

@@ -34,6 +34,7 @@
 import org.xml.sax.SAXException;
 
 import java.io.*;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * General xml utilities.
@@ -59,6 +60,11 @@ public class XMLUtils {
   public static final String VALIDATION =
       "http://xml.org/sax/features/validation";
 
+  private static final AtomicBoolean CAN_SET_TRANSFORMER_ACCESS_EXTERNAL_DTD =
+          new AtomicBoolean(true);
+  private static final AtomicBoolean CAN_SET_TRANSFORMER_ACCESS_EXTERNAL_STYLESHEET =
+          new AtomicBoolean(true);
+
   /**
    * Transform input xml given a stylesheet.
    * 
@@ -143,8 +149,7 @@ public static TransformerFactory newSecureTransformerFactory()
           throws TransformerConfigurationException {
     TransformerFactory trfactory = TransformerFactory.newInstance();
     trfactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
-    bestEffortSetAttribute(trfactory, XMLConstants.ACCESS_EXTERNAL_DTD, "");
-    bestEffortSetAttribute(trfactory, XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "");
+    setOptionalSecureTransformerAttributes(trfactory);
     return trfactory;
   }
 
@@ -161,29 +166,45 @@ public static SAXTransformerFactory newSecureSAXTransformerFactory()
           throws TransformerConfigurationException {
     SAXTransformerFactory trfactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
     trfactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
-    bestEffortSetAttribute(trfactory, XMLConstants.ACCESS_EXTERNAL_DTD, "");
-    bestEffortSetAttribute(trfactory, XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "");
+    setOptionalSecureTransformerAttributes(trfactory);
     return trfactory;
   }
 
+  /**
+   * These attributes are recommended for maximum security but some JAXP transformers do
+   * not support them. If at any stage, we fail to set these attributes, then we won't try again
+   * for subsequent transformers.
+   *
+   * @param transformerFactory to update
+   */
+  private static void setOptionalSecureTransformerAttributes(
+          TransformerFactory transformerFactory) {
+    bestEffortSetAttribute(transformerFactory, CAN_SET_TRANSFORMER_ACCESS_EXTERNAL_DTD,
+            XMLConstants.ACCESS_EXTERNAL_DTD, "");
+    bestEffortSetAttribute(transformerFactory, CAN_SET_TRANSFORMER_ACCESS_EXTERNAL_STYLESHEET,
+            XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "");
+  }
+
   /**
    * Set an attribute value on a {@link TransformerFactory}. If the TransformerFactory
    * does not support the attribute, the method just returns <code>false</code> and
    * logs the issue at debug level.
    *
    * @param transformerFactory to update
+   * @param flag that indicates whether to do the update and the flag can be set to
+   *             <code>false</code> if an update fails
    * @param name of the attribute to set
    * @param value to set on the attribute
-   * @return whether the attribute was successfully set
    */
-  static boolean bestEffortSetAttribute(TransformerFactory transformerFactory,
-                                        String name, Object value) {
-    try {
-      transformerFactory.setAttribute(name, value);
-      return true;
-    } catch (Throwable t) {
-      LOG.debug("Issue setting TransformerFactory attribute {}: {}", name, t.toString());
+  static void bestEffortSetAttribute(TransformerFactory transformerFactory, AtomicBoolean flag,
+                                     String name, Object value) {
+    if (flag.get()) {
+      try {
+        transformerFactory.setAttribute(name, value);
+      } catch (Throwable t) {
+        flag.set(false);
+        LOG.debug("Issue setting TransformerFactory attribute {}: {}", name, t.toString());
+      }
     }
-    return false;
   }
 }

hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md

@@ -975,7 +975,7 @@ this will be in the bucket; the `rm` operation will then take time proportional
 to the size of the data. Furthermore, the deleted files will continue to incur
 storage costs.
 
-To avoid this, use the the `-skipTrash` option.
+To avoid this, use the `-skipTrash` option.
 
 ```bash
 hadoop fs -rm -skipTrash s3a://bucket/dataset

hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md

@@ -220,7 +220,7 @@ Each metrics record contains tags such as ProcessName, SessionId, and Hostname a
 | `WarmUpEDEKTimeNumOps` | Total number of warming up EDEK |
 | `WarmUpEDEKTimeAvgTime` | Average time of warming up EDEK in milliseconds |
 | `WarmUpEDEKTime`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of time spent in warming up EDEK in milliseconds (*num* seconds granularity). Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
-| `ResourceCheckTime`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of of NameNode resource check latency in milliseconds (*num* seconds granularity). Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
+| `ResourceCheckTime`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of NameNode resource check latency in milliseconds (*num* seconds granularity). Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
 | `EditLogTailTimeNumOps` | Total number of times the standby NameNode tailed the edit log |
 | `EditLogTailTimeAvgTime` | Average time (in milliseconds) spent by standby NameNode in tailing edit log |
 | `EditLogTailTime`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of time spent in tailing edit logs by standby NameNode in milliseconds (*num* seconds granularity). Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |

hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md

@@ -595,7 +595,7 @@ hadoop kdiag \
   --keytab zk.service.keytab --principal zookeeper/devix.example.org@REALM
 ```
 
-This attempts to to perform all diagnostics without failing early, load in
+This attempts to perform all diagnostics without failing early, load in
 the HDFS and YARN XML resources, require a minimum key length of 1024 bytes,
 and log in as the principal `zookeeper/devix.example.org@REALM`, whose key must be in
 the keytab `zk.service.keytab`

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md

@@ -501,7 +501,7 @@ Where
       def blocks(FS, p, s, s +  l)  = a list of the blocks containing data(FS, path)[s:s+l]
 
 
-Note that that as `length(FS, f) ` is defined as `0` if `isDir(FS, f)`, the result
+Note that as `length(FS, f) ` is defined as `0` if `isDir(FS, f)`, the result
 of `getFileBlockLocations()` on a directory is `[]`
 
 
@@ -707,7 +707,7 @@ This is a significant difference between the behavior of object stores
 and that of filesystems, as it allows >1 client to create a file with `overwrite=false`,
 and potentially confuse file/directory logic. In particular, using `create()` to acquire
 an exclusive lock on a file (whoever creates the file without an error is considered
-the holder of the lock) may not not a safe algorithm to use when working with object stores.
+the holder of the lock) may not be a safe algorithm to use when working with object stores.
 
 * Object stores may create an empty file as a marker when a file is created.
 However, object stores with `overwrite=true` semantics may not implement this atomically,

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstreambuilder.md

@@ -167,7 +167,7 @@ rather than just any FS-specific subclass implemented by the implementation
 custom subclasses.
 
 This is critical to ensure safe use of the feature: directory listing/
-status serialization/deserialization can result result in the `withFileStatus()`
+status serialization/deserialization can result in the `withFileStatus()`
 argument not being the custom subclass returned by the Filesystem instance's
 own `getFileStatus()`, `listFiles()`, `listLocatedStatus()` calls, etc.
 
@@ -686,4 +686,4 @@ public T load(FileSystem fs,
 *Note:* : in Hadoop 3.3.2 and earlier, the `withFileStatus(status)` call
 required a non-null parameter; this has since been relaxed.
 For maximum compatibility across versions, only invoke the method
-when the file status is known to be non-null.
+when the file status is known to be non-null.

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md

@@ -228,12 +228,12 @@ Accordingly: *Use if and only if you are confident that the conditions are met.*
 
 ### `fs.s3a.create.header` User-supplied header support
 
-Options with the prefix `fs.s3a.create.header.` will be added to to the
+Options with the prefix `fs.s3a.create.header.` will be added to the
 S3 object metadata as "user defined metadata".
 This metadata is visible to all applications. It can also be retrieved through the
 FileSystem/FileContext `listXAttrs()` and `getXAttrs()` API calls with the prefix `header.`
 
 When an object is renamed, the metadata is propagated the copy created.
 
 It is possible to probe an S3A Filesystem instance for this capability through
-the `hasPathCapability(path, "fs.s3a.create.header")` check.
+the `hasPathCapability(path, "fs.s3a.create.header")` check.

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/outputstream.md

@@ -980,7 +980,7 @@ throw `UnsupportedOperationException`.
 ### `StreamCapabilities`
 
 Implementors of filesystem clients SHOULD implement the `StreamCapabilities`
-interface and its `hasCapabilities()` method to to declare whether or not
+interface and its `hasCapabilities()` method to declare whether or not
 an output streams offer the visibility and durability guarantees of `Syncable`.
 
 Implementors of `StreamCapabilities.hasCapabilities()` MUST NOT declare that
@@ -1013,4 +1013,4 @@ all data to the datanodes.
 
 1. `close()` SHALL return once the guarantees of `hflush()` are met: the data is
    visible to others.
-1. For durability guarantees, `hsync()` MUST be called first.
+1. For durability guarantees, `hsync()` MUST be called first.

hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-configuration.md

@@ -143,7 +143,7 @@ too must have this context defined.
 
 ### Identifying the system accounts `hadoop.registry.system.acls`
 
-These are the the accounts which are given full access to the base of the
+These are the accounts which are given full access to the base of the
 registry. The Resource Manager needs this option to create the root paths.
 
 Client applications writing to the registry access to the nodes it creates.