Merge branch 'apache:trunk' into YARN-11239

Author: slfan1989

LICENSE-binary

@@ -210,9 +210,9 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.* (css and js
 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java
 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java
 
-com.aliyun:aliyun-java-sdk-core:3.4.0
-com.aliyun:aliyun-java-sdk-ecs:4.2.0
-com.aliyun:aliyun-java-sdk-ram:3.0.0
+com.aliyun:aliyun-java-sdk-core:4.5.10
+com.aliyun:aliyun-java-sdk-kms:2.11.0
+com.aliyun:aliyun-java-sdk-ram:3.1.0
 com.aliyun:aliyun-java-sdk-sts:3.0.0
 com.aliyun.oss:aliyun-sdk-oss:3.13.2
 com.amazonaws:aws-java-sdk-bundle:1.12.316
@@ -323,21 +323,21 @@ org.apache.htrace:htrace-core4:4.1.0-incubating
 org.apache.httpcomponents:httpclient:4.5.6
 org.apache.httpcomponents:httpcore:4.4.10
 org.apache.kafka:kafka-clients:2.8.2
-org.apache.kerby:kerb-admin:2.0.2
-org.apache.kerby:kerb-client:2.0.2
-org.apache.kerby:kerb-common:2.0.2
-org.apache.kerby:kerb-core:2.0.2
-org.apache.kerby:kerb-crypto:2.0.2
-org.apache.kerby:kerb-identity:2.0.2
-org.apache.kerby:kerb-server:2.0.2
-org.apache.kerby:kerb-simplekdc:2.0.2
-org.apache.kerby:kerb-util:2.0.2
-org.apache.kerby:kerby-asn1:2.0.2
-org.apache.kerby:kerby-config:2.0.2
-org.apache.kerby:kerby-pkix:2.0.2
-org.apache.kerby:kerby-util:2.0.2
-org.apache.kerby:kerby-xdr:2.0.2
-org.apache.kerby:token-provider:2.0.2
+org.apache.kerby:kerb-admin:2.0.3
+org.apache.kerby:kerb-client:2.0.3
+org.apache.kerby:kerb-common:2.0.3
+org.apache.kerby:kerb-core:2.0.3
+org.apache.kerby:kerb-crypto:2.0.3
+org.apache.kerby:kerb-identity:2.0.3
+org.apache.kerby:kerb-server:2.0.3
+org.apache.kerby:kerb-simplekdc:2.0.3
+org.apache.kerby:kerb-util:2.0.3
+org.apache.kerby:kerby-asn1:2.0.3
+org.apache.kerby:kerby-config:2.0.3
+org.apache.kerby:kerby-pkix:2.0.3
+org.apache.kerby:kerby-util:2.0.3
+org.apache.kerby:kerby-xdr:2.0.3
+org.apache.kerby:token-provider:2.0.3
 org.apache.solr:solr-solrj:8.8.2
 org.apache.yetus:audience-annotations:0.5.0
 org.apache.zookeeper:zookeeper:3.6.3
@@ -357,10 +357,13 @@ org.eclipse.jetty:jetty-xml:9.4.48.v20220622
 org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.48.v20220622
 org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.48.v20220622
 org.ehcache:ehcache:3.3.1
+org.ini4j:ini4j:0.5.4
+org.jetbrains.kotlin:kotlin-stdlib:1.4.10
+org.jetbrains.kotlin:kotlin-stdlib-common:1.4.10
 org.lz4:lz4-java:1.7.1
 org.objenesis:objenesis:2.6
 org.xerial.snappy:snappy-java:1.0.5
-org.yaml:snakeyaml:1.33
+org.yaml:snakeyaml:2.0
 org.wildfly.openssl:wildfly-openssl:1.1.3.Final
 
 
@@ -516,6 +519,8 @@ Eclipse Public License 1.0
 --------------------------
 
 junit:junit:4.13.2
+org.jacoco:org.jacoco.agent:0.8.5
+
 
 
 HSQL License

hadoop-common-project/hadoop-common/src/main/bin/hadoop

@@ -26,9 +26,9 @@ MYNAME="${BASH_SOURCE-$0}"
 function hadoop_usage
 {
   hadoop_add_option "buildpaths" "attempt to add class files from build tree"
-  hadoop_add_option "hostnames list[,of,host,names]" "hosts to use in slave mode"
+  hadoop_add_option "hostnames list[,of,host,names]" "hosts to use in worker mode"
   hadoop_add_option "loglevel level" "set the log4j level for this command"
-  hadoop_add_option "hosts filename" "list of hosts to use in slave mode"
+  hadoop_add_option "hosts filename" "list of hosts to use in worker mode"
   hadoop_add_option "workers" "turn on worker mode"
 
   hadoop_add_subcommand "checknative" client "check native Hadoop and compression libraries availability"

hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh

@@ -16,7 +16,7 @@
 # limitations under the License.
 
 
-# Run a Hadoop command on all slave hosts.
+# Run a Hadoop command on all worker hosts.
 
 function hadoop_usage
 {

hadoop-common-project/hadoop-common/src/main/conf/log4j.properties

@@ -75,14 +75,6 @@ log4j.appender.console.target=System.err
 log4j.appender.console.layout=org.apache.log4j.PatternLayout
 log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
 
-#
-# TaskLog Appender
-#
-log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
-
-log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
-log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-
 #
 # HDFS block state change log from block manager
 #

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java

@@ -34,6 +34,8 @@
 
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;
+import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -44,6 +46,7 @@
 import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
 import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.GenericsUtil;
 import org.apache.hadoop.util.ServletUtil;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
@@ -338,14 +341,18 @@ public void doGet(HttpServletRequest request, HttpServletResponse response
         out.println(MARKER
             + "Submitted Class Name: <b>" + logName + "</b><br />");
 
-        Logger log = Logger.getLogger(logName);
+        org.slf4j.Logger log = LoggerFactory.getLogger(logName);
         out.println(MARKER
             + "Log Class: <b>" + log.getClass().getName() +"</b><br />");
         if (level != null) {
           out.println(MARKER + "Submitted Level: <b>" + level + "</b><br />");
         }
 
-        process(log, level, out);
+        if (GenericsUtil.isLog4jLogger(logName)) {
+          process(Logger.getLogger(logName), level, out);
+        } else {
+          out.println("Sorry, setting log level is only supported for log4j loggers.<br />");
+        }
       }
 
       out.println(FORMS);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java

@@ -314,7 +314,8 @@ public static void login(final Configuration conf,
 
     String keytabFilename = conf.get(keytabFileKey);
     if (keytabFilename == null || keytabFilename.length() == 0) {
-      throw new IOException("Running in secure mode, but config doesn't have a keytab");
+      throw new IOException(
+          "Running in secure mode, but config doesn't have a keytab for key: " + keytabFileKey);
     }
 
     String principalConfig = conf.get(userNameKey, System

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericsUtil.java

@@ -20,6 +20,7 @@
 
 import java.lang.reflect.Array;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -33,6 +34,14 @@
 @InterfaceStability.Unstable
 public class GenericsUtil {
 
+  private static final String SLF4J_LOG4J_ADAPTER_CLASS = "org.slf4j.impl.Log4jLoggerAdapter";
+
+  /**
+   * Set to false only if log4j adapter class is not found in the classpath. Once set to false,
+   * the utility method should not bother re-loading class again.
+   */
+  private static final AtomicBoolean IS_LOG4J_LOGGER = new AtomicBoolean(true);
+
   /**
    * Returns the Class object (of type <code>Class&lt;T&gt;</code>) of the  
    * argument of type <code>T</code>. 
@@ -87,12 +96,27 @@ public static boolean isLog4jLogger(Class<?> clazz) {
     if (clazz == null) {
       return false;
     }
-    Logger log = LoggerFactory.getLogger(clazz);
+    return isLog4jLogger(clazz.getName());
+  }
+
+  /**
+   * Determine whether the log of the given logger is of Log4J implementation.
+   *
+   * @param logger the logger name, usually class name as string.
+   * @return true if the logger uses Log4J implementation.
+   */
+  public static boolean isLog4jLogger(String logger) {
+    if (logger == null || !IS_LOG4J_LOGGER.get()) {
+      return false;
+    }
+    Logger log = LoggerFactory.getLogger(logger);
     try {
-      Class log4jClass = Class.forName("org.slf4j.impl.Log4jLoggerAdapter");
+      Class<?> log4jClass = Class.forName(SLF4J_LOG4J_ADAPTER_CLASS);
       return log4jClass.isInstance(log);
     } catch (ClassNotFoundException e) {
+      IS_LOG4J_LOGGER.set(false);
       return false;
     }
   }
+
 }

hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md

@@ -370,6 +370,9 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
 |:---- |:---- |
 | `BytesWritten` | Total number of bytes written to DataNode |
 | `BytesRead` | Total number of bytes read from DataNode |
+| `ReadTransferRateNumOps` | Total number of data read transfers |
+| `ReadTransferRateAvgTime` | Average transfer rate of bytes read from DataNode, measured in bytes per second. |
+| `ReadTransferRate`*num*`s(50/75/90/95/99)thPercentileRate` | The 50/75/90/95/99th percentile of the transfer rate of bytes read from DataNode, measured in bytes per second. |
 | `BlocksWritten` | Total number of blocks written to DataNode |
 | `BlocksRead` | Total number of blocks read from DataNode |
 | `BlocksReplicated` | Total number of blocks replicated |

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericsUtil.java

@@ -140,7 +140,7 @@ public void testGetClass() {
 
   @Test
   public void testIsLog4jLogger() throws Exception {
-    assertFalse("False if clazz is null", GenericsUtil.isLog4jLogger(null));
+    assertFalse("False if clazz is null", GenericsUtil.isLog4jLogger((Class<?>) null));
     assertTrue("The implementation is Log4j",
         GenericsUtil.isLog4jLogger(TestGenericsUtil.class));
   }

hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java

@@ -20,7 +20,7 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
-import org.apache.log4j.PropertyConfigurator;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -104,8 +104,6 @@ public class KMSConfiguration {
 
   public static final boolean KEY_AUTHORIZATION_ENABLE_DEFAULT = true;
 
-  private static final String LOG4J_PROPERTIES = "kms-log4j.properties";
-
   static {
     Configuration.addDefaultResource(KMS_DEFAULT_XML);
     Configuration.addDefaultResource(KMS_SITE_XML);
@@ -163,31 +161,20 @@ public static boolean isACLsFileNewer(long time) {
     return newer;
   }
 
-  public static void initLogging() {
-    String confDir = System.getProperty(KMS_CONFIG_DIR);
-    if (confDir == null) {
-      throw new RuntimeException("System property '" +
-          KMSConfiguration.KMS_CONFIG_DIR + "' not defined");
+  /**
+   * Validate whether "kms.config.dir" and "log4j.configuration" are defined in the System
+   * properties. If not, abort the KMS WebServer.
+   */
+  public static void validateSystemProps() {
+    if (System.getProperty(KMS_CONFIG_DIR) == null) {
+      String errorMsg = "System property '" + KMS_CONFIG_DIR + "' not defined";
+      System.err.println("Aborting KMSWebServer because " + errorMsg);
+      throw new RuntimeException(errorMsg);
     }
     if (System.getProperty("log4j.configuration") == null) {
-      System.setProperty("log4j.defaultInitOverride", "true");
-      boolean fromClasspath = true;
-      File log4jConf = new File(confDir, LOG4J_PROPERTIES).getAbsoluteFile();
-      if (log4jConf.exists()) {
-        PropertyConfigurator.configureAndWatch(log4jConf.getPath(), 1000);
-        fromClasspath = false;
-      } else {
-        ClassLoader cl = Thread.currentThread().getContextClassLoader();
-        URL log4jUrl = cl.getResource(LOG4J_PROPERTIES);
-        if (log4jUrl != null) {
-          PropertyConfigurator.configure(log4jUrl);
-        }
-      }
-      LOG.debug("KMS log starting");
-      if (fromClasspath) {
-        LOG.warn("Log4j configuration file '{}' not found", LOG4J_PROPERTIES);
-        LOG.warn("Logging with INFO level to standard output");
-      }
+      String errorMsg = "System property 'log4j.configuration' not defined";
+      System.err.println("Aborting KMSWebServer because " + errorMsg);
+      throw new RuntimeException(errorMsg);
     }
   }
 }