Merge branch 'apache:trunk' into trunk

Author: PradhanPrerak

LICENSE-binary

@@ -229,8 +229,8 @@ com.fasterxml.woodstox:woodstox-core:5.4.0
 com.github.ben-manes.caffeine:caffeine:2.9.3
 com.github.davidmoten:rxjava-extras:0.8.0.17
 com.github.stephenc.jcip:jcip-annotations:1.0-1
-com.google:guice:4.0
-com.google:guice-servlet:4.0
+com.google:guice:5.1.0
+com.google:guice-servlet:5.1.0
 com.google.api.grpc:proto-google-common-protos:1.0.0
 com.google.code.gson:2.9.0
 com.google.errorprone:error_prone_annotations:2.2.0
@@ -362,7 +362,7 @@ org.objenesis:objenesis:2.6
 org.xerial.snappy:snappy-java:1.1.10.4
 org.yaml:snakeyaml:2.0
 org.wildfly.openssl:wildfly-openssl:1.1.3.Final
-software.amazon.awssdk:bundle:jar:2.24.6
+software.amazon.awssdk:bundle:jar:2.25.53
 
 
 --------------------------------------------------------------------------------
@@ -379,7 +379,7 @@ hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h
 
 com.github.luben:zstd-jni:1.5.2-1
-dnsjava:dnsjava:2.1.7
+dnsjava:dnsjava:3.6.1
 org.codehaus.woodstox:stax2-api:4.2.1
 
 

dev-support/bin/create-release

@@ -205,7 +205,7 @@ function set_defaults
   DOCKERRAN=false
 
   CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
-  if [ "$CPU_ARCH" = "aarch64" ]; then
+  if [[ "$CPU_ARCH" = "aarch64" || "$CPU_ARCH" = "arm64" ]]; then
     DOCKERFILE="${BASEDIR}/dev-support/docker/Dockerfile_aarch64"
   fi
 
@@ -514,7 +514,7 @@ function dockermode
 
     # we always force build with the OpenJDK JDK
     # but with the correct version
-    if [ "$CPU_ARCH" = "aarch64" ]; then
+    if [[ "$CPU_ARCH" = "aarch64" || "$CPU_ARCH" = "arm64" ]]; then
       echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-arm64"
     else
       echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"

dev-support/docker/Dockerfile_centos_7

@@ -30,6 +30,13 @@ COPY pkg-resolver pkg-resolver
 RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
     && chmod a+r pkg-resolver/*.json
 
+######
+# Centos 7 has reached its EOL and the packages
+# are no longer available on mirror.centos.org site.
+# Please see https://www.centos.org/centos-linux-eol/
+######
+RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:7
+
 ######
 # Install packages from yum
 ######
@@ -38,8 +45,13 @@ RUN yum update -y \
     && yum groupinstall -y "Development Tools" \
     && yum install -y \
         centos-release-scl \
-        python3 \
-    && yum install -y $(pkg-resolver/resolve.py centos:7)
+        python3
+
+# Apply the script again because centos-release-scl creates new YUM repo files
+RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:7
+
+# hadolint ignore=DL3008,SC2046
+RUN yum install -y $(pkg-resolver/resolve.py centos:7)
 
 # Set GCC 9 as the default C/C++ compiler
 RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc

dev-support/docker/pkg-resolver/install-yasm.sh

@@ -40,7 +40,7 @@ fi
 
 if [ "$version_to_install" == "1.2.0-4" ]; then
   mkdir -p /tmp/yasm &&
-    curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \
+    curl -L -s -S https://archives.fedoraproject.org/pub/archive/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \
       -o /tmp/yasm-1.2.0-4.el7.x86_64.rpm &&
     rpm -Uvh /tmp/yasm-1.2.0-4.el7.x86_64.rpm
 else

dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh

@@ -24,7 +24,7 @@ fi
 if [ "$1" == "centos:7" ] || [ "$1" == "centos:8" ]; then
   cd /etc/yum.repos.d/ || exit &&
     sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* &&
-    sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* &&
+    sed -i 's|# *baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* &&
     yum update -y &&
     cd /root || exit
 else

hadoop-client-modules/hadoop-client-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh

@@ -51,6 +51,8 @@ allowed_expr+="|^[^-]*-default.xml$"
 allowed_expr+="|^[^-]*-version-info.properties$"
 #   * Hadoop's application classloader properties file.
 allowed_expr+="|^org.apache.hadoop.application-classloader.properties$"
+# Comes from dnsjava, not sure if relocatable.
+allowed_expr+="|^messages.properties$"
 # public suffix list used by httpcomponents
 allowed_expr+="|^mozilla/$"
 allowed_expr+="|^mozilla/public-suffix-list.txt$"

hadoop-client-modules/hadoop-client-runtime/pom.xml

@@ -229,6 +229,8 @@
                         <exclude>jnamed*</exclude>
                         <exclude>lookup*</exclude>
                         <exclude>update*</exclude>
+                        <exclude>META-INF/versions/21/*</exclude>
+                        <exclude>META-INF/versions/21/**/*</exclude>
                       </excludes>
                     </filter>
                     <filter>
@@ -243,6 +245,7 @@
                       <excludes>
                         <exclude>META-INF/versions/9/module-info.class</exclude>
                         <exclude>META-INF/versions/11/module-info.class</exclude>
+                        <exclude>META-INF/versions/21/module-info.class</exclude>
                       </excludes>
                     </filter>
 

hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml

@@ -454,4 +454,10 @@
     <Class name="org.apache.hadoop.ipc.internal.ShadedProtobufHelper" />
     <Bug pattern="AT_OPERATION_SEQUENCE_ON_CONCURRENT_ABSTRACTION" />
   </Match>
+
+  <!-- class cast after an assignableFrom check. -->
+  <Match>
+    <Class name="org.apache.hadoop.util.dynamic.DynMethods" />
+    <Bug pattern="BC_UNCONFIRMED_CAST" />
+  </Match>
 </FindBugsFilter>

hadoop-common-project/hadoop-common/pom.xml

@@ -708,6 +708,17 @@
           </filesets>
         </configuration>
       </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <configuration>
+          <sourceFileExcludes>
+            <sourceFileExclude>**/FSProtos.java</sourceFileExclude>
+          </sourceFileExcludes>
+          <excludePackageNames>*.proto:*.tracing:*.protobuf</excludePackageNames>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 
@@ -1279,16 +1290,6 @@
               </execution>
              </executions>
           </plugin>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-javadoc-plugin</artifactId>
-            <configuration>
-              <sourceFileExcludes>
-                <sourceFileExclude>**/FSProtos.java</sourceFileExclude>
-              </sourceFileExcludes>
-              <excludePackageNames>*.proto:*.tracing:*.protobuf</excludePackageNames>
-            </configuration>
-          </plugin>
         </plugins>
       </build>
     </profile>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java

@@ -49,6 +49,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -99,6 +100,7 @@
 import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
 import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
+import org.apache.hadoop.util.ConfigurationHelper;
 import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringInterner;
@@ -1786,6 +1788,26 @@ public <T extends Enum<T>> T getEnum(String name, T defaultValue) {
       : Enum.valueOf(defaultValue.getDeclaringClass(), val);
   }
 
+  /**
+   * Build an enumset from a comma separated list of values.
+   * Case independent.
+   * Special handling of "*" meaning: all values.
+   * @param key key to look for
+   * @param enumClass class of enum
+   * @param ignoreUnknown should unknown values raise an exception?
+   * @return a mutable set of the identified enum values declared in the configuration
+   * @param <E> enumeration type
+   * @throws IllegalArgumentException if one of the entries was unknown and ignoreUnknown is false,
+   *           or there are two entries in the enum which differ only by case.
+   */
+  public <E extends Enum<E>> EnumSet<E> getEnumSet(
+      final String key,
+      final Class<E> enumClass,
+      final boolean ignoreUnknown) throws IllegalArgumentException {
+    final String value = get(key, "");
+    return ConfigurationHelper.parseEnumSet(key, value, enumClass, ignoreUnknown);
+  }
+
   enum ParsedTimeDuration {
     NS {
       TimeUnit unit() { return TimeUnit.NANOSECONDS; }