Merge branch 'apache:trunk' into HADOOP-19638

Author: slfan1989

.github/workflows/stale.yml

@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Close stale PRs
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  stale:
+    if: github.repository == 'apache/hadoop'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          stale-pr-message: >
+            We're closing this stale PR because it has been open for 100 days with
+            no activity. This isn't a judgement on the merit of the PR in any way.
+            It's just a way of keeping the PR queue manageable.
+
+            If you feel like this was a mistake, or you would like to continue
+            working on it, please feel free to re-open it and ask for a committer to
+            remove the stale tag and review again.
+
+            Thanks all for your contribution.
+          days-before-stale: 100
+          days-before-close: 0
+          close-pr-label: 'stale'

BUILDING.txt

@@ -31,7 +31,7 @@ On Linux / Mac:
 
     $ ./start-build-env.sh [OS platform]
 
-   - [OS Platform] One of [rockylinux_8, debian_10, ubuntu_20, ubuntu_24, windows_10].
+   - [OS Platform] One of [rockylinux_8, debian_11, ubuntu_20, ubuntu_24, windows_10].
         Default is 'ubuntu_20'.
         Note: Currently only default ('ubuntu_20') is supported on arm machine
 

LICENSE-binary

@@ -229,6 +229,25 @@ com.fasterxml.woodstox:woodstox-core:5.4.0
 com.github.ben-manes.caffeine:caffeine:2.9.3
 com.github.davidmoten:rxjava-extras:0.8.0.17
 com.github.stephenc.jcip:jcip-annotations:1.0-1
+com.google.api:api-common:2.47.2
+com.google.api-client:google-api-client:2.7.2
+com.google.api:gax:2.64.2
+com.google.api:gax-grpc:2.64.2
+com.google.api:gax-httpjson:2.64.2
+com.google.api.grpc:gapic-google-cloud-storage-v2:2.52.0
+com.google.api.grpc:grpc-google-cloud-storage-v2:2.52.0
+com.google.api.grpc:proto-google-cloud-monitoring-v3:3.52.0
+com.google.api.grpc:proto-google-cloud-storage-v2:2.52.0
+com.google.api.grpc:proto-google-common-protos:2.55.2
+com.google.api.grpc:proto-google-iam-v1:1.50.2
+com.google.apis:google-api-services-storage:v1-rev20250420-2.0.0
+com.google.auto.value:auto-value-annotations:1.11.0
+com.google.cloud:google-cloud-core-grpc:2.54.2
+com.google.cloud:google-cloud-core-http:2.54.2
+com.google.cloud:google-cloud-core:2.54.2
+com.google.cloud:google-cloud-monitoring:3.52.0
+com.google.cloud.opentelemetry:exporter-metrics:0.33.0
+com.google.errorprone:error_prone_annotations:2.36.0
 com.google:guice:5.1.0
 com.google:guice-servlet:5.1.0
 com.google.api.grpc:proto-google-common-protos:1.0.0
@@ -237,9 +256,18 @@ com.google.errorprone:error_prone_annotations:2.5.1
 com.google.j2objc:j2objc-annotations:1.3
 com.google.json-simple:json-simple:1.1.1
 com.google.guava:failureaccess:1.0
+com.google.guava:failureaccess:1.0.2
 com.google.guava:guava:20.0
 com.google.guava:guava:32.0.1-jre
+com.google.guava:guava:33.1.0-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
+com.google.http-client:google-http-client-apache-v2:1.46.3
+com.google.http-client:google-http-client-appengine:1.46.3
+com.google.http-client:google-http-client-gson:1.46.3
+com.google.http-client:google-http-client-jackson2:1.46.3
+com.google.http-client:google-http-client:1.46.3
+com.google.j2objc:j2objc-annotations:3.0.0
+com.google.oauth-client:google-oauth-client:1.37.0
 com.microsoft.azure:azure-storage:7.0.0
 com.nimbusds:nimbus-jose-jwt:9.37.2
 com.zaxxer:HikariCP:4.0.3
@@ -251,13 +279,29 @@ commons-daemon:commons-daemon:1.0.13
 commons-io:commons-io:2.16.1
 commons-net:commons-net:3.9.0
 de.ruedigermoeller:fst:2.50
+io.grpc:grpc-alts:1.70.0
 io.grpc:grpc-api:1.69.0
+io.grpc:grpc-api:1.70.0
+io.grpc:grpc-auth:1.70.0
 io.grpc:grpc-context:1.69.0
+io.grpc:grpc-context:1.70.0
 io.grpc:grpc-core:1.69.0
+io.grpc:grpc-core:1.70.0
+io.grpc:grpc-googleapis:1.70.0
+io.grpc:grpc-grpclb:1.70.0
+io.grpc:grpc-inprocess:1.70.0
 io.grpc:grpc-netty:1.69.0
+io.grpc:grpc-netty-shaded:1.70.0
+io.grpc:grpc-opentelemetry:1.70.0
 io.grpc:grpc-protobuf:1.69.0
+io.grpc:grpc-protobuf:1.70.0
 io.grpc:grpc-protobuf-lite:1.69.0
+io.grpc:grpc-rls:1.70.0
+io.grpc:grpc-services:1.70.0
 io.grpc:grpc-stub:1.69.0
+io.grpc:grpc-stub:1.70.0
+io.grpc:grpc-util:1.70.0
+io.grpc:grpc-xds:1.70.0
 io.netty:netty-all:4.1.118.Final
 io.netty:netty-buffer:4.1.118.Final
 io.netty:netty-codec:4.1.118.Final
@@ -289,7 +333,19 @@ io.netty:netty-transport-native-epoll:4.1.118.Final
 io.netty:netty-transport-native-kqueue:4.1.118.Final
 io.netty:netty-resolver-dns-native-macos:4.1.118.Final
 io.opencensus:opencensus-api:0.12.3
+io.opencensus:opencensus-api:0.31.1
 io.opencensus:opencensus-contrib-grpc-metrics:0.12.3
+io.opencensus:opencensus-contrib-http-util:0.31.1
+io.opentelemetry.contrib:opentelemetry-gcp-resources:1.37.0-alpha
+io.opentelemetry:opentelemetry-api:1.47.0
+io.opentelemetry:opentelemetry-context:1.47.0
+io.opentelemetry:opentelemetry-sdk-common:1.47.0
+io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.47.0
+io.opentelemetry:opentelemetry-sdk:1.47.0
+io.opentelemetry:opentelemetry-sdk-logs:1.47.0
+io.opentelemetry:opentelemetry-sdk-metrics:1.47.0
+io.opentelemetry:opentelemetry-sdk-trace:1.47.0
+io.opentelemetry.semconv:opentelemetry-semconv:1.29.0-alpha
 io.reactivex:rxjava:1.3.8
 io.reactivex:rxjava-string:1.1.1
 io.reactivex:rxnetty:0.4.20
@@ -340,6 +396,7 @@ org.apache.solr:solr-solrj:8.11.2
 org.apache.yetus:audience-annotations:0.5.0
 org.apache.zookeeper:zookeeper:3.8.4
 org.codehaus.jettison:jettison:1.5.4
+org.conscrypt:conscrypt-openjdk-uber:2.5.2
 org.eclipse.jetty:jetty-annotations:9.4.57.v20241219
 org.eclipse.jetty:jetty-http:9.4.57.v20241219
 org.eclipse.jetty:jetty-io:9.4.57.v20241219
@@ -394,18 +451,22 @@ hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js
 leveldb v1.13
 
+com.google.auth:google-auth-library-credentials:1.33.1
+com.google.auth:google-auth-library-oauth2-http:1.33.1
 com.google.protobuf:protobuf-java:2.5.0
 com.google.protobuf:protobuf-java:3.25.5
 com.google.re2j:re2j:1.1
 com.jcraft:jsch:0.1.55
 com.thoughtworks.paranamer:paranamer:2.3
 jakarta.activation:jakarta.activation-api:1.2.1
+org.checkerframework:checker-qual:3.49.0
 org.fusesource.leveldbjni:leveldbjni-all:1.8
 org.jline:jline:3.9.0
 org.hamcrest:hamcrest-core:1.3
 org.ow2.asm:asm:5.0.4
 org.ow2.asm:asm-commons:6.0
 org.ow2.asm:asm-tree:6.0
+org.threeten:threetenbp:1.7.0
 
 
 MIT License
@@ -510,6 +571,7 @@ Eclipse Distribution License (EDL) 1.0
 --------------------------
 
 jakarta.xml.bind:jakarta.xml.bind-api:2.3.3
+org.glassfish.jaxb:jaxb-runtime:2.3.9
 
 Eclipse Public License 1.0
 --------------------------

dev-support/Jenkinsfile

@@ -91,10 +91,10 @@ pipeline {
                     '''
                 }
 
-                dir("${WORKSPACE}/debian-10") {
+                dir("${WORKSPACE}/debian-11") {
                     sh '''#!/usr/bin/env bash
 
-                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10
+                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-11
                     '''
                 }
 
@@ -157,12 +157,12 @@ pipeline {
         // C++/C++ build/platform.
         // This stage serves as a means of cross platform validation, which is
         // really needed to ensure that any C++ related/platform change doesn't
-        // break the Hadoop build on Debian 10.
-        stage ('precommit-run Debian 10') {
+        // break the Hadoop build on Debian 11.
+        stage ('precommit-run Debian 11') {
             environment {
-                SOURCEDIR = "${WORKSPACE}/debian-10/src"
-                PATCHDIR = "${WORKSPACE}/debian-10/out"
-                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
+                SOURCEDIR = "${WORKSPACE}/debian-11/src"
+                PATCHDIR = "${WORKSPACE}/debian-11/out"
+                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_11"
                 IS_OPTIONAL = 1
             }
 
@@ -182,7 +182,7 @@ pipeline {
                 failure {
                     sh '''#!/usr/bin/env bash
 
-                    cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}"
+                    cp -Rp "${WORKSPACE}/debian-11/out" "${WORKSPACE}"
                     '''
                     archiveArtifacts "out/**"
                 }

dev-support/docker/Dockerfile_debian_10 renamed to dev-support/docker/Dockerfile_debian_11

@@ -17,7 +17,7 @@
 # Dockerfile for installing the necessary dependencies for building Hadoop.
 # See BUILDING.txt.
 
-FROM debian:10
+FROM debian:11
 
 WORKDIR /root
 
@@ -29,65 +29,63 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
 RUN echo APT::Install-Suggests "0"\; >>  /etc/apt/apt.conf.d/10disableextras
 
-ENV DEBIAN_FRONTEND noninteractive
-ENV DEBCONF_TERSE true
+ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBCONF_TERSE=true
 
 ######
 # Platform package dependency resolver
 ######
 COPY pkg-resolver pkg-resolver
 RUN chmod a+x pkg-resolver/install-pkg-resolver.sh
-RUN pkg-resolver/install-pkg-resolver.sh debian:10
+RUN pkg-resolver/install-pkg-resolver.sh debian:11
 
 ######
 # Install packages from apt
 ######
 # hadolint ignore=DL3008,SC2046
 RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:10) \
-    && echo 'deb http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list \
-    && apt-get -q update \
-    && apt-get -q install -y --no-install-recommends -t bullseye $(pkg-resolver/resolve.py --release=bullseye debian:10) \
+    && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:11) \
     && apt-get clean \
+    && update-java-alternatives -s java-1.11.0-openjdk-amd64 \
     && rm -rf /var/lib/apt/lists/*
 
 # TODO : Set locale
 
 ######
 # Set env vars required to build Hadoop
 ######
-ENV MAVEN_HOME /usr
+ENV MAVEN_HOME=/usr
 # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
-ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
+ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
 
 #######
 # Set env vars for SpotBugs 4.2.2
 #######
-ENV SPOTBUGS_HOME /opt/spotbugs
+ENV SPOTBUGS_HOME=/opt/spotbugs
 
 #######
 # Set env vars for Google Protobuf 3.21.12
 #######
-ENV PROTOBUF_HOME /opt/protobuf
-ENV PATH "${PATH}:/opt/protobuf/bin"
+ENV PROTOBUF_HOME=/opt/protobuf
+ENV PATH="${PATH}:/opt/protobuf/bin"
 
 ###
 # Avoid out of memory errors in builds
 ###
-ENV MAVEN_OPTS -Xms256m -Xmx3072m
+ENV MAVEN_OPTS="-Xms256m -Xmx3072m"
 
 # Skip gpg verification when downloading Yetus via yetus-wrapper
-ENV HADOOP_SKIP_YETUS_VERIFICATION true
+ENV HADOOP_SKIP_YETUS_VERIFICATION=true
 
 ####
 # Install packages
 ####
-RUN pkg-resolver/install-cmake.sh debian:10
-RUN pkg-resolver/install-spotbugs.sh debian:10
-RUN pkg-resolver/install-boost.sh debian:10
-RUN pkg-resolver/install-protobuf.sh debian:10
-RUN pkg-resolver/install-hadolint.sh debian:10
-RUN pkg-resolver/install-intel-isa-l.sh debian:10
+RUN pkg-resolver/install-cmake.sh debian:11
+RUN pkg-resolver/install-spotbugs.sh debian:11
+RUN pkg-resolver/install-boost.sh debian:11
+RUN pkg-resolver/install-protobuf.sh debian:11
+RUN pkg-resolver/install-hadolint.sh debian:11
+RUN pkg-resolver/install-intel-isa-l.sh debian:11
 
 ###
 # Everything past this point is either not needed for testing or breaks Yetus.

dev-support/docker/Dockerfile_rockylinux_8

@@ -30,6 +30,11 @@ COPY pkg-resolver pkg-resolver
 RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
     && chmod a+r pkg-resolver/*.json
 
+####
+# Install EPEL
+####
+RUN pkg-resolver/install-epel.sh rockylinux:8
+
 ######
 # Install packages from yum
 ######
@@ -38,11 +43,6 @@ RUN yum update -y \
     && yum install -y python3 \
     && yum install -y $(pkg-resolver/resolve.py rockylinux:8)
 
-####
-# Install EPEL
-####
-RUN pkg-resolver/install-epel.sh rockylinux:8
-
 RUN dnf --enablerepo=powertools install -y \
     doxygen \
     snappy-devel \

dev-support/docker/pkg-resolver/install-pkg-resolver.sh

@@ -24,7 +24,7 @@ fi
 chmod a+x pkg-resolver/*.sh pkg-resolver/*.py
 chmod a+r pkg-resolver/*.json
 
-if [ "$1" == "debian:10" ]; then
+if [ "$1" == "debian:11" ]; then
   apt-get -q update
   apt-get -q install -y --no-install-recommends python3 \
     python3-pip \