Skip to content

Commit f8e7496

Browse files
slfan1989slfan1989
authored
Merge branch 'apache:trunk' into HADOOP-19638 
2 parents 577479b + 960a2ef commit f8e7496

File tree

584 files changed

+13117
-9539
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

584 files changed

+13117
-9539
lines changed

‎BUILDING.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ On Linux / Mac:
3131

3232
$ ./start-build-env.sh [OS platform]
3333

34-
- [OS Platform] One of [centos_7, centos_8, debian_10, ubuntu_20, ubuntu_24, windows_10].
34+
- [OS Platform] One of [rockylinux_8, debian_10, ubuntu_20, ubuntu_24, windows_10].
3535
Default is 'ubuntu_20'.
3636
Note: Currently only default ('ubuntu_20') is supported on arm machine
3737

@@ -461,7 +461,7 @@ path. This is necessary at least for Homebrewed OpenSSL.
461461

462462
----------------------------------------------------------------------------------
463463

464-
Building on CentOS 8
464+
Building on Rocky Linux 8
465465

466466
----------------------------------------------------------------------------------
467467

@@ -482,7 +482,7 @@ Building on CentOS 8
482482
$ sudo make install
483483
$ cd ..
484484

485-
* Install libraries provided by CentOS 8.
485+
* Install libraries provided by Rocky Linux 8.
486486
$ sudo dnf install libtirpc-devel zlib-devel lz4-devel bzip2-devel openssl-devel cyrus-sasl-devel libpmem-devel
487487

488488
* Install GCC 9.3.0

‎dev-support/Jenkinsfile‎

Lines changed: 8 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -83,17 +83,11 @@ pipeline {
8383
// optional stages after the first one.
8484
stage ('setup sources') {
8585
steps {
86-
dir("${WORKSPACE}/centos-7") {
87-
sh '''#!/usr/bin/env bash
88-
89-
cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-7
90-
'''
91-
}
9286

93-
dir("${WORKSPACE}/centos-8") {
87+
dir("${WORKSPACE}/rockylinux-8") {
9488
sh '''#!/usr/bin/env bash
9589
96-
cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8
90+
cp -Rp ${WORKSPACE}/src ${WORKSPACE}/rockylinux-8
9791
'''
9892
}
9993

@@ -117,58 +111,12 @@ pipeline {
117111
// C++/C++ build/platform.
118112
// This stage serves as a means of cross platform validation, which is
119113
// really needed to ensure that any C++ related/platform change doesn't
120-
// break the Hadoop build on Centos 7.
121-
stage ('precommit-run Centos 7') {
122-
environment {
123-
SOURCEDIR = "${WORKSPACE}/centos-7/src"
124-
PATCHDIR = "${WORKSPACE}/centos-7/out"
125-
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_7"
126-
IS_OPTIONAL = 1
127-
}
128-
129-
steps {
130-
withCredentials(getGithubCreds()) {
131-
sh '''#!/usr/bin/env bash
132-
133-
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
134-
"${SOURCEDIR}/dev-support/jenkins.sh" run_ci
135-
'''
136-
}
137-
}
138-
139-
post {
140-
// Since this is an optional platform, we want to copy the artifacts
141-
// and archive it only if the build fails, to help with debugging.
142-
failure {
143-
sh '''#!/usr/bin/env bash
144-
145-
cp -Rp "${WORKSPACE}/centos-7/out" "${WORKSPACE}"
146-
'''
147-
archiveArtifacts "out/**"
148-
}
149-
150-
cleanup() {
151-
script {
152-
sh '''#!/usr/bin/env bash
153-
154-
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
155-
"${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
156-
'''
157-
}
158-
}
159-
}
160-
}
161-
162-
// This is an optional stage which runs only when there's a change in
163-
// C++/C++ build/platform.
164-
// This stage serves as a means of cross platform validation, which is
165-
// really needed to ensure that any C++ related/platform change doesn't
166-
// break the Hadoop build on Centos 8.
167-
stage ('precommit-run Centos 8') {
114+
// break the Hadoop build on Rocky Linux 8.
115+
stage ('precommit-run Rocky Linux 8') {
168116
environment {
169-
SOURCEDIR = "${WORKSPACE}/centos-8/src"
170-
PATCHDIR = "${WORKSPACE}/centos-8/out"
171-
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
117+
SOURCEDIR = "${WORKSPACE}/rockylinux-8/src"
118+
PATCHDIR = "${WORKSPACE}/rockylinux-8/out"
119+
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_rockylinux_8"
172120
IS_OPTIONAL = 1
173121
}
174122

@@ -188,7 +136,7 @@ pipeline {
188136
failure {
189137
sh '''#!/usr/bin/env bash
190138
191-
cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}"
139+
cp -Rp "${WORKSPACE}/rockylinux-8/out" "${WORKSPACE}"
192140
'''
193141
archiveArtifacts "out/**"
194142
}

‎dev-support/docker/Dockerfile_centos_7‎

Lines changed: 0 additions & 108 deletions
This file was deleted.

‎dev-support/docker/Dockerfile_centos_8‎ renamed to ‎dev-support/docker/Dockerfile_rockylinux_8‎

Lines changed: 45 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
# Dockerfile for installing the necessary dependencies for building Hadoop.
1818
# See BUILDING.txt.
1919

20-
FROM centos:8
20+
FROM rockylinux:8
2121

2222
WORKDIR /root
2323

@@ -30,25 +30,18 @@ COPY pkg-resolver pkg-resolver
3030
RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
3131
&& chmod a+r pkg-resolver/*.json
3232

33-
######
34-
# Centos 8 has reached its EOL and the packages
35-
# are no longer available on mirror.centos.org site.
36-
# Please see https://www.centos.org/centos-linux-eol/
37-
######
38-
RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:8
39-
4033
######
4134
# Install packages from yum
4235
######
4336
# hadolint ignore=DL3008,SC2046
4437
RUN yum update -y \
4538
&& yum install -y python3 \
46-
&& yum install -y $(pkg-resolver/resolve.py centos:8)
39+
&& yum install -y $(pkg-resolver/resolve.py rockylinux:8)
4740

4841
####
4942
# Install EPEL
5043
####
51-
RUN pkg-resolver/install-epel.sh centos:8
44+
RUN pkg-resolver/install-epel.sh rockylinux:8
5245

5346
RUN dnf --enablerepo=powertools install -y \
5447
doxygen \
@@ -69,50 +62,65 @@ SHELL ["/bin/bash", "--login", "-c"]
6962
# Set the environment variables needed for CMake
7063
# to find and use GCC 9 for compilation
7164
######
72-
ENV GCC_HOME "/opt/rh/gcc-toolset-9"
73-
ENV CC "${GCC_HOME}/root/usr/bin/gcc"
74-
ENV CXX "${GCC_HOME}/root/usr/bin/g++"
75-
ENV MODULES_RUN_QUARANTINE "LD_LIBRARY_PATH LD_PRELOAD"
76-
ENV MODULES_CMD "/usr/share/Modules/libexec/modulecmd.tcl"
77-
ENV SHLVL 1
78-
ENV MODULEPATH "/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles"
79-
ENV MODULEPATH_modshare "/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1"
80-
ENV MODULESHOME "/usr/share/Modules"
81-
ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
82-
ENV PCP_DIR "${GCC_HOME}/root"
83-
ENV MANPATH "${GCC_HOME}/root/usr/share/man::"
84-
ENV PATH "${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
85-
ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig"
86-
ENV INFOPATH "${GCC_HOME}/root/usr/share/info"
65+
ENV GCC_HOME="/opt/rh/gcc-toolset-9"
66+
ENV CC="${GCC_HOME}/root/usr/bin/gcc"
67+
ENV CXX="${GCC_HOME}/root/usr/bin/g++"
68+
ENV MODULES_RUN_QUARANTINE="LD_LIBRARY_PATH LD_PRELOAD"
69+
ENV MODULES_CMD="/usr/share/Modules/libexec/modulecmd.tcl"
70+
ENV SHLVL=1
71+
ENV MODULEPATH="/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles"
72+
ENV MODULEPATH_modshare="/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1"
73+
ENV MODULESHOME="/usr/share/Modules"
74+
ENV LD_LIBRARY_PATH="${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
75+
ENV PCP_DIR="${GCC_HOME}/root"
76+
ENV MANPATH="${GCC_HOME}/root/usr/share/man::"
77+
ENV PATH="${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
78+
ENV PKG_CONFIG_PATH="${GCC_HOME}/root/usr/lib64/pkgconfig"
79+
ENV INFOPATH="${GCC_HOME}/root/usr/share/info"
8780

8881
# TODO: Set locale
8982

9083
######
9184
# Set env vars required to build Hadoop
9285
######
93-
ENV MAVEN_HOME /opt/maven
94-
ENV PATH "${PATH}:${MAVEN_HOME}/bin"
86+
ENV MAVEN_HOME=/opt/maven
87+
ENV PATH="${PATH}:${MAVEN_HOME}/bin"
9588
# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
96-
ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
89+
ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
9790

9891
#######
9992
# Set env vars for SpotBugs
10093
#######
101-
ENV SPOTBUGS_HOME /opt/spotbugs
94+
ENV SPOTBUGS_HOME=/opt/spotbugs
10295

10396
#######
10497
# Set env vars for Google Protobuf 3.21.12
10598
#######
106-
ENV PROTOBUF_HOME /opt/protobuf
107-
ENV PATH "${PATH}:/opt/protobuf/bin"
99+
ENV PROTOBUF_HOME=/opt/protobuf
100+
ENV PATH="${PATH}:${PROTOBUF_HOME}/bin"
101+
102+
# Skip gpg verification when downloading Yetus via yetus-wrapper
103+
ENV HADOOP_SKIP_YETUS_VERIFICATION=true
108104

109105
######
110106
# Install packages
111107
######
112-
RUN pkg-resolver/install-maven.sh centos:8
113-
RUN pkg-resolver/install-cmake.sh centos:8
114-
RUN pkg-resolver/install-boost.sh centos:8
115-
RUN pkg-resolver/install-spotbugs.sh centos:8
116-
RUN pkg-resolver/install-protobuf.sh centos:8
117-
RUN pkg-resolver/install-zstandard.sh centos:8
108+
RUN pkg-resolver/install-maven.sh rockylinux:8
109+
RUN pkg-resolver/install-cmake.sh rockylinux:8
110+
RUN pkg-resolver/install-boost.sh rockylinux:8
111+
RUN pkg-resolver/install-spotbugs.sh rockylinux:8
112+
RUN pkg-resolver/install-protobuf.sh rockylinux:8
113+
RUN pkg-resolver/install-zstandard.sh rockylinux:8
118114
RUN pkg-resolver/install-common-pkgs.sh
115+
116+
###
117+
# Everything past this point is either not needed for testing or breaks Yetus.
118+
# So tell Yetus not to read the rest of the file:
119+
# YETUS CUT HERE
120+
###
121+
122+
# Add a welcome message and environment checks.
123+
COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
124+
RUN chmod 755 /root/hadoop_env_checks.sh
125+
# hadolint ignore=SC2016
126+
RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc

‎dev-support/docker/Dockerfile_windows_10‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,11 +74,11 @@ RUN powershell Invoke-WebRequest -Uri https://github.com/facebook/zstd/releases/
7474
RUN powershell Expand-Archive -Path $Env:TEMP\zstd-v1.5.4-win64.zip -DestinationPath "C:\ZStd"
7575
RUN setx PATH "%PATH%;C:\ZStd"
7676

77-
# Install libopenssl 3.1.4 needed for rsync 3.2.7.
78-
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libopenssl-3.1.4-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libopenssl-3.1.4-1-x86_64.pkg.tar.zst
79-
RUN powershell zstd -d $Env:TEMP\libopenssl-3.1.4-1-x86_64.pkg.tar.zst -o $Env:TEMP\libopenssl-3.1.4-1-x86_64.pkg.tar
77+
# Install libopenssl 3.5.2-1 needed for rsync 3.2.7.
78+
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libopenssl-3.5.2-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libopenssl-3.5.2-1-x86_64.pkg.tar.zst
79+
RUN powershell zstd -d $Env:TEMP\libopenssl-3.5.2-1-x86_64.pkg.tar.zst -o $Env:TEMP\libopenssl-3.5.2-1-x86_64.pkg.tar
8080
RUN powershell mkdir "C:\LibOpenSSL"
81-
RUN powershell tar -xvf $Env:TEMP\libopenssl-3.1.4-1-x86_64.pkg.tar -C "C:\LibOpenSSL"
81+
RUN powershell tar -xvf $Env:TEMP\libopenssl-3.5.2-1-x86_64.pkg.tar -C "C:\LibOpenSSL"
8282

8383
# Install libxxhash 0.8.3 needed for rsync 3.2.7.
8484
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libxxhash-0.8.3-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libxxhash-0.8.3-1-x86_64.pkg.tar.zst

‎dev-support/docker/pkg-resolver/install-epel.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ fi
4040

4141
if [ "$version_to_install" == "8" ]; then
4242
mkdir -p /tmp/epel &&
43-
curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \
43+
curl -L -s -S https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \
4444
-o /tmp/epel/epel-release-latest-8.noarch.rpm &&
4545
rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm
4646
else

0 commit comments

Comments
 (0)