Skip to content

Commit 5666603

Browse files
pan3793cnauroth
pan3793
authored and
cnauroth
committed
HADOOP-19661. Migrate CentOS 8 to Rocky Linux 8 in build env Dockerfile
Closes #7900 Signed-off-by: Shilun Fan <[email protected]> Signed-off-by: Chris Nauroth <[email protected]>
1 parent 631939f commit 5666603

File tree

8 files changed

+81
-106
lines changed

8 files changed

+81
-106
lines changed

‎BUILDING.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ On Linux / Mac:
3131

3232
$ ./start-build-env.sh [OS platform]
3333

34-
- [OS Platform] One of [centos_7, centos_8, debian_10, ubuntu_20, ubuntu_24, windows_10].
34+
- [OS Platform] One of [rockylinux_8, debian_10, ubuntu_20, ubuntu_24, windows_10].
3535
Default is 'ubuntu_20'.
3636
Note: Currently only default ('ubuntu_20') is supported on arm machine
3737

@@ -461,7 +461,7 @@ path. This is necessary at least for Homebrewed OpenSSL.
461461

462462
----------------------------------------------------------------------------------
463463

464-
Building on CentOS 8
464+
Building on Rocky Linux 8
465465

466466
----------------------------------------------------------------------------------
467467

@@ -482,7 +482,7 @@ Building on CentOS 8
482482
$ sudo make install
483483
$ cd ..
484484

485-
* Install libraries provided by CentOS 8.
485+
* Install libraries provided by Rocky Linux 8.
486486
$ sudo dnf install libtirpc-devel zlib-devel lz4-devel bzip2-devel openssl-devel cyrus-sasl-devel libpmem-devel
487487

488488
* Install GCC 9.3.0

‎dev-support/Jenkinsfile‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -84,10 +84,10 @@ pipeline {
8484
stage ('setup sources') {
8585
steps {
8686

87-
dir("${WORKSPACE}/centos-8") {
87+
dir("${WORKSPACE}/rockylinux-8") {
8888
sh '''#!/usr/bin/env bash
8989
90-
cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8
90+
cp -Rp ${WORKSPACE}/src ${WORKSPACE}/rockylinux-8
9191
'''
9292
}
9393

@@ -111,11 +111,11 @@ pipeline {
111111
// C++/C++ build/platform.
112112
// This stage serves as a means of cross platform validation, which is
113113
// really needed to ensure that any C++ related/platform change doesn't
114-
// break the Hadoop build on Centos 8.
115-
stage ('precommit-run Centos 8') {
114+
// break the Hadoop build on Rocky Linux 8.
115+
stage ('precommit-run Rocky Linux 8') {
116116
environment {
117-
SOURCEDIR = "${WORKSPACE}/centos-8/src"
118-
PATCHDIR = "${WORKSPACE}/centos-8/out"
117+
SOURCEDIR = "${WORKSPACE}/rockylinux-8/src"
118+
PATCHDIR = "${WORKSPACE}/rockylinux-8/out"
119119
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
120120
IS_OPTIONAL = 1
121121
}
@@ -136,7 +136,7 @@ pipeline {
136136
failure {
137137
sh '''#!/usr/bin/env bash
138138
139-
cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}"
139+
cp -Rp "${WORKSPACE}/rockylinux-8/out" "${WORKSPACE}"
140140
'''
141141
archiveArtifacts "out/**"
142142
}

‎dev-support/docker/Dockerfile_centos_8‎

Lines changed: 45 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
# Dockerfile for installing the necessary dependencies for building Hadoop.
1818
# See BUILDING.txt.
1919

20-
FROM centos:8
20+
FROM rockylinux:8
2121

2222
WORKDIR /root
2323

@@ -30,25 +30,18 @@ COPY pkg-resolver pkg-resolver
3030
RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
3131
&& chmod a+r pkg-resolver/*.json
3232

33-
######
34-
# Centos 8 has reached its EOL and the packages
35-
# are no longer available on mirror.centos.org site.
36-
# Please see https://www.centos.org/centos-linux-eol/
37-
######
38-
RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:8
39-
4033
######
4134
# Install packages from yum
4235
######
4336
# hadolint ignore=DL3008,SC2046
4437
RUN yum update -y \
4538
&& yum install -y python3 \
46-
&& yum install -y $(pkg-resolver/resolve.py centos:8)
39+
&& yum install -y $(pkg-resolver/resolve.py rockylinux:8)
4740

4841
####
4942
# Install EPEL
5043
####
51-
RUN pkg-resolver/install-epel.sh centos:8
44+
RUN pkg-resolver/install-epel.sh rockylinux:8
5245

5346
RUN dnf --enablerepo=powertools install -y \
5447
doxygen \
@@ -69,50 +62,65 @@ SHELL ["/bin/bash", "--login", "-c"]
6962
# Set the environment variables needed for CMake
7063
# to find and use GCC 9 for compilation
7164
######
72-
ENV GCC_HOME "/opt/rh/gcc-toolset-9"
73-
ENV CC "${GCC_HOME}/root/usr/bin/gcc"
74-
ENV CXX "${GCC_HOME}/root/usr/bin/g++"
75-
ENV MODULES_RUN_QUARANTINE "LD_LIBRARY_PATH LD_PRELOAD"
76-
ENV MODULES_CMD "/usr/share/Modules/libexec/modulecmd.tcl"
77-
ENV SHLVL 1
78-
ENV MODULEPATH "/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles"
79-
ENV MODULEPATH_modshare "/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1"
80-
ENV MODULESHOME "/usr/share/Modules"
81-
ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
82-
ENV PCP_DIR "${GCC_HOME}/root"
83-
ENV MANPATH "${GCC_HOME}/root/usr/share/man::"
84-
ENV PATH "${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
85-
ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig"
86-
ENV INFOPATH "${GCC_HOME}/root/usr/share/info"
65+
ENV GCC_HOME="/opt/rh/gcc-toolset-9"
66+
ENV CC="${GCC_HOME}/root/usr/bin/gcc"
67+
ENV CXX="${GCC_HOME}/root/usr/bin/g++"
68+
ENV MODULES_RUN_QUARANTINE="LD_LIBRARY_PATH LD_PRELOAD"
69+
ENV MODULES_CMD="/usr/share/Modules/libexec/modulecmd.tcl"
70+
ENV SHLVL=1
71+
ENV MODULEPATH="/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles"
72+
ENV MODULEPATH_modshare="/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1"
73+
ENV MODULESHOME="/usr/share/Modules"
74+
ENV LD_LIBRARY_PATH="${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
75+
ENV PCP_DIR="${GCC_HOME}/root"
76+
ENV MANPATH="${GCC_HOME}/root/usr/share/man::"
77+
ENV PATH="${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
78+
ENV PKG_CONFIG_PATH="${GCC_HOME}/root/usr/lib64/pkgconfig"
79+
ENV INFOPATH="${GCC_HOME}/root/usr/share/info"
8780

8881
# TODO: Set locale
8982

9083
######
9184
# Set env vars required to build Hadoop
9285
######
93-
ENV MAVEN_HOME /opt/maven
94-
ENV PATH "${PATH}:${MAVEN_HOME}/bin"
86+
ENV MAVEN_HOME=/opt/maven
87+
ENV PATH="${PATH}:${MAVEN_HOME}/bin"
9588
# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
96-
ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
89+
ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0
9790

9891
#######
9992
# Set env vars for SpotBugs
10093
#######
101-
ENV SPOTBUGS_HOME /opt/spotbugs
94+
ENV SPOTBUGS_HOME=/opt/spotbugs
10295

10396
#######
10497
# Set env vars for Google Protobuf 3.21.12
10598
#######
106-
ENV PROTOBUF_HOME /opt/protobuf
107-
ENV PATH "${PATH}:/opt/protobuf/bin"
99+
ENV PROTOBUF_HOME=/opt/protobuf
100+
ENV PATH="${PATH}:${PROTOBUF_HOME}/bin"
101+
102+
# Skip gpg verification when downloading Yetus via yetus-wrapper
103+
ENV HADOOP_SKIP_YETUS_VERIFICATION=true
108104

109105
######
110106
# Install packages
111107
######
112-
RUN pkg-resolver/install-maven.sh centos:8
113-
RUN pkg-resolver/install-cmake.sh centos:8
114-
RUN pkg-resolver/install-boost.sh centos:8
115-
RUN pkg-resolver/install-spotbugs.sh centos:8
116-
RUN pkg-resolver/install-protobuf.sh centos:8
117-
RUN pkg-resolver/install-zstandard.sh centos:8
108+
RUN pkg-resolver/install-maven.sh rockylinux:8
109+
RUN pkg-resolver/install-cmake.sh rockylinux:8
110+
RUN pkg-resolver/install-boost.sh rockylinux:8
111+
RUN pkg-resolver/install-spotbugs.sh rockylinux:8
112+
RUN pkg-resolver/install-protobuf.sh rockylinux:8
113+
RUN pkg-resolver/install-zstandard.sh rockylinux:8
118114
RUN pkg-resolver/install-common-pkgs.sh
115+
116+
###
117+
# Everything past this point is either not needed for testing or breaks Yetus.
118+
# So tell Yetus not to read the rest of the file:
119+
# YETUS CUT HERE
120+
###
121+
122+
# Add a welcome message and environment checks.
123+
COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
124+
RUN chmod 755 /root/hadoop_env_checks.sh
125+
# hadolint ignore=SC2016
126+
RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc

‎dev-support/docker/pkg-resolver/install-epel.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ fi
4040

4141
if [ "$version_to_install" == "8" ]; then
4242
mkdir -p /tmp/epel &&
43-
curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \
43+
curl -L -s -S https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \
4444
-o /tmp/epel/epel-release-latest-8.noarch.rpm &&
4545
rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm
4646
else

‎dev-support/docker/pkg-resolver/packages.json‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"ubuntu:focal": "ant",
55
"ubuntu:focal::arch64": "ant",
66
"ubuntu:noble": "ant",
7-
"centos:8": "ant"
7+
"rockylinux:8": "ant"
88
},
99
"apt-utils": {
1010
"debian:10": "apt-utils",
@@ -17,7 +17,7 @@
1717
"ubuntu:focal": "automake",
1818
"ubuntu:noble": "automake",
1919
"ubuntu:focal::arch64": "automake",
20-
"centos:8": "automake"
20+
"rockylinux:8": "automake"
2121
},
2222
"bats": {
2323
"debian:10": "bats",
@@ -48,7 +48,7 @@
4848
"bzip2",
4949
"libbz2-dev"
5050
],
51-
"centos:8": [
51+
"rockylinux:8": [
5252
"bzip2",
5353
"bzip2-devel"
5454
]
@@ -58,7 +58,7 @@
5858
"ubuntu:focal": "clang",
5959
"ubuntu:noble": "clang",
6060
"ubuntu:focal::arch64": "clang",
61-
"centos:8": "clang"
61+
"rockylinux:8": "clang"
6262
},
6363
"cmake": {
6464
"ubuntu:focal": "cmake",
@@ -82,7 +82,7 @@
8282
"curl",
8383
"libcurl4-openssl-dev"
8484
],
85-
"centos:8": [
85+
"rockylinux:8": [
8686
"curl",
8787
"libcurl-devel"
8888
]
@@ -94,7 +94,7 @@
9494
"ubuntu:focal::arch64": "doxygen"
9595
},
9696
"dnf": {
97-
"centos:8": "dnf"
97+
"rockylinux:8": "dnf"
9898
},
9999
"fuse": {
100100
"debian:10": [
@@ -113,7 +113,7 @@
113113
"fuse",
114114
"libfuse-dev"
115115
],
116-
"centos:8": [
116+
"rockylinux:8": [
117117
"fuse",
118118
"fuse-libs",
119119
"fuse-devel"
@@ -144,7 +144,7 @@
144144
"ubuntu:focal": "git",
145145
"ubuntu:noble": "git",
146146
"ubuntu:focal::arch64": "git",
147-
"centos:8": "git"
147+
"rockylinux:8": "git"
148148
},
149149
"gnupg-agent": {
150150
"debian:10": "gnupg-agent",
@@ -169,17 +169,17 @@
169169
"ubuntu:focal": "libtool",
170170
"ubuntu:noble": "libtool",
171171
"ubuntu:focal::arch64": "libtool",
172-
"centos:8": "libtool"
172+
"rockylinux:8": "libtool"
173173
},
174174
"openssl": {
175175
"debian:10": "libssl-dev",
176176
"ubuntu:focal": "libssl-dev",
177177
"ubuntu:noble": "libssl-dev",
178178
"ubuntu:focal::arch64": "libssl-dev",
179-
"centos:8": "openssl-devel"
179+
"rockylinux:8": "openssl-devel"
180180
},
181181
"perl": {
182-
"centos:7": [
182+
"rockylinux:7": [
183183
"perl-CPAN",
184184
"perl-devel"
185185
]
@@ -207,7 +207,7 @@
207207
"ubuntu:focal": "libsasl2-dev",
208208
"ubuntu:noble": "libsasl2-dev",
209209
"ubuntu:focal::arch64": "libsasl2-dev",
210-
"centos:8": "cyrus-sasl-devel"
210+
"rockylinux:8": "cyrus-sasl-devel"
211211
},
212212
"snappy": {
213213
"debian:10": "libsnappy-dev",
@@ -232,7 +232,7 @@
232232
"libzstd-dev",
233233
"zlib1g-dev"
234234
],
235-
"centos:8": [
235+
"rockylinux:8": [
236236
"zlib-devel",
237237
"lz4-devel"
238238
]
@@ -244,15 +244,15 @@
244244
"ubuntu:focal::arch64": "locales"
245245
},
246246
"libtirpc-devel": {
247-
"centos:8": "libtirpc-devel",
247+
"rockylinux:8": "libtirpc-devel",
248248
"ubuntu:noble": "libtirpc-dev"
249249
},
250250
"make": {
251251
"debian:10": "make",
252252
"ubuntu:focal": "make",
253253
"ubuntu:noble": "make",
254254
"ubuntu:focal::arch64": "make",
255-
"centos:8": "make"
255+
"rockylinux:8": "make"
256256
},
257257
"maven": {
258258
"debian:10": "maven",
@@ -285,14 +285,14 @@
285285
"ubuntu:focal": "pinentry-curses",
286286
"ubuntu:noble": "pinentry-curses",
287287
"ubuntu:focal::arch64": "pinentry-curses",
288-
"centos:8": "pinentry-curses"
288+
"rockylinux:8": "pinentry-curses"
289289
},
290290
"pkg-config": {
291291
"debian:10": "pkg-config",
292292
"ubuntu:focal": "pkg-config",
293293
"ubuntu:noble": "pkg-config",
294294
"ubuntu:focal::arch64": "pkg-config",
295-
"centos:8": "pkg-config"
295+
"rockylinux:8": "pkg-config"
296296
},
297297
"python": {
298298
"debian:10": [
@@ -324,7 +324,7 @@
324324
"python3-setuptools",
325325
"python3-wheel"
326326
],
327-
"centos:8": [
327+
"rockylinux:8": [
328328
"python3",
329329
"python3-pip",
330330
"python3-setuptools",
@@ -336,7 +336,7 @@
336336
"ubuntu:focal": "rsync",
337337
"ubuntu:noble": "rsync",
338338
"ubuntu:focal::arch64": "rsync",
339-
"centos:8": "rsync"
339+
"rockylinux:8": "rsync"
340340
},
341341
"shellcheck": {
342342
"debian:10": "shellcheck",
@@ -345,7 +345,7 @@
345345
"ubuntu:focal::arch64": "shellcheck"
346346
},
347347
"shasum": {
348-
"centos:8": "perl-Digest-SHA"
348+
"rockylinux:8": "perl-Digest-SHA"
349349
},
350350
"software-properties-common": {
351351
"debian:10": "software-properties-common",
@@ -358,14 +358,14 @@
358358
"ubuntu:focal": "sudo",
359359
"ubuntu:noble": "sudo",
360360
"ubuntu:focal::arch64": "sudo",
361-
"centos:8": "sudo"
361+
"rockylinux:8": "sudo"
362362
},
363363
"valgrind": {
364364
"debian:10": "valgrind",
365365
"ubuntu:focal": "valgrind",
366366
"ubuntu:noble": "valgrind",
367367
"ubuntu:focal::arch64": "valgrind",
368-
"centos:8": "valgrind"
368+
"rockylinux:8": "valgrind"
369369
},
370370
"yasm": {
371371
"debian:10": "yasm",

0 commit comments

Comments
 (0)