Skip to content

Commit b1a7aa2

Browse files
committed
env/linux-ppc64/osuosl: upgrade linux/ppc64 hardware & OS, use Docker
Collaboration with @tiborvass at Docker who got Docker running on big-endian PPC64. Go for ppc64 doesn't support cgo or external linking, so runc doesn't work, but a new OCI-compliant runc implementation written in C (https://github.com/containers/crun) means we can run Docker after all. See NOTES & build-*.sh Then add a Dockerfile & associated cleanup in buildlet & stage0 to use rundockerbuildlet. Once done, might help with golang/go#35188, golang/go#32613, etc. Fixes golang/go#34830 Updates golang/go#21260 Change-Id: I43d7afa1d58bbdfa16e3c57670bc41f1d1932d80 Reviewed-on: https://go-review.googlesource.com/c/build/+/203886 Reviewed-by: Brad Fitzpatrick <[email protected]>
1 parent e2347b6 commit b1a7aa2

16 files changed

+251
-142
lines changed

cmd/buildlet/buildlet.go

Lines changed: 2 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ package main // import "golang.org/x/build/cmd/buildlet"
1414

1515
import (
1616
"archive/tar"
17-
"bufio"
1817
"bytes"
1918
"compress/gzip"
2019
"context"
@@ -43,7 +42,6 @@ import (
4342

4443
"cloud.google.com/go/compute/metadata"
4544
"golang.org/x/build/buildlet"
46-
"golang.org/x/build/internal/httpdl"
4745
"golang.org/x/build/pargzip"
4846
)
4947

@@ -257,45 +255,6 @@ func initGorootBootstrap() {
257255

258256
// Default if not otherwise configured in dashboard/builders.go:
259257
os.Setenv("GOROOT_BOOTSTRAP", filepath.Join(*workDir, "go1.4"))
260-
261-
if runtime.GOOS == "linux" && runtime.GOARCH == "ppc64" {
262-
downloadBootstrapGoroot("/usr/local/go-bootstrap", "https://storage.googleapis.com/go-builder-data/gobootstrap-linux-ppc64.tar.gz")
263-
}
264-
}
265-
266-
func downloadBootstrapGoroot(destDir, url string) {
267-
tarPath := destDir + ".tar.gz"
268-
origInfo, err := os.Stat(tarPath)
269-
if err != nil && !os.IsNotExist(err) {
270-
log.Fatalf("Checking for tar existence: %v", err)
271-
}
272-
if err := httpdl.Download(tarPath, url); err != nil {
273-
log.Fatalf("Downloading %s to %s: %v", url, tarPath, err)
274-
}
275-
newInfo, err := os.Stat(tarPath)
276-
if err != nil {
277-
log.Fatalf("Stat after download: %v", err)
278-
}
279-
if os.SameFile(origInfo, newInfo) {
280-
// The file on disk was unmodified, so we probably untarred it already.
281-
return
282-
}
283-
if err := os.RemoveAll(destDir); err != nil {
284-
log.Fatal(err)
285-
}
286-
if err := os.MkdirAll(destDir, 0755); err != nil {
287-
log.Fatal(err)
288-
}
289-
f, err := os.Open(tarPath)
290-
if err != nil {
291-
log.Fatalf("Opening after download: %v", err)
292-
}
293-
defer f.Close()
294-
if err := untar(f, destDir); err != nil {
295-
os.Remove(tarPath)
296-
os.RemoveAll(destDir)
297-
log.Fatalf("Untarring %s: %v", url, err)
298-
}
299258
}
300259

301260
func listenForCoordinator() {
@@ -1205,7 +1164,7 @@ func doHalt() {
12051164
err = errors.New("not respecting -halt flag on macOS in unknown environment")
12061165
}
12071166
default:
1208-
err = errors.New("No system-specific halt command run; will just end buildlet process.")
1167+
err = errors.New("no system-specific halt command run; will just end buildlet process")
12091168
}
12101169
log.Printf("Shutdown: %v", err)
12111170
log.Printf("Ending buildlet process post-halt")
@@ -1614,15 +1573,6 @@ func (pw *plan9LogWriter) Write(p []byte) (n int, err error) {
16141573
return pw.w.Write(pw.buf[:n+1])
16151574
}
16161575

1617-
func requireTrailerSupport() {
1618-
// Depend on a symbol that was added after HTTP Trailer support was
1619-
// implemented (4b96409 Dec 29 2014) so that this function will fail
1620-
// to compile without Trailer support.
1621-
// bufio.Reader.Discard was added by ee2ecc4 Jan 7 2015.
1622-
var r bufio.Reader
1623-
_ = r.Discard
1624-
}
1625-
16261576
var killProcessTree = killProcessTreeUnix
16271577

16281578
func killProcessTreeUnix(p *os.Process) error {
@@ -1804,7 +1754,7 @@ func appendSSHAuthorizedKey(sshUser, authKey string) error {
18041754
}
18051755
if runtime.GOOS == "windows" {
18061756
if res, err := exec.Command("icacls.exe", authFile, "/grant", `NT SERVICE\sshd:(R)`).CombinedOutput(); err != nil {
1807-
return fmt.Errorf("setting permissions on authorized_keys with: %v\n%s.", err, res)
1757+
return fmt.Errorf("setting permissions on authorized_keys with: %v\n%s", err, res)
18081758
}
18091759
}
18101760
return nil

cmd/buildlet/stage0/stage0.go

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,6 @@ func main() {
9090
default:
9191
panic(fmt.Sprintf("unknown/unspecified $GO_BUILDER_ENV value %q", env))
9292
}
93-
case "linux/ppc64":
94-
initOregonStatePPC64()
9593
case "darwin/amd64":
9694
// The MacStadium builders' baked-in stage0.sh
9795
// bootstrap file doesn't set GO_BUILDER_ENV
@@ -169,6 +167,8 @@ Download:
169167
cmd.Args = append(cmd.Args, reverseHostTypeArgs(buildEnv)...)
170168
case "host-linux-ppc64le-osu": // power8
171169
cmd.Args = append(cmd.Args, reverseHostTypeArgs(buildEnv)...)
170+
case "host-linux-ppc64-osu":
171+
cmd.Args = append(cmd.Args, reverseHostTypeArgs(buildEnv)...)
172172
}
173173
switch osArch {
174174
case "linux/s390x":
@@ -189,10 +189,6 @@ Download:
189189
default:
190190
panic(fmt.Sprintf("unknown/unspecified $GO_BUILDER_ENV value %q", env))
191191
}
192-
case "linux/ppc64":
193-
// Assume OSU (osuosl.org) host type for now. If we get more, use
194-
// GO_BUILD_HOST_TYPE (see above) and check that.
195-
cmd.Args = append(cmd.Args, reverseHostTypeArgs("host-linux-ppc64-osu")...)
196192
case "solaris/amd64", "illumos/amd64":
197193
hostType := buildEnv
198194
cmd.Args = append(cmd.Args, reverseHostTypeArgs(hostType)...)
@@ -409,11 +405,6 @@ func initBootstrapDir(destDir, tgzCache string) {
409405
log.Printf("untarred %s to %s in %v", tgzCache, destDir, time.Since(t1).Round(time.Second/10))
410406
}
411407

412-
func initOregonStatePPC64() {
413-
aptGetInstall("gcc", "strace", "libc6-dev", "gdb")
414-
initBootstrapDir("/usr/local/go-bootstrap", "/usr/local/go-bootstrap.tar.gz")
415-
}
416-
417408
func isUnix() bool {
418409
switch runtime.GOOS {
419410
case "plan9", "windows":

cmd/coordinator/status.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -427,7 +427,7 @@ func newPacketHealthChecker() *healthChecker {
427427
func newOSUPPC64Checker() *healthChecker {
428428
var hosts []string
429429
for i := 1; i <= expectedHosts("host-linux-ppc64-osu"); i++ {
430-
name := fmt.Sprintf("host-linux-ppc64-osu:go-be-%v", i)
430+
name := fmt.Sprintf("host-linux-ppc64-osu:ppc64_%02d", i)
431431
hosts = append(hosts, name)
432432
}
433433
return &healthChecker{

cmd/rundockerbuildlet/rundockerbuildlet.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,11 +150,13 @@ func checkFix() error {
150150
}
151151
cmd := exec.Command("docker", "run",
152152
"-d",
153-
"--memory="+*memory,
154153
"--name="+name,
155154
"-v", filepath.Dir(keyFile)+":/buildkey/",
156155
"-e", "HOSTNAME="+name,
157156
"--tmpfs=/workdir:rw,exec")
157+
if *memory != "" {
158+
cmd.Args = append(cmd.Args, "--memory="+*memory)
159+
}
158160
if *builderEnv != "" {
159161
cmd.Args = append(cmd.Args, "-e", "GO_BUILDER_ENV="+*builderEnv)
160162
}

env/linux-ppc64/osuosl/Dockerfile

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Copyright 2019 The Go Authors. All rights reserved.
2+
# Use of this source code is governed by a BSD-style
3+
# license that can be found in the LICENSE file.
4+
5+
FROM tiborvass/ubuntu:xenial-ppc64 # built locally from build-ubuntu.sh
6+
7+
ENV DEBIAN_FRONTEND noninteractive
8+
9+
RUN apt-get update && \
10+
apt-get install --yes \
11+
gcc curl strace \
12+
ca-certificates netbase \
13+
procps lsof psmisc \
14+
libc6-dev gdb \
15+
openssh-server
16+
17+
RUN mkdir /usr/local/go-bootstrap && \
18+
curl --silent https://storage.googleapis.com/go-builder-data/gobootstrap-linux-ppc64.tar.gz | \
19+
tar -C /usr/local/go-bootstrap -zxv
20+
21+
ENV GOROOT_BOOTSTRAP /usr/local/go-bootstrap
22+
RUN curl -o /usr/local/bin/stage0 https://storage.googleapis.com/go-builder-data/buildlet-stage0.linux-ppc64 && \
23+
chmod +x /usr/local/bin/stage0
24+
25+
ENV GO_BUILDER_ENV host-linux-ppc64-osu
26+
27+
ENV GO_BUILD_KEY_DELETE_AFTER_READ true
28+
ENV GO_BUILD_KEY_PATH /buildkey/gobuildkey
29+
30+
CMD ["/usr/local/bin/stage0"]
31+

env/linux-ppc64/osuosl/NOTES

Lines changed: 19 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
The linux-ppc64 buildlets run on PPC64 VMs at osuosl.org (OSU Open Source Lab).
22

3-
They run Debian jessie.
4-
53
Filing tickets: https://support.osuosl.org/
64
Ticket username: [email protected]
75
Ticket password: http://go/pw-osuosl-ppc64
@@ -23,44 +21,31 @@ Machines:
2321
2422
2523

26-
Each was once configured by scping setup.bash to them, logging in to
27-
each, and running:
24+
# Installing Docker on the unsupported ppc64 platform
25+
26+
There are 3 main scripts, in chronological order:
27+
- build-golang.sh
28+
- build-docker.sh
29+
- build-ubuntu.sh
2830

29-
$ sudo ./setup.bash <BUILDKEY_HERE>
31+
## Go toolchain
3032

31-
TODO: these should be updated like linux-arm and linux-arm64 to use Docker
32-
per build. That is https://golang.org/issue/21189
33+
The build-golang.sh script will create a Go toolchain for ppc64 and is meant to be run on a system for which Go has official releases, such as linux/amd64.
3334

34-
Machine information.
35+
The resulting tgz archive (and sha256 checksum file) should be sent to the ppc64 machine
3536

36-
debian@go-be-1:~$ free
37-
total used free shared buffers cached
38-
Mem: 4176832 2045952 2130880 95616 74752 1638464
39-
-/+ buffers/cache: 332736 3844096
40-
Swap: 0 0 0
37+
## Docker
4138

42-
debian@go-be-1:~$ cat /proc/cpuinfo
43-
processor : 0
44-
cpu : POWER8 (architected), altivec supported
45-
clock : 3425.000000MHz
46-
revision : 2.1 (pvr 004b 0201)
39+
The build-docker.sh script assumes it is running on the target ppc64 machine, after the Go toolchain archive and corresponding checksum file were transferred to the current directory.
40+
It will install docker and its dependencies to /usr/local/bin/ but will not start docker.
4741

48-
processor : 1
49-
cpu : POWER8 (architected), altivec supported
50-
clock : 3425.000000MHz
51-
revision : 2.1 (pvr 004b 0201)
42+
## Ubuntu base image
5243

53-
timebase : 512000000
54-
platform : pSeries
55-
model : IBM pSeries (emulated by qemu)
56-
machine : CHRP IBM pSeries (emulated by qemu)
44+
The build-ubuntu.sh script is to show how the tiborvass/ubuntu:xenial-ppc64 base image was built.
45+
To build images with static binaries only, there is no need for an ubuntu base image.
46+
Keep in mind, that the ubuntu packages are all 32bit, even though the kernel is 64bit.
5747

58-
debian@go-be-1:~$ uname -a
59-
Linux go-be-1 3.16.0-4-powerpc64 #1 SMP Debian 3.16.36-1+deb8u1 (2016-09-03) ppc64 GNU/Linux
48+
# Cleanup
6049

61-
debian@go-be-1:~$ lsb_release -a
62-
No LSB modules are available.
63-
Distributor ID: Debian
64-
Description: Debian GNU/Linux 8.6 (jessie)
65-
Release: 8.6
66-
Codename: jessie
50+
The cleanup.sh script by default uninstalls from /usr/local/bin/
51+
`./cleanup.sh all` will however remove all intermediary files and folders such as extracted archives.
Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#!/bin/sh -ex
2+
3+
GO_VERSION=1.13.3
4+
CRUN_VERSION=6254263af0a41d3b78ed280f944acda10199d42a # latest released version doesn't work
5+
CONTAINERD_VERSION=v1.2.10
6+
DOCKER_VERSION=v19.03.4
7+
GOOS=linux
8+
GOARCH=ppc64
9+
10+
11+
# Setting up Go toolchain
12+
bootstrap=go-${GOOS}-${GOARCH}-bootstrap
13+
export GOROOT=$(pwd)/${bootstrap}
14+
export GOPATH=$(pwd)/go
15+
export PATH=$GOROOT/bin:$PATH
16+
17+
install_go() {
18+
if ! which go >/dev/null; then
19+
sha256sum -c $bootstrap.tgz.sha256
20+
tar xf $bootstrap.tgz
21+
else
22+
echo "skipping go installation"
23+
fi
24+
[ "$(go version)" = "go version go${GO_VERSION} ${GOOS}/${GOARCH}" ]
25+
}
26+
27+
install_crun() {
28+
if [ -e /usr/local/bin/runc ]; then
29+
echo "skipping runc installation"
30+
return
31+
fi
32+
# crun is a C-only implementation of the OCI runtime spec, compatible with runc
33+
git clone https://github.com/containers/crun && cd crun && git checkout "$CRUN_VERSION"
34+
sudo apt-get update && sudo apt-get install --no-install-recommends -y make gcc libc6-dev pkgconf libtool go-md2man libtool autoconf python3 automake libcap-dev libseccomp-dev libyajl-dev
35+
./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install
36+
37+
sudo ln -s crun /usr/local/bin/runc # hacky but will work
38+
}
39+
40+
install_containerd() {
41+
if [ -e /usr/local/bin/containerd ]; then
42+
echo "skipping containerd installation"
43+
return
44+
fi
45+
# installs a static build of containerd
46+
mkdir -p $GOPATH/src/github.com/containerd
47+
git clone --depth 1 -b "$CONTAINERD_VERSION" https://github.com/containerd/containerd $GOPATH/src/github.com/containerd/containerd
48+
cd $GOPATH/src/github.com/containerd/containerd
49+
patch -p1 < containerd.patch
50+
BUILDTAGS='netgo osusergo static_build no_btrfs' make && sudo make install
51+
}
52+
53+
install_docker() {
54+
# installs a static build of docker
55+
# unfortunately, need to build manually
56+
if [ -e /usr/local/bin/dockerd ]; then
57+
echo "skipping dockerd installation"
58+
else
59+
mkdir -p $GOPATH/src/github.com/docker
60+
git clone --depth=1 -b "$DOCKER_VERSION" https://github.com/docker/engine $GOPATH/src/github.com/docker/docker
61+
cd $GOPATH/src/github.com/docker/docker
62+
VERSION="$DOCKER_VERSION" GITCOMMIT=$(git rev-parse --short HEAD) bash ./hack/make/.go-autogen
63+
CGO_ENABLED=0 go build -o dockerd \
64+
-tags 'autogen netgo osusergo static_build exclude_graphdriver_devicemapper exclude_disk_quota' \
65+
-installsuffix netgo \
66+
github.com/docker/docker/cmd/dockerd
67+
sudo mv dockerd /usr/local/bin/dockerd
68+
fi
69+
if [ -e /usr/local/bin/docker ]; then
70+
echo "skipping dockerd installation"
71+
return
72+
fi
73+
git clone --depth=1 -b "$DOCKER_VERSION" https://github.com/docker/cli $GOPATH/src/github.com/docker/cli
74+
cd $GOPATH/src/github.com/docker/cli
75+
VERSION="$DOCKER_VERSION" GITCOMMIT=$(git rev-parse --short HEAD) ./scripts/build/binary
76+
sudo cp build/docker-${GOOS}-${GOARCH} /usr/local/bin/docker
77+
}
78+
79+
install_go
80+
install_crun
81+
install_containerd
82+
install_docker
83+
84+
sudo install docker.service /etc/systemd/user/docker.service
85+
sudo systemctl enable /etc/systemd/user/docker.service || true
86+
sudo systemctl restart docker
87+
88+
sudo docker version
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/sh -ex
2+
3+
GO_VERSION=1.13.3
4+
5+
echo "Meant to be run on a system on which Go has official releases and is installed"
6+
GOHOSTARCH=$(go env GOHOSTARCH)
7+
GOHOSTOS=$(go env GOHOSTOS)
8+
export GOOS=linux
9+
export GOARCH=ppc64
10+
targz=go${GO_VERSION}.${GOHOSTOS}-${GOHOSTARCH}.tar.gz
11+
wget https://dl.google.com/go/${targz}
12+
tar xf ${targz}
13+
( cd go/src; ./bootstrap.bash )
14+
bootstrap=go-$GOOS-$GOARCH-bootstrap
15+
rm -f ${bootstrap}.tbz # does not contain all that's needed
16+
tar czf ${bootstrap}.tgz ${bootstrap}
17+
sha256sum $bootstrap.tgz | tee $bootstrap.tgz.sha256
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/sh -ex
2+
3+
export rootfs=$(pwd)/ubuntu_xenial_1604
4+
if [ ! -e "$rootfs" ]; then
5+
(
6+
sudo -E sh -c 'debootstrap --variant=minbase --keyring=/usr/share/keyrings/ubuntu-archive-keyring.gpg --include ubuntu-keyring xenial $rootfs'
7+
cd "$rootfs"
8+
sudo rm -rf var/log/{dpkg,bootstrap,alternatives}.log var/cache/ldconfig/aux-cache var/cache/apt/* var/lib/apt/lists/* dev/* proc/* sys/*
9+
)
10+
else
11+
echo "skipping debootstrap"
12+
fi
13+
14+
cat > Dockerfile.xenial <<'EOF'
15+
FROM scratch
16+
COPY ./ubuntu_xenial_1604/ /
17+
CMD ["/bin/bash"]
18+
EOF
19+
20+
# always build with buildkit :)
21+
export DOCKER_BUILDKIT=1
22+
sudo -E docker build -f Dockerfile.xenial -t tiborvass/ubuntu:xenial-ppc64 .
23+
#sudo docker login
24+
#sudo docker push tiborvass/ubuntu:xenial-ppc64
25+
sudo docker run -it --rm tiborvass/ubuntu:xenial-ppc64 cat /etc/os-release

0 commit comments

Comments
 (0)