env/linux-ppc64/osuosl: upgrade linux/ppc64 hardware & OS, use Docker

Collaboration with @tiborvass at Docker who got Docker running on
big-endian PPC64. Go for ppc64 doesn't support cgo or external
linking, so runc doesn't work, but a new OCI-compliant runc
implementation written in C (https://github.com/containers/crun) means
we can run Docker after all. See NOTES & build-*.sh

Then add a Dockerfile & associated cleanup in buildlet & stage0 to use
rundockerbuildlet.

Once done, might help with golang/go#35188, golang/go#32613, etc.

Fixes golang/go#34830
Updates golang/go#21260

Change-Id: I43d7afa1d58bbdfa16e3c57670bc41f1d1932d80
Reviewed-on: https://go-review.googlesource.com/c/build/+/203886
Reviewed-by: Brad Fitzpatrick <[email protected]>

Author: bradfitz

cmd/buildlet/buildlet.go

@@ -14,7 +14,6 @@ package main // import "golang.org/x/build/cmd/buildlet"
 
 import (
 	"archive/tar"
-	"bufio"
 	"bytes"
 	"compress/gzip"
 	"context"
@@ -43,7 +42,6 @@ import (
 
 	"cloud.google.com/go/compute/metadata"
 	"golang.org/x/build/buildlet"
-	"golang.org/x/build/internal/httpdl"
 	"golang.org/x/build/pargzip"
 )
 
@@ -257,45 +255,6 @@ func initGorootBootstrap() {
 
 	// Default if not otherwise configured in dashboard/builders.go:
 	os.Setenv("GOROOT_BOOTSTRAP", filepath.Join(*workDir, "go1.4"))
-
-	if runtime.GOOS == "linux" && runtime.GOARCH == "ppc64" {
-		downloadBootstrapGoroot("/usr/local/go-bootstrap", "https://storage.googleapis.com/go-builder-data/gobootstrap-linux-ppc64.tar.gz")
-	}
-}
-
-func downloadBootstrapGoroot(destDir, url string) {
-	tarPath := destDir + ".tar.gz"
-	origInfo, err := os.Stat(tarPath)
-	if err != nil && !os.IsNotExist(err) {
-		log.Fatalf("Checking for tar existence: %v", err)
-	}
-	if err := httpdl.Download(tarPath, url); err != nil {
-		log.Fatalf("Downloading %s to %s: %v", url, tarPath, err)
-	}
-	newInfo, err := os.Stat(tarPath)
-	if err != nil {
-		log.Fatalf("Stat after download: %v", err)
-	}
-	if os.SameFile(origInfo, newInfo) {
-		// The file on disk was unmodified, so we probably untarred it already.
-		return
-	}
-	if err := os.RemoveAll(destDir); err != nil {
-		log.Fatal(err)
-	}
-	if err := os.MkdirAll(destDir, 0755); err != nil {
-		log.Fatal(err)
-	}
-	f, err := os.Open(tarPath)
-	if err != nil {
-		log.Fatalf("Opening after download: %v", err)
-	}
-	defer f.Close()
-	if err := untar(f, destDir); err != nil {
-		os.Remove(tarPath)
-		os.RemoveAll(destDir)
-		log.Fatalf("Untarring %s: %v", url, err)
-	}
 }
 
 func listenForCoordinator() {
@@ -1205,7 +1164,7 @@ func doHalt() {
 			err = errors.New("not respecting -halt flag on macOS in unknown environment")
 		}
 	default:
-		err = errors.New("No system-specific halt command run; will just end buildlet process.")
+		err = errors.New("no system-specific halt command run; will just end buildlet process")
 	}
 	log.Printf("Shutdown: %v", err)
 	log.Printf("Ending buildlet process post-halt")
@@ -1614,15 +1573,6 @@ func (pw *plan9LogWriter) Write(p []byte) (n int, err error) {
 	return pw.w.Write(pw.buf[:n+1])
 }
 
-func requireTrailerSupport() {
-	// Depend on a symbol that was added after HTTP Trailer support was
-	// implemented (4b96409 Dec 29 2014) so that this function will fail
-	// to compile without Trailer support.
-	// bufio.Reader.Discard was added by ee2ecc4 Jan 7 2015.
-	var r bufio.Reader
-	_ = r.Discard
-}
-
 var killProcessTree = killProcessTreeUnix
 
 func killProcessTreeUnix(p *os.Process) error {
@@ -1804,7 +1754,7 @@ func appendSSHAuthorizedKey(sshUser, authKey string) error {
 	}
 	if runtime.GOOS == "windows" {
 		if res, err := exec.Command("icacls.exe", authFile, "/grant", `NT SERVICE\sshd:(R)`).CombinedOutput(); err != nil {
-			return fmt.Errorf("setting permissions on authorized_keys with: %v\n%s.", err, res)
+			return fmt.Errorf("setting permissions on authorized_keys with: %v\n%s", err, res)
 		}
 	}
 	return nil

cmd/buildlet/stage0/stage0.go

@@ -90,8 +90,6 @@ func main() {
 		default:
 			panic(fmt.Sprintf("unknown/unspecified $GO_BUILDER_ENV value %q", env))
 		}
-	case "linux/ppc64":
-		initOregonStatePPC64()
 	case "darwin/amd64":
 		// The MacStadium builders' baked-in stage0.sh
 		// bootstrap file doesn't set GO_BUILDER_ENV
@@ -169,6 +167,8 @@ Download:
 		cmd.Args = append(cmd.Args, reverseHostTypeArgs(buildEnv)...)
 	case "host-linux-ppc64le-osu": // power8
 		cmd.Args = append(cmd.Args, reverseHostTypeArgs(buildEnv)...)
+	case "host-linux-ppc64-osu":
+		cmd.Args = append(cmd.Args, reverseHostTypeArgs(buildEnv)...)
 	}
 	switch osArch {
 	case "linux/s390x":
@@ -189,10 +189,6 @@ Download:
 		default:
 			panic(fmt.Sprintf("unknown/unspecified $GO_BUILDER_ENV value %q", env))
 		}
-	case "linux/ppc64":
-		// Assume OSU (osuosl.org) host type for now. If we get more, use
-		// GO_BUILD_HOST_TYPE (see above) and check that.
-		cmd.Args = append(cmd.Args, reverseHostTypeArgs("host-linux-ppc64-osu")...)
 	case "solaris/amd64", "illumos/amd64":
 		hostType := buildEnv
 		cmd.Args = append(cmd.Args, reverseHostTypeArgs(hostType)...)
@@ -409,11 +405,6 @@ func initBootstrapDir(destDir, tgzCache string) {
 	log.Printf("untarred %s to %s in %v", tgzCache, destDir, time.Since(t1).Round(time.Second/10))
 }
 
-func initOregonStatePPC64() {
-	aptGetInstall("gcc", "strace", "libc6-dev", "gdb")
-	initBootstrapDir("/usr/local/go-bootstrap", "/usr/local/go-bootstrap.tar.gz")
-}
-
 func isUnix() bool {
 	switch runtime.GOOS {
 	case "plan9", "windows":

cmd/coordinator/status.go

@@ -427,7 +427,7 @@ func newPacketHealthChecker() *healthChecker {
 func newOSUPPC64Checker() *healthChecker {
 	var hosts []string
 	for i := 1; i <= expectedHosts("host-linux-ppc64-osu"); i++ {
-		name := fmt.Sprintf("host-linux-ppc64-osu:go-be-%v", i)
+		name := fmt.Sprintf("host-linux-ppc64-osu:ppc64_%02d", i)
 		hosts = append(hosts, name)
 	}
 	return &healthChecker{

cmd/rundockerbuildlet/rundockerbuildlet.go

@@ -150,11 +150,13 @@ func checkFix() error {
 		}
 		cmd := exec.Command("docker", "run",
 			"-d",
-			"--memory="+*memory,
 			"--name="+name,
 			"-v", filepath.Dir(keyFile)+":/buildkey/",
 			"-e", "HOSTNAME="+name,
 			"--tmpfs=/workdir:rw,exec")
+		if *memory != "" {
+			cmd.Args = append(cmd.Args, "--memory="+*memory)
+		}
 		if *builderEnv != "" {
 			cmd.Args = append(cmd.Args, "-e", "GO_BUILDER_ENV="+*builderEnv)
 		}

env/linux-ppc64/osuosl/Dockerfile

@@ -0,0 +1,31 @@
+# Copyright 2019 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+FROM tiborvass/ubuntu:xenial-ppc64 # built locally from build-ubuntu.sh
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && \
+    apt-get install --yes \
+          gcc curl strace \
+          ca-certificates netbase \
+          procps lsof psmisc \
+          libc6-dev gdb \
+          openssh-server
+
+RUN mkdir /usr/local/go-bootstrap && \
+    curl --silent https://storage.googleapis.com/go-builder-data/gobootstrap-linux-ppc64.tar.gz | \
+    tar -C /usr/local/go-bootstrap -zxv
+
+ENV GOROOT_BOOTSTRAP /usr/local/go-bootstrap
+RUN curl -o  /usr/local/bin/stage0 https://storage.googleapis.com/go-builder-data/buildlet-stage0.linux-ppc64 && \
+    chmod +x /usr/local/bin/stage0
+
+ENV GO_BUILDER_ENV host-linux-ppc64-osu
+
+ENV GO_BUILD_KEY_DELETE_AFTER_READ true
+ENV GO_BUILD_KEY_PATH /buildkey/gobuildkey
+
+CMD ["/usr/local/bin/stage0"]
+

env/linux-ppc64/osuosl/NOTES

@@ -1,7 +1,5 @@
 The linux-ppc64 buildlets run on PPC64 VMs at osuosl.org (OSU Open Source Lab).
 
-They run Debian jessie.
-
 Filing tickets: https://support.osuosl.org/
     Ticket username: [email protected]
     Ticket password: http://go/pw-osuosl-ppc64
@@ -23,44 +21,31 @@ Machines:
   go-be-4: [email protected]
   go-be-5: [email protected]
 
-Each was once configured by scping setup.bash to them, logging in to
-each, and running:
+# Installing Docker on the unsupported ppc64 platform
+
+There are 3 main scripts, in chronological order:
+- build-golang.sh
+- build-docker.sh
+- build-ubuntu.sh
 
-$ sudo ./setup.bash <BUILDKEY_HERE>
+## Go toolchain
 
-TODO: these should be updated like linux-arm and linux-arm64 to use Docker
-      per build. That is https://golang.org/issue/21189
+The build-golang.sh script will create a Go toolchain for ppc64 and is meant to be run on a system for which Go has official releases, such as linux/amd64.
 
-Machine information.
+The resulting tgz archive (and sha256 checksum file) should be sent to the ppc64 machine
 
-debian@go-be-1:~$ free
-total       used       free     shared    buffers     cached
-Mem:       4176832    2045952    2130880      95616      74752    1638464
--/+ buffers/cache:     332736    3844096
-Swap:            0          0          0
+## Docker
 
-debian@go-be-1:~$ cat /proc/cpuinfo
-processor       : 0
-cpu             : POWER8 (architected), altivec supported
-clock           : 3425.000000MHz
-revision        : 2.1 (pvr 004b 0201)
+The build-docker.sh script assumes it is running on the target ppc64 machine, after the Go toolchain archive and corresponding checksum file were transferred to the current directory.
+It will install docker and its dependencies to /usr/local/bin/ but will not start docker.
 
-processor       : 1
-cpu             : POWER8 (architected), altivec supported
-clock           : 3425.000000MHz
-revision        : 2.1 (pvr 004b 0201)
+## Ubuntu base image
 
-timebase        : 512000000
-platform        : pSeries
-model           : IBM pSeries (emulated by qemu)
-machine         : CHRP IBM pSeries (emulated by qemu)
+The build-ubuntu.sh script is to show how the tiborvass/ubuntu:xenial-ppc64 base image was built.
+To build images with static binaries only, there is no need for an ubuntu base image.
+Keep in mind, that the ubuntu packages are all 32bit, even though the kernel is 64bit.
 
-debian@go-be-1:~$ uname -a
-Linux go-be-1 3.16.0-4-powerpc64 #1 SMP Debian 3.16.36-1+deb8u1 (2016-09-03) ppc64 GNU/Linux
+# Cleanup
 
-debian@go-be-1:~$ lsb_release -a
-No LSB modules are available.
-Distributor ID: Debian
-Description:    Debian GNU/Linux 8.6 (jessie)
-Release:        8.6
-Codename:       jessie
+The cleanup.sh script by default uninstalls from /usr/local/bin/
+`./cleanup.sh all` will however remove all intermediary files and folders such as extracted archives.

env/linux-ppc64/osuosl/build-docker.sh

@@ -0,0 +1,88 @@
+#!/bin/sh -ex
+
+GO_VERSION=1.13.3
+CRUN_VERSION=6254263af0a41d3b78ed280f944acda10199d42a # latest released version doesn't work
+CONTAINERD_VERSION=v1.2.10
+DOCKER_VERSION=v19.03.4
+GOOS=linux
+GOARCH=ppc64
+
+
+# Setting up Go toolchain
+bootstrap=go-${GOOS}-${GOARCH}-bootstrap
+export GOROOT=$(pwd)/${bootstrap}
+export GOPATH=$(pwd)/go
+export PATH=$GOROOT/bin:$PATH
+
+install_go() {
+	if ! which go >/dev/null; then
+		sha256sum -c $bootstrap.tgz.sha256
+		tar xf $bootstrap.tgz
+	else
+		echo "skipping go installation"
+	fi
+	[ "$(go version)" = "go version go${GO_VERSION} ${GOOS}/${GOARCH}" ]
+}
+
+install_crun() {
+	if [ -e /usr/local/bin/runc ]; then
+		echo "skipping runc installation"
+		return
+	fi
+	# crun is a C-only implementation of the OCI runtime spec, compatible with runc
+	git clone https://github.com/containers/crun && cd crun && git checkout "$CRUN_VERSION"
+	sudo apt-get update && sudo apt-get install --no-install-recommends -y make gcc libc6-dev pkgconf libtool go-md2man libtool autoconf python3 automake libcap-dev libseccomp-dev libyajl-dev
+	./autogen.sh && ./configure --prefix=/usr/local && make && sudo make install
+
+	sudo ln -s crun /usr/local/bin/runc    # hacky but will work
+}
+
+install_containerd() {
+	if [ -e /usr/local/bin/containerd ]; then
+		echo "skipping containerd installation"
+		return
+	fi
+	# installs a static build of containerd
+	mkdir -p $GOPATH/src/github.com/containerd
+	git clone --depth 1 -b "$CONTAINERD_VERSION" https://github.com/containerd/containerd $GOPATH/src/github.com/containerd/containerd
+	cd $GOPATH/src/github.com/containerd/containerd
+	patch -p1 < containerd.patch
+	BUILDTAGS='netgo osusergo static_build no_btrfs' make && sudo make install
+}
+
+install_docker() {
+	# installs a static build of docker
+	# unfortunately, need to build manually
+	if [ -e /usr/local/bin/dockerd ]; then
+		echo "skipping dockerd installation"
+	else
+		mkdir -p $GOPATH/src/github.com/docker
+		git clone --depth=1 -b "$DOCKER_VERSION" https://github.com/docker/engine $GOPATH/src/github.com/docker/docker
+		cd $GOPATH/src/github.com/docker/docker
+		VERSION="$DOCKER_VERSION" GITCOMMIT=$(git rev-parse --short HEAD) bash ./hack/make/.go-autogen
+		CGO_ENABLED=0 go build -o dockerd \
+			-tags 'autogen netgo osusergo static_build exclude_graphdriver_devicemapper exclude_disk_quota' \
+			-installsuffix netgo  \
+			github.com/docker/docker/cmd/dockerd
+		sudo mv dockerd /usr/local/bin/dockerd
+	fi
+	if [ -e /usr/local/bin/docker ]; then
+		echo "skipping dockerd installation"
+		return
+	fi
+	git clone --depth=1 -b "$DOCKER_VERSION" https://github.com/docker/cli $GOPATH/src/github.com/docker/cli
+	cd $GOPATH/src/github.com/docker/cli
+	VERSION="$DOCKER_VERSION" GITCOMMIT=$(git rev-parse --short HEAD) ./scripts/build/binary
+	sudo cp build/docker-${GOOS}-${GOARCH} /usr/local/bin/docker
+}
+
+install_go
+install_crun
+install_containerd
+install_docker
+
+sudo install docker.service /etc/systemd/user/docker.service
+sudo systemctl enable /etc/systemd/user/docker.service || true
+sudo systemctl restart docker
+
+sudo docker version

env/linux-ppc64/osuosl/build-golang.sh

@@ -0,0 +1,17 @@
+#!/bin/sh -ex
+
+GO_VERSION=1.13.3
+
+echo "Meant to be run on a system on which Go has official releases and is installed"
+GOHOSTARCH=$(go env GOHOSTARCH)
+GOHOSTOS=$(go env GOHOSTOS)
+export GOOS=linux
+export GOARCH=ppc64
+targz=go${GO_VERSION}.${GOHOSTOS}-${GOHOSTARCH}.tar.gz
+wget https://dl.google.com/go/${targz}
+tar xf ${targz}
+( cd go/src; ./bootstrap.bash )
+bootstrap=go-$GOOS-$GOARCH-bootstrap
+rm -f ${bootstrap}.tbz # does not contain all that's needed
+tar czf ${bootstrap}.tgz ${bootstrap}
+sha256sum $bootstrap.tgz | tee $bootstrap.tgz.sha256

env/linux-ppc64/osuosl/build-ubuntu.sh

@@ -0,0 +1,25 @@
+#!/bin/sh -ex
+
+export rootfs=$(pwd)/ubuntu_xenial_1604
+if [ ! -e "$rootfs" ]; then
+	(
+		sudo -E sh -c 'debootstrap --variant=minbase --keyring=/usr/share/keyrings/ubuntu-archive-keyring.gpg --include ubuntu-keyring xenial $rootfs'
+		cd "$rootfs"
+		sudo rm -rf var/log/{dpkg,bootstrap,alternatives}.log var/cache/ldconfig/aux-cache var/cache/apt/* var/lib/apt/lists/* dev/* proc/* sys/*
+	)
+else
+	echo "skipping debootstrap"
+fi
+
+cat > Dockerfile.xenial <<'EOF'
+FROM scratch
+COPY ./ubuntu_xenial_1604/ /
+CMD ["/bin/bash"]
+EOF
+
+# always build with buildkit :)
+export DOCKER_BUILDKIT=1
+sudo -E docker build -f Dockerfile.xenial -t tiborvass/ubuntu:xenial-ppc64 .
+#sudo docker login
+#sudo docker push tiborvass/ubuntu:xenial-ppc64
+sudo docker run -it --rm tiborvass/ubuntu:xenial-ppc64 cat /etc/os-release