cmd/dist: make toolchain build reproducible

- Build cmd with CGO_ENABLED=0. Doing so removes the C compiler
  toolchain from the reproducibility perimeter and also results in
  cmd/go and cmd/pprof binaries that are statically linked,
  so that they will run on a wider variety of systems.
  In particular the Linux versions will run on Alpine and NixOS
  without needing a simulation of libc.so.6.

  The potential downside of disabling cgo is that cmd/go and cmd/pprof
  use the pure Go network resolver instead of the host resolver on
  Unix systems. This means they will not be able to use non-DNS
  resolver mechanisms that may be specified in /etc/resolv.conf,
  such as mDNS. Neither program seems likely to need non-DNS names
  like those, however.

  macOS and Windows systems still use the host resolver, which they
  access without cgo.

- Build cmd with -trimpath when building a release.
  Doing so removes $GOPATH from the file name prefixes stored in the
  binary, so that the build directory does not leak into the final artifacts.

- When CC and CXX are empty, do not pick values to hard-code into
  the source tree and binaries. Instead, emit code that makes the
  right decision at runtime. In addition to reproducibility, this
  makes cross-compiled toolchains work better. A macOS toolchain
  cross-compiled on Linux will now correctly look for clang,
  instead of looking for gcc because it was built on Linux.

- Convert \ to / in file names stored in .a files.
  These are converted to / in the final binaries, but the hashes of
  the .a files affect the final build ID of the binaries. Without this
  change, builds of a Windows toolchain on Windows and non-Windows
  machines produce identical binaries except for the input hash part
  of the build ID.

- Due to the conversion of \ to / in .a files, convert back when
  reading inline bodies on Windows to preserve output file names
  in error messages.

Combined, these four changes (along with Go 1.20's removal of
installed pkg/**.a files and conversion of macOS net away from cgo)
make the output of make.bash fully reproducible, even when
cross-compiling: a released macOS toolchain built on Linux or Windows
will contain exactly the same bits as a released macOS toolchain
built on macOS.

The word "released" in the previous sentence is important.
For the build IDs in the binaries to work out the same on
both systems, a VERSION file must exist to provide a consistent
compiler build ID (instead of using a content hash of the binary).

For #24904.
Fixes #57007.

Change-Id: I665e1ef4ff207d6ff469452347dca5bfc81050e6
Reviewed-on: https://go-review.googlesource.com/c/go/+/454836
Reviewed-by: Bryan Mills <[email protected]>
Run-TryBot: Russ Cox <[email protected]>
Auto-Submit: Russ Cox <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>

Author: rsc

src/cmd/compile/internal/noder/reader.go

@@ -9,6 +9,7 @@ import (
 	"go/constant"
 	"internal/buildcfg"
 	"internal/pkgbits"
+	"path/filepath"
 	"strings"
 
 	"cmd/compile/internal/base"
@@ -268,13 +269,14 @@ func (pr *pkgReader) posBaseIdx(idx pkgbits.Index) *src.PosBase {
 	// "$GOROOT" to buildcfg.GOROOT is a close-enough approximation to
 	// satisfy this.
 	//
-	// TODO(mdempsky): De-duplicate this logic with similar logic in
-	// cmd/link/internal/ld's expandGoroot. However, this will probably
-	// require being more consistent about when we use native vs UNIX
-	// file paths.
+	// The export data format only ever uses slash paths
+	// (for cross-operating-system reproducible builds),
+	// but error messages need to use native paths (backslash on Windows)
+	// as if they had been specified on the command line.
+	// (The go command always passes native paths to the compiler.)
 	const dollarGOROOT = "$GOROOT"
 	if buildcfg.GOROOT != "" && strings.HasPrefix(filename, dollarGOROOT) {
-		filename = buildcfg.GOROOT + filename[len(dollarGOROOT):]
+		filename = filepath.FromSlash(buildcfg.GOROOT + filename[len(dollarGOROOT):])
 	}
 
 	if r.Bool() {

src/cmd/compile/internal/ssa/debug_lines_test.go

@@ -222,7 +222,7 @@ func testInlineStack(t *testing.T, file, function string, wantStacks [][]int) {
 	sortInlineStacks(gotStacks)
 	sortInlineStacks(wantStacks)
 	if !reflect.DeepEqual(wantStacks, gotStacks) {
-		t.Errorf("wanted inlines %+v but got %+v", wantStacks, gotStacks)
+		t.Errorf("wanted inlines %+v but got %+v\n%s", wantStacks, gotStacks, dumpBytes)
 	}
 
 }

src/cmd/dist/build.go

@@ -52,9 +52,8 @@ var (
 	defaultpkgconfig string
 	defaultldso      string
 
-	rebuildall   bool
-	defaultclang bool
-	noOpt        bool
+	rebuildall bool
+	noOpt      bool
 
 	vflag int // verbosity
 )
@@ -210,12 +209,8 @@ func xinit() {
 	gogcflags = os.Getenv("BOOT_GO_GCFLAGS")
 	goldflags = os.Getenv("BOOT_GO_LDFLAGS")
 
-	cc, cxx := "gcc", "g++"
-	if defaultclang {
-		cc, cxx = "clang", "clang++"
-	}
-	defaultcc = compilerEnv("CC", cc)
-	defaultcxx = compilerEnv("CXX", cxx)
+	defaultcc = compilerEnv("CC", "")
+	defaultcxx = compilerEnv("CXX", "")
 
 	b = os.Getenv("PKG_CONFIG")
 	if b == "" {
@@ -308,12 +303,34 @@ func compilerEnv(envName, def string) map[string]string {
 	return m
 }
 
+// clangos lists the operating systems where we prefer clang to gcc.
+var clangos = []string{
+	"darwin",  // macOS 10.9 and later require clang
+	"freebsd", // FreeBSD 10 and later do not ship gcc
+	"openbsd", // OpenBSD ships with GCC 4.2, which is now quite old.
+}
+
 // compilerEnvLookup returns the compiler settings for goos/goarch in map m.
-func compilerEnvLookup(m map[string]string, goos, goarch string) string {
+// kind is "CC" or "CXX".
+func compilerEnvLookup(kind string, m map[string]string, goos, goarch string) string {
 	if cc := m[goos+"/"+goarch]; cc != "" {
 		return cc
 	}
-	return m[""]
+	if cc := m[""]; cc != "" {
+		return cc
+	}
+	for _, os := range clangos {
+		if goos == os {
+			if kind == "CXX" {
+				return "clang++"
+			}
+			return "clang"
+		}
+	}
+	if kind == "CXX" {
+		return "g++"
+	}
+	return "gcc"
 }
 
 // rmworkdir deletes the work directory.
@@ -524,15 +541,25 @@ func setup() {
 		xremove(pathf("%s/bin/%s", goroot, old))
 	}
 
-	// For release, make sure excluded things are excluded.
+	// Special release-specific setup.
 	goversion := findgoversion()
-	if strings.HasPrefix(goversion, "release.") || (strings.HasPrefix(goversion, "go") && !strings.Contains(goversion, "beta")) {
+	isRelease := strings.HasPrefix(goversion, "release.") || (strings.HasPrefix(goversion, "go") && !strings.Contains(goversion, "beta"))
+	if isRelease {
+		// Make sure release-excluded things are excluded.
 		for _, dir := range unreleased {
 			if p := pathf("%s/%s", goroot, dir); isdir(p) {
 				fatalf("%s should not exist in release build", p)
 			}
 		}
 	}
+	if isRelease || os.Getenv("GO_BUILDER_NAME") != "" {
+		// Add -trimpath for reproducible builds of releases.
+		// Include builders so that -trimpath is well-tested ahead of releases.
+		// Do not include local development, so that people working in the
+		// main branch for day-to-day work on the Go toolchain itself can
+		// still have full paths for stack traces for compiler crashes and the like.
+		// toolenv = append(toolenv, "GOFLAGS=-trimpath")
+	}
 }
 
 /*
@@ -675,7 +702,7 @@ func runInstall(pkg string, ch chan struct{}) {
 		if goldflags != "" {
 			link = append(link, goldflags)
 		}
-		link = append(link, "-extld="+compilerEnvLookup(defaultcc, goos, goarch))
+		link = append(link, "-extld="+compilerEnvLookup("CC", defaultcc, goos, goarch))
 		link = append(link, "-L="+pathf("%s/pkg/obj/go-bootstrap/%s_%s", goroot, goos, goarch))
 		link = append(link, "-o", pathf("%s/%s%s", tooldir, elem, exe))
 		targ = len(link) - 1
@@ -1263,6 +1290,15 @@ func timelog(op, name string) {
 	fmt.Fprintf(timeLogFile, "%s %+.1fs %s %s\n", t.Format(time.UnixDate), t.Sub(timeLogStart).Seconds(), op, name)
 }
 
+// toolenv is the environment to use when building cmd.
+// We disable cgo to get static binaries for cmd/go and cmd/pprof,
+// so that they work on systems without the same dynamic libraries
+// as the original build system.
+// In release branches, we add -trimpath for reproducible builds.
+// In the main branch we leave it off, so that compiler crashes and
+// the like have full path names for easier navigation to source files.
+var toolenv = []string{"CGO_ENABLED=0"}
+
 var toolchain = []string{"cmd/asm", "cmd/cgo", "cmd/compile", "cmd/link"}
 
 // The bootstrap command runs a build from scratch,
@@ -1388,10 +1424,10 @@ func cmdbootstrap() {
 		xprintf("\n")
 	}
 	xprintf("Building Go toolchain2 using go_bootstrap and Go toolchain1.\n")
-	os.Setenv("CC", compilerEnvLookup(defaultcc, goos, goarch))
+	os.Setenv("CC", compilerEnvLookup("CC", defaultcc, goos, goarch))
 	// Now that cmd/go is in charge of the build process, enable GOEXPERIMENT.
 	os.Setenv("GOEXPERIMENT", goexperiment)
-	goInstall(goBootstrap, toolchain...)
+	goInstall(toolenv, goBootstrap, toolchain...)
 	if debug {
 		run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
 		copyfile(pathf("%s/compile2", tooldir), pathf("%s/compile", tooldir), writeExec)
@@ -1418,7 +1454,7 @@ func cmdbootstrap() {
 		xprintf("\n")
 	}
 	xprintf("Building Go toolchain3 using go_bootstrap and Go toolchain2.\n")
-	goInstall(goBootstrap, append([]string{"-a"}, toolchain...)...)
+	goInstall(toolenv, goBootstrap, append([]string{"-a"}, toolchain...)...)
 	if debug {
 		run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
 		copyfile(pathf("%s/compile3", tooldir), pathf("%s/compile", tooldir), writeExec)
@@ -1440,9 +1476,12 @@ func cmdbootstrap() {
 			xprintf("\n")
 		}
 		xprintf("Building packages and commands for host, %s/%s.\n", goos, goarch)
-		goInstall(goBootstrap, "std", "cmd")
-		checkNotStale(goBootstrap, "std", "cmd")
-		checkNotStale(cmdGo, "std", "cmd")
+		goInstall(nil, goBootstrap, "std")
+		goInstall(toolenv, goBootstrap, "cmd")
+		checkNotStale(nil, goBootstrap, "std")
+		checkNotStale(toolenv, goBootstrap, "cmd")
+		checkNotStale(nil, cmdGo, "std")
+		checkNotStale(toolenv, cmdGo, "cmd")
 
 		timelog("build", "target toolchain")
 		if vflag > 0 {
@@ -1452,17 +1491,19 @@ func cmdbootstrap() {
 		goarch = oldgoarch
 		os.Setenv("GOOS", goos)
 		os.Setenv("GOARCH", goarch)
-		os.Setenv("CC", compilerEnvLookup(defaultcc, goos, goarch))
+		os.Setenv("CC", compilerEnvLookup("CC", defaultcc, goos, goarch))
 		xprintf("Building packages and commands for target, %s/%s.\n", goos, goarch)
 	}
-	targets := []string{"std", "cmd"}
-	goInstall(goBootstrap, targets...)
-	checkNotStale(goBootstrap, append(toolchain, "runtime/internal/sys")...)
-	checkNotStale(goBootstrap, targets...)
-	checkNotStale(cmdGo, targets...)
+	goInstall(nil, goBootstrap, "std")
+	goInstall(toolenv, goBootstrap, "cmd")
+	checkNotStale(toolenv, goBootstrap, append(toolchain, "runtime/internal/sys")...)
+	checkNotStale(nil, goBootstrap, "std")
+	checkNotStale(toolenv, goBootstrap, "cmd")
+	checkNotStale(nil, cmdGo, "std")
+	checkNotStale(toolenv, cmdGo, "cmd")
 	if debug {
 		run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
-		checkNotStale(goBootstrap, append(toolchain, "runtime/internal/sys")...)
+		checkNotStale(toolenv, goBootstrap, append(toolchain, "runtime/internal/sys")...)
 		copyfile(pathf("%s/compile4", tooldir), pathf("%s/compile", tooldir), writeExec)
 	}
 
@@ -1492,8 +1533,8 @@ func cmdbootstrap() {
 		oldcc := os.Getenv("CC")
 		os.Setenv("GOOS", gohostos)
 		os.Setenv("GOARCH", gohostarch)
-		os.Setenv("CC", compilerEnvLookup(defaultcc, gohostos, gohostarch))
-		goCmd(cmdGo, "build", "-o", pathf("%s/go_%s_%s_exec%s", gorootBin, goos, goarch, exe), wrapperPath)
+		os.Setenv("CC", compilerEnvLookup("CC", defaultcc, gohostos, gohostarch))
+		goCmd(nil, cmdGo, "build", "-o", pathf("%s/go_%s_%s_exec%s", gorootBin, goos, goarch, exe), wrapperPath)
 		// Restore environment.
 		// TODO(elias.naur): support environment variables in goCmd?
 		os.Setenv("GOOS", goos)
@@ -1521,8 +1562,8 @@ func wrapperPathFor(goos, goarch string) string {
 	return ""
 }
 
-func goInstall(goBinary string, args ...string) {
-	goCmd(goBinary, "install", args...)
+func goInstall(env []string, goBinary string, args ...string) {
+	goCmd(env, goBinary, "install", args...)
 }
 
 func appendCompilerFlags(args []string) []string {
@@ -1535,7 +1576,7 @@ func appendCompilerFlags(args []string) []string {
 	return args
 }
 
-func goCmd(goBinary string, cmd string, args ...string) {
+func goCmd(env []string, goBinary string, cmd string, args ...string) {
 	goCmd := []string{goBinary, cmd}
 	if noOpt {
 		goCmd = append(goCmd, "-tags=noopt")
@@ -1550,18 +1591,18 @@ func goCmd(goBinary string, cmd string, args ...string) {
 		goCmd = append(goCmd, "-p=1")
 	}
 
-	run(workdir, ShowOutput|CheckExit, append(goCmd, args...)...)
+	runEnv(workdir, ShowOutput|CheckExit, env, append(goCmd, args...)...)
 }
 
-func checkNotStale(goBinary string, targets ...string) {
+func checkNotStale(env []string, goBinary string, targets ...string) {
 	goCmd := []string{goBinary, "list"}
 	if noOpt {
 		goCmd = append(goCmd, "-tags=noopt")
 	}
 	goCmd = appendCompilerFlags(goCmd)
 	goCmd = append(goCmd, "-f={{if .Stale}}\tSTALE {{.ImportPath}}: {{.StaleReason}}{{end}}")
 
-	out := run(workdir, CheckExit, append(goCmd, targets...)...)
+	out := runEnv(workdir, CheckExit, env, append(goCmd, targets...)...)
 	if strings.Contains(out, "\tSTALE ") {
 		os.Setenv("GODEBUG", "gocachehash=1")
 		for _, target := range []string{"runtime/internal/sys", "cmd/dist", "cmd/link"} {
@@ -1664,7 +1705,17 @@ func checkCC() {
 	if !needCC() {
 		return
 	}
-	cc, err := quotedSplit(defaultcc[""])
+	cc1 := defaultcc[""]
+	if cc1 == "" {
+		cc1 = "gcc"
+		for _, os := range clangos {
+			if gohostos == os {
+				cc1 = "clang"
+				break
+			}
+		}
+	}
+	cc, err := quotedSplit(cc1)
 	if err != nil {
 		fatalf("split CC: %v", err)
 	}

src/cmd/dist/buildgo.go

@@ -66,7 +66,26 @@ func defaultCCFunc(name string, defaultcc map[string]string) string {
 		fmt.Fprintf(&buf, "\tcase %q:\n\t\treturn %q\n", k, defaultcc[k])
 	}
 	fmt.Fprintf(&buf, "\t}\n")
-	fmt.Fprintf(&buf, "\treturn %q\n", defaultcc[""])
+	if cc := defaultcc[""]; cc != "" {
+		fmt.Fprintf(&buf, "\treturn %q\n", cc)
+	} else {
+		clang, gcc := "clang", "gcc"
+		if strings.HasSuffix(name, "CXX") {
+			clang, gcc = "clang++", "g++"
+		}
+		fmt.Fprintf(&buf, "\tswitch goos {\n")
+		fmt.Fprintf(&buf, "\tcase ")
+		for i, os := range clangos {
+			if i > 0 {
+				fmt.Fprintf(&buf, ", ")
+			}
+			fmt.Fprintf(&buf, "%q", os)
+		}
+		fmt.Fprintf(&buf, ":\n")
+		fmt.Fprintf(&buf, "\t\treturn %q\n", clang)
+		fmt.Fprintf(&buf, "\t}\n")
+		fmt.Fprintf(&buf, "\treturn %q\n", gcc)
+	}
 	fmt.Fprintf(&buf, "}\n")
 
 	return buf.String()

src/cmd/dist/main.go

@@ -59,15 +59,6 @@ func main() {
 	case "aix":
 		// uname -m doesn't work under AIX
 		gohostarch = "ppc64"
-	case "darwin":
-		// macOS 10.9 and later require clang
-		defaultclang = true
-	case "freebsd":
-		// Since FreeBSD 10 gcc is no longer part of the base system.
-		defaultclang = true
-	case "openbsd":
-		// OpenBSD ships with GCC 4.2, which is now quite old.
-		defaultclang = true
 	case "plan9":
 		gohostarch = os.Getenv("objtype")
 		if gohostarch == "" {