all: base64-encode binaries that will cause Apple notarization to fail

Starting with macOS 10.15 (Catalina), Apple now requires all software
distributed outside of the App Store to be notarized. Any binaries we
distribute must abide by a strict set of requirements like code-signing
and having a minimum target SDK of 10.9 (amongst others).

Apple’s notarization service will recursively inspect archives looking to
find notarization candidate binaries. If it finds a binary that does not
meet the requirements or is unable to decompress an archive, it will
reject the entire distribution. From cursory testing, it seems that the
service uses content sniffing to determine file types, so changing
the file extension will not work.

There are some binaries and archives included in our distribution that
are being detected by Apple’s service as potential candidates for
notarization or decompression. As these are files used by tests and some
are intentionally invalid, we don’t intend to ever make them compliant.

As a workaround for this, we base64-encode any binaries or archives that
Apple’s notarization service issues a warning for, as these warnings will
become errors in January 2020.

Updates #34986

Change-Id: I106fbb6227b61eb221755568f047ee11103c1680
Reviewed-on: https://go-review.googlesource.com/c/go/+/208118
Run-TryBot: Andrew Bonventre <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Brad Fitzpatrick <[email protected]>

Author: andybons

src/archive/zip/reader_test.go

@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"encoding/binary"
 	"encoding/hex"
+	"internal/obscuretestdata"
 	"io"
 	"io/ioutil"
 	"os"
@@ -19,11 +20,12 @@ import (
 )
 
 type ZipTest struct {
-	Name    string
-	Source  func() (r io.ReaderAt, size int64) // if non-nil, used instead of testdata/<Name> file
-	Comment string
-	File    []ZipTestFile
-	Error   error // the error that Opening this file should return
+	Name     string
+	Source   func() (r io.ReaderAt, size int64) // if non-nil, used instead of testdata/<Name> file
+	Comment  string
+	File     []ZipTestFile
+	Obscured bool  // needed for Apple notarization (golang.org/issue/34986)
+	Error    error // the error that Opening this file should return
 }
 
 type ZipTestFile struct {
@@ -189,8 +191,12 @@ var tests = []ZipTest{
 	},
 	{
 		// created by Go, before we wrote the "optional" data
-		// descriptor signatures (which are required by OS X)
-		Name: "go-no-datadesc-sig.zip",
+		// descriptor signatures (which are required by macOS).
+		// Use obscured file to avoid Apple’s notarization service
+		// rejecting the toolchain due to an inability to unzip this archive.
+		// See golang.org/issue/34986
+		Name:     "go-no-datadesc-sig.zip.base64",
+		Obscured: true,
 		File: []ZipTestFile{
 			{
 				Name:     "foo.txt",
@@ -208,7 +214,7 @@ var tests = []ZipTest{
 	},
 	{
 		// created by Go, after we wrote the "optional" data
-		// descriptor signatures (which are required by OS X)
+		// descriptor signatures (which are required by macOS)
 		Name: "go-with-datadesc-sig.zip",
 		File: []ZipTestFile{
 			{
@@ -496,8 +502,18 @@ func readTestZip(t *testing.T, zt ZipTest) {
 		rat, size := zt.Source()
 		z, err = NewReader(rat, size)
 	} else {
+		path := filepath.Join("testdata", zt.Name)
+		if zt.Obscured {
+			tf, err := obscuretestdata.DecodeToTempFile(path)
+			if err != nil {
+				t.Errorf("obscuretestdata.DecodeToTempFile(%s): %v", path, err)
+				return
+			}
+			defer os.Remove(tf)
+			path = tf
+		}
 		var rc *ReadCloser
-		rc, err = OpenReader(filepath.Join("testdata", zt.Name))
+		rc, err = OpenReader(path)
 		if err == nil {
 			defer rc.Close()
 			z = &rc.Reader

src/archive/zip/testdata/go-no-datadesc-sig.zip

@@ -0,0 +1 @@
+UEsDBBQACAAAAGWHaECoZTJ+BAAAAAQAAAAHABgAZm9vLnR4dFVUBQAD3lVZT3V4CwABBPUBAAAEFAAAAGZvbwqoZTJ+BAAAAAQAAABQSwMEFAAIAAAAZodoQOmzogQEAAAABAAAAAcAGABiYXIudHh0VVQFAAPgVVlPdXgLAAEE9QEAAAQUAAAAYmFyCumzogQEAAAABAAAAFBLAQIUAxQACAAAAGWHaECoZTJ+BAAAAAQAAAAHABgAAAAAAAAAAACkgQAAAABmb28udHh0VVQFAAPeVVlPdXgLAAEE9QEAAAQUAAAAUEsBAhQDFAAIAAAAZodoQOmzogQEAAAABAAAAAcAGAAAAAAAAAAAAKSBTQAAAGJhci50eHRVVAUAA+BVWU91eAsAAQT1AQAABBQAAABQSwUGAAAAAAIAAgCaAAAAmgAAAAAA

src/archive/zip/testdata/go-no-datadesc-sig.zip.base64

@@ -7,6 +7,7 @@ package buildid
 import (
 	"bytes"
 	"crypto/sha256"
+	"internal/obscuretestdata"
 	"io/ioutil"
 	"os"
 	"reflect"
@@ -19,13 +20,6 @@ const (
 )
 
 func TestReadFile(t *testing.T) {
-	var files = []string{
-		"p.a",
-		"a.elf",
-		"a.macho",
-		"a.pe",
-	}
-
 	f, err := ioutil.TempFile("", "buildid-test-")
 	if err != nil {
 		t.Fatal(err)
@@ -34,26 +28,43 @@ func TestReadFile(t *testing.T) {
 	defer os.Remove(tmp)
 	f.Close()
 
-	for _, f := range files {
-		id, err := ReadFile("testdata/" + f)
+	// Use obscured files to prevent Apple’s notarization service from
+	// mistaking them as candidates for notarization and rejecting the entire
+	// toolchain.
+	// See golang.org/issue/34986
+	var files = []string{
+		"p.a.base64",
+		"a.elf.base64",
+		"a.macho.base64",
+		"a.pe.base64",
+	}
+
+	for _, name := range files {
+		f, err := obscuretestdata.DecodeToTempFile("testdata/" + name)
+		if err != nil {
+			t.Errorf("obscuretestdata.DecodeToTempFile(testdata/%s): %v", name, err)
+			continue
+		}
+		defer os.Remove(f)
+		id, err := ReadFile(f)
 		if id != expectedID || err != nil {
 			t.Errorf("ReadFile(testdata/%s) = %q, %v, want %q, nil", f, id, err, expectedID)
 		}
 		old := readSize
 		readSize = 2048
-		id, err = ReadFile("testdata/" + f)
+		id, err = ReadFile(f)
 		readSize = old
 		if id != expectedID || err != nil {
-			t.Errorf("ReadFile(testdata/%s) [readSize=2k] = %q, %v, want %q, nil", f, id, err, expectedID)
+			t.Errorf("ReadFile(%s) [readSize=2k] = %q, %v, want %q, nil", f, id, err, expectedID)
 		}
 
-		data, err := ioutil.ReadFile("testdata/" + f)
+		data, err := ioutil.ReadFile(f)
 		if err != nil {
 			t.Fatal(err)
 		}
 		m, _, err := FindAndHash(bytes.NewReader(data), expectedID, 1024)
 		if err != nil {
-			t.Errorf("FindAndHash(testdata/%s): %v", f, err)
+			t.Errorf("FindAndHash(%s): %v", f, err)
 			continue
 		}
 		if err := ioutil.WriteFile(tmp, data, 0666); err != nil {
@@ -68,7 +79,7 @@ func TestReadFile(t *testing.T) {
 		err = Rewrite(tf, m, newID)
 		err2 := tf.Close()
 		if err != nil {
-			t.Errorf("Rewrite(testdata/%s): %v", f, err)
+			t.Errorf("Rewrite(%s): %v", f, err)
 			continue
 		}
 		if err2 != nil {
@@ -77,7 +88,7 @@ func TestReadFile(t *testing.T) {
 
 		id, err = ReadFile(tmp)
 		if id != newID || err != nil {
-			t.Errorf("ReadFile(testdata/%s after Rewrite) = %q, %v, want %q, nil", f, id, err, newID)
+			t.Errorf("ReadFile(%s after Rewrite) = %q, %v, want %q, nil", f, id, err, newID)
 		}
 	}
 }

src/cmd/internal/buildid/buildid_test.go

@@ -0,0 +1 @@
+TVqQAAMABAAAAAAA//8AAIsAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAA4fug4AtAnNIbgBTM0hVGhpcyBwcm9ncmFtIGNhbm5vdCBiZSBydW4gaW4gRE9TIG1vZGUuDQ0KJAAAAAAAAABQRQAAZIYEAAAAAAAADAAAAAAAAPAAIwILAgMAAAIAAAACAAAAAAAAcBAAAAAQAAAAAEAAAAAAAAAQAAAAAgAABAAAAAEAAAAEAAAAAAAAAABQAAAABgAAAAAAAAMAAAAAACAAAAAAAADgHwAAAAAAAAAQAAAAAAAAEAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAMAAAFAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAudGV4dAAAAMYBAAAAEAAAAAIAAAAGAAAAAAAAAAAAAAAAAABgAABgLmRhdGEAAADgAQAAACAAAAACAAAACAAAAAAAAAAAAAAAAAAAQAAAwC5pZGF0YQAAFAAAAAAwAAAAAgAAAAoAAAAAAAAAAAAAAAAAAEAAAMAuc3ltdGFiAAQAAAAAQAAAAAIAAAAMAAAAAAAAAAAAAAAAAAAAAABCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/yBHbyBidWlsZCBJRDogImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6LjEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDEyMzQiCiD/zMPMzMzMzMzMzMzMzMzMzMwBAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAEEAAAAAAAAAAAAAAAAAA+////wAAAQgCAAAAAAAAAAAQQAAAAAAAQAAAAAAAAABwEEAAAAAAAHgAAAAAAAAAcRBAAAAAAADIAAAAAAAAAAAQQAAAAAAAaAAAAAAAAABnRSMBAAAAAAAAAAAAAAAAAAAAAAAAAABnby5idWlsZGlkAAAAAAAAcBBAAAAAAACwAAAAAAAAAGdFIwG7AAAAvgAAAMEAAAAAAAAAAgAAAIAQQAAAAAAAgBBAAAAAAABtYWluLm1haW4AAAIBAAQBAAYBAAAAAAACAAAA0AAAAC9Vc2Vycy9yc2MvZ28vc3JjL2NtZC9pbnRlcm5hbC9idWlsZGlkL3Rlc3RkYXRhL3AuZ28AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgIEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAQQAAAAAAABgEAAAAAAAAGAQAAAAAAANAQQAAAAAAAAwAAAAAAAAADAAAAAAAAAIgRQAAAAAAAAgAAAAAAAAACAAAAAAAAAIwQQAAAAAAAABBAAAAAAABxEEAAAAAAAAAQQAAAAAAAgBBAAAAAAAAAIEAAAAAAAOAhQAAAAAAA4CFAAAAAAADgIUAAAAAAAOAhQAAAAAAA4CFAAAAAAADgIUAAAAAAAOAhQAAAAAAA4CFAAAAAAACJEEAAAAAAAIgQQAAAAAAAgBBAAAAAAAC4EEAAAAAAAKAQQAAAAAAAAQAAAAAAAAABAAAAAAAAALgQQAAAAAAAAAAAAAAAAAAAAAAAAAAAALgQQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=