[release-branch.go1.17] all: merge master (5805efc) into release-branch.go1.17

Merge List:

+ 2021-08-12 5805efc doc/go1.17: remove draft notice
+ 2021-08-12 39634e7 CONTRIBUTORS: update for the Go 1.17 release
+ 2021-08-12 095bb79 os/exec: re-enable LookPathTest/16
+ 2021-08-11 dea23e9 src/make.*: make --no-clean flag a no-op that prints a warning
+ 2021-08-11 d4c0ed2 doc/go1.17: linker passes -I to extld as -Wl,--dynamic-linker
+ 2021-08-10 1f9c9d8 doc: use "high address/low address" instead of "top/bottom"
+ 2021-08-09 f1dce31 cmd/go: with -mod=vendor, don't panic if there are duplicate requirements
+ 2021-08-09 7aeaad5 runtime/cgo: when using msan explicitly unpoison cgoCallers
+ 2021-08-08 507cc34 doc: add example for conversion from slice expressions to array ptr
+ 2021-08-07 891547e doc/go1.17: fix a typo introduced in CL 335135
+ 2021-08-06 8eaf4d1 make.bash: do not overwrite GO_LDSO if already set
+ 2021-08-06 63b968f doc/go1.17: clarify Modules changes
+ 2021-08-06 70546f6 runtime: allow arm64 SEH to be called if illegal instruction
+ 2021-08-05 fd45e26 runtime: warn that KeepAlive is not an unsafe.Pointer workaround
+ 2021-08-04 6e73886 net/http: speed up and deflake TestCancelRequestWhenSharingConnection
+ 2021-08-02 8a7ee4c io/fs: don't use absolute path in DirEntry.Name doc

Change-Id: Ie9ce48bf4e99457cde966c5f972b04140468fcf0

Author: dmitshur

CONTRIBUTORS

@@ -166,7 +166,7 @@ <h3 id="symbols">Symbols</h3>
 </li>
 
 <li>
-<code>SP</code>: Stack pointer: top of stack.
+<code>SP</code>: Stack pointer: the highest address within the local stack frame.
 </li>
 
 </ul>
@@ -216,7 +216,7 @@ <h3 id="symbols">Symbols</h3>
 The <code>SP</code> pseudo-register is a virtual stack pointer
 used to refer to frame-local variables and the arguments being
 prepared for function calls.
-It points to the top of the local stack frame, so references should use negative offsets
+It points to the highest address within the local stack frame, so references should use negative offsets
 in the range [−framesize, 0):
 <code>x-8(SP)</code>, <code>y-4(SP)</code>, and so on.
 </p>
@@ -409,7 +409,7 @@ <h3 id="directives">Directives</h3>
 (For <code>TEXT</code> items.)
 Don't insert the preamble to check if the stack must be split.
 The frame for the routine, plus anything it calls, must fit in the
-spare space at the top of the stack segment.
+spare space remaining in the current stack segment.
 Used to protect routines such as the stack splitting code itself.
 </li>
 <li>
@@ -460,7 +460,7 @@ <h3 id="directives">Directives</h3>
 <code>TOPFRAME</code> = 2048
 <br>
 (For <code>TEXT</code> items.)
-Function is the top of the call stack. Traceback should stop at this function.
+Function is the outermost frame of the call stack. Traceback should stop at this function.
 </li>
 </ul>
 

doc/asm.html

@@ -14,13 +14,13 @@
   main ul li { margin: 0.5em 0; }
 </style>
 
-<h2 id="introduction">DRAFT RELEASE NOTES — Introduction to Go 1.17</h2>
+<h2 id="introduction">Introduction to Go 1.17</h2>
 
 <p>
-  <strong>
-    Go 1.17 is not yet released. These are work-in-progress
-    release notes. Go 1.17 is expected to be released in August 2021.
-  </strong>
+  The latest Go release, version 1.17, arrives six months after <a href="/doc/go1.16">Go 1.16</a>.
+  Most of its changes are in the implementation of the toolchain, runtime, and libraries.
+  As always, the release maintains the Go 1 <a href="/doc/go1compat">promise of compatibility</a>.
+  We expect almost all Go programs to continue to compile and run as before.
 </p>
 
 <h2 id="language">Changes to the language</h2>
@@ -134,35 +134,54 @@ <h2 id="tools">Tools</h2>
 
 <h3 id="go-command">Go command</h3>
 
-<h4 id="lazy-loading">Lazy module loading</h4>
+<a id="lazy-loading"><!-- for existing links only --></a>
+<h4 id="graph-pruning">Pruned module graphs in <code>go 1.17</code> modules</h4>
 
 <p><!-- golang.org/issue/36460 -->
+  If a module specifies <code>go</code> <code>1.17</code> or higher, the module
+  graph includes only the <em>immediate</em> dependencies of
+  other <code>go</code> <code>1.17</code> modules, not their full transitive
+  dependencies. (See <a href="/ref/mod#graph-pruning">Module graph pruning</a>
+  for more detail.)
+</p>
+
+<p>
+  For the <code>go</code> command to correctly resolve transitive imports using
+  the pruned module graph, the <code>go.mod</code> file for each module needs to
+  include more detail about the transitive dependencies relevant to that module.
   If a module specifies <code>go</code> <code>1.17</code> or higher in its
-  <code>go.mod</code> file, its transitive requirements are now loaded lazily,
-  avoiding the need to download or read <code>go.mod</code> files for
-  otherwise-irrelevant dependencies. To support lazy loading, in Go 1.17 modules
-  the <code>go</code> command maintains <em>explicit</em> requirements in
-  the <code>go.mod</code> file for every dependency that provides any package
-  transitively imported by any package or test within the module.
-  See <a href="https://golang.org/design/36460-lazy-module-loading">the design
-  document</a> for more detail.
-  <!-- TODO(bcmills): replace the design-doc link with proper documentation. -->
+  <code>go.mod</code> file, its <code>go.mod</code> file now contains an
+  explicit <a href="/ref/mod#go-mod-file-require"><code>require</code>
+  directive</a> for every module that provides a transitively-imported package.
+  (In previous versions, the <code>go.mod</code> file typically only included
+  explicit requirements for <em>directly</em>-imported packages.)
+<p>
+
+<p>
+  Since the expanded <code>go.mod</code> file needed for module graph pruning
+  includes all of the dependencies needed to load the imports of any package in
+  the main module, if the main module specifies
+  <code>go</code> <code>1.17</code> or higher the <code>go</code> tool no longer
+  reads (or even downloads) <code>go.mod</code> files for dependencies if they
+  are not needed in order to complete the requested command.
+  (See <a href="/ref/mod#lazy-loading">Lazy loading</a>.)
 </p>
 
 <p><!-- golang.org/issue/45965 -->
-  Because the number of additional explicit requirements in the go.mod file may
-  be substantial, in a Go 1.17 module the newly-added requirements
-  on <em>indirect</em> dependencies are maintained in a
-  separate <code>require</code> block from the block containing direct
-  dependencies.
+  Because the number of explicit requirements may be substantially larger in an
+  expanded Go 1.17 <code>go.mod</code> file, the newly-added requirements
+  on <em>indirect</em> dependencies in a <code>go</code> <code>1.17</code>
+  module are maintained in a separate <code>require</code> block from the block
+  containing direct dependencies.
 </p>
 
 <p><!-- golang.org/issue/45094 -->
-  To facilitate the upgrade to lazy loading, the
-  <code>go</code> <code>mod</code> <code>tidy</code> subcommand now supports
-  a <code>-go</code> flag to set or change the <code>go</code> version in
-  the <code>go.mod</code> file. To enable lazy loading for an existing module
-  without changing the selected versions of its dependencies, run:
+  To facilitate the upgrade to Go 1.17 pruned module graphs, the
+  <a href="/ref/mod#go-mod-tidy"><code>go</code> <code>mod</code> <code>tidy</code></a>
+  subcommand now supports a <code>-go</code> flag to set or change
+  the <code>go</code> version in the <code>go.mod</code> file. To convert
+  the <code>go.mod</code> file for an existing module to Go 1.17 without
+  changing the selected versions of its dependencies, run:
 </p>
 
 <pre>
@@ -199,10 +218,10 @@ <h4 id="lazy-loading">Lazy module loading</h4>
 </p>
 
 <p><!-- golang.org/issue/46366 -->
-  The <code>go</code> <code>mod</code> <code>graph</code> subcommand also
-  supports the <code>-go</code> flag, which causes it to report the graph as
-  seen by the indicated Go version, showing dependencies that may otherwise be
-  pruned out by lazy loading.
+  The <a href="/ref/mod#go-mod-graph"><code>go</code> <code>mod</code> <code>graph</code></a>
+  subcommand also supports the <code>-go</code> flag, which causes it to report
+  the graph as seen by the indicated Go version, showing dependencies that may
+  otherwise be pruned out.
 </p>
 
 <h4 id="module-deprecation-comments">Module deprecation comments</h4>
@@ -270,7 +289,8 @@ <h4 id="vendor"><code>vendor</code> contents</h4>
 
 <p><!-- golang.org/issue/36876 -->
   If the main module specifies <code>go</code> <code>1.17</code> or higher,
-  <code>go</code> <code>mod</code> <code>vendor</code> now annotates
+  <a href="/ref/mod#go-mod-vendor"><code>go</code> <code>mod</code> <code>vendor</code></a>
+  now annotates
   <code>vendor/modules.txt</code> with the <code>go</code> version indicated by
   each vendored module in its own <code>go.mod</code> file. The annotated
   version is used when building the module's packages from vendored source code.
@@ -468,6 +488,15 @@ <h2 id="compiler">Compiler</h2>
   and compare functions by code pointer.
 </p>
 
+<h3 id="link">Linker</h3>
+
+<p><!-- CL 310349 -->
+  When the linker uses external linking mode, which is the default
+  when linking a program that uses cgo, and the linker is invoked
+  with a <code>-I</code> option, the option will now be passed to the
+  external linker as a <code>-Wl,--dynamic-linker</code> option.
+</p>
+
 <h2 id="library">Core library</h2>
 
 <h3 id="runtime/cgo"><a href="/pkg/runtime/cgo">Cgo</a></h3>

doc/go1.17.html

@@ -4329,6 +4329,7 @@ <h4 id="Conversions_from_slice_to_array_pointer">Conversions from slice to array
 <pre>
 s := make([]byte, 2, 4)
 s0 := (*[0]byte)(s)      // s0 != nil
+s1 := (*[1]byte)(s[1:])  // &amp;s1[0] == &amp;s[1]
 s2 := (*[2]byte)(s)      // &amp;s2[0] == &amp;s[0]
 s4 := (*[4]byte)(s)      // panics: len([4]byte) > len(s)
 

doc/go_spec.html

@@ -42,6 +42,7 @@ func TestMSAN(t *testing.T) {
 		{src: "msan5.go"},
 		{src: "msan6.go"},
 		{src: "msan7.go"},
+		{src: "msan8.go"},
 		{src: "msan_fail.go", wantErr: true},
 	}
 	for _, tc := range cases {

misc/cgo/testsanitizers/msan_test.go

@@ -0,0 +1,109 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+/*
+#include <pthread.h>
+#include <signal.h>
+#include <stdint.h>
+
+#include <sanitizer/msan_interface.h>
+
+// cgoTracebackArg is the type of the argument passed to msanGoTraceback.
+struct cgoTracebackArg {
+	uintptr_t context;
+	uintptr_t sigContext;
+	uintptr_t* buf;
+	uintptr_t max;
+};
+
+// msanGoTraceback is registered as the cgo traceback function.
+// This will be called when a signal occurs.
+void msanGoTraceback(void* parg) {
+	struct cgoTracebackArg* arg = (struct cgoTracebackArg*)(parg);
+        arg->buf[0] = 0;
+}
+
+// msanGoWait will be called with all registers undefined as far as
+// msan is concerned. It just waits for a signal.
+// Because the registers are msan-undefined, the signal handler will
+// be invoked with all registers msan-undefined.
+__attribute__((noinline))
+void msanGoWait(unsigned long a1, unsigned long a2, unsigned long a3, unsigned long a4, unsigned long a5, unsigned long a6) {
+	sigset_t mask;
+
+	sigemptyset(&mask);
+        sigsuspend(&mask);
+}
+
+// msanGoSignalThread is the thread ID of the msanGoLoop thread.
+static pthread_t msanGoSignalThread;
+
+// msanGoSignalThreadSet is used to record that msanGoSignalThread
+// has been initialized. This is accessed atomically.
+static int32_t msanGoSignalThreadSet;
+
+// uninit is explicitly poisoned, so that we can make all registers
+// undefined by calling msanGoWait.
+static unsigned long uninit;
+
+// msanGoLoop loops calling msanGoWait, with the arguments passed
+// such that msan thinks that they are undefined. msan permits
+// undefined values to be used as long as they are not used to
+// for conditionals or for memory access.
+void msanGoLoop() {
+	int i;
+
+	msanGoSignalThread = pthread_self();
+        __atomic_store_n(&msanGoSignalThreadSet, 1, __ATOMIC_SEQ_CST);
+
+	// Force uninit to be undefined for msan.
+	__msan_poison(&uninit, sizeof uninit);
+	for (i = 0; i < 100; i++) {
+		msanGoWait(uninit, uninit, uninit, uninit, uninit, uninit);
+        }
+}
+
+// msanGoReady returns whether msanGoSignalThread is set.
+int msanGoReady() {
+	return __atomic_load_n(&msanGoSignalThreadSet, __ATOMIC_SEQ_CST) != 0;
+}
+
+// msanGoSendSignal sends a signal to the msanGoLoop thread.
+void msanGoSendSignal() {
+	pthread_kill(msanGoSignalThread, SIGWINCH);
+}
+*/
+import "C"
+
+import (
+	"runtime"
+	"time"
+)
+
+func main() {
+	runtime.SetCgoTraceback(0, C.msanGoTraceback, nil, nil)
+
+	c := make(chan bool)
+	go func() {
+		defer func() { c <- true }()
+		C.msanGoLoop()
+	}()
+
+	for C.msanGoReady() == 0 {
+		time.Sleep(time.Microsecond)
+	}
+
+loop:
+	for {
+		select {
+		case <-c:
+			break loop
+		default:
+			C.msanGoSendSignal()
+			time.Sleep(time.Microsecond)
+		}
+	}
+}

misc/cgo/testsanitizers/testdata/msan8.go

@@ -1263,14 +1263,19 @@ func cmdbootstrap() {
 	timelog("start", "dist bootstrap")
 	defer timelog("end", "dist bootstrap")
 
-	var noBanner bool
+	var noBanner, noClean bool
 	var debug bool
 	flag.BoolVar(&rebuildall, "a", rebuildall, "rebuild all")
 	flag.BoolVar(&debug, "d", debug, "enable debugging of bootstrap process")
 	flag.BoolVar(&noBanner, "no-banner", noBanner, "do not print banner")
+	flag.BoolVar(&noClean, "no-clean", noClean, "print deprecation warning")
 
 	xflagparse(0)
 
+	if noClean {
+		xprintf("warning: --no-clean is deprecated and has no effect; use 'go install std cmd' instead\n")
+	}
+
 	// Set GOPATH to an internal directory. We shouldn't actually
 	// need to store files here, since the toolchain won't
 	// depend on modules outside of vendor directories, but if

src/cmd/dist/build.go

@@ -191,6 +191,19 @@ func (rs *Requirements) rootSelected(path string) (version string, ok bool) {
 	return "", false
 }
 
+// hasRedundantRoot returns true if the root list contains multiple requirements
+// of the same module or a requirement on any version of the main module.
+// Redundant requirements should be pruned, but they may influence version
+// selection.
+func (rs *Requirements) hasRedundantRoot() bool {
+	for i, m := range rs.rootModules {
+		if m.Path == Target.Path || (i > 0 && m.Path == rs.rootModules[i-1].Path) {
+			return true
+		}
+	}
+	return false
+}
+
 // Graph returns the graph of module requirements loaded from the current
 // root modules (as reported by RootModules).
 //
@@ -882,6 +895,12 @@ func updateLazyRoots(ctx context.Context, direct map[string]bool, rs *Requiremen
 		// and (trivially) version.
 
 		if !rootsUpgraded {
+			if cfg.BuildMod != "mod" {
+				// The only changes to the root set (if any) were to remove duplicates.
+				// The requirements are consistent (if perhaps redundant), so keep the
+				// original rs to preserve its ModuleGraph.
+				return rs, nil
+			}
 			// The root set has converged: every root going into this iteration was
 			// already at its selected version, although we have have removed other
 			// (redundant) roots for the same path.