runtime: properly model rwmutex in lock ranking

Currently, lock ranking doesn't really try to model rwmutex. It records
the internal locks rLock and wLock, but in a subpar fashion:

1. wLock is held from lock to unlock, so it works OK, but it conflates
   write locks of all rwmutexes as rwmutexW, rather than allowing
   different rwmutexes to have different rankings.
2. rLock is an internal implementation detail that is only taken when
   there is contention in rlock. As as result, the reader lock path is
   almost never checked.

Add proper modeling. rwmutexR and rwmutexW remain as the ranks of the
internal locks, which have their own ordering. The new init method is
passed the ranks of the higher level lock that this represents, just
like lockInit for mutex.

execW ordered before MALLOC captures the case from #64722. i.e., there
can be allocation between BeforeFork and AfterFork.

For #64722.

Cq-Include-Trybots: luci.golang.try:gotip-linux-amd64-staticlockranking
Change-Id: I23335b28faa42fb04f1bc9da02fdf54d1616cd28
Reviewed-on: https://go-review.googlesource.com/c/go/+/549536
Reviewed-by: Michael Knyszek <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>

Author: prattmic

src/runtime/export_test.go

@@ -586,6 +586,10 @@ type RWMutex struct {
 	rw rwmutex
 }
 
+func (rw *RWMutex) Init() {
+	rw.rw.init(lockRankTestR, lockRankTestW)
+}
+
 func (rw *RWMutex) RLock() {
 	rw.rw.rlock()
 }

src/runtime/lockrank.go

@@ -52,30 +52,40 @@ NONE <
   assistQueue,
   sweep;
 
+# Test only
+NONE < testR, testW;
+
 # Scheduler, timers, netpoll
-NONE < pollDesc, cpuprof, wakeableSleep;
+NONE <
+  allocmW,
+  execW,
+  cpuprof,
+  pollDesc,
+  wakeableSleep;
 assistQueue,
   cpuprof,
   forcegc,
   pollDesc, # pollDesc can interact with timers, which can lock sched.
   scavenge,
   sweep,
   sweepWaiters,
+  testR,
   wakeableSleep
+# Above SCHED are things that can call into the scheduler.
+< SCHED
+# Below SCHED is the scheduler implementation.
+< allocmR,
+  execR
 < sched;
 sched < allg, allp;
 allp, wakeableSleep < timers;
 timers < netpollInit;
 
 # Channels
-scavenge, sweep, wakeableSleep < hchan;
+scavenge, sweep, testR, wakeableSleep < hchan;
 NONE < notifyList;
 hchan, notifyList < sudog;
 
-# RWMutex
-NONE < rwmutexW;
-rwmutexW, sysmon < rwmutexR;
-
 # Semaphores
 NONE < root;
 
@@ -100,6 +110,9 @@ traceBuf < traceStrings;
 
 # Malloc
 allg,
+  allocmR,
+  execR, # May grow stack
+  execW, # May allocate after BeforeFork
   hchan,
   notifyList,
   reflectOffs,
@@ -136,7 +149,7 @@ gcBitsArenas,
 < STACKGROW
 # Below STACKGROW is the stack allocator/copying implementation.
 < gscan;
-gscan, rwmutexR < stackpool;
+gscan < stackpool;
 gscan < stackLarge;
 # Generally, hchan must be acquired before gscan. But in one case,
 # where we suspend a G and then shrink its stack, syncadjustsudogs
@@ -189,6 +202,18 @@ NONE < panic;
 panic < deadlock;
 # raceFini is only held while exiting.
 panic < raceFini;
+
+# RWMutex
+allocmW,
+  execW,
+  testW
+< rwmutexW;
+
+rwmutexW,
+  allocmR,
+  execR,
+  testR
+< rwmutexR;
 `
 
 // cyclicRanks lists lock ranks that allow multiple locks of the same

src/runtime/mklockrank.go

@@ -759,6 +759,8 @@ func schedinit() {
 	lockInit(&reflectOffs.lock, lockRankReflectOffs)
 	lockInit(&finlock, lockRankFin)
 	lockInit(&cpuprof.lock, lockRankCpuprof)
+	allocmLock.init(lockRankAllocmR, lockRankAllocmW)
+	execLock.init(lockRankExecR, lockRankExecW)
 	traceLockInit()
 	// Enforce that this lock is always a leaf lock.
 	// All of this lock's critical sections should be

src/runtime/proc.go

@@ -25,6 +25,37 @@ type rwmutex struct {
 
 	readerCount atomic.Int32 // number of pending readers
 	readerWait  atomic.Int32 // number of departing readers
+
+	readRank  lockRank // semantic lock rank for read locking
+	writeRank lockRank // semantic lock rank for write locking
+}
+
+// Lock ranking an rwmutex has two aspects:
+//
+// Semantic ranking: this rwmutex represents some higher level lock that
+// protects some resource (e.g., allocmLock protects creation of new Ms). The
+// read and write locks of that resource need to be represented in the lock
+// rank.
+//
+// Internal ranking: as an implementation detail, rwmutex uses two mutexes:
+// rLock and wLock. These have lock order requirements: wLock must be locked
+// before rLock. This also needs to be represented in the lock rank.
+//
+// Internal ranking is represented by assigning ranks rwmutexR and rwmutexW to
+// rLock and wLock, respectively.
+//
+// Semantic ranking is represented by acquiring readRank during read lock and
+// writeRank during write lock.
+//
+// readRank is always taken before rwmutexR and writeRank is always taken
+// before rwmutexW, so each unique rwmutex must record this order in the lock
+// ranking.
+func (rw *rwmutex) init(readRank, writeRank lockRank) {
+	rw.readRank = readRank
+	rw.writeRank = writeRank
+
+	lockInit(&rw.rLock, lockRankRwmutexR)
+	lockInit(&rw.wLock, lockRankRwmutexW)
 }
 
 const rwmutexMaxReaders = 1 << 30
@@ -36,10 +67,14 @@ func (rw *rwmutex) rlock() {
 	// deadlock (issue #20903). Alternatively, we could drop the P
 	// while sleeping.
 	acquirem()
+
+	acquireLockRank(rw.readRank)
+	lockWithRankMayAcquire(&rw.rLock, getLockRank(&rw.rLock))
+
 	if rw.readerCount.Add(1) < 0 {
 		// A writer is pending. Park on the reader queue.
 		systemstack(func() {
-			lockWithRank(&rw.rLock, lockRankRwmutexR)
+			lock(&rw.rLock)
 			if rw.readerPass > 0 {
 				// Writer finished.
 				rw.readerPass -= 1
@@ -67,26 +102,28 @@ func (rw *rwmutex) runlock() {
 		// A writer is pending.
 		if rw.readerWait.Add(-1) == 0 {
 			// The last reader unblocks the writer.
-			lockWithRank(&rw.rLock, lockRankRwmutexR)
+			lock(&rw.rLock)
 			w := rw.writer.ptr()
 			if w != nil {
 				notewakeup(&w.park)
 			}
 			unlock(&rw.rLock)
 		}
 	}
+	releaseLockRank(rw.readRank)
 	releasem(getg().m)
 }
 
 // lock locks rw for writing.
 func (rw *rwmutex) lock() {
 	// Resolve competition with other writers and stick to our P.
-	lockWithRank(&rw.wLock, lockRankRwmutexW)
+	acquireLockRank(rw.writeRank)
+	lock(&rw.wLock)
 	m := getg().m
 	// Announce that there is a pending writer.
 	r := rw.readerCount.Add(-rwmutexMaxReaders) + rwmutexMaxReaders
 	// Wait for any active readers to complete.
-	lockWithRank(&rw.rLock, lockRankRwmutexR)
+	lock(&rw.rLock)
 	if r != 0 && rw.readerWait.Add(r) != 0 {
 		// Wait for reader to wake us up.
 		systemstack(func() {
@@ -108,7 +145,7 @@ func (rw *rwmutex) unlock() {
 		throw("unlock of unlocked rwmutex")
 	}
 	// Unblock blocked readers.
-	lockWithRank(&rw.rLock, lockRankRwmutexR)
+	lock(&rw.rLock)
 	for rw.readers.ptr() != nil {
 		reader := rw.readers.ptr()
 		rw.readers = reader.schedlink
@@ -122,4 +159,5 @@ func (rw *rwmutex) unlock() {
 	unlock(&rw.rLock)
 	// Allow other writers to proceed.
 	unlock(&rw.wLock)
+	releaseLockRank(rw.writeRank)
 }

src/runtime/rwmutex.go

@@ -29,6 +29,7 @@ func parallelReader(m *RWMutex, clocked chan bool, cunlock *atomic.Bool, cdone c
 func doTestParallelReaders(numReaders int) {
 	GOMAXPROCS(numReaders + 1)
 	var m RWMutex
+	m.Init()
 	clocked := make(chan bool, numReaders)
 	var cunlock atomic.Bool
 	cdone := make(chan bool)
@@ -100,6 +101,7 @@ func HammerRWMutex(gomaxprocs, numReaders, num_iterations int) {
 	// Number of active readers + 10000 * number of active writers.
 	var activity int32
 	var rwm RWMutex
+	rwm.Init()
 	cdone := make(chan bool)
 	go writer(&rwm, num_iterations, &activity, cdone)
 	var i int
@@ -141,6 +143,7 @@ func BenchmarkRWMutexUncontended(b *testing.B) {
 	}
 	b.RunParallel(func(pb *testing.PB) {
 		var rwm PaddedRWMutex
+		rwm.Init()
 		for pb.Next() {
 			rwm.RLock()
 			rwm.RLock()
@@ -154,6 +157,7 @@ func BenchmarkRWMutexUncontended(b *testing.B) {
 
 func benchmarkRWMutex(b *testing.B, localWork, writeRatio int) {
 	var rwm RWMutex
+	rwm.Init()
 	b.RunParallel(func(pb *testing.PB) {
 		foo := 0
 		for pb.Next() {