Revert "runtime: use semaphore structure for futex locks"

This reverts commit dfb7073 (CL 585636).

Reason for revert: This is part of a patch series that changed the
handling of contended lock2/unlock2 calls, reducing the maximum
throughput of contended runtime.mutex values, and causing a performance
regression on applications where that is (or became) the bottleneck.

Updates #66999
Updates #67585

Change-Id: I3483bf0b85ba0b77204032a68b7cbe93f142703e
Reviewed-on: https://go-review.googlesource.com/c/go/+/589098
LUCI-TryBot-Result: Go LUCI <[email protected]>
Reviewed-by: Than McIntosh <[email protected]>
Reviewed-by: Michael Pratt <[email protected]>
Auto-Submit: Rhys Hiltner <[email protected]>

Author: rhysh

src/runtime/lock_futex.go

@@ -23,20 +23,19 @@ import (
 //		If any procs are sleeping on addr, wake up at most cnt.
 
 const (
-	mutex_locked   = 0x1
-	mutex_sleeping = 0x2 // Ensure futex's low 32 bits won't be all zeros
+	mutex_unlocked = 0
+	mutex_locked   = 1
+	mutex_sleeping = 2
 
 	active_spin     = 4
 	active_spin_cnt = 30
 	passive_spin    = 1
 )
 
-// The mutex.key holds two state flags in its lowest bits: When the mutex_locked
-// bit is set, the mutex is locked. When the mutex_sleeping bit is set, a thread
-// is waiting in futexsleep for the mutex to be available. These flags operate
-// independently: a thread can enter lock2, observe that another thread is
-// already asleep, and immediately try to grab the lock anyway without waiting
-// for its "fair" turn.
+// Possible lock states are mutex_unlocked, mutex_locked and mutex_sleeping.
+// mutex_sleeping means that there is presumably at least one sleeping thread.
+// Note that there can be spinning threads during all states - they do not
+// affect mutex's state.
 
 // We use the uintptr mutex.key and note.key as a uint32.
 //
@@ -55,16 +54,27 @@ func lock(l *mutex) {
 
 func lock2(l *mutex) {
 	gp := getg()
+
 	if gp.m.locks < 0 {
 		throw("runtime·lock: lock count")
 	}
 	gp.m.locks++
 
 	// Speculative grab for lock.
-	if atomic.Casuintptr(&l.key, 0, mutex_locked) {
+	v := atomic.Xchg(key32(&l.key), mutex_locked)
+	if v == mutex_unlocked {
 		return
 	}
 
+	// wait is either MUTEX_LOCKED or MUTEX_SLEEPING
+	// depending on whether there is a thread sleeping
+	// on this mutex. If we ever change l->key from
+	// MUTEX_SLEEPING to some other value, we must be
+	// careful to change it back to MUTEX_SLEEPING before
+	// returning, to ensure that the sleeping thread gets
+	// its wakeup call.
+	wait := v
+
 	timer := &lockTimer{lock: l}
 	timer.begin()
 	// On uniprocessors, no point spinning.
@@ -73,39 +83,37 @@ func lock2(l *mutex) {
 	if ncpu > 1 {
 		spin = active_spin
 	}
-Loop:
-	for i := 0; ; i++ {
-		v := atomic.Loaduintptr(&l.key)
-		if v&mutex_locked == 0 {
-			// Unlocked. Try to lock.
-			if atomic.Casuintptr(&l.key, v, v|mutex_locked) {
-				timer.end()
-				return
+	for {
+		// Try for lock, spinning.
+		for i := 0; i < spin; i++ {
+			for l.key == mutex_unlocked {
+				if atomic.Cas(key32(&l.key), mutex_unlocked, wait) {
+					timer.end()
+					return
+				}
 			}
-			i = 0
-		}
-		if i < spin {
 			procyield(active_spin_cnt)
-		} else if i < spin+passive_spin {
-			osyield()
-		} else {
-			// Someone else has it.
-			for {
-				head := v &^ (mutex_locked | mutex_sleeping)
-				if atomic.Casuintptr(&l.key, v, head|mutex_locked|mutex_sleeping) {
-					break
-				}
-				v = atomic.Loaduintptr(&l.key)
-				if v&mutex_locked == 0 {
-					continue Loop
+		}
+
+		// Try for lock, rescheduling.
+		for i := 0; i < passive_spin; i++ {
+			for l.key == mutex_unlocked {
+				if atomic.Cas(key32(&l.key), mutex_unlocked, wait) {
+					timer.end()
+					return
 				}
 			}
-			if v&mutex_locked != 0 {
-				// Queued. Wait.
-				futexsleep(key32(&l.key), uint32(v), -1)
-				i = 0
-			}
+			osyield()
 		}
+
+		// Sleep.
+		v = atomic.Xchg(key32(&l.key), mutex_sleeping)
+		if v == mutex_unlocked {
+			timer.end()
+			return
+		}
+		wait = mutex_sleeping
+		futexsleep(key32(&l.key), mutex_sleeping, -1)
 	}
 }
 
@@ -114,21 +122,12 @@ func unlock(l *mutex) {
 }
 
 func unlock2(l *mutex) {
-	for {
-		v := atomic.Loaduintptr(&l.key)
-		if v == mutex_locked {
-			if atomic.Casuintptr(&l.key, mutex_locked, 0) {
-				break
-			}
-		} else if v&mutex_locked == 0 {
-			throw("unlock of unlocked lock")
-		} else {
-			// Other M's are waiting for the lock.
-			if atomic.Casuintptr(&l.key, v, v&^mutex_locked) {
-				futexwakeup(key32(&l.key), 1)
-				break
-			}
-		}
+	v := atomic.Xchg(key32(&l.key), mutex_unlocked)
+	if v == mutex_unlocked {
+		throw("unlock of unlocked lock")
+	}
+	if v == mutex_sleeping {
+		futexwakeup(key32(&l.key), 1)
 	}
 
 	gp := getg()