Skip to content

Commit d881ed6

Browse files
rhyshgopherbot
authored andcommitted
runtime: double-link list of waiting Ms
When an M unlocks a contended mutex, it needs to consult a list of the Ms that had to wait during its critical section. This allows the M to attribute the appropriate amount of blame to the unlocking call stack. Mirroring the implementation for users' sync.Mutex contention (via sudog), we can (in a future commit) use the time that the head and tail of the wait list started waiting, and the number of waiters, to estimate the sum of the Ms' delays. When an M acquires the mutex, it needs to remove itself from the list of waiters. Since the futex-based lock implementation leaves the OS in control of the order of M wakeups, we need to be prepared for quickly (constant time) removing any M from the list. First, have each M add itself to a singly-linked wait list when it finds that its lock call will need to sleep. This case is safe against live-lock, since any delay to one M adding itself to the list would be due to another M making durable progress. Second, have the M that holds the lock (either right before releasing, or right after acquiring) update metadata on the list of waiting Ms to double-link the list and maintain a tail pointer and waiter count. That work is amortized-constant: we'll avoid contended locks becoming proportionally more contended and undergoing performance collapse. For #66999 Change-Id: If75cdea915afb59ccec47294e0b52c466aac8736 Reviewed-on: https://go-review.googlesource.com/c/go/+/585637 Reviewed-by: Dmitri Shuralyov <[email protected]> Reviewed-by: Michael Pratt <[email protected]> LUCI-TryBot-Result: Go LUCI <[email protected]> Auto-Submit: Rhys Hiltner <[email protected]>
1 parent dfb7073 commit d881ed6

File tree

3 files changed

+324
-25
lines changed

3 files changed

+324
-25
lines changed

src/runtime/lock_futex.go

Lines changed: 56 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,13 @@ const (
3737
// independently: a thread can enter lock2, observe that another thread is
3838
// already asleep, and immediately try to grab the lock anyway without waiting
3939
// for its "fair" turn.
40+
//
41+
// The rest of mutex.key holds a pointer to the head of a linked list of the Ms
42+
// that are waiting for the mutex. The pointer portion is set if and only if the
43+
// mutex_sleeping flag is set. Because the futex syscall operates on 32 bits but
44+
// a uintptr may be larger, the flag lets us be sure the futexsleep call will
45+
// only commit if the pointer portion is unset. Otherwise an M allocated at an
46+
// address like 0x123_0000_0000 might miss its wakeups.
4047

4148
// We use the uintptr mutex.key and note.key as a uint32.
4249
//
@@ -67,18 +74,53 @@ func lock2(l *mutex) {
6774

6875
timer := &lockTimer{lock: l}
6976
timer.begin()
77+
78+
// If a goroutine's stack needed to grow during a lock2 call, the M could
79+
// end up with two active lock2 calls (one each on curg and g0). If both are
80+
// contended, the call on g0 will corrupt mWaitList. Disable stack growth.
81+
stackguard0, throwsplit := gp.stackguard0, gp.throwsplit
82+
if gp == gp.m.curg {
83+
gp.stackguard0, gp.throwsplit = stackPreempt, true
84+
}
85+
7086
// On uniprocessors, no point spinning.
7187
// On multiprocessors, spin for ACTIVE_SPIN attempts.
7288
spin := 0
7389
if ncpu > 1 {
7490
spin = active_spin
7591
}
92+
var enqueued bool
7693
Loop:
7794
for i := 0; ; i++ {
7895
v := atomic.Loaduintptr(&l.key)
7996
if v&mutex_locked == 0 {
8097
// Unlocked. Try to lock.
8198
if atomic.Casuintptr(&l.key, v, v|mutex_locked) {
99+
// We now own the mutex
100+
v = v | mutex_locked
101+
for {
102+
old := v
103+
104+
head := muintptr(v &^ (mutex_sleeping | mutex_locked))
105+
fixMutexWaitList(head)
106+
if enqueued {
107+
head = removeMutexWaitList(head, gp.m)
108+
}
109+
110+
v = mutex_locked
111+
if head != 0 {
112+
v = v | uintptr(head) | mutex_sleeping
113+
}
114+
115+
if v == old || atomic.Casuintptr(&l.key, old, v) {
116+
gp.m.mWaitList.clearLinks()
117+
break
118+
}
119+
v = atomic.Loaduintptr(&l.key)
120+
}
121+
if gp == gp.m.curg {
122+
gp.stackguard0, gp.throwsplit = stackguard0, throwsplit
123+
}
82124
timer.end()
83125
return
84126
}
@@ -90,21 +132,28 @@ Loop:
90132
osyield()
91133
} else {
92134
// Someone else has it.
135+
// l->key points to a linked list of M's waiting
136+
// for this lock, chained through m->mWaitList.next.
137+
// Queue this M.
93138
for {
94139
head := v &^ (mutex_locked | mutex_sleeping)
95-
if atomic.Casuintptr(&l.key, v, head|mutex_locked|mutex_sleeping) {
96-
break
140+
if !enqueued {
141+
gp.m.mWaitList.next = muintptr(head)
142+
head = uintptr(unsafe.Pointer(gp.m))
143+
if atomic.Casuintptr(&l.key, v, head|mutex_locked|mutex_sleeping) {
144+
enqueued = true
145+
break
146+
}
147+
gp.m.mWaitList.next = 0
97148
}
98149
v = atomic.Loaduintptr(&l.key)
99150
if v&mutex_locked == 0 {
100151
continue Loop
101152
}
102153
}
103-
if v&mutex_locked != 0 {
104-
// Queued. Wait.
105-
futexsleep(key32(&l.key), uint32(v), -1)
106-
i = 0
107-
}
154+
// Queued. Wait.
155+
futexsleep(key32(&l.key), uint32(v), -1)
156+
i = 0
108157
}
109158
}
110159
}

src/runtime/lock_sema.go

Lines changed: 48 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -54,18 +54,49 @@ func lock2(l *mutex) {
5454

5555
timer := &lockTimer{lock: l}
5656
timer.begin()
57+
58+
// If a goroutine's stack needed to grow during a lock2 call, the M could
59+
// end up with two active lock2 calls (one each on curg and g0). If both are
60+
// contended, the call on g0 will corrupt mWaitList. Disable stack growth.
61+
stackguard0, throwsplit := gp.stackguard0, gp.throwsplit
62+
if gp == gp.m.curg {
63+
gp.stackguard0, gp.throwsplit = stackPreempt, true
64+
}
65+
5766
// On uniprocessor's, no point spinning.
5867
// On multiprocessors, spin for ACTIVE_SPIN attempts.
5968
spin := 0
6069
if ncpu > 1 {
6170
spin = active_spin
6271
}
72+
var enqueued bool
6373
Loop:
6474
for i := 0; ; i++ {
6575
v := atomic.Loaduintptr(&l.key)
6676
if v&locked == 0 {
6777
// Unlocked. Try to lock.
6878
if atomic.Casuintptr(&l.key, v, v|locked) {
79+
// We now own the mutex
80+
v = v | locked
81+
for {
82+
old := v
83+
84+
head := muintptr(v &^ locked)
85+
fixMutexWaitList(head)
86+
if enqueued {
87+
head = removeMutexWaitList(head, gp.m)
88+
}
89+
v = locked | uintptr(head)
90+
91+
if v == old || atomic.Casuintptr(&l.key, old, v) {
92+
gp.m.mWaitList.clearLinks()
93+
break
94+
}
95+
v = atomic.Loaduintptr(&l.key)
96+
}
97+
if gp == gp.m.curg {
98+
gp.stackguard0, gp.throwsplit = stackguard0, throwsplit
99+
}
69100
timer.end()
70101
return
71102
}
@@ -81,20 +112,29 @@ Loop:
81112
// for this lock, chained through m.mWaitList.next.
82113
// Queue this M.
83114
for {
84-
gp.m.mWaitList.next = muintptr(v &^ locked)
85-
if atomic.Casuintptr(&l.key, v, uintptr(unsafe.Pointer(gp.m))|locked) {
86-
break
115+
if !enqueued {
116+
gp.m.mWaitList.next = muintptr(v &^ locked)
117+
if atomic.Casuintptr(&l.key, v, uintptr(unsafe.Pointer(gp.m))|locked) {
118+
enqueued = true
119+
break
120+
}
121+
gp.m.mWaitList.next = 0
87122
}
123+
88124
v = atomic.Loaduintptr(&l.key)
89125
if v&locked == 0 {
90126
continue Loop
91127
}
92128
}
93-
if v&locked != 0 {
94-
// Queued. Wait.
95-
semasleep(-1)
96-
i = 0
97-
}
129+
// Queued. Wait.
130+
semasleep(-1)
131+
i = 0
132+
enqueued = false
133+
// unlock2 removed this M from the list (it was at the head). We
134+
// need to erase the metadata about its former position in the
135+
// list -- and since it's no longer a published member we can do
136+
// so without races.
137+
gp.m.mWaitList.clearLinks()
98138
}
99139
}
100140
}

0 commit comments

Comments
 (0)