Skip to content

Commit e09dbaa

Browse files
committed
runtime: schedule fractional workers on all Ps
Currently only a single P can run a fractional mark worker at a time. This doesn't let us spread out the load, so it gets concentrated on whatever unlucky P picks up the token to run a fractional worker. This can significantly delay goroutines on that P. This commit changes this scheduling rule so each P separately schedules fractional workers. This can significantly reduce the load on any individual P and allows workers to self-preempt earlier. It does have the downside that it's possible for all Ps to be in fractional workers simultaneously (an effect STW). Updates #21698. Change-Id: Ia1e300c422043fa62bb4e3dd23c6232d81e4419c Reviewed-on: https://go-review.googlesource.com/68574 Run-TryBot: Austin Clements <[email protected]> TryBot-Result: Gobot Gobot <[email protected]> Reviewed-by: Rick Hudson <[email protected]>
1 parent 28e1a8e commit e09dbaa

File tree

2 files changed

+30
-49
lines changed

2 files changed

+30
-49
lines changed

src/runtime/mgc.go

Lines changed: 26 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -396,23 +396,18 @@ type gcControllerState struct {
396396
assistBytesPerWork float64
397397

398398
// fractionalUtilizationGoal is the fraction of wall clock
399-
// time that should be spent in the fractional mark worker.
400-
// For example, if the overall mark utilization goal is 25%
401-
// and GOMAXPROCS is 6, one P will be a dedicated mark worker
402-
// and this will be set to 0.5 so that 50% of the time some P
403-
// is in a fractional mark worker. This is computed at the
404-
// beginning of each cycle.
399+
// time that should be spent in the fractional mark worker on
400+
// each P that isn't running a dedicated worker.
401+
//
402+
// For example, if the utilization goal is 25% and there are
403+
// no dedicated workers, this will be 0.25. If there goal is
404+
// 25%, there is one dedicated worker, and GOMAXPROCS is 5,
405+
// this will be 0.05 to make up the missing 5%.
406+
//
407+
// If this is zero, no fractional workers are needed.
405408
fractionalUtilizationGoal float64
406409

407410
_ [sys.CacheLineSize]byte
408-
409-
// fractionalMarkWorkersNeeded is the number of fractional
410-
// mark workers that need to be started. This is either 0 or
411-
// 1. This is potentially updated atomically at every
412-
// scheduling point (hence it gets its own cache line).
413-
fractionalMarkWorkersNeeded int64
414-
415-
_ [sys.CacheLineSize]byte
416411
}
417412

418413
// startCycle resets the GC controller's state and computes estimates
@@ -471,19 +466,15 @@ func (c *gcControllerState) startCycle() {
471466
// Too many dedicated workers.
472467
c.dedicatedMarkWorkersNeeded--
473468
}
474-
c.fractionalUtilizationGoal = totalUtilizationGoal - float64(c.dedicatedMarkWorkersNeeded)
469+
c.fractionalUtilizationGoal = (totalUtilizationGoal - float64(c.dedicatedMarkWorkersNeeded)) / float64(gomaxprocs)
475470
} else {
476471
c.fractionalUtilizationGoal = 0
477472
}
478-
if c.fractionalUtilizationGoal > 0 {
479-
c.fractionalMarkWorkersNeeded = 1
480-
} else {
481-
c.fractionalMarkWorkersNeeded = 0
482-
}
483473

484474
// Clear per-P state
485475
for _, p := range allp {
486476
p.gcAssistTime = 0
477+
p.gcFractionalMarkTime = 0
487478
}
488479

489480
// Compute initial values for controls that are updated
@@ -496,7 +487,7 @@ func (c *gcControllerState) startCycle() {
496487
work.initialHeapLive>>20, "->",
497488
memstats.next_gc>>20, " MB)",
498489
" workers=", c.dedicatedMarkWorkersNeeded,
499-
"+", c.fractionalMarkWorkersNeeded, "\n")
490+
"+", c.fractionalUtilizationGoal, "\n")
500491
}
501492
}
502493

@@ -702,31 +693,20 @@ func (c *gcControllerState) findRunnableGCWorker(_p_ *p) *g {
702693
// This P is now dedicated to marking until the end of
703694
// the concurrent mark phase.
704695
_p_.gcMarkWorkerMode = gcMarkWorkerDedicatedMode
696+
} else if c.fractionalUtilizationGoal == 0 {
697+
// No need for fractional workers.
698+
return nil
705699
} else {
706-
if !decIfPositive(&c.fractionalMarkWorkersNeeded) {
707-
// No more workers are need right now.
708-
return nil
709-
}
710-
711-
// This P has picked the token for the fractional worker.
712-
// Is the GC currently under or at the utilization goal?
713-
// If so, do more work.
700+
// Is this P behind on the fractional utilization
701+
// goal?
714702
//
715703
// This should be kept in sync with pollFractionalWorkerExit.
716-
717-
// TODO(austin): We could fast path this and basically
718-
// eliminate contention on c.fractionalMarkWorkersNeeded by
719-
// precomputing the minimum time at which it's worth
720-
// next scheduling the fractional worker. Then Ps
721-
// don't have to fight in the window where we've
722-
// passed that deadline and no one has started the
723-
// worker yet.
724-
delta := nanotime() - c.markStartTime
725-
if delta > 0 && float64(c.fractionalMarkTime)/float64(delta) > c.fractionalUtilizationGoal {
726-
// Nope, we'd overshoot the utilization goal
727-
atomic.Xaddint64(&c.fractionalMarkWorkersNeeded, +1)
704+
delta := nanotime() - gcController.markStartTime
705+
if delta > 0 && float64(_p_.gcFractionalMarkTime)/float64(delta) > c.fractionalUtilizationGoal {
706+
// Nope. No need to run a fractional worker.
728707
return nil
729708
}
709+
// Run a fractional worker.
730710
_p_.gcMarkWorkerMode = gcMarkWorkerFractionalMode
731711
}
732712

@@ -751,8 +731,7 @@ func pollFractionalWorkerExit() bool {
751731
return true
752732
}
753733
p := getg().m.p.ptr()
754-
// Account for time since starting this worker.
755-
selfTime := gcController.fractionalMarkTime + (now - p.gcMarkWorkerStartTime)
734+
selfTime := p.gcFractionalMarkTime + (now - p.gcMarkWorkerStartTime)
756735
// Add some slack to the utilization goal so that the
757736
// fractional worker isn't behind again the instant it exits.
758737
return float64(selfTime)/float64(delta) > 1.2*gcController.fractionalUtilizationGoal
@@ -1387,7 +1366,8 @@ top:
13871366
// TODO(austin): Should dedicated workers keep an eye on this
13881367
// and exit gcDrain promptly?
13891368
atomic.Xaddint64(&gcController.dedicatedMarkWorkersNeeded, -0xffffffff)
1390-
atomic.Xaddint64(&gcController.fractionalMarkWorkersNeeded, -0xffffffff)
1369+
prevFractionalGoal := gcController.fractionalUtilizationGoal
1370+
gcController.fractionalUtilizationGoal = 0
13911371

13921372
if !gcBlackenPromptly {
13931373
// Transition from mark 1 to mark 2.
@@ -1430,7 +1410,7 @@ top:
14301410

14311411
// Now we can start up mark 2 workers.
14321412
atomic.Xaddint64(&gcController.dedicatedMarkWorkersNeeded, 0xffffffff)
1433-
atomic.Xaddint64(&gcController.fractionalMarkWorkersNeeded, 0xffffffff)
1413+
gcController.fractionalUtilizationGoal = prevFractionalGoal
14341414

14351415
incnwait := atomic.Xadd(&work.nwait, +1)
14361416
if incnwait == work.nproc && !gcMarkWorkAvailable(nil) {
@@ -1849,7 +1829,7 @@ func gcBgMarkWorker(_p_ *p) {
18491829
atomic.Xaddint64(&gcController.dedicatedMarkWorkersNeeded, 1)
18501830
case gcMarkWorkerFractionalMode:
18511831
atomic.Xaddint64(&gcController.fractionalMarkTime, duration)
1852-
atomic.Xaddint64(&gcController.fractionalMarkWorkersNeeded, 1)
1832+
atomic.Xaddint64(&_p_.gcFractionalMarkTime, duration)
18531833
case gcMarkWorkerIdleMode:
18541834
atomic.Xaddint64(&gcController.idleMarkTime, duration)
18551835
}

src/runtime/runtime2.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -522,9 +522,10 @@ type p struct {
522522
palloc persistentAlloc // per-P to avoid mutex
523523

524524
// Per-P GC state
525-
gcAssistTime int64 // Nanoseconds in assistAlloc
526-
gcBgMarkWorker guintptr
527-
gcMarkWorkerMode gcMarkWorkerMode
525+
gcAssistTime int64 // Nanoseconds in assistAlloc
526+
gcFractionalMarkTime int64 // Nanoseconds in fractional mark worker
527+
gcBgMarkWorker guintptr
528+
gcMarkWorkerMode gcMarkWorkerMode
528529

529530
// gcMarkWorkerStartTime is the nanotime() at which this mark
530531
// worker started.

0 commit comments

Comments
 (0)