runtime: remove periodic scavenging

This change removes the periodic scavenger which goes over every span
in the heap and scavenges it if it hasn't been used for 5 minutes. It
should no longer be necessary if we have background scavenging
(follow-up).

For #30333.

Change-Id: Ic3a1a4e85409dc25719ba4593a3b60273a4c71e0
Reviewed-on: https://go-review.googlesource.com/c/go/+/143157
Run-TryBot: Michael Knyszek <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Austin Clements <[email protected]>

Author: mknyszek

src/runtime/mheap.go

@@ -394,7 +394,6 @@ type mspan struct {
 	divShift2   uint8      // for divide by elemsize - divMagic.shift2
 	scavenged   bool       // whether this span has had its pages released to the OS
 	elemsize    uintptr    // computed from sizeclass or from npages
-	unusedsince int64      // first time spotted by gc in mspanfree state
 	limit       uintptr    // end of data in span
 	speciallock mutex      // guards specials list
 	specials    *special   // linked list of special records sorted by offset.
@@ -1209,10 +1208,9 @@ HaveSpan:
 		// Also, scavenge may cause coalescing, so prevent
 		// coalescing with s by temporarily changing its state.
 		s.state = mSpanManual
-		h.scavengeLocked(s.npages * pageSize)
+		h.scavengeLocked(s.npages*pageSize, true)
 		s.state = mSpanFree
 	}
-	s.unusedsince = 0
 
 	h.setSpans(s.base(), npage, s)
 
@@ -1243,7 +1241,7 @@ func (h *mheap) grow(npage uintptr) bool {
 	// is proportional to the number of sysUnused() calls rather than
 	// the number of pages released, so we make fewer of those calls
 	// with larger spans.
-	h.scavengeLocked(size)
+	h.scavengeLocked(size, true)
 
 	// Create a fake "in use" span and free it, so that the
 	// right coalescing happens.
@@ -1253,7 +1251,7 @@ func (h *mheap) grow(npage uintptr) bool {
 	atomic.Store(&s.sweepgen, h.sweepgen)
 	s.state = mSpanInUse
 	h.pagesInUse += uint64(s.npages)
-	h.freeSpanLocked(s, false, true, 0)
+	h.freeSpanLocked(s, false, true)
 	return true
 }
 
@@ -1283,7 +1281,7 @@ func (h *mheap) freeSpan(s *mspan, large bool) {
 			// heap_scan changed.
 			gcController.revise()
 		}
-		h.freeSpanLocked(s, true, true, 0)
+		h.freeSpanLocked(s, true, true)
 		unlock(&h.lock)
 	})
 }
@@ -1304,12 +1302,12 @@ func (h *mheap) freeManual(s *mspan, stat *uint64) {
 	lock(&h.lock)
 	*stat -= uint64(s.npages << _PageShift)
 	memstats.heap_sys += uint64(s.npages << _PageShift)
-	h.freeSpanLocked(s, false, true, 0)
+	h.freeSpanLocked(s, false, true)
 	unlock(&h.lock)
 }
 
 // s must be on the busy list or unlinked.
-func (h *mheap) freeSpanLocked(s *mspan, acctinuse, acctidle bool, unusedsince int64) {
+func (h *mheap) freeSpanLocked(s *mspan, acctinuse, acctidle bool) {
 	switch s.state {
 	case mSpanManual:
 		if s.allocCount != 0 {
@@ -1337,13 +1335,6 @@ func (h *mheap) freeSpanLocked(s *mspan, acctinuse, acctidle bool, unusedsince i
 	}
 	s.state = mSpanFree
 
-	// Stamp newly unused spans. The scavenger will use that
-	// info to potentially give back some pages to the OS.
-	s.unusedsince = unusedsince
-	if unusedsince == 0 {
-		s.unusedsince = nanotime()
-	}
-
 	// Coalesce span with neighbors.
 	h.coalesce(s)
 
@@ -1353,15 +1344,23 @@ func (h *mheap) freeSpanLocked(s *mspan, acctinuse, acctidle bool, unusedsince i
 
 // scavengeLocked scavenges nbytes worth of spans in the free treap by
 // starting from the span with the highest base address and working down.
-// It then takes those spans and places them in scav. h must be locked.
-func (h *mheap) scavengeLocked(nbytes uintptr) {
+// It then takes those spans and places them in scav.
+//
+// useCredit determines whether a scavenging call should use the credit
+// system. In general, useCredit should be true except in special
+// circumstances.
+//
+// Returns the amount of memory scavenged in bytes. h must be locked.
+func (h *mheap) scavengeLocked(nbytes uintptr, useCredit bool) uintptr {
 	// Use up scavenge credit if there's any available.
-	if nbytes > h.scavengeCredit {
-		nbytes -= h.scavengeCredit
-		h.scavengeCredit = 0
-	} else {
-		h.scavengeCredit -= nbytes
-		return
+	if useCredit {
+		if nbytes > h.scavengeCredit {
+			nbytes -= h.scavengeCredit
+			h.scavengeCredit = 0
+		} else {
+			h.scavengeCredit -= nbytes
+			return nbytes
+		}
 	}
 	released := uintptr(0)
 	// Iterate over spans with huge pages first, then spans without.
@@ -1388,60 +1387,41 @@ func (h *mheap) scavengeLocked(nbytes uintptr) {
 			h.free.insert(s)
 		}
 	}
-	// If we over-scavenged, turn that extra amount into credit.
-	if released > nbytes {
-		h.scavengeCredit += released - nbytes
-	}
-}
-
-// scavengeAll visits each node in the unscav treap and scavenges the
-// treapNode's span. It then removes the scavenged span from
-// unscav and adds it into scav before continuing. h must be locked.
-func (h *mheap) scavengeAllLocked(now, limit uint64) uintptr {
-	// Iterate over the unscavenged spans in the treap scavenging spans
-	// if unused for at least limit time.
-	released := uintptr(0)
-	for t := h.free.start(treapIterScav, 0); t.valid(); {
-		s := t.span()
-		n := t.next()
-		if (now - uint64(s.unusedsince)) > limit {
-			start, end := s.physPageBounds()
-			if start < end {
-				h.free.erase(t)
-				released += s.scavenge()
-				// See (*mheap).scavengeLocked.
-				h.coalesce(s)
-				h.free.insert(s)
-			}
+	if useCredit {
+		// If we over-scavenged, turn that extra amount into credit.
+		if released > nbytes {
+			h.scavengeCredit += released - nbytes
 		}
-		t = n
 	}
 	return released
 }
 
-func (h *mheap) scavengeAll(k int32, now, limit uint64) {
+// scavengeAll visits each node in the free treap and scavenges the
+// treapNode's span. It then removes the scavenged span from
+// unscav and adds it into scav before continuing.
+func (h *mheap) scavengeAll() {
 	// Disallow malloc or panic while holding the heap lock. We do
 	// this here because this is an non-mallocgc entry-point to
 	// the mheap API.
 	gp := getg()
 	gp.m.mallocing++
 	lock(&h.lock)
-	released := h.scavengeAllLocked(now, limit)
+	released := h.scavengeLocked(^uintptr(0), false)
 	unlock(&h.lock)
 	gp.m.mallocing--
 
 	if debug.gctrace > 0 {
 		if released > 0 {
-			print("scvg", k, ": ", released>>20, " MB released\n")
+			print("forced scvg: ", released>>20, " MB released\n")
 		}
-		print("scvg", k, ": inuse: ", memstats.heap_inuse>>20, ", idle: ", memstats.heap_idle>>20, ", sys: ", memstats.heap_sys>>20, ", released: ", memstats.heap_released>>20, ", consumed: ", (memstats.heap_sys-memstats.heap_released)>>20, " (MB)\n")
+		print("forced scvg: inuse: ", memstats.heap_inuse>>20, ", idle: ", memstats.heap_idle>>20, ", sys: ", memstats.heap_sys>>20, ", released: ", memstats.heap_released>>20, ", consumed: ", (memstats.heap_sys-memstats.heap_released)>>20, " (MB)\n")
 	}
 }
 
 //go:linkname runtime_debug_freeOSMemory runtime/debug.freeOSMemory
 func runtime_debug_freeOSMemory() {
 	GC()
-	systemstack(func() { mheap_.scavengeAll(-1, ^uint64(0), 0) })
+	systemstack(func() { mheap_.scavengeAll() })
 }
 
 // Initialize a new span with the given start and npages.
@@ -1456,7 +1436,6 @@ func (span *mspan) init(base uintptr, npages uintptr) {
 	span.spanclass = 0
 	span.elemsize = 0
 	span.state = mSpanDead
-	span.unusedsince = 0
 	span.scavenged = false
 	span.speciallock.key = 0
 	span.specials = nil

src/runtime/proc.go

@@ -4282,19 +4282,6 @@ func sysmon() {
 	checkdead()
 	unlock(&sched.lock)
 
-	// If a heap span goes unused for 5 minutes after a garbage collection,
-	// we hand it back to the operating system.
-	scavengelimit := int64(5 * 60 * 1e9)
-
-	if debug.scavenge > 0 {
-		// Scavenge-a-lot for testing.
-		forcegcperiod = 10 * 1e6
-		scavengelimit = 20 * 1e6
-	}
-
-	lastscavenge := nanotime()
-	nscavenge := 0
-
 	lasttrace := int64(0)
 	idle := 0 // how many cycles in succession we had not wokeup somebody
 	delay := uint32(0)
@@ -4316,9 +4303,6 @@ func sysmon() {
 				// Make wake-up period small enough
 				// for the sampling to be correct.
 				maxsleep := forcegcperiod / 2
-				if scavengelimit < forcegcperiod {
-					maxsleep = scavengelimit / 2
-				}
 				shouldRelax := true
 				if osRelaxMinNS > 0 {
 					next := timeSleepUntil()
@@ -4381,12 +4365,6 @@ func sysmon() {
 			injectglist(&list)
 			unlock(&forcegc.lock)
 		}
-		// scavenge heap once in a while
-		if lastscavenge+scavengelimit/2 < now {
-			mheap_.scavengeAll(int32(nscavenge), uint64(now), uint64(scavengelimit))
-			lastscavenge = now
-			nscavenge++
-		}
 		if debug.schedtrace > 0 && lasttrace+int64(debug.schedtrace)*1000000 <= now {
 			lasttrace = now
 			schedtrace(debug.scheddetail > 0)