testing: stop rounding b.N

The original goal of rounding to readable b.N
was to make it easier to eyeball times.
However, proper analysis requires tooling
(such as benchstat) anyway.

Instead, take b.N as it comes.
This will reduce the impact of external noise
such as GC on benchmarks.

This requires reworking our iteration estimates.
We used to calculate the estimated ns/op
and then divide our target ns by that estimate.
However, this order of operations was destructive
when the ns/op was very small; rounding could
hide almost an order of magnitude of variation.
Instead, multiply first, then divide.
Also, make n an int64 to avoid overflow.

Prior to this change, we attempted to cap b.N at 1e9.
Due to rounding up, it was possible to get b.N as high as 2e9.
This change consistently enforces the 1e9 cap.

This change also reduces the wall time required to run benchmarks.

Here's the impact of this change on the wall time to run
all benchmarks once with benchtime=1s on some std packages:

name           old time/op       new time/op       delta
bytes                 306s ± 1%         238s ± 1%  -22.24%  (p=0.000 n=10+10)
encoding/json         112s ± 8%          99s ± 7%  -11.64%  (p=0.000 n=10+10)
net/http             54.7s ± 7%        44.9s ± 4%  -17.94%  (p=0.000 n=10+9)
runtime               957s ± 1%         714s ± 0%  -25.38%  (p=0.000 n=10+9)
strings               262s ± 1%         201s ± 1%  -23.27%  (p=0.000 n=10+10)
[Geo mean]            216s              172s       -20.23%

Updates #24735

Change-Id: I7e38efb8e23c804046bf4fc065b3f5f3991d0a15
Reviewed-on: https://go-review.googlesource.com/c/go/+/112155
Reviewed-by: Austin Clements <[email protected]>

Author: josharian

src/cmd/go/go_test.go

@@ -4947,14 +4947,14 @@ func TestTestRegexps(t *testing.T) {
     x_test.go:15: LOG: Y running N=10000
     x_test.go:15: LOG: Y running N=1000000
     x_test.go:15: LOG: Y running N=100000000
-    x_test.go:15: LOG: Y running N=2000000000
+    x_test.go:15: LOG: Y running N=1000000000
 --- BENCH: BenchmarkX/Y
     x_test.go:15: LOG: Y running N=1
     x_test.go:15: LOG: Y running N=100
     x_test.go:15: LOG: Y running N=10000
     x_test.go:15: LOG: Y running N=1000000
     x_test.go:15: LOG: Y running N=100000000
-    x_test.go:15: LOG: Y running N=2000000000
+    x_test.go:15: LOG: Y running N=1000000000
 --- BENCH: BenchmarkX
     x_test.go:13: LOG: X running N=1
 --- BENCH: BenchmarkXX

src/testing/benchmark.go

@@ -170,13 +170,6 @@ func (b *B) ReportAllocs() {
 	b.showAllocResult = true
 }
 
-func (b *B) nsPerOp() int64 {
-	if b.N <= 0 {
-		return 0
-	}
-	return b.duration.Nanoseconds() / int64(b.N)
-}
-
 // runN runs a single benchmark for the specified number of iterations.
 func (b *B) runN(n int) {
 	benchmarkLock.Lock()
@@ -199,53 +192,20 @@ func (b *B) runN(n int) {
 	}
 }
 
-func min(x, y int) int {
+func min(x, y int64) int64 {
 	if x > y {
 		return y
 	}
 	return x
 }
 
-func max(x, y int) int {
+func max(x, y int64) int64 {
 	if x < y {
 		return y
 	}
 	return x
 }
 
-// roundDown10 rounds a number down to the nearest power of 10.
-func roundDown10(n int) int {
-	var tens = 0
-	// tens = floor(log_10(n))
-	for n >= 10 {
-		n = n / 10
-		tens++
-	}
-	// result = 10^tens
-	result := 1
-	for i := 0; i < tens; i++ {
-		result *= 10
-	}
-	return result
-}
-
-// roundUp rounds x up to a number of the form [1eX, 2eX, 3eX, 5eX].
-func roundUp(n int) int {
-	base := roundDown10(n)
-	switch {
-	case n <= base:
-		return base
-	case n <= (2 * base):
-		return 2 * base
-	case n <= (3 * base):
-		return 3 * base
-	case n <= (5 * base):
-		return 5 * base
-	default:
-		return 10 * base
-	}
-}
-
 // run1 runs the first iteration of benchFunc. It reports whether more
 // iterations of this benchmarks should be run.
 func (b *B) run1() bool {
@@ -328,20 +288,31 @@ func (b *B) launch() {
 		b.runN(b.benchTime.n)
 	} else {
 		d := b.benchTime.d
-		for n := 1; !b.failed && b.duration < d && n < 1e9; {
+		for n := int64(1); !b.failed && b.duration < d && n < 1e9; {
 			last := n
 			// Predict required iterations.
-			n = int(d.Nanoseconds())
-			if nsop := b.nsPerOp(); nsop != 0 {
-				n /= int(nsop)
+			goalns := d.Nanoseconds()
+			prevIters := int64(b.N)
+			prevns := b.duration.Nanoseconds()
+			if prevns <= 0 {
+				// Round up, to avoid div by zero.
+				prevns = 1
 			}
+			// Order of operations matters.
+			// For very fast benchmarks, prevIters ~= prevns.
+			// If you divide first, you get 0 or 1,
+			// which can hide an order of magnitude in execution time.
+			// So multiply first, then divide.
+			n = goalns * prevIters / prevns
 			// Run more iterations than we think we'll need (1.2x).
+			n += n / 5
 			// Don't grow too fast in case we had timing errors previously.
+			n = min(n, 100*last)
 			// Be sure to run at least one more than last time.
-			n = max(min(n+n/5, 100*last), last+1)
-			// Round up to something easy to read.
-			n = roundUp(n)
-			b.runN(n)
+			n = max(n, last+1)
+			// Don't run more than 1e9 times. (This also keeps n in int range on 32 bit platforms.)
+			n = min(n, 1e9)
+			b.runN(int(n))
 		}
 	}
 	b.result = BenchmarkResult{b.N, b.duration, b.bytes, b.netAllocs, b.netBytes, b.extra}

src/testing/benchmark_test.go

@@ -14,57 +14,6 @@ import (
 	"text/template"
 )
 
-var roundDownTests = []struct {
-	v, expected int
-}{
-	{1, 1},
-	{9, 1},
-	{10, 10},
-	{11, 10},
-	{100, 100},
-	{101, 100},
-	{999, 100},
-	{1000, 1000},
-	{1001, 1000},
-}
-
-func TestRoundDown10(t *testing.T) {
-	for _, tt := range roundDownTests {
-		actual := testing.RoundDown10(tt.v)
-		if tt.expected != actual {
-			t.Errorf("roundDown10(%d): expected %d, actual %d", tt.v, tt.expected, actual)
-		}
-	}
-}
-
-var roundUpTests = []struct {
-	v, expected int
-}{
-	{0, 1},
-	{1, 1},
-	{2, 2},
-	{3, 3},
-	{5, 5},
-	{9, 10},
-	{999, 1000},
-	{1000, 1000},
-	{1400, 2000},
-	{1700, 2000},
-	{2700, 3000},
-	{4999, 5000},
-	{5000, 5000},
-	{5001, 10000},
-}
-
-func TestRoundUp(t *testing.T) {
-	for _, tt := range roundUpTests {
-		actual := testing.RoundUp(tt.v)
-		if tt.expected != actual {
-			t.Errorf("roundUp(%d): expected %d, actual %d", tt.v, tt.expected, actual)
-		}
-	}
-}
-
 var prettyPrintTests = []struct {
 	v        float64
 	expected string

src/testing/export_test.go

@@ -4,8 +4,4 @@
 
 package testing
 
-var (
-	RoundDown10 = roundDown10
-	RoundUp     = roundUp
-	PrettyPrint = prettyPrint
-)
+var PrettyPrint = prettyPrint