testing: make benchmarking faster

The number of estimated iterations required to reach the benchtime is multiplied by a safety margin (to avoid falling just short) and then rounded up to a readable number. With an accurate estimate, in the worse case, the resulting number of iterations could be 3.75x more than necessary: 1.5x for safety * 2.5x to round up (e.g. from 2eX+1 to 5eX).

This CL reduces the safety margin to 1.2x. Experimentation showed a diminishing margin of return past 1.2x, although the average case continued to show improvements down to 1.05x.

This CL also reduces the maximum round-up multiplier from 2.5x (from 2eX+1 to 5eX) to 2x, by allowing the number of iterations to be of the form 3eX.

Both changes improve benchmark wall clock times, and the effects are cumulative.

From 1.5x to 1.2x safety margin:

package		old s	new s	delta
bytes		163	125	-23%
encoding/json	27	21	-22%
net/http	42	36	-14%
runtime		463	418	-10%
strings		82	65	-21%

Allowing 3eX iterations:

package		old s	new s	delta
bytes		163	134	-18%
encoding/json	27	23	-15%
net/http	42	36	-14%
runtime		463	422	-9%
strings		82	72	-12%

Combined:

package		old s	new s	delta
bytes		163	112	-31%
encoding/json	27	20	-26%
net/http	42	30	-29%
runtime		463	346	-25%
strings		82	60	-27%

LGTM=crawshaw, r, rsc
R=golang-codereviews, crawshaw, r, rsc
CC=golang-codereviews
https://golang.org/cl/105990045

Author: josharian

src/pkg/testing/benchmark.go

@@ -157,14 +157,16 @@ func roundDown10(n int) int {
 	return result
 }
 
-// roundUp rounds x up to a number of the form [1eX, 2eX, 5eX].
+// roundUp rounds x up to a number of the form [1eX, 2eX, 3eX, 5eX].
 func roundUp(n int) int {
 	base := roundDown10(n)
 	switch {
 	case n <= base:
 		return base
 	case n <= (2 * base):
 		return 2 * base
+	case n <= (3 * base):
+		return 3 * base
 	case n <= (5 * base):
 		return 5 * base
 	default:
@@ -180,10 +182,10 @@ func (b *B) run() BenchmarkResult {
 }
 
 // launch launches the benchmark function.  It gradually increases the number
-// of benchmark iterations until the benchmark runs for a second in order
-// to get a reasonable measurement.  It prints timing information in this form
+// of benchmark iterations until the benchmark runs for the requested benchtime.
+// It prints timing information in this form
 //		testing.BenchmarkHello	100000		19 ns/op
-// launch is run by the fun function as a separate goroutine.
+// launch is run by the run function as a separate goroutine.
 func (b *B) launch() {
 	// Run the benchmark for a single iteration in case it's expensive.
 	n := 1
@@ -199,16 +201,16 @@ func (b *B) launch() {
 	d := *benchTime
 	for !b.failed && b.duration < d && n < 1e9 {
 		last := n
-		// Predict iterations/sec.
+		// Predict required iterations.
 		if b.nsPerOp() == 0 {
 			n = 1e9
 		} else {
 			n = int(d.Nanoseconds() / b.nsPerOp())
 		}
-		// Run more iterations than we think we'll need for a second (1.5x).
+		// Run more iterations than we think we'll need (1.2x).
 		// Don't grow too fast in case we had timing errors previously.
 		// Be sure to run at least one more than last time.
-		n = max(min(n+n/2, 100*last), last+1)
+		n = max(min(n+n/5, 100*last), last+1)
 		// Round up to something easy to read.
 		n = roundUp(n)
 		b.runN(n)

src/pkg/testing/benchmark_test.go

@@ -41,12 +41,14 @@ var roundUpTests = []struct {
 	{0, 1},
 	{1, 1},
 	{2, 2},
+	{3, 3},
 	{5, 5},
 	{9, 10},
 	{999, 1000},
 	{1000, 1000},
 	{1400, 2000},
 	{1700, 2000},
+	{2700, 3000},
 	{4999, 5000},
 	{5000, 5000},
 	{5001, 10000},