|
| 1 | +// Copyright 2017 The Go Authors. All rights reserved. |
| 2 | +// Use of this source code is governed by a BSD-style |
| 3 | +// license that can be found in the LICENSE file. |
| 4 | + |
| 5 | +package main |
| 6 | + |
| 7 | +import ( |
| 8 | + "bytes" |
| 9 | + "errors" |
| 10 | + "fmt" |
| 11 | + "io" |
| 12 | + "path" |
| 13 | + "strings" |
| 14 | + "time" |
| 15 | + |
| 16 | + "golang.org/x/build/buildlet" |
| 17 | +) |
| 18 | + |
| 19 | +// benchRuns is the number of times to run each benchmark binary |
| 20 | +const benchRuns = 5 |
| 21 | + |
| 22 | +type benchmarkItem struct { |
| 23 | + binary string // name of binary relative to goroot |
| 24 | + args []string // args to run binary with |
| 25 | + preamble string // string to print before benchmark results (e.g. "pkg: test/bench/go1\n") |
| 26 | + output []string // old, new benchmark output |
| 27 | + |
| 28 | + build func(bc *buildlet.Client, goroot string, w io.Writer) (remoteErr, err error) // how to build benchmark binary |
| 29 | +} |
| 30 | + |
| 31 | +func (b *benchmarkItem) name() string { |
| 32 | + return b.binary + " " + strings.Join(b.args, " ") |
| 33 | +} |
| 34 | + |
| 35 | +// buildGo1 builds the Go 1 benchmarks. |
| 36 | +func (st *buildStatus) buildGo1(bc *buildlet.Client, goroot string, w io.Writer) (remoteErr, err error) { |
| 37 | + workDir, err := bc.WorkDir() |
| 38 | + if err != nil { |
| 39 | + return nil, err |
| 40 | + } |
| 41 | + var found bool |
| 42 | + if err := bc.ListDir(path.Join(goroot, "test/bench/go1"), buildlet.ListDirOpts{}, func(e buildlet.DirEntry) { |
| 43 | + switch e.Name() { |
| 44 | + case "go1.test", "go1.test.exe": |
| 45 | + found = true |
| 46 | + } |
| 47 | + }); err != nil { |
| 48 | + return nil, err |
| 49 | + } |
| 50 | + if found { |
| 51 | + return nil, nil |
| 52 | + } |
| 53 | + return bc.Exec(path.Join(goroot, "bin", "go"), buildlet.ExecOpts{ |
| 54 | + Output: w, |
| 55 | + ExtraEnv: []string{"GOROOT=" + st.conf.FilePathJoin(workDir, goroot)}, |
| 56 | + Args: []string{"test", "-c"}, |
| 57 | + Dir: path.Join(goroot, "test/bench/go1"), |
| 58 | + }) |
| 59 | +} |
| 60 | + |
| 61 | +// buildXBenchmark builds a benchmark from x/benchmarks. |
| 62 | +func (st *buildStatus) buildXBenchmark(bc *buildlet.Client, goroot string, w io.Writer, rev, pkg, name string) (remoteErr, err error) { |
| 63 | + workDir, err := bc.WorkDir() |
| 64 | + if err != nil { |
| 65 | + return nil, err |
| 66 | + } |
| 67 | + if err := bc.ListDir("gopath/src/golang.org/x/benchmarks", buildlet.ListDirOpts{}, func(buildlet.DirEntry) {}); err != nil { |
| 68 | + if err := st.fetchSubrepo(bc, "benchmarks", rev); err != nil { |
| 69 | + return nil, err |
| 70 | + } |
| 71 | + } |
| 72 | + return bc.Exec(path.Join(goroot, "bin/go"), buildlet.ExecOpts{ |
| 73 | + Output: w, |
| 74 | + ExtraEnv: []string{ |
| 75 | + "GOROOT=" + st.conf.FilePathJoin(workDir, goroot), |
| 76 | + "GOPATH=" + st.conf.FilePathJoin(workDir, "gopath"), |
| 77 | + }, |
| 78 | + Args: []string{"build", "-o", st.conf.FilePathJoin(workDir, goroot, name), pkg}, |
| 79 | + }) |
| 80 | +} |
| 81 | + |
| 82 | +func (st *buildStatus) enumerateBenchmarks(bc *buildlet.Client) ([]*benchmarkItem, error) { |
| 83 | + workDir, err := bc.WorkDir() |
| 84 | + if err != nil { |
| 85 | + err = fmt.Errorf("buildBench, WorkDir: %v", err) |
| 86 | + return nil, err |
| 87 | + } |
| 88 | + // Fetch x/benchmarks |
| 89 | + rev := getRepoHead("benchmarks") |
| 90 | + if rev == "" { |
| 91 | + rev = "master" // should happen rarely; ok if it does. |
| 92 | + } |
| 93 | + |
| 94 | + if err := st.fetchSubrepo(bc, "benchmarks", rev); err != nil { |
| 95 | + return nil, err |
| 96 | + } |
| 97 | + |
| 98 | + var out []*benchmarkItem |
| 99 | + |
| 100 | + // These regexes shard the go1 tests so each shard takes about 20s, ensuring no test runs for |
| 101 | + for _, re := range []string{`^Benchmark[BF]`, `^Benchmark[HR]`, `^Benchmark[^BFHR]`} { |
| 102 | + out = append(out, &benchmarkItem{ |
| 103 | + binary: "test/bench/go1/go1.test", |
| 104 | + args: []string{"-test.bench", re, "-test.benchmem"}, |
| 105 | + preamble: "pkg: test/bench/go1\n", |
| 106 | + build: st.buildGo1, |
| 107 | + }) |
| 108 | + } |
| 109 | + |
| 110 | + // Enumerate x/benchmarks |
| 111 | + var buf bytes.Buffer |
| 112 | + remoteErr, err := bc.Exec("go/bin/go", buildlet.ExecOpts{ |
| 113 | + Output: &buf, |
| 114 | + ExtraEnv: []string{ |
| 115 | + "GOROOT=" + st.conf.FilePathJoin(workDir, "go"), |
| 116 | + "GOPATH=" + st.conf.FilePathJoin(workDir, "gopath"), |
| 117 | + }, |
| 118 | + Args: []string{"list", "-f", `{{if eq .Name "main"}}{{.ImportPath}}{{end}}`, "golang.org/x/benchmarks/..."}, |
| 119 | + }) |
| 120 | + if remoteErr != nil { |
| 121 | + return nil, remoteErr |
| 122 | + } |
| 123 | + if err != nil { |
| 124 | + return nil, err |
| 125 | + } |
| 126 | + for _, pkg := range strings.Fields(buf.String()) { |
| 127 | + pkg := pkg |
| 128 | + name := "bench-" + path.Base(pkg) + ".exe" |
| 129 | + out = append(out, &benchmarkItem{ |
| 130 | + binary: name, args: nil, build: func(bc *buildlet.Client, goroot string, w io.Writer) (error, error) { |
| 131 | + return st.buildXBenchmark(bc, goroot, w, rev, pkg, name) |
| 132 | + }}) |
| 133 | + } |
| 134 | + // TODO(quentin): Enumerate package benchmarks that were affected by the CL |
| 135 | + return out, nil |
| 136 | +} |
| 137 | + |
| 138 | +// runOneBenchBinary runs a binary on the buildlet and writes its output to w with a trailing newline. |
| 139 | +func (st *buildStatus) runOneBenchBinary(bc *buildlet.Client, w io.Writer, goroot string, path string, args []string) (remoteErr, err error) { |
| 140 | + defer w.Write([]byte{'\n'}) |
| 141 | + workDir, err := bc.WorkDir() |
| 142 | + if err != nil { |
| 143 | + return nil, fmt.Errorf("runOneBenchBinary, WorkDir: %v", err) |
| 144 | + } |
| 145 | + // Some benchmarks need GOROOT so they can invoke cmd/go. |
| 146 | + return bc.Exec(path, buildlet.ExecOpts{ |
| 147 | + Output: w, |
| 148 | + Args: args, |
| 149 | + Path: []string{"$WORKDIR/" + goroot + "/bin", "$PATH"}, |
| 150 | + ExtraEnv: []string{ |
| 151 | + "GOROOT=" + st.conf.FilePathJoin(workDir, goroot), |
| 152 | + }, |
| 153 | + }) |
| 154 | +} |
| 155 | + |
| 156 | +func (b *benchmarkItem) buildParent(st *buildStatus, bc *buildlet.Client, w io.Writer) error { |
| 157 | + pbr := st.builderRev // copy |
| 158 | + rev := st.trySet.ci.Revisions[st.trySet.ci.CurrentRevision] |
| 159 | + if rev.Commit == nil { |
| 160 | + return fmt.Errorf("commit information missing for revision %q", st.trySet.ci.CurrentRevision) |
| 161 | + } |
| 162 | + if len(rev.Commit.Parents) == 0 { |
| 163 | + // TODO(quentin): Log? |
| 164 | + return errors.New("commit has no parent") |
| 165 | + } |
| 166 | + pbr.rev = rev.Commit.Parents[0].CommitID |
| 167 | + if pbr.snapshotExists() { |
| 168 | + return bc.PutTarFromURL(pbr.snapshotURL(), "go-parent") |
| 169 | + } |
| 170 | + if err := bc.PutTar(versionTgz(pbr.rev), "go-parent"); err != nil { |
| 171 | + return err |
| 172 | + } |
| 173 | + srcTar, err := getSourceTgz(st, "go", pbr.rev) |
| 174 | + if err != nil { |
| 175 | + return err |
| 176 | + } |
| 177 | + if err := bc.PutTar(srcTar, "go-parent"); err != nil { |
| 178 | + return err |
| 179 | + } |
| 180 | + remoteErr, err := st.runMake(bc, "go-parent", w) |
| 181 | + if err != nil { |
| 182 | + return err |
| 183 | + } |
| 184 | + return remoteErr |
| 185 | +} |
| 186 | + |
| 187 | +// run runs all the iterations of this benchmark on bc. |
| 188 | +// Build output is sent to w. Benchmark output is stored in b.output. |
| 189 | +// TODO(quentin): Take a list of commits so this can be used for non-try runs. |
| 190 | +func (b *benchmarkItem) run(st *buildStatus, bc *buildlet.Client, w io.Writer) (remoteErr, err error) { |
| 191 | + // Ensure we have a built parent repo. |
| 192 | + if err := bc.ListDir("go-parent", buildlet.ListDirOpts{}, func(buildlet.DirEntry) {}); err != nil { |
| 193 | + sp := st.createSpan("bench_build_parent", bc.Name()) |
| 194 | + err := b.buildParent(st, bc, w) |
| 195 | + sp.done(err) |
| 196 | + if err != nil { |
| 197 | + return nil, err |
| 198 | + } |
| 199 | + } |
| 200 | + // Build benchmark. |
| 201 | + for _, goroot := range []string{"go", "go-parent"} { |
| 202 | + sp := st.createSpan("bench_build", fmt.Sprintf("%s/%s: %s", goroot, b.binary, bc.Name())) |
| 203 | + remoteErr, err = b.build(bc, goroot, w) |
| 204 | + sp.done(err) |
| 205 | + if remoteErr != nil || err != nil { |
| 206 | + return remoteErr, err |
| 207 | + } |
| 208 | + } |
| 209 | + |
| 210 | + type commit struct { |
| 211 | + path string |
| 212 | + out bytes.Buffer |
| 213 | + } |
| 214 | + commits := []*commit{ |
| 215 | + {path: "go-parent"}, |
| 216 | + {path: "go"}, |
| 217 | + } |
| 218 | + |
| 219 | + for _, c := range commits { |
| 220 | + c.out.WriteString(b.preamble) |
| 221 | + } |
| 222 | + |
| 223 | + // Run bench binaries and capture the results |
| 224 | + for i := 0; i < benchRuns; i++ { |
| 225 | + for _, c := range commits { |
| 226 | + fmt.Fprintf(&c.out, "iteration: %d\nstart-time: %s\n", i, time.Now().UTC().Format(time.RFC3339)) |
| 227 | + p := path.Join(c.path, b.binary) |
| 228 | + sp := st.createSpan("run_one_bench", p) |
| 229 | + remoteErr, err = st.runOneBenchBinary(bc, &c.out, c.path, p, b.args) |
| 230 | + sp.done(err) |
| 231 | + if err != nil || remoteErr != nil { |
| 232 | + c.out.WriteTo(w) |
| 233 | + return |
| 234 | + } |
| 235 | + } |
| 236 | + } |
| 237 | + b.output = []string{ |
| 238 | + commits[0].out.String(), |
| 239 | + commits[1].out.String(), |
| 240 | + } |
| 241 | + return nil, nil |
| 242 | +} |
0 commit comments