diff --git a/cmd/dep/integration_test.go b/cmd/dep/integration_test.go index b886596eb4..3a00cb4349 100644 --- a/cmd/dep/integration_test.go +++ b/cmd/dep/integration_test.go @@ -21,25 +21,37 @@ func TestIntegration(t *testing.T) { test.NeedsExternalNetwork(t) test.NeedsGit(t) - filepath.Walk(filepath.Join("testdata", "harness_tests"), func(path string, info os.FileInfo, err error) error { - if err != nil { - t.Fatal("error walking filepath") - } + err := forEachTestCase(func(wd, testName string) { + t.Run(testName, func(t *testing.T) { + t.Parallel() + + t.Run("external", testIntegration(testName, wd, true, execCmd)) + t.Run("internal", testIntegration(testName, wd, false, runMain)) + }) + }) + if err != nil { + t.Fatal("error walking filepath:", err) + } +} + +// forEachTestCase executes testCaseFunc with the current working directory and path to +// a `testcase.json` file, for each `testdata/harness_tests` test case. +func forEachTestCase(testCaseFunc func(wd, path string)) error { + wd, err := os.Getwd() + if err != nil { + panic(err) + } - wd, err := os.Getwd() + return filepath.Walk(filepath.Join("testdata", "harness_tests"), func(path string, info os.FileInfo, err error) error { if err != nil { - panic(err) + return err } if filepath.Base(path) == "testcase.json" { parse := strings.Split(path, string(filepath.Separator)) testName := strings.Join(parse[2:len(parse)-1], "/") - t.Run(testName, func(t *testing.T) { - t.Parallel() - t.Run("external", testIntegration(testName, wd, true, execCmd)) - t.Run("internal", testIntegration(testName, wd, false, runMain)) - }) + testCaseFunc(wd, testName) } return nil }) @@ -87,7 +99,7 @@ func testIntegration(name, wd string, externalProc bool, run test.RunFunc) func( // Set up environment testCase := test.NewTestCase(t, name, wd) defer testCase.Cleanup() - testProj := test.NewTestProject(t, testCase.InitialPath(), wd, externalProc, run) + testProj := test.NewTestProject(t, testCase.InitialPath(), wd, externalProc) defer testProj.Cleanup() // Create and checkout the vendor revisions @@ -107,7 +119,7 @@ func testIntegration(name, wd string, externalProc bool, run test.RunFunc) func( var err error for i, args := range testCase.Commands { - err = testProj.DoRun(args) + err = testProj.DoRun(args, run) if err != nil && i < len(testCase.Commands)-1 { t.Fatalf("cmd %s raised an unexpected error: %s", args[0], err.Error()) } @@ -128,3 +140,70 @@ func testIntegration(name, wd string, externalProc bool, run test.RunFunc) func( testCase.CompareVendorPaths(testProj.GetVendorPaths()) } } + +func BenchmarkIntegration(b *testing.B) { + err := forEachTestCase(func(wd, testName string) { + b.Run(testName, benchmarkIntegration(wd, testName)) + }) + if err != nil { + b.Fatal("error walking filepath:", err) + } +} + +func benchmarkIntegration(wd, testName string) func(b *testing.B) { + return func(b *testing.B) { + // Set up environment + testCase := test.NewTestCase(b, testName, wd) + defer testCase.Cleanup() + + testProj := test.NewTestProject(b, testCase.InitialPath(), wd, false) + defer testProj.Cleanup() + + // Create and checkout the vendor revisions + for ip, rev := range testCase.VendorInitial { + testProj.GetVendorGit(ip) + testProj.RunGit(testProj.VendorPath(ip), "checkout", rev) + } + + // Create and checkout the import revisions + for ip, rev := range testCase.GopathInitial { + testProj.RunGo("get", ip) + testProj.RunGit(testProj.Path("src", ip), "checkout", rev) + } + + // Run commands + testProj.RecordImportPaths() + + // Only measure the timing of calls to runMain. + b.StopTimer() + b.ResetTimer() + var err error + for i, args := range testCase.Commands { + // Execute DoRun, but measure only runMain. + err = testProj.DoRun(args, func(prog string, newargs []string, outW, errW io.Writer, dir string, env []string) error { + b.StartTimer() + err := runMain(prog, newargs, outW, errW, dir, env) + b.StopTimer() + return err + + }) + if err != nil && i < len(testCase.Commands)-1 { + b.Fatalf("cmd %s raised an unexpected error: %s", args[0], err.Error()) + } + } + + // Check error raised in final command + testCase.CompareError(err, testProj.GetStderr()) + + // Check output + testCase.CompareOutput(testProj.GetStdout()) + + // Check final manifest and lock + testCase.CompareFile(dep.ManifestName, testProj.ProjPath(dep.ManifestName)) + testCase.CompareFile(dep.LockName, testProj.ProjPath(dep.LockName)) + + // Check vendor paths + testProj.CompareImportPaths() + testCase.CompareVendorPaths(testProj.GetVendorPaths()) + } +} diff --git a/cmd/dep/remove_test.go b/cmd/dep/remove_test.go index 881ccd946c..73822406a6 100644 --- a/cmd/dep/remove_test.go +++ b/cmd/dep/remove_test.go @@ -28,7 +28,7 @@ func TestRemoveErrors(t *testing.T) { func removeErrors(name, wd string, externalProc bool, run test.RunFunc) func(*testing.T) { return func(t *testing.T) { testCase := test.NewTestCase(t, name, wd) - testProj := test.NewTestProject(t, testCase.InitialPath(), wd, externalProc, run) + testProj := test.NewTestProject(t, testCase.InitialPath(), wd, externalProc) defer testProj.Cleanup() // Create and checkout the vendor revisions @@ -43,19 +43,19 @@ func removeErrors(name, wd string, externalProc bool, run test.RunFunc) func(*te testProj.RunGit(testProj.Path("src", ip), "checkout", rev) } - if err := testProj.DoRun([]string{"remove", "-unused", "github.com/not/used"}); err == nil { + if err := testProj.DoRun([]string{"remove", "-unused", "github.com/not/used"}, run); err == nil { t.Fatal("rm with both -unused and arg should have failed") } - if err := testProj.DoRun([]string{"remove", "github.com/not/present"}); err == nil { + if err := testProj.DoRun([]string{"remove", "github.com/not/present"}, run); err == nil { t.Fatal("rm with arg not in manifest should have failed") } - if err := testProj.DoRun([]string{"remove", "github.com/not/used", "github.com/not/present"}); err == nil { + if err := testProj.DoRun([]string{"remove", "github.com/not/used", "github.com/not/present"}, run); err == nil { t.Fatal("rm with one arg not in manifest should have failed") } - if err := testProj.DoRun([]string{"remove", "github.com/sdboyer/deptest"}); err == nil { + if err := testProj.DoRun([]string{"remove", "github.com/sdboyer/deptest"}, run); err == nil { t.Fatal("rm of arg in manifest and imports should have failed without -force") } } diff --git a/internal/test/integration_testcase.go b/internal/test/integration_testcase.go index 241612b034..c56f58f6cf 100644 --- a/internal/test/integration_testcase.go +++ b/internal/test/integration_testcase.go @@ -23,7 +23,7 @@ var ( // IntegrationTestCase manages a test case directory structure and content type IntegrationTestCase struct { - t *testing.T + tb testing.TB name string rootPath string initialPath string @@ -35,10 +35,10 @@ type IntegrationTestCase struct { VendorFinal []string `json:"vendor-final"` } -func NewTestCase(t *testing.T, name, wd string) *IntegrationTestCase { +func NewTestCase(tb testing.TB, name, wd string) *IntegrationTestCase { rootPath := filepath.FromSlash(filepath.Join(wd, "testdata", "harness_tests", name)) n := &IntegrationTestCase{ - t: t, + tb: tb, name: name, rootPath: rootPath, initialPath: filepath.Join(rootPath, "initial"), @@ -78,7 +78,7 @@ func (tc *IntegrationTestCase) Cleanup() { j = append(j, '\n') err = ioutil.WriteFile(filepath.Join(tc.rootPath, "testcase.json"), j, 0666) if err != nil { - tc.t.Errorf("Failed to update testcase %s: %s", tc.name, err) + tc.tb.Errorf("Failed to update testcase %s: %s", tc.name, err) } } } @@ -92,39 +92,39 @@ func (tc *IntegrationTestCase) CompareFile(goldenPath, working string) { gotExists, got, err := getFile(working) if err != nil { - tc.t.Fatalf("Error reading project file %s: %s", goldenPath, err) + tc.tb.Fatalf("Error reading project file %s: %s", goldenPath, err) } wantExists, want, err := getFile(golden) if err != nil { - tc.t.Fatalf("Error reading testcase file %s: %s", goldenPath, err) + tc.tb.Fatalf("Error reading testcase file %s: %s", goldenPath, err) } if wantExists && gotExists { if want != got { if *UpdateGolden { if err := tc.WriteFile(golden, got); err != nil { - tc.t.Fatal(err) + tc.tb.Fatal(err) } } else { - tc.t.Errorf("expected %s, got %s", want, got) + tc.tb.Errorf("expected %s, got %s", want, got) } } } else if !wantExists && gotExists { if *UpdateGolden { if err := tc.WriteFile(golden, got); err != nil { - tc.t.Fatal(err) + tc.tb.Fatal(err) } } else { - tc.t.Errorf("%s created where none was expected", goldenPath) + tc.tb.Errorf("%s created where none was expected", goldenPath) } } else if wantExists && !gotExists { if *UpdateGolden { err := os.Remove(golden) if err != nil { - tc.t.Fatal(err) + tc.tb.Fatal(err) } } else { - tc.t.Errorf("%s not created where one was expected", goldenPath) + tc.tb.Errorf("%s not created where one was expected", goldenPath) } } } @@ -144,7 +144,7 @@ func (tc *IntegrationTestCase) CompareOutput(stdout string) { stdout = normalizeLines(stdout) if expStr != stdout { - tc.t.Errorf("expected: %q but got: %q", expStr, stdout) + tc.tb.Errorf("expected: %q but got: %q", expStr, stdout) } } @@ -164,12 +164,12 @@ func (tc *IntegrationTestCase) CompareError(err error, stderr string) { if wantExists && gotExists { if !strings.Contains(got, want) { - tc.t.Errorf("expected error containing %s, got error %s", want, got) + tc.tb.Errorf("expected error containing %s, got error %s", want, got) } } else if !wantExists && gotExists { - tc.t.Fatalf("error raised where none was expected: \n%v", stderr) + tc.tb.Fatalf("error raised where none was expected: \n%v", stderr) } else if wantExists && !gotExists { - tc.t.Error("error not raised where one was expected:", want) + tc.tb.Error("error not raised where one was expected:", want) } } @@ -179,11 +179,11 @@ func (tc *IntegrationTestCase) CompareVendorPaths(gotVendorPaths []string) { } else { wantVendorPaths := tc.VendorFinal if len(gotVendorPaths) != len(wantVendorPaths) { - tc.t.Fatalf("Wrong number of vendor paths created: want %d got %d", len(wantVendorPaths), len(gotVendorPaths)) + tc.tb.Fatalf("Wrong number of vendor paths created: want %d got %d", len(wantVendorPaths), len(gotVendorPaths)) } for ind := range gotVendorPaths { if gotVendorPaths[ind] != wantVendorPaths[ind] { - tc.t.Errorf("Mismatch in vendor paths created: want %s got %s", gotVendorPaths, wantVendorPaths) + tc.tb.Errorf("Mismatch in vendor paths created: want %s got %s", gotVendorPaths, wantVendorPaths) } } } diff --git a/internal/test/integration_testproj.go b/internal/test/integration_testproj.go index 85fbca8b5d..660a594814 100644 --- a/internal/test/integration_testproj.go +++ b/internal/test/integration_testproj.go @@ -28,7 +28,7 @@ type RunFunc func(prog string, newargs []string, outW, errW io.Writer, dir strin // IntegrationTestProject manages the "virtual" test project directory structure // and content type IntegrationTestProject struct { - t *testing.T + tb testing.TB h *Helper preImports []string tempdir string @@ -39,12 +39,11 @@ type IntegrationTestProject struct { run RunFunc } -func NewTestProject(t *testing.T, initPath, wd string, externalProc bool, run RunFunc) *IntegrationTestProject { +func NewTestProject(tb testing.TB, initPath, wd string, externalProc bool) *IntegrationTestProject { new := &IntegrationTestProject{ - t: t, + tb: tb, origWd: wd, env: os.Environ(), - run: run, } new.makeRootTempDir() new.TempDir(ProjectRoot, "vendor") @@ -80,7 +79,7 @@ func (p *IntegrationTestProject) ProjPath(args ...string) string { func (p *IntegrationTestProject) TempDir(args ...string) { fullPath := p.Path(args...) if err := os.MkdirAll(fullPath, 0755); err != nil && !os.IsExist(err) { - p.t.Fatalf("%+v", errors.Errorf("Unable to create temp directory: %s", fullPath)) + p.tb.Fatalf("%+v", errors.Errorf("Unable to create temp directory: %s", fullPath)) } } @@ -105,16 +104,15 @@ func (p *IntegrationTestProject) RunGo(args ...string) { cmd.Env = p.env status := cmd.Run() if p.stdout.Len() > 0 { - p.t.Log("go standard output:") - p.t.Log(p.stdout.String()) + p.tb.Log("go standard output:") + p.tb.Log(p.stdout.String()) } if p.stderr.Len() > 0 { - p.t.Log("go standard error:") - p.t.Log(p.stderr.String()) + p.tb.Log("go standard error:") + p.tb.Log(p.stderr.String()) } if status != nil { - p.t.Logf("go %v failed unexpectedly: %v", args, status) - p.t.FailNow() + p.tb.Fatalf("go %v failed unexpectedly: %v", args, status) } } @@ -129,17 +127,16 @@ func (p *IntegrationTestProject) RunGit(dir string, args ...string) { status := cmd.Run() if *PrintLogs { if p.stdout.Len() > 0 { - p.t.Logf("git %v standard output:", args) - p.t.Log(p.stdout.String()) + p.tb.Logf("git %v standard output:", args) + p.tb.Log(p.stdout.String()) } if p.stderr.Len() > 0 { - p.t.Logf("git %v standard error:", args) - p.t.Log(p.stderr.String()) + p.tb.Logf("git %v standard error:", args) + p.tb.Log(p.stderr.String()) } } if status != nil { - p.t.Logf("git %v failed unexpectedly: %v", args, status) - p.t.FailNow() + p.tb.Fatalf("git %v failed unexpectedly: %v", args, status) } } @@ -160,9 +157,9 @@ func (p *IntegrationTestProject) GetVendorGit(ip string) { p.RunGit(p.ProjPath("vendor", gitDir), "clone", "http://"+ip) } -func (p *IntegrationTestProject) DoRun(args []string) error { +func (p *IntegrationTestProject) DoRun(args []string, run RunFunc) error { if *PrintLogs { - p.t.Logf("running testdep %v", args) + p.tb.Logf("running testdep %v", args) } prog := filepath.Join(p.origWd, "testdep"+ExeSuffix) newargs := append([]string{args[0], "-v"}, args[1:]...) @@ -170,16 +167,16 @@ func (p *IntegrationTestProject) DoRun(args []string) error { p.stdout.Reset() p.stderr.Reset() - status := p.run(prog, newargs, &p.stdout, &p.stderr, p.ProjPath(""), p.env) + status := run(prog, newargs, &p.stdout, &p.stderr, p.ProjPath(""), p.env) if *PrintLogs { if p.stdout.Len() > 0 { - p.t.Log("standard output:") - p.t.Log(p.stdout.String()) + p.tb.Log("standard output:") + p.tb.Log(p.stdout.String()) } if p.stderr.Len() > 0 { - p.t.Log("standard error:") - p.t.Log(p.stderr.String()) + p.tb.Log("standard error:") + p.tb.Log(p.stderr.String()) } } return status @@ -269,11 +266,11 @@ func (p *IntegrationTestProject) CompareImportPaths() { wantImportPaths := p.preImports gotImportPaths := p.GetImportPaths() if len(gotImportPaths) != len(wantImportPaths) { - p.t.Fatalf("Import path count changed during command: pre %d post %d", len(wantImportPaths), len(gotImportPaths)) + p.tb.Fatalf("Import path count changed during command: pre %d post %d", len(wantImportPaths), len(gotImportPaths)) } for ind := range gotImportPaths { if gotImportPaths[ind] != wantImportPaths[ind] { - p.t.Errorf("Change in import paths during: pre %s post %s", gotImportPaths, wantImportPaths) + p.tb.Errorf("Change in import paths during: pre %s post %s", gotImportPaths, wantImportPaths) } } } @@ -297,7 +294,7 @@ func (p *IntegrationTestProject) Setenv(name, val string) { // Must gives a fatal error if err is not nil. func (p *IntegrationTestProject) Must(err error) { if err != nil { - p.t.Fatalf("%+v", err) + p.tb.Fatalf("%+v", err) } }