Merge pull request #402 from domgreen/test_harness_errors

Test harness can now assert if errors have been raised correctly

Author: sdboyer

cmd/dep/init.go

@@ -74,7 +74,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
 		return err
 	}
 	if mok {
-		return errors.Errorf("manifest file %q already exists", mf)
+		return errors.Errorf("manifest already exists: %s", mf)
 	}
 	// Manifest file does not exist.
 

cmd/dep/integration_test.go

@@ -19,6 +19,10 @@ func TestIntegration(t *testing.T) {
 	test.NeedsGit(t)
 
 	filepath.Walk(filepath.Join("testdata", "harness_tests"), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			t.Fatal("error walking filepath")
+		}
+
 		wd, err := os.Getwd()
 		if err != nil {
 			panic(err)
@@ -51,13 +55,18 @@ func TestIntegration(t *testing.T) {
 
 				// Run commands
 				testProj.RecordImportPaths()
-				for _, args := range testCase.Commands {
+
+				var err error
+				for i, args := range testCase.Commands {
 					err = testProj.DoRun(args)
-					if err != nil {
-						t.Fatalf("%v", err)
+					if err != nil && i < len(testCase.Commands)-1 {
+						t.Fatalf("cmd %s raised an unexpected error: %s", args[0], err.Error())
 					}
 				}
 
+				// Check error raised in final command
+				testCase.CompareError(err, testProj.GetStderr())
+
 				// Check final manifest and lock
 				testCase.CompareFile(dep.ManifestName, testProj.ProjPath(dep.ManifestName))
 				testCase.CompareFile(dep.LockName, testProj.ProjPath(dep.LockName))

cmd/dep/testdata/harness_tests/README.md

@@ -58,7 +58,8 @@ The `testcase.json` file has the following format:
         "github.com/sdboyer/deptestdos",
         "github.com/sdboyer/deptesttres",
         "github.com/sdboyer/deptestquatro"
-      ]
+      ],
+      "error-expected": "something went wrong"
     }
 
 All of the categories are optional - if the `imports` list for a test is empty,
@@ -72,9 +73,10 @@ The test procedure is as follows:
 4. Fetch the repos and versions in `gopath-initial` into `$TMPDIR/src` directory
 5. Fetch the repos and versions in `vendor-initial` to the project's `vendor` directory
 6. Run `commands` on the project, in declaration order
-7. Check the resulting files against those in the `final` input directory
-8. Check the `vendor` directory for the projects listed under `vendor-final`
-9. Check that there were no changes to `src` listings
-10. Clean up
+7. Ensure that, if any errors are raised, it is only by the final command and their string output matches `error-expected`
+8. Check the resulting files against those in the `final` input directory
+9. Check the `vendor` directory for the projects listed under `vendor-final`
+10. Check that there were no changes to `src` listings
+11. Clean up
 
 Note that for the remote fetches, only git repos are currently supported.

cmd/dep/testdata/harness_tests/init/manifest-exists/final/Gopkg.toml

@@ -0,0 +1,6 @@
+{
+   "commands": [
+     ["init"]
+   ],
+   "error-expected" : "manifest already exists:"
+ }

cmd/dep/testdata/harness_tests/init/manifest-exists/initial/Gopkg.toml

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strings"
 	"testing"
 )
 
@@ -27,6 +28,7 @@ type IntegrationTestCase struct {
 	initialPath   string
 	finalPath     string
 	Commands      [][]string        `json:"commands"`
+	ErrorExpected string            `json:"error-expected"`
 	GopathInitial map[string]string `json:"gopath-initial"`
 	VendorInitial map[string]string `json:"vendor-initial"`
 	VendorFinal   []string          `json:"vendor-final"`
@@ -126,6 +128,22 @@ func (tc *IntegrationTestCase) CompareFile(goldenPath, working string) {
 	}
 }
 
+// CompareError compares exected and actual error
+func (tc *IntegrationTestCase) CompareError(err error, stderr string) {
+	wantExists, want := tc.ErrorExpected != "", tc.ErrorExpected
+	gotExists, got := stderr != "" && err != nil, stderr
+
+	if wantExists && gotExists {
+		if !strings.Contains(got, want) {
+			tc.t.Errorf("expected error containing %s, got error %s", want, got)
+		}
+	} else if !wantExists && gotExists {
+		tc.t.Fatal("error raised where none was expected")
+	} else if wantExists && !gotExists {
+		tc.t.Error("error not raised where one was expected")
+	}
+}
+
 func (tc *IntegrationTestCase) CompareVendorPaths(gotVendorPaths []string) {
 	if *UpdateGolden {
 		tc.VendorFinal = gotVendorPaths

cmd/dep/testdata/harness_tests/init/manifest-exists/testcase.json

@@ -139,6 +139,11 @@ func (p *IntegrationTestProject) RunGit(dir string, args ...string) {
 	}
 }
 
+// GetStderr gets the Stderr output from test run
+func (p *IntegrationTestProject) GetStderr() string {
+	return p.stderr.String()
+}
+
 func (p *IntegrationTestProject) GetVendorGit(ip string) {
 	parse := strings.Split(ip, "/")
 	gitDir := strings.Join(parse[:len(parse)-1], string(filepath.Separator))