test harness should be able to expect errors

cmd/dep/init.go

@@ -72,7 +72,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
 		return err
 	}
 	if mok {
-		return fmt.Errorf("manifest file %q already exists", mf)
+		return errors.New("manifest file already exists")
 	}
 	// Manifest file does not exist.
 

cmd/dep/integration_test.go

@@ -51,10 +51,18 @@ func TestIntegration(t *testing.T) {
 
 				// Run commands
 				testProj.RecordImportPaths()
-				for _, args := range testCase.Commands {
-					testProj.DoRun(args)
+
+				var err error
+				for i, args := range testCase.Commands {
+					err = testProj.DoRun(args)
+					if err != nil && i < len(testCase.Commands)-1 {
+						t.Fatalf("cmd %s raised an unexpected error: %s", args[0], err.Error())
+					}
 				}
 
+				// Check errors from final command
+				testCase.CompareError(err, testProj.GetStderr())
+
 				// Check final manifest and lock
 				testCase.CompareFile("manifest.json", testProj.ProjPath("manifest.json"))
 				testCase.CompareFile("lock.json", testProj.ProjPath("lock.json"))

cmd/dep/testdata/harness_tests/README.md

@@ -58,7 +58,8 @@ The `testcase.json` file has the following format:
         "github.com/sdboyer/deptestdos",
         "github.com/sdboyer/deptesttres",
         "github.com/sdboyer/deptestquatro"
-      ]
+      ],
+      "error-expected": "something went wrong"
     }
 
 All of the categories are optional - if the `imports` list for a test is empty,
@@ -72,9 +73,10 @@ The test procedure is as follows:
 4. Fetch the repos and versions in `gopath-initial` into `$TMPDIR/src` directory
 5. Fetch the repos and versions in `vendor-initial` to the project's `vendor` directory
 6. Run `commands` on the project, in declaration order
-7. Check the resulting files against those in the `final` input directory
-8. Check the `vendor` directory for the projects listed under `vendor-final`
-9. Check that there were no changes to `src` listings
-10. Clean up
+7. Ensure that, if any errors are raised, it is only by the final command and their string output matches `error-expected`
+8. Check the resulting files against those in the `final` input directory
+9. Check the `vendor` directory for the projects listed under `vendor-final`
+10. Check that there were no changes to `src` listings
+11. Clean up
 
 Note that for the remote fetches, only git repos are currently supported.

cmd/dep/testdata/harness_tests/ensure/update/case1/testcase.json

@@ -1,6 +1,5 @@
 {
   "commands": [
-    ["init"],
     ["ensure", "-update", "github.com/sdboyer/deptest"]
   ],
   "vendor-final": [

cmd/dep/testdata/harness_tests/ensure/update/case2/testcase.json

@@ -1,6 +1,5 @@
 {
   "commands": [
-    ["init"],
     ["ensure", "-n", "-update", "github.com/sdboyer/deptest"]
   ]
 }

cmd/dep/testdata/harness_tests/init/manifest-exists/final/manifest.json

@@ -0,0 +1 @@
+{}

cmd/dep/testdata/harness_tests/init/manifest-exists/initial/manifest.json

@@ -0,0 +1 @@
+{}

cmd/dep/testdata/harness_tests/init/manifest-exists/testcase.json

@@ -0,0 +1,6 @@
+{
+  "commands": [
+    ["init"]
+  ],
+  "error-expected" : "manifest file already exists"
+}

test/integration_testcase.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strings"
 	"testing"
 )
 
@@ -27,6 +28,7 @@ type IntegrationTestCase struct {
 	initialPath   string
 	finalPath     string
 	Commands      [][]string        `json:"commands"`
+	ErrorExpected string            `json:"error-expected"`
 	GopathInitial map[string]string `json:"gopath-initial"`
 	VendorInitial map[string]string `json:"vendor-initial"`
 	VendorFinal   []string          `json:"vendor-final"`
@@ -56,6 +58,26 @@ func NewTestCase(t *testing.T, name string) *IntegrationTestCase {
 	return n
 }
 
+// CompareError compares expected error to error recived.
+func (tc *IntegrationTestCase) CompareError(err error, stderr string) {
+	if err == nil {
+		return
+	}
+
+	wantExists, want := tc.ErrorExpected != "", tc.ErrorExpected
+	gotExists, got := stderr != "", stderr
+
+	if wantExists && gotExists {
+		if !strings.Contains(got, want) {
+			tc.t.Errorf("expected error %s, got error %s", want, got)
+		}
+	} else if !wantExists && gotExists {
+		tc.t.Fatal("error raised where none was expected")
+	} else if wantExists && !gotExists {
+		tc.t.Error("error was not raised where one was expected")
+	}
+}
+
 var jsonNils *regexp.Regexp = regexp.MustCompile(`.*: null,.*\r?\n`)
 var jsonCmds *regexp.Regexp = regexp.MustCompile(`(?s)  "commands": \[(.*)  ],`)
 var jsonInds *regexp.Regexp = regexp.MustCompile(`(?s)\s*\n\s*`)
@@ -148,11 +170,12 @@ func (tc *IntegrationTestCase) WriteFile(src string, content string) error {
 	return err
 }
 
-func getFile(path string) (bool, string, error) {
-	_, err := os.Stat(path)
+func getFile(path string) (exists bool, contents string, err error) {
+	_, err = os.Stat(path)
 	if err != nil {
 		return false, "", nil
 	}
+
 	f, err := ioutil.ReadFile(path)
 	if err != nil {
 		return true, "", err

test/integration_testproj.go

@@ -74,6 +74,10 @@ func (p *IntegrationTestProject) RunGit(dir string, args ...string) {
 	p.h.RunGit(dir, args...)
 }
 
+func (p *IntegrationTestProject) GetStderr() string {
+	return p.h.getStderr()
+}
+
 func (p *IntegrationTestProject) GetVendorGit(ip string) {
 	parse := strings.Split(ip, "/")
 	gitDir := strings.Join(parse[:len(parse)-1], string(filepath.Separator))

test/test.go

@@ -506,12 +506,7 @@ func (h *Helper) Path(name string) string {
 		joined = filepath.Join(h.tempdir, name)
 	}
 
-	// Ensure it's the absolute, symlink-less path we're returning
-	abs, err := filepath.EvalSymlinks(joined)
-	if err != nil {
-		h.t.Fatalf("%+v", errors.Wrapf(err, "internal testsuite error: could not get absolute path for dir(%q)", joined))
-	}
-	return abs
+	return joined
 }
 
 // MustExist fails if path does not exist.