gps: export content of git submodules

The "git checkout-index" command is unaware of git submodules,
therefore their content was missed when exporting files.

With "git submodule foreach" the same operation (read-tree +
checkout-index) can also be done for all submodules, recursively. The
downside is that "foreach" invokes a shell command with certain
variables set. To achieve the desired functionality, we have to rely
on POSIX shell support.

OTOH, the command seems to get embedded by git in a larger shell
script because error messages when there are syntax errors show that the
command gets invoked via `eval`, so git already depends on a POSIX
shell.

Author: pohly

CHANGELOG.md

@@ -27,6 +27,7 @@ IMPROVEMENTS:
 * Hash digests of vendor contents are now stored in `Gopkg.lock`, and the contents of vendor are only rewritten on change or hash mismatch ([#1912](https://github.com/golang/dep/pull/1912)).
 * Don't exclude `Godeps` folder ([#1822](https://github.com/golang/dep/issues/1822)).
 * Add project-package relationship graph support in graphviz ([#1588](https://github.com/golang/dep/pull/1588)).
+* Export the content of git submodules to the vendor directory ([#1909](https://github.com/golang/dep/pull/1909)).
 
 WIP:
 * Enable importing external configuration from dependencies during init (#1277). This is feature flagged and disabled by default.

cmd/dep/integration_test.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/golang/dep"
 	"github.com/golang/dep/internal/fs"
 	"github.com/golang/dep/internal/test"
 	"github.com/golang/dep/internal/test/integration"
@@ -210,45 +211,60 @@ func testIntegration(name, relPath, wd string, run integration.RunFunc) func(t *
 			testCase.CompareOutput(testProj.GetStdout())
 		}
 
+		// Determine how the test case specifies the expected
+		// content: either it lists just some projects, or it
+		// provides a complete reference directory.
+		reference := len(testCase.VendorFinal) == 1 && testCase.VendorFinal[0] == "compare"
+
 		// Check vendor paths
 		testProj.CompareImportPaths()
-		testCase.CompareVendorPaths(testProj.GetVendorPaths())
+		if !reference {
+			testCase.CompareVendorPaths(testProj.GetVendorPaths())
+		}
 
-		if *test.UpdateGolden {
-			// Update all files in the 'final' directory, removing those which
-			// no longer should exist.
-			if err := os.RemoveAll(testCase.FinalPath()); err != nil {
-				t.Fatalf("error removing 'final' directory: %s", err)
-			}
-			if err := fs.CopyTree(testCase.FinalPath(), testProj.ProjPath()); err != nil {
-				t.Fatalf("error copying into 'final' directory: %s", err)
-			}
-		} else {
-			// Compare all files from either of the two trees.
-			files := make(map[string]bool)
-			findFiles := func(dir string) error {
-				return filepath.Walk(dir,
-					func(path string, info os.FileInfo, err error) error {
-						if err != nil {
-							return err
-						}
-						if dir == path {
+		if reference {
+			// Check all files.
+			if *test.UpdateGolden {
+				// Update all files in the 'final' directory, removing those which
+				// no longer should exist.
+				if err := os.RemoveAll(testCase.FinalPath()); err != nil {
+					t.Fatalf("error removing 'final' directory: %s", err)
+				}
+				if err := fs.CopyTree(testCase.FinalPath(), testProj.ProjPath()); err != nil {
+					t.Fatalf("error copying into 'final' directory: %s", err)
+				}
+			} else {
+				// Compare all files from either of the two trees.
+				files := make(map[string]bool)
+				findFiles := func(dir string) error {
+					return filepath.Walk(dir,
+						func(path string, info os.FileInfo, err error) error {
+							if err != nil {
+								return err
+							}
+							if dir == path {
+								return nil
+							}
+							if !info.IsDir() {
+								localpath := path[len(dir)+1:]
+								files[localpath] = true
+							}
 							return nil
-						}
-						if !info.IsDir() {
-							localpath := path[len(dir)+1:]
-							files[localpath] = true
-						}
-						return nil
-					})
-			}
-			findFiles(testCase.FinalPath())
-			findFiles(testProj.ProjPath())
-
-			for localpath, _ := range files {
-				t.Logf("comparing %s", localpath)
-				testCase.CompareFile(localpath, testProj.ProjPath(localpath))
+						})
+				}
+				findFiles(testCase.FinalPath())
+				findFiles(testProj.ProjPath())
+
+				t.Logf("checking against %s", testCase.FinalPath())
+				for localpath, _ := range files {
+					t.Logf("comparing %s", localpath)
+					testCase.CompareFile(localpath, testProj.ProjPath(localpath))
+				}
 			}
+		} else {
+			// Check final manifest and lock
+			testCase.CompareFile(dep.ManifestName, testProj.ProjPath(dep.ManifestName))
+			testCase.CompareFile(dep.LockName, testProj.ProjPath(dep.LockName))
 		}
 	}
 }

cmd/dep/testdata/harness_tests/ensure/submodules/case1/final/Gopkg.lock

@@ -0,0 +1,34 @@
+# Gopkg.toml example
+#
+# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
+
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/pohly/deptestmodules"
+
+[prune]
+  go-tests = true
+  unused-packages = true

cmd/dep/testdata/harness_tests/ensure/submodules/case1/final/Gopkg.toml

@@ -0,0 +1,8 @@
+package main
+
+import (
+	_ "github.com/pohly/deptestmodules/pkg/foo"
+)
+
+func main() {
+}

cmd/dep/testdata/harness_tests/ensure/submodules/case1/final/main.go

@@ -0,0 +1,8 @@
+package main
+
+import (
+	_ "github.com/pohly/deptestmodules/pkg/foo"
+)
+
+func main() {
+}

cmd/dep/testdata/harness_tests/ensure/submodules/case1/final/vendor/github.com/pohly/deptestmodules/pkg/foo/foo.go

@@ -0,0 +1,9 @@
+{
+    "commands": [
+        ["init"],
+        ["ensure"]
+    ],
+    "vendor-final": [
+        "compare"
+    ]
+}

cmd/dep/testdata/harness_tests/ensure/submodules/case1/initial/main.go

@@ -203,6 +203,34 @@ func (s *gitSource) exportRevisionTo(ctx context.Context, rev Revision, to strin
 		}
 	}
 
+	// Now do the same for any submodule, using a shell command that "git submodule"
+	// iterates over.
+	{
+		cmd := commandContext(ctx, "git", "submodule", "foreach", "--recursive",
+			// The command gets called in the working tree
+			// of the git submodule.  The .git is a text
+			// file which contains a relative link to the
+			// git directory. We need to follow that and
+			// there do the same "git read-tree + git
+			// checkout-index" with a temporary index as
+			// for the main repo.
+			//
+			// $toplevel is our initial root directory
+			// plus any intermediate git repo (i.e. if
+			// root contains "foo" and "foo" contains
+			// "bar", then we first get called with
+			// "<root>" when exporting "foo" and then with
+			// "<root>/foo" for bar. By stripping our root
+			// directory we get a relative path that we can
+			// append to the target directory.
+			"(cd $(grep gitdir .git | sed -e 's;gitdir: *;;' .git) && mv index origindex && trap 'mv origindex index' EXIT && git read-tree \"$sha1\" && git checkout-index -a --prefix=\""+to+"$(echo \"$toplevel\" | sed -e 's;"+r.LocalPath()+";;')/$path/\")",
+		)
+		cmd.SetDir(r.LocalPath())
+		if out, err := cmd.CombinedOutput(); err != nil {
+			return errors.Wrap(err, string(out))
+		}
+	}
+
 	return nil
 }
 

cmd/dep/testdata/harness_tests/ensure/submodules/case1/testcase.json

@@ -698,6 +698,9 @@ func volumeName(path string) (v string) {
 func CopyTree(dest, src string) error {
 	return filepath.Walk(src,
 		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return nil
+			}
 			var localpath string
 			if path != src {
 				localpath = path[len(src)+1:]