internal/task: create a basic Git wrapper, use it for the fake Gerrit

For golang/go#59717

Change-Id: Ie0e01a82a5acebf279c1f770dee53f37fc4e7800
Reviewed-on: https://go-review.googlesource.com/c/build/+/486515
Run-TryBot: Heschi Kreinick <[email protected]>
Auto-Submit: Heschi Kreinick <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>

Author: heschi

internal/relui/buildrelease_test.go

@@ -103,7 +103,7 @@ type releaseTestDeps struct {
 	publishedFiles map[string]task.WebsiteFile
 }
 
-func newReleaseTestDeps(t *testing.T, previousTag, wantVersion string) *releaseTestDeps {
+func newReleaseTestDeps(t *testing.T, previousTag string, major int, wantVersion string) *releaseTestDeps {
 	if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
 		t.Skip("Requires bash shell scripting support.")
 	}
@@ -177,6 +177,7 @@ esac
 	goRepo := task.NewFakeRepo(t, "go")
 	base := goRepo.Commit(goFiles)
 	goRepo.Tag(previousTag, base)
+	goRepo.Branch(fmt.Sprintf("release-branch.go1.%d", major), base)
 	dlRepo := task.NewFakeRepo(t, "dl")
 	toolsRepo := task.NewFakeRepo(t, "tools")
 	toolsRepo1 := toolsRepo.Commit(map[string]string{
@@ -256,7 +257,7 @@ esac
 }
 
 func testRelease(t *testing.T, prevTag string, major int, wantVersion string, kind task.ReleaseKind) {
-	deps := newReleaseTestDeps(t, prevTag, wantVersion)
+	deps := newReleaseTestDeps(t, prevTag, major, wantVersion)
 	wd := workflow.New()
 
 	deps.gerrit.wantReviewers = []string{"heschi", "dmitshur"}
@@ -417,7 +418,7 @@ func testRelease(t *testing.T, prevTag string, major int, wantVersion string, ki
 }
 
 func testSecurity(t *testing.T, mergeFixes bool) {
-	deps := newReleaseTestDeps(t, "go1.17", "go1.18rc1")
+	deps := newReleaseTestDeps(t, "go1.17", 18, "go1.18rc1")
 
 	// Set up the fake merge process. Once we stop to ask for approval, commit
 	// the fix to the public server.
@@ -431,7 +432,7 @@ func testSecurity(t *testing.T, mergeFixes bool) {
 	defaultApprove := deps.buildTasks.ApproveAction
 	deps.buildTasks.ApproveAction = func(tc *workflow.TaskContext) error {
 		if mergeFixes {
-			deps.goRepo.Commit(securityFix)
+			deps.goRepo.CommitOnBranch("release-branch.go1.18", securityFix)
 		}
 		return defaultApprove(tc)
 	}
@@ -468,7 +469,7 @@ func testSecurity(t *testing.T, mergeFixes bool) {
 }
 
 func TestAdvisoryTrybotFail(t *testing.T) {
-	deps := newReleaseTestDeps(t, "go1.17", "go1.18rc1")
+	deps := newReleaseTestDeps(t, "go1.17", 18, "go1.18rc1")
 	defaultApprove := deps.buildTasks.ApproveAction
 	var trybotApprovals atomic.Int32
 	deps.buildTasks.ApproveAction = func(ctx *workflow.TaskContext) error {

internal/task/fakes.go

@@ -10,6 +10,7 @@ import (
 	"compress/gzip"
 	"context"
 	"crypto/sha256"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -348,67 +349,70 @@ type FakeGerrit struct {
 }
 
 type FakeRepo struct {
-	t       *testing.T
-	name    string
-	seq     int
-	history []string // oldest to newest
-	content map[string]map[string]string
-	tags    map[string]string
+	t    *testing.T
+	name string
+	dir  *GitDir
 }
 
 func NewFakeRepo(t *testing.T, name string) *FakeRepo {
-	return &FakeRepo{
-		t:       t,
-		name:    name,
-		content: map[string]map[string]string{},
-		tags:    map[string]string{},
+	if _, err := exec.LookPath("git"); errors.Is(err, exec.ErrNotFound) {
+		t.Skip("test requires git")
 	}
+
+	r := &FakeRepo{
+		t:    t,
+		name: name,
+		dir:  &GitDir{&Git{}, t.TempDir()},
+	}
+	t.Cleanup(func() { r.dir.Close() })
+	r.runGit("init", "-b", "master")
+	r.runGit("commit", "--allow-empty", "--allow-empty-message", "-m", "")
+	return r
+}
+
+func (repo *FakeRepo) runGit(args ...string) []byte {
+	repo.t.Helper()
+	out, err := repo.dir.RunCommand(context.Background(), args...)
+	if err != nil {
+		repo.t.Fatal(err)
+	}
+	return out
 }
 
 func (repo *FakeRepo) Commit(contents map[string]string) string {
-	rev := fmt.Sprintf("%v~%v", repo.name, repo.seq)
-	repo.seq++
+	return repo.CommitOnBranch("master", contents)
+}
 
-	newContent := map[string]string{}
-	if len(repo.history) != 0 {
-		for k, v := range repo.content[repo.history[len(repo.history)-1]] {
-			newContent[k] = v
-		}
-	}
+func (repo *FakeRepo) CommitOnBranch(branch string, contents map[string]string) string {
+	repo.runGit("switch", branch)
 	for k, v := range contents {
-		newContent[k] = v
+		full := filepath.Join(repo.dir.dir, k)
+		if err := os.MkdirAll(filepath.Dir(full), 0777); err != nil {
+			repo.t.Fatal(err)
+		}
+		if err := os.WriteFile(full, []byte(v), 0777); err != nil {
+			repo.t.Fatal(err)
+		}
 	}
-	repo.content[rev] = newContent
-	repo.history = append(repo.history, rev)
-	return rev
+	repo.runGit("add", ".")
+	repo.runGit("commit", "--allow-empty-message", "-m", "")
+	return strings.TrimSpace(string(repo.runGit("rev-parse", "HEAD")))
+}
+
+func (repo *FakeRepo) History() []string {
+	return strings.Split(string(repo.runGit("log", "--format=%H")), "\n")
 }
 
 func (repo *FakeRepo) Tag(tag, commit string) {
-	if _, ok := repo.content[commit]; !ok {
-		repo.t.Fatalf("commit %q does not exist on repo %q", commit, repo.name)
-	}
-	if _, ok := repo.tags[tag]; ok {
-		repo.t.Fatalf("tag %q already exists on repo %q", commit, repo.name)
-	}
-	repo.tags[tag] = commit
+	repo.runGit("tag", tag, commit)
 }
 
-// GetRepoContent returns the content of repo based on the value of commit:
-// - commit is "master": return content of the most recent revision
-// - commit is tag: return content of the repo associating with the commit that the tag maps to
-// - commit is neither "master" or tag: return content of the repo associated with that commit
-func (repo *FakeRepo) GetRepoContent(commit string) (map[string]string, error) {
-	rev := commit
-	if commit == "master" {
-		l := len(repo.history)
-		if l == 0 {
-			return nil, fmt.Errorf("repo %v history is empty", repo.name)
-		}
-		rev = repo.history[l-1]
-	} else if val, ok := repo.tags[commit]; ok {
-		rev = val
-	}
-	return repo.content[rev], nil
+func (repo *FakeRepo) Branch(branch, commit string) {
+	repo.runGit("branch", branch, commit)
+}
+
+func (repo *FakeRepo) ReadFile(commit, file string) ([]byte, error) {
+	return repo.dir.RunCommand(context.Background(), "show", commit+":"+file)
 }
 
 var _ GerritClient = (*FakeGerrit)(nil)
@@ -434,55 +438,42 @@ func (g *FakeGerrit) ReadBranchHead(ctx context.Context, project, branch string)
 	if err != nil {
 		return "", err
 	}
-	return repo.history[len(repo.history)-1], nil
+	out, err := repo.dir.RunCommand(ctx, "rev-parse", "refs/heads/"+branch)
+	return strings.TrimSpace(string(out)), err
 }
 
 func (g *FakeGerrit) ReadFile(ctx context.Context, project string, commit string, file string) ([]byte, error) {
 	repo, err := g.repo(project)
 	if err != nil {
 		return nil, err
 	}
-	repoContent, err := repo.GetRepoContent(commit)
-	if err != nil {
-		return nil, err
-	}
-	fileContent := repoContent[file]
-	if fileContent == "" {
-		return nil, fmt.Errorf("commit/file not found %v at %v: %w", file, commit, gerrit.ErrResourceNotExist)
-	}
-	return []byte(fileContent), nil
+	return repo.ReadFile(commit, file)
 }
 
 func (g *FakeGerrit) ListTags(ctx context.Context, project string) ([]string, error) {
 	repo, err := g.repo(project)
 	if err != nil {
 		return nil, err
 	}
-	var tags []string
-	for k := range repo.tags {
-		tags = append(tags, k)
-	}
-	return tags, nil
+	out, err := repo.dir.RunCommand(ctx, "tag", "-l")
+	return strings.Split(strings.TrimSpace(string(out)), "\n"), err
 }
 
 func (g *FakeGerrit) GetTag(ctx context.Context, project string, tag string) (gerrit.TagInfo, error) {
 	repo, err := g.repo(project)
 	if err != nil {
 		return gerrit.TagInfo{}, err
 	}
-	if commit, ok := repo.tags[tag]; ok {
-		return gerrit.TagInfo{Revision: commit}, nil
-	} else {
-		return gerrit.TagInfo{}, fmt.Errorf("tag not found: %w", gerrit.ErrResourceNotExist)
-	}
+	out, err := repo.dir.RunCommand(ctx, "rev-parse", "refs/tags/"+tag)
+	return gerrit.TagInfo{Revision: strings.TrimSpace(string(out))}, err
 }
 
 func (g *FakeGerrit) CreateAutoSubmitChange(_ *wf.TaskContext, input gerrit.ChangeInput, reviewers []string, contents map[string]string) (string, error) {
 	repo, err := g.repo(input.Project)
 	if err != nil {
 		return "", err
 	}
-	commit := repo.Commit(contents)
+	commit := repo.CommitOnBranch(input.Branch, contents)
 	return "cl_" + commit, nil
 }
 
@@ -504,9 +495,23 @@ func (g *FakeGerrit) GetCommitsInRefs(ctx context.Context, project string, commi
 	if err != nil {
 		return nil, err
 	}
+	refSet := map[string]bool{}
+	for _, ref := range refs {
+		refSet[ref] = true
+	}
+
 	result := map[string][]string{}
-	for _, commit := range repo.history {
-		result[commit] = []string{"master"}
+	for _, commit := range commits {
+		out, err := repo.dir.RunCommand(ctx, "branch", "--format=%(refname)", "--contains="+commit)
+		if err != nil {
+			return nil, err
+		}
+		for _, branch := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+			branch := strings.TrimSpace(branch)
+			if refSet[branch] {
+				result[commit] = append(result[commit], branch)
+			}
+		}
 	}
 	return result, nil
 }
@@ -527,12 +532,12 @@ func (g *FakeGerrit) serveHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	rev := strings.TrimSuffix(parts[3], ".tar.gz")
-	repoContent, err := repo.GetRepoContent(rev)
+	archive, err := repo.dir.RunCommand(r.Context(), "archive", "--format=tgz", rev)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		return
 	}
-	ServeTarball("", repoContent, w, r)
+	http.ServeContent(w, r, parts[3], time.Now(), bytes.NewReader(archive))
 }
 
 func (*FakeGerrit) QueryChanges(_ context.Context, query string) ([]*gerrit.ChangeInfo, error) {

internal/task/git.go

@@ -0,0 +1,93 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package task
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+
+	"golang.org/x/oauth2"
+)
+
+// A Git manages a set of Git repositories.
+type Git struct {
+	ts         oauth2.TokenSource
+	cookieFile string
+}
+
+// UseOAuth2Auth configures Git authentication using ts.
+func (g *Git) UseOAuth2Auth(ts oauth2.TokenSource) error {
+	g.ts = ts
+	f, err := os.CreateTemp("", "gitcookies")
+	if err != nil {
+		return err
+	}
+	g.cookieFile = f.Name()
+	return f.Close()
+}
+
+// Clone checks out the repository at origin into a temporary directory owned
+// by the resulting GitDir.
+func (g *Git) Clone(ctx context.Context, origin string) (*GitDir, error) {
+	dir, err := os.MkdirTemp("", "relui-git-clone-*")
+	if err != nil {
+		return nil, err
+	}
+	if _, err := g.run(ctx, "", "clone", origin, dir); err != nil {
+		return nil, err
+	}
+	return &GitDir{g, dir}, err
+}
+
+func (g *Git) run(ctx context.Context, dir string, args ...string) ([]byte, error) {
+	stdout := &bytes.Buffer{}
+	stderr := &bytes.Buffer{}
+	if err := g.runGitStreamed(ctx, stdout, stderr, dir, args...); err != nil {
+		return stdout.Bytes(), fmt.Errorf("git command failed: %v, stderr %v", err, stderr.String())
+	}
+	return stdout.Bytes(), nil
+}
+
+func (g *Git) runGitStreamed(ctx context.Context, stdout, stderr io.Writer, dir string, args ...string) error {
+	if g.ts != nil {
+		tok, err := g.ts.Token()
+		if err != nil {
+			return err
+		}
+		// https://github.com/curl/curl/blob/master/docs/HTTP-COOKIES.md
+		cookieLine := fmt.Sprintf(".googlesource.com TRUE / TRUE %v o %v\n", tok.Expiry.Unix(), tok.AccessToken)
+		if err := os.WriteFile(g.cookieFile, []byte(cookieLine), 0o700); err != nil {
+			return fmt.Errorf("error writing git cookies: %v", err)
+		}
+		args = append([]string{"-c", "http.cookiefile=" + g.cookieFile}, args...)
+	}
+
+	cmd := exec.CommandContext(ctx, "git", args...)
+	cmd.Dir = dir
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	return cmd.Run()
+}
+
+// A GitDir is a single Git repository.
+type GitDir struct {
+	git *Git
+	dir string
+}
+
+// RunCommand runs a Git command, returning its stdout if it succeeds, or an
+// error containing its stderr if it fails.
+func (g *GitDir) RunCommand(ctx context.Context, args ...string) ([]byte, error) {
+	return g.git.run(ctx, g.dir, args...)
+}
+
+// Close cleans up the repository.
+func (g *GitDir) Close() error {
+	return os.RemoveAll(g.dir)
+}

internal/task/tagx_test.go

@@ -389,7 +389,7 @@ func newTagXTestDeps(t *testing.T, repos ...*FakeRepo) *tagXTestDeps {
 			st := &types.BuildStatus{
 				Builders: builders,
 			}
-			for _, commit := range r.history {
+			for _, commit := range r.History() {
 				st.Revisions = append(st.Revisions, types.BuildRevision{
 					Repo:       r.name,
 					Revision:   commit,