From 3101e91c20e678f45c1059dab5b9c5b3148f5a03 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 27 Mar 2022 20:57:49 +0100 Subject: [PATCH 1/9] Use full output of git show-ref --tags to get tags for PushUpdateAddTag Strangely #19038 appears to relate to an issue whereby a tag appears to be listed in `git show-ref --tags` but then does not appear when `git show-ref --tags -- short_name` is called. As a solution though I propose to stop the second call as it is unnecessary and only likely to cause problems. Fix #19038 Signed-off-by: Andrew Thornton --- modules/git/repo_branch_nogogit.go | 24 +++++++++++++++--------- modules/git/repo_tag.go | 14 ++++++++++++++ modules/repository/repo.go | 28 +++++++++++++++------------- services/repository/branch.go | 2 +- 4 files changed, 45 insertions(+), 23 deletions(-) diff --git a/modules/git/repo_branch_nogogit.go b/modules/git/repo_branch_nogogit.go index 66990add6fd98..94388b5592da7 100644 --- a/modules/git/repo_branch_nogogit.go +++ b/modules/git/repo_branch_nogogit.go @@ -68,13 +68,18 @@ func (repo *Repository) GetBranchNames(skip, limit int) ([]string, int, error) { } // WalkReferences walks all the references from the repository -func WalkReferences(ctx context.Context, repoPath string, walkfn func(string) error) (int, error) { +func WalkReferences(ctx context.Context, repoPath string, walkfn func(sha1, refname string) error) (int, error) { return walkShowRef(ctx, repoPath, "", 0, 0, walkfn) } +// WalkReferences walks all the references from the repository +func (repo *Repository) WalkReferences(arg string, skip, limit int, walkfn func(sha1, refname string) error) (int, error) { + return walkShowRef(repo.Ctx, repo.Path, arg, skip, limit, walkfn) +} + // callShowRef return refs, if limit = 0 it will not limit func callShowRef(ctx context.Context, repoPath, prefix, arg string, skip, limit int) (branchNames []string, countAll int, err error) { - countAll, err = walkShowRef(ctx, repoPath, arg, skip, limit, func(branchName string) error { + countAll, err = walkShowRef(ctx, repoPath, arg, skip, limit, func(_, branchName string) error { branchName = strings.TrimPrefix(branchName, prefix) branchNames = append(branchNames, branchName) @@ -83,7 +88,7 @@ func callShowRef(ctx context.Context, repoPath, prefix, arg string, skip, limit return } -func walkShowRef(ctx context.Context, repoPath, arg string, skip, limit int, walkfn func(string) error) (countAll int, err error) { +func walkShowRef(ctx context.Context, repoPath, arg string, skip, limit int, walkfn func(sha1, refname string) error) (countAll int, err error) { stdoutReader, stdoutWriter := io.Pipe() defer func() { _ = stdoutReader.Close() @@ -130,11 +135,7 @@ func walkShowRef(ctx context.Context, repoPath, arg string, skip, limit int, wal for limit == 0 || i < skip+limit { // The output of show-ref is simply a list: // SP LF - _, err := bufReader.ReadSlice(' ') - for err == bufio.ErrBufferFull { - // This shouldn't happen but we'll tolerate it for the sake of peace - _, err = bufReader.ReadSlice(' ') - } + sha, err := bufReader.ReadString(' ') if err == io.EOF { return i, nil } @@ -154,7 +155,12 @@ func walkShowRef(ctx context.Context, repoPath, arg string, skip, limit int, wal if len(branchName) > 0 { branchName = branchName[:len(branchName)-1] } - err = walkfn(branchName) + + if len(sha) > 0 { + sha = sha[:len(sha)-1] + } + + err = walkfn(sha, branchName) if err != nil { return i, err } diff --git a/modules/git/repo_tag.go b/modules/git/repo_tag.go index afeb7f5df8a37..c832ab2061be6 100644 --- a/modules/git/repo_tag.go +++ b/modules/git/repo_tag.go @@ -159,6 +159,20 @@ func (repo *Repository) GetTag(name string) (*Tag, error) { return tag, nil } +// GetTagWithID returns a Git tag by given name. +func (repo *Repository) GetTagWithID(idStr, name string) (*Tag, error) { + id, err := NewIDFromString(idStr) + if err != nil { + return nil, err + } + + tag, err := repo.getTag(id, name) + if err != nil { + return nil, err + } + return tag, nil +} + // GetTagInfos returns all tag infos of the repository. func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, int, error) { // TODO this a slow implementation, makes one git command per tag diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 98388bbd7b2ef..6539ff67682b0 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -278,23 +278,25 @@ func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository) } } } - tags, err := gitRepo.GetTags(0, 0) - if err != nil { - return fmt.Errorf("unable to GetTags in Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) - } - for _, tagName := range tags { - if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok { - if err := PushUpdateAddTag(repo, gitRepo, tagName); err != nil { - return fmt.Errorf("unable to PushUpdateAddTag: %q to Repo[%d:%s/%s]: %w", tagName, repo.ID, repo.OwnerName, repo.Name, err) - } + + _, err := gitRepo.WalkReferences("--tags", 0, 0, func(sha1, refname string) error { + tagName := strings.TrimPrefix(refname, git.TagPrefix) + if _, ok := existingRelTags[strings.ToLower(tagName)]; ok { + return nil } - } - return nil + + if err := PushUpdateAddTag(repo, gitRepo, tagName, sha1, refname); err != nil { + return fmt.Errorf("unable to PushUpdateAddTag: %q to Repo[%d:%s/%s]: %w", tagName, repo.ID, repo.OwnerName, repo.Name, err) + } + + return nil + }) + return err } // PushUpdateAddTag must be called for any push actions to add tag -func PushUpdateAddTag(repo *repo_model.Repository, gitRepo *git.Repository, tagName string) error { - tag, err := gitRepo.GetTag(tagName) +func PushUpdateAddTag(repo *repo_model.Repository, gitRepo *git.Repository, tagName, sha1, refname string) error { + tag, err := gitRepo.GetTagWithID(sha1, tagName) if err != nil { return fmt.Errorf("unable to GetTag: %w", err) } diff --git a/services/repository/branch.go b/services/repository/branch.go index b1a6dafb58d61..6667cdee61a2d 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -55,7 +55,7 @@ func GetBranches(ctx context.Context, repo *repo_model.Repository, skip, limit i // checkBranchName validates branch name with existing repository branches func checkBranchName(ctx context.Context, repo *repo_model.Repository, name string) error { - _, err := git.WalkReferences(ctx, repo.RepoPath(), func(refName string) error { + _, err := git.WalkReferences(ctx, repo.RepoPath(), func(_, refName string) error { branchRefName := strings.TrimPrefix(refName, git.BranchPrefix) switch { case branchRefName == name: From 90da6536b510078c4b072bfd92706214b7950366 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 27 Mar 2022 21:56:56 +0100 Subject: [PATCH 2/9] fixup! Use full output of git show-ref --tags to get tags for PushUpdateAddTag --- modules/git/repo_branch_gogit.go | 38 ++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/modules/git/repo_branch_gogit.go b/modules/git/repo_branch_gogit.go index 57952bcc640ca..a9b11f90d98ae 100644 --- a/modules/git/repo_branch_gogit.go +++ b/modules/git/repo_branch_gogit.go @@ -13,6 +13,7 @@ import ( "strings" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/storer" ) // IsObjectExist returns true if given reference exists in the repository. @@ -82,7 +83,7 @@ func (repo *Repository) GetBranchNames(skip, limit int) ([]string, int, error) { } // WalkReferences walks all the references from the repository -func WalkReferences(ctx context.Context, repoPath string, walkfn func(string) error) (int, error) { +func WalkReferences(ctx context.Context, repoPath string, walkfn func(sha1, refname string) error) (int, error) { repo := RepositoryFromContext(ctx, repoPath) if repo == nil { var err error @@ -101,9 +102,42 @@ func WalkReferences(ctx context.Context, repoPath string, walkfn func(string) er defer iter.Close() err = iter.ForEach(func(ref *plumbing.Reference) error { - err := walkfn(string(ref.Name())) + err := walkfn(ref.Hash().String(), string(ref.Name())) i++ return err }) return i, err } + +// WalkReferences walks all the references from the repository +func (repo *Repository) WalkReferences(arg string, skip, limit int, walkfn func(sha1, refname string) error) (int, error) { + i := 0 + var iter storer.ReferenceIter + var err error + if arg == "--tags" { + iter, err = repo.gogitRepo.Tags() + } else { + iter, err = repo.gogitRepo.References() + } + if err != nil { + return i, err + } + defer iter.Close() + + err = iter.ForEach(func(ref *plumbing.Reference) error { + if i < skip { + i++ + return nil + } + err := walkfn(ref.Hash().String(), string(ref.Name())) + i++ + if err != nil { + return err + } + if limit != 0 && i >= skip+limit { + return storer.ErrStop + } + return nil + }) + return i, err +} From d9a11c1fc529057c2667c192fa988950594672a3 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 27 Mar 2022 22:17:26 +0100 Subject: [PATCH 3/9] reuse cat-file --batch and --batch-check for tags Signed-off-by: Andrew Thornton --- modules/git/repo_tag.go | 77 ----------------------- modules/git/repo_tag_gogit.go | 76 ++++++++++++++++++++++ modules/git/repo_tag_nogogit.go | 108 ++++++++++++++++++++++++++++++++ 3 files changed, 184 insertions(+), 77 deletions(-) diff --git a/modules/git/repo_tag.go b/modules/git/repo_tag.go index c832ab2061be6..4cf6ed416c9a4 100644 --- a/modules/git/repo_tag.go +++ b/modules/git/repo_tag.go @@ -10,7 +10,6 @@ import ( "fmt" "strings" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" ) @@ -34,69 +33,6 @@ func (repo *Repository) CreateAnnotatedTag(name, message, revision string) error return err } -func (repo *Repository) getTag(tagID SHA1, name string) (*Tag, error) { - t, ok := repo.tagCache.Get(tagID.String()) - if ok { - log.Debug("Hit cache: %s", tagID) - tagClone := *t.(*Tag) - tagClone.Name = name // This is necessary because lightweight tags may have same id - return &tagClone, nil - } - - tp, err := repo.GetTagType(tagID) - if err != nil { - return nil, err - } - - // Get the commit ID and tag ID (may be different for annotated tag) for the returned tag object - commitIDStr, err := repo.GetTagCommitID(name) - if err != nil { - // every tag should have a commit ID so return all errors - return nil, err - } - commitID, err := NewIDFromString(commitIDStr) - if err != nil { - return nil, err - } - - // If type is "commit, the tag is a lightweight tag - if ObjectType(tp) == ObjectCommit { - commit, err := repo.GetCommit(commitIDStr) - if err != nil { - return nil, err - } - tag := &Tag{ - Name: name, - ID: tagID, - Object: commitID, - Type: tp, - Tagger: commit.Committer, - Message: commit.Message(), - } - - repo.tagCache.Set(tagID.String(), tag) - return tag, nil - } - - // The tag is an annotated tag with a message. - data, err := NewCommand(repo.Ctx, "cat-file", "-p", tagID.String()).RunInDirBytes(repo.Path) - if err != nil { - return nil, err - } - - tag, err := parseTagData(data) - if err != nil { - return nil, err - } - - tag.Name = name - tag.ID = tagID - tag.Type = tp - - repo.tagCache.Set(tagID.String(), tag) - return tag, nil -} - // GetTagNameBySHA returns the name of a tag from its tag object SHA or commit SHA func (repo *Repository) GetTagNameBySHA(sha string) (string, error) { if len(sha) < 5 { @@ -206,19 +142,6 @@ func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, int, error) { return tags, tagsTotal, nil } -// GetTagType gets the type of the tag, either commit (simple) or tag (annotated) -func (repo *Repository) GetTagType(id SHA1) (string, error) { - // Get tag type - stdout, err := NewCommand(repo.Ctx, "cat-file", "-t", id.String()).RunInDir(repo.Path) - if err != nil { - return "", err - } - if len(stdout) == 0 { - return "", ErrNotExist{ID: id.String()} - } - return strings.TrimSpace(stdout), nil -} - // GetAnnotatedTag returns a Git tag by its SHA, must be an annotated tag func (repo *Repository) GetAnnotatedTag(sha string) (*Tag, error) { id, err := NewIDFromString(sha) diff --git a/modules/git/repo_tag_gogit.go b/modules/git/repo_tag_gogit.go index ff8a6d53eee6e..e795db608dba5 100644 --- a/modules/git/repo_tag_gogit.go +++ b/modules/git/repo_tag_gogit.go @@ -11,6 +11,7 @@ package git import ( "strings" + "code.gitea.io/gitea/modules/log" "github.com/go-git/go-git/v5/plumbing" ) @@ -53,3 +54,78 @@ func (repo *Repository) GetTags(skip, limit int) ([]string, error) { return tagNames, nil } + +// GetTagType gets the type of the tag, either commit (simple) or tag (annotated) +func (repo *Repository) GetTagType(id SHA1) (string, error) { + // Get tag type + stdout, err := NewCommand(repo.Ctx, "cat-file", "-t", id.String()).RunInDir(repo.Path) + if err != nil { + return "", err + } + if len(stdout) == 0 { + return "", ErrNotExist{ID: id.String()} + } + return strings.TrimSpace(stdout), nil +} + +func (repo *Repository) getTag(tagID SHA1, name string) (*Tag, error) { + t, ok := repo.tagCache.Get(tagID.String()) + if ok { + log.Debug("Hit cache: %s", tagID) + tagClone := *t.(*Tag) + tagClone.Name = name // This is necessary because lightweight tags may have same id + return &tagClone, nil + } + + tp, err := repo.GetTagType(tagID) + if err != nil { + return nil, err + } + + // Get the commit ID and tag ID (may be different for annotated tag) for the returned tag object + commitIDStr, err := repo.GetTagCommitID(name) + if err != nil { + // every tag should have a commit ID so return all errors + return nil, err + } + commitID, err := NewIDFromString(commitIDStr) + if err != nil { + return nil, err + } + + // If type is "commit, the tag is a lightweight tag + if ObjectType(tp) == ObjectCommit { + commit, err := repo.GetCommit(commitIDStr) + if err != nil { + return nil, err + } + tag := &Tag{ + Name: name, + ID: tagID, + Object: commitID, + Type: tp, + Tagger: commit.Committer, + Message: commit.Message(), + } + + repo.tagCache.Set(tagID.String(), tag) + return tag, nil + } + + gogitTag, err := repo.gogitRepo.TagObject(tagID) + if err != nil { + return nil, err + } + + tag := &Tag{ + Name: name, + ID: tagID, + Object: gogitTag.Target, + Type: tp, + Tagger: &gogitTag.Tagger, + Message: gogitTag.Message, + } + + repo.tagCache.Set(tagID.String(), tag) + return tag, nil +} diff --git a/modules/git/repo_tag_nogogit.go b/modules/git/repo_tag_nogogit.go index 1a23755aa666b..f2da7d88572b3 100644 --- a/modules/git/repo_tag_nogogit.go +++ b/modules/git/repo_tag_nogogit.go @@ -8,6 +8,13 @@ package git +import ( + "errors" + "io" + + "code.gitea.io/gitea/modules/log" +) + // IsTagExist returns true if given tag exists in the repository. func (repo *Repository) IsTagExist(name string) bool { if name == "" { @@ -23,3 +30,104 @@ func (repo *Repository) GetTags(skip, limit int) (tags []string, err error) { tags, _, err = callShowRef(repo.Ctx, repo.Path, TagPrefix, "--tags", skip, limit) return } + +// GetTagType gets the type of the tag, either commit (simple) or tag (annotated) +func (repo *Repository) GetTagType(id SHA1) (string, error) { + wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx) + defer cancel() + _, err := wr.Write([]byte(id.String() + "\n")) + if err != nil { + return "", err + } + _, typ, _, err := ReadBatchLine(rd) + if IsErrNotExist(err) { + return "", ErrNotExist{ID: id.String()} + } + return typ, nil +} + +func (repo *Repository) getTag(tagID SHA1, name string) (*Tag, error) { + t, ok := repo.tagCache.Get(tagID.String()) + if ok { + log.Debug("Hit cache: %s", tagID) + tagClone := *t.(*Tag) + tagClone.Name = name // This is necessary because lightweight tags may have same id + return &tagClone, nil + } + + tp, err := repo.GetTagType(tagID) + if err != nil { + return nil, err + } + + // Get the commit ID and tag ID (may be different for annotated tag) for the returned tag object + commitIDStr, err := repo.GetTagCommitID(name) + if err != nil { + // every tag should have a commit ID so return all errors + return nil, err + } + commitID, err := NewIDFromString(commitIDStr) + if err != nil { + return nil, err + } + + // If type is "commit, the tag is a lightweight tag + if ObjectType(tp) == ObjectCommit { + commit, err := repo.GetCommit(commitIDStr) + if err != nil { + return nil, err + } + tag := &Tag{ + Name: name, + ID: tagID, + Object: commitID, + Type: tp, + Tagger: commit.Committer, + Message: commit.Message(), + } + + repo.tagCache.Set(tagID.String(), tag) + return tag, nil + } + + // The tag is an annotated tag with a message. + wr, rd, cancel := repo.CatFileBatch(repo.Ctx) + defer cancel() + + if _, err := wr.Write([]byte(tagID.String() + "\n")); err != nil { + return nil, err + } + _, typ, size, err := ReadBatchLine(rd) + if err != nil { + if errors.Is(err, io.EOF) || IsErrNotExist(err) { + return nil, ErrNotExist{ID: tagID.String()} + } + return nil, err + } + if typ != "tag" { + return nil, ErrNotExist{ID: tagID.String()} + } + + // then we need to parse the tag + // and load the commit + data, err := io.ReadAll(io.LimitReader(rd, size)) + if err != nil { + return nil, err + } + _, err = rd.Discard(1) + if err != nil { + return nil, err + } + + tag, err := parseTagData(data) + if err != nil { + return nil, err + } + + tag.Name = name + tag.ID = tagID + tag.Type = tp + + repo.tagCache.Set(tagID.String(), tag) + return tag, nil +} From e8a0a21f516ba1a2d1d270648fb6ab67fef09f5a Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 28 Mar 2022 20:50:28 +0100 Subject: [PATCH 4/9] Write the commit-graph after migration Signed-off-by: Andrew Thornton --- modules/repository/repo.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 6539ff67682b0..2bea900d38e48 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -80,6 +80,12 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, return repo, fmt.Errorf("Clone: %v", err) } + if git.CheckGitVersionAtLeast("2.18") == nil { + if _, err := git.NewCommand(ctx, "commit-graph", "write").RunInDir(repoPath); err != nil { + return repo, fmt.Errorf("unable to write commit-graph: %w", err) + } + } + if opts.Wiki { wikiPath := repo_model.WikiPath(u.Name, opts.RepoName) wikiRemotePath := WikiRemoteURL(ctx, opts.CloneAddr) @@ -101,6 +107,11 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } } } + if git.CheckGitVersionAtLeast("2.18") == nil { + if _, err := git.NewCommand(ctx, "commit-graph", "write").RunInDir(wikiPath); err != nil { + return repo, fmt.Errorf("unable to write commit-graph: %w", err) + } + } } if repo.OwnerID == u.ID { From 6456768805bc78e4534d28e8ef969f0f288a2638 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 29 Mar 2022 02:02:18 +0100 Subject: [PATCH 5/9] as per review Signed-off-by: Andrew Thornton --- modules/git/repo_branch_gogit.go | 10 +++++++--- modules/git/repo_branch_nogogit.go | 13 ++++++++++++- modules/repository/repo.go | 2 +- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/modules/git/repo_branch_gogit.go b/modules/git/repo_branch_gogit.go index a9b11f90d98ae..59ae0eaa09b7d 100644 --- a/modules/git/repo_branch_gogit.go +++ b/modules/git/repo_branch_gogit.go @@ -83,6 +83,7 @@ func (repo *Repository) GetBranchNames(skip, limit int) ([]string, int, error) { } // WalkReferences walks all the references from the repository +// refType should be empty, ObjectTag or ObjectBranch. All other values are equivalent to empty. func WalkReferences(ctx context.Context, repoPath string, walkfn func(sha1, refname string) error) (int, error) { repo := RepositoryFromContext(ctx, repoPath) if repo == nil { @@ -110,13 +111,16 @@ func WalkReferences(ctx context.Context, repoPath string, walkfn func(sha1, refn } // WalkReferences walks all the references from the repository -func (repo *Repository) WalkReferences(arg string, skip, limit int, walkfn func(sha1, refname string) error) (int, error) { +func (repo *Repository) WalkReferences(arg ObjectType, skip, limit int, walkfn func(sha1, refname string) error) (int, error) { i := 0 var iter storer.ReferenceIter var err error - if arg == "--tags" { + switch arg { + case ObjectTag: iter, err = repo.gogitRepo.Tags() - } else { + case ObjectBranch: + iter, err = repo.gogitRepo.Branches() + default: iter, err = repo.gogitRepo.References() } if err != nil { diff --git a/modules/git/repo_branch_nogogit.go b/modules/git/repo_branch_nogogit.go index 94388b5592da7..f595b6d9a86ad 100644 --- a/modules/git/repo_branch_nogogit.go +++ b/modules/git/repo_branch_nogogit.go @@ -73,7 +73,18 @@ func WalkReferences(ctx context.Context, repoPath string, walkfn func(sha1, refn } // WalkReferences walks all the references from the repository -func (repo *Repository) WalkReferences(arg string, skip, limit int, walkfn func(sha1, refname string) error) (int, error) { +// refType should be empty, ObjectTag or ObjectBranch. All other values are equivalent to empty. +func (repo *Repository) WalkReferences(refType ObjectType, skip, limit int, walkfn func(sha1, refname string) error) (int, error) { + var arg string + switch refType { + case ObjectTag: + arg = "--tags" + case ObjectBranch: + arg = "--heads" + default: + arg = "" + } + return walkShowRef(repo.Ctx, repo.Path, arg, skip, limit, walkfn) } diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 2bea900d38e48..a3aa73aae0c0a 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -290,7 +290,7 @@ func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository) } } - _, err := gitRepo.WalkReferences("--tags", 0, 0, func(sha1, refname string) error { + _, err := gitRepo.WalkReferences(git.ObjectTag, 0, 0, func(sha1, refname string) error { tagName := strings.TrimPrefix(refname, git.TagPrefix) if _, ok := existingRelTags[strings.ToLower(tagName)]; ok { return nil From 83a8a620aad7affae459228b5674ac6148362c36 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 29 Mar 2022 02:06:45 +0100 Subject: [PATCH 6/9] Also write commit-graph on mirror pull Signed-off-by: Andrew Thornton --- services/mirror/mirror_pull.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index d142a48ca0760..fdf73b23131bb 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -204,6 +204,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo) + gitArgs := []string{"remote", "update"} if m.EnablePrune { gitArgs = append(gitArgs, "--prune") @@ -276,6 +277,12 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo } output := stderrBuilder.String() + if git.CheckGitVersionAtLeast("2.18") == nil { + if _, err := git.NewCommand(ctx, "commit-graph", "write").RunInDir(repoPath); err != nil { + log.Error("SyncMirrors [repo: %-v]: Unable to write commit-graph Error %v", m.Repo, err) + } + } + gitRepo, err := git.OpenRepositoryCtx(ctx, repoPath) if err != nil { log.Error("SyncMirrors [repo: %-v]: failed to OpenRepository: %v", m.Repo, err) @@ -368,6 +375,12 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo } return nil, false } + + if git.CheckGitVersionAtLeast("2.18") == nil { + if _, err := git.NewCommand(ctx, "commit-graph", "write").RunInDir(wikiPath); err != nil { + log.Error("SyncMirrors [repo: %-v]: Unable to write commit-graph Error %v", m.Repo, err) + } + } } log.Trace("SyncMirrors [repo: %-v Wiki]: git remote update complete", m.Repo) } From 425a5ac0ab3e10f3848627dd165cce1d54fd3ba1 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 29 Mar 2022 02:07:38 +0100 Subject: [PATCH 7/9] as per review Signed-off-by: Andrew Thornton --- modules/git/repo_tag.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/git/repo_tag.go b/modules/git/repo_tag.go index 4cf6ed416c9a4..d1b076ffc3f8f 100644 --- a/modules/git/repo_tag.go +++ b/modules/git/repo_tag.go @@ -95,7 +95,7 @@ func (repo *Repository) GetTag(name string) (*Tag, error) { return tag, nil } -// GetTagWithID returns a Git tag by given name. +// GetTagWithID returns a Git tag by given name and ID func (repo *Repository) GetTagWithID(idStr, name string) (*Tag, error) { id, err := NewIDFromString(idStr) if err != nil { From 8375ac4c5ac55b696ff4d87d53275505bf6cebdd Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 29 Mar 2022 02:21:13 +0100 Subject: [PATCH 8/9] improve gogit gettagtype Signed-off-by: Andrew Thornton --- modules/git/repo_tag_gogit.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/modules/git/repo_tag_gogit.go b/modules/git/repo_tag_gogit.go index e795db608dba5..f2d0fc8fc9b3c 100644 --- a/modules/git/repo_tag_gogit.go +++ b/modules/git/repo_tag_gogit.go @@ -58,14 +58,15 @@ func (repo *Repository) GetTags(skip, limit int) ([]string, error) { // GetTagType gets the type of the tag, either commit (simple) or tag (annotated) func (repo *Repository) GetTagType(id SHA1) (string, error) { // Get tag type - stdout, err := NewCommand(repo.Ctx, "cat-file", "-t", id.String()).RunInDir(repo.Path) + obj, err := repo.gogitRepo.Object(plumbing.AnyObject, id) if err != nil { + if err == plumbing.ErrReferenceNotFound { + return "", &ErrNotExist{ID: id.String()} + } return "", err } - if len(stdout) == 0 { - return "", ErrNotExist{ID: id.String()} - } - return strings.TrimSpace(stdout), nil + + return obj.Type().String(), nil } func (repo *Repository) getTag(tagID SHA1, name string) (*Tag, error) { @@ -114,6 +115,10 @@ func (repo *Repository) getTag(tagID SHA1, name string) (*Tag, error) { gogitTag, err := repo.gogitRepo.TagObject(tagID) if err != nil { + if err == plumbing.ErrReferenceNotFound { + return nil, &ErrNotExist{ID: tagID.String()} + } + return nil, err } From 9113f7f713749352e5adeb718108c0be7e2f3898 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 29 Mar 2022 13:18:31 +0200 Subject: [PATCH 9/9] dedup code --- modules/git/repo_commitgraph.go | 21 +++++++++++++++++++++ modules/git/repo_tag_gogit.go | 1 + modules/repository/repo.go | 12 ++++-------- services/mirror/mirror_pull.go | 12 ++++-------- 4 files changed, 30 insertions(+), 16 deletions(-) create mode 100644 modules/git/repo_commitgraph.go diff --git a/modules/git/repo_commitgraph.go b/modules/git/repo_commitgraph.go new file mode 100644 index 0000000000000..5549c92591254 --- /dev/null +++ b/modules/git/repo_commitgraph.go @@ -0,0 +1,21 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package git + +import ( + "context" + "fmt" +) + +// WriteCommitGraph write commit graph to speed up repo access +// this requires git v2.18 to be installed +func WriteCommitGraph(ctx context.Context, repoPath string) error { + if CheckGitVersionAtLeast("2.18") == nil { + if _, err := NewCommand(ctx, "commit-graph", "write").RunInDir(repoPath); err != nil { + return fmt.Errorf("unable to write commit-graph for '%s' : %w", repoPath, err) + } + } + return nil +} diff --git a/modules/git/repo_tag_gogit.go b/modules/git/repo_tag_gogit.go index f2d0fc8fc9b3c..5c87e914c063f 100644 --- a/modules/git/repo_tag_gogit.go +++ b/modules/git/repo_tag_gogit.go @@ -12,6 +12,7 @@ import ( "strings" "code.gitea.io/gitea/modules/log" + "github.com/go-git/go-git/v5/plumbing" ) diff --git a/modules/repository/repo.go b/modules/repository/repo.go index a3aa73aae0c0a..302f6c40b1116 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -80,10 +80,8 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, return repo, fmt.Errorf("Clone: %v", err) } - if git.CheckGitVersionAtLeast("2.18") == nil { - if _, err := git.NewCommand(ctx, "commit-graph", "write").RunInDir(repoPath); err != nil { - return repo, fmt.Errorf("unable to write commit-graph: %w", err) - } + if err := git.WriteCommitGraph(ctx, repoPath); err != nil { + return repo, err } if opts.Wiki { @@ -107,10 +105,8 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } } } - if git.CheckGitVersionAtLeast("2.18") == nil { - if _, err := git.NewCommand(ctx, "commit-graph", "write").RunInDir(wikiPath); err != nil { - return repo, fmt.Errorf("unable to write commit-graph: %w", err) - } + if err := git.WriteCommitGraph(ctx, wikiPath); err != nil { + return repo, err } } diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index fdf73b23131bb..a8a646a51bc1a 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -277,10 +277,8 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo } output := stderrBuilder.String() - if git.CheckGitVersionAtLeast("2.18") == nil { - if _, err := git.NewCommand(ctx, "commit-graph", "write").RunInDir(repoPath); err != nil { - log.Error("SyncMirrors [repo: %-v]: Unable to write commit-graph Error %v", m.Repo, err) - } + if err := git.WriteCommitGraph(ctx, repoPath); err != nil { + log.Error("SyncMirrors [repo: %-v]: %v", m.Repo, err) } gitRepo, err := git.OpenRepositoryCtx(ctx, repoPath) @@ -376,10 +374,8 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo return nil, false } - if git.CheckGitVersionAtLeast("2.18") == nil { - if _, err := git.NewCommand(ctx, "commit-graph", "write").RunInDir(wikiPath); err != nil { - log.Error("SyncMirrors [repo: %-v]: Unable to write commit-graph Error %v", m.Repo, err) - } + if err := git.WriteCommitGraph(ctx, wikiPath); err != nil { + log.Error("SyncMirrors [repo: %-v]: %v", m.Repo, err) } } log.Trace("SyncMirrors [repo: %-v Wiki]: git remote update complete", m.Repo)