Skip to content

Commit e232c49

Browse files
lafrikslafriks
authored
Sync releases table with tags on push and for mirrors (#2459) (#2554)
* Sync releases table with tags on push and for mirrors * Code style fixes * Fix api to return only releases * Optimize release creation and update Minimize posibility of race conditions * Fix release lower tag name updating * handle tag reference update by addionally comparing commit id
1 parent 25e71ad commit e232c49

File tree

10 files changed

+366
-118
lines changed

10 files changed

+366
-118
lines changed

‎models/migrations/migrations.go

+2
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,8 @@ var migrations = []Migration{
128128
NewMigration("remove commits and settings unit types", removeCommitsUnitType),
129129
// v43 -> v44
130130
NewMigration("fix protected branch can push value to false", fixProtectedBranchCanPushValue),
131+
// v42 -> v43
132+
NewMigration("add tags to releases and sync existing repositories", releaseAddColumnIsTagAndSyncTags),
131133
}
132134

133135
// Migrate database to current version

‎models/migrations/v42.go

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
// Copyright 2017 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package migrations
6+
7+
import (
8+
"fmt"
9+
10+
"code.gitea.io/git"
11+
"code.gitea.io/gitea/models"
12+
"code.gitea.io/gitea/modules/log"
13+
14+
"github.com/go-xorm/xorm"
15+
)
16+
17+
// ReleaseV39 describes the added field for Release
18+
type ReleaseV39 struct {
19+
IsTag bool `xorm:"NOT NULL DEFAULT false"`
20+
}
21+
22+
// TableName will be invoked by XORM to customrize the table name
23+
func (*ReleaseV39) TableName() string {
24+
return "release"
25+
}
26+
27+
func releaseAddColumnIsTagAndSyncTags(x *xorm.Engine) error {
28+
if err := x.Sync2(new(ReleaseV39)); err != nil {
29+
return fmt.Errorf("Sync2: %v", err)
30+
}
31+
32+
// For the sake of SQLite3, we can't use x.Iterate here.
33+
offset := 0
34+
pageSize := 20
35+
for {
36+
repos := make([]*models.Repository, 0, pageSize)
37+
if err := x.Table("repository").Asc("id").Limit(pageSize, offset).Find(&repos); err != nil {
38+
return fmt.Errorf("select repos [offset: %d]: %v", offset, err)
39+
}
40+
for _, repo := range repos {
41+
gitRepo, err := git.OpenRepository(repo.RepoPath())
42+
if err != nil {
43+
log.Warn("OpenRepository: %v", err)
44+
continue
45+
}
46+
47+
if err = models.SyncReleasesWithTags(repo, gitRepo); err != nil {
48+
log.Warn("SyncReleasesWithTags: %v", err)
49+
}
50+
}
51+
if len(repos) < pageSize {
52+
break
53+
}
54+
offset += pageSize
55+
}
56+
return nil
57+
}

‎models/release.go

+75-14
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ type Release struct {
3434
NumCommitsBehind int64 `xorm:"-"`
3535
Note string `xorm:"TEXT"`
3636
IsDraft bool `xorm:"NOT NULL DEFAULT false"`
37-
IsPrerelease bool
37+
IsPrerelease bool `xorm:"NOT NULL DEFAULT false"`
38+
IsTag bool `xorm:"NOT NULL DEFAULT false"`
3839

3940
Attachments []*Attachment `xorm:"-"`
4041

@@ -139,17 +140,18 @@ func createTag(gitRepo *git.Repository, rel *Release) error {
139140
}
140141
return err
141142
}
142-
} else {
143-
commit, err := gitRepo.GetTagCommit(rel.TagName)
144-
if err != nil {
145-
return fmt.Errorf("GetTagCommit: %v", err)
146-
}
143+
rel.LowerTagName = strings.ToLower(rel.TagName)
144+
}
145+
commit, err := gitRepo.GetTagCommit(rel.TagName)
146+
if err != nil {
147+
return fmt.Errorf("GetTagCommit: %v", err)
148+
}
147149

148-
rel.Sha1 = commit.ID.String()
149-
rel.NumCommits, err = commit.CommitsCount()
150-
if err != nil {
151-
return fmt.Errorf("CommitsCount: %v", err)
152-
}
150+
rel.Sha1 = commit.ID.String()
151+
rel.CreatedUnix = commit.Author.When.Unix()
152+
rel.NumCommits, err = commit.CommitsCount()
153+
if err != nil {
154+
return fmt.Errorf("CommitsCount: %v", err)
153155
}
154156
}
155157
return nil
@@ -236,6 +238,7 @@ func GetReleaseByID(id int64) (*Release, error) {
236238
// FindReleasesOptions describes the conditions to Find releases
237239
type FindReleasesOptions struct {
238240
IncludeDrafts bool
241+
IncludeTags bool
239242
TagNames []string
240243
}
241244

@@ -246,6 +249,9 @@ func (opts *FindReleasesOptions) toConds(repoID int64) builder.Cond {
246249
if !opts.IncludeDrafts {
247250
cond = cond.And(builder.Eq{"is_draft": false})
248251
}
252+
if !opts.IncludeTags {
253+
cond = cond.And(builder.Eq{"is_tag": false})
254+
}
249255
if len(opts.TagNames) > 0 {
250256
cond = cond.And(builder.In("tag_name", opts.TagNames))
251257
}
@@ -361,6 +367,8 @@ func UpdateRelease(gitRepo *git.Repository, rel *Release, attachmentUUIDs []stri
361367
if err = createTag(gitRepo, rel); err != nil {
362368
return err
363369
}
370+
rel.LowerTagName = strings.ToLower(rel.TagName)
371+
364372
_, err = x.Id(rel.ID).AllCols().Update(rel)
365373
if err != nil {
366374
return err
@@ -397,11 +405,64 @@ func DeleteReleaseByID(id int64, u *User, delTag bool) error {
397405
if err != nil && !strings.Contains(stderr, "not found") {
398406
return fmt.Errorf("git tag -d: %v - %s", err, stderr)
399407
}
400-
}
401408

402-
if _, err = x.Id(rel.ID).Delete(new(Release)); err != nil {
403-
return fmt.Errorf("Delete: %v", err)
409+
if _, err = x.Id(rel.ID).Delete(new(Release)); err != nil {
410+
return fmt.Errorf("Delete: %v", err)
411+
}
412+
} else {
413+
rel.IsTag = true
414+
rel.IsDraft = false
415+
rel.IsPrerelease = false
416+
rel.Title = ""
417+
rel.Note = ""
418+
419+
if _, err = x.Id(rel.ID).AllCols().Update(rel); err != nil {
420+
return fmt.Errorf("Update: %v", err)
421+
}
404422
}
405423

406424
return nil
407425
}
426+
427+
// SyncReleasesWithTags synchronizes release table with repository tags
428+
func SyncReleasesWithTags(repo *Repository, gitRepo *git.Repository) error {
429+
existingRelTags := make(map[string]struct{})
430+
opts := FindReleasesOptions{IncludeDrafts: true, IncludeTags: true}
431+
for page := 1; ; page++ {
432+
rels, err := GetReleasesByRepoID(repo.ID, opts, page, 100)
433+
if err != nil {
434+
return fmt.Errorf("GetReleasesByRepoID: %v", err)
435+
}
436+
if len(rels) == 0 {
437+
break
438+
}
439+
for _, rel := range rels {
440+
if rel.IsDraft {
441+
continue
442+
}
443+
commitID, err := gitRepo.GetTagCommitID(rel.TagName)
444+
if err != nil {
445+
return fmt.Errorf("GetTagCommitID: %v", err)
446+
}
447+
if !gitRepo.IsTagExist(rel.TagName) || commitID != rel.Sha1 {
448+
if err := pushUpdateDeleteTag(repo, gitRepo, rel.TagName); err != nil {
449+
return fmt.Errorf("pushUpdateDeleteTag: %v", err)
450+
}
451+
} else {
452+
existingRelTags[strings.ToLower(rel.TagName)] = struct{}{}
453+
}
454+
}
455+
}
456+
tags, err := gitRepo.GetTags()
457+
if err != nil {
458+
return fmt.Errorf("GetTags: %v", err)
459+
}
460+
for _, tagName := range tags {
461+
if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok {
462+
if err := pushUpdateAddTag(repo, gitRepo, tagName); err != nil {
463+
return fmt.Errorf("pushUpdateAddTag: %v", err)
464+
}
465+
}
466+
}
467+
return nil
468+
}

‎models/repo.go

+4
Original file line numberDiff line numberDiff line change
@@ -912,6 +912,10 @@ func MigrateRepository(u *User, opts MigrateRepoOptions) (*Repository, error) {
912912
if headBranch != nil {
913913
repo.DefaultBranch = headBranch.Name
914914
}
915+
916+
if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
917+
log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
918+
}
915919
}
916920

917921
if err = repo.UpdateSize(); err != nil {

‎models/repo_mirror.go

+10
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/go-xorm/xorm"
1414
"gopkg.in/ini.v1"
1515

16+
"code.gitea.io/git"
1617
"code.gitea.io/gitea/modules/log"
1718
"code.gitea.io/gitea/modules/process"
1819
"code.gitea.io/gitea/modules/setting"
@@ -148,6 +149,15 @@ func (m *Mirror) runSync() bool {
148149
return false
149150
}
150151

152+
gitRepo, err := git.OpenRepository(repoPath)
153+
if err != nil {
154+
log.Error(4, "OpenRepository: %v", err)
155+
return false
156+
}
157+
if err = SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
158+
log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
159+
}
160+
151161
if err := m.Repo.UpdateSize(); err != nil {
152162
log.Error(4, "Failed to update size for mirror repository: %v", err)
153163
}

‎models/update.go

+100-5
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,93 @@ func PushUpdate(branch string, opt PushUpdateOptions) error {
8181
return nil
8282
}
8383

84+
func pushUpdateDeleteTag(repo *Repository, gitRepo *git.Repository, tagName string) error {
85+
rel, err := GetRelease(repo.ID, tagName)
86+
if err != nil {
87+
if IsErrReleaseNotExist(err) {
88+
return nil
89+
}
90+
return fmt.Errorf("GetRelease: %v", err)
91+
}
92+
if rel.IsTag {
93+
if _, err = x.Id(rel.ID).Delete(new(Release)); err != nil {
94+
return fmt.Errorf("Delete: %v", err)
95+
}
96+
} else {
97+
rel.IsDraft = true
98+
rel.NumCommits = 0
99+
rel.Sha1 = ""
100+
if _, err = x.Id(rel.ID).AllCols().Update(rel); err != nil {
101+
return fmt.Errorf("Update: %v", err)
102+
}
103+
}
104+
105+
return nil
106+
}
107+
108+
func pushUpdateAddTag(repo *Repository, gitRepo *git.Repository, tagName string) error {
109+
rel, err := GetRelease(repo.ID, tagName)
110+
if err != nil && !IsErrReleaseNotExist(err) {
111+
return fmt.Errorf("GetRelease: %v", err)
112+
}
113+
114+
tag, err := gitRepo.GetTag(tagName)
115+
if err != nil {
116+
return fmt.Errorf("GetTag: %v", err)
117+
}
118+
commit, err := tag.Commit()
119+
if err != nil {
120+
return fmt.Errorf("Commit: %v", err)
121+
}
122+
tagCreatedUnix := commit.Author.When.Unix()
123+
124+
author, err := GetUserByEmail(commit.Author.Email)
125+
if err != nil && !IsErrUserNotExist(err) {
126+
return fmt.Errorf("GetUserByEmail: %v", err)
127+
}
128+
129+
commitsCount, err := commit.CommitsCount()
130+
if err != nil {
131+
return fmt.Errorf("CommitsCount: %v", err)
132+
}
133+
134+
if rel == nil {
135+
rel = &Release{
136+
RepoID: repo.ID,
137+
Title: "",
138+
TagName: tagName,
139+
LowerTagName: strings.ToLower(tagName),
140+
Target: "",
141+
Sha1: commit.ID.String(),
142+
NumCommits: commitsCount,
143+
Note: "",
144+
IsDraft: false,
145+
IsPrerelease: false,
146+
IsTag: true,
147+
CreatedUnix: tagCreatedUnix,
148+
}
149+
if author != nil {
150+
rel.PublisherID = author.ID
151+
}
152+
153+
if _, err = x.InsertOne(rel); err != nil {
154+
return fmt.Errorf("InsertOne: %v", err)
155+
}
156+
} else {
157+
rel.Sha1 = commit.ID.String()
158+
rel.CreatedUnix = tagCreatedUnix
159+
rel.NumCommits = commitsCount
160+
rel.IsDraft = false
161+
if rel.IsTag && author != nil {
162+
rel.PublisherID = author.ID
163+
}
164+
if _, err = x.Id(rel.ID).AllCols().Update(rel); err != nil {
165+
return fmt.Errorf("Update: %v", err)
166+
}
167+
}
168+
return nil
169+
}
170+
84171
func pushUpdate(opts PushUpdateOptions) (repo *Repository, err error) {
85172
isNewRef := opts.OldCommitID == git.EmptySHA
86173
isDelRef := opts.NewCommitID == git.EmptySHA
@@ -106,23 +193,31 @@ func pushUpdate(opts PushUpdateOptions) (repo *Repository, err error) {
106193
return nil, fmt.Errorf("GetRepositoryByName: %v", err)
107194
}
108195

196+
gitRepo, err := git.OpenRepository(repoPath)
197+
if err != nil {
198+
return nil, fmt.Errorf("OpenRepository: %v", err)
199+
}
200+
109201
if isDelRef {
202+
// Tag has been deleted
203+
if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
204+
err = pushUpdateDeleteTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):])
205+
if err != nil {
206+
return nil, fmt.Errorf("pushUpdateDeleteTag: %v", err)
207+
}
208+
}
110209
log.GitLogger.Info("Reference '%s' has been deleted from '%s/%s' by %s",
111210
opts.RefFullName, opts.RepoUserName, opts.RepoName, opts.PusherName)
112211
return repo, nil
113212
}
114213

115-
gitRepo, err := git.OpenRepository(repoPath)
116-
if err != nil {
117-
return nil, fmt.Errorf("OpenRepository: %v", err)
118-
}
119-
120214
if err = repo.UpdateSize(); err != nil {
121215
log.Error(4, "Failed to update size for repository: %v", err)
122216
}
123217

124218
// Push tags.
125219
if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
220+
pushUpdateAddTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):])
126221
if err := CommitRepoAction(CommitRepoActionOptions{
127222
PusherName: opts.PusherName,
128223
RepoOwnerID: owner.ID,

‎modules/context/repo.go

+1
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,7 @@ func RepoAssignment() macaron.Handler {
347347

348348
count, err := models.GetReleaseCountByRepoID(ctx.Repo.Repository.ID, models.FindReleasesOptions{
349349
IncludeDrafts: false,
350+
IncludeTags: true,
350351
})
351352
if err != nil {
352353
ctx.Handle(500, "GetReleaseCountByRepoID", err)

0 commit comments

Comments
 (0)