From c808a5d38f70f96e9c35ca9ed20a17bdf2feeaa1 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 22 Oct 2020 13:30:56 +0800 Subject: [PATCH 1/3] Keep database transactions not too big --- models/migrations/v156.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/models/migrations/v156.go b/models/migrations/v156.go index dc9c20188dcce..acdee695cd261 100644 --- a/models/migrations/v156.go +++ b/models/migrations/v156.go @@ -50,9 +50,7 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error { sess := x.NewSession() defer sess.Close() - if err := sess.Begin(); err != nil { - return err - } + var ( gitRepoCache = make(map[int64]*git.Repository) @@ -70,6 +68,10 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error { for start := 0; ; start += batchSize { releases := make([]*Release, 0, batchSize) + if err := sess.Begin(); err != nil { + return err + } + if err := sess.Limit(batchSize, start).Asc("id").Where("is_tag=?", true).Find(&releases); err != nil { return err } @@ -134,7 +136,11 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error { return err } } + + if err := sess.Commit(); err != nil { + return err + } } - return sess.Commit() + return nil } From 0ae7bd2e9036f9576ed4a54e47ab97e9f4e7834e Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 22 Oct 2020 10:36:07 +0100 Subject: [PATCH 2/3] Fix #13255 Signed-off-by: Andrew Thornton --- models/migrations/v156.go | 43 ++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/models/migrations/v156.go b/models/migrations/v156.go index acdee695cd261..a5f9d6711516d 100644 --- a/models/migrations/v156.go +++ b/models/migrations/v156.go @@ -10,6 +10,7 @@ import ( "strings" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "xorm.io/xorm" @@ -35,9 +36,10 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error { } type Repository struct { - ID int64 - OwnerID int64 - Name string + ID int64 + OwnerID int64 + OwnerName string + Name string } type User struct { @@ -50,13 +52,10 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error { sess := x.NewSession() defer sess.Close() - - var ( gitRepoCache = make(map[int64]*git.Repository) gitRepo *git.Repository repoCache = make(map[int64]*Repository) - userCache = make(map[int64]*User) ok bool err error ) @@ -90,26 +89,32 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error { if err != nil { return err } else if !has { - return fmt.Errorf("Repository %d is not exist", release.RepoID) + log.Warn("Release[%d] is orphaned and refers to non-existing repository %d", release.ID, release.RepoID) + log.Warn("This release should be deleted") + continue } - repoCache[release.RepoID] = repo - } + if repo.OwnerName == "" { + // v120.go migration may not have been run correctly - we'll just replicate it here + // because this appears to be a common-ish problem. + if _, err := sess.Exec("UPDATE repository SET owner_name = (SELECT name FROM `user` WHERE `user`.id = repository.owner_id)"); err != nil { + return err + } - user, ok := userCache[repo.OwnerID] - if !ok { - user = new(User) - has, err := sess.ID(repo.OwnerID).Get(user) - if err != nil { - return err - } else if !has { - return fmt.Errorf("User %d is not exist", repo.OwnerID) + if _, err := sess.ID(release.RepoID).Get(repo); err != nil { + return err + } } - userCache[repo.OwnerID] = user + repoCache[release.RepoID] = repo + } + + if repo.OwnerName == "" { + log.Warn("Release[%d] refers to Repo[%d] Name: %s with OwnerID: %d but does not have OwnerName set", release.ID, release.RepoID, repo.Name, repo.OwnerID) + continue } - gitRepo, err = git.OpenRepository(repoPath(user.Name, repo.Name)) + gitRepo, err = git.OpenRepository(repoPath(repo.OwnerName, repo.Name)) if err != nil { return err } From 199bf5d08e96d1ef710aaf4963dda53e7182c190 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 22 Oct 2020 13:29:54 +0100 Subject: [PATCH 3/3] Only cache the last repo Signed-off-by: Andrew Thornton --- models/migrations/v156.go | 64 ++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/models/migrations/v156.go b/models/migrations/v156.go index a5f9d6711516d..6092a37d554aa 100644 --- a/models/migrations/v156.go +++ b/models/migrations/v156.go @@ -53,15 +53,12 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error { defer sess.Close() var ( - gitRepoCache = make(map[int64]*git.Repository) - gitRepo *git.Repository - repoCache = make(map[int64]*Repository) - ok bool - err error + repo *Repository + gitRepo *git.Repository ) defer func() { - for i := range gitRepoCache { - gitRepoCache[i].Close() + if gitRepo != nil { + gitRepo.Close() } }() for start := 0; ; start += batchSize { @@ -71,7 +68,7 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error { return err } - if err := sess.Limit(batchSize, start).Asc("id").Where("is_tag=?", true).Find(&releases); err != nil { + if err := sess.Limit(batchSize, start).Asc("repo_id", "id").Where("is_tag=?", true).Find(&releases); err != nil { return err } @@ -80,45 +77,36 @@ func fixPublisherIDforTagReleases(x *xorm.Engine) error { } for _, release := range releases { - gitRepo, ok = gitRepoCache[release.RepoID] - if !ok { - repo, ok := repoCache[release.RepoID] - if !ok { - repo = new(Repository) - has, err := sess.ID(release.RepoID).Get(repo) - if err != nil { + if repo == nil || repo.ID != release.RepoID { + if gitRepo != nil { + gitRepo.Close() + gitRepo = nil + } + repo = new(Repository) + has, err := sess.ID(release.RepoID).Get(repo) + if err != nil { + return err + } else if !has { + log.Warn("Release[%d] is orphaned and refers to non-existing repository %d", release.ID, release.RepoID) + log.Warn("This release should be deleted") + continue + } + + if repo.OwnerName == "" { + // v120.go migration may not have been run correctly - we'll just replicate it here + // because this appears to be a common-ish problem. + if _, err := sess.Exec("UPDATE repository SET owner_name = (SELECT name FROM `user` WHERE `user`.id = repository.owner_id)"); err != nil { return err - } else if !has { - log.Warn("Release[%d] is orphaned and refers to non-existing repository %d", release.ID, release.RepoID) - log.Warn("This release should be deleted") - continue } - if repo.OwnerName == "" { - // v120.go migration may not have been run correctly - we'll just replicate it here - // because this appears to be a common-ish problem. - if _, err := sess.Exec("UPDATE repository SET owner_name = (SELECT name FROM `user` WHERE `user`.id = repository.owner_id)"); err != nil { - return err - } - - if _, err := sess.ID(release.RepoID).Get(repo); err != nil { - return err - } + if _, err := sess.ID(release.RepoID).Get(repo); err != nil { + return err } - - repoCache[release.RepoID] = repo } - - if repo.OwnerName == "" { - log.Warn("Release[%d] refers to Repo[%d] Name: %s with OwnerID: %d but does not have OwnerName set", release.ID, release.RepoID, repo.Name, repo.OwnerID) - continue - } - gitRepo, err = git.OpenRepository(repoPath(repo.OwnerName, repo.Name)) if err != nil { return err } - gitRepoCache[release.RepoID] = gitRepo } commit, err := gitRepo.GetTagCommit(release.TagName)