From ac61c17c14df46425337e7d8d93e0a988d623c68 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 15 Apr 2022 17:33:11 -0400 Subject: [PATCH 01/31] Update pushmirror model to allow sync-on-push --- models/repo/pushmirror.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/repo/pushmirror.go b/models/repo/pushmirror.go index 048c0c3487b75..776c4b2e6edf4 100644 --- a/models/repo/pushmirror.go +++ b/models/repo/pushmirror.go @@ -23,6 +23,7 @@ type PushMirror struct { Repo *Repository `xorm:"-"` RemoteName string + SyncOnPush bool Interval time.Duration CreatedUnix timeutil.TimeStamp `xorm:"created"` LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` From 04d7bd5fd07e1caf0df929c8778b1dccb01e570c Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 15 Apr 2022 17:42:45 -0400 Subject: [PATCH 02/31] Move mirror push functions to a pushmirror service The functions related push mirrors will be called by notification module, when sync-on-push is supported. So relevant functions are moved to pushmirror service package to avoid circular imports. --- services/mirror/mirror_push.go | 248 +--------------------------- services/pushmirror/mirror_push.go | 256 +++++++++++++++++++++++++++++ 2 files changed, 262 insertions(+), 242 deletions(-) create mode 100644 services/pushmirror/mirror_push.go diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 2927bed72b279..6d0df8a7ff3bb 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -5,252 +5,16 @@ package mirror import ( - "context" - "errors" - "fmt" - "io" - "regexp" - "strings" - "time" - - repo_model "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/lfs" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/process" - "code.gitea.io/gitea/modules/repository" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" + // Implementations related to push mirrors are in `services/pushmirror` package + // to avoid circular imports + "code.gitea.io/gitea/services/pushmirror" ) -var stripExitStatus = regexp.MustCompile(`exit status \d+ - `) - // AddPushMirrorRemote registers the push mirror remote. -func AddPushMirrorRemote(ctx context.Context, m *repo_model.PushMirror, addr string) error { - addRemoteAndConfig := func(addr, path string) error { - cmd := git.NewCommand(ctx, "remote", "add", "--mirror=push", m.RemoteName, addr) - if strings.Contains(addr, "://") && strings.Contains(addr, "@") { - cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=push %s [repo_path: %s]", m.RemoteName, util.SanitizeCredentialURLs(addr), path)) - } else { - cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=push %s [repo_path: %s]", m.RemoteName, addr, path)) - } - if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: path}); err != nil { - return err - } - if _, _, err := git.NewCommand(ctx, "config", "--add", "remote."+m.RemoteName+".push", "+refs/heads/*:refs/heads/*").RunStdString(&git.RunOpts{Dir: path}); err != nil { - return err - } - if _, _, err := git.NewCommand(ctx, "config", "--add", "remote."+m.RemoteName+".push", "+refs/tags/*:refs/tags/*").RunStdString(&git.RunOpts{Dir: path}); err != nil { - return err - } - return nil - } - - if err := addRemoteAndConfig(addr, m.Repo.RepoPath()); err != nil { - return err - } - - if m.Repo.HasWiki() { - wikiRemoteURL := repository.WikiRemoteURL(ctx, addr) - if len(wikiRemoteURL) > 0 { - if err := addRemoteAndConfig(wikiRemoteURL, m.Repo.WikiPath()); err != nil { - return err - } - } - } - - return nil -} +var AddPushMirrorRemote = pushmirror.AddPushMirrorRemote // RemovePushMirrorRemote removes the push mirror remote. -func RemovePushMirrorRemote(ctx context.Context, m *repo_model.PushMirror) error { - cmd := git.NewCommand(ctx, "remote", "rm", m.RemoteName) - _ = m.GetRepository() - - if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: m.Repo.RepoPath()}); err != nil { - return err - } - - if m.Repo.HasWiki() { - if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: m.Repo.WikiPath()}); err != nil { - // The wiki remote may not exist - log.Warn("Wiki Remote[%d] could not be removed: %v", m.ID, err) - } - } - - return nil -} +var RemovePushMirrorRemote = pushmirror.RemovePushMirrorRemote // SyncPushMirror starts the sync of the push mirror and schedules the next run. -func SyncPushMirror(ctx context.Context, mirrorID int64) bool { - log.Trace("SyncPushMirror [mirror: %d]", mirrorID) - defer func() { - err := recover() - if err == nil { - return - } - // There was a panic whilst syncPushMirror... - log.Error("PANIC whilst syncPushMirror[%d] Panic: %v\nStacktrace: %s", mirrorID, err, log.Stack(2)) - }() - - m, err := repo_model.GetPushMirrorByID(mirrorID) - if err != nil { - log.Error("GetPushMirrorByID [%d]: %v", mirrorID, err) - return false - } - - _ = m.GetRepository() - - m.LastError = "" - - ctx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Syncing PushMirror %s/%s to %s", m.Repo.OwnerName, m.Repo.Name, m.RemoteName)) - defer finished() - - log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Running Sync", m.ID, m.Repo) - err = runPushSync(ctx, m) - if err != nil { - log.Error("SyncPushMirror [mirror: %d][repo: %-v]: %v", m.ID, m.Repo, err) - m.LastError = stripExitStatus.ReplaceAllLiteralString(err.Error(), "") - } - - m.LastUpdateUnix = timeutil.TimeStampNow() - - if err := repo_model.UpdatePushMirror(m); err != nil { - log.Error("UpdatePushMirror [%d]: %v", m.ID, err) - - return false - } - - log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Finished", m.ID, m.Repo) - - return err == nil -} - -func runPushSync(ctx context.Context, m *repo_model.PushMirror) error { - timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second - - performPush := func(path string) error { - remoteURL, err := git.GetRemoteURL(ctx, path, m.RemoteName) - if err != nil { - log.Error("GetRemoteAddress(%s) Error %v", path, err) - return errors.New("Unexpected error") - } - - if setting.LFS.StartServer { - log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) - - gitRepo, err := git.OpenRepository(ctx, path) - if err != nil { - log.Error("OpenRepository: %v", err) - return errors.New("Unexpected error") - } - defer gitRepo.Close() - - endpoint := lfs.DetermineEndpoint(remoteURL.String(), "") - lfsClient := lfs.NewClient(endpoint, nil) - if err := pushAllLFSObjects(ctx, gitRepo, lfsClient); err != nil { - return util.SanitizeErrorCredentialURLs(err) - } - } - - log.Trace("Pushing %s mirror[%d] remote %s", path, m.ID, m.RemoteName) - - if err := git.Push(ctx, path, git.PushOptions{ - Remote: m.RemoteName, - Force: true, - Mirror: true, - Timeout: timeout, - }); err != nil { - log.Error("Error pushing %s mirror[%d] remote %s: %v", path, m.ID, m.RemoteName, err) - - return util.SanitizeErrorCredentialURLs(err) - } - - return nil - } - - err := performPush(m.Repo.RepoPath()) - if err != nil { - return err - } - - if m.Repo.HasWiki() { - wikiPath := m.Repo.WikiPath() - _, err := git.GetRemoteAddress(ctx, wikiPath, m.RemoteName) - if err == nil { - err := performPush(wikiPath) - if err != nil { - return err - } - } else { - log.Trace("Skipping wiki: No remote configured") - } - } - - return nil -} - -func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, lfsClient lfs.Client) error { - contentStore := lfs.NewContentStore() - - pointerChan := make(chan lfs.PointerBlob) - errChan := make(chan error, 1) - go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) - - uploadObjects := func(pointers []lfs.Pointer) error { - err := lfsClient.Upload(ctx, pointers, func(p lfs.Pointer, objectError error) (io.ReadCloser, error) { - if objectError != nil { - return nil, objectError - } - - content, err := contentStore.Get(p) - if err != nil { - log.Error("Error reading LFS object %v: %v", p, err) - } - return content, err - }) - if err != nil { - select { - case <-ctx.Done(): - return nil - default: - } - } - return err - } - - var batch []lfs.Pointer - for pointerBlob := range pointerChan { - exists, err := contentStore.Exists(pointerBlob.Pointer) - if err != nil { - log.Error("Error checking if LFS object %v exists: %v", pointerBlob.Pointer, err) - return err - } - if !exists { - log.Trace("Skipping missing LFS object %v", pointerBlob.Pointer) - continue - } - - batch = append(batch, pointerBlob.Pointer) - if len(batch) >= lfsClient.BatchSize() { - if err := uploadObjects(batch); err != nil { - return err - } - batch = nil - } - } - if len(batch) > 0 { - if err := uploadObjects(batch); err != nil { - return err - } - } - - err, has := <-errChan - if has { - log.Error("Error enumerating LFS objects for repository: %v", err) - return err - } - - return nil -} +var SyncPushMirror = pushmirror.SyncPushMirror diff --git a/services/pushmirror/mirror_push.go b/services/pushmirror/mirror_push.go new file mode 100644 index 0000000000000..70c7a11b9a42e --- /dev/null +++ b/services/pushmirror/mirror_push.go @@ -0,0 +1,256 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package pushmirror + +import ( + "context" + "errors" + "fmt" + "io" + "regexp" + "strings" + "time" + + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" +) + +var stripExitStatus = regexp.MustCompile(`exit status \d+ - `) + +// AddPushMirrorRemote registers the push mirror remote. +func AddPushMirrorRemote(ctx context.Context, m *repo_model.PushMirror, addr string) error { + addRemoteAndConfig := func(addr, path string) error { + cmd := git.NewCommand(ctx, "remote", "add", "--mirror=push", m.RemoteName, addr) + if strings.Contains(addr, "://") && strings.Contains(addr, "@") { + cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=push %s [repo_path: %s]", m.RemoteName, util.SanitizeCredentialURLs(addr), path)) + } else { + cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=push %s [repo_path: %s]", m.RemoteName, addr, path)) + } + if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: path}); err != nil { + return err + } + if _, _, err := git.NewCommand(ctx, "config", "--add", "remote."+m.RemoteName+".push", "+refs/heads/*:refs/heads/*").RunStdString(&git.RunOpts{Dir: path}); err != nil { + return err + } + if _, _, err := git.NewCommand(ctx, "config", "--add", "remote."+m.RemoteName+".push", "+refs/tags/*:refs/tags/*").RunStdString(&git.RunOpts{Dir: path}); err != nil { + return err + } + return nil + } + + if err := addRemoteAndConfig(addr, m.Repo.RepoPath()); err != nil { + return err + } + + if m.Repo.HasWiki() { + wikiRemoteURL := repository.WikiRemoteURL(ctx, addr) + if len(wikiRemoteURL) > 0 { + if err := addRemoteAndConfig(wikiRemoteURL, m.Repo.WikiPath()); err != nil { + return err + } + } + } + + return nil +} + +// RemovePushMirrorRemote removes the push mirror remote. +func RemovePushMirrorRemote(ctx context.Context, m *repo_model.PushMirror) error { + cmd := git.NewCommand(ctx, "remote", "rm", m.RemoteName) + _ = m.GetRepository() + + if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: m.Repo.RepoPath()}); err != nil { + return err + } + + if m.Repo.HasWiki() { + if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: m.Repo.WikiPath()}); err != nil { + // The wiki remote may not exist + log.Warn("Wiki Remote[%d] could not be removed: %v", m.ID, err) + } + } + + return nil +} + +// SyncPushMirror starts the sync of the push mirror and schedules the next run. +func SyncPushMirror(ctx context.Context, mirrorID int64) bool { + log.Trace("SyncPushMirror [mirror: %d]", mirrorID) + defer func() { + err := recover() + if err == nil { + return + } + // There was a panic whilst syncPushMirror... + log.Error("PANIC whilst syncPushMirror[%d] Panic: %v\nStacktrace: %s", mirrorID, err, log.Stack(2)) + }() + + m, err := repo_model.GetPushMirrorByID(mirrorID) + if err != nil { + log.Error("GetPushMirrorByID [%d]: %v", mirrorID, err) + return false + } + + _ = m.GetRepository() + + m.LastError = "" + + ctx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Syncing PushMirror %s/%s to %s", m.Repo.OwnerName, m.Repo.Name, m.RemoteName)) + defer finished() + + log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Running Sync", m.ID, m.Repo) + err = runPushSync(ctx, m) + if err != nil { + log.Error("SyncPushMirror [mirror: %d][repo: %-v]: %v", m.ID, m.Repo, err) + m.LastError = stripExitStatus.ReplaceAllLiteralString(err.Error(), "") + } + + m.LastUpdateUnix = timeutil.TimeStampNow() + + if err := repo_model.UpdatePushMirror(m); err != nil { + log.Error("UpdatePushMirror [%d]: %v", m.ID, err) + + return false + } + + log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Finished", m.ID, m.Repo) + + return err == nil +} + +func runPushSync(ctx context.Context, m *repo_model.PushMirror) error { + timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second + + performPush := func(path string) error { + remoteURL, err := git.GetRemoteURL(ctx, path, m.RemoteName) + if err != nil { + log.Error("GetRemoteAddress(%s) Error %v", path, err) + return errors.New("Unexpected error") + } + + if setting.LFS.StartServer { + log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) + + gitRepo, err := git.OpenRepository(ctx, path) + if err != nil { + log.Error("OpenRepository: %v", err) + return errors.New("Unexpected error") + } + defer gitRepo.Close() + + endpoint := lfs.DetermineEndpoint(remoteURL.String(), "") + lfsClient := lfs.NewClient(endpoint, nil) + if err := pushAllLFSObjects(ctx, gitRepo, lfsClient); err != nil { + return util.SanitizeErrorCredentialURLs(err) + } + } + + log.Trace("Pushing %s mirror[%d] remote %s", path, m.ID, m.RemoteName) + + if err := git.Push(ctx, path, git.PushOptions{ + Remote: m.RemoteName, + Force: true, + Mirror: true, + Timeout: timeout, + }); err != nil { + log.Error("Error pushing %s mirror[%d] remote %s: %v", path, m.ID, m.RemoteName, err) + + return util.SanitizeErrorCredentialURLs(err) + } + + return nil + } + + err := performPush(m.Repo.RepoPath()) + if err != nil { + return err + } + + if m.Repo.HasWiki() { + wikiPath := m.Repo.WikiPath() + _, err := git.GetRemoteAddress(ctx, wikiPath, m.RemoteName) + if err == nil { + err := performPush(wikiPath) + if err != nil { + return err + } + } else { + log.Trace("Skipping wiki: No remote configured") + } + } + + return nil +} + +func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, lfsClient lfs.Client) error { + contentStore := lfs.NewContentStore() + + pointerChan := make(chan lfs.PointerBlob) + errChan := make(chan error, 1) + go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) + + uploadObjects := func(pointers []lfs.Pointer) error { + err := lfsClient.Upload(ctx, pointers, func(p lfs.Pointer, objectError error) (io.ReadCloser, error) { + if objectError != nil { + return nil, objectError + } + + content, err := contentStore.Get(p) + if err != nil { + log.Error("Error reading LFS object %v: %v", p, err) + } + return content, err + }) + if err != nil { + select { + case <-ctx.Done(): + return nil + default: + } + } + return err + } + + var batch []lfs.Pointer + for pointerBlob := range pointerChan { + exists, err := contentStore.Exists(pointerBlob.Pointer) + if err != nil { + log.Error("Error checking if LFS object %v exists: %v", pointerBlob.Pointer, err) + return err + } + if !exists { + log.Trace("Skipping missing LFS object %v", pointerBlob.Pointer) + continue + } + + batch = append(batch, pointerBlob.Pointer) + if len(batch) >= lfsClient.BatchSize() { + if err := uploadObjects(batch); err != nil { + return err + } + batch = nil + } + } + if len(batch) > 0 { + if err := uploadObjects(batch); err != nil { + return err + } + } + + err, has := <-errChan + if has { + log.Error("Error enumerating LFS objects for repository: %v", err) + return err + } + + return nil +} From bd5541101a2179657485f267a174f006cb3a0ef5 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 15 Apr 2022 18:04:51 -0400 Subject: [PATCH 03/31] Implement sync-on-push via a mirrorNotifier --- modules/notification/mirror/mirror.go | 44 +++++++++++++++++++++++++++ modules/notification/notification.go | 2 ++ 2 files changed, 46 insertions(+) create mode 100644 modules/notification/mirror/mirror.go diff --git a/modules/notification/mirror/mirror.go b/modules/notification/mirror/mirror.go new file mode 100644 index 0000000000000..1850aeb1022d9 --- /dev/null +++ b/modules/notification/mirror/mirror.go @@ -0,0 +1,44 @@ +package mirror + +import ( + "fmt" + + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/notification/base" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/repository" + pushmirror_service "code.gitea.io/gitea/services/pushmirror" +) + +type mirrorNotifier struct { + base.NullNotifier +} + +var _ base.Notifier = &mirrorNotifier{} + +// NewNotifier create a new mirrorNotifier notifier +func NewNotifier() base.Notifier { + return &mirrorNotifier{} +} + +func (m *mirrorNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) { + ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("mirrorNotifier.NotifyPushCommits User: %s[%d] in %s[%d]", pusher.Name, pusher.ID, repo.FullName(), repo.ID)) + defer finished() + + pushMirrors, err := repo_model.GetPushMirrorsByRepoID(repo.ID) + if err != nil { + log.Error("repo_model.GetPushMirrorsByRepoID failed: %v", err) + return + } + + for _, mirror := range pushMirrors { + if mirror.SyncOnPush { + // TODO: push mirror likely will benefit from using a queue + pushmirror_service.SyncPushMirror(ctx, mirror.ID) + } + } + return +} diff --git a/modules/notification/notification.go b/modules/notification/notification.go index d60a880bec6a2..bdfed90b7864e 100644 --- a/modules/notification/notification.go +++ b/modules/notification/notification.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/notification/base" "code.gitea.io/gitea/modules/notification/indexer" "code.gitea.io/gitea/modules/notification/mail" + "code.gitea.io/gitea/modules/notification/mirror" "code.gitea.io/gitea/modules/notification/ui" "code.gitea.io/gitea/modules/notification/webhook" "code.gitea.io/gitea/modules/repository" @@ -37,6 +38,7 @@ func NewContext() { RegisterNotifier(indexer.NewNotifier()) RegisterNotifier(webhook.NewNotifier()) RegisterNotifier(action.NewNotifier()) + RegisterNotifier(mirror.NewNotifier()) } // NotifyCreateIssueComment notifies issue comment related message to notifiers From 8f87595ea095427f8cbb498a4619118bc5806672 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 15 Apr 2022 18:29:37 -0400 Subject: [PATCH 04/31] Fix lint-backend issues --- modules/notification/mirror/mirror.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/notification/mirror/mirror.go b/modules/notification/mirror/mirror.go index 1850aeb1022d9..8af43feafd6a2 100644 --- a/modules/notification/mirror/mirror.go +++ b/modules/notification/mirror/mirror.go @@ -1,3 +1,7 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + package mirror import ( @@ -40,5 +44,4 @@ func (m *mirrorNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_m pushmirror_service.SyncPushMirror(ctx, mirror.ID) } } - return } From eb25993ce8096de24d7e5b53626fd5f87a5cea72 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 15 Apr 2022 18:39:34 -0400 Subject: [PATCH 05/31] Add migration for PushMirror struct --- models/migrations/migrations.go | 2 ++ models/migrations/v218.go | 35 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 models/migrations/v218.go diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index edd4beb451bc5..22b12569a3353 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -391,6 +391,8 @@ var migrations = []Migration{ NewMigration("Improve Action table indices", improveActionTableIndices), // v217 -> v218 NewMigration("Alter hook_task table TEXT fields to LONGTEXT", alterHookTaskTextFieldsToLongText), + // v218 -> v219 + NewMigration("Add sync_on_push column to push_mirror table", addSyncOnPushColForPushMirror), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v218.go b/models/migrations/v218.go new file mode 100644 index 0000000000000..626e92a1ce0ba --- /dev/null +++ b/models/migrations/v218.go @@ -0,0 +1,35 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + "time" + + "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/timeutil" + "xorm.io/xorm" +) + +func addSyncOnPushColForPushMirror(x *xorm.Engine) error { + type PushMirror struct { + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"INDEX"` + Repo *repo.Repository `xorm:"-"` + RemoteName string + + SyncOnPush bool + Interval time.Duration + CreatedUnix timeutil.TimeStamp `xorm:"created"` + LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` + LastError string `xorm:"text"` + } + + if err := x.Sync2(new(PushMirror)); err != nil { + return fmt.Errorf("sync2: %v", err) + } + + return nil +} From bcabe51739b49db46908cb320504b37b0b712749 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 17 Apr 2022 18:52:04 -0400 Subject: [PATCH 06/31] Improve filtering for push mirrors --- models/repo/pushmirror.go | 9 +++++++++ modules/notification/mirror/mirror.go | 11 +++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/models/repo/pushmirror.go b/models/repo/pushmirror.go index 776c4b2e6edf4..502458bef6330 100644 --- a/models/repo/pushmirror.go +++ b/models/repo/pushmirror.go @@ -94,6 +94,15 @@ func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) { return mirrors, db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID).Find(&mirrors) } +// GetPushMirrorsByRepoIDWithSyncOnPush returns push-mirror information of a repository, +// filtered by sync_on_push. +func GetPushMirrorsByRepoIDWithSyncOnPush(repoID int64, syncOnPush bool) ([]*PushMirror, error) { + mirrors := make([]*PushMirror, 0, 10) + return mirrors, db.GetEngine(db.DefaultContext). + Where("repo_id=? AND sync_on_push=?", repoID, syncOnPush). + Find(&mirrors) +} + // PushMirrorsIterate iterates all push-mirror repositories. func PushMirrorsIterate(limit int, f func(idx int, bean interface{}) error) error { return db.GetEngine(db.DefaultContext). diff --git a/modules/notification/mirror/mirror.go b/modules/notification/mirror/mirror.go index 8af43feafd6a2..7a8ab9da94b1b 100644 --- a/modules/notification/mirror/mirror.go +++ b/modules/notification/mirror/mirror.go @@ -32,16 +32,15 @@ func (m *mirrorNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_m ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("mirrorNotifier.NotifyPushCommits User: %s[%d] in %s[%d]", pusher.Name, pusher.ID, repo.FullName(), repo.ID)) defer finished() - pushMirrors, err := repo_model.GetPushMirrorsByRepoID(repo.ID) + syncOnPush := true + pushMirrors, err := repo_model.GetPushMirrorsByRepoIDWithSyncOnPush(repo.ID, syncOnPush) if err != nil { - log.Error("repo_model.GetPushMirrorsByRepoID failed: %v", err) + log.Error("repo_model.GetPushMirrorsByRepoIDWithSyncOnPush failed: %v", err) return } for _, mirror := range pushMirrors { - if mirror.SyncOnPush { - // TODO: push mirror likely will benefit from using a queue - pushmirror_service.SyncPushMirror(ctx, mirror.ID) - } + // TODO: push mirror likely will benefit from using a queue + pushmirror_service.SyncPushMirror(ctx, mirror.ID) } } From c6a4f305bc94ab94b5665b546297dc70d7a9fea9 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 3 Jun 2022 01:22:38 -0400 Subject: [PATCH 07/31] Refactor to use modules/mirror Some functions in services/mirror are moved to modules/mirror package. This approach solves the circular imports problem without introducting a seperate pushmirror package. --- .../mirror}/mirror_push.go | 64 +----------------- modules/notification/mirror/mirror.go | 4 +- services/mirror/mirror.go | 3 +- services/mirror/mirror_push.go | 67 +++++++++++++++++-- 4 files changed, 65 insertions(+), 73 deletions(-) rename {services/pushmirror => modules/mirror}/mirror_push.go (68%) diff --git a/services/pushmirror/mirror_push.go b/modules/mirror/mirror_push.go similarity index 68% rename from services/pushmirror/mirror_push.go rename to modules/mirror/mirror_push.go index 70c7a11b9a42e..1f76af1c8fde1 100644 --- a/services/pushmirror/mirror_push.go +++ b/modules/mirror/mirror_push.go @@ -1,8 +1,4 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package pushmirror +package mirror import ( "context" @@ -10,7 +6,6 @@ import ( "fmt" "io" "regexp" - "strings" "time" repo_model "code.gitea.io/gitea/models/repo" @@ -18,7 +13,6 @@ import ( "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" - "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" @@ -26,62 +20,6 @@ import ( var stripExitStatus = regexp.MustCompile(`exit status \d+ - `) -// AddPushMirrorRemote registers the push mirror remote. -func AddPushMirrorRemote(ctx context.Context, m *repo_model.PushMirror, addr string) error { - addRemoteAndConfig := func(addr, path string) error { - cmd := git.NewCommand(ctx, "remote", "add", "--mirror=push", m.RemoteName, addr) - if strings.Contains(addr, "://") && strings.Contains(addr, "@") { - cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=push %s [repo_path: %s]", m.RemoteName, util.SanitizeCredentialURLs(addr), path)) - } else { - cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=push %s [repo_path: %s]", m.RemoteName, addr, path)) - } - if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: path}); err != nil { - return err - } - if _, _, err := git.NewCommand(ctx, "config", "--add", "remote."+m.RemoteName+".push", "+refs/heads/*:refs/heads/*").RunStdString(&git.RunOpts{Dir: path}); err != nil { - return err - } - if _, _, err := git.NewCommand(ctx, "config", "--add", "remote."+m.RemoteName+".push", "+refs/tags/*:refs/tags/*").RunStdString(&git.RunOpts{Dir: path}); err != nil { - return err - } - return nil - } - - if err := addRemoteAndConfig(addr, m.Repo.RepoPath()); err != nil { - return err - } - - if m.Repo.HasWiki() { - wikiRemoteURL := repository.WikiRemoteURL(ctx, addr) - if len(wikiRemoteURL) > 0 { - if err := addRemoteAndConfig(wikiRemoteURL, m.Repo.WikiPath()); err != nil { - return err - } - } - } - - return nil -} - -// RemovePushMirrorRemote removes the push mirror remote. -func RemovePushMirrorRemote(ctx context.Context, m *repo_model.PushMirror) error { - cmd := git.NewCommand(ctx, "remote", "rm", m.RemoteName) - _ = m.GetRepository() - - if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: m.Repo.RepoPath()}); err != nil { - return err - } - - if m.Repo.HasWiki() { - if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: m.Repo.WikiPath()}); err != nil { - // The wiki remote may not exist - log.Warn("Wiki Remote[%d] could not be removed: %v", m.ID, err) - } - } - - return nil -} - // SyncPushMirror starts the sync of the push mirror and schedules the next run. func SyncPushMirror(ctx context.Context, mirrorID int64) bool { log.Trace("SyncPushMirror [mirror: %d]", mirrorID) diff --git a/modules/notification/mirror/mirror.go b/modules/notification/mirror/mirror.go index 7a8ab9da94b1b..7727bf30001fc 100644 --- a/modules/notification/mirror/mirror.go +++ b/modules/notification/mirror/mirror.go @@ -11,10 +11,10 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" + mirror_module "code.gitea.io/gitea/modules/mirror" "code.gitea.io/gitea/modules/notification/base" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/repository" - pushmirror_service "code.gitea.io/gitea/services/pushmirror" ) type mirrorNotifier struct { @@ -41,6 +41,6 @@ func (m *mirrorNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_m for _, mirror := range pushMirrors { // TODO: push mirror likely will benefit from using a queue - pushmirror_service.SyncPushMirror(ctx, mirror.ID) + mirror_module.SyncPushMirror(ctx, mirror.ID) } } diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 013adac0f4ed1..c7a5107fe0f37 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -11,6 +11,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" + mirror_module "code.gitea.io/gitea/modules/mirror" "code.gitea.io/gitea/modules/queue" "code.gitea.io/gitea/modules/setting" ) @@ -41,7 +42,7 @@ func doMirrorSync(ctx context.Context, req *SyncRequest) { } switch req.Type { case PushMirrorType: - _ = SyncPushMirror(ctx, req.ReferenceID) + _ = mirror_module.SyncPushMirror(ctx, req.ReferenceID) case PullMirrorType: _ = SyncPullMirror(ctx, req.ReferenceID) default: diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 6d0df8a7ff3bb..a52eb76d525f3 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -5,16 +5,69 @@ package mirror import ( - // Implementations related to push mirrors are in `services/pushmirror` package - // to avoid circular imports - "code.gitea.io/gitea/services/pushmirror" + "context" + "fmt" + "strings" + + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/util" ) // AddPushMirrorRemote registers the push mirror remote. -var AddPushMirrorRemote = pushmirror.AddPushMirrorRemote +func AddPushMirrorRemote(ctx context.Context, m *repo_model.PushMirror, addr string) error { + addRemoteAndConfig := func(addr, path string) error { + cmd := git.NewCommand(ctx, "remote", "add", "--mirror=push", m.RemoteName, addr) + if strings.Contains(addr, "://") && strings.Contains(addr, "@") { + cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=push %s [repo_path: %s]", m.RemoteName, util.SanitizeCredentialURLs(addr), path)) + } else { + cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=push %s [repo_path: %s]", m.RemoteName, addr, path)) + } + if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: path}); err != nil { + return err + } + if _, _, err := git.NewCommand(ctx, "config", "--add", "remote."+m.RemoteName+".push", "+refs/heads/*:refs/heads/*").RunStdString(&git.RunOpts{Dir: path}); err != nil { + return err + } + if _, _, err := git.NewCommand(ctx, "config", "--add", "remote."+m.RemoteName+".push", "+refs/tags/*:refs/tags/*").RunStdString(&git.RunOpts{Dir: path}); err != nil { + return err + } + return nil + } + + if err := addRemoteAndConfig(addr, m.Repo.RepoPath()); err != nil { + return err + } + + if m.Repo.HasWiki() { + wikiRemoteURL := repository.WikiRemoteURL(ctx, addr) + if len(wikiRemoteURL) > 0 { + if err := addRemoteAndConfig(wikiRemoteURL, m.Repo.WikiPath()); err != nil { + return err + } + } + } + + return nil +} // RemovePushMirrorRemote removes the push mirror remote. -var RemovePushMirrorRemote = pushmirror.RemovePushMirrorRemote +func RemovePushMirrorRemote(ctx context.Context, m *repo_model.PushMirror) error { + cmd := git.NewCommand(ctx, "remote", "rm", m.RemoteName) + _ = m.GetRepository() + + if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: m.Repo.RepoPath()}); err != nil { + return err + } + + if m.Repo.HasWiki() { + if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: m.Repo.WikiPath()}); err != nil { + // The wiki remote may not exist + log.Warn("Wiki Remote[%d] could not be removed: %v", m.ID, err) + } + } -// SyncPushMirror starts the sync of the push mirror and schedules the next run. -var SyncPushMirror = pushmirror.SyncPushMirror + return nil +} From 2978942cb2f414ec4f8cb086196b9ee5027e189c Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 3 Jun 2022 01:31:44 -0400 Subject: [PATCH 08/31] Update tests after moving code to modules/mirror --- integrations/mirror_push_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integrations/mirror_push_test.go b/integrations/mirror_push_test.go index a73b69e7869dd..47af00f2d802a 100644 --- a/integrations/mirror_push_test.go +++ b/integrations/mirror_push_test.go @@ -17,10 +17,10 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + mirror_module "code.gitea.io/gitea/modules/mirror" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/services/migrations" - mirror_service "code.gitea.io/gitea/services/mirror" "github.com/stretchr/testify/assert" ) @@ -51,7 +51,7 @@ func testMirrorPush(t *testing.T, u *url.URL) { assert.NoError(t, err) assert.Len(t, mirrors, 1) - ok := mirror_service.SyncPushMirror(context.Background(), mirrors[0].ID) + ok := mirror_module.SyncPushMirror(context.Background(), mirrors[0].ID) assert.True(t, ok) srcGitRepo, err := git.OpenRepository(git.DefaultContext, srcRepo.RepoPath()) From 6c0e7219d8c11d0d9c68964cdc92119c8c2e3cdc Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 3 Jun 2022 01:55:37 -0400 Subject: [PATCH 09/31] Include copyright --- modules/mirror/mirror_push.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/mirror/mirror_push.go b/modules/mirror/mirror_push.go index 1f76af1c8fde1..5d732bfe30f2a 100644 --- a/modules/mirror/mirror_push.go +++ b/modules/mirror/mirror_push.go @@ -1,3 +1,7 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + package mirror import ( From 08364738c981361ec841f14d46dc627d80af0f02 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Wed, 29 Jun 2022 03:31:22 -0400 Subject: [PATCH 10/31] Add 'sync on commit' checkbox in web UI --- options/locale/locale_en-US.ini | 3 ++- templates/repo/settings/options.tmpl | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index eb7ae4774313b..1834a369070fc 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -861,8 +861,9 @@ default_branch = Default Branch default_branch_helper = The default branch is the base branch for pull requests and code commits. mirror_prune = Prune mirror_prune_desc = Remove obsolete remote-tracking references -mirror_interval = Mirror Interval (valid time units are 'h', 'm', 's'). 0 to disable automatic sync. (Minimum interval: %s) +mirror_interval = Mirror Interval (valid time units are 'h', 'm', 's'). 0 to disable periodic sync. (Minimum interval: %s) mirror_interval_invalid = The mirror interval is not valid. +mirror_sync_on_commit = Sync when new commit is pushed mirror_address = Clone From URL mirror_address_desc = Put any required credentials in the Authorization section. mirror_address_url_invalid = The provided url is invalid. You must escape all components of the url correctly. diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index e76aba761a28d..944ece8a36493 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -219,6 +219,12 @@ +
+
+ + +
+
From 5a8d2b537c99129530902686304a2044a38f845d Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Wed, 29 Jun 2022 03:46:33 -0400 Subject: [PATCH 11/31] Update checkbox input --- templates/repo/settings/options.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 944ece8a36493..664261d2c66c4 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -221,7 +221,7 @@
- +
From 88f50e9efdd4017a651c06a9a4f5957625b29339 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Wed, 29 Jun 2022 03:46:53 -0400 Subject: [PATCH 12/31] Store sync on commit setting --- routers/web/repo/setting.go | 1 + services/forms/repo_form.go | 35 ++++++++++++++++++----------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index fae62c102077c..92792d65be7b3 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -360,6 +360,7 @@ func SettingsPost(ctx *context.Context) { RepoID: repo.ID, Repo: repo, RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix), + SyncOnPush: form.PushMirrorSyncOnCommit, Interval: interval, } if err := repo_model.InsertPushMirror(m); err != nil { diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index c9327bbd9b0f8..afecc205f31e8 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -115,23 +115,24 @@ func ParseRemoteAddr(remoteAddr, authUsername, authPassword string) (string, err // RepoSettingForm form for changing repository settings type RepoSettingForm struct { - RepoName string `binding:"Required;AlphaDashDot;MaxSize(100)"` - Description string `binding:"MaxSize(255)"` - Website string `binding:"ValidUrl;MaxSize(255)"` - Interval string - MirrorAddress string - MirrorUsername string - MirrorPassword string - LFS bool `form:"mirror_lfs"` - LFSEndpoint string `form:"mirror_lfs_endpoint"` - PushMirrorID string - PushMirrorAddress string - PushMirrorUsername string - PushMirrorPassword string - PushMirrorInterval string - Private bool - Template bool - EnablePrune bool + RepoName string `binding:"Required;AlphaDashDot;MaxSize(100)"` + Description string `binding:"MaxSize(255)"` + Website string `binding:"ValidUrl;MaxSize(255)"` + Interval string + MirrorAddress string + MirrorUsername string + MirrorPassword string + LFS bool `form:"mirror_lfs"` + LFSEndpoint string `form:"mirror_lfs_endpoint"` + PushMirrorID string + PushMirrorAddress string + PushMirrorUsername string + PushMirrorPassword string + PushMirrorSyncOnCommit bool + PushMirrorInterval string + Private bool + Template bool + EnablePrune bool // Advanced settings EnableWiki bool From c9d63935186674c1248c272dff3f9a22092361ed Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Wed, 29 Jun 2022 03:55:25 -0400 Subject: [PATCH 13/31] Use more generic name SyncOnCommit --- models/migrations/v218.go | 2 +- models/repo/pushmirror.go | 10 +++++----- modules/notification/mirror/mirror.go | 6 +++--- routers/web/repo/setting.go | 10 +++++----- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/models/migrations/v218.go b/models/migrations/v218.go index 626e92a1ce0ba..0b7f71feeb928 100644 --- a/models/migrations/v218.go +++ b/models/migrations/v218.go @@ -20,7 +20,7 @@ func addSyncOnPushColForPushMirror(x *xorm.Engine) error { Repo *repo.Repository `xorm:"-"` RemoteName string - SyncOnPush bool + SyncOnCommit bool Interval time.Duration CreatedUnix timeutil.TimeStamp `xorm:"created"` LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` diff --git a/models/repo/pushmirror.go b/models/repo/pushmirror.go index 502458bef6330..c5b8d51fd0944 100644 --- a/models/repo/pushmirror.go +++ b/models/repo/pushmirror.go @@ -23,7 +23,7 @@ type PushMirror struct { Repo *Repository `xorm:"-"` RemoteName string - SyncOnPush bool + SyncOnCommit bool Interval time.Duration CreatedUnix timeutil.TimeStamp `xorm:"created"` LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` @@ -94,12 +94,12 @@ func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) { return mirrors, db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID).Find(&mirrors) } -// GetPushMirrorsByRepoIDWithSyncOnPush returns push-mirror information of a repository, -// filtered by sync_on_push. -func GetPushMirrorsByRepoIDWithSyncOnPush(repoID int64, syncOnPush bool) ([]*PushMirror, error) { +// GetPushMirrorsByRepoIDWithSyncOnCommit returns push-mirror information of a repository, +// filtered by sync_on_commit. +func GetPushMirrorsByRepoIDWithSyncOnCommit(repoID int64, syncOnCommit bool) ([]*PushMirror, error) { mirrors := make([]*PushMirror, 0, 10) return mirrors, db.GetEngine(db.DefaultContext). - Where("repo_id=? AND sync_on_push=?", repoID, syncOnPush). + Where("repo_id=? AND sync_on_commit=?", repoID, syncOnCommit). Find(&mirrors) } diff --git a/modules/notification/mirror/mirror.go b/modules/notification/mirror/mirror.go index 7727bf30001fc..89ce11d70bcc1 100644 --- a/modules/notification/mirror/mirror.go +++ b/modules/notification/mirror/mirror.go @@ -32,10 +32,10 @@ func (m *mirrorNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_m ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("mirrorNotifier.NotifyPushCommits User: %s[%d] in %s[%d]", pusher.Name, pusher.ID, repo.FullName(), repo.ID)) defer finished() - syncOnPush := true - pushMirrors, err := repo_model.GetPushMirrorsByRepoIDWithSyncOnPush(repo.ID, syncOnPush) + syncOnCommit := true + pushMirrors, err := repo_model.GetPushMirrorsByRepoIDWithSyncOnCommit(repo.ID, syncOnCommit) if err != nil { - log.Error("repo_model.GetPushMirrorsByRepoIDWithSyncOnPush failed: %v", err) + log.Error("repo_model.GetPushMirrorsByRepoIDWithSyncOnCommit failed: %v", err) return } diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 92792d65be7b3..3a30bd02be0aa 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -357,11 +357,11 @@ func SettingsPost(ctx *context.Context) { } m := &repo_model.PushMirror{ - RepoID: repo.ID, - Repo: repo, - RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix), - SyncOnPush: form.PushMirrorSyncOnCommit, - Interval: interval, + RepoID: repo.ID, + Repo: repo, + RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix), + SyncOnCommit: form.PushMirrorSyncOnCommit, + Interval: interval, } if err := repo_model.InsertPushMirror(m); err != nil { ctx.ServerError("InsertPushMirror", err) From f2de0a090401f0e1860648169af8e0f8a27a0259 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Wed, 29 Jun 2022 23:14:13 -0400 Subject: [PATCH 14/31] Trigger mirror on pull requests --- modules/notification/mirror/mirror.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/modules/notification/mirror/mirror.go b/modules/notification/mirror/mirror.go index 89ce11d70bcc1..ead7fb8e2fa9c 100644 --- a/modules/notification/mirror/mirror.go +++ b/modules/notification/mirror/mirror.go @@ -5,8 +5,10 @@ package mirror import ( + "context" "fmt" + issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/graceful" @@ -32,8 +34,19 @@ func (m *mirrorNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_m ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("mirrorNotifier.NotifyPushCommits User: %s[%d] in %s[%d]", pusher.Name, pusher.ID, repo.FullName(), repo.ID)) defer finished() + syncPushMirrorWithSyncOnCommit(ctx, repo.ID) +} + +func (m *mirrorNotifier) NotifyMergePullRequest(pr *issues_model.PullRequest, doer *user_model.User) { + ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("webhook.NotifyMergePullRequest Pull[%d] #%d in [%d]", pr.ID, pr.Index, pr.BaseRepoID)) + defer finished() + + syncPushMirrorWithSyncOnCommit(ctx, pr.BaseRepoID) +} + +func syncPushMirrorWithSyncOnCommit(ctx context.Context, repoID int64) { syncOnCommit := true - pushMirrors, err := repo_model.GetPushMirrorsByRepoIDWithSyncOnCommit(repo.ID, syncOnCommit) + pushMirrors, err := repo_model.GetPushMirrorsByRepoIDWithSyncOnCommit(repoID, syncOnCommit) if err != nil { log.Error("repo_model.GetPushMirrorsByRepoIDWithSyncOnCommit failed: %v", err) return From 0f745bdef71c94c130503e1cdf26fb36952272ea Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Mon, 4 Jul 2022 13:13:39 -0400 Subject: [PATCH 15/31] Fix table column name --- models/migrations/migrations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 60490524a1468..a3af6ed990ccb 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -397,7 +397,7 @@ var migrations = []Migration{ // v218 -> v219 NewMigration("Improve Action table indices v2", improveActionTableIndices), // v219 -> v220 - NewMigration("Add sync_on_push column to push_mirror table", addSyncOnPushColForPushMirror), + NewMigration("Add sync_on_commit column to push_mirror table", addSyncOnPushColForPushMirror), } // GetCurrentDBVersion returns the current db version From 6f66d4493ece7ca0727634a21152f6e5b1225234 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Mon, 4 Jul 2022 13:21:46 -0400 Subject: [PATCH 16/31] Update sync_on_commit column to false --- models/migrations/v219.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/models/migrations/v219.go b/models/migrations/v219.go index 0b7f71feeb928..73c134fb07fb2 100644 --- a/models/migrations/v219.go +++ b/models/migrations/v219.go @@ -27,9 +27,19 @@ func addSyncOnPushColForPushMirror(x *xorm.Engine) error { LastError string `xorm:"text"` } - if err := x.Sync2(new(PushMirror)); err != nil { + session := x.NewSession() + defer session.Close() + if err := session.Begin(); err != nil { + return err + } + + if err := session.Sync2(new(PushMirror)); err != nil { return fmt.Errorf("sync2: %v", err) } - return nil + if _, err := session.Exec("UPDATE push_mirror SET sync_on_commit = 0"); err != nil { + return err + } + + return session.Commit() } From cb517b279325666d5952eb98b2d7f7afa280a0bf Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Mon, 4 Jul 2022 14:34:42 -0400 Subject: [PATCH 17/31] Make use of mirror queue --- integrations/mirror_push_test.go | 4 +- modules/mirror/mirror.go | 75 ++++++++++ modules/mirror/mirror_push.go | 198 -------------------------- modules/notification/mirror/mirror.go | 3 +- routers/api/v1/repo/mirror.go | 4 +- routers/web/repo/setting.go | 5 +- services/mirror/mirror.go | 88 ++---------- services/mirror/mirror_push.go | 183 ++++++++++++++++++++++++ 8 files changed, 281 insertions(+), 279 deletions(-) create mode 100644 modules/mirror/mirror.go delete mode 100644 modules/mirror/mirror_push.go diff --git a/integrations/mirror_push_test.go b/integrations/mirror_push_test.go index 47af00f2d802a..a73b69e7869dd 100644 --- a/integrations/mirror_push_test.go +++ b/integrations/mirror_push_test.go @@ -17,10 +17,10 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" - mirror_module "code.gitea.io/gitea/modules/mirror" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/services/migrations" + mirror_service "code.gitea.io/gitea/services/mirror" "github.com/stretchr/testify/assert" ) @@ -51,7 +51,7 @@ func testMirrorPush(t *testing.T, u *url.URL) { assert.NoError(t, err) assert.Len(t, mirrors, 1) - ok := mirror_module.SyncPushMirror(context.Background(), mirrors[0].ID) + ok := mirror_service.SyncPushMirror(context.Background(), mirrors[0].ID) assert.True(t, ok) srcGitRepo, err := git.OpenRepository(git.DefaultContext, srcRepo.RepoPath()) diff --git a/modules/mirror/mirror.go b/modules/mirror/mirror.go new file mode 100644 index 0000000000000..f3c56102dc9f9 --- /dev/null +++ b/modules/mirror/mirror.go @@ -0,0 +1,75 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package mirror + +import ( + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/queue" + "code.gitea.io/gitea/modules/setting" +) + +var mirrorQueue queue.UniqueQueue + +// SyncType type of sync request +type SyncType int + +const ( + // PullMirrorType for pull mirrors + PullMirrorType SyncType = iota + // PushMirrorType for push mirrors + PushMirrorType +) + +// SyncRequest for the mirror queue +type SyncRequest struct { + Type SyncType + ReferenceID int64 // RepoID for pull mirror, MirrorID fro push mirror +} + +// StartSyncMirrors starts a go routine to sync the mirrors +func StartSyncMirrors(queueHandle func(data ...queue.Data) []queue.Data) { + if !setting.Mirror.Enabled { + return + } + mirrorQueue = queue.CreateUniqueQueue("mirror", queueHandle, new(SyncRequest)) + + go graceful.GetManager().RunWithShutdownFns(mirrorQueue.Run) +} + +// AddPullMirrorToQueue adds repoID to mirror queue +func AddPullMirrorToQueue(repoID int64) { + if !setting.Mirror.Enabled { + return + } + go func() { + err := PushToQueue(PullMirrorType, repoID) + if err != nil { + log.Error("Unable to push sync request for to the queue for pull mirror repo[%d]: Error: %v", repoID, err) + return + } + }() +} + +// AddPushMirrorToQueue adds the push mirror to the queue +func AddPushMirrorToQueue(mirrorID int64) { + if !setting.Mirror.Enabled { + return + } + go func() { + err := PushToQueue(PushMirrorType, mirrorID) + if err != nil { + log.Error("Unable to push sync request to the queue for pull mirror repo[%d]: Error: %v", mirrorID, err) + } + }() +} + +// PushToQueue adds the sync request to the queue +func PushToQueue(mirrorType SyncType, referenceID int64) error { + return mirrorQueue.Push(&SyncRequest{ + Type: mirrorType, + ReferenceID: referenceID, + }) +} diff --git a/modules/mirror/mirror_push.go b/modules/mirror/mirror_push.go deleted file mode 100644 index 5d732bfe30f2a..0000000000000 --- a/modules/mirror/mirror_push.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package mirror - -import ( - "context" - "errors" - "fmt" - "io" - "regexp" - "time" - - repo_model "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/lfs" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/process" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" -) - -var stripExitStatus = regexp.MustCompile(`exit status \d+ - `) - -// SyncPushMirror starts the sync of the push mirror and schedules the next run. -func SyncPushMirror(ctx context.Context, mirrorID int64) bool { - log.Trace("SyncPushMirror [mirror: %d]", mirrorID) - defer func() { - err := recover() - if err == nil { - return - } - // There was a panic whilst syncPushMirror... - log.Error("PANIC whilst syncPushMirror[%d] Panic: %v\nStacktrace: %s", mirrorID, err, log.Stack(2)) - }() - - m, err := repo_model.GetPushMirrorByID(mirrorID) - if err != nil { - log.Error("GetPushMirrorByID [%d]: %v", mirrorID, err) - return false - } - - _ = m.GetRepository() - - m.LastError = "" - - ctx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Syncing PushMirror %s/%s to %s", m.Repo.OwnerName, m.Repo.Name, m.RemoteName)) - defer finished() - - log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Running Sync", m.ID, m.Repo) - err = runPushSync(ctx, m) - if err != nil { - log.Error("SyncPushMirror [mirror: %d][repo: %-v]: %v", m.ID, m.Repo, err) - m.LastError = stripExitStatus.ReplaceAllLiteralString(err.Error(), "") - } - - m.LastUpdateUnix = timeutil.TimeStampNow() - - if err := repo_model.UpdatePushMirror(m); err != nil { - log.Error("UpdatePushMirror [%d]: %v", m.ID, err) - - return false - } - - log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Finished", m.ID, m.Repo) - - return err == nil -} - -func runPushSync(ctx context.Context, m *repo_model.PushMirror) error { - timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second - - performPush := func(path string) error { - remoteURL, err := git.GetRemoteURL(ctx, path, m.RemoteName) - if err != nil { - log.Error("GetRemoteAddress(%s) Error %v", path, err) - return errors.New("Unexpected error") - } - - if setting.LFS.StartServer { - log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) - - gitRepo, err := git.OpenRepository(ctx, path) - if err != nil { - log.Error("OpenRepository: %v", err) - return errors.New("Unexpected error") - } - defer gitRepo.Close() - - endpoint := lfs.DetermineEndpoint(remoteURL.String(), "") - lfsClient := lfs.NewClient(endpoint, nil) - if err := pushAllLFSObjects(ctx, gitRepo, lfsClient); err != nil { - return util.SanitizeErrorCredentialURLs(err) - } - } - - log.Trace("Pushing %s mirror[%d] remote %s", path, m.ID, m.RemoteName) - - if err := git.Push(ctx, path, git.PushOptions{ - Remote: m.RemoteName, - Force: true, - Mirror: true, - Timeout: timeout, - }); err != nil { - log.Error("Error pushing %s mirror[%d] remote %s: %v", path, m.ID, m.RemoteName, err) - - return util.SanitizeErrorCredentialURLs(err) - } - - return nil - } - - err := performPush(m.Repo.RepoPath()) - if err != nil { - return err - } - - if m.Repo.HasWiki() { - wikiPath := m.Repo.WikiPath() - _, err := git.GetRemoteAddress(ctx, wikiPath, m.RemoteName) - if err == nil { - err := performPush(wikiPath) - if err != nil { - return err - } - } else { - log.Trace("Skipping wiki: No remote configured") - } - } - - return nil -} - -func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, lfsClient lfs.Client) error { - contentStore := lfs.NewContentStore() - - pointerChan := make(chan lfs.PointerBlob) - errChan := make(chan error, 1) - go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) - - uploadObjects := func(pointers []lfs.Pointer) error { - err := lfsClient.Upload(ctx, pointers, func(p lfs.Pointer, objectError error) (io.ReadCloser, error) { - if objectError != nil { - return nil, objectError - } - - content, err := contentStore.Get(p) - if err != nil { - log.Error("Error reading LFS object %v: %v", p, err) - } - return content, err - }) - if err != nil { - select { - case <-ctx.Done(): - return nil - default: - } - } - return err - } - - var batch []lfs.Pointer - for pointerBlob := range pointerChan { - exists, err := contentStore.Exists(pointerBlob.Pointer) - if err != nil { - log.Error("Error checking if LFS object %v exists: %v", pointerBlob.Pointer, err) - return err - } - if !exists { - log.Trace("Skipping missing LFS object %v", pointerBlob.Pointer) - continue - } - - batch = append(batch, pointerBlob.Pointer) - if len(batch) >= lfsClient.BatchSize() { - if err := uploadObjects(batch); err != nil { - return err - } - batch = nil - } - } - if len(batch) > 0 { - if err := uploadObjects(batch); err != nil { - return err - } - } - - err, has := <-errChan - if has { - log.Error("Error enumerating LFS objects for repository: %v", err) - return err - } - - return nil -} diff --git a/modules/notification/mirror/mirror.go b/modules/notification/mirror/mirror.go index ead7fb8e2fa9c..2e9a7e112b1f3 100644 --- a/modules/notification/mirror/mirror.go +++ b/modules/notification/mirror/mirror.go @@ -53,7 +53,6 @@ func syncPushMirrorWithSyncOnCommit(ctx context.Context, repoID int64) { } for _, mirror := range pushMirrors { - // TODO: push mirror likely will benefit from using a queue - mirror_module.SyncPushMirror(ctx, mirror.ID) + mirror_module.AddPushMirrorToQueue(mirror.ID) } } diff --git a/routers/api/v1/repo/mirror.go b/routers/api/v1/repo/mirror.go index 1af63c55be01a..3d29383550c32 100644 --- a/routers/api/v1/repo/mirror.go +++ b/routers/api/v1/repo/mirror.go @@ -11,8 +11,8 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/context" + mirror_module "code.gitea.io/gitea/modules/mirror" "code.gitea.io/gitea/modules/setting" - mirror_service "code.gitea.io/gitea/services/mirror" ) // MirrorSync adds a mirrored repository to the sync queue @@ -59,7 +59,7 @@ func MirrorSync(ctx *context.APIContext) { return } - mirror_service.StartToMirror(repo.ID) + mirror_module.AddPullMirrorToQueue(repo.ID) ctx.Status(http.StatusOK) } diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 3a30bd02be0aa..5ded0561c65b5 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -29,6 +29,7 @@ import ( "code.gitea.io/gitea/modules/indexer/stats" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" + mirror_module "code.gitea.io/gitea/modules/mirror" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" @@ -272,7 +273,7 @@ func SettingsPost(ctx *context.Context) { return } - mirror_service.StartToMirror(repo.ID) + mirror_module.AddPullMirrorToQueue(repo.ID) ctx.Flash.Info(ctx.Tr("repo.settings.mirror_sync_in_progress")) ctx.Redirect(repo.Link() + "/settings") @@ -289,7 +290,7 @@ func SettingsPost(ctx *context.Context) { return } - mirror_service.AddPushMirrorToQueue(m.ID) + mirror_module.AddPushMirrorToQueue(m.ID) ctx.Flash.Info(ctx.Tr("repo.settings.mirror_sync_in_progress")) ctx.Redirect(repo.Link() + "/settings") diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index c7a5107fe0f37..8321829ad26ba 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -16,34 +16,16 @@ import ( "code.gitea.io/gitea/modules/setting" ) -var mirrorQueue queue.UniqueQueue - -// SyncType type of sync request -type SyncType int - -const ( - // PullMirrorType for pull mirrors - PullMirrorType SyncType = iota - // PushMirrorType for push mirrors - PushMirrorType -) - -// SyncRequest for the mirror queue -type SyncRequest struct { - Type SyncType - ReferenceID int64 // RepoID for pull mirror, MirrorID fro push mirror -} - // doMirrorSync causes this request to mirror itself -func doMirrorSync(ctx context.Context, req *SyncRequest) { +func doMirrorSync(ctx context.Context, req *mirror_module.SyncRequest) { if req.ReferenceID == 0 { log.Warn("Skipping mirror sync request, no mirror ID was specified") return } switch req.Type { - case PushMirrorType: - _ = mirror_module.SyncPushMirror(ctx, req.ReferenceID) - case PullMirrorType: + case mirror_module.PushMirrorType: + _ = SyncPushMirror(ctx, req.ReferenceID) + case mirror_module.PullMirrorType: _ = SyncPullMirror(ctx, req.ReferenceID) default: log.Error("Unknown Request type in queue: %v for MirrorID[%d]", req.Type, req.ReferenceID) @@ -61,28 +43,26 @@ func Update(ctx context.Context, pullLimit, pushLimit int) error { log.Trace("Doing: Update") handler := func(idx int, bean interface{}) error { - var item SyncRequest var repo *repo_model.Repository + var mirrorType mirror_module.SyncType + var referenceID int64 + if m, ok := bean.(*repo_model.Mirror); ok { if m.GetRepository() == nil { log.Error("Disconnected mirror found: %d", m.ID) return nil } repo = m.Repo - item = SyncRequest{ - Type: PullMirrorType, - ReferenceID: m.RepoID, - } + mirrorType = mirror_module.PullMirrorType + referenceID = m.RepoID } else if m, ok := bean.(*repo_model.PushMirror); ok { if m.GetRepository() == nil { log.Error("Disconnected push-mirror found: %d", m.ID) return nil } repo = m.Repo - item = SyncRequest{ - Type: PushMirrorType, - ReferenceID: m.ID, - } + mirrorType = mirror_module.PushMirrorType + referenceID = m.ID } else { log.Error("Unknown bean: %v", bean) return nil @@ -96,9 +76,9 @@ func Update(ctx context.Context, pullLimit, pushLimit int) error { } // Push to the Queue - if err := mirrorQueue.Push(&item); err != nil { + if err := mirror_module.PushToQueue(mirrorType, referenceID); err != nil { if err == queue.ErrAlreadyInQueue { - if item.Type == PushMirrorType { + if mirrorType == mirror_module.PushMirrorType { log.Trace("PushMirrors for %-v already queued for sync", repo) } else { log.Trace("PullMirrors for %-v already queued for sync", repo) @@ -143,7 +123,7 @@ func Update(ctx context.Context, pullLimit, pushLimit int) error { func queueHandle(data ...queue.Data) []queue.Data { for _, datum := range data { - req := datum.(*SyncRequest) + req := datum.(*mirror_module.SyncRequest) doMirrorSync(graceful.GetManager().ShutdownContext(), req) } return nil @@ -151,43 +131,5 @@ func queueHandle(data ...queue.Data) []queue.Data { // InitSyncMirrors initializes a go routine to sync the mirrors func InitSyncMirrors() { - if !setting.Mirror.Enabled { - return - } - mirrorQueue = queue.CreateUniqueQueue("mirror", queueHandle, new(SyncRequest)) - - go graceful.GetManager().RunWithShutdownFns(mirrorQueue.Run) -} - -// StartToMirror adds repoID to mirror queue -func StartToMirror(repoID int64) { - if !setting.Mirror.Enabled { - return - } - go func() { - err := mirrorQueue.Push(&SyncRequest{ - Type: PullMirrorType, - ReferenceID: repoID, - }) - if err != nil { - log.Error("Unable to push sync request for to the queue for pull mirror repo[%d]: Error: %v", repoID, err) - return - } - }() -} - -// AddPushMirrorToQueue adds the push mirror to the queue -func AddPushMirrorToQueue(mirrorID int64) { - if !setting.Mirror.Enabled { - return - } - go func() { - err := mirrorQueue.Push(&SyncRequest{ - Type: PushMirrorType, - ReferenceID: mirrorID, - }) - if err != nil { - log.Error("Unable to push sync request to the queue for pull mirror repo[%d]: Error: %v", mirrorID, err) - } - }() + mirror_module.StartSyncMirrors(queueHandle) } diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index a52eb76d525f3..2927bed72b279 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -6,16 +6,26 @@ package mirror import ( "context" + "errors" "fmt" + "io" + "regexp" "strings" + "time" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" ) +var stripExitStatus = regexp.MustCompile(`exit status \d+ - `) + // AddPushMirrorRemote registers the push mirror remote. func AddPushMirrorRemote(ctx context.Context, m *repo_model.PushMirror, addr string) error { addRemoteAndConfig := func(addr, path string) error { @@ -71,3 +81,176 @@ func RemovePushMirrorRemote(ctx context.Context, m *repo_model.PushMirror) error return nil } + +// SyncPushMirror starts the sync of the push mirror and schedules the next run. +func SyncPushMirror(ctx context.Context, mirrorID int64) bool { + log.Trace("SyncPushMirror [mirror: %d]", mirrorID) + defer func() { + err := recover() + if err == nil { + return + } + // There was a panic whilst syncPushMirror... + log.Error("PANIC whilst syncPushMirror[%d] Panic: %v\nStacktrace: %s", mirrorID, err, log.Stack(2)) + }() + + m, err := repo_model.GetPushMirrorByID(mirrorID) + if err != nil { + log.Error("GetPushMirrorByID [%d]: %v", mirrorID, err) + return false + } + + _ = m.GetRepository() + + m.LastError = "" + + ctx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Syncing PushMirror %s/%s to %s", m.Repo.OwnerName, m.Repo.Name, m.RemoteName)) + defer finished() + + log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Running Sync", m.ID, m.Repo) + err = runPushSync(ctx, m) + if err != nil { + log.Error("SyncPushMirror [mirror: %d][repo: %-v]: %v", m.ID, m.Repo, err) + m.LastError = stripExitStatus.ReplaceAllLiteralString(err.Error(), "") + } + + m.LastUpdateUnix = timeutil.TimeStampNow() + + if err := repo_model.UpdatePushMirror(m); err != nil { + log.Error("UpdatePushMirror [%d]: %v", m.ID, err) + + return false + } + + log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Finished", m.ID, m.Repo) + + return err == nil +} + +func runPushSync(ctx context.Context, m *repo_model.PushMirror) error { + timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second + + performPush := func(path string) error { + remoteURL, err := git.GetRemoteURL(ctx, path, m.RemoteName) + if err != nil { + log.Error("GetRemoteAddress(%s) Error %v", path, err) + return errors.New("Unexpected error") + } + + if setting.LFS.StartServer { + log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) + + gitRepo, err := git.OpenRepository(ctx, path) + if err != nil { + log.Error("OpenRepository: %v", err) + return errors.New("Unexpected error") + } + defer gitRepo.Close() + + endpoint := lfs.DetermineEndpoint(remoteURL.String(), "") + lfsClient := lfs.NewClient(endpoint, nil) + if err := pushAllLFSObjects(ctx, gitRepo, lfsClient); err != nil { + return util.SanitizeErrorCredentialURLs(err) + } + } + + log.Trace("Pushing %s mirror[%d] remote %s", path, m.ID, m.RemoteName) + + if err := git.Push(ctx, path, git.PushOptions{ + Remote: m.RemoteName, + Force: true, + Mirror: true, + Timeout: timeout, + }); err != nil { + log.Error("Error pushing %s mirror[%d] remote %s: %v", path, m.ID, m.RemoteName, err) + + return util.SanitizeErrorCredentialURLs(err) + } + + return nil + } + + err := performPush(m.Repo.RepoPath()) + if err != nil { + return err + } + + if m.Repo.HasWiki() { + wikiPath := m.Repo.WikiPath() + _, err := git.GetRemoteAddress(ctx, wikiPath, m.RemoteName) + if err == nil { + err := performPush(wikiPath) + if err != nil { + return err + } + } else { + log.Trace("Skipping wiki: No remote configured") + } + } + + return nil +} + +func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, lfsClient lfs.Client) error { + contentStore := lfs.NewContentStore() + + pointerChan := make(chan lfs.PointerBlob) + errChan := make(chan error, 1) + go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) + + uploadObjects := func(pointers []lfs.Pointer) error { + err := lfsClient.Upload(ctx, pointers, func(p lfs.Pointer, objectError error) (io.ReadCloser, error) { + if objectError != nil { + return nil, objectError + } + + content, err := contentStore.Get(p) + if err != nil { + log.Error("Error reading LFS object %v: %v", p, err) + } + return content, err + }) + if err != nil { + select { + case <-ctx.Done(): + return nil + default: + } + } + return err + } + + var batch []lfs.Pointer + for pointerBlob := range pointerChan { + exists, err := contentStore.Exists(pointerBlob.Pointer) + if err != nil { + log.Error("Error checking if LFS object %v exists: %v", pointerBlob.Pointer, err) + return err + } + if !exists { + log.Trace("Skipping missing LFS object %v", pointerBlob.Pointer) + continue + } + + batch = append(batch, pointerBlob.Pointer) + if len(batch) >= lfsClient.BatchSize() { + if err := uploadObjects(batch); err != nil { + return err + } + batch = nil + } + } + if len(batch) > 0 { + if err := uploadObjects(batch); err != nil { + return err + } + } + + err, has := <-errChan + if has { + log.Error("Error enumerating LFS objects for repository: %v", err) + return err + } + + return nil +} From 3b0bd598e3bf582aacc0a862e9fcff8571cb0825 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Mon, 4 Jul 2022 14:38:24 -0400 Subject: [PATCH 18/31] Use boolean keyword FALSE --- models/migrations/v219.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrations/v219.go b/models/migrations/v219.go index 73c134fb07fb2..c51c10ff8804f 100644 --- a/models/migrations/v219.go +++ b/models/migrations/v219.go @@ -37,7 +37,7 @@ func addSyncOnPushColForPushMirror(x *xorm.Engine) error { return fmt.Errorf("sync2: %v", err) } - if _, err := session.Exec("UPDATE push_mirror SET sync_on_commit = 0"); err != nil { + if _, err := session.Exec("UPDATE push_mirror SET sync_on_commit = FALSE"); err != nil { return err } From 28925ad48f5d77659f1699f00cfbb1b641858a4a Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Mon, 4 Jul 2022 15:07:53 -0400 Subject: [PATCH 19/31] Use False only for postgres --- models/migrations/v219.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/models/migrations/v219.go b/models/migrations/v219.go index c51c10ff8804f..58fe85bc43541 100644 --- a/models/migrations/v219.go +++ b/models/migrations/v219.go @@ -9,6 +9,7 @@ import ( "time" "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "xorm.io/xorm" ) @@ -37,8 +38,14 @@ func addSyncOnPushColForPushMirror(x *xorm.Engine) error { return fmt.Errorf("sync2: %v", err) } - if _, err := session.Exec("UPDATE push_mirror SET sync_on_commit = FALSE"); err != nil { - return err + if setting.Database.UsePostgreSQL { // PostgreSQL uses Boolean type + if _, err := session.Exec("UPDATE push_mirror SET sync_on_commit = FALSE"); err != nil { + return err + } + } else { + if _, err := session.Exec("UPDATE push_mirror SET sync_on_commit = 0"); err != nil { + return err + } } return session.Commit() From 436013ea5589a010b0d7ebe64592108e4cc62c1e Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 5 Jul 2022 03:14:06 -0400 Subject: [PATCH 20/31] Remove MergePullRequest event --- modules/notification/mirror/mirror.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/modules/notification/mirror/mirror.go b/modules/notification/mirror/mirror.go index 2e9a7e112b1f3..9310760d68f6d 100644 --- a/modules/notification/mirror/mirror.go +++ b/modules/notification/mirror/mirror.go @@ -8,7 +8,6 @@ import ( "context" "fmt" - issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/graceful" @@ -37,13 +36,6 @@ func (m *mirrorNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_m syncPushMirrorWithSyncOnCommit(ctx, repo.ID) } -func (m *mirrorNotifier) NotifyMergePullRequest(pr *issues_model.PullRequest, doer *user_model.User) { - ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("webhook.NotifyMergePullRequest Pull[%d] #%d in [%d]", pr.ID, pr.Index, pr.BaseRepoID)) - defer finished() - - syncPushMirrorWithSyncOnCommit(ctx, pr.BaseRepoID) -} - func syncPushMirrorWithSyncOnCommit(ctx context.Context, repoID int64) { syncOnCommit := true pushMirrors, err := repo_model.GetPushMirrorsByRepoIDWithSyncOnCommit(repoID, syncOnCommit) From 2bd23e6f7919706065db6ec15142f60d27794a5f Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 5 Jul 2022 03:25:20 -0400 Subject: [PATCH 21/31] Sync with mirror on SyncPushCommits event --- modules/notification/mirror/mirror.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/notification/mirror/mirror.go b/modules/notification/mirror/mirror.go index 9310760d68f6d..c16f1f138841d 100644 --- a/modules/notification/mirror/mirror.go +++ b/modules/notification/mirror/mirror.go @@ -36,6 +36,13 @@ func (m *mirrorNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_m syncPushMirrorWithSyncOnCommit(ctx, repo.ID) } +func (m *mirrorNotifier) NotifySyncPushCommits(pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) { + ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("webhook.NotifySyncPushCommits User: %s[%d] in %s[%d]", pusher.Name, pusher.ID, repo.FullName(), repo.ID)) + defer finished() + + syncPushMirrorWithSyncOnCommit(ctx, repo.ID) +} + func syncPushMirrorWithSyncOnCommit(ctx context.Context, repoID int64) { syncOnCommit := true pushMirrors, err := repo_model.GetPushMirrorsByRepoIDWithSyncOnCommit(repoID, syncOnCommit) From 9cc7738854d9060525972ef12556d6e03731b686 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 5 Jul 2022 03:26:12 -0400 Subject: [PATCH 22/31] Remove unused context --- modules/notification/mirror/mirror.go | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/modules/notification/mirror/mirror.go b/modules/notification/mirror/mirror.go index c16f1f138841d..b3bd5eda9f615 100644 --- a/modules/notification/mirror/mirror.go +++ b/modules/notification/mirror/mirror.go @@ -5,16 +5,11 @@ package mirror import ( - "context" - "fmt" - repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" mirror_module "code.gitea.io/gitea/modules/mirror" "code.gitea.io/gitea/modules/notification/base" - "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/repository" ) @@ -30,20 +25,14 @@ func NewNotifier() base.Notifier { } func (m *mirrorNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) { - ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("mirrorNotifier.NotifyPushCommits User: %s[%d] in %s[%d]", pusher.Name, pusher.ID, repo.FullName(), repo.ID)) - defer finished() - - syncPushMirrorWithSyncOnCommit(ctx, repo.ID) + syncPushMirrorWithSyncOnCommit(repo.ID) } func (m *mirrorNotifier) NotifySyncPushCommits(pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) { - ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("webhook.NotifySyncPushCommits User: %s[%d] in %s[%d]", pusher.Name, pusher.ID, repo.FullName(), repo.ID)) - defer finished() - - syncPushMirrorWithSyncOnCommit(ctx, repo.ID) + syncPushMirrorWithSyncOnCommit(repo.ID) } -func syncPushMirrorWithSyncOnCommit(ctx context.Context, repoID int64) { +func syncPushMirrorWithSyncOnCommit(repoID int64) { syncOnCommit := true pushMirrors, err := repo_model.GetPushMirrorsByRepoIDWithSyncOnCommit(repoID, syncOnCommit) if err != nil { From 207e0fe679444cec78e7830a2032c4f7da5ad927 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 5 Jul 2022 03:56:54 -0400 Subject: [PATCH 23/31] Fill input box with value --- templates/repo/settings/options.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 664261d2c66c4..095ce0b203391 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -221,7 +221,7 @@
- +
From 08ceff9d7d3f609af1192a63769e8943018d515b Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 5 Jul 2022 11:04:14 -0400 Subject: [PATCH 24/31] Fix a func naming glitch in migration --- models/migrations/migrations.go | 2 +- models/migrations/v219.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index a3af6ed990ccb..1b2a743b6d357 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -397,7 +397,7 @@ var migrations = []Migration{ // v218 -> v219 NewMigration("Improve Action table indices v2", improveActionTableIndices), // v219 -> v220 - NewMigration("Add sync_on_commit column to push_mirror table", addSyncOnPushColForPushMirror), + NewMigration("Add sync_on_commit column to push_mirror table", addSyncOnCommitColForPushMirror), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v219.go b/models/migrations/v219.go index 58fe85bc43541..ce14927d2e880 100644 --- a/models/migrations/v219.go +++ b/models/migrations/v219.go @@ -14,7 +14,7 @@ import ( "xorm.io/xorm" ) -func addSyncOnPushColForPushMirror(x *xorm.Engine) error { +func addSyncOnCommitColForPushMirror(x *xorm.Engine) error { type PushMirror struct { ID int64 `xorm:"pk autoincr"` RepoID int64 `xorm:"INDEX"` From 2cc1297dbab64d0654d061fe78a2f2f02092bd9a Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 5 Jul 2022 13:47:51 -0400 Subject: [PATCH 25/31] Apply not null and default to SyncOnCommit --- models/migrations/v219.go | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/models/migrations/v219.go b/models/migrations/v219.go index ce14927d2e880..40f22baf1448a 100644 --- a/models/migrations/v219.go +++ b/models/migrations/v219.go @@ -9,7 +9,6 @@ import ( "time" "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "xorm.io/xorm" ) @@ -21,32 +20,15 @@ func addSyncOnCommitColForPushMirror(x *xorm.Engine) error { Repo *repo.Repository `xorm:"-"` RemoteName string - SyncOnCommit bool + SyncOnCommit bool `xorm:"NOT NULL DEFAULT false"` Interval time.Duration CreatedUnix timeutil.TimeStamp `xorm:"created"` LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` LastError string `xorm:"text"` } - session := x.NewSession() - defer session.Close() - if err := session.Begin(); err != nil { - return err - } - - if err := session.Sync2(new(PushMirror)); err != nil { + if err := x.Sync2(new(PushMirror)); err != nil { return fmt.Errorf("sync2: %v", err) } - - if setting.Database.UsePostgreSQL { // PostgreSQL uses Boolean type - if _, err := session.Exec("UPDATE push_mirror SET sync_on_commit = FALSE"); err != nil { - return err - } - } else { - if _, err := session.Exec("UPDATE push_mirror SET sync_on_commit = 0"); err != nil { - return err - } - } - - return session.Commit() + return nil } From 92603fc5da83c6289d1b67d6569ae47cfa2c253f Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 5 Jul 2022 19:41:57 -0400 Subject: [PATCH 26/31] Apply suggestions from code review Co-authored-by: delvh --- models/migrations/v219.go | 5 +---- models/repo/pushmirror.go | 7 +++---- modules/mirror/mirror.go | 22 ++++++++-------------- modules/notification/mirror/mirror.go | 9 ++++----- options/locale/locale_en-US.ini | 2 +- templates/repo/settings/options.tmpl | 2 +- 6 files changed, 18 insertions(+), 29 deletions(-) diff --git a/models/migrations/v219.go b/models/migrations/v219.go index 40f22baf1448a..e126a5ee06bf7 100644 --- a/models/migrations/v219.go +++ b/models/migrations/v219.go @@ -27,8 +27,5 @@ func addSyncOnCommitColForPushMirror(x *xorm.Engine) error { LastError string `xorm:"text"` } - if err := x.Sync2(new(PushMirror)); err != nil { - return fmt.Errorf("sync2: %v", err) - } - return nil + return x.Sync2(new(PushMirror)) } diff --git a/models/repo/pushmirror.go b/models/repo/pushmirror.go index c5b8d51fd0944..fbf0bb2588624 100644 --- a/models/repo/pushmirror.go +++ b/models/repo/pushmirror.go @@ -94,12 +94,11 @@ func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) { return mirrors, db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID).Find(&mirrors) } -// GetPushMirrorsByRepoIDWithSyncOnCommit returns push-mirror information of a repository, -// filtered by sync_on_commit. -func GetPushMirrorsByRepoIDWithSyncOnCommit(repoID int64, syncOnCommit bool) ([]*PushMirror, error) { +// GetPushMirrorsByRepoIDWithSyncOnCommit returns push-mirrors for this repo that should be updated by new commits +func GetPushMirrorsSyncedOnCommit(repoID int64) ([]*PushMirror, error) { mirrors := make([]*PushMirror, 0, 10) return mirrors, db.GetEngine(db.DefaultContext). - Where("repo_id=? AND sync_on_commit=?", repoID, syncOnCommit). + Where("repo_id=? AND sync_on_commit=?", repoID, true). Find(&mirrors) } diff --git a/modules/mirror/mirror.go b/modules/mirror/mirror.go index f3c56102dc9f9..d73592725d994 100644 --- a/modules/mirror/mirror.go +++ b/modules/mirror/mirror.go @@ -26,7 +26,7 @@ const ( // SyncRequest for the mirror queue type SyncRequest struct { Type SyncType - ReferenceID int64 // RepoID for pull mirror, MirrorID fro push mirror + ReferenceID int64 // RepoID for pull mirror, MirrorID for push mirror } // StartSyncMirrors starts a go routine to sync the mirrors @@ -41,31 +41,25 @@ func StartSyncMirrors(queueHandle func(data ...queue.Data) []queue.Data) { // AddPullMirrorToQueue adds repoID to mirror queue func AddPullMirrorToQueue(repoID int64) { - if !setting.Mirror.Enabled { - return - } - go func() { - err := PushToQueue(PullMirrorType, repoID) - if err != nil { - log.Error("Unable to push sync request for to the queue for pull mirror repo[%d]: Error: %v", repoID, err) - return - } - }() + addMirrorToQueue(PullMirrorType, repoID) } // AddPushMirrorToQueue adds the push mirror to the queue func AddPushMirrorToQueue(mirrorID int64) { + addMirrorToQueue(PushMirrorType, mirrorID) +} + +func addMirrorToQueue(mirrorType SyncType, referenceID int64) { if !setting.Mirror.Enabled { return } go func() { - err := PushToQueue(PushMirrorType, mirrorID) + err := PushToQueue(syncType, referenceID) if err != nil { - log.Error("Unable to push sync request to the queue for pull mirror repo[%d]: Error: %v", mirrorID, err) + log.Error("Unable to push sync request for to the queue for pull mirror repo[%d]. Error: %v", referenceID, err) } }() } - // PushToQueue adds the sync request to the queue func PushToQueue(mirrorType SyncType, referenceID int64) error { return mirrorQueue.Push(&SyncRequest{ diff --git a/modules/notification/mirror/mirror.go b/modules/notification/mirror/mirror.go index b3bd5eda9f615..646b09a4ab6e6 100644 --- a/modules/notification/mirror/mirror.go +++ b/modules/notification/mirror/mirror.go @@ -24,19 +24,18 @@ func NewNotifier() base.Notifier { return &mirrorNotifier{} } -func (m *mirrorNotifier) NotifyPushCommits(pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) { +func (m *mirrorNotifier) NotifyPushCommits(_ *user_model.User, repo *repo_model.Repository, _ *repository.PushUpdateOptions, _ *repository.PushCommits) { syncPushMirrorWithSyncOnCommit(repo.ID) } -func (m *mirrorNotifier) NotifySyncPushCommits(pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) { +func (m *mirrorNotifier) NotifySyncPushCommits(_ *user_model.User, repo *repo_model.Repository, _ *repository.PushUpdateOptions, _ *repository.PushCommits) { syncPushMirrorWithSyncOnCommit(repo.ID) } func syncPushMirrorWithSyncOnCommit(repoID int64) { - syncOnCommit := true - pushMirrors, err := repo_model.GetPushMirrorsByRepoIDWithSyncOnCommit(repoID, syncOnCommit) + pushMirrors, err := repo_model.GetPushMirrorsSyncedOnCommit(repoID) if err != nil { - log.Error("repo_model.GetPushMirrorsByRepoIDWithSyncOnCommit failed: %v", err) + log.Error("repo_model.GetPushMirrorsSyncedOnCommit failed: %v", err) return } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 1834a369070fc..464a7d396c0aa 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -863,7 +863,7 @@ mirror_prune = Prune mirror_prune_desc = Remove obsolete remote-tracking references mirror_interval = Mirror Interval (valid time units are 'h', 'm', 's'). 0 to disable periodic sync. (Minimum interval: %s) mirror_interval_invalid = The mirror interval is not valid. -mirror_sync_on_commit = Sync when new commit is pushed +mirror_sync_on_commit = Sync when commits are pushed mirror_address = Clone From URL mirror_address_desc = Put any required credentials in the Authorization section. mirror_address_url_invalid = The provided url is invalid. You must escape all components of the url correctly. diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 095ce0b203391..98cf4f88c870f 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -222,7 +222,7 @@
- +
From 67fa2092c725880a872ef5eba9abb4a26dee640d Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 5 Jul 2022 19:42:47 -0400 Subject: [PATCH 27/31] Fix minor errors from code review --- models/migrations/v219.go | 1 - modules/mirror/mirror.go | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/migrations/v219.go b/models/migrations/v219.go index e126a5ee06bf7..2e9f886647f89 100644 --- a/models/migrations/v219.go +++ b/models/migrations/v219.go @@ -5,7 +5,6 @@ package migrations import ( - "fmt" "time" "code.gitea.io/gitea/models/repo" diff --git a/modules/mirror/mirror.go b/modules/mirror/mirror.go index d73592725d994..180da512961bb 100644 --- a/modules/mirror/mirror.go +++ b/modules/mirror/mirror.go @@ -49,7 +49,7 @@ func AddPushMirrorToQueue(mirrorID int64) { addMirrorToQueue(PushMirrorType, mirrorID) } -func addMirrorToQueue(mirrorType SyncType, referenceID int64) { +func addMirrorToQueue(syncType SyncType, referenceID int64) { if !setting.Mirror.Enabled { return } @@ -60,6 +60,7 @@ func addMirrorToQueue(mirrorType SyncType, referenceID int64) { } }() } + // PushToQueue adds the sync request to the queue func PushToQueue(mirrorType SyncType, referenceID int64) error { return mirrorQueue.Push(&SyncRequest{ From 740408ee61ee1895f35fcccea2b34276a8ce2c41 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 5 Jul 2022 20:17:22 -0400 Subject: [PATCH 28/31] Enable sync on commit by default --- models/migrations/v219.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrations/v219.go b/models/migrations/v219.go index 2e9f886647f89..7b2eaa3292704 100644 --- a/models/migrations/v219.go +++ b/models/migrations/v219.go @@ -19,7 +19,7 @@ func addSyncOnCommitColForPushMirror(x *xorm.Engine) error { Repo *repo.Repository `xorm:"-"` RemoteName string - SyncOnCommit bool `xorm:"NOT NULL DEFAULT false"` + SyncOnCommit bool `xorm:"NOT NULL DEFAULT true"` Interval time.Duration CreatedUnix timeutil.TimeStamp `xorm:"created"` LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` From 2210235611ec3848e5dbe7f2afd61d5f20e04a87 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Wed, 6 Jul 2022 00:17:47 -0400 Subject: [PATCH 29/31] Set SyncOnCommit's default to true in models --- models/repo/pushmirror.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/repo/pushmirror.go b/models/repo/pushmirror.go index fbf0bb2588624..79cb3aebe610c 100644 --- a/models/repo/pushmirror.go +++ b/models/repo/pushmirror.go @@ -23,7 +23,7 @@ type PushMirror struct { Repo *Repository `xorm:"-"` RemoteName string - SyncOnCommit bool + SyncOnCommit bool `xorm:"NOT NULL DEFAULT true"` Interval time.Duration CreatedUnix timeutil.TimeStamp `xorm:"created"` LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` From bc1afc1bce748e306f769cce2118080a545f190a Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Wed, 6 Jul 2022 22:06:57 -0400 Subject: [PATCH 30/31] Update comment to match function name Co-authored-by: zeripath --- models/repo/pushmirror.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/repo/pushmirror.go b/models/repo/pushmirror.go index 79cb3aebe610c..0a7dea79c93b3 100644 --- a/models/repo/pushmirror.go +++ b/models/repo/pushmirror.go @@ -94,7 +94,7 @@ func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) { return mirrors, db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID).Find(&mirrors) } -// GetPushMirrorsByRepoIDWithSyncOnCommit returns push-mirrors for this repo that should be updated by new commits +// GetPushMirrorsSyncedOnCommit returns push-mirrors for this repo that should be updated by new commits func GetPushMirrorsSyncedOnCommit(repoID int64) ([]*PushMirror, error) { mirrors := make([]*PushMirror, 0, 10) return mirrors, db.GetEngine(db.DefaultContext). From b7331e9d7a9e13af8cadfb7745683bca09ee3dcc Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 8 Jul 2022 13:21:45 -0400 Subject: [PATCH 31/31] Apply suggestions from code review Co-authored-by: delvh --- modules/mirror/mirror.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/mirror/mirror.go b/modules/mirror/mirror.go index 180da512961bb..b261bd0242634 100644 --- a/modules/mirror/mirror.go +++ b/modules/mirror/mirror.go @@ -54,8 +54,7 @@ func addMirrorToQueue(syncType SyncType, referenceID int64) { return } go func() { - err := PushToQueue(syncType, referenceID) - if err != nil { + if err := PushToQueue(syncType, referenceID); err != nil { log.Error("Unable to push sync request for to the queue for pull mirror repo[%d]. Error: %v", referenceID, err) } }()