From deff356e31d6481eb354c6b2b06b3291799ff291 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 5 Aug 2019 09:36:04 +0800 Subject: [PATCH 01/19] update migrated issues/comments when login as github --- models/issue.go | 13 ++++++++++++- models/issue_comment.go | 11 +++++++++++ modules/migrations/update.go | 21 +++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 modules/migrations/update.go diff --git a/models/issue.go b/models/issue.go index 8ce7d496ab465..078229c5cf18d 100644 --- a/models/issue.go +++ b/models/issue.go @@ -32,7 +32,7 @@ type Issue struct { PosterID int64 `xorm:"INDEX"` Poster *User `xorm:"-"` OriginalAuthor string - OriginalAuthorID int64 + OriginalAuthorID int64 `xorm:"index"` Title string `xorm:"name"` Content string `xorm:"TEXT"` RenderedContent string `xorm:"-"` @@ -1947,3 +1947,14 @@ func (issue *Issue) ResolveMentionsByVisibility(ctx DBContext, doer *User, menti return } + +// UpdateIssuesMigrations updates issues' migrations information +func UpdateIssuesMigrations(repoID, originalAuthorID, posterID int64) error { + _, err := x.Table("issue"). + Where("repo_id = ?", repoID). + And("original_author_id = ?", originalAuthorID). + Update(map[string]interface{}{ + "poster_id": posterID, + }) + return err +} diff --git a/models/issue_comment.go b/models/issue_comment.go index 7d38302b98dbb..e033274b725c7 100644 --- a/models/issue_comment.go +++ b/models/issue_comment.go @@ -1022,3 +1022,14 @@ func fetchCodeCommentsByReview(e Engine, issue *Issue, currentUser *User, review func FetchCodeComments(issue *Issue, currentUser *User) (CodeComments, error) { return fetchCodeComments(x, issue, currentUser) } + +// UpdateIssuesMigrations updates issues' migrations information +func UpdateCommentsMigrations(repoID, originalAuthorID, posterID int64) error { + _, err := x.Table("comment"). + Where("issue_id IN (SELECT id FROM issue WHERE repo_id = ?)", repoID). + And("original_author_id = ?", originalAuthorID). + Update(map[string]interface{}{ + "poster_id": posterID, + }) + return err +} diff --git a/modules/migrations/update.go b/modules/migrations/update.go new file mode 100644 index 0000000000000..3165650289ec1 --- /dev/null +++ b/modules/migrations/update.go @@ -0,0 +1,21 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import "code.gitea.io/gitea/models" + +// UpdateGithubMigrations will update posterid on issues/comments/prs when github user +// login or when migrating +func UpdateGithubMigrations(repoID, githubUserID, userID int64) error { + if err := models.UpdateIssuesMigrations(repoID, githubUserID, userID); err != nil { + return err + } + + if err := models.UpdateCommentsMigrations(repoID, githubUserID, userID); err != nil { + return err + } + + return nil +} From 74b7181dca86b83baaf830d07f06950780f65b4a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 10 Aug 2019 18:22:15 +0800 Subject: [PATCH 02/19] add get userid when migrating or login with github oauth2 --- models/external_login_user.go | 38 ++++++++- models/repo.go | 29 ++++--- modules/migrations/gitea.go | 149 ++++++++++++++++++++++++---------- modules/structs/repo.go | 13 +++ routers/api/v1/repo/repo.go | 36 ++++---- routers/user/auth.go | 18 ++++ 6 files changed, 210 insertions(+), 73 deletions(-) diff --git a/models/external_login_user.go b/models/external_login_user.go index 21a3cbbd312bc..e9185fbf70bcd 100644 --- a/models/external_login_user.go +++ b/models/external_login_user.go @@ -4,13 +4,32 @@ package models -import "github.com/markbates/goth" +import ( + "time" + + "github.com/markbates/goth" +) // ExternalLoginUser makes the connecting between some existing user and additional external login sources type ExternalLoginUser struct { - ExternalID string `xorm:"pk NOT NULL"` - UserID int64 `xorm:"INDEX NOT NULL"` - LoginSourceID int64 `xorm:"pk NOT NULL"` + ExternalID string `xorm:"pk NOT NULL"` + UserID int64 `xorm:"INDEX NOT NULL"` + LoginSourceID int64 `xorm:"pk NOT NULL"` + RawData map[string]interface{} + Provider string `xorm:"index"` + Email string + Name string + FirstName string + LastName string + NickName string + Description string + ExternalUserID string `xorm:"VARCHAR(50) index"` + AvatarURL string + Location string + AccessToken string + AccessTokenSecret string + RefreshToken string + ExpiresAt time.Time } // GetExternalLogin checks if a externalID in loginSourceID scope already exists @@ -72,3 +91,14 @@ func removeAllAccountLinks(e Engine, user *User) error { _, err := e.Delete(&ExternalLoginUser{UserID: user.ID}) return err } + +// GetUserIDByExternalUserID get user id according to provider and userID +func GetUserIDByExternalUserID(provider string, userID string) (int64, error) { + var id int64 + _, err := x.Select("user_id").Where("provider=?", provider). + And("external_user_id=?", userID).Get(&id) + if err != nil { + return 0, err + } + return id, nil +} diff --git a/models/repo.go b/models/repo.go index 23b1c2ef52c4d..f5b03806be4da 100644 --- a/models/repo.go +++ b/models/repo.go @@ -32,6 +32,7 @@ import ( "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/sync" "code.gitea.io/gitea/modules/timeutil" @@ -137,16 +138,17 @@ const ( // Repository represents a git repository. type Repository struct { - ID int64 `xorm:"pk autoincr"` - OwnerID int64 `xorm:"UNIQUE(s) index"` - OwnerName string `xorm:"-"` - Owner *User `xorm:"-"` - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - Name string `xorm:"INDEX NOT NULL"` - Description string `xorm:"TEXT"` - Website string `xorm:"VARCHAR(2048)"` - OriginalURL string `xorm:"VARCHAR(2048)"` - DefaultBranch string + ID int64 `xorm:"pk autoincr"` + OwnerID int64 `xorm:"UNIQUE(s) index"` + OwnerName string `xorm:"-"` + Owner *User `xorm:"-"` + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Name string `xorm:"INDEX NOT NULL"` + Description string `xorm:"TEXT"` + Website string `xorm:"VARCHAR(2048)"` + OriginalServiceType structs.GitServiceType `xorm:"index"` + OriginalURL string `xorm:"VARCHAR(2048)"` + DefaultBranch string NumWatches int NumStars int @@ -2780,3 +2782,10 @@ func (repo *Repository) GetOriginalURLHostname() string { return u.Host } + +// FindMigratedRepositoryIDs find all migrated +func FindMigratedRepositoryIDs(tp structs.GitServiceType) ([]int64, error) { + var ids []int64 + err := x.Select("id").Where("original_service_type = ?", tp).Table("repository").Find(&ids) + return ids, err +} diff --git a/modules/migrations/gitea.go b/modules/migrations/gitea.go index ab3b0b9f694ad..936a37c743bfb 100644 --- a/modules/migrations/gitea.go +++ b/modules/migrations/gitea.go @@ -43,6 +43,7 @@ type GiteaLocalUploader struct { issues sync.Map gitRepo *git.Repository prHeadCache map[string]struct{} + userMap map[int64]int64 // external user id mapping to user id } // NewGiteaLocalUploader creates an gitea Uploader via gitea API v1 @@ -109,13 +110,15 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate } r, err = models.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{ - RepoName: g.repoName, - Description: repo.Description, - Mirror: repo.IsMirror, - CloneAddr: remoteAddr, - Private: repo.IsPrivate, - Wiki: opts.Wiki, - Releases: opts.Releases, // if didn't get releases, then sync them from tags + RepoName: g.repoName, + Description: repo.Description, + OriginalURL: repo.OriginalURL, + GitServiceType: opts.GitServiceType, + Mirror: repo.IsMirror, + CloneAddr: remoteAddr, + Private: repo.IsPrivate, + Wiki: opts.Wiki, + Releases: opts.Releases, // if didn't get releases, then sync them from tags }) g.repo = r @@ -284,20 +287,38 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { } var is = models.Issue{ - RepoID: g.repo.ID, - Repo: g.repo, - Index: issue.Number, - PosterID: g.doer.ID, - OriginalAuthor: issue.PosterName, - OriginalAuthorID: issue.PosterID, - Title: issue.Title, - Content: issue.Content, - IsClosed: issue.State == "closed", - IsLocked: issue.IsLocked, - MilestoneID: milestoneID, - Labels: labels, - CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()), + RepoID: g.repo.ID, + Repo: g.repo, + Index: issue.Number, + Title: issue.Title, + Content: issue.Content, + IsClosed: issue.State == "closed", + IsLocked: issue.IsLocked, + MilestoneID: milestoneID, + Labels: labels, + CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()), + } + + userid, ok := g.userMap[issue.PosterID] + if !ok { + var err error + userid, err = models.GetUserIDByExternalUserID("github", fmt.Sprintf("%v", issue.PosterID)) + if err != nil { + log.Error("GetUserIDByExternalUserID: %v", err) + } + if userid > 0 { + g.userMap[issue.PosterID] = userid + } + } + + if userid > 0 { + is.PosterID = userid + } else { + is.PosterID = g.doer.ID + is.OriginalAuthor = issue.PosterName + is.OriginalAuthorID = issue.PosterID } + if issue.Closed != nil { is.ClosedUnix = timeutil.TimeStamp(issue.Closed.Unix()) } @@ -331,15 +352,34 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { issueID = issueIDStr.(int64) } - cms = append(cms, &models.Comment{ - IssueID: issueID, - Type: models.CommentTypeComment, - PosterID: g.doer.ID, - OriginalAuthor: comment.PosterName, - OriginalAuthorID: comment.PosterID, - Content: comment.Content, - CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()), - }) + userid, ok := g.userMap[comment.PosterID] + if !ok { + var err error + userid, err = models.GetUserIDByExternalUserID("github", fmt.Sprintf("%v", comment.PosterID)) + if err != nil { + log.Error("GetUserIDByExternalUserID: %v", err) + } + if userid > 0 { + g.userMap[comment.PosterID] = userid + } + } + + cm := models.Comment{ + IssueID: issueID, + Type: models.CommentTypeComment, + Content: comment.Content, + CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()), + } + + if userid > 0 { + cm.PosterID = userid + } else { + cm.PosterID = g.doer.ID + cm.OriginalAuthor = comment.PosterName + cm.OriginalAuthorID = comment.PosterID + } + + cms = append(cms, &cm) // TODO: Reactions } @@ -460,6 +500,40 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR head = pr.Head.Ref } + var issue = models.Issue{ + RepoID: g.repo.ID, + Repo: g.repo, + Title: pr.Title, + Index: pr.Number, + Content: pr.Content, + MilestoneID: milestoneID, + IsPull: true, + IsClosed: pr.State == "closed", + IsLocked: pr.IsLocked, + Labels: labels, + CreatedUnix: timeutil.TimeStamp(pr.Created.Unix()), + } + + userid, ok := g.userMap[pr.PosterID] + if !ok { + var err error + userid, err = models.GetUserIDByExternalUserID("github", fmt.Sprintf("%v", pr.PosterID)) + if err != nil { + log.Error("GetUserIDByExternalUserID: %v", err) + } + if userid > 0 { + g.userMap[pr.PosterID] = userid + } + } + + if userid > 0 { + issue.PosterID = userid + } else { + issue.PosterID = g.doer.ID + issue.OriginalAuthor = pr.PosterName + issue.OriginalAuthorID = pr.PosterID + } + var pullRequest = models.PullRequest{ HeadRepoID: g.repo.ID, HeadBranch: head, @@ -470,22 +544,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR Index: pr.Number, HasMerged: pr.Merged, - Issue: &models.Issue{ - RepoID: g.repo.ID, - Repo: g.repo, - Title: pr.Title, - Index: pr.Number, - PosterID: g.doer.ID, - OriginalAuthor: pr.PosterName, - OriginalAuthorID: pr.PosterID, - Content: pr.Content, - MilestoneID: milestoneID, - IsPull: true, - IsClosed: pr.State == "closed", - IsLocked: pr.IsLocked, - Labels: labels, - CreatedUnix: timeutil.TimeStamp(pr.Created.Unix()), - }, + Issue: &issue, } if pullRequest.Issue.IsClosed && pr.Closed != nil { diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 57f1768a0b943..d2aa2593b1f9b 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -153,6 +153,17 @@ type EditRepoOption struct { Archived *bool `json:"archived,omitempty"` } +// GitServiceType +type GitServiceType int + +const ( + PlainGitService GitServiceType = iota // 0 plain git service + GithubService // 1 github.com + GiteaService // 2 gitea service + GitlabService // 3 gitlab service + GogsService // 4 gogs service +) + // MigrateRepoOption options for migrating a repository from an external service type MigrateRepoOption struct { // required: true @@ -166,6 +177,8 @@ type MigrateRepoOption struct { Mirror bool `json:"mirror"` Private bool `json:"private"` Description string `json:"description"` + OriginalURL string + GitServiceType GitServiceType Wiki bool Issues bool Milestones bool diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index 08c0635bc3128..e7c1577d5a809 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/validation" @@ -397,21 +398,28 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) { return } + var gitServiceType = structs.PlainGitService + // TODO: this should be chosen by UI when from a customerize git service domain + if strings.HasPrefix(remoteAddr, "https://github.com") || strings.HasPrefix(remoteAddr, "http://github.com") { + gitServiceType = structs.GithubService + } + var opts = migrations.MigrateOptions{ - CloneAddr: remoteAddr, - RepoName: form.RepoName, - Description: form.Description, - Private: form.Private || setting.Repository.ForcePrivate, - Mirror: form.Mirror, - AuthUsername: form.AuthUsername, - AuthPassword: form.AuthPassword, - Wiki: form.Wiki, - Issues: form.Issues, - Milestones: form.Milestones, - Labels: form.Labels, - Comments: true, - PullRequests: form.PullRequests, - Releases: form.Releases, + CloneAddr: remoteAddr, + RepoName: form.RepoName, + Description: form.Description, + Private: form.Private || setting.Repository.ForcePrivate, + Mirror: form.Mirror, + AuthUsername: form.AuthUsername, + AuthPassword: form.AuthPassword, + Wiki: form.Wiki, + Issues: form.Issues, + Milestones: form.Milestones, + Labels: form.Labels, + Comments: true, + PullRequests: form.PullRequests, + Releases: form.Releases, + GitServiceType: gitServiceType, } if opts.Mirror { opts.Issues = false diff --git a/routers/user/auth.go b/routers/user/auth.go index 3def867f6456d..cec1e88250e8a 100644 --- a/routers/user/auth.go +++ b/routers/user/auth.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "net/http" + "strconv" "strings" "code.gitea.io/gitea/models" @@ -17,8 +18,10 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/recaptcha" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/mailer" @@ -627,6 +630,21 @@ func handleOAuth2SignIn(u *models.User, gothUser goth.User, ctx *context.Context return } + // update user's migrated comments and issues original id + if gothUser.Provider == "github" { + ids, err := models.FindMigratedRepositoryIDs(structs.GithubService) + if err != nil { + log.Error("FindMigratedRepositoryIDs failed: %v", err) + } else { + for _, id := range ids { + userID, _ := strconv.ParseInt(gothUser.UserID, 10, 64) + if err = migrations.UpdateGithubMigrations(id, userID, u.ID); err != nil { + log.Error("UpdateGithubMigrations repo %v, github user id %v, user id %v failed: %v", id, gothUser.UserID, u.ID, err) + } + } + } + } + ctx.Redirect(setting.AppSubURL + "/") } else { ctx.ServerError("UserSignIn", err) From dc0377b7bd4848b8a813e1e6095c2a8d5671d733 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 10 Aug 2019 19:44:14 +0800 Subject: [PATCH 03/19] fix lint --- models/issue_comment.go | 2 +- modules/migrations/update.go | 6 +----- modules/structs/repo.go | 14 ++++++++------ 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/models/issue_comment.go b/models/issue_comment.go index e033274b725c7..61f666432e50b 100644 --- a/models/issue_comment.go +++ b/models/issue_comment.go @@ -1023,7 +1023,7 @@ func FetchCodeComments(issue *Issue, currentUser *User) (CodeComments, error) { return fetchCodeComments(x, issue, currentUser) } -// UpdateIssuesMigrations updates issues' migrations information +// UpdateCommentsMigrations updates comments' migrations information func UpdateCommentsMigrations(repoID, originalAuthorID, posterID int64) error { _, err := x.Table("comment"). Where("issue_id IN (SELECT id FROM issue WHERE repo_id = ?)", repoID). diff --git a/modules/migrations/update.go b/modules/migrations/update.go index 3165650289ec1..8c59ef9ac48be 100644 --- a/modules/migrations/update.go +++ b/modules/migrations/update.go @@ -13,9 +13,5 @@ func UpdateGithubMigrations(repoID, githubUserID, userID int64) error { return err } - if err := models.UpdateCommentsMigrations(repoID, githubUserID, userID); err != nil { - return err - } - - return nil + return models.UpdateCommentsMigrations(repoID, githubUserID, userID) } diff --git a/modules/structs/repo.go b/modules/structs/repo.go index d2aa2593b1f9b..3e8e3675a7c0d 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -153,15 +153,17 @@ type EditRepoOption struct { Archived *bool `json:"archived,omitempty"` } -// GitServiceType +// GitServiceType represents a git service type GitServiceType int +// enumerate all GitServiceType const ( - PlainGitService GitServiceType = iota // 0 plain git service - GithubService // 1 github.com - GiteaService // 2 gitea service - GitlabService // 3 gitlab service - GogsService // 4 gogs service + NotMigrated GitServiceType = iota // 0 not migrated from external sites + PlainGitService // 1 plain git service + GithubService // 2 github.com + GiteaService // 3 gitea service + GitlabService // 4 gitlab service + GogsService // 5 gogs service ) // MigrateRepoOption options for migrating a repository from an external service From 37eb777a13b8ab34e93dedb343cf347c486e3984 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 14 Sep 2019 20:44:40 +0800 Subject: [PATCH 04/19] add migrations for repository service type --- models/migrations/migrations.go | 2 ++ models/migrations/v100.go | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 models/migrations/v100.go diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index ef5cd377a6c11..60a416c6e92ea 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -254,6 +254,8 @@ var migrations = []Migration{ NewMigration("add original author name and id on migrated release", addOriginalAuthorOnMigratedReleases), // v99 -> v100 NewMigration("add task table and status column for repository table", addTaskTable), + // v100 -> v101 + NewMigration("update migration repositories' service type", updateMigrationServiceTypes), } // Migrate database to current version diff --git a/models/migrations/v100.go b/models/migrations/v100.go new file mode 100644 index 0000000000000..df0b36a609a37 --- /dev/null +++ b/models/migrations/v100.go @@ -0,0 +1,42 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "strings" + + "code.gitea.io/gitea/modules/structs" + + "github.com/go-xorm/xorm" +) + +func updateMigrationServiceTypes(x *xorm.Engine) error { + for { + sql := "SELECT id, original_url FROM WHERE original_url <> '' LIMIT 50 ORDER BY id" + var results = make([]struct { + ID int64 + OriginalURL string + }, 0, 50) + err := x.SQL(sql).Find(results) + if err != nil { + return err + } + if len(results) == 0 { + return nil + } + + for _, res := range results { + u := strings.ToLower(res.OriginalURL) + var serviceType = structs.PlainGitService + if strings.HasPrefix(u, "https://github.com") || strings.HasPrefix(u, "https://github.com") { + serviceType = structs.GithubService + } + _, err = x.Exec("UPDATE repository SET original_service_type = ? WHERE id = ?", serviceType, res.ID) + if err != nil { + return err + } + } + } +} From a2fbab23910cc13c7a3d538565d29dbda77c15c2 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 15 Sep 2019 16:44:25 +0800 Subject: [PATCH 05/19] fix build --- models/migrations/v100.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrations/v100.go b/models/migrations/v100.go index df0b36a609a37..891eeb5f4667c 100644 --- a/models/migrations/v100.go +++ b/models/migrations/v100.go @@ -30,7 +30,7 @@ func updateMigrationServiceTypes(x *xorm.Engine) error { for _, res := range results { u := strings.ToLower(res.OriginalURL) var serviceType = structs.PlainGitService - if strings.HasPrefix(u, "https://github.com") || strings.HasPrefix(u, "https://github.com") { + if strings.HasPrefix(u, "https://github.com") || strings.HasPrefix(u, "http://github.com") { serviceType = structs.GithubService } _, err = x.Exec("UPDATE repository SET original_service_type = ? WHERE id = ?", serviceType, res.ID) From 191be942849495f9f714285089cd5b1b92429144 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 24 Sep 2019 15:10:09 +0800 Subject: [PATCH 06/19] remove unnecessary dependencies on migrations --- models/migrations/v100.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/models/migrations/v100.go b/models/migrations/v100.go index 891eeb5f4667c..fc410a90a935b 100644 --- a/models/migrations/v100.go +++ b/models/migrations/v100.go @@ -7,8 +7,6 @@ package migrations import ( "strings" - "code.gitea.io/gitea/modules/structs" - "github.com/go-xorm/xorm" ) @@ -27,11 +25,14 @@ func updateMigrationServiceTypes(x *xorm.Engine) error { return nil } + const PlainGitService = 1 // 1 plain git service + const GithubService = 2 // 2 github.com + for _, res := range results { u := strings.ToLower(res.OriginalURL) - var serviceType = structs.PlainGitService + var serviceType = PlainGitService if strings.HasPrefix(u, "https://github.com") || strings.HasPrefix(u, "http://github.com") { - serviceType = structs.GithubService + serviceType = GithubService } _, err = x.Exec("UPDATE repository SET original_service_type = ? WHERE id = ?", serviceType, res.ID) if err != nil { From c883433f6b71ea80d801eff4d15e8a455533448f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 26 Sep 2019 20:53:10 +0800 Subject: [PATCH 07/19] add cron task to update migrations poster ids and fix posterid when migrating --- custom/conf/app.ini.sample | 5 + .../doc/advanced/config-cheat-sheet.en-us.md | 4 + .../doc/advanced/config-cheat-sheet.zh-cn.md | 6 +- models/external_login_user.go | 107 ++++++++++++++++-- models/issue.go | 4 +- models/issue_comment.go | 4 +- models/migrations/v100.go | 58 ++++++++-- models/repo.go | 8 +- modules/cron/cron.go | 23 +++- modules/migrations/base/downloader.go | 3 + modules/migrations/gitea.go | 54 ++++++--- modules/migrations/github.go | 6 + modules/migrations/migrate.go | 9 ++ modules/migrations/update.go | 74 +++++++++++- modules/setting/cron.go | 8 ++ modules/structs/repo.go | 14 +++ routers/user/auth.go | 106 +++++++++-------- 17 files changed, 395 insertions(+), 98 deletions(-) diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index dd14089d2b06d..05e9f658f50f0 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -690,6 +690,11 @@ SCHEDULE = @every 24h ; or only create new users if UPDATE_EXISTING is set to false UPDATE_EXISTING = true +; Update migrated repositories' issues and comments' posterid, it will always be started at start. +[cron.update_migration_post_id] +; Interval as a duration between each synchronization (default every 10m) +SCHEDULE = @every 10m + [git] ; The path of git executable. If empty, Gitea searches through the PATH environment. PATH = diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index ed34be032bbda..1a8c40df35063 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -419,6 +419,10 @@ NB: You must `REDIRECT_MACARON_LOG` and have `DISABLE_ROUTER_LOG` set to `false` - `RUN_AT_START`: **true**: Run repository statistics check at start time. - `SCHEDULE`: **@every 24h**: Cron syntax for scheduling repository statistics check. +### Cron - Update Migration Poster ID (`cron.update_migration_post_id`) + +- `SCHEDULE`: **@every 10m** : Interval as a duration between each synchronization, it will always be started at start. + ## Git (`git`) - `PATH`: **""**: The path of git executable. If empty, Gitea searches through the PATH environment. diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index 01ba821a47a38..d35eb17777ae3 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -196,7 +196,11 @@ menu: ### Cron - Repository Statistics Check (`cron.check_repo_stats`) - `RUN_AT_START`: 是否启动时自动运行仓库统计。 -- `SCHEDULE`: 藏亏统计时的Cron 语法，比如：`@every 24h`. +- `SCHEDULE`: 仓库统计时的Cron 语法，比如：`@every 24h`. + +### Cron - Update Migration Poster ID (`cron.update_migration_post_id`) + +- `SCHEDULE`: **@every 10m** : 每次统计的间隔时间。此任务总是在启动时自动进行。 ## Git (`git`) diff --git a/models/external_login_user.go b/models/external_login_user.go index e9185fbf70bcd..d49961ee058fd 100644 --- a/models/external_login_user.go +++ b/models/external_login_user.go @@ -8,22 +8,22 @@ import ( "time" "github.com/markbates/goth" + "xorm.io/builder" ) // ExternalLoginUser makes the connecting between some existing user and additional external login sources type ExternalLoginUser struct { - ExternalID string `xorm:"pk NOT NULL"` - UserID int64 `xorm:"INDEX NOT NULL"` - LoginSourceID int64 `xorm:"pk NOT NULL"` - RawData map[string]interface{} - Provider string `xorm:"index"` + ExternalID string `xorm:"VARCHAR(50) pk NOT NULL"` + UserID int64 `xorm:"INDEX NOT NULL"` + LoginSourceID int64 `xorm:"pk NOT NULL"` + RawData map[string]interface{} `xorm:"TEXT"` + Provider string `xorm:"index"` Email string Name string FirstName string LastName string NickName string Description string - ExternalUserID string `xorm:"VARCHAR(50) index"` AvatarURL string Location string AccessToken string @@ -59,11 +59,28 @@ func LinkAccountToUser(user *User, gothUser goth.User) error { } externalLoginUser := &ExternalLoginUser{ - ExternalID: gothUser.UserID, - UserID: user.ID, - LoginSourceID: loginSource.ID, + ExternalID: gothUser.UserID, + UserID: user.ID, + LoginSourceID: loginSource.ID, + RawData: gothUser.RawData, + Provider: gothUser.Provider, + Email: gothUser.Email, + Name: gothUser.Name, + FirstName: gothUser.FirstName, + LastName: gothUser.LastName, + NickName: gothUser.NickName, + Description: gothUser.Description, + AvatarURL: gothUser.AvatarURL, + Location: gothUser.Location, + AccessToken: gothUser.AccessToken, + AccessTokenSecret: gothUser.AccessTokenSecret, + RefreshToken: gothUser.RefreshToken, + ExpiresAt: gothUser.ExpiresAt, } - has, err := x.Get(externalLoginUser) + + has, err := x.Where("external_id=? AND login_source_id=?", gothUser.UserID, loginSource.ID). + NoAutoCondition(). + Exist(externalLoginUser) if err != nil { return err } else if has { @@ -95,10 +112,76 @@ func removeAllAccountLinks(e Engine, user *User) error { // GetUserIDByExternalUserID get user id according to provider and userID func GetUserIDByExternalUserID(provider string, userID string) (int64, error) { var id int64 - _, err := x.Select("user_id").Where("provider=?", provider). - And("external_user_id=?", userID).Get(&id) + _, err := x.Table("external_login_user"). + Select("user_id"). + Where("provider=?", provider). + And("external_id=?", userID). + Get(&id) if err != nil { return 0, err } return id, nil } + +// UpdateExternalUser updates external user's information +func UpdateExternalUser(user *User, gothUser goth.User) error { + loginSource, err := GetActiveOAuth2LoginSourceByName(gothUser.Provider) + if err != nil { + return err + } + externalLoginUser := &ExternalLoginUser{ + ExternalID: gothUser.UserID, + UserID: user.ID, + LoginSourceID: loginSource.ID, + RawData: gothUser.RawData, + Provider: gothUser.Provider, + Email: gothUser.Email, + Name: gothUser.Name, + FirstName: gothUser.FirstName, + LastName: gothUser.LastName, + NickName: gothUser.NickName, + Description: gothUser.Description, + AvatarURL: gothUser.AvatarURL, + Location: gothUser.Location, + AccessToken: gothUser.AccessToken, + AccessTokenSecret: gothUser.AccessTokenSecret, + RefreshToken: gothUser.RefreshToken, + ExpiresAt: gothUser.ExpiresAt, + } + + has, err := x.Where("external_id=? AND login_source_id=?", gothUser.UserID, loginSource.ID). + NoAutoCondition(). + Exist(externalLoginUser) + if err != nil { + return err + } else if !has { + return ErrExternalLoginUserNotExist{user.ID, loginSource.ID} + } + + _, err = x.Where("external_id=? AND login_source_id=?", gothUser.UserID, loginSource.ID).AllCols().Update(externalLoginUser) + return err +} + +type FindExternalUserOptions struct { + Provider string + Limit int + Start int +} + +func (opts FindExternalUserOptions) toConds() builder.Cond { + var cond = builder.NewCond() + if len(opts.Provider) > 0 { + cond = cond.And(builder.Eq{"provider": opts.Provider}) + } + return cond +} + +// FindExternalUsersByProvider represents external users via provider +func FindExternalUsersByProvider(opts FindExternalUserOptions) ([]ExternalLoginUser, error) { + var users []ExternalLoginUser + err := x.Where(opts.toConds()).Limit(opts.Limit, opts.Start).Find(&users) + if err != nil { + return nil, err + } + return users, nil +} diff --git a/models/issue.go b/models/issue.go index 078229c5cf18d..ebe45e3102a8c 100644 --- a/models/issue.go +++ b/models/issue.go @@ -1954,7 +1954,9 @@ func UpdateIssuesMigrations(repoID, originalAuthorID, posterID int64) error { Where("repo_id = ?", repoID). And("original_author_id = ?", originalAuthorID). Update(map[string]interface{}{ - "poster_id": posterID, + "poster_id": posterID, + "original_author": "", + "original_author_id": 0, }) return err } diff --git a/models/issue_comment.go b/models/issue_comment.go index 61f666432e50b..1457b858095d8 100644 --- a/models/issue_comment.go +++ b/models/issue_comment.go @@ -1029,7 +1029,9 @@ func UpdateCommentsMigrations(repoID, originalAuthorID, posterID int64) error { Where("issue_id IN (SELECT id FROM issue WHERE repo_id = ?)", repoID). And("original_author_id = ?", originalAuthorID). Update(map[string]interface{}{ - "poster_id": posterID, + "poster_id": posterID, + "original_author": "", + "original_author_id": 0, }) return err } diff --git a/models/migrations/v100.go b/models/migrations/v100.go index fc410a90a935b..36043c32b4b87 100644 --- a/models/migrations/v100.go +++ b/models/migrations/v100.go @@ -5,33 +5,51 @@ package migrations import ( + "net/url" "strings" + "time" "github.com/go-xorm/xorm" ) func updateMigrationServiceTypes(x *xorm.Engine) error { + type Repository struct { + ID int64 + OriginalServiceType int `xorm:"index default(0)"` + OriginalURL string `xorm:"VARCHAR(2048)"` + } + + if err := x.Sync2(new(Repository)); err != nil { + return err + } + + var last int + const batchSize = 50 for { - sql := "SELECT id, original_url FROM WHERE original_url <> '' LIMIT 50 ORDER BY id" - var results = make([]struct { - ID int64 - OriginalURL string - }, 0, 50) - err := x.SQL(sql).Find(results) + var results = make([]Repository, 0, batchSize) + err := x.Where("original_url <> '' AND original_url IS NOT NULL"). + And("original_service_type = 0 OR original_service_type IS NULL"). + OrderBy("id"). + Limit(batchSize, last). + Find(&results) if err != nil { return err } if len(results) == 0 { - return nil + break } + last += len(results) const PlainGitService = 1 // 1 plain git service const GithubService = 2 // 2 github.com for _, res := range results { - u := strings.ToLower(res.OriginalURL) + u, err := url.Parse(res.OriginalURL) + if err != nil { + return err + } var serviceType = PlainGitService - if strings.HasPrefix(u, "https://github.com") || strings.HasPrefix(u, "http://github.com") { + if strings.EqualFold(u.Host, "github.com") { serviceType = GithubService } _, err = x.Exec("UPDATE repository SET original_service_type = ? WHERE id = ?", serviceType, res.ID) @@ -40,4 +58,26 @@ func updateMigrationServiceTypes(x *xorm.Engine) error { } } } + + type ExternalLoginUser struct { + ExternalID string `xorm:"pk NOT NULL"` + UserID int64 `xorm:"INDEX NOT NULL"` + LoginSourceID int64 `xorm:"pk NOT NULL"` + RawData map[string]interface{} `xorm:"TEXT"` + Provider string `xorm:"index VARCHAR(25)"` + Email string + Name string + FirstName string + LastName string + NickName string + Description string + AvatarURL string + Location string + AccessToken string + AccessTokenSecret string + RefreshToken string + ExpiresAt time.Time + } + + return x.Sync2(new(ExternalLoginUser)) } diff --git a/models/repo.go b/models/repo.go index f5b03806be4da..6be0caa2584ea 100644 --- a/models/repo.go +++ b/models/repo.go @@ -2784,8 +2784,12 @@ func (repo *Repository) GetOriginalURLHostname() string { } // FindMigratedRepositoryIDs find all migrated -func FindMigratedRepositoryIDs(tp structs.GitServiceType) ([]int64, error) { +func FindMigratedRepositoryIDs(tp structs.GitServiceType, limit, start int) ([]int64, error) { var ids []int64 - err := x.Select("id").Where("original_service_type = ?", tp).Table("repository").Find(&ids) + err := x.Select("id"). + Where("original_service_type = ?", tp). + Limit(limit, start). + Table("repository"). + Find(&ids) return ids, err } diff --git a/modules/cron/cron.go b/modules/cron/cron.go index 089f0fa767a6f..795fafb51fdaa 100644 --- a/modules/cron/cron.go +++ b/modules/cron/cron.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/sync" mirror_service "code.gitea.io/gitea/services/mirror" @@ -18,12 +19,13 @@ import ( ) const ( - mirrorUpdate = "mirror_update" - gitFsck = "git_fsck" - checkRepos = "check_repos" - archiveCleanup = "archive_cleanup" - syncExternalUsers = "sync_external_users" - deletedBranchesCleanup = "deleted_branches_cleanup" + mirrorUpdate = "mirror_update" + gitFsck = "git_fsck" + checkRepos = "check_repos" + archiveCleanup = "archive_cleanup" + syncExternalUsers = "sync_external_users" + deletedBranchesCleanup = "deleted_branches_cleanup" + updateMigrationPosterID = "update_migration_post_id" ) var c = cron.New() @@ -117,6 +119,15 @@ func NewContext() { go WithUnique(deletedBranchesCleanup, models.RemoveOldDeletedBranches)() } } + + entry, err = c.AddFunc("Update migrated repositories' issues and comments' posterid", setting.Cron.UpdateMigrationPosterID.Schedule, WithUnique(updateMigrationPosterID, migrations.UpdateMigrationPosterID)) + if err != nil { + log.Fatal("Cron[Update migrated repositories]: %v", err) + } + entry.Prev = time.Now() + entry.ExecTimes++ + go WithUnique(updateMigrationPosterID, migrations.UpdateMigrationPosterID)() + c.Start() } diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index ab5ca6dec80ff..69c2adb9e9503 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -5,6 +5,8 @@ package base +import "code.gitea.io/gitea/modules/structs" + // Downloader downloads the site repo informations type Downloader interface { GetRepoInfo() (*Repository, error) @@ -21,4 +23,5 @@ type Downloader interface { type DownloaderFactory interface { Match(opts MigrateOptions) (bool, error) New(opts MigrateOptions) (Downloader, error) + GitServiceType() structs.GitServiceType } diff --git a/modules/migrations/gitea.go b/modules/migrations/gitea.go index 936a37c743bfb..4accfc281b79e 100644 --- a/modules/migrations/gitea.go +++ b/modules/migrations/gitea.go @@ -34,16 +34,17 @@ var ( // GiteaLocalUploader implements an Uploader to gitea sites type GiteaLocalUploader struct { - doer *models.User - repoOwner string - repoName string - repo *models.Repository - labels sync.Map - milestones sync.Map - issues sync.Map - gitRepo *git.Repository - prHeadCache map[string]struct{} - userMap map[int64]int64 // external user id mapping to user id + doer *models.User + repoOwner string + repoName string + repo *models.Repository + labels sync.Map + milestones sync.Map + issues sync.Map + gitRepo *git.Repository + prHeadCache map[string]struct{} + userMap map[int64]int64 // external user id mapping to user id + gitServiceType structs.GitServiceType } // NewGiteaLocalUploader creates an gitea Uploader via gitea API v1 @@ -53,6 +54,7 @@ func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *Gitea repoOwner: repoOwner, repoName: repoName, prHeadCache: make(map[string]struct{}), + userMap: make(map[int64]int64), } } @@ -300,9 +302,10 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { } userid, ok := g.userMap[issue.PosterID] - if !ok { + tp := g.gitServiceType.Name() + if !ok && tp != "" { var err error - userid, err = models.GetUserIDByExternalUserID("github", fmt.Sprintf("%v", issue.PosterID)) + userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", issue.PosterID)) if err != nil { log.Error("GetUserIDByExternalUserID: %v", err) } @@ -353,9 +356,10 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { } userid, ok := g.userMap[comment.PosterID] - if !ok { + tp := g.gitServiceType.Name() + if !ok && tp != "" { var err error - userid, err = models.GetUserIDByExternalUserID("github", fmt.Sprintf("%v", comment.PosterID)) + userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", comment.PosterID)) if err != nil { log.Error("GetUserIDByExternalUserID: %v", err) } @@ -395,6 +399,28 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error if err != nil { return err } + + userid, ok := g.userMap[pr.PosterID] + tp := g.gitServiceType.Name() + if !ok && tp != "" { + var err error + userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID)) + if err != nil { + log.Error("GetUserIDByExternalUserID: %v", err) + } + if userid > 0 { + g.userMap[pr.PosterID] = userid + } + } + + if userid > 0 { + gpr.Issue.PosterID = userid + } else { + gpr.Issue.PosterID = g.doer.ID + gpr.Issue.OriginalAuthor = pr.PosterName + gpr.Issue.OriginalAuthorID = pr.PosterID + } + gprs = append(gprs, gpr) } if err := models.InsertPullRequests(gprs...); err != nil { diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 1c5d96c03d471..c3bc2cd873418 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/structs" "github.com/google/go-github/v24/github" "golang.org/x/oauth2" @@ -58,6 +59,11 @@ func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Download return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, oldOwner, oldName), nil } +// GitServiceType returns the type of git service +func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType { + return structs.GithubService +} + // GithubDownloaderV3 implements a Downloader interface to get repository informations // from github via APIv3 type GithubDownloaderV3 struct { diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 3f5c0d1118c32..0cf06be6b2283 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/structs" ) // MigrateOptions is equal to base.MigrateOptions @@ -30,6 +31,7 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt var ( downloader base.Downloader uploader = NewGiteaLocalUploader(doer, ownerName, opts.RepoName) + theFactory base.DownloaderFactory ) for _, factory := range factories { @@ -40,6 +42,7 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt if err != nil { return nil, err } + theFactory = factory break } } @@ -52,10 +55,16 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt opts.Comments = false opts.Issues = false opts.PullRequests = false + opts.GitServiceType = structs.PlainGitService downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr) log.Trace("Will migrate from git: %s", opts.CloneAddr) + } else { + if opts.GitServiceType == structs.NotMigrated { + opts.GitServiceType = theFactory.GitServiceType() + } } + uploader.gitServiceType = opts.GitServiceType if err := migrateRepository(downloader, uploader, opts); err != nil { if err1 := uploader.Rollback(); err1 != nil { log.Error("rollback failed: %v", err1) diff --git a/modules/migrations/update.go b/modules/migrations/update.go index 8c59ef9ac48be..7b22a93a30b86 100644 --- a/modules/migrations/update.go +++ b/modules/migrations/update.go @@ -4,14 +4,78 @@ package migrations -import "code.gitea.io/gitea/models" +import ( + "strconv" -// UpdateGithubMigrations will update posterid on issues/comments/prs when github user + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/structs" +) + +// UpdateRepoMigrations will update posterid on issues/comments/prs when external login user // login or when migrating -func UpdateGithubMigrations(repoID, githubUserID, userID int64) error { - if err := models.UpdateIssuesMigrations(repoID, githubUserID, userID); err != nil { +func UpdateRepoMigrations(repoID, externalUserID, userID int64) error { + if err := models.UpdateIssuesMigrations(repoID, externalUserID, userID); err != nil { return err } - return models.UpdateCommentsMigrations(repoID, githubUserID, userID) + return models.UpdateCommentsMigrations(repoID, externalUserID, userID) +} + +// UpdateMigrationPosterID updates all migrated repositories' issues and comments posterID +func UpdateMigrationPosterID() { + if err := updateMigrationPosterIDByGitService(structs.GithubService); err != nil { + log.Error("updateMigrationPosterIDByGitService failed: %v", err) + } +} + +func updateMigrationPosterIDByGitService(tp structs.GitServiceType) error { + provider := tp.Name() + if len(provider) == 0 { + return nil + } + + var repoStart int + const batchSize = 100 + for { + ids, err := models.FindMigratedRepositoryIDs(tp, batchSize, repoStart) + if err != nil { + return err + } + + var start int + for { + users, err := models.FindExternalUsersByProvider(models.FindExternalUserOptions{ + Provider: provider, + Start: start, + Limit: batchSize, + }) + if err != nil { + return err + } + + for _, user := range users { + externalUserID, err := strconv.ParseInt(user.ExternalID, 10, 64) + if err != nil { + log.Warn("Parse externalUser %#v 's userID failed: %v", user, err) + continue + } + for _, id := range ids { + if err = UpdateRepoMigrations(id, externalUserID, user.UserID); err != nil { + log.Error("UpdateRepoMigrations repo %v, github user id %v, user id %v failed: %v", id, user.ExternalID, user.UserID, err) + } + } + } + + if len(users) < batchSize { + break + } + start += len(users) + } + + if len(ids) < batchSize { + return nil + } + repoStart += len(ids) + } } diff --git a/modules/setting/cron.go b/modules/setting/cron.go index c544c6c22878e..9a00a4eec2c8f 100644 --- a/modules/setting/cron.go +++ b/modules/setting/cron.go @@ -49,6 +49,9 @@ var ( Schedule string OlderThan time.Duration } `ini:"cron.deleted_branches_cleanup"` + UpdateMigrationPosterID struct { + Schedule string + } `ini:"cron.update_migration_poster_id"` }{ UpdateMirror: struct { Enabled bool @@ -114,6 +117,11 @@ var ( Schedule: "@every 24h", OlderThan: 24 * time.Hour, }, + UpdateMigrationPosterID: struct { + Schedule string + }{ + Schedule: "@every 10m", + }, } ) diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 3e8e3675a7c0d..5befdf5242dcb 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -166,6 +166,20 @@ const ( GogsService // 5 gogs service ) +// Name represents the service type's name +// WARNNING: the name have to be equal to that on goth's library +func (gt GitServiceType) Name() string { + switch gt { + case GithubService: + return "github" + case GiteaService: + return "gitea" + case GitlabService: + return "gitlab" + } + return "" +} + // MigrateRepoOption options for migrating a repository from an external service type MigrateRepoOption struct { // required: true diff --git a/routers/user/auth.go b/routers/user/auth.go index cec1e88250e8a..a32bbdd9df4b2 100644 --- a/routers/user/auth.go +++ b/routers/user/auth.go @@ -18,7 +18,6 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/recaptcha" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" @@ -604,51 +603,57 @@ func handleOAuth2SignIn(u *models.User, gothUser goth.User, ctx *context.Context // Instead, redirect them to the 2FA authentication page. _, err = models.GetTwoFactorByUID(u.ID) if err != nil { - if models.IsErrTwoFactorNotEnrolled(err) { - err = ctx.Session.Set("uid", u.ID) - if err != nil { - log.Error(fmt.Sprintf("Error setting session: %v", err)) - } - err = ctx.Session.Set("uname", u.Name) - if err != nil { - log.Error(fmt.Sprintf("Error setting session: %v", err)) - } + if !models.IsErrTwoFactorNotEnrolled(err) { + ctx.ServerError("UserSignIn", err) + return + } - // Clear whatever CSRF has right now, force to generate a new one - ctx.SetCookie(setting.CSRFCookieName, "", -1, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true) + err = ctx.Session.Set("uid", u.ID) + if err != nil { + log.Error(fmt.Sprintf("Error setting session: %v", err)) + } + err = ctx.Session.Set("uname", u.Name) + if err != nil { + log.Error(fmt.Sprintf("Error setting session: %v", err)) + } - // Register last login - u.SetLastLogin() - if err := models.UpdateUserCols(u, "last_login_unix"); err != nil { - ctx.ServerError("UpdateUserCols", err) - return - } + // Clear whatever CSRF has right now, force to generate a new one + ctx.SetCookie(setting.CSRFCookieName, "", -1, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true) - if redirectTo := ctx.GetCookie("redirect_to"); len(redirectTo) > 0 { - ctx.SetCookie("redirect_to", "", -1, setting.AppSubURL, "", setting.SessionConfig.Secure, true) - ctx.RedirectToFirst(redirectTo) - return - } + // Register last login + u.SetLastLogin() + if err := models.UpdateUserCols(u, "last_login_unix"); err != nil { + ctx.ServerError("UpdateUserCols", err) + return + } - // update user's migrated comments and issues original id - if gothUser.Provider == "github" { - ids, err := models.FindMigratedRepositoryIDs(structs.GithubService) - if err != nil { - log.Error("FindMigratedRepositoryIDs failed: %v", err) - } else { - for _, id := range ids { - userID, _ := strconv.ParseInt(gothUser.UserID, 10, 64) - if err = migrations.UpdateGithubMigrations(id, userID, u.ID); err != nil { - log.Error("UpdateGithubMigrations repo %v, github user id %v, user id %v failed: %v", id, gothUser.UserID, u.ID, err) - } + // update user's migrated comments and issues original id + if gothUser.Provider == "github" { + ids, err := models.FindMigratedRepositoryIDs(structs.GithubService) + if err != nil { + log.Error("FindMigratedRepositoryIDs failed: %v", err) + } else { + for _, id := range ids { + userID, _ := strconv.ParseInt(gothUser.UserID, 10, 64) + if err = migrations.UpdateGithubMigrations(id, userID, u.ID); err != nil { + log.Error("UpdateGithubMigrations repo %v, github user id %v, user id %v failed: %v", id, gothUser.UserID, u.ID, err) } } } + } - ctx.Redirect(setting.AppSubURL + "/") - } else { - ctx.ServerError("UserSignIn", err) + // update external user information + if err := models.UpdateExternalUser(u, gothUser); err != nil { + log.Error("UpdateExternalUser failed: %v", err) + } + + if redirectTo := ctx.GetCookie("redirect_to"); len(redirectTo) > 0 { + ctx.SetCookie("redirect_to", "", -1, setting.AppSubURL, "", setting.SessionConfig.Secure, true) + ctx.RedirectToFirst(redirectTo) + return } + + ctx.Redirect(setting.AppSubURL + "/") return } @@ -693,7 +698,7 @@ func oAuth2UserLoginCallback(loginSource *models.LoginSource, request *http.Requ } if hasUser { - return user, goth.User{}, nil + return user, gothUser, nil } // search in external linked users @@ -707,7 +712,7 @@ func oAuth2UserLoginCallback(loginSource *models.LoginSource, request *http.Requ } if hasUser { user, err = models.GetUserByID(externalLoginUser.UserID) - return user, goth.User{}, err + return user, gothUser, err } // no user found to login @@ -807,16 +812,18 @@ func LinkAccountPostSignIn(ctx *context.Context, signInForm auth.SignInForm) { // Instead, redirect them to the 2FA authentication page. _, err = models.GetTwoFactorByUID(u.ID) if err != nil { - if models.IsErrTwoFactorNotEnrolled(err) { - err = models.LinkAccountToUser(u, gothUser.(goth.User)) - if err != nil { - ctx.ServerError("UserLinkAccount", err) - } else { - handleSignIn(ctx, u, signInForm.Remember) - } - } else { + if !models.IsErrTwoFactorNotEnrolled(err) { + ctx.ServerError("UserLinkAccount", err) + return + } + + err = models.LinkAccountToUser(u, gothUser.(goth.User)) + if err != nil { ctx.ServerError("UserLinkAccount", err) + return } + + handleSignIn(ctx, u, signInForm.Remember) return } @@ -965,6 +972,11 @@ func LinkAccountPostRegister(ctx *context.Context, cpt *captcha.Captcha, form au } } + // update external user information + if err := models.UpdateExternalUser(u, gothUser.(goth.User)); err != nil { + log.Error("UpdateExternalUser failed: %v", err) + } + // Send confirmation email if setting.Service.RegisterEmailConfirm && u.ID > 1 { mailer.SendActivateAccountMail(ctx.Locale, u) From a36ef4a8ee6cec38d89564a8fb3a1995337754d2 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 26 Sep 2019 21:09:10 +0800 Subject: [PATCH 08/19] fix lint --- modules/migrations/migrate.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 0cf06be6b2283..bbc1dc2d56107 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -58,10 +58,8 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt opts.GitServiceType = structs.PlainGitService downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr) log.Trace("Will migrate from git: %s", opts.CloneAddr) - } else { - if opts.GitServiceType == structs.NotMigrated { - opts.GitServiceType = theFactory.GitServiceType() - } + } else if opts.GitServiceType == structs.NotMigrated { + opts.GitServiceType = theFactory.GitServiceType() } uploader.gitServiceType = opts.GitServiceType From 8d3e224cbb91050591e37959ae723ac532ba84a5 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 26 Sep 2019 21:12:47 +0800 Subject: [PATCH 09/19] fix lint --- models/external_login_user.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/external_login_user.go b/models/external_login_user.go index d49961ee058fd..2c36d4a1de6e0 100644 --- a/models/external_login_user.go +++ b/models/external_login_user.go @@ -162,6 +162,7 @@ func UpdateExternalUser(user *User, gothUser goth.User) error { return err } +// FindExternalUserOptions represents an options to find external users type FindExternalUserOptions struct { Provider string Limit int From 508c661ef000d861efa9d9f50a5421376351074d Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 27 Sep 2019 10:25:36 +0800 Subject: [PATCH 10/19] improve code --- custom/conf/app.ini.sample | 6 +- .../doc/advanced/config-cheat-sheet.en-us.md | 2 +- .../doc/advanced/config-cheat-sheet.zh-cn.md | 2 +- models/external_login_user.go | 43 +++++-------- models/issue.go | 7 ++- models/issue_comment.go | 16 +++-- models/repo.go | 11 ---- modules/migrations/github.go | 2 +- modules/migrations/update.go | 56 +++++------------ modules/setting/cron.go | 2 +- modules/structs/repo.go | 8 +++ routers/api/v1/repo/repo.go | 5 +- routers/user/auth.go | 24 ++----- services/externalaccount/user.go | 62 +++++++++++++++++++ 14 files changed, 130 insertions(+), 116 deletions(-) create mode 100644 services/externalaccount/user.go diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 05e9f658f50f0..fd8d928ede544 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -690,10 +690,10 @@ SCHEDULE = @every 24h ; or only create new users if UPDATE_EXISTING is set to false UPDATE_EXISTING = true -; Update migrated repositories' issues and comments' posterid, it will always be started at start. +; Update migrated repositories' issues and comments' posterid, it will always attempt synchronization when the instance starts. [cron.update_migration_post_id] -; Interval as a duration between each synchronization (default every 10m) -SCHEDULE = @every 10m +; Interval as a duration between each synchronization. (default every 24h) +SCHEDULE = @every 24h [git] ; The path of git executable. If empty, Gitea searches through the PATH environment. diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 1a8c40df35063..b927793a50811 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -421,7 +421,7 @@ NB: You must `REDIRECT_MACARON_LOG` and have `DISABLE_ROUTER_LOG` set to `false` ### Cron - Update Migration Poster ID (`cron.update_migration_post_id`) -- `SCHEDULE`: **@every 10m** : Interval as a duration between each synchronization, it will always be started at start. +- `SCHEDULE`: **@every 24h** : Interval as a duration between each synchronization, it will always attempt synchronization when the instance starts. ## Git (`git`) diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index d35eb17777ae3..f2804b68cb8e2 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -200,7 +200,7 @@ menu: ### Cron - Update Migration Poster ID (`cron.update_migration_post_id`) -- `SCHEDULE`: **@every 10m** : 每次统计的间隔时间。此任务总是在启动时自动进行。 +- `SCHEDULE`: **@every 24h** : 每次统计的间隔时间。此任务总是在启动时自动进行。 ## Git (`git`) diff --git a/models/external_login_user.go b/models/external_login_user.go index 2c36d4a1de6e0..17d5c4c288597 100644 --- a/models/external_login_user.go +++ b/models/external_login_user.go @@ -7,6 +7,7 @@ package models import ( "time" + "code.gitea.io/gitea/modules/structs" "github.com/markbates/goth" "xorm.io/builder" ) @@ -51,40 +52,15 @@ func ListAccountLinks(user *User) ([]*ExternalLoginUser, error) { return externalAccounts, nil } -// LinkAccountToUser link the gothUser to the user -func LinkAccountToUser(user *User, gothUser goth.User) error { - loginSource, err := GetActiveOAuth2LoginSourceByName(gothUser.Provider) - if err != nil { - return err - } - - externalLoginUser := &ExternalLoginUser{ - ExternalID: gothUser.UserID, - UserID: user.ID, - LoginSourceID: loginSource.ID, - RawData: gothUser.RawData, - Provider: gothUser.Provider, - Email: gothUser.Email, - Name: gothUser.Name, - FirstName: gothUser.FirstName, - LastName: gothUser.LastName, - NickName: gothUser.NickName, - Description: gothUser.Description, - AvatarURL: gothUser.AvatarURL, - Location: gothUser.Location, - AccessToken: gothUser.AccessToken, - AccessTokenSecret: gothUser.AccessTokenSecret, - RefreshToken: gothUser.RefreshToken, - ExpiresAt: gothUser.ExpiresAt, - } - - has, err := x.Where("external_id=? AND login_source_id=?", gothUser.UserID, loginSource.ID). +// LinkExternalToUser link the external user to the user +func LinkExternalToUser(user *User, externalLoginUser *ExternalLoginUser) error { + has, err := x.Where("external_id=? AND login_source_id=?", externalLoginUser.ExternalID, externalLoginUser.LoginSourceID). NoAutoCondition(). Exist(externalLoginUser) if err != nil { return err } else if has { - return ErrExternalLoginUserAlreadyExist{gothUser.UserID, user.ID, loginSource.ID} + return ErrExternalLoginUserAlreadyExist{externalLoginUser.ExternalID, user.ID, externalLoginUser.LoginSourceID} } _, err = x.Insert(externalLoginUser) @@ -186,3 +162,12 @@ func FindExternalUsersByProvider(opts FindExternalUserOptions) ([]ExternalLoginU } return users, nil } + +// UpdateMigrationsByType updates all migrated repositories' posterid from gitServiceType to replace originalAuthorID to posterID +func UpdateMigrationsByType(tp structs.GitServiceType, externalUserID, userID int64) error { + if err := UpdateIssuesMigrationsByType(tp, externalUserID, userID); err != nil { + return err + } + + return UpdateCommentsMigrationsByType(tp, externalUserID, userID) +} diff --git a/models/issue.go b/models/issue.go index ebe45e3102a8c..fc675a3ffbe2a 100644 --- a/models/issue.go +++ b/models/issue.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" @@ -1948,10 +1949,10 @@ func (issue *Issue) ResolveMentionsByVisibility(ctx DBContext, doer *User, menti return } -// UpdateIssuesMigrations updates issues' migrations information -func UpdateIssuesMigrations(repoID, originalAuthorID, posterID int64) error { +// UpdateIssuesMigrationsByType updates all migrated repositories' issues from gitServiceType to replace originalAuthorID to posterID +func UpdateIssuesMigrationsByType(gitServiceType structs.GitServiceType, originalAuthorID, posterID int64) error { _, err := x.Table("issue"). - Where("repo_id = ?", repoID). + Where("repo_id IN (SELECT id FROM repository WHERE original_service_type = ?)", gitServiceType). And("original_author_id = ?", originalAuthorID). Update(map[string]interface{}{ "poster_id": posterID, diff --git a/models/issue_comment.go b/models/issue_comment.go index 1457b858095d8..3a090c3b19fd3 100644 --- a/models/issue_comment.go +++ b/models/issue_comment.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/references" + "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" @@ -1023,11 +1024,18 @@ func FetchCodeComments(issue *Issue, currentUser *User) (CodeComments, error) { return fetchCodeComments(x, issue, currentUser) } -// UpdateCommentsMigrations updates comments' migrations information -func UpdateCommentsMigrations(repoID, originalAuthorID, posterID int64) error { +// UpdateCommentsMigrationsByType updates comments' migrations information via given git service type and original id and poster id +func UpdateCommentsMigrationsByType(tp structs.GitServiceType, originalAuthorID, posterID int64) error { _, err := x.Table("comment"). - Where("issue_id IN (SELECT id FROM issue WHERE repo_id = ?)", repoID). - And("original_author_id = ?", originalAuthorID). + Where(builder.In("issue_id", + builder.Select("issue.id"). + From("issue"). + InnerJoin("repository", "issue.repo_id = repository.id"). + Where(builder.Eq{ + "repository.original_service_type": tp, + }), + )). + And("comment.original_author_id = ?", originalAuthorID). Update(map[string]interface{}{ "poster_id": posterID, "original_author": "", diff --git a/models/repo.go b/models/repo.go index 6be0caa2584ea..aa2cf06f326dd 100644 --- a/models/repo.go +++ b/models/repo.go @@ -2782,14 +2782,3 @@ func (repo *Repository) GetOriginalURLHostname() string { return u.Host } - -// FindMigratedRepositoryIDs find all migrated -func FindMigratedRepositoryIDs(tp structs.GitServiceType, limit, start int) ([]int64, error) { - var ids []int64 - err := x.Select("id"). - Where("original_service_type = ?", tp). - Limit(limit, start). - Table("repository"). - Find(&ids) - return ids, err -} diff --git a/modules/migrations/github.go b/modules/migrations/github.go index c3bc2cd873418..00d137a3de69a 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -40,7 +40,7 @@ func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error return false, err } - return u.Host == "github.com" && opts.AuthUsername != "", nil + return strings.EqualFold(u.Host, "github.com") && opts.AuthUsername != "", nil } // New returns a Downloader related to this factory according MigrateOptions diff --git a/modules/migrations/update.go b/modules/migrations/update.go index 7b22a93a30b86..08c4ad701eef9 100644 --- a/modules/migrations/update.go +++ b/modules/migrations/update.go @@ -12,16 +12,6 @@ import ( "code.gitea.io/gitea/modules/structs" ) -// UpdateRepoMigrations will update posterid on issues/comments/prs when external login user -// login or when migrating -func UpdateRepoMigrations(repoID, externalUserID, userID int64) error { - if err := models.UpdateIssuesMigrations(repoID, externalUserID, userID); err != nil { - return err - } - - return models.UpdateCommentsMigrations(repoID, externalUserID, userID) -} - // UpdateMigrationPosterID updates all migrated repositories' issues and comments posterID func UpdateMigrationPosterID() { if err := updateMigrationPosterIDByGitService(structs.GithubService); err != nil { @@ -35,47 +25,33 @@ func updateMigrationPosterIDByGitService(tp structs.GitServiceType) error { return nil } - var repoStart int const batchSize = 100 + var start int for { - ids, err := models.FindMigratedRepositoryIDs(tp, batchSize, repoStart) + users, err := models.FindExternalUsersByProvider(models.FindExternalUserOptions{ + Provider: provider, + Start: start, + Limit: batchSize, + }) if err != nil { return err } - var start int - for { - users, err := models.FindExternalUsersByProvider(models.FindExternalUserOptions{ - Provider: provider, - Start: start, - Limit: batchSize, - }) + for _, user := range users { + externalUserID, err := strconv.ParseInt(user.ExternalID, 10, 64) if err != nil { - return err + log.Warn("Parse externalUser %#v 's userID failed: %v", user, err) + continue } - - for _, user := range users { - externalUserID, err := strconv.ParseInt(user.ExternalID, 10, 64) - if err != nil { - log.Warn("Parse externalUser %#v 's userID failed: %v", user, err) - continue - } - for _, id := range ids { - if err = UpdateRepoMigrations(id, externalUserID, user.UserID); err != nil { - log.Error("UpdateRepoMigrations repo %v, github user id %v, user id %v failed: %v", id, user.ExternalID, user.UserID, err) - } - } - } - - if len(users) < batchSize { - break + if err := models.UpdateMigrationsByType(tp, externalUserID, user.UserID); err != nil { + log.Error("UpdateMigrationsByType type %v, github user id %v, user id %v failed: %v", tp, user.ExternalID, user.UserID, err) } - start += len(users) } - if len(ids) < batchSize { - return nil + if len(users) < batchSize { + break } - repoStart += len(ids) + start += len(users) } + return nil } diff --git a/modules/setting/cron.go b/modules/setting/cron.go index 9a00a4eec2c8f..77f55168aa3b7 100644 --- a/modules/setting/cron.go +++ b/modules/setting/cron.go @@ -120,7 +120,7 @@ var ( UpdateMigrationPosterID: struct { Schedule string }{ - Schedule: "@every 10m", + Schedule: "@every 24h", }, } ) diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 5befdf5242dcb..59af47ee00adb 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -180,6 +180,14 @@ func (gt GitServiceType) Name() string { return "" } +var ( + // all git services supported to migrate issues/labels/prs and etc. + // TODO: add to this list after new git service added + SupportedFullGitService = []GitServiceType{ + GithubService, + } +) + // MigrateRepoOption options for migrating a repository from an external service type MigrateRepoOption struct { // required: true diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index e7c1577d5a809..a4417107ee384 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -8,6 +8,7 @@ package repo import ( "fmt" "net/http" + "net/url" "strings" "code.gitea.io/gitea/models" @@ -399,8 +400,8 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) { } var gitServiceType = structs.PlainGitService - // TODO: this should be chosen by UI when from a customerize git service domain - if strings.HasPrefix(remoteAddr, "https://github.com") || strings.HasPrefix(remoteAddr, "http://github.com") { + u, err := url.Parse(remoteAddr) + if err == nil && strings.EqualFold(u.Host, "github.com") { gitServiceType = structs.GithubService } diff --git a/routers/user/auth.go b/routers/user/auth.go index a32bbdd9df4b2..212d535a06498 100644 --- a/routers/user/auth.go +++ b/routers/user/auth.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "net/http" - "strconv" "strings" "code.gitea.io/gitea/models" @@ -20,9 +19,9 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/recaptcha" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/services/externalaccount" "code.gitea.io/gitea/services/mailer" "gitea.com/macaron/captcha" @@ -279,7 +278,7 @@ func TwoFactorPost(ctx *context.Context, form auth.TwoFactorAuthForm) { return } - err = models.LinkAccountToUser(u, gothUser.(goth.User)) + err = externalaccount.LinkAccountToUser(u, gothUser.(goth.User)) if err != nil { ctx.ServerError("UserSignIn", err) return @@ -454,7 +453,7 @@ func U2FSign(ctx *context.Context, signResp u2f.SignResponse) { return } - err = models.LinkAccountToUser(user, gothUser.(goth.User)) + err = externalaccount.LinkAccountToUser(user, gothUser.(goth.User)) if err != nil { ctx.ServerError("UserSignIn", err) return @@ -627,21 +626,6 @@ func handleOAuth2SignIn(u *models.User, gothUser goth.User, ctx *context.Context return } - // update user's migrated comments and issues original id - if gothUser.Provider == "github" { - ids, err := models.FindMigratedRepositoryIDs(structs.GithubService) - if err != nil { - log.Error("FindMigratedRepositoryIDs failed: %v", err) - } else { - for _, id := range ids { - userID, _ := strconv.ParseInt(gothUser.UserID, 10, 64) - if err = migrations.UpdateGithubMigrations(id, userID, u.ID); err != nil { - log.Error("UpdateGithubMigrations repo %v, github user id %v, user id %v failed: %v", id, gothUser.UserID, u.ID, err) - } - } - } - } - // update external user information if err := models.UpdateExternalUser(u, gothUser); err != nil { log.Error("UpdateExternalUser failed: %v", err) @@ -817,7 +801,7 @@ func LinkAccountPostSignIn(ctx *context.Context, signInForm auth.SignInForm) { return } - err = models.LinkAccountToUser(u, gothUser.(goth.User)) + err = externalaccount.LinkAccountToUser(u, gothUser.(goth.User)) if err != nil { ctx.ServerError("UserLinkAccount", err) return diff --git a/services/externalaccount/user.go b/services/externalaccount/user.go new file mode 100644 index 0000000000000..b4dc7cf0f8987 --- /dev/null +++ b/services/externalaccount/user.go @@ -0,0 +1,62 @@ +package externalaccount + +import ( + "strconv" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/structs" + + "github.com/markbates/goth" +) + +// LinkAccountToUser link the gothUser to the user +func LinkAccountToUser(user *models.User, gothUser goth.User) error { + loginSource, err := models.GetActiveOAuth2LoginSourceByName(gothUser.Provider) + if err != nil { + return err + } + + externalLoginUser := &models.ExternalLoginUser{ + ExternalID: gothUser.UserID, + UserID: user.ID, + LoginSourceID: loginSource.ID, + RawData: gothUser.RawData, + Provider: gothUser.Provider, + Email: gothUser.Email, + Name: gothUser.Name, + FirstName: gothUser.FirstName, + LastName: gothUser.LastName, + NickName: gothUser.NickName, + Description: gothUser.Description, + AvatarURL: gothUser.AvatarURL, + Location: gothUser.Location, + AccessToken: gothUser.AccessToken, + AccessTokenSecret: gothUser.AccessTokenSecret, + RefreshToken: gothUser.RefreshToken, + ExpiresAt: gothUser.ExpiresAt, + } + + if err := models.LinkExternalToUser(user, externalLoginUser); err != nil { + return err + } + + externalID, err := strconv.ParseInt(externalLoginUser.ExternalID, 10, 64) + if err != nil { + return err + } + + var tp structs.GitServiceType + for _, s := range structs.SupportedFullGitService { + if strings.EqualFold(s.Name(), gothUser.Provider) { + tp = s + break + } + } + + if tp.Name() != "" { + return models.UpdateMigrationsByType(tp, externalID, user.ID) + } + + return nil +} From 16a6e624591f4308842db088ebd08445c7280d5b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 27 Sep 2019 10:30:22 +0800 Subject: [PATCH 11/19] fix lint --- modules/structs/repo.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 59af47ee00adb..9f1b7529c3df2 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -176,6 +176,8 @@ func (gt GitServiceType) Name() string { return "gitea" case GitlabService: return "gitlab" + case GogsService: + return "gogs" } return "" } From d03d5330785acee168717a5e4ae25bccaf0dc1a0 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 27 Sep 2019 11:02:00 +0800 Subject: [PATCH 12/19] improve code --- .../doc/advanced/config-cheat-sheet.zh-cn.md | 2 +- models/external_login_user.go | 4 ++-- models/migrations/v99.go | 21 +++++++++++++++++++ services/externalaccount/user.go | 4 ++++ 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index f2804b68cb8e2..ab73e2059eca0 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -200,7 +200,7 @@ menu: ### Cron - Update Migration Poster ID (`cron.update_migration_post_id`) -- `SCHEDULE`: **@every 24h** : 每次统计的间隔时间。此任务总是在启动时自动进行。 +- `SCHEDULE`: **@every 24h** : 每次同步的间隔时间。此任务总是在启动时自动进行。 ## Git (`git`) diff --git a/models/external_login_user.go b/models/external_login_user.go index 17d5c4c288597..058b37418b850 100644 --- a/models/external_login_user.go +++ b/models/external_login_user.go @@ -14,11 +14,11 @@ import ( // ExternalLoginUser makes the connecting between some existing user and additional external login sources type ExternalLoginUser struct { - ExternalID string `xorm:"VARCHAR(50) pk NOT NULL"` + ExternalID string `xorm:"pk NOT NULL"` UserID int64 `xorm:"INDEX NOT NULL"` LoginSourceID int64 `xorm:"pk NOT NULL"` RawData map[string]interface{} `xorm:"TEXT"` - Provider string `xorm:"index"` + Provider string `xorm:"index VARCHAR(25)"` Email string Name string FirstName string diff --git a/models/migrations/v99.go b/models/migrations/v99.go index 3eb287af6c960..c377eb67491be 100644 --- a/models/migrations/v99.go +++ b/models/migrations/v99.go @@ -26,8 +26,29 @@ func addTaskTable(x *xorm.Engine) error { Created timeutil.TimeStamp `xorm:"created"` } +<<<<<<< HEAD type Repository struct { Status int `xorm:"NOT NULL DEFAULT 0"` +======= + type ExternalLoginUser struct { + ExternalID string `xorm:"pk NOT NULL"` + UserID int64 `xorm:"INDEX NOT NULL"` + LoginSourceID int64 `xorm:"pk NOT NULL"` + RawData map[string]interface{} `xorm:"TEXT"` + Provider string `xorm:"index VARCHAR(25)"` + Email string + Name string + FirstName string + LastName string + NickName string + Description string + AvatarURL string + Location string + AccessToken string + AccessTokenSecret string + RefreshToken string + ExpiresAt time.Time +>>>>>>> improve code } return x.Sync2(new(Task), new(Repository)) diff --git a/services/externalaccount/user.go b/services/externalaccount/user.go index b4dc7cf0f8987..800546f123114 100644 --- a/services/externalaccount/user.go +++ b/services/externalaccount/user.go @@ -1,3 +1,7 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + package externalaccount import ( From 84a731590e71f891bb7505608e4fb175bec7fc1c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 6 Oct 2019 09:49:10 +0800 Subject: [PATCH 13/19] replace releases publish id to actual author id --- models/external_login_user.go | 6 ++++- models/release.go | 13 ++++++++++ modules/migrations/gitea.go | 46 ++++++++++++++++++++++++----------- 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/models/external_login_user.go b/models/external_login_user.go index 058b37418b850..7b58bd0a7d140 100644 --- a/models/external_login_user.go +++ b/models/external_login_user.go @@ -169,5 +169,9 @@ func UpdateMigrationsByType(tp structs.GitServiceType, externalUserID, userID in return err } - return UpdateCommentsMigrationsByType(tp, externalUserID, userID) + if err := UpdateCommentsMigrationsByType(tp, externalUserID, userID); err != nil { + return err + } + + return UpdateReleasesMigrationsByType(tp, externalUserID, userID) } diff --git a/models/release.go b/models/release.go index 243cc2fa3c31a..295f2027b72d8 100644 --- a/models/release.go +++ b/models/release.go @@ -366,3 +366,16 @@ func SyncReleasesWithTags(repo *Repository, gitRepo *git.Repository) error { } return nil } + +// UpdateReleasesMigrationsByType updates all migrated repositories' releases from gitServiceType to replace originalAuthorID to posterID +func UpdateReleasesMigrationsByType(gitServiceType GitServiceType, originalAuthorID, posterID int64) error { + _, err := x.Table("release"). + Where("repo_id IN (SELECT id FROM repository WHERE original_service_type = ?)", gitServiceType). + And("original_author_id = ?", originalAuthorID). + Update(map[string]interface{}{ + "publisher_id": posterID, + "original_author": "", + "original_author_id": 0, + }) + return err +} diff --git a/modules/migrations/gitea.go b/modules/migrations/gitea.go index 4accfc281b79e..2452a7a88322f 100644 --- a/modules/migrations/gitea.go +++ b/modules/migrations/gitea.go @@ -198,20 +198,38 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { var rels = make([]*models.Release, 0, len(releases)) for _, release := range releases { var rel = models.Release{ - RepoID: g.repo.ID, - PublisherID: g.doer.ID, - TagName: release.TagName, - LowerTagName: strings.ToLower(release.TagName), - Target: release.TargetCommitish, - Title: release.Name, - Sha1: release.TargetCommitish, - Note: release.Body, - IsDraft: release.Draft, - IsPrerelease: release.Prerelease, - IsTag: false, - CreatedUnix: timeutil.TimeStamp(release.Created.Unix()), - OriginalAuthor: release.PublisherName, - OriginalAuthorID: release.PublisherID, + RepoID: g.repo.ID, + TagName: release.TagName, + LowerTagName: strings.ToLower(release.TagName), + Target: release.TargetCommitish, + Title: release.Name, + Sha1: release.TargetCommitish, + Note: release.Body, + IsDraft: release.Draft, + IsPrerelease: release.Prerelease, + IsTag: false, + CreatedUnix: timeutil.TimeStamp(release.Created.Unix()), + } + + userid, ok := g.userMap[release.PublisherID] + tp := g.gitServiceType.Name() + if !ok && tp != "" { + var err error + userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", release.PublisherID)) + if err != nil { + log.Error("GetUserIDByExternalUserID: %v", err) + } + if userid > 0 { + g.userMap[release.PublisherID] = userid + } + } + + if userid > 0 { + rel.PublisherID = userid + } else { + rel.PublisherID = g.doer.ID + rel.OriginalAuthor = release.PublisherName + rel.OriginalAuthorID = release.PublisherID } // calc NumCommits From 776869af68a9013b66c4080d87d3fe5b7eb11851 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 14 Oct 2019 00:44:10 +0800 Subject: [PATCH 14/19] fix import --- models/external_login_user.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/external_login_user.go b/models/external_login_user.go index 7b58bd0a7d140..0a076eada619d 100644 --- a/models/external_login_user.go +++ b/models/external_login_user.go @@ -8,6 +8,7 @@ import ( "time" "code.gitea.io/gitea/modules/structs" + "github.com/markbates/goth" "xorm.io/builder" ) From fc5e550ebf1d1d69ba6bf6712838b187ad3875d8 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 14 Oct 2019 09:27:42 +0800 Subject: [PATCH 15/19] fix bug --- models/migrations/v99.go | 21 --------------------- models/release.go | 3 ++- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/models/migrations/v99.go b/models/migrations/v99.go index c377eb67491be..3eb287af6c960 100644 --- a/models/migrations/v99.go +++ b/models/migrations/v99.go @@ -26,29 +26,8 @@ func addTaskTable(x *xorm.Engine) error { Created timeutil.TimeStamp `xorm:"created"` } -<<<<<<< HEAD type Repository struct { Status int `xorm:"NOT NULL DEFAULT 0"` -======= - type ExternalLoginUser struct { - ExternalID string `xorm:"pk NOT NULL"` - UserID int64 `xorm:"INDEX NOT NULL"` - LoginSourceID int64 `xorm:"pk NOT NULL"` - RawData map[string]interface{} `xorm:"TEXT"` - Provider string `xorm:"index VARCHAR(25)"` - Email string - Name string - FirstName string - LastName string - NickName string - Description string - AvatarURL string - Location string - AccessToken string - AccessTokenSecret string - RefreshToken string - ExpiresAt time.Time ->>>>>>> improve code } return x.Sync2(new(Task), new(Repository)) diff --git a/models/release.go b/models/release.go index 295f2027b72d8..03685e0a44c79 100644 --- a/models/release.go +++ b/models/release.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" @@ -368,7 +369,7 @@ func SyncReleasesWithTags(repo *Repository, gitRepo *git.Repository) error { } // UpdateReleasesMigrationsByType updates all migrated repositories' releases from gitServiceType to replace originalAuthorID to posterID -func UpdateReleasesMigrationsByType(gitServiceType GitServiceType, originalAuthorID, posterID int64) error { +func UpdateReleasesMigrationsByType(gitServiceType structs.GitServiceType, originalAuthorID, posterID int64) error { _, err := x.Table("release"). Where("repo_id IN (SELECT id FROM repository WHERE original_service_type = ?)", gitServiceType). And("original_author_id = ?", originalAuthorID). From 4d4c83735fffed1b9568f8b9955288efdc0ba8d2 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 14 Oct 2019 09:43:20 +0800 Subject: [PATCH 16/19] fix lint --- modules/structs/repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 9f1b7529c3df2..be6a3d4b43e4f 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -183,7 +183,7 @@ func (gt GitServiceType) Name() string { } var ( - // all git services supported to migrate issues/labels/prs and etc. + // SupportedFullGitService represents all git services supported to migrate issues/labels/prs and etc. // TODO: add to this list after new git service added SupportedFullGitService = []GitServiceType{ GithubService, From 8e76be373049e06bfaf797cdd863a863c516bcc1 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 14 Oct 2019 10:08:46 +0800 Subject: [PATCH 17/19] fix rawdata definition --- models/external_login_user.go | 2 +- models/migrations/v100.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/external_login_user.go b/models/external_login_user.go index 0a076eada619d..48122776d8f36 100644 --- a/models/external_login_user.go +++ b/models/external_login_user.go @@ -18,7 +18,7 @@ type ExternalLoginUser struct { ExternalID string `xorm:"pk NOT NULL"` UserID int64 `xorm:"INDEX NOT NULL"` LoginSourceID int64 `xorm:"pk NOT NULL"` - RawData map[string]interface{} `xorm:"TEXT"` + RawData map[string]interface{} `xorm:"TEXT JSON"` Provider string `xorm:"index VARCHAR(25)"` Email string Name string diff --git a/models/migrations/v100.go b/models/migrations/v100.go index 36043c32b4b87..ac3b73e2ad2a9 100644 --- a/models/migrations/v100.go +++ b/models/migrations/v100.go @@ -63,7 +63,7 @@ func updateMigrationServiceTypes(x *xorm.Engine) error { ExternalID string `xorm:"pk NOT NULL"` UserID int64 `xorm:"INDEX NOT NULL"` LoginSourceID int64 `xorm:"pk NOT NULL"` - RawData map[string]interface{} `xorm:"TEXT"` + RawData map[string]interface{} `xorm:"TEXT JSON"` Provider string `xorm:"index VARCHAR(25)"` Email string Name string From 4e7165a3a1f3f499b9649ea8be9f81655d5e643c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 14 Oct 2019 10:47:04 +0800 Subject: [PATCH 18/19] fix some bugs --- models/external_login_user.go | 5 ++++- modules/migrations/update.go | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/models/external_login_user.go b/models/external_login_user.go index 48122776d8f36..5058fd1b4b781 100644 --- a/models/external_login_user.go +++ b/models/external_login_user.go @@ -157,7 +157,10 @@ func (opts FindExternalUserOptions) toConds() builder.Cond { // FindExternalUsersByProvider represents external users via provider func FindExternalUsersByProvider(opts FindExternalUserOptions) ([]ExternalLoginUser, error) { var users []ExternalLoginUser - err := x.Where(opts.toConds()).Limit(opts.Limit, opts.Start).Find(&users) + err := x.Where(opts.toConds()). + Limit(opts.Limit, opts.Start). + Asc("id"). + Find(&users) if err != nil { return nil, err } diff --git a/modules/migrations/update.go b/modules/migrations/update.go index 08c4ad701eef9..205ebe689123a 100644 --- a/modules/migrations/update.go +++ b/modules/migrations/update.go @@ -14,8 +14,10 @@ import ( // UpdateMigrationPosterID updates all migrated repositories' issues and comments posterID func UpdateMigrationPosterID() { - if err := updateMigrationPosterIDByGitService(structs.GithubService); err != nil { - log.Error("updateMigrationPosterIDByGitService failed: %v", err) + for _, gitService := range structs.SupportedFullGitService { + if err := updateMigrationPosterIDByGitService(gitService); err != nil { + log.Error("updateMigrationPosterIDByGitService failed: %v", err) + } } } From a8773de895a6e2437b18bcf91e9cd488751770c4 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 14 Oct 2019 13:06:49 +0800 Subject: [PATCH 19/19] fix error message --- modules/migrations/update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/migrations/update.go b/modules/migrations/update.go index 205ebe689123a..df626ddd955e9 100644 --- a/modules/migrations/update.go +++ b/modules/migrations/update.go @@ -46,7 +46,7 @@ func updateMigrationPosterIDByGitService(tp structs.GitServiceType) error { continue } if err := models.UpdateMigrationsByType(tp, externalUserID, user.UserID); err != nil { - log.Error("UpdateMigrationsByType type %v, github user id %v, user id %v failed: %v", tp, user.ExternalID, user.UserID, err) + log.Error("UpdateMigrationsByType type %s external user id %v to local user id %v failed: %v", tp.Name(), user.ExternalID, user.UserID, err) } }