Skip to content

Commit e308d25

Browse files
authored
Cache repository default branch commit status to reduce query on commit status table (#29444)
After repository commit status has been introduced on dashaboard, the most top SQL comes from `GetLatestCommitStatusForPairs`. This PR adds a cache for the repository's default branch's latest combined commit status. When a new commit status updated, the cache will be marked as invalid. <img width="998" alt="image" src="https://github.com/go-gitea/gitea/assets/81045/76759de7-3a83-4d54-8571-278f5422aed3">
1 parent 90a3f2d commit e308d25

File tree

4 files changed

+145
-70
lines changed

4 files changed

+145
-70
lines changed

routers/api/v1/repo/status.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import (
1414
"code.gitea.io/gitea/routers/api/v1/utils"
1515
"code.gitea.io/gitea/services/context"
1616
"code.gitea.io/gitea/services/convert"
17-
files_service "code.gitea.io/gitea/services/repository/files"
17+
commitstatus_service "code.gitea.io/gitea/services/repository/commitstatus"
1818
)
1919

2020
// NewCommitStatus creates a new CommitStatus
@@ -64,7 +64,7 @@ func NewCommitStatus(ctx *context.APIContext) {
6464
Description: form.Description,
6565
Context: form.Context,
6666
}
67-
if err := files_service.CreateCommitStatus(ctx, ctx.Repo.Repository, ctx.Doer, sha, status); err != nil {
67+
if err := commitstatus_service.CreateCommitStatus(ctx, ctx.Repo.Repository, ctx.Doer, sha, status); err != nil {
6868
ctx.Error(http.StatusInternalServerError, "CreateCommitStatus", err)
6969
return
7070
}

routers/web/repo/repo.go

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import (
3535
"code.gitea.io/gitea/services/forms"
3636
repo_service "code.gitea.io/gitea/services/repository"
3737
archiver_service "code.gitea.io/gitea/services/repository/archiver"
38+
commitstatus_service "code.gitea.io/gitea/services/repository/commitstatus"
3839
)
3940

4041
const (
@@ -634,30 +635,14 @@ func SearchRepo(ctx *context.Context) {
634635
return
635636
}
636637

637-
// collect the latest commit of each repo
638-
// at most there are dozens of repos (limited by MaxResponseItems), so it's not a big problem at the moment
639-
repoBranchNames := make(map[int64]string, len(repos))
640-
for _, repo := range repos {
641-
repoBranchNames[repo.ID] = repo.DefaultBranch
642-
}
643-
644-
repoIDsToLatestCommitSHAs, err := git_model.FindBranchesByRepoAndBranchName(ctx, repoBranchNames)
638+
latestCommitStatuses, err := commitstatus_service.FindReposLastestCommitStatuses(ctx, repos)
645639
if err != nil {
646-
log.Error("FindBranchesByRepoAndBranchName: %v", err)
647-
return
648-
}
649-
650-
// call the database O(1) times to get the commit statuses for all repos
651-
repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoIDsToLatestCommitSHAs, db.ListOptionsAll)
652-
if err != nil {
653-
log.Error("GetLatestCommitStatusForPairs: %v", err)
640+
log.Error("FindReposLastestCommitStatuses: %v", err)
654641
return
655642
}
656643

657644
results := make([]*repo_service.WebSearchRepository, len(repos))
658645
for i, repo := range repos {
659-
latestCommitStatus := git_model.CalcCommitStatus(repoToItsLatestCommitStatuses[repo.ID])
660-
661646
results[i] = &repo_service.WebSearchRepository{
662647
Repository: &api.Repository{
663648
ID: repo.ID,
@@ -671,8 +656,11 @@ func SearchRepo(ctx *context.Context) {
671656
Link: repo.Link(),
672657
Internal: !repo.IsPrivate && repo.Owner.Visibility == api.VisibleTypePrivate,
673658
},
674-
LatestCommitStatus: latestCommitStatus,
675-
LocaleLatestCommitStatus: latestCommitStatus.LocaleString(ctx.Locale),
659+
}
660+
661+
if latestCommitStatuses[i] != nil {
662+
results[i].LatestCommitStatus = latestCommitStatuses[i]
663+
results[i].LocaleLatestCommitStatus = latestCommitStatuses[i].LocaleString(ctx.Locale)
676664
}
677665
}
678666

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
// Copyright 2024 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package commitstatus
5+
6+
import (
7+
"context"
8+
"crypto/sha256"
9+
"fmt"
10+
11+
"code.gitea.io/gitea/models/db"
12+
git_model "code.gitea.io/gitea/models/git"
13+
repo_model "code.gitea.io/gitea/models/repo"
14+
user_model "code.gitea.io/gitea/models/user"
15+
"code.gitea.io/gitea/modules/cache"
16+
"code.gitea.io/gitea/modules/git"
17+
"code.gitea.io/gitea/modules/gitrepo"
18+
"code.gitea.io/gitea/modules/log"
19+
api "code.gitea.io/gitea/modules/structs"
20+
"code.gitea.io/gitea/services/automerge"
21+
)
22+
23+
func getCacheKey(repoID int64, brancheName string) string {
24+
hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%d:%s", repoID, brancheName)))
25+
return fmt.Sprintf("commit_status:%x", hashBytes)
26+
}
27+
28+
func updateCommitStatusCache(ctx context.Context, repoID int64, branchName string, status api.CommitStatusState) error {
29+
c := cache.GetCache()
30+
return c.Put(getCacheKey(repoID, branchName), string(status), 3*24*60)
31+
}
32+
33+
func deleteCommitStatusCache(ctx context.Context, repoID int64, branchName string) error {
34+
c := cache.GetCache()
35+
return c.Delete(getCacheKey(repoID, branchName))
36+
}
37+
38+
// CreateCommitStatus creates a new CommitStatus given a bunch of parameters
39+
// NOTE: All text-values will be trimmed from whitespaces.
40+
// Requires: Repo, Creator, SHA
41+
func CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, creator *user_model.User, sha string, status *git_model.CommitStatus) error {
42+
repoPath := repo.RepoPath()
43+
44+
// confirm that commit is exist
45+
gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo)
46+
if err != nil {
47+
return fmt.Errorf("OpenRepository[%s]: %w", repoPath, err)
48+
}
49+
defer closer.Close()
50+
51+
objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName)
52+
53+
commit, err := gitRepo.GetCommit(sha)
54+
if err != nil {
55+
return fmt.Errorf("GetCommit[%s]: %w", sha, err)
56+
}
57+
if len(sha) != objectFormat.FullLength() {
58+
// use complete commit sha
59+
sha = commit.ID.String()
60+
}
61+
62+
if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
63+
Repo: repo,
64+
Creator: creator,
65+
SHA: commit.ID,
66+
CommitStatus: status,
67+
}); err != nil {
68+
return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
69+
}
70+
71+
defaultBranchCommit, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
72+
if err != nil {
73+
return fmt.Errorf("GetBranchCommit[%s]: %w", repo.DefaultBranch, err)
74+
}
75+
76+
if commit.ID.String() == defaultBranchCommit.ID.String() { // since one commit status updated, the combined commit status should be invalid
77+
if err := deleteCommitStatusCache(ctx, repo.ID, repo.DefaultBranch); err != nil {
78+
log.Error("deleteCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err)
79+
}
80+
}
81+
82+
if status.State.IsSuccess() {
83+
if err := automerge.MergeScheduledPullRequest(ctx, sha, repo); err != nil {
84+
return fmt.Errorf("MergeScheduledPullRequest[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
85+
}
86+
}
87+
88+
return nil
89+
}
90+
91+
// FindReposLastestCommitStatuses loading repository default branch latest combinded commit status with cache
92+
func FindReposLastestCommitStatuses(ctx context.Context, repos []*repo_model.Repository) ([]*git_model.CommitStatus, error) {
93+
results := make([]*git_model.CommitStatus, len(repos))
94+
c := cache.GetCache()
95+
96+
for i, repo := range repos {
97+
status, ok := c.Get(getCacheKey(repo.ID, repo.DefaultBranch)).(string)
98+
if ok && status != "" {
99+
results[i] = &git_model.CommitStatus{State: api.CommitStatusState(status)}
100+
}
101+
}
102+
103+
// collect the latest commit of each repo
104+
// at most there are dozens of repos (limited by MaxResponseItems), so it's not a big problem at the moment
105+
repoBranchNames := make(map[int64]string, len(repos))
106+
for i, repo := range repos {
107+
if results[i] == nil {
108+
repoBranchNames[repo.ID] = repo.DefaultBranch
109+
}
110+
}
111+
112+
repoIDsToLatestCommitSHAs, err := git_model.FindBranchesByRepoAndBranchName(ctx, repoBranchNames)
113+
if err != nil {
114+
return nil, fmt.Errorf("FindBranchesByRepoAndBranchName: %v", err)
115+
}
116+
117+
// call the database O(1) times to get the commit statuses for all repos
118+
repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoIDsToLatestCommitSHAs, db.ListOptionsAll)
119+
if err != nil {
120+
return nil, fmt.Errorf("GetLatestCommitStatusForPairs: %v", err)
121+
}
122+
123+
for i, repo := range repos {
124+
if results[i] == nil {
125+
results[i] = git_model.CalcCommitStatus(repoToItsLatestCommitStatuses[repo.ID])
126+
if results[i].State != "" {
127+
if err := updateCommitStatusCache(ctx, repo.ID, repo.DefaultBranch, results[i].State); err != nil {
128+
log.Error("updateCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err)
129+
}
130+
}
131+
}
132+
}
133+
134+
return results, nil
135+
}

services/repository/files/commit.go

Lines changed: 0 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -5,61 +5,13 @@ package files
55

66
import (
77
"context"
8-
"fmt"
98

109
asymkey_model "code.gitea.io/gitea/models/asymkey"
11-
git_model "code.gitea.io/gitea/models/git"
1210
repo_model "code.gitea.io/gitea/models/repo"
13-
user_model "code.gitea.io/gitea/models/user"
1411
"code.gitea.io/gitea/modules/git"
15-
"code.gitea.io/gitea/modules/gitrepo"
1612
"code.gitea.io/gitea/modules/structs"
17-
"code.gitea.io/gitea/services/automerge"
1813
)
1914

20-
// CreateCommitStatus creates a new CommitStatus given a bunch of parameters
21-
// NOTE: All text-values will be trimmed from whitespaces.
22-
// Requires: Repo, Creator, SHA
23-
func CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, creator *user_model.User, sha string, status *git_model.CommitStatus) error {
24-
repoPath := repo.RepoPath()
25-
26-
// confirm that commit is exist
27-
gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo)
28-
if err != nil {
29-
return fmt.Errorf("OpenRepository[%s]: %w", repoPath, err)
30-
}
31-
defer closer.Close()
32-
33-
objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName)
34-
35-
commit, err := gitRepo.GetCommit(sha)
36-
if err != nil {
37-
gitRepo.Close()
38-
return fmt.Errorf("GetCommit[%s]: %w", sha, err)
39-
} else if len(sha) != objectFormat.FullLength() {
40-
// use complete commit sha
41-
sha = commit.ID.String()
42-
}
43-
gitRepo.Close()
44-
45-
if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
46-
Repo: repo,
47-
Creator: creator,
48-
SHA: commit.ID,
49-
CommitStatus: status,
50-
}); err != nil {
51-
return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
52-
}
53-
54-
if status.State.IsSuccess() {
55-
if err := automerge.MergeScheduledPullRequest(ctx, sha, repo); err != nil {
56-
return fmt.Errorf("MergeScheduledPullRequest[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
57-
}
58-
}
59-
60-
return nil
61-
}
62-
6315
// CountDivergingCommits determines how many commits a branch is ahead or behind the repository's base branch
6416
func CountDivergingCommits(ctx context.Context, repo *repo_model.Repository, branch string) (*git.DivergeObject, error) {
6517
divergence, err := git.GetDivergingCommits(ctx, repo.RepoPath(), repo.DefaultBranch, branch)

0 commit comments

Comments
 (0)