Skip to content

Commit a2cfcdb

Browse files
zeripathzeripath
authored
Slightly simplify LastCommitCache (#20444)
The LastCommitCache code is a little complex and there is unnecessary duplication between the gogit and nogogit variants. This PR adds the LastCommitCache as a field to the git.Repository and pre-creates it in the ReferencesGit helpers etc. There has been some simplification and unification of the variant code. Signed-off-by: Andrew Thornton <[email protected]>
1 parent 690272d commit a2cfcdb

19 files changed

+177
-182
lines changed

‎modules/context/repo.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1001,6 +1001,8 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context
10011001
return
10021002
}
10031003
ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount
1004+
ctx.Repo.GitRepo.LastCommitCache = git.NewLastCommitCache(ctx.Repo.CommitsCount, ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, cache.GetCache())
1005+
10041006
return cancel
10051007
}
10061008
}

‎modules/git/commit.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@ func (c *Commit) ParentCount() int {
8080

8181
// GetCommitByPath return the commit of relative path object.
8282
func (c *Commit) GetCommitByPath(relpath string) (*Commit, error) {
83+
if c.repo.LastCommitCache != nil {
84+
return c.repo.LastCommitCache.GetCommitByPath(c.ID.String(), relpath)
85+
}
8386
return c.repo.getCommitByPathWithID(c.ID, relpath)
8487
}
8588

‎modules/git/commit_info_gogit.go

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import (
1717
)
1818

1919
// GetCommitsInfo gets information of all commits that are corresponding to these entries
20-
func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) {
20+
func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string) ([]CommitInfo, *Commit, error) {
2121
entryPaths := make([]string, len(tes)+1)
2222
// Get the commit for the treePath itself
2323
entryPaths[0] = ""
@@ -35,15 +35,15 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
3535
return nil, nil, err
3636
}
3737

38-
var revs map[string]*object.Commit
39-
if cache != nil {
38+
var revs map[string]*Commit
39+
if commit.repo.LastCommitCache != nil {
4040
var unHitPaths []string
41-
revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
41+
revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, commit.repo.LastCommitCache)
4242
if err != nil {
4343
return nil, nil, err
4444
}
4545
if len(unHitPaths) > 0 {
46-
revs2, err := GetLastCommitForPaths(ctx, cache, c, treePath, unHitPaths)
46+
revs2, err := GetLastCommitForPaths(ctx, commit.repo.LastCommitCache, c, treePath, unHitPaths)
4747
if err != nil {
4848
return nil, nil, err
4949
}
@@ -68,8 +68,7 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
6868
}
6969

7070
// Check if we have found a commit for this entry in time
71-
if rev, ok := revs[entry.Name()]; ok {
72-
entryCommit := convertCommit(rev)
71+
if entryCommit, ok := revs[entry.Name()]; ok {
7372
commitsInfo[i].Commit = entryCommit
7473
}
7574

@@ -96,10 +95,10 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
9695
// get it for free during the tree traversal and it's used for listing
9796
// pages to display information about newest commit for a given path.
9897
var treeCommit *Commit
98+
var ok bool
9999
if treePath == "" {
100100
treeCommit = commit
101-
} else if rev, ok := revs[""]; ok {
102-
treeCommit = convertCommit(rev)
101+
} else if treeCommit, ok = revs[""]; ok {
103102
treeCommit.repo = commit.repo
104103
}
105104
return commitsInfo, treeCommit, nil
@@ -155,16 +154,16 @@ func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[
155154
return hashes, nil
156155
}
157156

158-
func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*object.Commit, []string, error) {
157+
func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) {
159158
var unHitEntryPaths []string
160-
results := make(map[string]*object.Commit)
159+
results := make(map[string]*Commit)
161160
for _, p := range paths {
162161
lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
163162
if err != nil {
164163
return nil, nil, err
165164
}
166165
if lastCommit != nil {
167-
results[p] = lastCommit.(*object.Commit)
166+
results[p] = lastCommit
168167
continue
169168
}
170169

@@ -175,7 +174,7 @@ func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cac
175174
}
176175

177176
// GetLastCommitForPaths returns last commit information
178-
func GetLastCommitForPaths(ctx context.Context, cache *LastCommitCache, c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
177+
func GetLastCommitForPaths(ctx context.Context, cache *LastCommitCache, c cgobject.CommitNode, treePath string, paths []string) (map[string]*Commit, error) {
179178
refSha := c.ID().String()
180179

181180
// We do a tree traversal with nodes sorted by commit time
@@ -293,13 +292,13 @@ heaploop:
293292
}
294293

295294
// Post-processing
296-
result := make(map[string]*object.Commit)
295+
result := make(map[string]*Commit)
297296
for path, commitNode := range resultNodes {
298-
var err error
299-
result[path], err = commitNode.Commit()
297+
commit, err := commitNode.Commit()
300298
if err != nil {
301299
return nil, err
302300
}
301+
result[path] = convertCommit(commit)
303302
}
304303

305304
return result, nil

‎modules/git/commit_info_nogogit.go

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import (
1717
)
1818

1919
// GetCommitsInfo gets information of all commits that are corresponding to these entries
20-
func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) {
20+
func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string) ([]CommitInfo, *Commit, error) {
2121
entryPaths := make([]string, len(tes)+1)
2222
// Get the commit for the treePath itself
2323
entryPaths[0] = ""
@@ -28,15 +28,15 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
2828
var err error
2929

3030
var revs map[string]*Commit
31-
if cache != nil {
31+
if commit.repo.LastCommitCache != nil {
3232
var unHitPaths []string
33-
revs, unHitPaths, err = getLastCommitForPathsByCache(ctx, commit.ID.String(), treePath, entryPaths, cache)
33+
revs, unHitPaths, err = getLastCommitForPathsByCache(ctx, commit.ID.String(), treePath, entryPaths, commit.repo.LastCommitCache)
3434
if err != nil {
3535
return nil, nil, err
3636
}
3737
if len(unHitPaths) > 0 {
3838
sort.Strings(unHitPaths)
39-
commits, err := GetLastCommitForPaths(ctx, cache, commit, treePath, unHitPaths)
39+
commits, err := GetLastCommitForPaths(ctx, commit, treePath, unHitPaths)
4040
if err != nil {
4141
return nil, nil, err
4242
}
@@ -47,7 +47,7 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
4747
}
4848
} else {
4949
sort.Strings(entryPaths)
50-
revs, err = GetLastCommitForPaths(ctx, nil, commit, treePath, entryPaths)
50+
revs, err = GetLastCommitForPaths(ctx, commit, treePath, entryPaths)
5151
}
5252
if err != nil {
5353
return nil, nil, err
@@ -99,18 +99,15 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
9999
}
100100

101101
func getLastCommitForPathsByCache(ctx context.Context, commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) {
102-
wr, rd, cancel := cache.repo.CatFileBatch(ctx)
103-
defer cancel()
104-
105102
var unHitEntryPaths []string
106103
results := make(map[string]*Commit)
107104
for _, p := range paths {
108-
lastCommit, err := cache.Get(commitID, path.Join(treePath, p), wr, rd)
105+
lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
109106
if err != nil {
110107
return nil, nil, err
111108
}
112109
if lastCommit != nil {
113-
results[p] = lastCommit.(*Commit)
110+
results[p] = lastCommit
114111
continue
115112
}
116113

@@ -121,9 +118,9 @@ func getLastCommitForPathsByCache(ctx context.Context, commitID, treePath string
121118
}
122119

123120
// GetLastCommitForPaths returns last commit information
124-
func GetLastCommitForPaths(ctx context.Context, cache *LastCommitCache, commit *Commit, treePath string, paths []string) (map[string]*Commit, error) {
121+
func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string, paths []string) (map[string]*Commit, error) {
125122
// We read backwards from the commit to obtain all of the commits
126-
revs, err := WalkGitLog(ctx, cache, commit.repo, commit, treePath, paths...)
123+
revs, err := WalkGitLog(ctx, commit.repo, commit, treePath, paths...)
127124
if err != nil {
128125
return nil, err
129126
}

‎modules/git/commit_info_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ func testGetCommitsInfo(t *testing.T, repo1 *Repository) {
9191
}
9292

9393
// FIXME: Context.TODO() - if graceful has started we should use its Shutdown context otherwise use install signals in TestMain.
94-
commitsInfo, treeCommit, err := entries.GetCommitsInfo(context.TODO(), commit, testCase.Path, nil)
94+
commitsInfo, treeCommit, err := entries.GetCommitsInfo(context.TODO(), commit, testCase.Path)
9595
assert.NoError(t, err, "Unable to get commit information for entries of subtree: %s in commit: %s from testcase due to error: %v", testCase.Path, testCase.CommitID, err)
9696
if err != nil {
9797
t.FailNow()
@@ -170,7 +170,7 @@ func BenchmarkEntries_GetCommitsInfo(b *testing.B) {
170170
b.ResetTimer()
171171
b.Run(benchmark.name, func(b *testing.B) {
172172
for i := 0; i < b.N; i++ {
173-
_, _, err := entries.GetCommitsInfo(context.Background(), commit, "", nil)
173+
_, _, err := entries.GetCommitsInfo(context.Background(), commit, "")
174174
if err != nil {
175175
b.Fatal(err)
176176
}

‎modules/git/last_commit_cache.go

Lines changed: 84 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"fmt"
1010

1111
"code.gitea.io/gitea/modules/log"
12+
"code.gitea.io/gitea/modules/setting"
1213
)
1314

1415
// Cache represents a caching interface
@@ -19,16 +20,96 @@ type Cache interface {
1920
Get(key string) interface{}
2021
}
2122

22-
func (c *LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string {
23-
hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath)))
23+
func getCacheKey(repoPath, commitID, entryPath string) string {
24+
hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, commitID, entryPath)))
2425
return fmt.Sprintf("last_commit:%x", hashBytes)
2526
}
2627

28+
// LastCommitCache represents a cache to store last commit
29+
type LastCommitCache struct {
30+
repoPath string
31+
ttl func() int64
32+
repo *Repository
33+
commitCache map[string]*Commit
34+
cache Cache
35+
}
36+
37+
// NewLastCommitCache creates a new last commit cache for repo
38+
func NewLastCommitCache(count int64, repoPath string, gitRepo *Repository, cache Cache) *LastCommitCache {
39+
if cache == nil {
40+
return nil
41+
}
42+
if !setting.CacheService.LastCommit.Enabled || count < setting.CacheService.LastCommit.CommitsCount {
43+
return nil
44+
}
45+
46+
return &LastCommitCache{
47+
repoPath: repoPath,
48+
repo: gitRepo,
49+
ttl: setting.LastCommitCacheTTLSeconds,
50+
cache: cache,
51+
}
52+
}
53+
2754
// Put put the last commit id with commit and entry path
2855
func (c *LastCommitCache) Put(ref, entryPath, commitID string) error {
2956
if c == nil || c.cache == nil {
3057
return nil
3158
}
3259
log.Debug("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
33-
return c.cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl())
60+
return c.cache.Put(getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl())
61+
}
62+
63+
// Get gets the last commit information by commit id and entry path
64+
func (c *LastCommitCache) Get(ref, entryPath string) (*Commit, error) {
65+
if c == nil || c.cache == nil {
66+
return nil, nil
67+
}
68+
69+
commitID, ok := c.cache.Get(getCacheKey(c.repoPath, ref, entryPath)).(string)
70+
if !ok || commitID == "" {
71+
return nil, nil
72+
}
73+
74+
log.Debug("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, commitID)
75+
if c.commitCache != nil {
76+
if commit, ok := c.commitCache[commitID]; ok {
77+
log.Debug("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, commitID)
78+
return commit, nil
79+
}
80+
}
81+
82+
commit, err := c.repo.GetCommit(commitID)
83+
if err != nil {
84+
return nil, err
85+
}
86+
if c.commitCache == nil {
87+
c.commitCache = make(map[string]*Commit)
88+
}
89+
c.commitCache[commitID] = commit
90+
return commit, nil
91+
}
92+
93+
// GetCommitByPath gets the last commit for the entry in the provided commit
94+
func (c *LastCommitCache) GetCommitByPath(commitID, entryPath string) (*Commit, error) {
95+
sha1, err := NewIDFromString(commitID)
96+
if err != nil {
97+
return nil, err
98+
}
99+
100+
lastCommit, err := c.Get(sha1.String(), entryPath)
101+
if err != nil || lastCommit != nil {
102+
return lastCommit, err
103+
}
104+
105+
lastCommit, err = c.repo.getCommitByPathWithID(sha1, entryPath)
106+
if err != nil {
107+
return nil, err
108+
}
109+
110+
if err := c.Put(commitID, entryPath, lastCommit.ID.String()); err != nil {
111+
log.Error("Unable to cache %s as the last commit for %q in %s %s. Error %v", lastCommit.ID.String(), entryPath, commitID, c.repoPath, err)
112+
}
113+
114+
return lastCommit, nil
34115
}

0 commit comments

Comments
 (0)