Improve generational GC and only check repos with LFSMetaObjects

Signed-off-by: Andrew Thornton <[email protected]>

Author: zeripath

custom/conf/app.example.ini

@@ -2188,6 +2188,11 @@ ROUTER = console
 ;OLDER_THAN = 168h
 ;; Only attempt to garbage collect LFSMetaObjects that have not been attempted to be garbage collected for this long (default 3 days)
 ;LAST_UPDATED_MORE_THAN_AGO = 72h
+; Minimum number of stale LFSMetaObjects to check per repo. Set to `0` to always check all.
+;NUMBER_TO_CHECK_PER_REPO = 100
+;Check at least this proportion of LFSMetaObjects per repo. (This may cause all stale LFSMetaObjects to be checked.)
+;PROPORTION_TO_CHECK_PER_REPO = 0.6
+
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -1032,6 +1032,8 @@ Default templates for project boards:
 - `SCHEDULE`: **@every 24h**: Cron syntax to set how often to check.
 - `OLDER_THAN`: **168h**: Only attempt to garbage collect LFSMetaObjects older than this (default 7 days)
 - `LAST_UPDATED_MORE_THAN_AGO`: **72h**: Only attempt to garbage collect LFSMetaObjects that have not been attempted to be garbage collected for this long (default 3 days)
+- `NUMBER_TO_CHECK_PER_REPO`: **100**: Minimum number of stale LFSMetaObjects to check per repo. Set to `0` to always check all.
+- `PROPORTION_TO_CHECK_PER_REPO`: **0.6**: Check at least this proportion of LFSMetaObjects per repo. (This may cause all stale LFSMetaObjects to be checked.)
 
 ## Git (`git`)
 

models/git/lfs.go

@@ -335,9 +335,45 @@ func GetRepoLFSSize(ctx context.Context, repoID int64) (int64, error) {
 	return lfsSize, nil
 }
 
+// IterateRepositoryIDsWithLFSMetaObjects iterates across the repositories that have LFSMetaObjects
+func IterateRepositoryIDsWithLFSMetaObjects(ctx context.Context, f func(ctx context.Context, repoID, count int64) error) error {
+	batchSize := setting.Database.IterateBufferSize
+	sess := db.GetEngine(ctx)
+	id := int64(0)
+	type RepositoryCount struct {
+		RepositoryID int64
+		Count        int64
+	}
+	for {
+		counts := make([]*RepositoryCount, 0, batchSize)
+		sess.Select("repository_id, COUNT(id) AS count").
+			Table("lfs_meta_object").
+			Where("repository_id > ?", id).
+			GroupBy("repository_id").
+			OrderBy("repository_id ASC")
+
+		if err := sess.Limit(batchSize, 0).Find(&counts); err != nil {
+			return err
+		}
+		if len(counts) == 0 {
+			return nil
+		}
+
+		for _, count := range counts {
+			if err := f(ctx, count.RepositoryID, count.Count); err != nil {
+				return err
+			}
+		}
+		id = counts[len(counts)-1].RepositoryID
+	}
+}
+
+// IterateLFSMetaObjectsForRepoOptions provides options for IterateLFSMetaObjectsForRepo
 type IterateLFSMetaObjectsForRepoOptions struct {
-	OlderThan               time.Time
-	UpdatedLessRecentlyThan time.Time
+	OlderThan                 time.Time
+	UpdatedLessRecentlyThan   time.Time
+	OrderByUpdated            bool
+	LoopFunctionAlwaysUpdates bool
 }
 
 // IterateLFSMetaObjectsForRepo provides a iterator for LFSMetaObjects per Repo
@@ -354,7 +390,6 @@ func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(cont
 
 	for {
 		beans := make([]*CountLFSMetaObject, 0, batchSize)
-		// SELECT `lfs_meta_object`.*, COUNT(`l1`.id) as `count` FROM lfs_meta_object INNER JOIN lfs_meta_object AS l1 ON l1.oid = lfs_meta_object.oid WHERE lfs_meta_object.repository_id = ? GROUP BY lfs_meta_object.id
 		sess := engine.Select("`lfs_meta_object`.*, COUNT(`l1`.oid) AS `count`").
 			Join("INNER", "`lfs_meta_object` AS l1", "`lfs_meta_object`.oid = `l1`.oid").
 			Where("`lfs_meta_object`.repository_id = ?", repoID)
@@ -364,16 +399,22 @@ func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(cont
 		if !opts.UpdatedLessRecentlyThan.IsZero() {
 			sess.And("`lfs_meta_object`.updated_unix < ?", opts.UpdatedLessRecentlyThan)
 		}
-		sess.And("`lfs_meta_object`.id > ?", id)
 		sess.GroupBy("`lfs_meta_object`.id")
-		sess.OrderBy("`lfs_meta_object`.id ASC")
+		if opts.OrderByUpdated {
+			sess.OrderBy("`lfs_meta_object`.updated_unix ASC")
+		} else {
+			sess.And("`lfs_meta_object`.id > ?", id)
+			sess.OrderBy("`lfs_meta_object`.id ASC")
+		}
 		if err := sess.Limit(batchSize, start).Find(&beans); err != nil {
 			return err
 		}
 		if len(beans) == 0 {
 			return nil
 		}
-		start += len(beans)
+		if !opts.LoopFunctionAlwaysUpdates {
+			start += len(beans)
+		}
 
 		for _, bean := range beans {
 			if err := f(ctx, &bean.LFSMetaObject, bean.Count); err != nil {

modules/doctor/lfs.go

@@ -41,7 +41,7 @@ func garbageCollectLFSCheck(ctx context.Context, logger log.Logger, autofix bool
 		//
 		// It is likely that a week is potentially excessive but it should definitely be enough that any
 		// unassociated LFS object is genuinely unassociated.
-		OlderThan: 24 * time.Hour * 7,
+		OlderThan: time.Now().Add(-24 * time.Hour * 7),
 		// We don't set the UpdatedLessRecentlyThan because we want to do a full GC
 	}); err != nil {
 		return err

services/cron/tasks_extended.go

@@ -181,10 +181,12 @@ func registerGCLFS() {
 	}
 	type GCLFSConfig struct {
 		OlderThanConfig
-		LastUpdatedMoreThanAgo time.Duration
+		LastUpdatedMoreThanAgo   time.Duration
+		NumberToCheckPerRepo     int64
+		ProportionToCheckPerRepo float64
 	}
 
-	RegisterTaskFatal("delete_old_system_notices", &GCLFSConfig{
+	RegisterTaskFatal("gc_lfs", &GCLFSConfig{
 		OlderThanConfig: OlderThanConfig{
 			BaseConfig: BaseConfig{
 				Enabled:    false,
@@ -202,13 +204,15 @@ func registerGCLFS() {
 			OlderThan: 24 * time.Hour * 7,
 		},
 		// Only GC things that haven't been looked at in the past 3 days
-		LastUpdatedMoreThanAgo: 24 * time.Hour * 3,
+		LastUpdatedMoreThanAgo:   24 * time.Hour * 3,
+		NumberToCheckPerRepo:     100,
+		ProportionToCheckPerRepo: 0.6,
 	}, func(ctx context.Context, _ *user_model.User, config Config) error {
 		gcLFSConfig := config.(*GCLFSConfig)
 		return repo_service.GarbageCollectLFSMetaObjects(ctx, repo_service.GarbageCollectLFSMetaObjectsOptions{
-			AutoFix:                true,
-			OlderThan:              gcLFSConfig.OlderThan,
-			LastUpdatedMoreThanAgo: gcLFSConfig.LastUpdatedMoreThanAgo,
+			AutoFix:                 true,
+			OlderThan:               time.Now().Add(-gcLFSConfig.OlderThan),
+			UpdatedLessRecentlyThan: time.Now().Add(-gcLFSConfig.LastUpdatedMoreThanAgo),
 		})
 	})
 }

services/repository/lfs.go

@@ -5,27 +5,26 @@ package repository
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
-	"code.gitea.io/gitea/models/db"
 	git_model "code.gitea.io/gitea/models/git"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
-
-	"xorm.io/builder"
 )
 
 // GarbageCollectLFSMetaObjectsOptions provides options for GarbageCollectLFSMetaObjects function
 type GarbageCollectLFSMetaObjectsOptions struct {
-	RepoID                 int64
-	Logger                 log.Logger
-	AutoFix                bool
-	OlderThan              time.Duration
-	LastUpdatedMoreThanAgo time.Duration
+	Logger                   log.Logger
+	AutoFix                  bool
+	OlderThan                time.Time
+	UpdatedLessRecentlyThan  time.Time
+	NumberToCheckPerRepo     int64
+	ProportionToCheckPerRepo float64
 }
 
 // GarbageCollectLFSMetaObjects garbage collects LFS objects for all repositories
@@ -40,29 +39,25 @@ func GarbageCollectLFSMetaObjects(ctx context.Context, opts GarbageCollectLFSMet
 		return nil
 	}
 
-	if opts.RepoID == 0 {
-		repo, err := repo_model.GetRepositoryByID(ctx, opts.RepoID)
+	return git_model.IterateRepositoryIDsWithLFSMetaObjects(ctx, func(ctx context.Context, repoID, count int64) error {
+		repo, err := repo_model.GetRepositoryByID(ctx, repoID)
 		if err != nil {
 			return err
 		}
-		return GarbageCollectLFSMetaObjectsForRepo(ctx, repo, opts)
-	}
 
-	return db.Iterate(
-		ctx,
-		builder.And(builder.Gt{"id": 0}),
-		func(ctx context.Context, repo *repo_model.Repository) error {
-			return GarbageCollectLFSMetaObjectsForRepo(ctx, repo, opts)
-		},
-	)
+		if newMinimum := int64(float64(count) * opts.ProportionToCheckPerRepo); newMinimum > opts.NumberToCheckPerRepo && opts.NumberToCheckPerRepo != 0 {
+			opts.NumberToCheckPerRepo = newMinimum
+		}
+		return GarbageCollectLFSMetaObjectsForRepo(ctx, repo, opts)
+	})
 }
 
 // GarbageCollectLFSMetaObjectsForRepo garbage collects LFS objects for a specific repository
 func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.Repository, opts GarbageCollectLFSMetaObjectsOptions) error {
 	if opts.Logger != nil {
 		opts.Logger.Info("Checking %-v", repo)
 	}
-	total, orphaned, collected, deleted := 0, 0, 0, 0
+	total, orphaned, collected, deleted := int64(0), 0, 0, 0
 	if opts.Logger != nil {
 		defer func() {
 			if orphaned == 0 {
@@ -83,18 +78,12 @@ func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.R
 	defer gitRepo.Close()
 
 	store := lfs.NewContentStore()
+	errStop := errors.New("STOPERR")
 
-	var olderThan time.Time
-	var updatedLessRecentlyThan time.Time
-
-	if opts.OlderThan > 0 {
-		olderThan = time.Now().Add(opts.OlderThan)
-	}
-	if opts.LastUpdatedMoreThanAgo > 0 {
-		updatedLessRecentlyThan = time.Now().Add(opts.LastUpdatedMoreThanAgo)
-	}
-
-	return git_model.IterateLFSMetaObjectsForRepo(ctx, repo.ID, func(ctx context.Context, metaObject *git_model.LFSMetaObject, count int64) error {
+	err = git_model.IterateLFSMetaObjectsForRepo(ctx, repo.ID, func(ctx context.Context, metaObject *git_model.LFSMetaObject, count int64) error {
+		if opts.NumberToCheckPerRepo > 0 && total > opts.NumberToCheckPerRepo {
+			return errStop
+		}
 		total++
 		pointerSha := git.ComputeBlobHash([]byte(metaObject.Pointer.StringContent()))
 
@@ -133,7 +122,19 @@ func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.R
 		//
 		// It is likely that a week is potentially excessive but it should definitely be enough that any
 		// unassociated LFS object is genuinely unassociated.
-		OlderThan:               olderThan,
-		UpdatedLessRecentlyThan: updatedLessRecentlyThan,
+		OlderThan:                 opts.OlderThan,
+		UpdatedLessRecentlyThan:   opts.UpdatedLessRecentlyThan,
+		OrderByUpdated:            true,
+		LoopFunctionAlwaysUpdates: true,
 	})
+
+	if err == errStop {
+		if opts.Logger != nil {
+			opts.Logger.Info("Processing stopped at %d total LFSMetaObjects in %-v", total, repo)
+		}
+		return nil
+	} else if err != nil {
+		return err
+	}
+	return nil
 }