From c5bc0de86b077778b68768a29495e64ec8bc0492 Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Tue, 25 Jul 2023 12:27:39 +0800 Subject: [PATCH 1/6] fix time type --- models/git/lfs.go | 5 ++--- services/repository/lfs.go | 5 +++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/models/git/lfs.go b/models/git/lfs.go index 7d3da72a9410c..7eda3d6cb0823 100644 --- a/models/git/lfs.go +++ b/models/git/lfs.go @@ -6,7 +6,6 @@ package git import ( "context" "fmt" - "time" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/perm" @@ -370,8 +369,8 @@ func IterateRepositoryIDsWithLFSMetaObjects(ctx context.Context, f func(ctx cont // IterateLFSMetaObjectsForRepoOptions provides options for IterateLFSMetaObjectsForRepo type IterateLFSMetaObjectsForRepoOptions struct { - OlderThan time.Time - UpdatedLessRecentlyThan time.Time + OlderThan timeutil.TimeStamp + UpdatedLessRecentlyThan timeutil.TimeStamp OrderByUpdated bool LoopFunctionAlwaysUpdates bool } diff --git a/services/repository/lfs.go b/services/repository/lfs.go index 0bd4d53a5c4da..8e654b6f13bcd 100644 --- a/services/repository/lfs.go +++ b/services/repository/lfs.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" ) // GarbageCollectLFSMetaObjectsOptions provides options for GarbageCollectLFSMetaObjects function @@ -120,8 +121,8 @@ func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.R // // It is likely that a week is potentially excessive but it should definitely be enough that any // unassociated LFS object is genuinely unassociated. - OlderThan: opts.OlderThan, - UpdatedLessRecentlyThan: opts.UpdatedLessRecentlyThan, + OlderThan: timeutil.TimeStamp(opts.OlderThan.Unix()), + UpdatedLessRecentlyThan: timeutil.TimeStamp(opts.UpdatedLessRecentlyThan.Unix()), OrderByUpdated: true, LoopFunctionAlwaysUpdates: true, }) From bade6126fbf1169bd2652190569c2da51d348603 Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Tue, 25 Jul 2023 12:27:58 +0800 Subject: [PATCH 2/6] add missing xorm extends tag --- models/git/lfs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/git/lfs.go b/models/git/lfs.go index 7eda3d6cb0823..e8192f92c5c68 100644 --- a/models/git/lfs.go +++ b/models/git/lfs.go @@ -381,8 +381,8 @@ func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(cont batchSize := setting.Database.IterateBufferSize engine := db.GetEngine(ctx) type CountLFSMetaObject struct { - Count int64 - LFSMetaObject + Count int64 + LFSMetaObject `xorm:"extends"` } id := int64(0) From ce5c3eb76ccd9a4ff9b1957f45126bdc66d1f2a1 Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Tue, 25 Jul 2023 16:12:12 +0800 Subject: [PATCH 3/6] add test --- tests/integration/lfs_gc_test.go | 38 ++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/integration/lfs_gc_test.go diff --git a/tests/integration/lfs_gc_test.go b/tests/integration/lfs_gc_test.go new file mode 100644 index 0000000000000..c0bc9b6e53084 --- /dev/null +++ b/tests/integration/lfs_gc_test.go @@ -0,0 +1,38 @@ +package integration + +import ( + "context" + "testing" + "time" + + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + repo_service "code.gitea.io/gitea/services/repository" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/assert" +) + +func TestLFSMetaObjectGC(t *testing.T) { + defer tests.PrepareTestEnv(t)() + + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "repo1") + assert.NoError(t, err) + + // add lfs object + lfsContent := []byte("gitea1") + lfsOid := storeObjectInRepo(t, repo.ID, &lfsContent) + + // gc + err = repo_service.GarbageCollectLFSMetaObjects(context.Background(), repo_service.GarbageCollectLFSMetaObjectsOptions{ + AutoFix: true, + OlderThan: time.Now().Add(7 * 24 * time.Hour).Add(5 * 24 * time.Hour), + UpdatedLessRecentlyThan: time.Now().Add(7 * 24 * time.Hour).Add(3 * 24 * time.Hour), + }) + assert.NoError(t, err) + + // lfs meta has been deleted + _, err = git_model.GetLFSMetaObjectByOid(db.DefaultContext, repo.ID, lfsOid) + assert.ErrorIs(t, err, git_model.ErrLFSObjectNotExist) +} From 4f367291002e7628c98aecf11aeb6149f4ab8cfa Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Tue, 25 Jul 2023 16:50:02 +0800 Subject: [PATCH 4/6] move to unit test --- services/repository/lfs_test.go | 58 ++++++++++++++++++++++++++++++++ tests/integration/lfs_gc_test.go | 38 --------------------- 2 files changed, 58 insertions(+), 38 deletions(-) create mode 100644 services/repository/lfs_test.go delete mode 100644 tests/integration/lfs_gc_test.go diff --git a/services/repository/lfs_test.go b/services/repository/lfs_test.go new file mode 100644 index 0000000000000..8f3555ec83bfc --- /dev/null +++ b/services/repository/lfs_test.go @@ -0,0 +1,58 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "bytes" + "context" + "testing" + "time" + + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/lfs" + + "github.com/stretchr/testify/assert" +) + +func TestGarbageCollectLFSMetaObjects(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "repo1") + assert.NoError(t, err) + + // add lfs object + lfsContent := []byte("gitea1") + lfsOid := storeObjectInRepo(t, repo.ID, &lfsContent) + + // gc + err = GarbageCollectLFSMetaObjects(context.Background(), GarbageCollectLFSMetaObjectsOptions{ + AutoFix: true, + OlderThan: time.Now().Add(7 * 24 * time.Hour).Add(5 * 24 * time.Hour), + UpdatedLessRecentlyThan: time.Now().Add(7 * 24 * time.Hour).Add(3 * 24 * time.Hour), + }) + assert.NoError(t, err) + + // lfs meta has been deleted + _, err = git_model.GetLFSMetaObjectByOid(db.DefaultContext, repo.ID, lfsOid) + assert.ErrorIs(t, err, git_model.ErrLFSObjectNotExist) +} + +func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string { + pointer, err := lfs.GeneratePointer(bytes.NewReader(*content)) + assert.NoError(t, err) + + _, err = git_model.NewLFSMetaObject(db.DefaultContext, &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: repositoryID}) + assert.NoError(t, err) + contentStore := lfs.NewContentStore() + exist, err := contentStore.Exists(pointer) + assert.NoError(t, err) + if !exist { + err := contentStore.Put(pointer, bytes.NewReader(*content)) + assert.NoError(t, err) + } + return pointer.Oid +} diff --git a/tests/integration/lfs_gc_test.go b/tests/integration/lfs_gc_test.go deleted file mode 100644 index c0bc9b6e53084..0000000000000 --- a/tests/integration/lfs_gc_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package integration - -import ( - "context" - "testing" - "time" - - "code.gitea.io/gitea/models/db" - git_model "code.gitea.io/gitea/models/git" - repo_model "code.gitea.io/gitea/models/repo" - repo_service "code.gitea.io/gitea/services/repository" - "code.gitea.io/gitea/tests" - - "github.com/stretchr/testify/assert" -) - -func TestLFSMetaObjectGC(t *testing.T) { - defer tests.PrepareTestEnv(t)() - - repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "repo1") - assert.NoError(t, err) - - // add lfs object - lfsContent := []byte("gitea1") - lfsOid := storeObjectInRepo(t, repo.ID, &lfsContent) - - // gc - err = repo_service.GarbageCollectLFSMetaObjects(context.Background(), repo_service.GarbageCollectLFSMetaObjectsOptions{ - AutoFix: true, - OlderThan: time.Now().Add(7 * 24 * time.Hour).Add(5 * 24 * time.Hour), - UpdatedLessRecentlyThan: time.Now().Add(7 * 24 * time.Hour).Add(3 * 24 * time.Hour), - }) - assert.NoError(t, err) - - // lfs meta has been deleted - _, err = git_model.GetLFSMetaObjectByOid(db.DefaultContext, repo.ID, lfsOid) - assert.ErrorIs(t, err, git_model.ErrLFSObjectNotExist) -} From 52e6d05a5d5b5c737e2407ab2a2cae1ac518c99f Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Tue, 25 Jul 2023 17:37:52 +0800 Subject: [PATCH 5/6] enable lfs in test --- services/repository/lfs_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/repository/lfs_test.go b/services/repository/lfs_test.go index 8f3555ec83bfc..a18557c76aff0 100644 --- a/services/repository/lfs_test.go +++ b/services/repository/lfs_test.go @@ -14,12 +14,14 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" ) func TestGarbageCollectLFSMetaObjects(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) + setting.LFS.StartServer = true repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "repo1") assert.NoError(t, err) From 777a8a88f012ba206f0271e043d6b1dd6f589f99 Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Wed, 26 Jul 2023 09:03:52 +0800 Subject: [PATCH 6/6] init lfs --- services/repository/lfs_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/repository/lfs_test.go b/services/repository/lfs_test.go index a18557c76aff0..e88befdfefe96 100644 --- a/services/repository/lfs_test.go +++ b/services/repository/lfs_test.go @@ -15,13 +15,17 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "github.com/stretchr/testify/assert" ) func TestGarbageCollectLFSMetaObjects(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) + setting.LFS.StartServer = true + err := storage.Init() + assert.NoError(t, err) repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "repo1") assert.NoError(t, err)