From dfc292bb62c378241557c4ab4b491184a35d5ced Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 6 Sep 2021 12:47:40 +0800 Subject: [PATCH 1/3] Fix storage Iterate bug and Add storage doctor to delete garbage attachments --- cmd/doctor.go | 1 - models/attachment.go | 5 +++ modules/doctor/storage.go | 71 +++++++++++++++++++++++++++++++++++++++ modules/storage/minio.go | 4 +-- 4 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 modules/doctor/storage.go diff --git a/cmd/doctor.go b/cmd/doctor.go index 0152aebe39959..2e7927543481c 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -124,7 +124,6 @@ func runRecreateTable(ctx *cli.Context) error { } func runDoctor(ctx *cli.Context) error { - // Silence the default loggers log.DelNamedLogger("console") log.DelNamedLogger(log.DEFAULT) diff --git a/models/attachment.go b/models/attachment.go index e12609f8108f4..4e0ccba5a0cc2 100644 --- a/models/attachment.go +++ b/models/attachment.go @@ -144,6 +144,11 @@ func GetAttachmentByUUID(uuid string) (*Attachment, error) { return getAttachmentByUUID(x, uuid) } +// ExistAttachmentsByUUID returns true if attachment is exist by given UUID +func ExistAttachmentsByUUID(uuid string) (bool, error) { + return x.Where("`uuid`=?", uuid).Exist(new(Attachment)) +} + // GetAttachmentByReleaseIDFileName returns attachment by given releaseId and fileName. func GetAttachmentByReleaseIDFileName(releaseID int64, fileName string) (*Attachment, error) { return getAttachmentByReleaseIDFileName(x, releaseID, fileName) diff --git a/modules/doctor/storage.go b/modules/doctor/storage.go new file mode 100644 index 0000000000000..ecfc89dd680b3 --- /dev/null +++ b/modules/doctor/storage.go @@ -0,0 +1,71 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package doctor + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/storage" +) + +func checkStorageFiles(logger log.Logger, autofix bool) error { + if err := storage.Init(); err != nil { + logger.Error("storage.Init failed: %v", err) + return err + } + + var total, garbageNum int + var deletePaths []string + if err := storage.Attachments.IterateObjects(func(p string, obj storage.Object) error { + total++ + stat, err := obj.Stat() + if err != nil { + return err + } + exist, err := models.ExistAttachmentsByUUID(stat.Name()) + if err != nil { + return err + } + if !exist { + garbageNum++ + if autofix { + deletePaths = append(deletePaths, p) + } + } + return nil + }); err != nil { + logger.Error("storage.Attachments.IterateObjects failed: %v", err) + return err + } + + if garbageNum > 0 { + if autofix { + var deletedNum int + for _, p := range deletePaths { + if err := storage.Attachments.Delete(p); err != nil { + log.Error("Delete attachment %s failed: %v", p, err) + } else { + deletedNum++ + } + } + logger.Info("%d missed information attachment detected, %d deleted.", garbageNum, deletedNum) + } else { + logger.Warn("Checked %d attachment, %d missed information.", total, garbageNum) + } + } + return nil +} + +func init() { + Register(&Check{ + Title: "Check if there is garbage storage files", + Name: "storages", + IsDefault: false, + Run: checkStorageFiles, + AbortIfFailed: false, + SkipDatabaseInitialization: false, + Priority: 1, + }) +} diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 724445c0ab9de..f78ba6aa27511 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -151,7 +151,7 @@ type minioFileInfo struct { } func (m minioFileInfo) Name() string { - return m.ObjectInfo.Key + return path.Base(m.ObjectInfo.Key) } func (m minioFileInfo) Size() int64 { @@ -219,7 +219,7 @@ func (m *MinioStorage) IterateObjects(fn func(path string, obj Object) error) er } if err := func(object *minio.Object, fn func(path string, obj Object) error) error { defer object.Close() - return fn(strings.TrimPrefix(m.basePath, mObjInfo.Key), &minioObject{object}) + return fn(strings.TrimPrefix(mObjInfo.Key, m.basePath), &minioObject{object}) }(object, fn); err != nil { return convertMinioErr(err) } From 4b556358112de9eaba135bc81cb6c2202b72a49a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 6 Sep 2021 13:25:41 +0800 Subject: [PATCH 2/3] Close object when used --- modules/doctor/storage.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/modules/doctor/storage.go b/modules/doctor/storage.go index ecfc89dd680b3..5b5f453b0b56b 100644 --- a/modules/doctor/storage.go +++ b/modules/doctor/storage.go @@ -10,15 +10,12 @@ import ( "code.gitea.io/gitea/modules/storage" ) -func checkStorageFiles(logger log.Logger, autofix bool) error { - if err := storage.Init(); err != nil { - logger.Error("storage.Init failed: %v", err) - return err - } - +func checkAttachmentStorageFiles(logger log.Logger, autofix bool) error { var total, garbageNum int var deletePaths []string if err := storage.Attachments.IterateObjects(func(p string, obj storage.Object) error { + defer obj.Close() + total++ stat, err := obj.Stat() if err != nil { @@ -58,6 +55,15 @@ func checkStorageFiles(logger log.Logger, autofix bool) error { return nil } +func checkStorageFiles(logger log.Logger, autofix bool) error { + if err := storage.Init(); err != nil { + logger.Error("storage.Init failed: %v", err) + return err + } + + return checkAttachmentStorageFiles(logger, autofix) +} + func init() { Register(&Check{ Title: "Check if there is garbage storage files", From 6b237c5ca0d51fbd630a8ab716c2451dff4788f3 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 6 Sep 2021 16:02:41 +0200 Subject: [PATCH 3/3] Update modules/doctor/storage.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Alexey 〒erentyev --- modules/doctor/storage.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/doctor/storage.go b/modules/doctor/storage.go index 5b5f453b0b56b..cb06d6620498c 100644 --- a/modules/doctor/storage.go +++ b/modules/doctor/storage.go @@ -60,7 +60,6 @@ func checkStorageFiles(logger log.Logger, autofix bool) error { logger.Error("storage.Init failed: %v", err) return err } - return checkAttachmentStorageFiles(logger, autofix) }