From f8a93c113414944ad755fbb9258560f37c1dc0f2 Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Tue, 25 Jul 2023 22:27:28 +0800 Subject: [PATCH 01/10] add artifact retention support and clean up cron task --- models/actions/artifact.go | 20 ++++++++++++++- modules/setting/actions.go | 14 ++++++++--- options/locale/locale_en-US.ini | 1 + routers/api/actions/artifacts.go | 3 ++- services/actions/cleanup.go | 42 ++++++++++++++++++++++++++++++++ services/cron/tasks_basic.go | 18 ++++++++++++++ 6 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 services/actions/cleanup.go diff --git a/models/actions/artifact.go b/models/actions/artifact.go index 800dcd0d50a7a..4c5fed9131d8e 100644 --- a/models/actions/artifact.go +++ b/models/actions/artifact.go @@ -9,6 +9,7 @@ package actions import ( "context" "errors" + "time" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/timeutil" @@ -22,6 +23,8 @@ const ( ArtifactStatusUploadConfirmed = 2 // ArtifactStatusUploadError is the status of an artifact upload that is errored ArtifactStatusUploadError = 3 + // ArtifactStatusExpired is the status of an artifact that is expired + ArtifactStatusExpired = 4 ) func init() { @@ -45,9 +48,10 @@ type ActionArtifact struct { Status int64 `xorm:"index"` // The status of the artifact, uploading, expired or need-delete CreatedUnix timeutil.TimeStamp `xorm:"created"` UpdatedUnix timeutil.TimeStamp `xorm:"updated index"` + ExpiredUnix timeutil.TimeStamp `xorm:"index"` // The time when the artifact will be expired } -func CreateArtifact(ctx context.Context, t *ActionTask, artifactName, artifactPath string) (*ActionArtifact, error) { +func CreateArtifact(ctx context.Context, t *ActionTask, artifactName, artifactPath string, expiredDays int64) (*ActionArtifact, error) { if err := t.LoadJob(ctx); err != nil { return nil, err } @@ -62,6 +66,7 @@ func CreateArtifact(ctx context.Context, t *ActionTask, artifactName, artifactPa OwnerID: t.OwnerID, CommitSHA: t.CommitSHA, Status: ArtifactStatusUploadPending, + ExpiredUnix: timeutil.TimeStamp(time.Now().Unix() + 3600*24*expiredDays), } if _, err := db.GetEngine(ctx).Insert(artifact); err != nil { return nil, err @@ -149,3 +154,16 @@ func ListArtifactsByRunIDAndName(ctx context.Context, runID int64, name string) arts := make([]*ActionArtifact, 0, 10) return arts, db.GetEngine(ctx).Where("run_id=? AND artifact_name=?", runID, name).Find(&arts) } + +// ListExpiredArtifacts returns all expired artifacts but not deleted +func ListExpiredArtifacts(ctx context.Context) ([]*ActionArtifact, error) { + arts := make([]*ActionArtifact, 0, 10) + return arts, db.GetEngine(ctx). + Where("expired_unix < ? AND status = ?", timeutil.TimeStamp(time.Now().Unix()), ArtifactStatusUploadConfirmed).Find(&arts) +} + +// SetArtifactExpired sets an artifact to expired +func SetArtifactExpired(ctx context.Context, artifactID int64) error { + _, err := db.GetEngine(ctx).Where("id=?", artifactID).Cols("status").Update(&ActionArtifact{Status: ArtifactStatusExpired}) + return err +} diff --git a/modules/setting/actions.go b/modules/setting/actions.go index a13330dcd18a3..bfc502c0cbea5 100644 --- a/modules/setting/actions.go +++ b/modules/setting/actions.go @@ -13,10 +13,11 @@ import ( // Actions settings var ( Actions = struct { - LogStorage *Storage // how the created logs should be stored - ArtifactStorage *Storage // how the created artifacts should be stored - Enabled bool - DefaultActionsURL defaultActionsURL `ini:"DEFAULT_ACTIONS_URL"` + LogStorage *Storage // how the created logs should be stored + ArtifactStorage *Storage // how the created artifacts should be stored + ArtifactRetentionDays int64 `ini:"ARTIFACT_RETENTION_DAYS"` + Enabled bool + DefaultActionsURL defaultActionsURL `ini:"DEFAULT_ACTIONS_URL"` }{ Enabled: false, DefaultActionsURL: defaultActionsURLGitHub, @@ -76,5 +77,10 @@ func loadActionsFrom(rootCfg ConfigProvider) error { Actions.ArtifactStorage, err = getStorage(rootCfg, "actions_artifacts", "", actionsSec) + // default to 90 days in Github Actions + if Actions.ArtifactRetentionDays <= 0 { + Actions.ArtifactRetentionDays = 90 + } + return err } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index a76750e44fbd3..b657d0a1ab036 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2688,6 +2688,7 @@ dashboard.reinit_missing_repos = Reinitialize all missing Git repositories for w dashboard.sync_external_users = Synchronize external user data dashboard.cleanup_hook_task_table = Cleanup hook_task table dashboard.cleanup_packages = Cleanup expired packages +dashboard.cleanup_actions = Cleanup actions expired logs and artifacts dashboard.server_uptime = Server Uptime dashboard.current_goroutine = Current Goroutines dashboard.current_memory_usage = Current Memory Usage diff --git a/routers/api/actions/artifacts.go b/routers/api/actions/artifacts.go index 946ea11e75271..0b401c0ff996a 100644 --- a/routers/api/actions/artifacts.go +++ b/routers/api/actions/artifacts.go @@ -220,7 +220,8 @@ func (ar artifactRoutes) uploadArtifact(ctx *ArtifactContext) { } // create or get artifact with name and path - artifact, err := actions.CreateArtifact(ctx, task, artifactName, artifactPath) + expiredDays := setting.Actions.ArtifactRetentionDays + artifact, err := actions.CreateArtifact(ctx, task, artifactName, artifactPath, expiredDays) if err != nil { log.Error("Error create or get artifact: %v", err) ctx.Error(http.StatusInternalServerError, "Error create or get artifact") diff --git a/services/actions/cleanup.go b/services/actions/cleanup.go new file mode 100644 index 0000000000000..81ba65c910f41 --- /dev/null +++ b/services/actions/cleanup.go @@ -0,0 +1,42 @@ +package actions + +import ( + "context" + "time" + + "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/storage" +) + +// Cleanup removes expired actions logs, data and artifacts +func Cleanup(taskCtx context.Context, olderThan time.Duration) error { + + // clean up expired artifacts + if err := CleanupArtifacts(taskCtx); err != nil { + return err + } + + return nil +} + +// CleanupArtifacts removes expired artifacts and set records expired status +func CleanupArtifacts(taskCtx context.Context) error { + artifacts, err := actions.ListExpiredArtifacts(taskCtx) + if err != nil { + return err + } + log.Info("Found %d expired artifacts", len(artifacts)) + for _, artifact := range artifacts { + if err := storage.ActionsArtifacts.Delete(artifact.StoragePath); err != nil { + log.Error("Cannot delete artifact %d: %v", artifact.ID, err) + continue + } + if err := actions.SetArtifactExpired(taskCtx, artifact.ID); err != nil { + log.Error("Cannot set artifact %d expired: %v", artifact.ID, err) + continue + } + log.Info("Artifact %d set expired", artifact.ID) + } + return nil +} diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go index 2e6560ec0c9dd..81a676c570ab4 100644 --- a/services/cron/tasks_basic.go +++ b/services/cron/tasks_basic.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/models/webhook" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/actions" "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/migrations" mirror_service "code.gitea.io/gitea/services/mirror" @@ -156,6 +157,20 @@ func registerCleanupPackages() { }) } +func registerActionsCleanup() { + RegisterTaskFatal("cleanup_actions", &OlderThanConfig{ + BaseConfig: BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@midnight", + }, + OlderThan: 24 * time.Hour, + }, func(ctx context.Context, _ *user_model.User, config Config) error { + realConfig := config.(*OlderThanConfig) + return actions.Cleanup(ctx, realConfig.OlderThan) + }) +} + func initBasicTasks() { if setting.Mirror.Enabled { registerUpdateMirrorTask() @@ -172,4 +187,7 @@ func initBasicTasks() { if setting.Packages.Enabled { registerCleanupPackages() } + if setting.Actions.Enabled { + registerActionsCleanup() + } } From 0c4aaf15c5cc1c5ed9769c0aec019943e98bd2a3 Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Mon, 31 Jul 2023 21:57:09 +0800 Subject: [PATCH 02/10] support custom retention days from upload-actions --- routers/api/actions/artifacts.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/routers/api/actions/artifacts.go b/routers/api/actions/artifacts.go index 0b401c0ff996a..c9b443ecc0de4 100644 --- a/routers/api/actions/artifacts.go +++ b/routers/api/actions/artifacts.go @@ -70,6 +70,7 @@ import ( "strings" "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" @@ -170,8 +171,9 @@ func (ar artifactRoutes) buildArtifactURL(runID int64, artifactHash, suffix stri } type getUploadArtifactRequest struct { - Type string - Name string + Type string + Name string + RetentionDays int64 } type getUploadArtifactResponse struct { @@ -180,7 +182,7 @@ type getUploadArtifactResponse struct { // getUploadArtifactURL generates a URL for uploading an artifact func (ar artifactRoutes) getUploadArtifactURL(ctx *ArtifactContext) { - _, runID, ok := validateRunID(ctx) + task, runID, ok := validateRunID(ctx) if !ok { return } @@ -192,6 +194,12 @@ func (ar artifactRoutes) getUploadArtifactURL(ctx *ArtifactContext) { return } + // set retention days + if req.RetentionDays > 0 { + cacheKey := fmt.Sprintf("actions_artifact_retention_days_%d_%d", task.ID, runID) + cache.GetCache().Put(cacheKey, req.RetentionDays, 0) + } + // use md5(artifact_name) to create upload url artifactHash := fmt.Sprintf("%x", md5.Sum([]byte(req.Name))) resp := getUploadArtifactResponse{ @@ -219,8 +227,14 @@ func (ar artifactRoutes) uploadArtifact(ctx *ArtifactContext) { return } - // create or get artifact with name and path + // get artifact retention days expiredDays := setting.Actions.ArtifactRetentionDays + cacheKey := fmt.Sprintf("actions_artifact_retention_days_%d_%d", task.ID, runID) + if v := cache.GetCache().Get(cacheKey); v != nil { + expiredDays = v.(int64) + } + + // create or get artifact with name and path artifact, err := actions.CreateArtifact(ctx, task, artifactName, artifactPath, expiredDays) if err != nil { log.Error("Error create or get artifact: %v", err) From d6a7200ecdb8baf6f1c07aa6ed49ac29dc5e4919 Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Thu, 3 Aug 2023 21:05:15 +0800 Subject: [PATCH 03/10] show unclickable text when artifact is expxired in repo view page --- models/actions/artifact.go | 5 +++-- routers/api/actions/artifacts.go | 4 +++- routers/web/repo/actions/view.go | 14 ++++++++++---- services/actions/cleanup.go | 7 ++----- web_src/js/components/RepoActionView.vue | 5 ++++- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/models/actions/artifact.go b/models/actions/artifact.go index 4c5fed9131d8e..80aa4a48f8efa 100644 --- a/models/actions/artifact.go +++ b/models/actions/artifact.go @@ -131,15 +131,16 @@ func ListUploadedArtifactsByRunID(ctx context.Context, runID int64) ([]*ActionAr type ActionArtifactMeta struct { ArtifactName string FileSize int64 + Status int64 } // ListUploadedArtifactsMeta returns all uploaded artifacts meta of a run func ListUploadedArtifactsMeta(ctx context.Context, runID int64) ([]*ActionArtifactMeta, error) { arts := make([]*ActionArtifactMeta, 0, 10) return arts, db.GetEngine(ctx).Table("action_artifact"). - Where("run_id=? AND status=?", runID, ArtifactStatusUploadConfirmed). + Where("run_id=? AND (status=? OR status=?)", runID, ArtifactStatusUploadConfirmed, ArtifactStatusExpired). GroupBy("artifact_name"). - Select("artifact_name, sum(file_size) as file_size"). + Select("artifact_name, sum(file_size) as file_size, max(status) as status"). Find(&arts) } diff --git a/routers/api/actions/artifacts.go b/routers/api/actions/artifacts.go index c9b443ecc0de4..27cfa9713b983 100644 --- a/routers/api/actions/artifacts.go +++ b/routers/api/actions/artifacts.go @@ -197,7 +197,9 @@ func (ar artifactRoutes) getUploadArtifactURL(ctx *ArtifactContext) { // set retention days if req.RetentionDays > 0 { cacheKey := fmt.Sprintf("actions_artifact_retention_days_%d_%d", task.ID, runID) - cache.GetCache().Put(cacheKey, req.RetentionDays, 0) + if err := cache.GetCache().Put(cacheKey, req.RetentionDays, 0); err != nil { + log.Warn("Error set cache %s: %v", cacheKey, err) + } } // use md5(artifact_name) to create upload url diff --git a/routers/web/repo/actions/view.go b/routers/web/repo/actions/view.go index abb1f6b66b979..21e174c0e0a5f 100644 --- a/routers/web/repo/actions/view.go +++ b/routers/web/repo/actions/view.go @@ -481,8 +481,9 @@ type ArtifactsViewResponse struct { } type ArtifactsViewItem struct { - Name string `json:"name"` - Size int64 `json:"size"` + Name string `json:"name"` + Size int64 `json:"size"` + Status string `json:"status"` } func ArtifactsView(ctx *context_module.Context) { @@ -505,9 +506,14 @@ func ArtifactsView(ctx *context_module.Context) { Artifacts: make([]*ArtifactsViewItem, 0, len(artifacts)), } for _, art := range artifacts { + status := "completed" + if art.Status == actions_model.ArtifactStatusExpired { + status = "expired" + } artifactsResponse.Artifacts = append(artifactsResponse.Artifacts, &ArtifactsViewItem{ - Name: art.ArtifactName, - Size: art.FileSize, + Name: art.ArtifactName, + Size: art.FileSize, + Status: status, }) } ctx.JSON(http.StatusOK, artifactsResponse) diff --git a/services/actions/cleanup.go b/services/actions/cleanup.go index 81ba65c910f41..82dd38d58d351 100644 --- a/services/actions/cleanup.go +++ b/services/actions/cleanup.go @@ -11,13 +11,10 @@ import ( // Cleanup removes expired actions logs, data and artifacts func Cleanup(taskCtx context.Context, olderThan time.Duration) error { + // clean up expired actions logs // clean up expired artifacts - if err := CleanupArtifacts(taskCtx); err != nil { - return err - } - - return nil + return CleanupArtifacts(taskCtx) } // CleanupArtifacts removes expired artifacts and set records expired status diff --git a/web_src/js/components/RepoActionView.vue b/web_src/js/components/RepoActionView.vue index 08cead08c58e5..cc90b4e36b2b2 100644 --- a/web_src/js/components/RepoActionView.vue +++ b/web_src/js/components/RepoActionView.vue @@ -50,9 +50,12 @@ From f690e2aa4009ce2c0d20e36401b9a3ecf0cc1172 Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Thu, 3 Aug 2023 21:29:30 +0800 Subject: [PATCH 04/10] missing license header --- services/actions/cleanup.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/actions/cleanup.go b/services/actions/cleanup.go index 82dd38d58d351..8d674380698f5 100644 --- a/services/actions/cleanup.go +++ b/services/actions/cleanup.go @@ -1,3 +1,6 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + package actions import ( From c94b16386645a6bef06e15a7b88afbe6c3bb0e26 Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Mon, 7 Aug 2023 22:15:07 +0800 Subject: [PATCH 05/10] use query string to pass through retention days to next uploading request, drop cache use --- routers/api/actions/artifacts.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/routers/api/actions/artifacts.go b/routers/api/actions/artifacts.go index 27cfa9713b983..308c02e20debd 100644 --- a/routers/api/actions/artifacts.go +++ b/routers/api/actions/artifacts.go @@ -70,7 +70,6 @@ import ( "strings" "code.gitea.io/gitea/models/actions" - "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" @@ -182,7 +181,7 @@ type getUploadArtifactResponse struct { // getUploadArtifactURL generates a URL for uploading an artifact func (ar artifactRoutes) getUploadArtifactURL(ctx *ArtifactContext) { - task, runID, ok := validateRunID(ctx) + _, runID, ok := validateRunID(ctx) if !ok { return } @@ -195,17 +194,16 @@ func (ar artifactRoutes) getUploadArtifactURL(ctx *ArtifactContext) { } // set retention days + req.RetentionDays = 17 + retentionQuery := "" if req.RetentionDays > 0 { - cacheKey := fmt.Sprintf("actions_artifact_retention_days_%d_%d", task.ID, runID) - if err := cache.GetCache().Put(cacheKey, req.RetentionDays, 0); err != nil { - log.Warn("Error set cache %s: %v", cacheKey, err) - } + retentionQuery = fmt.Sprintf("?retentionDays=%d", req.RetentionDays) } // use md5(artifact_name) to create upload url artifactHash := fmt.Sprintf("%x", md5.Sum([]byte(req.Name))) resp := getUploadArtifactResponse{ - FileContainerResourceURL: ar.buildArtifactURL(runID, artifactHash, "upload"), + FileContainerResourceURL: ar.buildArtifactURL(runID, artifactHash, "upload"+retentionQuery), } log.Debug("[artifact] get upload url: %s", resp.FileContainerResourceURL) ctx.JSON(http.StatusOK, resp) @@ -231,10 +229,16 @@ func (ar artifactRoutes) uploadArtifact(ctx *ArtifactContext) { // get artifact retention days expiredDays := setting.Actions.ArtifactRetentionDays - cacheKey := fmt.Sprintf("actions_artifact_retention_days_%d_%d", task.ID, runID) - if v := cache.GetCache().Get(cacheKey); v != nil { - expiredDays = v.(int64) + if queryRetentionDays := ctx.Req.URL.Query().Get("retentionDays"); queryRetentionDays != "" { + expiredDays, err = strconv.ParseInt(queryRetentionDays, 10, 64) + if err != nil { + log.Error("Error parse retention days: %v", err) + ctx.Error(http.StatusBadRequest, "Error parse retention days") + return + } } + log.Debug("[artifact] upload chunk, name: %s, path: %s, size: %d, retention days: %d", + artifactName, artifactPath, fileRealTotalSize, expiredDays) // create or get artifact with name and path artifact, err := actions.CreateArtifact(ctx, task, artifactName, artifactPath, expiredDays) From 0016a7a0b4fccf8b479b580e0e94fbf8d965e1bf Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Thu, 10 Aug 2023 20:54:59 +0800 Subject: [PATCH 06/10] use query string to pass artifact retention days, fix test failures --- routers/api/actions/artifacts.go | 1 - services/actions/cleanup.go | 2 +- .../integration/api_actions_artifact_test.go | 42 ++++++++++++++++++- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/routers/api/actions/artifacts.go b/routers/api/actions/artifacts.go index 308c02e20debd..c45dc667afcdb 100644 --- a/routers/api/actions/artifacts.go +++ b/routers/api/actions/artifacts.go @@ -194,7 +194,6 @@ func (ar artifactRoutes) getUploadArtifactURL(ctx *ArtifactContext) { } // set retention days - req.RetentionDays = 17 retentionQuery := "" if req.RetentionDays > 0 { retentionQuery = fmt.Sprintf("?retentionDays=%d", req.RetentionDays) diff --git a/services/actions/cleanup.go b/services/actions/cleanup.go index 8d674380698f5..ee27f08a4cb34 100644 --- a/services/actions/cleanup.go +++ b/services/actions/cleanup.go @@ -14,7 +14,7 @@ import ( // Cleanup removes expired actions logs, data and artifacts func Cleanup(taskCtx context.Context, olderThan time.Duration) error { - // clean up expired actions logs + // TODO: clean up expired actions logs // clean up expired artifacts return CleanupArtifacts(taskCtx) diff --git a/tests/integration/api_actions_artifact_test.go b/tests/integration/api_actions_artifact_test.go index 6590ca667cf59..101bedde02788 100644 --- a/tests/integration/api_actions_artifact_test.go +++ b/tests/integration/api_actions_artifact_test.go @@ -18,8 +18,9 @@ type uploadArtifactResponse struct { } type getUploadArtifactRequest struct { - Type string - Name string + Type string + Name string + RetentionDays int64 } func TestActionsArtifactUploadSingleFile(t *testing.T) { @@ -252,3 +253,40 @@ func TestActionsArtifactDownloadMultiFiles(t *testing.T) { assert.Equal(t, resp.Body.String(), body) } } + +func TestActionsArtifactUploadWithRetentionDays(t *testing.T) { + defer tests.PrepareTestEnv(t)() + + // acquire artifact upload url + req := NewRequestWithJSON(t, "POST", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts", getUploadArtifactRequest{ + Type: "actions_storage", + Name: "artifact-retention-days", + RetentionDays: 9, + }) + req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a") + resp := MakeRequest(t, req, http.StatusOK) + var uploadResp uploadArtifactResponse + DecodeJSON(t, resp, &uploadResp) + assert.Contains(t, uploadResp.FileContainerResourceURL, "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts") + assert.Contains(t, uploadResp.FileContainerResourceURL, "?retentionDays=9") + + // get upload url + idx := strings.Index(uploadResp.FileContainerResourceURL, "/api/actions_pipeline/_apis/pipelines/") + url := uploadResp.FileContainerResourceURL[idx:] + "&itemPath=artifact-retention-days/abc.txt" + + // upload artifact chunk + body := strings.Repeat("A", 1024) + req = NewRequestWithBody(t, "PUT", url, strings.NewReader(body)) + req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a") + req.Header.Add("Content-Range", "bytes 0-1023/1024") + req.Header.Add("x-tfs-filelength", "1024") + req.Header.Add("x-actions-results-md5", "1HsSe8LeLWh93ILaw1TEFQ==") // base64(md5(body)) + MakeRequest(t, req, http.StatusOK) + + t.Logf("Create artifact confirm") + + // confirm artifact upload + req = NewRequest(t, "PATCH", "/api/actions_pipeline/_apis/pipelines/workflows/791/artifacts?artifactName=artifact-retention-days") + req = addTokenAuthHeader(req, "Bearer 8061e833a55f6fc0157c98b883e91fcfeeb1a71a") + MakeRequest(t, req, http.StatusOK) +} From 5140cdcef36ee3b21025a83ee60e9a1f468f00f5 Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Thu, 24 Aug 2023 21:53:59 +0800 Subject: [PATCH 07/10] update list artifact methods --- models/actions/artifact.go | 6 +++--- services/actions/cleanup.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/models/actions/artifact.go b/models/actions/artifact.go index 80aa4a48f8efa..2890f1757bceb 100644 --- a/models/actions/artifact.go +++ b/models/actions/artifact.go @@ -156,8 +156,8 @@ func ListArtifactsByRunIDAndName(ctx context.Context, runID int64, name string) return arts, db.GetEngine(ctx).Where("run_id=? AND artifact_name=?", runID, name).Find(&arts) } -// ListExpiredArtifacts returns all expired artifacts but not deleted -func ListExpiredArtifacts(ctx context.Context) ([]*ActionArtifact, error) { +// ListNeedExpiredArtifacts returns all need expired artifacts but not deleted +func ListNeedExpiredArtifacts(ctx context.Context) ([]*ActionArtifact, error) { arts := make([]*ActionArtifact, 0, 10) return arts, db.GetEngine(ctx). Where("expired_unix < ? AND status = ?", timeutil.TimeStamp(time.Now().Unix()), ArtifactStatusUploadConfirmed).Find(&arts) @@ -165,6 +165,6 @@ func ListExpiredArtifacts(ctx context.Context) ([]*ActionArtifact, error) { // SetArtifactExpired sets an artifact to expired func SetArtifactExpired(ctx context.Context, artifactID int64) error { - _, err := db.GetEngine(ctx).Where("id=?", artifactID).Cols("status").Update(&ActionArtifact{Status: ArtifactStatusExpired}) + _, err := db.GetEngine(ctx).Where("id=? AND status = ?", artifactID, ArtifactStatusUploadConfirmed).Cols("status").Update(&ActionArtifact{Status: ArtifactStatusExpired}) return err } diff --git a/services/actions/cleanup.go b/services/actions/cleanup.go index ee27f08a4cb34..785eeb5838ea9 100644 --- a/services/actions/cleanup.go +++ b/services/actions/cleanup.go @@ -22,7 +22,7 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { // CleanupArtifacts removes expired artifacts and set records expired status func CleanupArtifacts(taskCtx context.Context) error { - artifacts, err := actions.ListExpiredArtifacts(taskCtx) + artifacts, err := actions.ListNeedExpiredArtifacts(taskCtx) if err != nil { return err } From 9c4738aa77f35d5b619f27d930776028aeba7a0f Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Thu, 31 Aug 2023 20:17:40 +0800 Subject: [PATCH 08/10] add expired_unix column in artifact table migration --- models/migrations/migrations.go | 2 ++ models/migrations/v1_21/v274.go | 36 +++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 models/migrations/v1_21/v274.go diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 9f4acda23699a..928fd10deecb6 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -528,6 +528,8 @@ var migrations = []Migration{ NewMigration("Add Version to ActionRun table", v1_21.AddVersionToActionRunTable), // v273 -> v274 NewMigration("Add Action Schedule Table", v1_21.AddActionScheduleTable), + // v274 -> v275 + NewMigration("Add ExpiredUnix Column to Artifact Table", v1_21.AddExpiredUnixColumnInActionArtifactTable), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v1_21/v274.go b/models/migrations/v1_21/v274.go new file mode 100644 index 0000000000000..d1238002cfc41 --- /dev/null +++ b/models/migrations/v1_21/v274.go @@ -0,0 +1,36 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_21 //nolint +import ( + "time" + + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func AddExpiredUnixColumnInActionArtifactTable(x *xorm.Engine) error { + type ActionArtifact struct { + ExpiredUnix timeutil.TimeStamp `xorm:"index"` // time when the artifact will be expired + } + if err := x.Sync(new(ActionArtifact)); err != nil { + return err + } + return updateArtifactsExpiredUnixTo90Days(x) +} + +func updateArtifactsExpiredUnixTo90Days(x *xorm.Engine) error { + sess := x.NewSession() + defer sess.Close() + + if err := sess.Begin(); err != nil { + return err + } + expired_time := time.Now().AddDate(0, 0, 90).Unix() + if _, err := sess.Exec(`UPDATE action_artifact SET expired_unix=? WHERE status='2' AND expired_unix is NULL`, expired_time); err != nil { + return err + } + + return sess.Commit() +} From bac5707f6da0d40137ba7a4472884425ce781e22 Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Thu, 31 Aug 2023 21:44:51 +0800 Subject: [PATCH 09/10] fix lint problem --- models/migrations/v1_21/v274.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/migrations/v1_21/v274.go b/models/migrations/v1_21/v274.go index d1238002cfc41..df5994f159ffb 100644 --- a/models/migrations/v1_21/v274.go +++ b/models/migrations/v1_21/v274.go @@ -27,8 +27,8 @@ func updateArtifactsExpiredUnixTo90Days(x *xorm.Engine) error { if err := sess.Begin(); err != nil { return err } - expired_time := time.Now().AddDate(0, 0, 90).Unix() - if _, err := sess.Exec(`UPDATE action_artifact SET expired_unix=? WHERE status='2' AND expired_unix is NULL`, expired_time); err != nil { + expiredTime := time.Now().AddDate(0, 0, 90).Unix() + if _, err := sess.Exec(`UPDATE action_artifact SET expired_unix=? WHERE status='2' AND expired_unix is NULL`, expiredTime); err != nil { return err } From 786fa2e8033fbd5dc741cde4e0e3bd6545cb25f0 Mon Sep 17 00:00:00 2001 From: fuxiaohei Date: Mon, 4 Sep 2023 20:51:24 +0800 Subject: [PATCH 10/10] add docs for artifact clean task --- custom/conf/app.example.ini | 2 ++ .../config-cheat-sheet.en-us.md | 7 +++++++ models/actions/artifact.go | 19 +++++++++---------- models/migrations/migrations.go | 2 +- routers/api/actions/artifacts_chunks.go | 2 +- routers/web/repo/actions/view.go | 2 +- 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index dd673190aa745..a2fab2fd50931 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2564,6 +2564,8 @@ LEVEL = Info ;; ;; Default platform to get action plugins, `github` for `https://github.com`, `self` for the current Gitea instance. ;DEFAULT_ACTIONS_URL = github +;; Default artifact retention time in days, default is 90 days +;ARTIFACT_RETENTION_DAYS = 90 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/docs/content/administration/config-cheat-sheet.en-us.md b/docs/content/administration/config-cheat-sheet.en-us.md index 4158f14cb1ddd..7e8befb8b7e67 100644 --- a/docs/content/administration/config-cheat-sheet.en-us.md +++ b/docs/content/administration/config-cheat-sheet.en-us.md @@ -955,6 +955,12 @@ Default templates for project boards: - `SCHEDULE`: **@midnight** : Interval as a duration between each synchronization, it will always attempt synchronization when the instance starts. - `UPDATE_EXISTING`: **true**: Create new users, update existing user data and disable users that are not in external source anymore (default) or only create new users if UPDATE_EXISTING is set to false. +## Cron - Cleanup Expired Actions Assets (`cron.cleanup_actions`) + +- `ENABLED`: **true**: Enable cleanup expired actions assets job. +- `RUN_AT_START`: **true**: Run job at start time (if ENABLED). +- `SCHEDULE`: **@midnight** : Cron syntax for the job. + ### Extended cron tasks (not enabled by default) #### Cron - Garbage collect all repositories (`cron.git_gc_repos`) @@ -1381,6 +1387,7 @@ PROXY_HOSTS = *.github.com - `DEFAULT_ACTIONS_URL`: **github**: Default platform to get action plugins, `github` for `https://github.com`, `self` for the current Gitea instance. - `STORAGE_TYPE`: **local**: Storage type for actions logs, `local` for local disk or `minio` for s3 compatible object storage service, default is `local` or other name defined with `[storage.xxx]` - `MINIO_BASE_PATH`: **actions_log/**: Minio base path on the bucket only available when STORAGE_TYPE is `minio` +- `ARTIFACT_RETENTION_DAYS`: **90**: Number of days to keep artifacts. Set to 0 to disable artifact retention. Default is 90 days if not set. `DEFAULT_ACTIONS_URL` indicates where the Gitea Actions runners should find the actions with relative path. For example, `uses: actions/checkout@v3` means `https://github.com/actions/checkout@v3` since the value of `DEFAULT_ACTIONS_URL` is `github`. diff --git a/models/actions/artifact.go b/models/actions/artifact.go index 2890f1757bceb..849a90fd10a4f 100644 --- a/models/actions/artifact.go +++ b/models/actions/artifact.go @@ -16,15 +16,14 @@ import ( "code.gitea.io/gitea/modules/util" ) +// ArtifactStatus is the status of an artifact, uploading, expired or need-delete +type ArtifactStatus int64 + const ( - // ArtifactStatusUploadPending is the status of an artifact upload that is pending - ArtifactStatusUploadPending = 1 - // ArtifactStatusUploadConfirmed is the status of an artifact upload that is confirmed - ArtifactStatusUploadConfirmed = 2 - // ArtifactStatusUploadError is the status of an artifact upload that is errored - ArtifactStatusUploadError = 3 - // ArtifactStatusExpired is the status of an artifact that is expired - ArtifactStatusExpired = 4 + ArtifactStatusUploadPending ArtifactStatus = iota + 1 // 1， ArtifactStatusUploadPending is the status of an artifact upload that is pending + ArtifactStatusUploadConfirmed // 2， ArtifactStatusUploadConfirmed is the status of an artifact upload that is confirmed + ArtifactStatusUploadError // 3， ArtifactStatusUploadError is the status of an artifact upload that is errored + ArtifactStatusExpired // 4, ArtifactStatusExpired is the status of an artifact that is expired ) func init() { @@ -65,7 +64,7 @@ func CreateArtifact(ctx context.Context, t *ActionTask, artifactName, artifactPa RepoID: t.RepoID, OwnerID: t.OwnerID, CommitSHA: t.CommitSHA, - Status: ArtifactStatusUploadPending, + Status: int64(ArtifactStatusUploadPending), ExpiredUnix: timeutil.TimeStamp(time.Now().Unix() + 3600*24*expiredDays), } if _, err := db.GetEngine(ctx).Insert(artifact); err != nil { @@ -165,6 +164,6 @@ func ListNeedExpiredArtifacts(ctx context.Context) ([]*ActionArtifact, error) { // SetArtifactExpired sets an artifact to expired func SetArtifactExpired(ctx context.Context, artifactID int64) error { - _, err := db.GetEngine(ctx).Where("id=? AND status = ?", artifactID, ArtifactStatusUploadConfirmed).Cols("status").Update(&ActionArtifact{Status: ArtifactStatusExpired}) + _, err := db.GetEngine(ctx).Where("id=? AND status = ?", artifactID, ArtifactStatusUploadConfirmed).Cols("status").Update(&ActionArtifact{Status: int64(ArtifactStatusExpired)}) return err } diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 928fd10deecb6..40df1cd624347 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -529,7 +529,7 @@ var migrations = []Migration{ // v273 -> v274 NewMigration("Add Action Schedule Table", v1_21.AddActionScheduleTable), // v274 -> v275 - NewMigration("Add ExpiredUnix Column to Artifact Table", v1_21.AddExpiredUnixColumnInActionArtifactTable), + NewMigration("Add Actions artifacts expiration date", v1_21.AddExpiredUnixColumnInActionArtifactTable), } // GetCurrentDBVersion returns the current db version diff --git a/routers/api/actions/artifacts_chunks.go b/routers/api/actions/artifacts_chunks.go index 30d31b4d75695..458d671cff6d8 100644 --- a/routers/api/actions/artifacts_chunks.go +++ b/routers/api/actions/artifacts_chunks.go @@ -179,7 +179,7 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st // save storage path to artifact log.Debug("[artifact] merge chunks to artifact: %d, %s", artifact.ID, storagePath) artifact.StoragePath = storagePath - artifact.Status = actions.ArtifactStatusUploadConfirmed + artifact.Status = int64(actions.ArtifactStatusUploadConfirmed) if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil { return fmt.Errorf("update artifact error: %v", err) } diff --git a/routers/web/repo/actions/view.go b/routers/web/repo/actions/view.go index e3ea7ec99bafe..a9c2858303aac 100644 --- a/routers/web/repo/actions/view.go +++ b/routers/web/repo/actions/view.go @@ -512,7 +512,7 @@ func ArtifactsView(ctx *context_module.Context) { } for _, art := range artifacts { status := "completed" - if art.Status == actions_model.ArtifactStatusExpired { + if art.Status == int64(actions_model.ArtifactStatusExpired) { status = "expired" } artifactsResponse.Artifacts = append(artifactsResponse.Artifacts, &ArtifactsViewItem{