From bb96349d0958d6842a09543ca0e01a16ca2106a3 Mon Sep 17 00:00:00 2001 From: Antoine GIRARD Date: Sat, 7 Nov 2020 22:52:16 +0100 Subject: [PATCH 1/4] rebase on master --- models/migrations/migrations.go | 2 ++ models/migrations/v160.go | 16 ++++++++++++++++ models/repo.go | 22 +++++++++++++++++++--- modules/auth/repo_form.go | 2 ++ modules/git/repo.go | 7 ++++++- modules/repository/create.go | 1 + modules/structs/repo.go | 4 ++++ options/locale/locale_en-US.ini | 3 +++ routers/api/v1/repo/repo.go | 4 ++++ routers/private/hook.go | 21 +++++++++++++++++++++ routers/repo/repo.go | 1 + routers/repo/setting.go | 15 +++++++++++++++ templates/repo/settings/options.tmpl | 6 +++++- 13 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 models/migrations/v160.go diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 5a2646474c77f..bf340ce4dedad 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -254,6 +254,8 @@ var migrations = []Migration{ NewMigration("code comment replies should have the commitID of the review they are replying to", updateCodeCommentReplies), // v159 -> v160 NewMigration("update reactions constraint", updateReactionConstraint), + // v160 -> v161 + NewMigration("add size limit on repository", addSizeLimitOnRepo), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v160.go b/models/migrations/v160.go new file mode 100644 index 0000000000000..82cc528bb9cd4 --- /dev/null +++ b/models/migrations/v160.go @@ -0,0 +1,16 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import "xorm.io/xorm" + +func addSizeLimitOnRepo(x *xorm.Engine) error { + type Repository struct { + ID int64 `xorm:"pk autoincr"` + SizeLimit int64 `xorm:"NOT NULL DEFAULT 0"` + } + + return x.Sync2(new(Repository)) +} diff --git a/models/repo.go b/models/repo.go index 2b53ac666cbe7..55cbcec70ebbf 100644 --- a/models/repo.go +++ b/models/repo.go @@ -230,6 +230,7 @@ type Repository struct { TemplateID int64 `xorm:"INDEX"` TemplateRepo *Repository `xorm:"-"` Size int64 `xorm:"NOT NULL DEFAULT 0"` + SizeLimit int64 `xorm:"NOT NULL DEFAULT 0"` CodeIndexerStatus *RepoIndexerStatus `xorm:"-"` StatsIndexerStatus *RepoIndexerStatus `xorm:"-"` IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true"` @@ -860,20 +861,29 @@ func (repo *Repository) IsOwnedBy(userID int64) bool { return repo.OwnerID == userID } -func (repo *Repository) updateSize(e Engine) error { +func (repo *Repository) computeSize() (int64, error) { size, err := util.GetDirectorySize(repo.RepoPath()) if err != nil { - return fmt.Errorf("updateSize: %v", err) + return 0, fmt.Errorf("computeSize: %v", err) } objs, err := repo.GetLFSMetaObjects(-1, 0) if err != nil { - return fmt.Errorf("updateSize: GetLFSMetaObjects: %v", err) + return 0, fmt.Errorf("computeSize: GetLFSMetaObjects: %v", err) } for _, obj := range objs { size += obj.Size } + return size, nil +} + +func (repo *Repository) updateSize(e Engine) error { + size, err := repo.computeSize() + if err != nil { + return fmt.Errorf("updateSize: %v", err) + } + repo.Size = size _, err = e.ID(repo.ID).Cols("size").Update(repo) return err @@ -884,6 +894,11 @@ func (repo *Repository) UpdateSize(ctx DBContext) error { return repo.updateSize(ctx.e) } +// RepoSizeIsOversized return if is over size limitation +func (repo *Repository) RepoSizeIsOversized(additionalSize int64) bool { + return repo.SizeLimit > 0 && repo.Size+additionalSize > repo.SizeLimit +} + // CanUserFork returns true if specified user can fork repository. func (repo *Repository) CanUserFork(user *User) (bool, error) { if user == nil { @@ -1101,6 +1116,7 @@ type CreateRepoOptions struct { AutoInit bool Status RepositoryStatus TrustModel TrustModelType + SizeLimit int64 } // GetRepoInitFile returns repository init files diff --git a/modules/auth/repo_form.go b/modules/auth/repo_form.go index f27812bb1b964..1a9e0e1b81161 100644 --- a/modules/auth/repo_form.go +++ b/modules/auth/repo_form.go @@ -48,6 +48,7 @@ type CreateRepoForm struct { Avatar bool Labels bool TrustModel string + SizeLimit int64 } // Validate validates the fields @@ -126,6 +127,7 @@ type RepoSettingForm struct { Private bool Template bool EnablePrune bool + RepoSizeLimit int64 // Advanced settings EnableWiki bool diff --git a/modules/git/repo.go b/modules/git/repo.go index ae370d3da973a..3e7868ce90c15 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -352,8 +352,13 @@ const ( // CountObjects returns the results of git count-objects on the repoPath func CountObjects(repoPath string) (*CountObject, error) { + return CountObjectsWithEnv(repoPath, nil) +} + +// CountObjectsWithEnv returns the results of git count-objects on the repoPath with custom env setup +func CountObjectsWithEnv(repoPath string, env []string) (*CountObject, error) { cmd := NewCommand("count-objects", "-v") - stdout, err := cmd.RunInDir(repoPath) + stdout, err := cmd.RunInDirWithEnv(repoPath, env) if err != nil { return nil, err } diff --git a/modules/repository/create.go b/modules/repository/create.go index 1408637815d33..8fc73439befe0 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -45,6 +45,7 @@ func CreateRepository(doer, u *models.User, opts models.CreateRepoOptions) (*mod Status: opts.Status, IsEmpty: !opts.AutoInit, TrustModel: opts.TrustModel, + SizeLimit: opts.SizeLimit, } if err := models.WithTx(func(ctx models.DBContext) error { diff --git a/modules/structs/repo.go b/modules/structs/repo.go index c12f8e1c18e11..615c1d65e9488 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -122,6 +122,8 @@ type CreateRepoOption struct { // TrustModel of the repository // enum: default,collaborator,committer,collaboratorcommitter TrustModel string `json:"trust_model"` + // SizeLimit of the repository + SizeLimit int64 `json:"size_limit"` } // EditRepoOption options when editing a repository's properties @@ -168,6 +170,8 @@ type EditRepoOption struct { AllowSquash *bool `json:"allow_squash_merge,omitempty"` // set to `true` to archive this repository. Archived *bool `json:"archived,omitempty"` + // SizeLimit of the repository. + SizeLimit *int64 `json:"size_limit,omitempty"` } // CreateBranchRepoOption options when creating a branch in a repository diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index f4cdcac427938..79d561961948b 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -653,6 +653,7 @@ owner = Owner repo_name = Repository Name repo_name_helper = Good repository names use short, memorable and unique keywords. repo_size = Repository Size +repo_size_limit = Repository Size Limit template = Template template_select = Select a template. template_helper = Make repository a template @@ -733,6 +734,8 @@ archive.pull.nocomment = This repo is archived. You cannot comment on pull reque form.reach_limit_of_creation = You have already reached your limit of %d repositories. form.name_reserved = The repository name '%s' is reserved. form.name_pattern_not_allowed = The pattern '%s' is not allowed in a repository name. +form.repo_size_limit_negative = Repository size limitation cannot be negative. +form.repo_size_limit_only_by_admins = Only administrators can change the repository size limitation. need_auth = Clone Authorization migrate_options = Migration Options diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index 116e413125bf7..6f110bbb55329 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -246,6 +246,7 @@ func CreateUserRepo(ctx *context.APIContext, owner *models.User, opt api.CreateR DefaultBranch: opt.DefaultBranch, TrustModel: models.ToTrustModel(opt.TrustModel), IsTemplate: opt.Template, + SizeLimit: opt.SizeLimit, }) if err != nil { if models.IsErrRepoAlreadyExist(err) { @@ -568,6 +569,9 @@ func updateBasicProperties(ctx *context.APIContext, opts api.EditRepoOption) err repo.DefaultBranch = *opts.DefaultBranch } + if opts.SizeLimit != nil { + repo.SizeLimit = *opts.SizeLimit + } if err := models.UpdateRepository(repo, visibilityChanged); err != nil { ctx.Error(http.StatusInternalServerError, "UpdateRepository", err) return err diff --git a/routers/private/hook.go b/routers/private/hook.go index dac39407562fc..313ab7a5bb64b 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -155,12 +155,33 @@ func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) { private.GitQuarantinePath+"="+opts.GitQuarantinePath) } + pushSize, err := git.CountObjectsWithEnv(repo.RepoPath(), env) + if err != nil { + log.Error("Unable to get repository size with env %v: %s Error: %v", repo.RepoPath(), env, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": err.Error(), + }) + return + } + log.Trace("Push size %d", pushSize.Size) + // Iterate across the provided old commit IDs for i := range opts.OldCommitIDs { oldCommitID := opts.OldCommitIDs[i] newCommitID := opts.NewCommitIDs[i] refFullName := opts.RefFullNames[i] + //Check size + if newCommitID != git.EmptySHA && repo.RepoSizeIsOversized(pushSize.Size) { //Check next size if we are not deleting a reference + log.Warn("Forbidden: new repo size is over limitation: %d", repo.SizeLimit) + ctx.JSON(http.StatusForbidden, map[string]interface{}{ + "err": fmt.Sprintf("new repo size is over limitation: %d", repo.SizeLimit), + }) + } + //TODO investigate why on force push some git objects are not cleaned on server side. + //TODO corner-case force push and branch creation -> git.EmptySHA == oldCommitID + //TODO calculate pushed LFS objects size + branchName := strings.TrimPrefix(refFullName, git.BranchPrefix) if branchName == repo.DefaultBranch && newCommitID == git.EmptySHA { log.Warn("Forbidden: Branch: %s is the default branch in %-v and cannot be deleted", branchName, repo) diff --git a/routers/repo/repo.go b/routers/repo/repo.go index 2614389aaaca8..7df42e3030b03 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -250,6 +250,7 @@ func CreatePost(ctx *context.Context, form auth.CreateRepoForm) { AutoInit: form.AutoInit, IsTemplate: form.Template, TrustModel: models.ToTrustModel(form.TrustModel), + SizeLimit: form.SizeLimit, }) if err == nil { log.Trace("Repository created [%d]: %s/%s", repo.ID, ctxUser.Name, repo.Name) diff --git a/routers/repo/setting.go b/routers/repo/setting.go index 368879234bcde..90f357355cb75 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -50,6 +50,7 @@ func Settings(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("repo.settings") ctx.Data["PageIsSettingsOptions"] = true ctx.Data["ForcePrivate"] = setting.Repository.ForcePrivate + ctx.Data["Err_RepoSize"] = ctx.Repo.Repository.RepoSizeIsOversized(ctx.Repo.Repository.SizeLimit / 10) // less than 10% left signing, _ := models.SigningKey(ctx.Repo.Repository.RepoPath()) ctx.Data["SigningKeyAvailable"] = len(signing) > 0 @@ -64,6 +65,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { ctx.Data["PageIsSettingsOptions"] = true repo := ctx.Repo.Repository + ctx.Data["Err_RepoSize"] = repo.RepoSizeIsOversized(repo.SizeLimit / 10) // less than 10% left switch ctx.Query("action") { case "update": @@ -128,6 +130,19 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { return } + if form.RepoSizeLimit < 0 { + ctx.Data["Err_RepoSizeLimit"] = true + ctx.RenderWithErr(ctx.Tr("repo.form.repo_size_limit_negative"), tplSettingsOptions, &form) + return + } + + if !ctx.User.IsAdmin && repo.SizeLimit != form.RepoSizeLimit { + ctx.Data["Err_RepoSizeLimit"] = true + ctx.RenderWithErr(ctx.Tr("repo.form.repo_size_limit_only_by_admins"), tplSettingsOptions, &form) + return + } + repo.SizeLimit = form.RepoSizeLimit + repo.IsPrivate = form.Private if err := models.UpdateRepository(repo, visibilityChanged); err != nil { ctx.ServerError("UpdateRepository", err) diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 96414731bc982..60e1c32237c2e 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -17,7 +17,11 @@
- {{SizeFmt .Repository.Size}} + {{SizeFmt .Repository.Size}}{{if .Repository.SizeLimit}}/{{SizeFmt .Repository.SizeLimit}}{{end}} +
+
+ +
From 56b92a17e0d7597777fc946078d364509afde920 Mon Sep 17 00:00:00 2001 From: Antoine GIRARD Date: Sat, 7 Nov 2020 23:03:44 +0100 Subject: [PATCH 2/4] generate swagger --- templates/swagger/v1_json.tmpl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index e759a1558ce08..3e2967f02e386 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -12271,6 +12271,12 @@ "type": "string", "x-go-name": "Readme" }, + "size_limit": { + "description": "SizeLimit of the repository", + "type": "integer", + "format": "int64", + "x-go-name": "SizeLimit" + }, "template": { "description": "Whether the repository is template", "type": "boolean", @@ -13029,6 +13035,12 @@ "type": "boolean", "x-go-name": "Private" }, + "size_limit": { + "description": "SizeLimit of the repository.", + "type": "integer", + "format": "int64", + "x-go-name": "SizeLimit" + }, "template": { "description": "either `true` to make this repository a template or `false` to make it a normal repository", "type": "boolean", From 2f655456d63d503a5632083ceadd1511c47bb051 Mon Sep 17 00:00:00 2001 From: Antoine GIRARD Date: Sat, 7 Nov 2020 23:18:52 +0100 Subject: [PATCH 3/4] add some tests --- .../api_helper_for_declarative_test.go | 14 ++++++++ integrations/git_test.go | 36 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/integrations/api_helper_for_declarative_test.go b/integrations/api_helper_for_declarative_test.go index b8e513958e841..977d3b46f8b54 100644 --- a/integrations/api_helper_for_declarative_test.go +++ b/integrations/api_helper_for_declarative_test.go @@ -371,3 +371,17 @@ func doAPIAddRepoToOrganizationTeam(ctx APITestContext, teamID int64, orgName, r ctx.Session.MakeRequest(t, req, http.StatusNoContent) } } + +func doAPISetRepoSizeLimit(ctx APITestContext, owner, repo string, size int64) func(*testing.T) { + return func(t *testing.T) { + urlStr := fmt.Sprintf("/api/v1/repos/%s/%s?token=%s", + owner, repo, ctx.Token) + req := NewRequestWithJSON(t, http.MethodPatch, urlStr, &api.EditRepoOption{SizeLimit: &size}) + + if ctx.ExpectedCode != 0 { + ctx.Session.MakeRequest(t, req, ctx.ExpectedCode) + return + } + ctx.Session.MakeRequest(t, req, 200) + } +} diff --git a/integrations/git_test.go b/integrations/git_test.go index c3c11268296f9..8fc9eadf9dea2 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -68,6 +68,34 @@ func testGit(t *testing.T, u *url.URL) { rawTest(t, &httpContext, little, big, littleLFS, bigLFS) mediaTest(t, &httpContext, little, big, littleLFS, bigLFS) + t.Run("SizeLimit", func(t *testing.T) { + t.Run("Under", func(t *testing.T) { + PrintCurrentTest(t) + doCommitAndPush(t, littleSize, dstPath, "data-file-") + }) + t.Run("Over", func(t *testing.T) { + PrintCurrentTest(t) + doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, littleSize) + //TODO fix this doCommitAndPushWithExpectedError(t, bigSize, dstPath, "data-file-") + }) + t.Run("UnderAfterResize", func(t *testing.T) { + PrintCurrentTest(t) + doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, bigSize*10) + doCommitAndPush(t, littleSize, dstPath, "data-file-") + }) + t.Run("Deletion", func(t *testing.T) { + PrintCurrentTest(t) + //TODO delete a file + //doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") + }) + //TODO delete branch + //TODO delete tag + //TODO add big commit that will be over with the push + //TODO add lfs + //TODO remove lfs + //TODO add missing case + }) + t.Run("BranchProtectMerge", doBranchProtectPRMerge(&httpContext, dstPath)) t.Run("MergeFork", func(t *testing.T) { defer PrintCurrentTest(t)() @@ -291,6 +319,14 @@ func doCommitAndPush(t *testing.T, size int, repoPath, prefix string) string { return name } +func doCommitAndPushWithExpectedError(t *testing.T, size int, repoPath, prefix string) string { + name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", prefix) + assert.NoError(t, err) + _, err = git.NewCommand("push", "origin", "master").RunInDir(repoPath) //Push + assert.Error(t, err) + return name +} + func generateCommitWithNewData(size int, repoPath, email, fullName, prefix string) (string, error) { //Generate random file bufSize := 4 * 1024 From 7c31cfcbe3fb92539a2f4697e0bbc3ffb0be1681 Mon Sep 17 00:00:00 2001 From: Antoine GIRARD Date: Sat, 7 Nov 2020 23:30:54 +0100 Subject: [PATCH 4/4] add some tests --- integrations/git_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integrations/git_test.go b/integrations/git_test.go index 8fc9eadf9dea2..25cdc70096ed8 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -76,7 +76,7 @@ func testGit(t *testing.T, u *url.URL) { t.Run("Over", func(t *testing.T) { PrintCurrentTest(t) doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, littleSize) - //TODO fix this doCommitAndPushWithExpectedError(t, bigSize, dstPath, "data-file-") + doCommitAndPushWithExpectedError(t, bigSize, dstPath, "data-file-") }) t.Run("UnderAfterResize", func(t *testing.T) { PrintCurrentTest(t) @@ -85,8 +85,7 @@ func testGit(t *testing.T, u *url.URL) { }) t.Run("Deletion", func(t *testing.T) { PrintCurrentTest(t) - //TODO delete a file - //doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") + //TODO doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") }) //TODO delete branch //TODO delete tag