Limit repository size

integrations/api_helper_for_declarative_test.go

@@ -371,3 +371,17 @@ func doAPIAddRepoToOrganizationTeam(ctx APITestContext, teamID int64, orgName, r
 		ctx.Session.MakeRequest(t, req, http.StatusNoContent)
 	}
 }
+
+func doAPISetRepoSizeLimit(ctx APITestContext, owner, repo string, size int64) func(*testing.T) {
+	return func(t *testing.T) {
+		urlStr := fmt.Sprintf("/api/v1/repos/%s/%s?token=%s",
+			owner, repo, ctx.Token)
+		req := NewRequestWithJSON(t, http.MethodPatch, urlStr, &api.EditRepoOption{SizeLimit: &size})
+
+		if ctx.ExpectedCode != 0 {
+			ctx.Session.MakeRequest(t, req, ctx.ExpectedCode)
+			return
+		}
+		ctx.Session.MakeRequest(t, req, 200)
+	}
+}

integrations/git_test.go

@@ -68,6 +68,33 @@ func testGit(t *testing.T, u *url.URL) {
 		rawTest(t, &httpContext, little, big, littleLFS, bigLFS)
 		mediaTest(t, &httpContext, little, big, littleLFS, bigLFS)
 
+		t.Run("SizeLimit", func(t *testing.T) {
+			t.Run("Under", func(t *testing.T) {
+				PrintCurrentTest(t)
+				doCommitAndPush(t, littleSize, dstPath, "data-file-")
+			})
+			t.Run("Over", func(t *testing.T) {
+				PrintCurrentTest(t)
+				doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, littleSize)
+				doCommitAndPushWithExpectedError(t, bigSize, dstPath, "data-file-")
+			})
+			t.Run("UnderAfterResize", func(t *testing.T) {
+				PrintCurrentTest(t)
+				doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, bigSize*10)
+				doCommitAndPush(t, littleSize, dstPath, "data-file-")
+			})
+			t.Run("Deletion", func(t *testing.T) {
+				PrintCurrentTest(t)
+				//TODO doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-")
+			})
+			//TODO delete branch
+			//TODO delete tag
+			//TODO add big commit that will be over with the push
+			//TODO add lfs
+			//TODO remove lfs
+			//TODO add missing case
+		})
+
 		t.Run("BranchProtectMerge", doBranchProtectPRMerge(&httpContext, dstPath))
 		t.Run("MergeFork", func(t *testing.T) {
 			defer PrintCurrentTest(t)()
@@ -291,6 +318,14 @@ func doCommitAndPush(t *testing.T, size int, repoPath, prefix string) string {
 	return name
 }
 
+func doCommitAndPushWithExpectedError(t *testing.T, size int, repoPath, prefix string) string {
+	name, err := generateCommitWithNewData(size, repoPath, "[email protected]", "User Two", prefix)
+	assert.NoError(t, err)
+	_, err = git.NewCommand("push", "origin", "master").RunInDir(repoPath) //Push
+	assert.Error(t, err)
+	return name
+}
+
 func generateCommitWithNewData(size int, repoPath, email, fullName, prefix string) (string, error) {
 	//Generate random file
 	bufSize := 4 * 1024

models/migrations/migrations.go

@@ -254,6 +254,8 @@ var migrations = []Migration{
 	NewMigration("code comment replies should have the commitID of the review they are replying to", updateCodeCommentReplies),
 	// v159 -> v160
 	NewMigration("update reactions constraint", updateReactionConstraint),
+	// v160 -> v161
+	NewMigration("add size limit on repository", addSizeLimitOnRepo),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v160.go

@@ -0,0 +1,16 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import "xorm.io/xorm"
+
+func addSizeLimitOnRepo(x *xorm.Engine) error {
+	type Repository struct {
+		ID        int64 `xorm:"pk autoincr"`
+		SizeLimit int64 `xorm:"NOT NULL DEFAULT 0"`
+	}
+
+	return x.Sync2(new(Repository))
+}

models/repo.go

@@ -230,6 +230,7 @@ type Repository struct {
 	TemplateID                      int64              `xorm:"INDEX"`
 	TemplateRepo                    *Repository        `xorm:"-"`
 	Size                            int64              `xorm:"NOT NULL DEFAULT 0"`
+	SizeLimit                       int64              `xorm:"NOT NULL DEFAULT 0"`
 	CodeIndexerStatus               *RepoIndexerStatus `xorm:"-"`
 	StatsIndexerStatus              *RepoIndexerStatus `xorm:"-"`
 	IsFsckEnabled                   bool               `xorm:"NOT NULL DEFAULT true"`
@@ -860,20 +861,29 @@ func (repo *Repository) IsOwnedBy(userID int64) bool {
 	return repo.OwnerID == userID
 }
 
-func (repo *Repository) updateSize(e Engine) error {
+func (repo *Repository) computeSize() (int64, error) {
 	size, err := util.GetDirectorySize(repo.RepoPath())
 	if err != nil {
-		return fmt.Errorf("updateSize: %v", err)
+		return 0, fmt.Errorf("computeSize: %v", err)
 	}
 
 	objs, err := repo.GetLFSMetaObjects(-1, 0)
 	if err != nil {
-		return fmt.Errorf("updateSize: GetLFSMetaObjects: %v", err)
+		return 0, fmt.Errorf("computeSize: GetLFSMetaObjects: %v", err)
 	}
 	for _, obj := range objs {
 		size += obj.Size
 	}
 
+	return size, nil
+}
+
+func (repo *Repository) updateSize(e Engine) error {
+	size, err := repo.computeSize()
+	if err != nil {
+		return fmt.Errorf("updateSize: %v", err)
+	}
+
 	repo.Size = size
 	_, err = e.ID(repo.ID).Cols("size").Update(repo)
 	return err
@@ -884,6 +894,11 @@ func (repo *Repository) UpdateSize(ctx DBContext) error {
 	return repo.updateSize(ctx.e)
 }
 
+// RepoSizeIsOversized return if is over size limitation
+func (repo *Repository) RepoSizeIsOversized(additionalSize int64) bool {
+	return repo.SizeLimit > 0 && repo.Size+additionalSize > repo.SizeLimit
+}
+
 // CanUserFork returns true if specified user can fork repository.
 func (repo *Repository) CanUserFork(user *User) (bool, error) {
 	if user == nil {
@@ -1101,6 +1116,7 @@ type CreateRepoOptions struct {
 	AutoInit       bool
 	Status         RepositoryStatus
 	TrustModel     TrustModelType
+	SizeLimit      int64
 }
 
 // GetRepoInitFile returns repository init files

modules/auth/repo_form.go

@@ -48,6 +48,7 @@ type CreateRepoForm struct {
 	Avatar       bool
 	Labels       bool
 	TrustModel   string
+	SizeLimit    int64
 }
 
 // Validate validates the fields
@@ -126,6 +127,7 @@ type RepoSettingForm struct {
 	Private        bool
 	Template       bool
 	EnablePrune    bool
+	RepoSizeLimit  int64
 
 	// Advanced settings
 	EnableWiki                       bool

modules/git/repo.go

@@ -352,8 +352,13 @@ const (
 
 // CountObjects returns the results of git count-objects on the repoPath
 func CountObjects(repoPath string) (*CountObject, error) {
+	return CountObjectsWithEnv(repoPath, nil)
+}
+
+// CountObjectsWithEnv returns the results of git count-objects on the repoPath with custom env setup
+func CountObjectsWithEnv(repoPath string, env []string) (*CountObject, error) {
 	cmd := NewCommand("count-objects", "-v")
-	stdout, err := cmd.RunInDir(repoPath)
+	stdout, err := cmd.RunInDirWithEnv(repoPath, env)
 	if err != nil {
 		return nil, err
 	}

modules/repository/create.go

@@ -45,6 +45,7 @@ func CreateRepository(doer, u *models.User, opts models.CreateRepoOptions) (*mod
 		Status:                          opts.Status,
 		IsEmpty:                         !opts.AutoInit,
 		TrustModel:                      opts.TrustModel,
+		SizeLimit:                       opts.SizeLimit,
 	}
 
 	if err := models.WithTx(func(ctx models.DBContext) error {

modules/structs/repo.go

@@ -122,6 +122,8 @@ type CreateRepoOption struct {
 	// TrustModel of the repository
 	// enum: default,collaborator,committer,collaboratorcommitter
 	TrustModel string `json:"trust_model"`
+	// SizeLimit of the repository
+	SizeLimit int64 `json:"size_limit"`
 }
 
 // EditRepoOption options when editing a repository's properties
@@ -168,6 +170,8 @@ type EditRepoOption struct {
 	AllowSquash *bool `json:"allow_squash_merge,omitempty"`
 	// set to `true` to archive this repository.
 	Archived *bool `json:"archived,omitempty"`
+	// SizeLimit of the repository.
+	SizeLimit *int64 `json:"size_limit,omitempty"`
 }
 
 // CreateBranchRepoOption options when creating a branch in a repository

options/locale/locale_en-US.ini

@@ -653,6 +653,7 @@ owner = Owner
 repo_name = Repository Name
 repo_name_helper = Good repository names use short, memorable and unique keywords.
 repo_size = Repository Size
+repo_size_limit = Repository Size Limit
 template = Template
 template_select = Select a template.
 template_helper = Make repository a template
@@ -733,6 +734,8 @@ archive.pull.nocomment = This repo is archived. You cannot comment on pull reque
 form.reach_limit_of_creation = You have already reached your limit of %d repositories.
 form.name_reserved = The repository name '%s' is reserved.
 form.name_pattern_not_allowed = The pattern '%s' is not allowed in a repository name.
+form.repo_size_limit_negative = Repository size limitation cannot be negative.
+form.repo_size_limit_only_by_admins = Only administrators can change the repository size limitation.
 
 need_auth = Clone Authorization
 migrate_options = Migration Options

routers/api/v1/repo/repo.go

@@ -246,6 +246,7 @@ func CreateUserRepo(ctx *context.APIContext, owner *models.User, opt api.CreateR
 		DefaultBranch: opt.DefaultBranch,
 		TrustModel:    models.ToTrustModel(opt.TrustModel),
 		IsTemplate:    opt.Template,
+		SizeLimit:     opt.SizeLimit,
 	})
 	if err != nil {
 		if models.IsErrRepoAlreadyExist(err) {
@@ -568,6 +569,9 @@ func updateBasicProperties(ctx *context.APIContext, opts api.EditRepoOption) err
 		repo.DefaultBranch = *opts.DefaultBranch
 	}
 
+	if opts.SizeLimit != nil {
+		repo.SizeLimit = *opts.SizeLimit
+	}
 	if err := models.UpdateRepository(repo, visibilityChanged); err != nil {
 		ctx.Error(http.StatusInternalServerError, "UpdateRepository", err)
 		return err

routers/private/hook.go

@@ -155,12 +155,33 @@ func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) {
 			private.GitQuarantinePath+"="+opts.GitQuarantinePath)
 	}
 
+	pushSize, err := git.CountObjectsWithEnv(repo.RepoPath(), env)
+	if err != nil {
+		log.Error("Unable to get repository size with env %v: %s Error: %v", repo.RepoPath(), env, err)
+		ctx.JSON(http.StatusInternalServerError, map[string]interface{}{
+			"err": err.Error(),
+		})
+		return
+	}
+	log.Trace("Push size %d", pushSize.Size)
+
 	// Iterate across the provided old commit IDs
 	for i := range opts.OldCommitIDs {
 		oldCommitID := opts.OldCommitIDs[i]
 		newCommitID := opts.NewCommitIDs[i]
 		refFullName := opts.RefFullNames[i]
 
+		//Check size
+		if newCommitID != git.EmptySHA && repo.RepoSizeIsOversized(pushSize.Size) { //Check next size if we are not deleting a reference
+			log.Warn("Forbidden: new repo size is over limitation: %d", repo.SizeLimit)
+			ctx.JSON(http.StatusForbidden, map[string]interface{}{
+				"err": fmt.Sprintf("new repo size is over limitation: %d", repo.SizeLimit),
+			})
+		}
+		//TODO investigate why on force push some git objects are not cleaned on server side.
+		//TODO corner-case force push and branch creation -> git.EmptySHA == oldCommitID
+		//TODO calculate pushed LFS objects size
+
 		branchName := strings.TrimPrefix(refFullName, git.BranchPrefix)
 		if branchName == repo.DefaultBranch && newCommitID == git.EmptySHA {
 			log.Warn("Forbidden: Branch: %s is the default branch in %-v and cannot be deleted", branchName, repo)

routers/repo/repo.go

@@ -250,6 +250,7 @@ func CreatePost(ctx *context.Context, form auth.CreateRepoForm) {
 			AutoInit:      form.AutoInit,
 			IsTemplate:    form.Template,
 			TrustModel:    models.ToTrustModel(form.TrustModel),
+			SizeLimit:     form.SizeLimit,
 		})
 		if err == nil {
 			log.Trace("Repository created [%d]: %s/%s", repo.ID, ctxUser.Name, repo.Name)

routers/repo/setting.go

@@ -50,6 +50,7 @@ func Settings(ctx *context.Context) {
 	ctx.Data["Title"] = ctx.Tr("repo.settings")
 	ctx.Data["PageIsSettingsOptions"] = true
 	ctx.Data["ForcePrivate"] = setting.Repository.ForcePrivate
+	ctx.Data["Err_RepoSize"] = ctx.Repo.Repository.RepoSizeIsOversized(ctx.Repo.Repository.SizeLimit / 10) // less than 10% left
 
 	signing, _ := models.SigningKey(ctx.Repo.Repository.RepoPath())
 	ctx.Data["SigningKeyAvailable"] = len(signing) > 0
@@ -64,6 +65,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) {
 	ctx.Data["PageIsSettingsOptions"] = true
 
 	repo := ctx.Repo.Repository
+	ctx.Data["Err_RepoSize"] = repo.RepoSizeIsOversized(repo.SizeLimit / 10) // less than 10% left
 
 	switch ctx.Query("action") {
 	case "update":
@@ -128,6 +130,19 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) {
 			return
 		}
 
+		if form.RepoSizeLimit < 0 {
+			ctx.Data["Err_RepoSizeLimit"] = true
+			ctx.RenderWithErr(ctx.Tr("repo.form.repo_size_limit_negative"), tplSettingsOptions, &form)
+			return
+		}
+
+		if !ctx.User.IsAdmin && repo.SizeLimit != form.RepoSizeLimit {
+			ctx.Data["Err_RepoSizeLimit"] = true
+			ctx.RenderWithErr(ctx.Tr("repo.form.repo_size_limit_only_by_admins"), tplSettingsOptions, &form)
+			return
+		}
+		repo.SizeLimit = form.RepoSizeLimit
+
 		repo.IsPrivate = form.Private
 		if err := models.UpdateRepository(repo, visibilityChanged); err != nil {
 			ctx.ServerError("UpdateRepository", err)

templates/repo/settings/options.tmpl

@@ -17,7 +17,11 @@
 				</div>
 				<div class="inline field">
 					<label>{{.i18n.Tr "repo.repo_size"}}</label>
-					<span>{{SizeFmt .Repository.Size}}</span>
+					<span {{if .Err_RepoSize}}class="ui text red"{{end}}>{{SizeFmt .Repository.Size}}{{if .Repository.SizeLimit}}/{{SizeFmt .Repository.SizeLimit}}{{end}}</span>
+				</div>
+				<div class="field {{if .Err_RepoSizeLimit}}error{{end}}" {{if not .IsAdmin}}style="display:none;"{{end}}>
+					<label for="repo_size_limit">{{.i18n.Tr "repo.repo_size_limit"}}</label>
+					<input id="repo_size_limit" name="repo_size_limit" type="number" value="{{.Repository.SizeLimit}}" data-repo-size-limit="{{.Repository.SizeLimit}}">
 				</div>
 				<div class="inline field">
 					<label>{{.i18n.Tr "repo.template"}}</label>

templates/swagger/v1_json.tmpl

@@ -12271,6 +12271,12 @@
           "type": "string",
           "x-go-name": "Readme"
         },
+        "size_limit": {
+          "description": "SizeLimit of the repository",
+          "type": "integer",
+          "format": "int64",
+          "x-go-name": "SizeLimit"
+        },
         "template": {
           "description": "Whether the repository is template",
           "type": "boolean",
@@ -13029,6 +13035,12 @@
           "type": "boolean",
           "x-go-name": "Private"
         },
+        "size_limit": {
+          "description": "SizeLimit of the repository.",
+          "type": "integer",
+          "format": "int64",
+          "x-go-name": "SizeLimit"
+        },
         "template": {
           "description": "either `true` to make this repository a template or `false` to make it a normal repository",
           "type": "boolean",