Refactor git storage and command

cmd/serv.go

@@ -10,8 +10,6 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"os/exec"
-	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
@@ -21,11 +19,11 @@ import (
 	git_model "code.gitea.io/gitea/models/git"
 	"code.gitea.io/gitea/models/perm"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/git/storage"
 	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/pprof"
 	"code.gitea.io/gitea/modules/private"
-	"code.gitea.io/gitea/modules/process"
 	repo_module "code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/services/lfs"
@@ -70,7 +68,7 @@ func setup(logPath string, debug bool) {
 
 	// Check if setting.RepoRootPath exists. It could be the case that it doesn't exist, this can happen when
 	// `[repository]` `ROOT` is a relative path and $GITEA_WORK_DIR isn't passed to the SSH connection.
-	if _, err := os.Stat(setting.RepoRootPath); err != nil {
+	if err := storage.CheckStats(); err != nil {
 		if os.IsNotExist(err) {
 			_ = fail("Incorrect configuration, no repository directory.", "Directory `[repository].ROOT` %q was not found, please check if $GITEA_WORK_DIR is passed to the SSH connection or make `[repository].ROOT` an absolute value.", setting.RepoRootPath)
 		} else {
@@ -291,47 +289,19 @@ func runServ(c *cli.Context) error {
 		return nil
 	}
 
-	var gitcmd *exec.Cmd
-	gitBinPath := filepath.Dir(git.GitExecutable) // e.g. /usr/bin
-	gitBinVerb := filepath.Join(gitBinPath, verb) // e.g. /usr/bin/git-upload-pack
-	if _, err := os.Stat(gitBinVerb); err != nil {
-		// if the command "git-upload-pack" doesn't exist, try to split "git-upload-pack" to use the sub-command with git
-		// ps: Windows only has "git.exe" in the bin path, so Windows always uses this way
-		verbFields := strings.SplitN(verb, "-", 2)
-		if len(verbFields) == 2 {
-			// use git binary with the sub-command part: "C:\...\bin\git.exe", "upload-pack", ...
-			gitcmd = exec.CommandContext(ctx, git.GitExecutable, verbFields[1], repoPath)
-		}
-	}
-	if gitcmd == nil {
-		// by default, use the verb (it has been checked above by allowedCommands)
-		gitcmd = exec.CommandContext(ctx, gitBinVerb, repoPath)
-	}
-
-	process.SetSysProcAttribute(gitcmd)
-	gitcmd.Dir = setting.RepoRootPath
-	gitcmd.Stdout = os.Stdout
-	gitcmd.Stdin = os.Stdin
-	gitcmd.Stderr = os.Stderr
-	gitcmd.Env = append(gitcmd.Env, os.Environ()...)
-	gitcmd.Env = append(gitcmd.Env,
-		repo_module.EnvRepoIsWiki+"="+strconv.FormatBool(results.IsWiki),
-		repo_module.EnvRepoName+"="+results.RepoName,
-		repo_module.EnvRepoUsername+"="+results.OwnerName,
-		repo_module.EnvPusherName+"="+results.UserName,
-		repo_module.EnvPusherEmail+"="+results.UserEmail,
-		repo_module.EnvPusherID+"="+strconv.FormatInt(results.UserID, 10),
-		repo_module.EnvRepoID+"="+strconv.FormatInt(results.RepoID, 10),
-		repo_module.EnvPRID+"="+fmt.Sprintf("%d", 0),
-		repo_module.EnvDeployKeyID+"="+fmt.Sprintf("%d", results.DeployKeyID),
-		repo_module.EnvKeyID+"="+fmt.Sprintf("%d", results.KeyID),
-		repo_module.EnvAppURL+"="+setting.AppURL,
-	)
-	// to avoid breaking, here only use the minimal environment variables for the "gitea serv" command.
-	// it could be re-considered whether to use the same git.CommonGitCmdEnvs() as "git" command later.
-	gitcmd.Env = append(gitcmd.Env, git.CommonCmdServEnvs()...)
-
-	if err = gitcmd.Run(); err != nil {
+	if err := git.RunServCommand(ctx, verb, repoPath, []string{
+		repo_module.EnvRepoIsWiki + "=" + strconv.FormatBool(results.IsWiki),
+		repo_module.EnvRepoName + "=" + results.RepoName,
+		repo_module.EnvRepoUsername + "=" + results.OwnerName,
+		repo_module.EnvPusherName + "=" + results.UserName,
+		repo_module.EnvPusherEmail + "=" + results.UserEmail,
+		repo_module.EnvPusherID + "=" + strconv.FormatInt(results.UserID, 10),
+		repo_module.EnvRepoID + "=" + strconv.FormatInt(results.RepoID, 10),
+		repo_module.EnvPRID + "=" + fmt.Sprintf("%d", 0),
+		repo_module.EnvDeployKeyID + "=" + fmt.Sprintf("%d", results.DeployKeyID),
+		repo_module.EnvKeyID + "=" + fmt.Sprintf("%d", results.KeyID),
+		repo_module.EnvAppURL + "=" + setting.AppURL,
+	}); err != nil {
 		return fail("Internal error", "Failed to execute git command: %v", err)
 	}
 

models/migrations/base/tests.go

@@ -16,6 +16,7 @@ import (
 	"code.gitea.io/gitea/models/unittest"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/git/storage"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 
@@ -32,25 +33,22 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En
 	ourSkip := 2
 	ourSkip += skip
 	deferFn := PrintCurrentTest(t, ourSkip)
-	assert.NoError(t, os.RemoveAll(setting.RepoRootPath))
-	assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath))
-	ownerDirs, err := os.ReadDir(setting.RepoRootPath)
+	assert.NoError(t, storage.RemoveAll(""))
+	assert.NoError(t, storage.CopyDir(filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), ""))
+	ownerDirs, err := storage.ReadDir("")
 	if err != nil {
 		assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
 	}
 	for _, ownerDir := range ownerDirs {
 		if !ownerDir.Type().IsDir() {
 			continue
 		}
-		repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
+		repoDirs, err := storage.ReadDir(ownerDir.Name())
 		if err != nil {
 			assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
 		}
 		for _, repoDir := range repoDirs {
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755)
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755)
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755)
-			_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755)
+			storage.MakeRepoDir(path.Join(ownerDir.Name(), repoDir.Name()))
 		}
 	}
 
@@ -159,7 +157,7 @@ func MainTest(m *testing.M) {
 
 	exitStatus := m.Run()
 
-	if err := removeAllWithRetry(setting.RepoRootPath); err != nil {
+	if err := storage.RemoveAll(""); err != nil {
 		fmt.Fprintf(os.Stderr, "os.RemoveAll: %v\n", err)
 	}
 	if err := removeAllWithRetry(tmpDataPath); err != nil {

models/migrations/v1_12/v128.go

@@ -6,7 +6,7 @@ package v1_12 //nolint
 import (
 	"fmt"
 	"math"
-	"path/filepath"
+	"path"
 	"strings"
 	"time"
 
@@ -75,24 +75,24 @@ func FixMergeBase(x *xorm.Engine) error {
 				log.Error("Missing base repo with id %d for PR ID %d", pr.BaseRepoID, pr.ID)
 				continue
 			}
-			userPath := filepath.Join(setting.RepoRootPath, strings.ToLower(baseRepo.OwnerName))
-			repoPath := filepath.Join(userPath, strings.ToLower(baseRepo.Name)+".git")
+
+			repoRelPath := path.Join(strings.ToLower(baseRepo.OwnerName), strings.ToLower(baseRepo.Name)+".git")
 
 			gitRefName := fmt.Sprintf("refs/pull/%d/head", pr.Index)
 
 			if !pr.HasMerged {
 				var err error
-				pr.MergeBase, _, err = git.NewCommand(git.DefaultContext, "merge-base").AddDashesAndList(pr.BaseBranch, gitRefName).RunStdString(&git.RunOpts{Dir: repoPath})
+				pr.MergeBase, _, err = git.NewCommand(git.DefaultContext, "merge-base").AddDashesAndList(pr.BaseBranch, gitRefName).RunStdString(&git.RunOpts{Dir: repoRelPath})
 				if err != nil {
 					var err2 error
-					pr.MergeBase, _, err2 = git.NewCommand(git.DefaultContext, "rev-parse").AddDynamicArguments(git.BranchPrefix + pr.BaseBranch).RunStdString(&git.RunOpts{Dir: repoPath})
+					pr.MergeBase, _, err2 = git.NewCommand(git.DefaultContext, "rev-parse").AddDynamicArguments(git.BranchPrefix + pr.BaseBranch).RunStdString(&git.RunOpts{Dir: repoRelPath})
 					if err2 != nil {
 						log.Error("Unable to get merge base for PR ID %d, Index %d in %s/%s. Error: %v & %v", pr.ID, pr.Index, baseRepo.OwnerName, baseRepo.Name, err, err2)
 						continue
 					}
 				}
 			} else {
-				parentsString, _, err := git.NewCommand(git.DefaultContext, "rev-list", "--parents", "-n", "1").AddDynamicArguments(pr.MergedCommitID).RunStdString(&git.RunOpts{Dir: repoPath})
+				parentsString, _, err := git.NewCommand(git.DefaultContext, "rev-list", "--parents", "-n", "1").AddDynamicArguments(pr.MergedCommitID).RunStdString(&git.RunOpts{Dir: repoRelPath})
 				if err != nil {
 					log.Error("Unable to get parents for merged PR ID %d, Index %d in %s/%s. Error: %v", pr.ID, pr.Index, baseRepo.OwnerName, baseRepo.Name, err)
 					continue
@@ -106,7 +106,7 @@ func FixMergeBase(x *xorm.Engine) error {
 				refs = append(refs, gitRefName)
 				cmd := git.NewCommand(git.DefaultContext, "merge-base").AddDashesAndList(refs...)
 
-				pr.MergeBase, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoPath})
+				pr.MergeBase, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoRelPath})
 				if err != nil {
 					log.Error("Unable to get merge base for merged PR ID %d, Index %d in %s/%s. Error: %v", pr.ID, pr.Index, baseRepo.OwnerName, baseRepo.Name, err)
 					continue

models/migrations/v1_12/v134.go

@@ -6,7 +6,7 @@ package v1_12 //nolint
 import (
 	"fmt"
 	"math"
-	"path/filepath"
+	"path"
 	"strings"
 	"time"
 
@@ -74,12 +74,12 @@ func RefixMergeBase(x *xorm.Engine) error {
 				log.Error("Missing base repo with id %d for PR ID %d", pr.BaseRepoID, pr.ID)
 				continue
 			}
-			userPath := filepath.Join(setting.RepoRootPath, strings.ToLower(baseRepo.OwnerName))
-			repoPath := filepath.Join(userPath, strings.ToLower(baseRepo.Name)+".git")
+
+			repoRelPath := path.Join(strings.ToLower(baseRepo.OwnerName), strings.ToLower(baseRepo.Name)+".git")
 
 			gitRefName := fmt.Sprintf("refs/pull/%d/head", pr.Index)
 
-			parentsString, _, err := git.NewCommand(git.DefaultContext, "rev-list", "--parents", "-n", "1").AddDynamicArguments(pr.MergedCommitID).RunStdString(&git.RunOpts{Dir: repoPath})
+			parentsString, _, err := git.NewCommand(git.DefaultContext, "rev-list", "--parents", "-n", "1").AddDynamicArguments(pr.MergedCommitID).RunStdString(&git.RunOpts{Dir: repoRelPath})
 			if err != nil {
 				log.Error("Unable to get parents for merged PR ID %d, Index %d in %s/%s. Error: %v", pr.ID, pr.Index, baseRepo.OwnerName, baseRepo.Name, err)
 				continue
@@ -94,7 +94,7 @@ func RefixMergeBase(x *xorm.Engine) error {
 			refs = append(refs, gitRefName)
 			cmd := git.NewCommand(git.DefaultContext, "merge-base").AddDashesAndList(refs...)
 
-			pr.MergeBase, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoPath})
+			pr.MergeBase, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoRelPath})
 			if err != nil {
 				log.Error("Unable to get merge base for merged PR ID %d, Index %d in %s/%s. Error: %v", pr.ID, pr.Index, baseRepo.OwnerName, baseRepo.Name, err)
 				continue

models/migrations/v1_12/v136.go

@@ -6,7 +6,7 @@ package v1_12 //nolint
 import (
 	"fmt"
 	"math"
-	"path/filepath"
+	"path"
 	"strings"
 	"time"
 
@@ -85,12 +85,12 @@ func AddCommitDivergenceToPulls(x *xorm.Engine) error {
 				log.Error("Missing base repo with id %d for PR ID %d", pr.BaseRepoID, pr.ID)
 				continue
 			}
-			userPath := filepath.Join(setting.RepoRootPath, strings.ToLower(baseRepo.OwnerName))
-			repoPath := filepath.Join(userPath, strings.ToLower(baseRepo.Name)+".git")
+
+			repoRelPath := path.Join(strings.ToLower(baseRepo.OwnerName), strings.ToLower(baseRepo.Name)+".git")
 
 			gitRefName := fmt.Sprintf("refs/pull/%d/head", pr.Index)
 
-			divergence, err := git.GetDivergingCommits(graceful.GetManager().HammerContext(), repoPath, pr.BaseBranch, gitRefName)
+			divergence, err := git.GetDivergingCommits(graceful.GetManager().HammerContext(), repoRelPath, pr.BaseBranch, gitRefName)
 			if err != nil {
 				log.Warn("Could not recalculate Divergence for pull: %d", pr.ID)
 				pr.CommitsAhead = 0

models/repo/repo.go

@@ -9,13 +9,13 @@ import (
 	"html/template"
 	"net"
 	"net/url"
-	"path/filepath"
 	"strconv"
 	"strings"
 
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/models/unit"
 	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/git/storage"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/setting"
@@ -470,14 +470,9 @@ func (repo *Repository) IsGenerated() bool {
 	return repo.TemplateID != 0
 }
 
-// RepoPath returns repository path by given user and repository name.
-func RepoPath(userName, repoName string) string { //revive:disable-line:exported
-	return filepath.Join(user_model.UserPath(userName), strings.ToLower(repoName)+".git")
-}
-
 // RepoPath returns the repository path
 func (repo *Repository) RepoPath() string {
-	return RepoPath(repo.OwnerName, repo.Name)
+	return storage.RepoPath(repo.OwnerName, repo.Name)
 }
 
 // Link returns the repository relative url
@@ -685,7 +680,7 @@ func IsRepositoryExist(ctx context.Context, u *user_model.User, repoName string)
 	if err != nil {
 		return false, err
 	}
-	isDir, err := util.IsDir(RepoPath(u.Name, repoName))
+	isDir, err := util.IsDir(storage.RepoPath(u.Name, repoName))
 	return has && isDir, err
 }
 

models/repo/update.go

@@ -11,6 +11,7 @@ import (
 
 	"code.gitea.io/gitea/models/db"
 	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/git/storage"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/util"
 )
@@ -123,10 +124,10 @@ func CheckCreateRepository(doer, u *user_model.User, name string, overwriteOrAdo
 		return ErrRepoAlreadyExist{u.Name, name}
 	}
 
-	repoPath := RepoPath(u.Name, name)
-	isExist, err := util.IsExist(repoPath)
+	repoRelPath := storage.RepoRelPath(u.Name, name)
+	isExist, err := storage.IsExist(repoRelPath)
 	if err != nil {
-		log.Error("Unable to check if %s exists. Error: %v", repoPath, err)
+		log.Error("Unable to check if %s exists. Error: %v", repoRelPath, err)
 		return err
 	}
 	if !overwriteOrAdopt && isExist {
@@ -154,19 +155,19 @@ func ChangeRepositoryName(doer *user_model.User, repo *Repository, newRepoName s
 		return ErrRepoAlreadyExist{repo.Owner.Name, newRepoName}
 	}
 
-	newRepoPath := RepoPath(repo.Owner.Name, newRepoName)
+	newRepoPath := storage.RepoPath(repo.Owner.Name, newRepoName)
 	if err = util.Rename(repo.RepoPath(), newRepoPath); err != nil {
 		return fmt.Errorf("rename repository directory: %w", err)
 	}
 
-	wikiPath := repo.WikiPath()
-	isExist, err := util.IsExist(wikiPath)
+	wikiRelPath := storage.WikiRelPath(repo.OwnerName, repo.Name)
+	isExist, err := storage.IsExist(wikiRelPath)
 	if err != nil {
-		log.Error("Unable to check if %s exists. Error: %v", wikiPath, err)
+		log.Error("Unable to check if %s exists. Error: %v", wikiRelPath, err)
 		return err
 	}
 	if isExist {
-		if err = util.Rename(wikiPath, WikiPath(repo.Owner.Name, newRepoName)); err != nil {
+		if err = storage.Rename(wikiRelPath, storage.WikiRelPath(repo.Owner.Name, newRepoName)); err != nil {
 			return fmt.Errorf("rename repository wiki: %w", err)
 		}
 	}

models/repo/wiki.go

@@ -6,10 +6,8 @@ package repo
 
 import (
 	"fmt"
-	"path/filepath"
-	"strings"
 
-	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/git/storage"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/util"
 )
@@ -76,14 +74,9 @@ func (repo *Repository) WikiCloneLink() *CloneLink {
 	return repo.cloneLink(true)
 }
 
-// WikiPath returns wiki data path by given user and repository name.
-func WikiPath(userName, repoName string) string {
-	return filepath.Join(user_model.UserPath(userName), strings.ToLower(repoName)+".wiki.git")
-}
-
 // WikiPath returns wiki data path for given repository.
 func (repo *Repository) WikiPath() string {
-	return WikiPath(repo.OwnerName, repo.Name)
+	return storage.WikiPath(repo.OwnerName, repo.Name)
 }
 
 // HasWiki returns true if repository has wiki.

models/repo/wiki_test.go

@@ -9,6 +9,7 @@ import (
 
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/models/unittest"
+	"code.gitea.io/gitea/modules/git/storage"
 	"code.gitea.io/gitea/modules/setting"
 
 	"github.com/stretchr/testify/assert"
@@ -25,8 +26,7 @@ func TestRepository_WikiCloneLink(t *testing.T) {
 
 func TestWikiPath(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
-	expected := filepath.Join(setting.RepoRootPath, "user2/repo1.wiki.git")
-	assert.Equal(t, expected, repo_model.WikiPath("user2", "repo1"))
+	assert.Equal(t, "user2/repo1.wiki.git", storage.WikiRelPath("user2", "repo1"))
 }
 
 func TestRepository_WikiPath(t *testing.T) {