From ce91449b2bbfc289d94b3d36d43cbd3640461795 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 4 Jan 2023 15:26:16 +0800 Subject: [PATCH 1/7] add command runner --- modules/git/command.go | 143 ++++++++++++---------------------- modules/git/command_runner.go | 123 +++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+), 95 deletions(-) create mode 100644 modules/git/command_runner.go diff --git a/modules/git/command.go b/modules/git/command.go index d88fcd1a8c807..2a83eee4a6f52 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -17,7 +17,6 @@ import ( "unsafe" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/util" ) @@ -34,21 +33,18 @@ const DefaultLocale = "C" // Command represents a command with its subcommands or arguments. type Command struct { - name string args []string parentContext context.Context desc string globalArgsLength int brokenArgs []string + commandRunner CommandRunner } type CmdArg string func (c *Command) String() string { - if len(c.args) == 0 { - return c.name - } - return fmt.Sprintf("%s %s", c.name, strings.Join(c.args, " ")) + return fmt.Sprintf("%s %s", c.commandRunner.String(), strings.Join(c.args, " ")) } // NewCommand creates and returns a new Git Command based on given command and arguments. @@ -63,10 +59,9 @@ func NewCommand(ctx context.Context, args ...CmdArg) *Command { cargs = append(cargs, string(arg)) } return &Command{ - name: GitExecutable, - args: cargs, parentContext: ctx, globalArgsLength: len(globalCommandArgs), + commandRunner: newLocalCommandRunner(ctx), } } @@ -84,9 +79,9 @@ func NewCommandContextNoGlobals(ctx context.Context, args ...CmdArg) *Command { cargs = append(cargs, string(arg)) } return &Command{ - name: GitExecutable, args: cargs, parentContext: ctx, + commandRunner: newLocalCommandRunner(ctx), } } @@ -192,91 +187,6 @@ func CommonCmdServEnvs() []string { var ErrBrokenCommand = errors.New("git command is broken") -// Run runs the command with the RunOpts -func (c *Command) Run(opts *RunOpts) error { - if len(c.brokenArgs) != 0 { - log.Error("git command is broken: %s, broken args: %s", c.String(), strings.Join(c.brokenArgs, " ")) - return ErrBrokenCommand - } - if opts == nil { - opts = &RunOpts{} - } - - // We must not change the provided options - timeout := opts.Timeout - if timeout <= 0 { - timeout = defaultCommandExecutionTimeout - } - - if len(opts.Dir) == 0 { - log.Debug("%s", c) - } else { - log.Debug("%s: %v", opts.Dir, c) - } - - desc := c.desc - if desc == "" { - args := c.args[c.globalArgsLength:] - var argSensitiveURLIndexes []int - for i, arg := range c.args { - if strings.Contains(arg, "://") && strings.Contains(arg, "@") { - argSensitiveURLIndexes = append(argSensitiveURLIndexes, i) - } - } - if len(argSensitiveURLIndexes) > 0 { - args = make([]string, len(c.args)) - copy(args, c.args) - for _, urlArgIndex := range argSensitiveURLIndexes { - args[urlArgIndex] = util.SanitizeCredentialURLs(args[urlArgIndex]) - } - } - desc = fmt.Sprintf("%s %s [repo_path: %s]", c.name, strings.Join(args, " "), opts.Dir) - } - - var ctx context.Context - var cancel context.CancelFunc - var finished context.CancelFunc - - if opts.UseContextTimeout { - ctx, cancel, finished = process.GetManager().AddContext(c.parentContext, desc) - } else { - ctx, cancel, finished = process.GetManager().AddContextTimeout(c.parentContext, timeout, desc) - } - defer finished() - - cmd := exec.CommandContext(ctx, c.name, c.args...) - if opts.Env == nil { - cmd.Env = os.Environ() - } else { - cmd.Env = opts.Env - } - - process.SetSysProcAttribute(cmd) - cmd.Env = append(cmd.Env, CommonGitCmdEnvs()...) - cmd.Dir = opts.Dir - cmd.Stdout = opts.Stdout - cmd.Stderr = opts.Stderr - cmd.Stdin = opts.Stdin - if err := cmd.Start(); err != nil { - return err - } - - if opts.PipelineFunc != nil { - err := opts.PipelineFunc(ctx, cancel) - if err != nil { - cancel() - _ = cmd.Wait() - return err - } - } - - if err := cmd.Wait(); err != nil && ctx.Err() != context.DeadlineExceeded { - return err - } - - return ctx.Err() -} - type RunStdError interface { error Unwrap() error @@ -318,6 +228,49 @@ func bytesToString(b []byte) string { return *(*string)(unsafe.Pointer(&b)) // that's what Golang's strings.Builder.String() does (go/src/strings/builder.go) } +// Run runs the command with the RunOpts +func (c *Command) Run(opts *RunOpts) error { + if len(c.brokenArgs) != 0 { + log.Error("git command is broken: %s, broken args: %s", c.String(), strings.Join(c.brokenArgs, " ")) + return ErrBrokenCommand + } + if opts == nil { + opts = &RunOpts{} + } + + // We must not change the provided options + timeout := opts.Timeout + if timeout <= 0 { + timeout = defaultCommandExecutionTimeout + } + + if len(opts.Dir) == 0 { + log.Debug("%s", c) + } else { + log.Debug("%s: %v", opts.Dir, c) + } + + if desc == "" { + args := args[c.globalArgsLength:] + var argSensitiveURLIndexes []int + for i, arg := range args { + if strings.Contains(arg, "://") && strings.Contains(arg, "@") { + argSensitiveURLIndexes = append(argSensitiveURLIndexes, i) + } + } + if len(argSensitiveURLIndexes) > 0 { + args = make([]string, len(args)) + copy(args, args) + for _, urlArgIndex := range argSensitiveURLIndexes { + args[urlArgIndex] = util.SanitizeCredentialURLs(args[urlArgIndex]) + } + } + desc = fmt.Sprintf("%s %s [repo_path: %s]", c.name, strings.Join(args, " "), opts.Dir) + } + + return c.commandRunner.Run(c.parentContext, c.desc, c.args, opts, timeout) +} + // RunStdString runs the command with options and returns stdout/stderr as string. and store stderr to returned error (err combined with stderr). func (c *Command) RunStdString(opts *RunOpts) (stdout, stderr string, runErr RunStdError) { stdoutBytes, stderrBytes, err := c.RunStdBytes(opts) @@ -354,7 +307,7 @@ func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS PipelineFunc: opts.PipelineFunc, } - err := c.Run(newOpts) + err := c.commandRunner.Run(c.parentContext, c.desc, c.args, newOpts) stderr = stderrBuf.Bytes() if err != nil { return nil, stderr, &runStdError{err: err, stderr: bytesToString(stderr)} diff --git a/modules/git/command_runner.go b/modules/git/command_runner.go new file mode 100644 index 0000000000000..62a8a5fcc815c --- /dev/null +++ b/modules/git/command_runner.go @@ -0,0 +1,123 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package git + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/util" +) + +type CommandRunner interface { + String() string + Run(ctx context.Context, desc string, args []string, opts *RunOpts) error +} + +// localCommandRunner represents a command with its subcommands or arguments. +type localCommandRunner struct { + name string + globalArgsLength int +} + +var _ CommandRunner = &localCommandRunner{} + +func newLocalCommandRunner(ctx context.Context) *localCommandRunner { + return &localCommandRunner{ + name: GitExecutable, + globalArgsLength: len(globalCommandArgs), + } +} + +func (c *localCommandRunner) String() string { + return c.name +} + +// Run runs the command with the RunOpts +func (c *localCommandRunner) Run(ctx context.Context, desc string, args []string, opts *RunOpts) error { + if len(c.brokenArgs) != 0 { + log.Error("git command is broken: %s, broken args: %s", c.String(), strings.Join(c.brokenArgs, " ")) + return ErrBrokenCommand + } + if opts == nil { + opts = &RunOpts{} + } + + // We must not change the provided options + timeout := opts.Timeout + if timeout <= 0 { + timeout = defaultCommandExecutionTimeout + } + + if len(opts.Dir) == 0 { + log.Debug("%s", c) + } else { + log.Debug("%s: %v", opts.Dir, c) + } + + if desc == "" { + args := args[c.globalArgsLength:] + var argSensitiveURLIndexes []int + for i, arg := range args { + if strings.Contains(arg, "://") && strings.Contains(arg, "@") { + argSensitiveURLIndexes = append(argSensitiveURLIndexes, i) + } + } + if len(argSensitiveURLIndexes) > 0 { + args = make([]string, len(args)) + copy(args, args) + for _, urlArgIndex := range argSensitiveURLIndexes { + args[urlArgIndex] = util.SanitizeCredentialURLs(args[urlArgIndex]) + } + } + desc = fmt.Sprintf("%s %s [repo_path: %s]", c.name, strings.Join(args, " "), opts.Dir) + } + + var cancel context.CancelFunc + var finished context.CancelFunc + + if opts.UseContextTimeout { + ctx, cancel, finished = process.GetManager().AddContext(ctx, desc) + } else { + ctx, cancel, finished = process.GetManager().AddContextTimeout(ctx, timeout, desc) + } + defer finished() + + cmd := exec.CommandContext(ctx, c.name, args...) + if opts.Env == nil { + cmd.Env = os.Environ() + } else { + cmd.Env = opts.Env + } + + process.SetSysProcAttribute(cmd) + cmd.Env = append(cmd.Env, CommonGitCmdEnvs()...) + cmd.Dir = opts.Dir + cmd.Stdout = opts.Stdout + cmd.Stderr = opts.Stderr + cmd.Stdin = opts.Stdin + if err := cmd.Start(); err != nil { + return err + } + + if opts.PipelineFunc != nil { + err := opts.PipelineFunc(ctx, cancel) + if err != nil { + cancel() + _ = cmd.Wait() + return err + } + } + + if err := cmd.Wait(); err != nil && ctx.Err() != context.DeadlineExceeded { + return err + } + + return ctx.Err() +} From ed509fe939470e30a7d918e3b92940248310314b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 8 Jan 2023 21:27:09 +0800 Subject: [PATCH 2/7] abstract command runner and limit git path --- modules/git/batch_reader.go | 5 +- modules/git/command.go | 113 +++++++++++++++++++++------------- modules/git/command_runner.go | 105 +++++++++++++++---------------- modules/git/git.go | 42 ------------- routers/web/repo/http.go | 1 - 5 files changed, 119 insertions(+), 147 deletions(-) diff --git a/modules/git/batch_reader.go b/modules/git/batch_reader.go index 75539c0d0a935..be6d7d5460adc 100644 --- a/modules/git/batch_reader.go +++ b/modules/git/batch_reader.go @@ -32,7 +32,6 @@ type WriteCloserError interface { func EnsureValidGitRepository(ctx context.Context, repoPath string) error { stderr := strings.Builder{} err := NewCommand(ctx, "rev-parse"). - SetDescription(fmt.Sprintf("%s rev-parse [repo_path: %s]", GitExecutable, repoPath)). Run(&RunOpts{ Dir: repoPath, Stderr: &stderr, @@ -68,7 +67,7 @@ func CatFileBatchCheck(ctx context.Context, repoPath string) (WriteCloserError, go func() { stderr := strings.Builder{} err := NewCommand(ctx, "cat-file", "--batch-check"). - SetDescription(fmt.Sprintf("%s cat-file --batch-check [repo_path: %s] (%s:%d)", GitExecutable, repoPath, filename, line)). + SetExtraDescription(fmt.Sprintf("(%s:%d)", filename, line)). Run(&RunOpts{ Dir: repoPath, Stdin: batchStdinReader, @@ -118,7 +117,7 @@ func CatFileBatch(ctx context.Context, repoPath string) (WriteCloserError, *bufi go func() { stderr := strings.Builder{} err := NewCommand(ctx, "cat-file", "--batch"). - SetDescription(fmt.Sprintf("%s cat-file --batch [repo_path: %s] (%s:%d)", GitExecutable, repoPath, filename, line)). + SetExtraDescription(fmt.Sprintf("(%s:%d)", filename, line)). Run(&RunOpts{ Dir: repoPath, Stdin: batchStdinReader, diff --git a/modules/git/command.go b/modules/git/command.go index 2a83eee4a6f52..73f6a00542dbe 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -17,6 +17,7 @@ import ( "unsafe" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/util" ) @@ -36,6 +37,7 @@ type Command struct { args []string parentContext context.Context desc string + extraDesc string globalArgsLength int brokenArgs []string commandRunner CommandRunner @@ -59,6 +61,7 @@ func NewCommand(ctx context.Context, args ...CmdArg) *Command { cargs = append(cargs, string(arg)) } return &Command{ + args: cargs, parentContext: ctx, globalArgsLength: len(globalCommandArgs), commandRunner: newLocalCommandRunner(ctx), @@ -98,6 +101,13 @@ func (c *Command) SetDescription(desc string) *Command { return c } +// SetExtraDescription sets the description for this command which be returned on +// c.String() +func (c *Command) SetExtraDescription(extraDesc string) *Command { + c.extraDesc = extraDesc + return c +} + // AddArguments adds new git argument(s) to the command. Each argument must be safe to be trusted. // User-provided arguments should be passed to AddDynamicArguments instead. func (c *Command) AddArguments(args ...CmdArg) *Command { @@ -187,6 +197,64 @@ func CommonCmdServEnvs() []string { var ErrBrokenCommand = errors.New("git command is broken") +// Run runs the command with the RunOpts +func (c *Command) Run(opts *RunOpts) error { + if len(c.brokenArgs) != 0 { + log.Error("git command is broken: %s, broken args: %s", c.String(), strings.Join(c.brokenArgs, " ")) + return ErrBrokenCommand + } + if opts == nil { + opts = &RunOpts{} + } + + // We must not change the provided options + timeout := opts.Timeout + if timeout <= 0 { + timeout = defaultCommandExecutionTimeout + } + + if len(opts.Dir) == 0 { + log.Debug("%s", c) + } else { + log.Debug("%s: %v", opts.Dir, c) + } + + desc := c.desc + if desc == "" { + args := c.args[c.globalArgsLength:] + var argSensitiveURLIndexes []int + for i, arg := range c.args { + if strings.Contains(arg, "://") && strings.Contains(arg, "@") { + argSensitiveURLIndexes = append(argSensitiveURLIndexes, i) + } + } + if len(argSensitiveURLIndexes) > 0 { + args = make([]string, len(c.args)) + copy(args, c.args) + for _, urlArgIndex := range argSensitiveURLIndexes { + args[urlArgIndex] = util.SanitizeCredentialURLs(args[urlArgIndex]) + } + } + desc = fmt.Sprintf("%s %s [repo_path: %s]", c.commandRunner.String(), strings.Join(args, " "), opts.Dir) + } + if c.extraDesc != "" { + desc = fmt.Sprintf("%s %s", desc, c.extraDesc) + } + + var ctx context.Context + var cancel context.CancelFunc + var finished context.CancelFunc + + if opts.UseContextTimeout { + ctx, cancel, finished = process.GetManager().AddContext(c.parentContext, desc) + } else { + ctx, cancel, finished = process.GetManager().AddContextTimeout(c.parentContext, timeout, desc) + } + defer finished() + + return c.commandRunner.Run(ctx, c.args, opts, cancel) +} + type RunStdError interface { error Unwrap() error @@ -228,49 +296,6 @@ func bytesToString(b []byte) string { return *(*string)(unsafe.Pointer(&b)) // that's what Golang's strings.Builder.String() does (go/src/strings/builder.go) } -// Run runs the command with the RunOpts -func (c *Command) Run(opts *RunOpts) error { - if len(c.brokenArgs) != 0 { - log.Error("git command is broken: %s, broken args: %s", c.String(), strings.Join(c.brokenArgs, " ")) - return ErrBrokenCommand - } - if opts == nil { - opts = &RunOpts{} - } - - // We must not change the provided options - timeout := opts.Timeout - if timeout <= 0 { - timeout = defaultCommandExecutionTimeout - } - - if len(opts.Dir) == 0 { - log.Debug("%s", c) - } else { - log.Debug("%s: %v", opts.Dir, c) - } - - if desc == "" { - args := args[c.globalArgsLength:] - var argSensitiveURLIndexes []int - for i, arg := range args { - if strings.Contains(arg, "://") && strings.Contains(arg, "@") { - argSensitiveURLIndexes = append(argSensitiveURLIndexes, i) - } - } - if len(argSensitiveURLIndexes) > 0 { - args = make([]string, len(args)) - copy(args, args) - for _, urlArgIndex := range argSensitiveURLIndexes { - args[urlArgIndex] = util.SanitizeCredentialURLs(args[urlArgIndex]) - } - } - desc = fmt.Sprintf("%s %s [repo_path: %s]", c.name, strings.Join(args, " "), opts.Dir) - } - - return c.commandRunner.Run(c.parentContext, c.desc, c.args, opts, timeout) -} - // RunStdString runs the command with options and returns stdout/stderr as string. and store stderr to returned error (err combined with stderr). func (c *Command) RunStdString(opts *RunOpts) (stdout, stderr string, runErr RunStdError) { stdoutBytes, stderrBytes, err := c.RunStdBytes(opts) @@ -307,7 +332,7 @@ func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS PipelineFunc: opts.PipelineFunc, } - err := c.commandRunner.Run(c.parentContext, c.desc, c.args, newOpts) + err := c.Run(newOpts) stderr = stderrBuf.Bytes() if err != nil { return nil, stderr, &runStdError{err: err, stderr: bytesToString(stderr)} diff --git a/modules/git/command_runner.go b/modules/git/command_runner.go index 62a8a5fcc815c..a3c26eb7925cc 100644 --- a/modules/git/command_runner.go +++ b/modules/git/command_runner.go @@ -8,30 +8,69 @@ import ( "fmt" "os" "os/exec" - "strings" + "runtime" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" - "code.gitea.io/gitea/modules/util" + + "github.com/hashicorp/go-version" ) +// gitExecutable is the command name of git +// Could be updated to an absolute path while initialization +var gitExecutable = "git" + +// SetExecutablePath changes the path of git executable and checks the file permission and version. +func SetExecutablePath(path string) error { + // If path is empty, we use the default value of gitExecutable "git" to search for the location of git. + if path != "" { + gitExecutable = path + } + absPath, err := exec.LookPath(gitExecutable) + if err != nil { + return fmt.Errorf("git not found: %w", err) + } + gitExecutable = absPath + + _, err = loadGitVersion() + if err != nil { + return fmt.Errorf("unable to load git version: %w", err) + } + + versionRequired, err := version.NewVersion(RequiredVersion) + if err != nil { + return err + } + + if gitVersion.LessThan(versionRequired) { + moreHint := "get git: https://git-scm.com/download/" + if runtime.GOOS == "linux" { + // there are a lot of CentOS/RHEL users using old git, so we add a special hint for them + if _, err = os.Stat("/etc/redhat-release"); err == nil { + // ius.io is the recommended official(git-scm.com) method to install git + moreHint = "get git: https://git-scm.com/download/linux and https://ius.io" + } + } + return fmt.Errorf("installed git version %q is not supported, Gitea requires git version >= %q, %s", gitVersion.Original(), RequiredVersion, moreHint) + } + + return nil +} + type CommandRunner interface { String() string - Run(ctx context.Context, desc string, args []string, opts *RunOpts) error + Run(ctx context.Context, args []string, opts *RunOpts, cancel context.CancelFunc) error } // localCommandRunner represents a command with its subcommands or arguments. type localCommandRunner struct { - name string - globalArgsLength int + name string } var _ CommandRunner = &localCommandRunner{} func newLocalCommandRunner(ctx context.Context) *localCommandRunner { return &localCommandRunner{ - name: GitExecutable, - globalArgsLength: len(globalCommandArgs), + name: gitExecutable, } } @@ -40,55 +79,7 @@ func (c *localCommandRunner) String() string { } // Run runs the command with the RunOpts -func (c *localCommandRunner) Run(ctx context.Context, desc string, args []string, opts *RunOpts) error { - if len(c.brokenArgs) != 0 { - log.Error("git command is broken: %s, broken args: %s", c.String(), strings.Join(c.brokenArgs, " ")) - return ErrBrokenCommand - } - if opts == nil { - opts = &RunOpts{} - } - - // We must not change the provided options - timeout := opts.Timeout - if timeout <= 0 { - timeout = defaultCommandExecutionTimeout - } - - if len(opts.Dir) == 0 { - log.Debug("%s", c) - } else { - log.Debug("%s: %v", opts.Dir, c) - } - - if desc == "" { - args := args[c.globalArgsLength:] - var argSensitiveURLIndexes []int - for i, arg := range args { - if strings.Contains(arg, "://") && strings.Contains(arg, "@") { - argSensitiveURLIndexes = append(argSensitiveURLIndexes, i) - } - } - if len(argSensitiveURLIndexes) > 0 { - args = make([]string, len(args)) - copy(args, args) - for _, urlArgIndex := range argSensitiveURLIndexes { - args[urlArgIndex] = util.SanitizeCredentialURLs(args[urlArgIndex]) - } - } - desc = fmt.Sprintf("%s %s [repo_path: %s]", c.name, strings.Join(args, " "), opts.Dir) - } - - var cancel context.CancelFunc - var finished context.CancelFunc - - if opts.UseContextTimeout { - ctx, cancel, finished = process.GetManager().AddContext(ctx, desc) - } else { - ctx, cancel, finished = process.GetManager().AddContextTimeout(ctx, timeout, desc) - } - defer finished() - +func (c *localCommandRunner) Run(ctx context.Context, args []string, opts *RunOpts, cancel context.CancelFunc) error { cmd := exec.CommandContext(ctx, c.name, args...) if opts.Env == nil { cmd.Env = os.Environ() diff --git a/modules/git/git.go b/modules/git/git.go index f5919d82dcae1..b5c38ec9f1627 100644 --- a/modules/git/git.go +++ b/modules/git/git.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "os" - "os/exec" "path/filepath" "regexp" "runtime" @@ -26,10 +25,6 @@ import ( const RequiredVersion = "2.0.0" var ( - // GitExecutable is the command name of git - // Could be updated to an absolute path while initialization - GitExecutable = "git" - // DefaultContext is the default context to run git commands in, must be initialized by git.InitXxx DefaultContext context.Context @@ -71,43 +66,6 @@ func loadGitVersion() (*version.Version, error) { return gitVersion, err } -// SetExecutablePath changes the path of git executable and checks the file permission and version. -func SetExecutablePath(path string) error { - // If path is empty, we use the default value of GitExecutable "git" to search for the location of git. - if path != "" { - GitExecutable = path - } - absPath, err := exec.LookPath(GitExecutable) - if err != nil { - return fmt.Errorf("git not found: %w", err) - } - GitExecutable = absPath - - _, err = loadGitVersion() - if err != nil { - return fmt.Errorf("unable to load git version: %w", err) - } - - versionRequired, err := version.NewVersion(RequiredVersion) - if err != nil { - return err - } - - if gitVersion.LessThan(versionRequired) { - moreHint := "get git: https://git-scm.com/download/" - if runtime.GOOS == "linux" { - // there are a lot of CentOS/RHEL users using old git, so we add a special hint for them - if _, err = os.Stat("/etc/redhat-release"); err == nil { - // ius.io is the recommended official(git-scm.com) method to install git - moreHint = "get git: https://git-scm.com/download/linux and https://ius.io" - } - } - return fmt.Errorf("installed git version %q is not supported, Gitea requires git version >= %q, %s", gitVersion.Original(), RequiredVersion, moreHint) - } - - return nil -} - // VersionInfo returns git version information func VersionInfo() string { if gitVersion == nil { diff --git a/routers/web/repo/http.go b/routers/web/repo/http.go index b2a49e3e3a103..13b180eaced48 100644 --- a/routers/web/repo/http.go +++ b/routers/web/repo/http.go @@ -471,7 +471,6 @@ func serviceRPC(ctx gocontext.Context, h serviceHandler, service string) { var stderr bytes.Buffer cmd := git.NewCommand(h.r.Context(), git.CmdArgCheck(service), "--stateless-rpc").AddDynamicArguments(h.dir) - cmd.SetDescription(fmt.Sprintf("%s %s %s [repo_path: %s]", git.GitExecutable, service, "--stateless-rpc", h.dir)) if err := cmd.Run(&git.RunOpts{ Dir: h.dir, Env: append(os.Environ(), h.environ...), From b9015772e8c9bcc8efefdf5b919b911e3b6b2511 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 8 Jan 2023 22:32:44 +0800 Subject: [PATCH 3/7] git stoarge --- models/migrations/base/tests.go | 12 +++--- models/migrations/v1_12/v128.go | 14 +++---- models/migrations/v1_12/v134.go | 10 ++--- models/migrations/v1_12/v136.go | 8 ++-- models/repo/repo.go | 11 ++--- models/repo/update.go | 7 ++-- models/repo/wiki.go | 11 +---- models/repo_transfer.go | 17 ++++---- models/unittest/testdb.go | 26 +++++------- models/user/user.go | 11 ++--- modules/context/api.go | 4 +- modules/context/repo.go | 7 ++-- modules/git/command_runner.go | 6 ++- modules/git/storage/legacy.go | 35 ++++++++++++++++ modules/git/storage/storage.go | 42 +++++++++++++++++++ modules/repository/create.go | 5 ++- modules/repository/init.go | 3 +- modules/repository/push.go | 4 +- modules/repository/repo.go | 5 ++- routers/api/v1/admin/adopt.go | 5 ++- routers/api/v1/repo/file.go | 3 +- routers/api/v1/repo/pull.go | 5 ++- routers/web/admin/repos.go | 3 +- routers/web/repo/blame.go | 4 +- routers/web/repo/http.go | 5 ++- routers/web/user/setting/adopt.go | 4 +- routers/web/user/setting/profile.go | 3 +- services/org/org.go | 8 ++-- services/repository/adopt.go | 5 ++- services/repository/fork.go | 5 ++- services/user/user.go | 11 ++--- .../migration-test/migration_test.go | 12 +++--- 32 files changed, 191 insertions(+), 120 deletions(-) create mode 100644 modules/git/storage/legacy.go create mode 100644 modules/git/storage/storage.go diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index a9bcd20f674b1..7c1576e31928b 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -32,9 +33,9 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En ourSkip := 2 ourSkip += skip deferFn := PrintCurrentTest(t, ourSkip) - assert.NoError(t, os.RemoveAll(setting.RepoRootPath)) + assert.NoError(t, storage.GetStorage().RemoveAll()) assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath)) - ownerDirs, err := os.ReadDir(setting.RepoRootPath) + ownerDirs, err := storage.GetStorage().ReadDir("") if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } @@ -42,15 +43,12 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En if !ownerDir.Type().IsDir() { continue } - repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name())) + repoDirs, err := storage.GetStorage().ReadDir(ownerDir.Name()) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755) + storage.GetStorage().MakeDir(path.Join(ownerDir.Name(), repoDir.Name())) } } diff --git a/models/migrations/v1_12/v128.go b/models/migrations/v1_12/v128.go index 44d44a26c549f..c22df75850ab7 100644 --- a/models/migrations/v1_12/v128.go +++ b/models/migrations/v1_12/v128.go @@ -6,7 +6,7 @@ package v1_12 //nolint import ( "fmt" "math" - "path/filepath" + "path" "strings" "time" @@ -75,24 +75,24 @@ func FixMergeBase(x *xorm.Engine) error { log.Error("Missing base repo with id %d for PR ID %d", pr.BaseRepoID, pr.ID) continue } - userPath := filepath.Join(setting.RepoRootPath, strings.ToLower(baseRepo.OwnerName)) - repoPath := filepath.Join(userPath, strings.ToLower(baseRepo.Name)+".git") + + repoRelPath := path.Join(strings.ToLower(baseRepo.OwnerName), strings.ToLower(baseRepo.Name)+".git") gitRefName := fmt.Sprintf("refs/pull/%d/head", pr.Index) if !pr.HasMerged { var err error - pr.MergeBase, _, err = git.NewCommand(git.DefaultContext, "merge-base").AddDashesAndList(pr.BaseBranch, gitRefName).RunStdString(&git.RunOpts{Dir: repoPath}) + pr.MergeBase, _, err = git.NewCommand(git.DefaultContext, "merge-base").AddDashesAndList(pr.BaseBranch, gitRefName).RunStdString(&git.RunOpts{Dir: repoRelPath}) if err != nil { var err2 error - pr.MergeBase, _, err2 = git.NewCommand(git.DefaultContext, "rev-parse").AddDynamicArguments(git.BranchPrefix + pr.BaseBranch).RunStdString(&git.RunOpts{Dir: repoPath}) + pr.MergeBase, _, err2 = git.NewCommand(git.DefaultContext, "rev-parse").AddDynamicArguments(git.BranchPrefix + pr.BaseBranch).RunStdString(&git.RunOpts{Dir: repoRelPath}) if err2 != nil { log.Error("Unable to get merge base for PR ID %d, Index %d in %s/%s. Error: %v & %v", pr.ID, pr.Index, baseRepo.OwnerName, baseRepo.Name, err, err2) continue } } } else { - parentsString, _, err := git.NewCommand(git.DefaultContext, "rev-list", "--parents", "-n", "1").AddDynamicArguments(pr.MergedCommitID).RunStdString(&git.RunOpts{Dir: repoPath}) + parentsString, _, err := git.NewCommand(git.DefaultContext, "rev-list", "--parents", "-n", "1").AddDynamicArguments(pr.MergedCommitID).RunStdString(&git.RunOpts{Dir: repoRelPath}) if err != nil { log.Error("Unable to get parents for merged PR ID %d, Index %d in %s/%s. Error: %v", pr.ID, pr.Index, baseRepo.OwnerName, baseRepo.Name, err) continue @@ -106,7 +106,7 @@ func FixMergeBase(x *xorm.Engine) error { refs = append(refs, gitRefName) cmd := git.NewCommand(git.DefaultContext, "merge-base").AddDashesAndList(refs...) - pr.MergeBase, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoPath}) + pr.MergeBase, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoRelPath}) if err != nil { log.Error("Unable to get merge base for merged PR ID %d, Index %d in %s/%s. Error: %v", pr.ID, pr.Index, baseRepo.OwnerName, baseRepo.Name, err) continue diff --git a/models/migrations/v1_12/v134.go b/models/migrations/v1_12/v134.go index 3d1c82f09e164..4de0542fd2524 100644 --- a/models/migrations/v1_12/v134.go +++ b/models/migrations/v1_12/v134.go @@ -6,7 +6,7 @@ package v1_12 //nolint import ( "fmt" "math" - "path/filepath" + "path" "strings" "time" @@ -74,12 +74,12 @@ func RefixMergeBase(x *xorm.Engine) error { log.Error("Missing base repo with id %d for PR ID %d", pr.BaseRepoID, pr.ID) continue } - userPath := filepath.Join(setting.RepoRootPath, strings.ToLower(baseRepo.OwnerName)) - repoPath := filepath.Join(userPath, strings.ToLower(baseRepo.Name)+".git") + + repoRelPath := path.Join(strings.ToLower(baseRepo.OwnerName), strings.ToLower(baseRepo.Name)+".git") gitRefName := fmt.Sprintf("refs/pull/%d/head", pr.Index) - parentsString, _, err := git.NewCommand(git.DefaultContext, "rev-list", "--parents", "-n", "1").AddDynamicArguments(pr.MergedCommitID).RunStdString(&git.RunOpts{Dir: repoPath}) + parentsString, _, err := git.NewCommand(git.DefaultContext, "rev-list", "--parents", "-n", "1").AddDynamicArguments(pr.MergedCommitID).RunStdString(&git.RunOpts{Dir: repoRelPath}) if err != nil { log.Error("Unable to get parents for merged PR ID %d, Index %d in %s/%s. Error: %v", pr.ID, pr.Index, baseRepo.OwnerName, baseRepo.Name, err) continue @@ -94,7 +94,7 @@ func RefixMergeBase(x *xorm.Engine) error { refs = append(refs, gitRefName) cmd := git.NewCommand(git.DefaultContext, "merge-base").AddDashesAndList(refs...) - pr.MergeBase, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoPath}) + pr.MergeBase, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoRelPath}) if err != nil { log.Error("Unable to get merge base for merged PR ID %d, Index %d in %s/%s. Error: %v", pr.ID, pr.Index, baseRepo.OwnerName, baseRepo.Name, err) continue diff --git a/models/migrations/v1_12/v136.go b/models/migrations/v1_12/v136.go index 0cecba7be9395..83603a6b9becc 100644 --- a/models/migrations/v1_12/v136.go +++ b/models/migrations/v1_12/v136.go @@ -6,7 +6,7 @@ package v1_12 //nolint import ( "fmt" "math" - "path/filepath" + "path" "strings" "time" @@ -85,12 +85,12 @@ func AddCommitDivergenceToPulls(x *xorm.Engine) error { log.Error("Missing base repo with id %d for PR ID %d", pr.BaseRepoID, pr.ID) continue } - userPath := filepath.Join(setting.RepoRootPath, strings.ToLower(baseRepo.OwnerName)) - repoPath := filepath.Join(userPath, strings.ToLower(baseRepo.Name)+".git") + + repoRelPath := path.Join(strings.ToLower(baseRepo.OwnerName), strings.ToLower(baseRepo.Name)+".git") gitRefName := fmt.Sprintf("refs/pull/%d/head", pr.Index) - divergence, err := git.GetDivergingCommits(graceful.GetManager().HammerContext(), repoPath, pr.BaseBranch, gitRefName) + divergence, err := git.GetDivergingCommits(graceful.GetManager().HammerContext(), repoRelPath, pr.BaseBranch, gitRefName) if err != nil { log.Warn("Could not recalculate Divergence for pull: %d", pr.ID) pr.CommitsAhead = 0 diff --git a/models/repo/repo.go b/models/repo/repo.go index e5e1ac43b41fd..4ad0955761b5a 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -9,13 +9,13 @@ import ( "html/template" "net" "net/url" - "path/filepath" "strconv" "strings" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" @@ -471,14 +471,9 @@ func (repo *Repository) IsGenerated() bool { return repo.TemplateID != 0 } -// RepoPath returns repository path by given user and repository name. -func RepoPath(userName, repoName string) string { //revive:disable-line:exported - return filepath.Join(user_model.UserPath(userName), strings.ToLower(repoName)+".git") -} - // RepoPath returns the repository path func (repo *Repository) RepoPath() string { - return RepoPath(repo.OwnerName, repo.Name) + return storage.RepoPath(repo.OwnerName, repo.Name) } // Link returns the repository link @@ -686,7 +681,7 @@ func IsRepositoryExist(ctx context.Context, u *user_model.User, repoName string) if err != nil { return false, err } - isDir, err := util.IsDir(RepoPath(u.Name, repoName)) + isDir, err := util.IsDir(storage.RepoPath(u.Name, repoName)) return has && isDir, err } diff --git a/models/repo/update.go b/models/repo/update.go index 3aef280ff80bc..d0a87d9c3b972 100644 --- a/models/repo/update.go +++ b/models/repo/update.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models/db" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" ) @@ -123,7 +124,7 @@ func CheckCreateRepository(doer, u *user_model.User, name string, overwriteOrAdo return ErrRepoAlreadyExist{u.Name, name} } - repoPath := RepoPath(u.Name, name) + repoPath := storage.RepoPath(u.Name, name) isExist, err := util.IsExist(repoPath) if err != nil { log.Error("Unable to check if %s exists. Error: %v", repoPath, err) @@ -154,7 +155,7 @@ func ChangeRepositoryName(doer *user_model.User, repo *Repository, newRepoName s return ErrRepoAlreadyExist{repo.Owner.Name, newRepoName} } - newRepoPath := RepoPath(repo.Owner.Name, newRepoName) + newRepoPath := storage.RepoPath(repo.Owner.Name, newRepoName) if err = util.Rename(repo.RepoPath(), newRepoPath); err != nil { return fmt.Errorf("rename repository directory: %w", err) } @@ -166,7 +167,7 @@ func ChangeRepositoryName(doer *user_model.User, repo *Repository, newRepoName s return err } if isExist { - if err = util.Rename(wikiPath, WikiPath(repo.Owner.Name, newRepoName)); err != nil { + if err = util.Rename(wikiPath, storage.WikiPath(repo.Owner.Name, newRepoName)); err != nil { return fmt.Errorf("rename repository wiki: %w", err) } } diff --git a/models/repo/wiki.go b/models/repo/wiki.go index b378666a20620..241e6cd5797aa 100644 --- a/models/repo/wiki.go +++ b/models/repo/wiki.go @@ -6,10 +6,8 @@ package repo import ( "fmt" - "path/filepath" - "strings" - user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" ) @@ -76,14 +74,9 @@ func (repo *Repository) WikiCloneLink() *CloneLink { return repo.cloneLink(true) } -// WikiPath returns wiki data path by given user and repository name. -func WikiPath(userName, repoName string) string { - return filepath.Join(user_model.UserPath(userName), strings.ToLower(repoName)+".wiki.git") -} - // WikiPath returns wiki data path for given repository. func (repo *Repository) WikiPath() string { - return WikiPath(repo.OwnerName, repo.Name) + return storage.WikiPath(repo.OwnerName, repo.Name) } // HasWiki returns true if repository has wiki. diff --git a/models/repo_transfer.go b/models/repo_transfer.go index 27a77f9b8cc9f..aefad7485b97a 100644 --- a/models/repo_transfer.go +++ b/models/repo_transfer.go @@ -14,6 +14,7 @@ import ( access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" @@ -215,16 +216,16 @@ func TransferOwnership(doer *user_model.User, newOwnerName string, repo *repo_mo } if repoRenamed { - if err := util.Rename(repo_model.RepoPath(newOwnerName, repo.Name), repo_model.RepoPath(oldOwnerName, repo.Name)); err != nil { + if err := util.Rename(storage.RepoPath(newOwnerName, repo.Name), storage.RepoPath(oldOwnerName, repo.Name)); err != nil { log.Critical("Unable to move repository %s/%s directory from %s back to correct place %s: %v", oldOwnerName, repo.Name, - repo_model.RepoPath(newOwnerName, repo.Name), repo_model.RepoPath(oldOwnerName, repo.Name), err) + storage.RepoPath(newOwnerName, repo.Name), storage.RepoPath(oldOwnerName, repo.Name), err) } } if wikiRenamed { - if err := util.Rename(repo_model.WikiPath(newOwnerName, repo.Name), repo_model.WikiPath(oldOwnerName, repo.Name)); err != nil { + if err := util.Rename(storage.WikiPath(newOwnerName, repo.Name), storage.WikiPath(oldOwnerName, repo.Name)); err != nil { log.Critical("Unable to move wiki for repository %s/%s directory from %s back to correct place %s: %v", oldOwnerName, repo.Name, - repo_model.WikiPath(newOwnerName, repo.Name), repo_model.WikiPath(oldOwnerName, repo.Name), err) + storage.WikiPath(newOwnerName, repo.Name), storage.WikiPath(oldOwnerName, repo.Name), err) } } @@ -374,25 +375,25 @@ func TransferOwnership(doer *user_model.User, newOwnerName string, repo *repo_mo } // Rename remote repository to new path and delete local copy. - dir := user_model.UserPath(newOwner.Name) + dir := storage.UserPath(newOwner.Name) if err := os.MkdirAll(dir, os.ModePerm); err != nil { return fmt.Errorf("Failed to create dir %s: %w", dir, err) } - if err := util.Rename(repo_model.RepoPath(oldOwner.Name, repo.Name), repo_model.RepoPath(newOwner.Name, repo.Name)); err != nil { + if err := util.Rename(storage.RepoPath(oldOwner.Name, repo.Name), storage.RepoPath(newOwner.Name, repo.Name)); err != nil { return fmt.Errorf("rename repository directory: %w", err) } repoRenamed = true // Rename remote wiki repository to new path and delete local copy. - wikiPath := repo_model.WikiPath(oldOwner.Name, repo.Name) + wikiPath := storage.WikiPath(oldOwner.Name, repo.Name) if isExist, err := util.IsExist(wikiPath); err != nil { log.Error("Unable to check if %s exists. Error: %v", wikiPath, err) return err } else if isExist { - if err := util.Rename(wikiPath, repo_model.WikiPath(newOwner.Name, repo.Name)); err != nil { + if err := util.Rename(wikiPath, storage.WikiPath(newOwner.Name, repo.Name)); err != nil { return fmt.Errorf("rename repository wiki: %w", err) } wikiRenamed = true diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index 87d9304618b00..8b2008f3ff263 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "os" + "path" "path/filepath" "testing" @@ -14,8 +15,9 @@ import ( system_model "code.gitea.io/gitea/models/system" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/storage" + object_storage "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" @@ -106,7 +108,7 @@ func MainTest(m *testing.M, testOpts *TestOptions) { setting.Git.HomePath = filepath.Join(setting.AppDataPath, "home") - if err = storage.Init(); err != nil { + if err = object_storage.Init(); err != nil { fatalTestError("storage.Init: %v\n", err) } if err = system_model.Init(); err != nil { @@ -123,7 +125,7 @@ func MainTest(m *testing.M, testOpts *TestOptions) { if err = git.InitFull(context.Background()); err != nil { fatalTestError("git.Init: %v\n", err) } - ownerDirs, err := os.ReadDir(setting.RepoRootPath) + ownerDirs, err := storage.GetStorage().ReadDir("") if err != nil { fatalTestError("unable to read the new repo root: %v\n", err) } @@ -131,15 +133,12 @@ func MainTest(m *testing.M, testOpts *TestOptions) { if !ownerDir.Type().IsDir() { continue } - repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name())) + repoDirs, err := storage.GetStorage().ReadDir(ownerDir.Name()) if err != nil { fatalTestError("unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755) + storage.GetStorage().MakeDir(path.Join(ownerDir.Name(), repoDir.Name())) } } @@ -201,22 +200,19 @@ func PrepareTestDatabase() error { // by tests that use the above MainTest(..) function. func PrepareTestEnv(t testing.TB) { assert.NoError(t, PrepareTestDatabase()) - assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) + assert.NoError(t, storage.GetStorage().RemoveAll()) metaPath := filepath.Join(giteaRoot, "tests", "gitea-repositories-meta") assert.NoError(t, CopyDir(metaPath, setting.RepoRootPath)) - ownerDirs, err := os.ReadDir(setting.RepoRootPath) + ownerDirs, err := storage.GetStorage().ReadDir("") assert.NoError(t, err) for _, ownerDir := range ownerDirs { if !ownerDir.Type().IsDir() { continue } - repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name())) + repoDirs, err := storage.GetStorage().ReadDir(ownerDir.Name()) assert.NoError(t, err) for _, repoDir := range repoDirs { - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755) + storage.GetStorage().MakeDir(path.Join(ownerDir.Name(), repoDir.Name())) } } diff --git a/models/user/user.go b/models/user/user.go index 825223201b679..51c635b44483d 100644 --- a/models/user/user.go +++ b/models/user/user.go @@ -12,7 +12,6 @@ import ( "fmt" "net/url" "os" - "path/filepath" "strings" "time" @@ -23,6 +22,7 @@ import ( "code.gitea.io/gitea/modules/auth/openid" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" @@ -840,7 +840,7 @@ func ChangeUserName(u *User, newUserName string) (err error) { } // Do not fail if directory does not exist - if err = util.Rename(UserPath(oldUserName), UserPath(newUserName)); err != nil && !os.IsNotExist(err) { + if err = storage.Rename(oldUserName, newUserName); err != nil && !os.IsNotExist(err) { return fmt.Errorf("Rename user directory: %w", err) } @@ -849,7 +849,7 @@ func ChangeUserName(u *User, newUserName string) (err error) { } if err = committer.Commit(); err != nil { - if err2 := util.Rename(UserPath(newUserName), UserPath(oldUserName)); err2 != nil && !os.IsNotExist(err2) { + if err2 := storage.Rename(newUserName, oldUserName); err2 != nil && !os.IsNotExist(err2) { log.Critical("Unable to rollback directory change during failed username change from: %s to: %s. DB Error: %v. Filesystem Error: %v", oldUserName, newUserName, err, err2) return fmt.Errorf("failed to rollback directory change during failed username change from: %s to: %s. DB Error: %w. Filesystem Error: %v", oldUserName, newUserName, err, err2) } @@ -996,11 +996,6 @@ func GetInactiveUsers(ctx context.Context, olderThan time.Duration) ([]*User, er Find(&users) } -// UserPath returns the path absolute path of user repositories. -func UserPath(userName string) string { //revive:disable-line:exported - return filepath.Join(setting.RepoRootPath, strings.ToLower(userName)) -} - // GetUserByID returns the user object by given ID if exists. func GetUserByID(ctx context.Context, id int64) (*User, error) { u := new(User) diff --git a/modules/context/api.go b/modules/context/api.go index 3f52c54d4cfdf..e6ea178f01fb2 100644 --- a/modules/context/api.go +++ b/modules/context/api.go @@ -12,9 +12,9 @@ import ( "strings" "code.gitea.io/gitea/models/auth" - repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/httpcache" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -331,7 +331,7 @@ func ReferencesGitRepo(allowEmpty ...bool) func(ctx *APIContext) (cancel context // For API calls. if ctx.Repo.GitRepo == nil { - repoPath := repo_model.RepoPath(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name) + repoPath := storage.RepoPath(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name) gitRepo, err := git.OpenRepository(ctx, repoPath) if err != nil { ctx.Error(http.StatusInternalServerError, "RepoRef Invalid repo "+repoPath, err) diff --git a/modules/context/repo.go b/modules/context/repo.go index d15d28cab7834..67285d6f902e7 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -23,6 +23,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" code_indexer "code.gitea.io/gitea/modules/indexer/code" "code.gitea.io/gitea/modules/issue/template" "code.gitea.io/gitea/modules/log" @@ -624,7 +625,7 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) { return } - gitRepo, err := git.OpenRepository(ctx, repo_model.RepoPath(userName, repoName)) + gitRepo, err := git.OpenRepository(ctx, storage.RepoPath(userName, repoName)) if err != nil { if strings.Contains(err.Error(), "repository does not exist") || strings.Contains(err.Error(), "no such file or directory") { log.Error("Repository %-v has a broken repository on the file system: %s Error: %v", ctx.Repo.Repository, ctx.Repo.Repository.RepoPath(), err) @@ -637,7 +638,7 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) { } return } - ctx.ServerError("RepoAssignment Invalid repo "+repo_model.RepoPath(userName, repoName), err) + ctx.ServerError("RepoAssignment Invalid repo "+storage.RepoPath(userName, repoName), err) return } if ctx.Repo.GitRepo != nil { @@ -884,7 +885,7 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context ) if ctx.Repo.GitRepo == nil { - repoPath := repo_model.RepoPath(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name) + repoPath := storage.RepoPath(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name) ctx.Repo.GitRepo, err = git.OpenRepository(ctx, repoPath) if err != nil { ctx.ServerError("RepoRef Invalid repo "+repoPath, err) diff --git a/modules/git/command_runner.go b/modules/git/command_runner.go index a3c26eb7925cc..10e056b3aa835 100644 --- a/modules/git/command_runner.go +++ b/modules/git/command_runner.go @@ -8,9 +8,11 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "runtime" "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/setting" "github.com/hashicorp/go-version" ) @@ -89,7 +91,9 @@ func (c *localCommandRunner) Run(ctx context.Context, args []string, opts *RunOp process.SetSysProcAttribute(cmd) cmd.Env = append(cmd.Env, CommonGitCmdEnvs()...) - cmd.Dir = opts.Dir + if !filepath.IsAbs(opts.Dir) { + cmd.Dir = filepath.Join(setting.RepoRootPath, opts.Dir) + } cmd.Stdout = opts.Stdout cmd.Stderr = opts.Stderr cmd.Stdin = opts.Stdin diff --git a/modules/git/storage/legacy.go b/modules/git/storage/legacy.go new file mode 100644 index 0000000000000..b628384fe3138 --- /dev/null +++ b/modules/git/storage/legacy.go @@ -0,0 +1,35 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package storage + +import ( + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +func absPath(path string) string { + return filepath.Join(setting.RepoRootPath, path) +} + +// UserPath returns the path absolute path of user repositories. +func UserPath(userName string) string { //revive:disable-line:exported + return absPath(strings.ToLower(userName)) +} + +// RepoPath returns repository path by given user and repository name. +func RepoPath(userName, repoName string) string { //revive:disable-line:exported + return filepath.Join(UserPath(userName), strings.ToLower(repoName)+".git") +} + +// WikiPath returns wiki data path by given user and repository name. +func WikiPath(userName, repoName string) string { + return filepath.Join(UserPath(userName), strings.ToLower(repoName)+".wiki.git") +} + +func Rename(oldPath, newPath string) error { + return util.Rename(absPath(oldPath), absPath(newPath)) +} diff --git a/modules/git/storage/storage.go b/modules/git/storage/storage.go new file mode 100644 index 0000000000000..4eeade23f1c46 --- /dev/null +++ b/modules/git/storage/storage.go @@ -0,0 +1,42 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package storage + +import ( + "io/fs" + "os" + "path/filepath" + + "code.gitea.io/gitea/modules/setting" +) + +type Storage interface { + MakeDir(repoRelPath string) error + RemoveAll() error + ReadDir(owner string) ([]fs.DirEntry, error) +} + +type LocalStorage struct{} + +var _ Storage = &LocalStorage{} + +func (l *LocalStorage) MakeDir(repoRelPath string) error { + _ = os.MkdirAll(filepath.Join(absPath(repoRelPath), "objects", "pack"), 0o755) + _ = os.MkdirAll(filepath.Join(absPath(repoRelPath), "objects", "info"), 0o755) + _ = os.MkdirAll(filepath.Join(absPath(repoRelPath), "refs", "heads"), 0o755) + _ = os.MkdirAll(filepath.Join(absPath(repoRelPath), "refs", "tag"), 0o755) + return nil +} + +func (l *LocalStorage) RemoveAll() error { + return os.RemoveAll(setting.RepoRootPath) +} + +func (l *LocalStorage) ReadDir(owner string) ([]fs.DirEntry, error) { + return os.ReadDir(absPath(owner)) +} + +func GetStorage() Storage { + return &LocalStorage{} +} diff --git a/modules/repository/create.go b/modules/repository/create.go index 1e157ec85eff8..8d486ffb75316 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -22,6 +22,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/models/webhook" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" @@ -44,7 +45,7 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re } } - repoPath := repo_model.RepoPath(u.Name, repo.Name) + repoPath := storage.RepoPath(u.Name, repo.Name) isExist, err := util.IsExist(repoPath) if err != nil { log.Error("Unable to check if %s exists. Error: %v", repoPath, err) @@ -220,7 +221,7 @@ func CreateRepository(doer, u *user_model.User, opts CreateRepoOptions) (*repo_m return nil } - repoPath := repo_model.RepoPath(u.Name, repo.Name) + repoPath := storage.RepoPath(u.Name, repo.Name) isExist, err := util.IsExist(repoPath) if err != nil { log.Error("Unable to check if %s exists. Error: %v", repoPath, err) diff --git a/modules/repository/init.go b/modules/repository/init.go index 59284a5bafc81..94551be7323be 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -18,6 +18,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" @@ -362,7 +363,7 @@ func initRepoCommit(ctx context.Context, tmpPath string, repo *repo_model.Reposi func checkInitRepository(ctx context.Context, owner, name string) (err error) { // Somehow the directory could exist. - repoPath := repo_model.RepoPath(owner, name) + repoPath := storage.RepoPath(owner, name) isExist, err := util.IsExist(repoPath) if err != nil { log.Error("Unable to check if %s exists. Error: %v", repoPath, err) diff --git a/modules/repository/push.go b/modules/repository/push.go index 1fa711b359c34..bffbf7297b333 100644 --- a/modules/repository/push.go +++ b/modules/repository/push.go @@ -5,9 +5,9 @@ package repository import ( "context" + "path" "strings" - repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" ) @@ -104,7 +104,7 @@ func IsForcePush(ctx context.Context, opts *PushUpdateOptions) (bool, error) { } output, _, err := git.NewCommand(ctx, "rev-list", "--max-count=1").AddDynamicArguments(opts.OldCommitID, "^"+opts.NewCommitID). - RunStdString(&git.RunOpts{Dir: repo_model.RepoPath(opts.RepoUserName, opts.RepoName)}) + RunStdString(&git.RunOpts{Dir: path.Join(opts.RepoUserName, opts.RepoName)}) if err != nil { return false, err } else if len(output) > 0 { diff --git a/modules/repository/repo.go b/modules/repository/repo.go index a90eb8a764ac9..05b3c38908dcc 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -20,6 +20,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migration" @@ -54,7 +55,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, repo *repo_model.Repository, opts migration.MigrateOptions, httpTransport *http.Transport, ) (*repo_model.Repository, error) { - repoPath := repo_model.RepoPath(u.Name, opts.RepoName) + repoPath := storage.RepoPath(u.Name, opts.RepoName) if u.IsOrganization() { t, err := organization.OrgFromUser(u).GetOwnerTeam() @@ -90,7 +91,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } if opts.Wiki { - wikiPath := repo_model.WikiPath(u.Name, opts.RepoName) + wikiPath := storage.WikiPath(u.Name, opts.RepoName) wikiRemotePath := WikiRemoteURL(ctx, opts.CloneAddr) if len(wikiRemotePath) > 0 { if err := util.RemoveAll(wikiPath); err != nil { diff --git a/routers/api/v1/admin/adopt.go b/routers/api/v1/admin/adopt.go index 0e4e498e98243..eb32a557e9c20 100644 --- a/routers/api/v1/admin/adopt.go +++ b/routers/api/v1/admin/adopt.go @@ -9,6 +9,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git/storage" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/routers/api/v1/utils" @@ -100,7 +101,7 @@ func AdoptRepository(ctx *context.APIContext) { ctx.InternalServerError(err) return } - isDir, err := util.IsDir(repo_model.RepoPath(ctxUser.Name, repoName)) + isDir, err := util.IsDir(storage.RepoPath(ctxUser.Name, repoName)) if err != nil { ctx.InternalServerError(err) return @@ -162,7 +163,7 @@ func DeleteUnadoptedRepository(ctx *context.APIContext) { ctx.InternalServerError(err) return } - isDir, err := util.IsDir(repo_model.RepoPath(ctxUser.Name, repoName)) + isDir, err := util.IsDir(storage.RepoPath(ctxUser.Name, repoName)) if err != nil { ctx.InternalServerError(err) return diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index 623fe18e5d26d..11f5828b884bf 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" + git_storage "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/httpcache" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" @@ -283,7 +284,7 @@ func GetArchive(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - repoPath := repo_model.RepoPath(ctx.Params(":username"), ctx.Params(":reponame")) + repoPath := git_storage.RepoPath(ctx.Params(":username"), ctx.Params(":reponame")) if ctx.Repo.GitRepo == nil { gitRepo, err := git.OpenRepository(ctx, repoPath) if err != nil { diff --git a/routers/api/v1/repo/pull.go b/routers/api/v1/repo/pull.go index 1b1aba17d9299..e4e58ec8e7bdc 100644 --- a/routers/api/v1/repo/pull.go +++ b/routers/api/v1/repo/pull.go @@ -22,6 +22,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/setting" @@ -983,7 +984,7 @@ func parseCompareInfo(ctx *context.APIContext, form api.CreatePullRequestOption) headRepo = ctx.Repo.Repository headGitRepo = ctx.Repo.GitRepo } else { - headGitRepo, err = git.OpenRepository(ctx, repo_model.RepoPath(headUser.Name, headRepo.Name)) + headGitRepo, err = git.OpenRepository(ctx, storage.RepoPath(headUser.Name, headRepo.Name)) if err != nil { ctx.Error(http.StatusInternalServerError, "OpenRepository", err) return nil, nil, nil, nil, "", "" @@ -1035,7 +1036,7 @@ func parseCompareInfo(ctx *context.APIContext, form api.CreatePullRequestOption) return nil, nil, nil, nil, "", "" } - compareInfo, err := headGitRepo.GetCompareInfo(repo_model.RepoPath(baseRepo.Owner.Name, baseRepo.Name), baseBranch, headBranch, false, false) + compareInfo, err := headGitRepo.GetCompareInfo(storage.RepoPath(baseRepo.Owner.Name, baseRepo.Name), baseBranch, headBranch, false, false) if err != nil { headGitRepo.Close() ctx.Error(http.StatusInternalServerError, "GetCompareInfo", err) diff --git a/routers/web/admin/repos.go b/routers/web/admin/repos.go index dc5432c54998c..0eaff46cbefcd 100644 --- a/routers/web/admin/repos.go +++ b/routers/web/admin/repos.go @@ -13,6 +13,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" @@ -139,7 +140,7 @@ func AdoptOrDeleteRepository(ctx *context.Context) { ctx.ServerError("IsRepositoryExist", err) return } - isDir, err := util.IsDir(repo_model.RepoPath(ctxUser.Name, repoName)) + isDir, err := util.IsDir(storage.RepoPath(ctxUser.Name, repoName)) if err != nil { ctx.ServerError("IsDir", err) return diff --git a/routers/web/repo/blame.go b/routers/web/repo/blame.go index 50bfa9d3bd289..9c733d7be7ebd 100644 --- a/routers/web/repo/blame.go +++ b/routers/web/repo/blame.go @@ -10,12 +10,12 @@ import ( "net/url" "strings" - repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/highlight" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/templates" @@ -106,7 +106,7 @@ func RefBlame(ctx *context.Context) { return } - blameReader, err := git.CreateBlameReader(ctx, repo_model.RepoPath(userName, repoName), commitID, fileName) + blameReader, err := git.CreateBlameReader(ctx, storage.RepoPath(userName, repoName), commitID, fileName) if err != nil { ctx.NotFound("CreateBlameReader", err) return diff --git a/routers/web/repo/http.go b/routers/web/repo/http.go index 13b180eaced48..59e7132d9b04f 100644 --- a/routers/web/repo/http.go +++ b/routers/web/repo/http.go @@ -25,6 +25,7 @@ import ( "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" @@ -281,9 +282,9 @@ func httpBase(ctx *context.Context) (h *serviceHandler) { r.URL.Path = strings.ToLower(r.URL.Path) // blue: In case some repo name has upper case name - dir := repo_model.RepoPath(username, reponame) + dir := storage.RepoPath(username, reponame) if isWiki { - dir = repo_model.RepoPath(username, wikiRepoName) + dir = storage.RepoPath(username, wikiRepoName) } return &serviceHandler{cfg, w, r, dir, cfg.Env} diff --git a/routers/web/user/setting/adopt.go b/routers/web/user/setting/adopt.go index 0aaf5920bc6d1..5058e1a5c8aa4 100644 --- a/routers/web/user/setting/adopt.go +++ b/routers/web/user/setting/adopt.go @@ -7,8 +7,8 @@ import ( "path/filepath" repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git/storage" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -28,7 +28,7 @@ func AdoptOrDeleteRepository(ctx *context.Context) { action := ctx.FormString("action") ctxUser := ctx.Doer - root := user_model.UserPath(ctxUser.LowerName) + root := storage.UserPath(ctxUser.LowerName) // check not a repo has, err := repo_model.IsRepositoryExist(ctx, ctxUser, dir) diff --git a/routers/web/user/setting/profile.go b/routers/web/user/setting/profile.go index ef45ad8a862d1..00e1dd4f9c1f0 100644 --- a/routers/web/user/setting/profile.go +++ b/routers/web/user/setting/profile.go @@ -20,6 +20,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" @@ -279,7 +280,7 @@ func Repos(ctx *context.Context) { repoNames := make([]string, 0, setting.UI.Admin.UserPagingNum) repos := map[string]*repo_model.Repository{} // We're going to iterate by pagesize. - root := user_model.UserPath(ctxUser.Name) + root := storage.UserPath(ctxUser.Name) if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if err != nil { if os.IsNotExist(err) { diff --git a/services/org/org.go b/services/org/org.go index e45fb305debe8..7f0446712bcad 100644 --- a/services/org/org.go +++ b/services/org/org.go @@ -11,8 +11,8 @@ import ( "code.gitea.io/gitea/models/organization" packages_model "code.gitea.io/gitea/models/packages" repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/git/storage" + object_storage "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" ) @@ -50,7 +50,7 @@ func DeleteOrganization(org *organization.Organization) error { // FIXME: system notice // Note: There are something just cannot be roll back, // so just keep error logs of those operations. - path := user_model.UserPath(org.Name) + path := storage.UserPath(org.Name) if err := util.RemoveAll(path); err != nil { return fmt.Errorf("Failed to RemoveAll %s: %w", path, err) @@ -58,7 +58,7 @@ func DeleteOrganization(org *organization.Organization) error { if len(org.Avatar) > 0 { avatarPath := org.CustomAvatarRelativePath() - if err := storage.Avatars.Delete(avatarPath); err != nil { + if err := object_storage.Avatars.Delete(avatarPath); err != nil { return fmt.Errorf("Failed to remove %s: %w", avatarPath, err) } } diff --git a/services/repository/adopt.go b/services/repository/adopt.go index 93eeb56456f45..8a958d2438d43 100644 --- a/services/repository/adopt.go +++ b/services/repository/adopt.go @@ -16,6 +16,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" repo_module "code.gitea.io/gitea/modules/repository" @@ -54,7 +55,7 @@ func AdoptRepository(doer, u *user_model.User, opts repo_module.CreateRepoOption } if err := db.WithTx(db.DefaultContext, func(ctx context.Context) error { - repoPath := repo_model.RepoPath(u.Name, repo.Name) + repoPath := storage.RepoPath(u.Name, repo.Name) isExist, err := util.IsExist(repoPath) if err != nil { log.Error("Unable to check if %s exists. Error: %v", repoPath, err) @@ -193,7 +194,7 @@ func DeleteUnadoptedRepository(doer, u *user_model.User, repoName string) error return err } - repoPath := repo_model.RepoPath(u.Name, repoName) + repoPath := storage.RepoPath(u.Name, repoName) isExist, err := util.IsExist(repoPath) if err != nil { log.Error("Unable to check if %s exists. Error: %v", repoPath, err) diff --git a/services/repository/fork.go b/services/repository/fork.go index ad534be887f1b..678617f3ba3f7 100644 --- a/services/repository/fork.go +++ b/services/repository/fork.go @@ -14,6 +14,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" repo_module "code.gitea.io/gitea/modules/repository" @@ -92,7 +93,7 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork return } - repoPath := repo_model.RepoPath(owner.Name, repo.Name) + repoPath := storage.RepoPath(owner.Name, repo.Name) if exists, _ := util.IsExist(repoPath); !exists { return @@ -134,7 +135,7 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork needsRollback = true - repoPath := repo_model.RepoPath(owner.Name, repo.Name) + repoPath := storage.RepoPath(owner.Name, repo.Name) if stdout, _, err := git.NewCommand(txCtx, "clone", "--bare").AddDynamicArguments(oldRepoPath, repoPath). SetDescription(fmt.Sprintf("ForkRepository(git clone): %s to %s", opts.BaseRepo.FullName(), repo.FullName())). diff --git a/services/user/user.go b/services/user/user.go index c95eb67a851e9..659088119c5e5 100644 --- a/services/user/user.go +++ b/services/user/user.go @@ -20,9 +20,10 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/avatar" "code.gitea.io/gitea/modules/eventsource" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/storage" + object_storage "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/packages" ) @@ -181,7 +182,7 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { // Note: There are something just cannot be roll back, // so just keep error logs of those operations. - path := user_model.UserPath(u.Name) + path := storage.UserPath(u.Name) if err := util.RemoveAll(path); err != nil { err = fmt.Errorf("Failed to RemoveAll %s: %w", path, err) _ = system_model.CreateNotice(ctx, system_model.NoticeTask, fmt.Sprintf("delete user '%s': %v", u.Name, err)) @@ -190,7 +191,7 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { if u.Avatar != "" { avatarPath := u.CustomAvatarRelativePath() - if err := storage.Avatars.Delete(avatarPath); err != nil { + if err := object_storage.Avatars.Delete(avatarPath); err != nil { err = fmt.Errorf("Failed to remove %s: %w", avatarPath, err) _ = system_model.CreateNotice(ctx, system_model.NoticeTask, fmt.Sprintf("delete user '%s': %v", u.Name, err)) return err @@ -245,7 +246,7 @@ func UploadAvatar(u *user_model.User, data []byte) error { return fmt.Errorf("updateUser: %w", err) } - if err := storage.SaveFrom(storage.Avatars, u.CustomAvatarRelativePath(), func(w io.Writer) error { + if err := object_storage.SaveFrom(object_storage.Avatars, u.CustomAvatarRelativePath(), func(w io.Writer) error { if err := png.Encode(w, *m); err != nil { log.Error("Encode: %v", err) } @@ -262,7 +263,7 @@ func DeleteAvatar(u *user_model.User) error { aPath := u.CustomAvatarRelativePath() log.Trace("DeleteAvatar[%d]: %s", u.ID, aPath) if len(u.Avatar) > 0 { - if err := storage.Avatars.Delete(aPath); err != nil { + if err := object_storage.Avatars.Delete(aPath); err != nil { return fmt.Errorf("Failed to remove %s: %w", aPath, err) } } diff --git a/tests/integration/migration-test/migration_test.go b/tests/integration/migration-test/migration_test.go index 170a6dd444c3f..65acb3b5015f4 100644 --- a/tests/integration/migration-test/migration_test.go +++ b/tests/integration/migration-test/migration_test.go @@ -24,6 +24,7 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/tests" @@ -60,9 +61,9 @@ func initMigrationTest(t *testing.T) func() { setting.LoadForTest() assert.True(t, len(setting.RepoRootPath) != 0) - assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) + assert.NoError(t, storage.GetStorage().RemoveAll()) assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath)) - ownerDirs, err := os.ReadDir(setting.RepoRootPath) + ownerDirs, err := storage.GetStorage().ReadDir("") if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } @@ -70,15 +71,12 @@ func initMigrationTest(t *testing.T) func() { if !ownerDir.Type().IsDir() { continue } - repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name())) + repoDirs, err := storage.GetStorage().ReadDir(ownerDir.Name()) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755) + storage.GetStorage().MakeDir(path.Join(ownerDir.Name(), repoDir.Name())) } } From 47701604d6b29c5fe442555d2972d48042c75879 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 26 Mar 2023 01:42:16 +0800 Subject: [PATCH 4/7] adjustment --- models/migrations/base/tests.go | 4 +-- models/unittest/testdb.go | 6 ++-- modules/git/storage/storage.go | 34 +++++++++++++++------ {models/unittest => modules/util}/fscopy.go | 15 ++++----- modules/util/legacy.go | 31 ------------------- tests/test_utils.go | 27 +++++++--------- 6 files changed, 46 insertions(+), 71 deletions(-) rename {models/unittest => modules/util}/fscopy.go (83%) diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index 0f05bafaa762a..b20e00e23b88a 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -33,8 +33,8 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En ourSkip := 2 ourSkip += skip deferFn := PrintCurrentTest(t, ourSkip) - assert.NoError(t, storage.GetStorage().RemoveAll()) - assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath)) + assert.NoError(t, storage.GetStorage().RemoveAllRepos()) + assert.NoError(t, storage.GetStorage().CopyDir(filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) ownerDirs, err := storage.GetStorage().ReadDir("") if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index ad6a010859fa0..0068efd904ab7 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -122,7 +122,7 @@ func MainTest(m *testing.M, testOpts *TestOptions) { if err = util.RemoveAll(repoRootPath); err != nil { fatalTestError("util.RemoveAll: %v\n", err) } - if err = CopyDir(filepath.Join(testOpts.GiteaRootPath, "tests", "gitea-repositories-meta"), setting.RepoRootPath); err != nil { + if err = util.CopyDir(filepath.Join(testOpts.GiteaRootPath, "tests", "gitea-repositories-meta"), setting.RepoRootPath); err != nil { fatalTestError("util.CopyDir: %v\n", err) } @@ -204,9 +204,9 @@ func PrepareTestDatabase() error { // by tests that use the above MainTest(..) function. func PrepareTestEnv(t testing.TB) { assert.NoError(t, PrepareTestDatabase()) - assert.NoError(t, storage.GetStorage().RemoveAll()) + assert.NoError(t, storage.GetStorage().RemoveAllRepos()) metaPath := filepath.Join(giteaRoot, "tests", "gitea-repositories-meta") - assert.NoError(t, CopyDir(metaPath, setting.RepoRootPath)) + assert.NoError(t, storage.GetStorage().CopyDir(metaPath, "")) ownerDirs, err := storage.GetStorage().ReadDir("") assert.NoError(t, err) for _, ownerDir := range ownerDirs { diff --git a/modules/git/storage/storage.go b/modules/git/storage/storage.go index 4eeade23f1c46..086053c131961 100644 --- a/modules/git/storage/storage.go +++ b/modules/git/storage/storage.go @@ -9,34 +9,48 @@ import ( "path/filepath" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) type Storage interface { MakeDir(repoRelPath string) error - RemoveAll() error + RemoveAllRepos() error ReadDir(owner string) ([]fs.DirEntry, error) + CopyDir(source, target string) error } -type LocalStorage struct{} +type LocalStorage struct { + repoRootPath string +} var _ Storage = &LocalStorage{} +func (l *LocalStorage) absPath(relPath string) string { + return filepath.Join(l.repoRootPath, relPath) +} + func (l *LocalStorage) MakeDir(repoRelPath string) error { - _ = os.MkdirAll(filepath.Join(absPath(repoRelPath), "objects", "pack"), 0o755) - _ = os.MkdirAll(filepath.Join(absPath(repoRelPath), "objects", "info"), 0o755) - _ = os.MkdirAll(filepath.Join(absPath(repoRelPath), "refs", "heads"), 0o755) - _ = os.MkdirAll(filepath.Join(absPath(repoRelPath), "refs", "tag"), 0o755) + _ = os.MkdirAll(filepath.Join(l.absPath(repoRelPath), "objects", "pack"), 0o755) + _ = os.MkdirAll(filepath.Join(l.absPath(repoRelPath), "objects", "info"), 0o755) + _ = os.MkdirAll(filepath.Join(l.absPath(repoRelPath), "refs", "heads"), 0o755) + _ = os.MkdirAll(filepath.Join(l.absPath(repoRelPath), "refs", "tag"), 0o755) return nil } -func (l *LocalStorage) RemoveAll() error { - return os.RemoveAll(setting.RepoRootPath) +func (l *LocalStorage) RemoveAllRepos() error { + return os.RemoveAll(l.absPath("")) } func (l *LocalStorage) ReadDir(owner string) ([]fs.DirEntry, error) { - return os.ReadDir(absPath(owner)) + return os.ReadDir(l.absPath(owner)) +} + +func (l *LocalStorage) CopyDir(source, target string) error { + return util.CopyDir(source, l.absPath(target)) } func GetStorage() Storage { - return &LocalStorage{} + return &LocalStorage{ + repoRootPath: setting.RepoRootPath, + } } diff --git a/models/unittest/fscopy.go b/modules/util/fscopy.go similarity index 83% rename from models/unittest/fscopy.go rename to modules/util/fscopy.go index 74b12d5057791..408abcb751c50 100644 --- a/models/unittest/fscopy.go +++ b/modules/util/fscopy.go @@ -1,7 +1,7 @@ // Copyright 2022 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package unittest +package util import ( "errors" @@ -9,12 +9,10 @@ import ( "os" "path" "strings" - - "code.gitea.io/gitea/modules/util" ) -// Copy copies file from source to target path. -func Copy(src, dest string) error { +// CopyFile copies file from source to target path. +func CopyFile(src, dest string) error { // Gather file information to set back later. si, err := os.Lstat(src) if err != nil { @@ -48,7 +46,6 @@ func Copy(src, dest string) error { return err } - // Set back file information. if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil { return err } @@ -64,7 +61,7 @@ func Copy(src, dest string) error { func CopyDir(srcPath, destPath string, filters ...func(filePath string) bool) error { // Check if target directory exists. if _, err := os.Stat(destPath); !errors.Is(err, os.ErrNotExist) { - return util.NewAlreadyExistErrorf("file or directory already exists: %s", destPath) + return NewAlreadyExistErrorf("file or directory already exists: %s", destPath) } err := os.MkdirAll(destPath, os.ModePerm) @@ -73,7 +70,7 @@ func CopyDir(srcPath, destPath string, filters ...func(filePath string) bool) er } // Gather directory info. - infos, err := util.StatDir(srcPath, true) + infos, err := StatDir(srcPath, true) if err != nil { return err } @@ -92,7 +89,7 @@ func CopyDir(srcPath, destPath string, filters ...func(filePath string) bool) er if strings.HasSuffix(info, "/") { err = os.MkdirAll(curPath, os.ModePerm) } else { - err = Copy(path.Join(srcPath, info), curPath) + err = CopyFile(path.Join(srcPath, info), curPath) } if err != nil { return err diff --git a/modules/util/legacy.go b/modules/util/legacy.go index 2ea293a2be856..d0af2d6bd6ad7 100644 --- a/modules/util/legacy.go +++ b/modules/util/legacy.go @@ -8,39 +8,8 @@ import ( "crypto/cipher" "crypto/rand" "errors" - "io" - "os" ) -// CopyFile copies file from source to target path. -func CopyFile(src, dest string) error { - si, err := os.Lstat(src) - if err != nil { - return err - } - - sr, err := os.Open(src) - if err != nil { - return err - } - defer sr.Close() - - dw, err := os.Create(dest) - if err != nil { - return err - } - defer dw.Close() - - if _, err = io.Copy(dw, sr); err != nil { - return err - } - - if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil { - return err - } - return os.Chmod(dest, si.Mode()) -} - // AESGCMEncrypt (from legacy package): encrypts plaintext with the given key using AES in GCM mode. should be replaced. func AESGCMEncrypt(key, plaintext []byte) ([]byte, error) { block, err := aes.NewCipher(key) diff --git a/tests/test_utils.go b/tests/test_utils.go index 102dd3d298fb0..c0bc789971a40 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/git" + git_storage "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/queue" @@ -174,9 +175,9 @@ func PrepareTestEnv(t testing.TB, skip ...int) func() { assert.NoError(t, unittest.LoadFixtures()) // load git repo fixtures - assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath)) - ownerDirs, err := os.ReadDir(setting.RepoRootPath) + assert.NoError(t, git_storage.GetStorage().RemoveAllRepos()) + assert.NoError(t, git_storage.GetStorage().CopyDir(filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) + ownerDirs, err := git_storage.GetStorage().ReadDir("") if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } @@ -184,15 +185,12 @@ func PrepareTestEnv(t testing.TB, skip ...int) func() { if !ownerDir.Type().IsDir() { continue } - repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name())) + repoDirs, err := git_storage.GetStorage().ReadDir(ownerDir.Name()) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755) + git_storage.GetStorage().MakeDir(path.Join(ownerDir.Name(), repoDir.Name())) } } @@ -231,9 +229,9 @@ func ResetFixtures(t *testing.T) { assert.NoError(t, unittest.LoadFixtures()) // load git repo fixtures - assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath)) - ownerDirs, err := os.ReadDir(setting.RepoRootPath) + assert.NoError(t, git_storage.GetStorage().RemoveAllRepos()) + assert.NoError(t, git_storage.GetStorage().CopyDir(filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) + ownerDirs, err := git_storage.GetStorage().ReadDir("") if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } @@ -241,15 +239,12 @@ func ResetFixtures(t *testing.T) { if !ownerDir.Type().IsDir() { continue } - repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name())) + repoDirs, err := git_storage.GetStorage().ReadDir(ownerDir.Name()) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755) + _ = git_storage.GetStorage().MakeDir(path.Join(ownerDir.Name(), repoDir.Name())) } } From 3ec3a230a07ca67d48c627bdcdb45310eb297404 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 26 Mar 2023 11:23:21 +0800 Subject: [PATCH 5/7] fix --- tests/integration/migration-test/migration_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/migration-test/migration_test.go b/tests/integration/migration-test/migration_test.go index c5fd0398eb58b..740467c2763c3 100644 --- a/tests/integration/migration-test/migration_test.go +++ b/tests/integration/migration-test/migration_test.go @@ -20,7 +20,6 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/migrations" migrate_base "code.gitea.io/gitea/models/migrations/base" - "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" @@ -61,8 +60,8 @@ func initMigrationTest(t *testing.T) func() { setting.InitProviderAndLoadCommonSettingsForTest() assert.True(t, len(setting.RepoRootPath) != 0) - assert.NoError(t, storage.GetStorage().RemoveAll()) - assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath)) + assert.NoError(t, storage.GetStorage().RemoveAllRepos()) + assert.NoError(t, storage.GetStorage().CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) ownerDirs, err := storage.GetStorage().ReadDir("") if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) From 8f7d44436a30e95895a267165fabcd65b6b7eab1 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 26 Mar 2023 20:13:13 +0800 Subject: [PATCH 6/7] Move more related code to git storage --- models/migrations/base/tests.go | 12 +++--- models/repo/wiki_test.go | 3 +- models/unittest/testdb.go | 19 ++++++---- modules/git/storage/legacy.go | 5 --- modules/git/storage/storage.go | 37 ++++++++++++++++++- .../migration-test/migration_test.go | 12 +++--- 6 files changed, 61 insertions(+), 27 deletions(-) diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index b20e00e23b88a..3f905a4c2fb3c 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -33,9 +33,9 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En ourSkip := 2 ourSkip += skip deferFn := PrintCurrentTest(t, ourSkip) - assert.NoError(t, storage.GetStorage().RemoveAllRepos()) - assert.NoError(t, storage.GetStorage().CopyDir(filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) - ownerDirs, err := storage.GetStorage().ReadDir("") + assert.NoError(t, storage.RemoveAllRepos()) + assert.NoError(t, storage.CopyDir(filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) + ownerDirs, err := storage.ReadDir("") if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } @@ -43,12 +43,12 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En if !ownerDir.Type().IsDir() { continue } - repoDirs, err := storage.GetStorage().ReadDir(ownerDir.Name()) + repoDirs, err := storage.ReadDir(ownerDir.Name()) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - storage.GetStorage().MakeDir(path.Join(ownerDir.Name(), repoDir.Name())) + storage.MakeDir(path.Join(ownerDir.Name(), repoDir.Name())) } } @@ -157,7 +157,7 @@ func MainTest(m *testing.M) { exitStatus := m.Run() - if err := removeAllWithRetry(setting.RepoRootPath); err != nil { + if err := storage.RemoveAllRepos(); err != nil { fmt.Fprintf(os.Stderr, "os.RemoveAll: %v\n", err) } if err := removeAllWithRetry(tmpDataPath); err != nil { diff --git a/models/repo/wiki_test.go b/models/repo/wiki_test.go index 629986f741a5e..f3a44663d6c34 100644 --- a/models/repo/wiki_test.go +++ b/models/repo/wiki_test.go @@ -9,6 +9,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" @@ -26,7 +27,7 @@ func TestRepository_WikiCloneLink(t *testing.T) { func TestWikiPath(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) expected := filepath.Join(setting.RepoRootPath, "user2/repo1.wiki.git") - assert.Equal(t, expected, repo_model.WikiPath("user2", "repo1")) + assert.Equal(t, expected, storage.WikiPath("user2", "repo1")) } func TestRepository_WikiPath(t *testing.T) { diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index 0068efd904ab7..90c4de5a9f082 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -129,7 +129,7 @@ func MainTest(m *testing.M, testOpts *TestOptions) { if err = git.InitFull(context.Background()); err != nil { fatalTestError("git.Init: %v\n", err) } - ownerDirs, err := storage.GetStorage().ReadDir("") + ownerDirs, err := storage.ReadDir("") if err != nil { fatalTestError("unable to read the new repo root: %v\n", err) } @@ -137,12 +137,14 @@ func MainTest(m *testing.M, testOpts *TestOptions) { if !ownerDir.Type().IsDir() { continue } - repoDirs, err := storage.GetStorage().ReadDir(ownerDir.Name()) + repoDirs, err := storage.ReadDir(ownerDir.Name()) if err != nil { fatalTestError("unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - storage.GetStorage().MakeDir(path.Join(ownerDir.Name(), repoDir.Name())) + if err := storage.MakeDir(path.Join(ownerDir.Name(), repoDir.Name())); err != nil { + fatalTestError("create directories failed: %v\n", err) + } } } @@ -204,19 +206,20 @@ func PrepareTestDatabase() error { // by tests that use the above MainTest(..) function. func PrepareTestEnv(t testing.TB) { assert.NoError(t, PrepareTestDatabase()) - assert.NoError(t, storage.GetStorage().RemoveAllRepos()) + assert.NoError(t, storage.RemoveAllRepos()) metaPath := filepath.Join(giteaRoot, "tests", "gitea-repositories-meta") - assert.NoError(t, storage.GetStorage().CopyDir(metaPath, "")) - ownerDirs, err := storage.GetStorage().ReadDir("") + assert.NoError(t, storage.CopyDir(metaPath, "")) + ownerDirs, err := storage.ReadDir("") assert.NoError(t, err) for _, ownerDir := range ownerDirs { if !ownerDir.Type().IsDir() { continue } - repoDirs, err := storage.GetStorage().ReadDir(ownerDir.Name()) + repoDirs, err := storage.ReadDir(ownerDir.Name()) assert.NoError(t, err) for _, repoDir := range repoDirs { - storage.GetStorage().MakeDir(path.Join(ownerDir.Name(), repoDir.Name())) + err = storage.MakeDir(path.Join(ownerDir.Name(), repoDir.Name())) + assert.NoError(t, err) } } diff --git a/modules/git/storage/legacy.go b/modules/git/storage/legacy.go index b628384fe3138..6a4c9079c201f 100644 --- a/modules/git/storage/legacy.go +++ b/modules/git/storage/legacy.go @@ -8,7 +8,6 @@ import ( "strings" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" ) func absPath(path string) string { @@ -29,7 +28,3 @@ func RepoPath(userName, repoName string) string { //revive:disable-line:exported func WikiPath(userName, repoName string) string { return filepath.Join(UserPath(userName), strings.ToLower(repoName)+".wiki.git") } - -func Rename(oldPath, newPath string) error { - return util.Rename(absPath(oldPath), absPath(newPath)) -} diff --git a/modules/git/storage/storage.go b/modules/git/storage/storage.go index 086053c131961..7c44fc0b4dfe2 100644 --- a/modules/git/storage/storage.go +++ b/modules/git/storage/storage.go @@ -13,10 +13,12 @@ import ( ) type Storage interface { + IsValid() bool MakeDir(repoRelPath string) error RemoveAllRepos() error ReadDir(owner string) ([]fs.DirEntry, error) CopyDir(source, target string) error + Rename(oldPath, newPath string) error } type LocalStorage struct { @@ -29,6 +31,10 @@ func (l *LocalStorage) absPath(relPath string) string { return filepath.Join(l.repoRootPath, relPath) } +func (l *LocalStorage) IsValid() bool { + return len(l.repoRootPath) != 0 +} + func (l *LocalStorage) MakeDir(repoRelPath string) error { _ = os.MkdirAll(filepath.Join(l.absPath(repoRelPath), "objects", "pack"), 0o755) _ = os.MkdirAll(filepath.Join(l.absPath(repoRelPath), "objects", "info"), 0o755) @@ -38,6 +44,7 @@ func (l *LocalStorage) MakeDir(repoRelPath string) error { } func (l *LocalStorage) RemoveAllRepos() error { + // removeAllWithRetry(setting.RepoRootPath) return os.RemoveAll(l.absPath("")) } @@ -49,8 +56,36 @@ func (l *LocalStorage) CopyDir(source, target string) error { return util.CopyDir(source, l.absPath(target)) } -func GetStorage() Storage { +func (l *LocalStorage) Rename(oldPath, newPath string) error { + return util.Rename(l.absPath(oldPath), l.absPath(newPath)) +} + +func getStorage() Storage { return &LocalStorage{ repoRootPath: setting.RepoRootPath, } } + +func IsReposValid() bool { + return getStorage().IsValid() +} + +func MakeDir(repoRelPath string) error { + return getStorage().MakeDir(repoRelPath) +} + +func RemoveAllRepos() error { + return getStorage().RemoveAllRepos() +} + +func ReadDir(owner string) ([]fs.DirEntry, error) { + return getStorage().ReadDir(owner) +} + +func CopyDir(source, target string) error { + return getStorage().CopyDir(source, target) +} + +func Rename(oldPath, newPath string) error { + return getStorage().Rename(oldPath, newPath) +} diff --git a/tests/integration/migration-test/migration_test.go b/tests/integration/migration-test/migration_test.go index 740467c2763c3..192650a37802b 100644 --- a/tests/integration/migration-test/migration_test.go +++ b/tests/integration/migration-test/migration_test.go @@ -59,10 +59,10 @@ func initMigrationTest(t *testing.T) func() { setting.InitProviderAndLoadCommonSettingsForTest() - assert.True(t, len(setting.RepoRootPath) != 0) - assert.NoError(t, storage.GetStorage().RemoveAllRepos()) - assert.NoError(t, storage.GetStorage().CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) - ownerDirs, err := storage.GetStorage().ReadDir("") + assert.True(t, ) + assert.NoError(t, storage.RemoveAllRepos()) + assert.NoError(t, storage.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) + ownerDirs, err := storage.ReadDir("") if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } @@ -70,12 +70,12 @@ func initMigrationTest(t *testing.T) func() { if !ownerDir.Type().IsDir() { continue } - repoDirs, err := storage.GetStorage().ReadDir(ownerDir.Name()) + repoDirs, err := storage.ReadDir(ownerDir.Name()) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - storage.GetStorage().MakeDir(path.Join(ownerDir.Name(), repoDir.Name())) + storage.MakeDir(path.Join(ownerDir.Name(), repoDir.Name())) } } From 79b1df5587886efada9314305708e7abb0fa34d6 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 27 Mar 2023 15:22:16 +0800 Subject: [PATCH 7/7] Move more related code to git storage --- cmd/serv.go | 3 +- models/migrations/base/tests.go | 6 ++-- models/repo/update.go | 14 ++++---- models/repo/wiki_test.go | 3 +- models/repo_transfer.go | 26 +++++++-------- models/unittest/testdb.go | 10 +++--- modules/git/storage/legacy.go | 18 ++++++++++- modules/git/storage/storage.go | 57 +++++++++++++++++++++++++-------- modules/repository/repo.go | 12 ++++--- services/org/org.go | 5 ++- services/user/user.go | 5 ++- 11 files changed, 102 insertions(+), 57 deletions(-) diff --git a/cmd/serv.go b/cmd/serv.go index 70b2937cc25d6..190875c52e1df 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -19,6 +19,7 @@ import ( git_model "code.gitea.io/gitea/models/git" "code.gitea.io/gitea/models/perm" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/pprof" @@ -67,7 +68,7 @@ func setup(logPath string, debug bool) { // Check if setting.RepoRootPath exists. It could be the case that it doesn't exist, this can happen when // `[repository]` `ROOT` is a relative path and $GITEA_WORK_DIR isn't passed to the SSH connection. - if _, err := os.Stat(setting.RepoRootPath); err != nil { + if err := storage.CheckStats(); err != nil { if os.IsNotExist(err) { _ = fail("Incorrect configuration, no repository directory.", "Directory `[repository].ROOT` %q was not found, please check if $GITEA_WORK_DIR is passed to the SSH connection or make `[repository].ROOT` an absolute value.", setting.RepoRootPath) } else { diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index 3f905a4c2fb3c..6ab205c5320ab 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -33,7 +33,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En ourSkip := 2 ourSkip += skip deferFn := PrintCurrentTest(t, ourSkip) - assert.NoError(t, storage.RemoveAllRepos()) + assert.NoError(t, storage.RemoveAll("")) assert.NoError(t, storage.CopyDir(filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) ownerDirs, err := storage.ReadDir("") if err != nil { @@ -48,7 +48,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - storage.MakeDir(path.Join(ownerDir.Name(), repoDir.Name())) + storage.MakeRepoDir(path.Join(ownerDir.Name(), repoDir.Name())) } } @@ -157,7 +157,7 @@ func MainTest(m *testing.M) { exitStatus := m.Run() - if err := storage.RemoveAllRepos(); err != nil { + if err := storage.RemoveAll(""); err != nil { fmt.Fprintf(os.Stderr, "os.RemoveAll: %v\n", err) } if err := removeAllWithRetry(tmpDataPath); err != nil { diff --git a/models/repo/update.go b/models/repo/update.go index fecfa4ad88693..c15210f84932f 100644 --- a/models/repo/update.go +++ b/models/repo/update.go @@ -124,10 +124,10 @@ func CheckCreateRepository(doer, u *user_model.User, name string, overwriteOrAdo return ErrRepoAlreadyExist{u.Name, name} } - repoPath := storage.RepoPath(u.Name, name) - isExist, err := util.IsExist(repoPath) + repoRelPath := storage.RepoRelPath(u.Name, name) + isExist, err := storage.IsExist(repoRelPath) if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) + log.Error("Unable to check if %s exists. Error: %v", repoRelPath, err) return err } if !overwriteOrAdopt && isExist { @@ -160,14 +160,14 @@ func ChangeRepositoryName(doer *user_model.User, repo *Repository, newRepoName s return fmt.Errorf("rename repository directory: %w", err) } - wikiPath := repo.WikiPath() - isExist, err := util.IsExist(wikiPath) + wikiRelPath := storage.WikiRelPath(repo.OwnerName, repo.Name) + isExist, err := storage.IsExist(wikiRelPath) if err != nil { - log.Error("Unable to check if %s exists. Error: %v", wikiPath, err) + log.Error("Unable to check if %s exists. Error: %v", wikiRelPath, err) return err } if isExist { - if err = util.Rename(wikiPath, storage.WikiPath(repo.Owner.Name, newRepoName)); err != nil { + if err = storage.Rename(wikiRelPath, storage.WikiRelPath(repo.Owner.Name, newRepoName)); err != nil { return fmt.Errorf("rename repository wiki: %w", err) } } diff --git a/models/repo/wiki_test.go b/models/repo/wiki_test.go index f3a44663d6c34..c7255e2e76b48 100644 --- a/models/repo/wiki_test.go +++ b/models/repo/wiki_test.go @@ -26,8 +26,7 @@ func TestRepository_WikiCloneLink(t *testing.T) { func TestWikiPath(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - expected := filepath.Join(setting.RepoRootPath, "user2/repo1.wiki.git") - assert.Equal(t, expected, storage.WikiPath("user2", "repo1")) + assert.Equal(t, "user2/repo1.wiki.git", storage.WikiRelPath("user2", "repo1")) } func TestRepository_WikiPath(t *testing.T) { diff --git a/models/repo_transfer.go b/models/repo_transfer.go index aefad7485b97a..de8c1809c57b9 100644 --- a/models/repo_transfer.go +++ b/models/repo_transfer.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "os" + "strings" "code.gitea.io/gitea/models/db" issues_model "code.gitea.io/gitea/models/issues" @@ -17,7 +18,6 @@ import ( "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" ) // RepoTransfer is used to manage repository transfers @@ -216,16 +216,16 @@ func TransferOwnership(doer *user_model.User, newOwnerName string, repo *repo_mo } if repoRenamed { - if err := util.Rename(storage.RepoPath(newOwnerName, repo.Name), storage.RepoPath(oldOwnerName, repo.Name)); err != nil { + if err := storage.Rename(storage.RepoRelPath(newOwnerName, repo.Name), storage.RepoRelPath(oldOwnerName, repo.Name)); err != nil { log.Critical("Unable to move repository %s/%s directory from %s back to correct place %s: %v", oldOwnerName, repo.Name, - storage.RepoPath(newOwnerName, repo.Name), storage.RepoPath(oldOwnerName, repo.Name), err) + storage.RepoRelPath(newOwnerName, repo.Name), storage.RepoRelPath(oldOwnerName, repo.Name), err) } } if wikiRenamed { - if err := util.Rename(storage.WikiPath(newOwnerName, repo.Name), storage.WikiPath(oldOwnerName, repo.Name)); err != nil { + if err := storage.Rename(storage.WikiRelPath(newOwnerName, repo.Name), storage.WikiRelPath(oldOwnerName, repo.Name)); err != nil { log.Critical("Unable to move wiki for repository %s/%s directory from %s back to correct place %s: %v", oldOwnerName, repo.Name, - storage.WikiPath(newOwnerName, repo.Name), storage.WikiPath(oldOwnerName, repo.Name), err) + storage.WikiRelPath(newOwnerName, repo.Name), storage.WikiRelPath(oldOwnerName, repo.Name), err) } } @@ -375,25 +375,25 @@ func TransferOwnership(doer *user_model.User, newOwnerName string, repo *repo_mo } // Rename remote repository to new path and delete local copy. - dir := storage.UserPath(newOwner.Name) + userRelPath := strings.ToLower(newOwner.Name) - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return fmt.Errorf("Failed to create dir %s: %w", dir, err) + if err := storage.MakeDir(userRelPath, os.ModePerm); err != nil { + return fmt.Errorf("Failed to create dir %s: %w", userRelPath, err) } - if err := util.Rename(storage.RepoPath(oldOwner.Name, repo.Name), storage.RepoPath(newOwner.Name, repo.Name)); err != nil { + if err := storage.Rename(storage.RepoRelPath(oldOwner.Name, repo.Name), storage.RepoRelPath(newOwner.Name, repo.Name)); err != nil { return fmt.Errorf("rename repository directory: %w", err) } repoRenamed = true // Rename remote wiki repository to new path and delete local copy. - wikiPath := storage.WikiPath(oldOwner.Name, repo.Name) + wikiRelPath := storage.WikiRelPath(oldOwner.Name, repo.Name) - if isExist, err := util.IsExist(wikiPath); err != nil { - log.Error("Unable to check if %s exists. Error: %v", wikiPath, err) + if isExist, err := storage.IsExist(wikiRelPath); err != nil { + log.Error("Unable to check if %s exists. Error: %v", wikiRelPath, err) return err } else if isExist { - if err := util.Rename(wikiPath, storage.WikiPath(newOwner.Name, repo.Name)); err != nil { + if err := storage.Rename(wikiRelPath, storage.WikiRelPath(newOwner.Name, repo.Name)); err != nil { return fmt.Errorf("rename repository wiki: %w", err) } wikiRenamed = true diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index 90c4de5a9f082..1d911c1dfd323 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -119,10 +119,10 @@ func MainTest(m *testing.M, testOpts *TestOptions) { fatalTestError("models.Init: %v\n", err) } - if err = util.RemoveAll(repoRootPath); err != nil { + if err = storage.RemoveAll(""); err != nil { fatalTestError("util.RemoveAll: %v\n", err) } - if err = util.CopyDir(filepath.Join(testOpts.GiteaRootPath, "tests", "gitea-repositories-meta"), setting.RepoRootPath); err != nil { + if err = storage.CopyDir(filepath.Join(testOpts.GiteaRootPath, "tests", "gitea-repositories-meta"), ""); err != nil { fatalTestError("util.CopyDir: %v\n", err) } @@ -142,7 +142,7 @@ func MainTest(m *testing.M, testOpts *TestOptions) { fatalTestError("unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - if err := storage.MakeDir(path.Join(ownerDir.Name(), repoDir.Name())); err != nil { + if err := storage.MakeRepoDir(path.Join(ownerDir.Name(), repoDir.Name())); err != nil { fatalTestError("create directories failed: %v\n", err) } } @@ -206,7 +206,7 @@ func PrepareTestDatabase() error { // by tests that use the above MainTest(..) function. func PrepareTestEnv(t testing.TB) { assert.NoError(t, PrepareTestDatabase()) - assert.NoError(t, storage.RemoveAllRepos()) + assert.NoError(t, storage.RemoveAll("")) metaPath := filepath.Join(giteaRoot, "tests", "gitea-repositories-meta") assert.NoError(t, storage.CopyDir(metaPath, "")) ownerDirs, err := storage.ReadDir("") @@ -218,7 +218,7 @@ func PrepareTestEnv(t testing.TB) { repoDirs, err := storage.ReadDir(ownerDir.Name()) assert.NoError(t, err) for _, repoDir := range repoDirs { - err = storage.MakeDir(path.Join(ownerDir.Name(), repoDir.Name())) + err = storage.MakeRepoDir(path.Join(ownerDir.Name(), repoDir.Name())) assert.NoError(t, err) } } diff --git a/modules/git/storage/legacy.go b/modules/git/storage/legacy.go index 6a4c9079c201f..b59590b1e2dcf 100644 --- a/modules/git/storage/legacy.go +++ b/modules/git/storage/legacy.go @@ -4,6 +4,7 @@ package storage import ( + "path" "path/filepath" "strings" @@ -14,17 +15,32 @@ func absPath(path string) string { return filepath.Join(setting.RepoRootPath, path) } +// UserRelPath returns the path relative path of user repositories. +func UserRelPath(userName string) string { + return strings.ToLower(userName) +} + // UserPath returns the path absolute path of user repositories. func UserPath(userName string) string { //revive:disable-line:exported return absPath(strings.ToLower(userName)) } +// RepoRelPath returns repository relative path by given user and repository name. +func RepoRelPath(userName, repoName string) string { + return path.Join(strings.ToLower(userName), strings.ToLower(repoName)+".git") +} + // RepoPath returns repository path by given user and repository name. func RepoPath(userName, repoName string) string { //revive:disable-line:exported return filepath.Join(UserPath(userName), strings.ToLower(repoName)+".git") } +// WikiRelPath returns wiki repository relative path by given user and repository name. +func WikiRelPath(userName, repoName string) string { + return path.Join(strings.ToLower(userName), strings.ToLower(repoName)+".wiki.git") +} + // WikiPath returns wiki data path by given user and repository name. func WikiPath(userName, repoName string) string { - return filepath.Join(UserPath(userName), strings.ToLower(repoName)+".wiki.git") + return filepath.Join(UserPath(userName)) } diff --git a/modules/git/storage/storage.go b/modules/git/storage/storage.go index 7c44fc0b4dfe2..00ca2313ae03b 100644 --- a/modules/git/storage/storage.go +++ b/modules/git/storage/storage.go @@ -13,9 +13,13 @@ import ( ) type Storage interface { - IsValid() bool - MakeDir(repoRelPath string) error - RemoveAllRepos() error + IsConfigured() bool + CheckStats() error + IsExist(path string) (bool, error) + // MakeDir(repoRelPath string) error + MakeDir(dir string, perm os.FileMode) error + MakeRepoDir(repoRelPath string) error + RemoveAll(path string) error ReadDir(owner string) ([]fs.DirEntry, error) CopyDir(source, target string) error Rename(oldPath, newPath string) error @@ -31,11 +35,24 @@ func (l *LocalStorage) absPath(relPath string) string { return filepath.Join(l.repoRootPath, relPath) } -func (l *LocalStorage) IsValid() bool { +func (l *LocalStorage) IsConfigured() bool { return len(l.repoRootPath) != 0 } -func (l *LocalStorage) MakeDir(repoRelPath string) error { +func (l *LocalStorage) CheckStats() error { + _, err := os.Stat(l.repoRootPath) + return err +} + +func (l *LocalStorage) IsExist(path string) (bool, error) { + return util.IsExist(path) +} + +func (l *LocalStorage) MakeDir(dir string, perm os.FileMode) error { + return os.MkdirAll(l.absPath(dir), perm) +} + +func (l *LocalStorage) MakeRepoDir(repoRelPath string) error { _ = os.MkdirAll(filepath.Join(l.absPath(repoRelPath), "objects", "pack"), 0o755) _ = os.MkdirAll(filepath.Join(l.absPath(repoRelPath), "objects", "info"), 0o755) _ = os.MkdirAll(filepath.Join(l.absPath(repoRelPath), "refs", "heads"), 0o755) @@ -43,9 +60,9 @@ func (l *LocalStorage) MakeDir(repoRelPath string) error { return nil } -func (l *LocalStorage) RemoveAllRepos() error { - // removeAllWithRetry(setting.RepoRootPath) - return os.RemoveAll(l.absPath("")) +func (l *LocalStorage) RemoveAll(path string) error { + // TODO: removeAllWithRetry(l.absPath(path)) + return os.RemoveAll(l.absPath(path)) } func (l *LocalStorage) ReadDir(owner string) ([]fs.DirEntry, error) { @@ -66,16 +83,28 @@ func getStorage() Storage { } } -func IsReposValid() bool { - return getStorage().IsValid() +func IsConfigured() bool { + return getStorage().IsConfigured() +} + +func CheckStats() error { + return getStorage().CheckStats() +} + +func IsExist(path string) (bool, error) { + return getStorage().IsExist(path) +} + +func MakeDir(dir string, perm os.FileMode) error { + return getStorage().MakeDir(dir, perm) } -func MakeDir(repoRelPath string) error { - return getStorage().MakeDir(repoRelPath) +func MakeRepoDir(repoRelPath string) error { + return getStorage().MakeRepoDir(repoRelPath) } -func RemoveAllRepos() error { - return getStorage().RemoveAllRepos() +func RemoveAll(p string) error { + return getStorage().RemoveAll(p) } func ReadDir(owner string) ([]fs.DirEntry, error) { diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 7629c8fd957fe..ea72b0bb65531 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -91,13 +91,15 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } if opts.Wiki { - wikiPath := storage.WikiPath(u.Name, opts.RepoName) + wikiRelPath := storage.WikiRelPath(u.Name, opts.RepoName) wikiRemotePath := WikiRemoteURL(ctx, opts.CloneAddr) if len(wikiRemotePath) > 0 { - if err := util.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) + if err := storage.RemoveAll(wikiRelPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %w", wikiRelPath, err) } + // FIXME: use storage + wikiPath := storage.WikiPath(u.Name, opts.RepoName) if err := git.Clone(ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, @@ -106,8 +108,8 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, SkipTLSVerify: setting.Migrations.SkipTLSVerify, }); err != nil { log.Warn("Clone wiki: %v", err) - if err := util.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) + if err := storage.RemoveAll(wikiRelPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %w", wikiRelPath, err) } } else { if err := git.WriteCommitGraph(ctx, wikiPath); err != nil { diff --git a/services/org/org.go b/services/org/org.go index 7f0446712bcad..1b44243a06f33 100644 --- a/services/org/org.go +++ b/services/org/org.go @@ -13,7 +13,6 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git/storage" object_storage "code.gitea.io/gitea/modules/storage" - "code.gitea.io/gitea/modules/util" ) // DeleteOrganization completely and permanently deletes everything of organization. @@ -50,9 +49,9 @@ func DeleteOrganization(org *organization.Organization) error { // FIXME: system notice // Note: There are something just cannot be roll back, // so just keep error logs of those operations. - path := storage.UserPath(org.Name) + path := storage.UserRelPath(org.Name) - if err := util.RemoveAll(path); err != nil { + if err := storage.RemoveAll(path); err != nil { return fmt.Errorf("Failed to RemoveAll %s: %w", path, err) } diff --git a/services/user/user.go b/services/user/user.go index 73e5419a79237..972c909a3b95c 100644 --- a/services/user/user.go +++ b/services/user/user.go @@ -24,7 +24,6 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" object_storage "code.gitea.io/gitea/modules/storage" - "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/packages" ) @@ -198,8 +197,8 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { // Note: There are something just cannot be roll back, // so just keep error logs of those operations. - path := storage.UserPath(u.Name) - if err := util.RemoveAll(path); err != nil { + path := storage.UserRelPath(u.Name) + if err := storage.RemoveAll(path); err != nil { err = fmt.Errorf("Failed to RemoveAll %s: %w", path, err) _ = system_model.CreateNotice(ctx, system_model.NoticeTask, fmt.Sprintf("delete user '%s': %v", u.Name, err)) return err