From 34b7aababfc486f91fba217a1f8203df26c0ffdf Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 4 Jun 2023 22:18:22 +0800 Subject: [PATCH 01/10] Reduce using reporootpath directly --- models/migrations/base/tests.go | 16 +- models/migrations/v1_12/v128.go | 6 +- models/migrations/v1_12/v134.go | 6 +- models/migrations/v1_12/v136.go | 7 +- models/migrations/v1_14/v156.go | 14 +- models/repo/repo.go | 11 +- models/repo/update.go | 20 +-- models/repo/wiki.go | 13 +- models/repo/wiki_test.go | 8 +- models/repo_transfer.go | 27 ++-- models/unittest/testdb.go | 37 +++-- models/user/user.go | 6 - modules/context/api.go | 4 +- modules/context/repo.go | 7 +- modules/git/storage/legacy.go | 35 +++++ modules/git/storage/storage.go | 146 ++++++++++++++++++ modules/repository/create.go | 21 +-- modules/repository/init.go | 9 +- modules/repository/repo.go | 22 +-- {models/unittest => modules/util}/fscopy.go | 8 +- routers/api/v1/admin/adopt.go | 6 +- routers/api/v1/repo/file.go | 3 +- routers/api/v1/repo/pull.go | 5 +- routers/web/admin/repos.go | 4 +- routers/web/repo/blame.go | 4 +- routers/web/repo/http.go | 5 +- routers/web/user/setting/adopt.go | 8 +- routers/web/user/setting/profile.go | 3 +- services/org/org.go | 8 +- services/release/release_test.go | 7 +- services/repository/adopt.go | 18 ++- services/repository/adopt_test.go | 5 +- services/repository/files/content_test.go | 4 +- services/repository/fork.go | 10 +- services/repository/transfer_test.go | 6 +- services/user/user.go | 10 +- .../integration/api_repo_lfs_migrate_test.go | 4 +- .../migration-test/migration_test.go | 16 +- tests/integration/mirror_pull_test.go | 3 +- tests/integration/pull_merge_test.go | 5 +- tests/test_utils.go | 14 +- 41 files changed, 363 insertions(+), 208 deletions(-) create mode 100644 modules/git/storage/legacy.go create mode 100644 modules/git/storage/storage.go rename {models/unittest => modules/util}/fscopy.go (91%) diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index dd99a1eda280c..4c828339b01f7 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/testlogger" @@ -35,9 +36,9 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En ourSkip := 2 ourSkip += skip deferFn := testlogger.PrintCurrentTest(t, ourSkip) - assert.NoError(t, os.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath)) - ownerDirs, err := os.ReadDir(setting.RepoRootPath) + assert.NoError(t, storage.RemoveAll("")) + assert.NoError(t, storage.CopyDir(filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) + ownerDirs, err := storage.ReadDir("") if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } @@ -45,15 +46,12 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En if !ownerDir.Type().IsDir() { continue } - repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name())) + repoDirs, err := storage.ReadDir(ownerDir.Name()) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755) + assert.NoError(t, storage.MakeRepoDir(path.Join(ownerDir.Name(), repoDir.Name()))) } } @@ -160,7 +158,7 @@ func MainTest(m *testing.M) { exitStatus := m.Run() - if err := removeAllWithRetry(setting.RepoRootPath); err != nil { + if err := storage.RemoveAll(""); err != nil { fmt.Fprintf(os.Stderr, "os.RemoveAll: %v\n", err) } if err := removeAllWithRetry(tmpDataPath); err != nil { diff --git a/models/migrations/v1_12/v128.go b/models/migrations/v1_12/v128.go index 44d44a26c549f..b086067cfd43b 100644 --- a/models/migrations/v1_12/v128.go +++ b/models/migrations/v1_12/v128.go @@ -6,11 +6,11 @@ package v1_12 //nolint import ( "fmt" "math" - "path/filepath" "strings" "time" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -75,8 +75,8 @@ func FixMergeBase(x *xorm.Engine) error { log.Error("Missing base repo with id %d for PR ID %d", pr.BaseRepoID, pr.ID) continue } - userPath := filepath.Join(setting.RepoRootPath, strings.ToLower(baseRepo.OwnerName)) - repoPath := filepath.Join(userPath, strings.ToLower(baseRepo.Name)+".git") + + repoPath := storage.RepoPath(baseRepo.OwnerName, baseRepo.Name) gitRefName := fmt.Sprintf("refs/pull/%d/head", pr.Index) diff --git a/models/migrations/v1_12/v134.go b/models/migrations/v1_12/v134.go index 3d1c82f09e164..d005197e11922 100644 --- a/models/migrations/v1_12/v134.go +++ b/models/migrations/v1_12/v134.go @@ -6,11 +6,11 @@ package v1_12 //nolint import ( "fmt" "math" - "path/filepath" "strings" "time" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -74,8 +74,8 @@ func RefixMergeBase(x *xorm.Engine) error { log.Error("Missing base repo with id %d for PR ID %d", pr.BaseRepoID, pr.ID) continue } - userPath := filepath.Join(setting.RepoRootPath, strings.ToLower(baseRepo.OwnerName)) - repoPath := filepath.Join(userPath, strings.ToLower(baseRepo.Name)+".git") + + repoPath := storage.RepoPath(baseRepo.OwnerName, baseRepo.Name) gitRefName := fmt.Sprintf("refs/pull/%d/head", pr.Index) diff --git a/models/migrations/v1_12/v136.go b/models/migrations/v1_12/v136.go index 0cecba7be9395..5fbc6fd8ccb91 100644 --- a/models/migrations/v1_12/v136.go +++ b/models/migrations/v1_12/v136.go @@ -6,11 +6,10 @@ package v1_12 //nolint import ( "fmt" "math" - "path/filepath" - "strings" "time" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -85,8 +84,8 @@ func AddCommitDivergenceToPulls(x *xorm.Engine) error { log.Error("Missing base repo with id %d for PR ID %d", pr.BaseRepoID, pr.ID) continue } - userPath := filepath.Join(setting.RepoRootPath, strings.ToLower(baseRepo.OwnerName)) - repoPath := filepath.Join(userPath, strings.ToLower(baseRepo.Name)+".git") + + repoPath := storage.RepoPath(baseRepo.OwnerName, baseRepo.Name) gitRefName := fmt.Sprintf("refs/pull/%d/head", pr.Index) diff --git a/models/migrations/v1_14/v156.go b/models/migrations/v1_14/v156.go index 2cf4954a15f58..f75688b3cb737 100644 --- a/models/migrations/v1_14/v156.go +++ b/models/migrations/v1_14/v156.go @@ -5,25 +5,15 @@ package v1_14 //nolint import ( "fmt" - "path/filepath" "strings" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" "xorm.io/xorm" ) -// Copy paste from models/repo.go because we cannot import models package -func repoPath(userName, repoName string) string { - return filepath.Join(userPath(userName), strings.ToLower(repoName)+".git") -} - -func userPath(userName string) string { - return filepath.Join(setting.RepoRootPath, strings.ToLower(userName)) -} - func FixPublisherIDforTagReleases(x *xorm.Engine) error { type Release struct { ID int64 @@ -108,7 +98,7 @@ func FixPublisherIDforTagReleases(x *xorm.Engine) error { return err } } - gitRepo, err = git.OpenRepository(git.DefaultContext, repoPath(repo.OwnerName, repo.Name)) + gitRepo, err = git.OpenRepository(git.DefaultContext, storage.RepoPath(repo.OwnerName, repo.Name)) if err != nil { log.Error("Error whilst opening git repo for [%d]%s/%s. Error: %v", repo.ID, repo.OwnerName, repo.Name, err) return err diff --git a/models/repo/repo.go b/models/repo/repo.go index d3e6daa95b53a..43a5e401f500e 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -9,13 +9,13 @@ import ( "html/template" "net" "net/url" - "path/filepath" "strconv" "strings" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" @@ -469,14 +469,9 @@ func (repo *Repository) IsGenerated() bool { return repo.TemplateID != 0 } -// RepoPath returns repository path by given user and repository name. -func RepoPath(userName, repoName string) string { //revive:disable-line:exported - return filepath.Join(user_model.UserPath(userName), strings.ToLower(repoName)+".git") -} - // RepoPath returns the repository path func (repo *Repository) RepoPath() string { - return RepoPath(repo.OwnerName, repo.Name) + return storage.RepoPath(repo.OwnerName, repo.Name) } // Link returns the repository relative url @@ -730,7 +725,7 @@ func IsRepositoryModelOrDirExist(ctx context.Context, u *user_model.User, repoNa if err != nil { return false, err } - isDir, err := util.IsDir(RepoPath(u.Name, repoName)) + isDir, err := storage.IsDir(storage.RepoRelPath(u.Name, repoName)) return has || isDir, err } diff --git a/models/repo/update.go b/models/repo/update.go index 4894e0a1b9c5f..188496dd63f31 100644 --- a/models/repo/update.go +++ b/models/repo/update.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models/db" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" ) @@ -123,10 +124,10 @@ func CheckCreateRepository(doer, u *user_model.User, name string, overwriteOrAdo return ErrRepoAlreadyExist{u.Name, name} } - repoPath := RepoPath(u.Name, name) - isExist, err := util.IsExist(repoPath) + repoRelPath := storage.RepoRelPath(u.Name, name) + isExist, err := storage.IsExist(repoRelPath) if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) + log.Error("Unable to check if %s exists. Error: %v", repoRelPath, err) return err } if !overwriteOrAdopt && isExist { @@ -154,19 +155,20 @@ func ChangeRepositoryName(doer *user_model.User, repo *Repository, newRepoName s return ErrRepoAlreadyExist{repo.Owner.Name, newRepoName} } - newRepoPath := RepoPath(repo.Owner.Name, newRepoName) - if err = util.Rename(repo.RepoPath(), newRepoPath); err != nil { + repoRelPath := storage.RepoRelPath(repo.OwnerName, repo.Name) + newRepoRelPath := storage.RepoRelPath(repo.Owner.Name, newRepoName) + if err = storage.Rename(repoRelPath, newRepoRelPath); err != nil { return fmt.Errorf("rename repository directory: %w", err) } - wikiPath := repo.WikiPath() - isExist, err := util.IsExist(wikiPath) + wikiRelPath := storage.WikiRelPath(repo.OwnerName, repo.Name) + isExist, err := storage.IsExist(wikiRelPath) if err != nil { - log.Error("Unable to check if %s exists. Error: %v", wikiPath, err) + log.Error("Unable to check if %s exists. Error: %v", wikiRelPath, err) return err } if isExist { - if err = util.Rename(wikiPath, WikiPath(repo.Owner.Name, newRepoName)); err != nil { + if err = storage.Rename(wikiRelPath, storage.WikiRelPath(repo.Owner.Name, newRepoName)); err != nil { return fmt.Errorf("rename repository wiki: %w", err) } } diff --git a/models/repo/wiki.go b/models/repo/wiki.go index b378666a20620..131668fe05225 100644 --- a/models/repo/wiki.go +++ b/models/repo/wiki.go @@ -6,10 +6,8 @@ package repo import ( "fmt" - "path/filepath" - "strings" - user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" ) @@ -76,19 +74,14 @@ func (repo *Repository) WikiCloneLink() *CloneLink { return repo.cloneLink(true) } -// WikiPath returns wiki data path by given user and repository name. -func WikiPath(userName, repoName string) string { - return filepath.Join(user_model.UserPath(userName), strings.ToLower(repoName)+".wiki.git") -} - // WikiPath returns wiki data path for given repository. func (repo *Repository) WikiPath() string { - return WikiPath(repo.OwnerName, repo.Name) + return storage.WikiPath(repo.OwnerName, repo.Name) } // HasWiki returns true if repository has wiki. func (repo *Repository) HasWiki() bool { - isDir, err := util.IsDir(repo.WikiPath()) + isDir, err := storage.IsDir(storage.WikiRelPath(repo.OwnerName, repo.Name)) if err != nil { log.Error("Unable to check if %s is a directory: %v", repo.WikiPath(), err) } diff --git a/models/repo/wiki_test.go b/models/repo/wiki_test.go index 629986f741a5e..cc8460c5a2d1f 100644 --- a/models/repo/wiki_test.go +++ b/models/repo/wiki_test.go @@ -4,12 +4,11 @@ package repo_test import ( - "path/filepath" "testing" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/git/storage" "github.com/stretchr/testify/assert" ) @@ -25,14 +24,13 @@ func TestRepository_WikiCloneLink(t *testing.T) { func TestWikiPath(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - expected := filepath.Join(setting.RepoRootPath, "user2/repo1.wiki.git") - assert.Equal(t, expected, repo_model.WikiPath("user2", "repo1")) + assert.Equal(t, "user2/repo1.wiki.git", storage.WikiRelPath("user2", "repo1")) } func TestRepository_WikiPath(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) - expected := filepath.Join(setting.RepoRootPath, "user2/repo1.wiki.git") + expected := storage.LocalPath("user2/repo1.wiki.git") assert.Equal(t, expected, repo.WikiPath()) } diff --git a/models/repo_transfer.go b/models/repo_transfer.go index 1c873cec57624..1ab0b72237c9d 100644 --- a/models/repo_transfer.go +++ b/models/repo_transfer.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "os" + "strings" "code.gitea.io/gitea/models/db" issues_model "code.gitea.io/gitea/models/issues" @@ -14,9 +15,9 @@ import ( access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" ) // RepoTransfer is used to manage repository transfers @@ -215,16 +216,16 @@ func TransferOwnership(doer *user_model.User, newOwnerName string, repo *repo_mo } if repoRenamed { - if err := util.Rename(repo_model.RepoPath(newOwnerName, repo.Name), repo_model.RepoPath(oldOwnerName, repo.Name)); err != nil { + if err := storage.Rename(storage.RepoRelPath(newOwnerName, repo.Name), storage.RepoRelPath(oldOwnerName, repo.Name)); err != nil { log.Critical("Unable to move repository %s/%s directory from %s back to correct place %s: %v", oldOwnerName, repo.Name, - repo_model.RepoPath(newOwnerName, repo.Name), repo_model.RepoPath(oldOwnerName, repo.Name), err) + storage.RepoRelPath(newOwnerName, repo.Name), storage.RepoRelPath(oldOwnerName, repo.Name), err) } } if wikiRenamed { - if err := util.Rename(repo_model.WikiPath(newOwnerName, repo.Name), repo_model.WikiPath(oldOwnerName, repo.Name)); err != nil { + if err := storage.Rename(storage.WikiRelPath(newOwnerName, repo.Name), storage.WikiRelPath(oldOwnerName, repo.Name)); err != nil { log.Critical("Unable to move wiki for repository %s/%s directory from %s back to correct place %s: %v", oldOwnerName, repo.Name, - repo_model.WikiPath(newOwnerName, repo.Name), repo_model.WikiPath(oldOwnerName, repo.Name), err) + storage.WikiRelPath(newOwnerName, repo.Name), storage.WikiRelPath(oldOwnerName, repo.Name), err) } } @@ -374,25 +375,25 @@ func TransferOwnership(doer *user_model.User, newOwnerName string, repo *repo_mo } // Rename remote repository to new path and delete local copy. - dir := user_model.UserPath(newOwner.Name) + userRelPath := strings.ToLower(newOwner.Name) - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return fmt.Errorf("Failed to create dir %s: %w", dir, err) + if err := storage.MakeDir(userRelPath, os.ModePerm); err != nil { + return fmt.Errorf("Failed to create dir %s: %w", userRelPath, err) } - if err := util.Rename(repo_model.RepoPath(oldOwner.Name, repo.Name), repo_model.RepoPath(newOwner.Name, repo.Name)); err != nil { + if err := storage.Rename(storage.RepoRelPath(oldOwner.Name, repo.Name), storage.RepoRelPath(newOwner.Name, repo.Name)); err != nil { return fmt.Errorf("rename repository directory: %w", err) } repoRenamed = true // Rename remote wiki repository to new path and delete local copy. - wikiPath := repo_model.WikiPath(oldOwner.Name, repo.Name) + wikiRelPath := storage.WikiRelPath(oldOwner.Name, repo.Name) - if isExist, err := util.IsExist(wikiPath); err != nil { - log.Error("Unable to check if %s exists. Error: %v", wikiPath, err) + if isExist, err := storage.IsExist(wikiRelPath); err != nil { + log.Error("Unable to check if %s exists. Error: %v", wikiRelPath, err) return err } else if isExist { - if err := util.Rename(wikiPath, repo_model.WikiPath(newOwner.Name, repo.Name)); err != nil { + if err := storage.Rename(wikiRelPath, storage.WikiRelPath(newOwner.Name, repo.Name)); err != nil { return fmt.Errorf("rename repository wiki: %w", err) } wikiRenamed = true diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index 5351ff1139989..cf23e02335353 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -8,6 +8,7 @@ import ( "fmt" "log" "os" + "path" "path/filepath" "strings" "testing" @@ -17,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/auth/password/hash" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/git" + git_storage "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" @@ -139,17 +141,17 @@ func MainTest(m *testing.M, testOpts *TestOptions) { fatalTestError("models.Init: %v\n", err) } - if err = util.RemoveAll(repoRootPath); err != nil { - fatalTestError("util.RemoveAll: %v\n", err) + if err = git_storage.RemoveAll(""); err != nil { + fatalTestError("git_storage.RemoveAll: %v\n", err) } - if err = CopyDir(filepath.Join(testOpts.GiteaRootPath, "tests", "gitea-repositories-meta"), setting.RepoRootPath); err != nil { - fatalTestError("util.CopyDir: %v\n", err) + if err = git_storage.CopyDir(filepath.Join(testOpts.GiteaRootPath, "tests", "gitea-repositories-meta"), ""); err != nil { + fatalTestError("git_storage.CopyDir: %v\n", err) } if err = git.InitFull(context.Background()); err != nil { fatalTestError("git.Init: %v\n", err) } - ownerDirs, err := os.ReadDir(setting.RepoRootPath) + ownerDirs, err := git_storage.ReadDir("") if err != nil { fatalTestError("unable to read the new repo root: %v\n", err) } @@ -157,15 +159,14 @@ func MainTest(m *testing.M, testOpts *TestOptions) { if !ownerDir.Type().IsDir() { continue } - repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name())) + repoDirs, err := git_storage.ReadDir(ownerDir.Name()) if err != nil { fatalTestError("unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755) + if err := git_storage.MakeRepoDir(path.Join(ownerDir.Name(), repoDir.Name())); err != nil { + fatalTestError("create directories failed: %v\n", err) + } } } @@ -183,7 +184,7 @@ func MainTest(m *testing.M, testOpts *TestOptions) { } } - if err = util.RemoveAll(repoRootPath); err != nil { + if err = git_storage.RemoveAll(""); err != nil { fatalTestError("util.RemoveAll: %v\n", err) } if err = util.RemoveAll(appDataPath); err != nil { @@ -230,22 +231,20 @@ func PrepareTestDatabase() error { // by tests that use the above MainTest(..) function. func PrepareTestEnv(t testing.TB) { assert.NoError(t, PrepareTestDatabase()) - assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) + assert.NoError(t, git_storage.RemoveAll("")) metaPath := filepath.Join(giteaRoot, "tests", "gitea-repositories-meta") - assert.NoError(t, CopyDir(metaPath, setting.RepoRootPath)) - ownerDirs, err := os.ReadDir(setting.RepoRootPath) + assert.NoError(t, git_storage.CopyDir(metaPath, "")) + ownerDirs, err := git_storage.ReadDir("") assert.NoError(t, err) for _, ownerDir := range ownerDirs { if !ownerDir.Type().IsDir() { continue } - repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name())) + repoDirs, err := git_storage.ReadDir(ownerDir.Name()) assert.NoError(t, err) for _, repoDir := range repoDirs { - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755) + err = git_storage.MakeRepoDir(path.Join(ownerDir.Name(), repoDir.Name())) + assert.NoError(t, err) } } diff --git a/models/user/user.go b/models/user/user.go index 2077d55f513e0..2afd85f72c18d 100644 --- a/models/user/user.go +++ b/models/user/user.go @@ -9,7 +9,6 @@ import ( "encoding/hex" "fmt" "net/url" - "path/filepath" "strings" "time" @@ -899,11 +898,6 @@ func GetInactiveUsers(ctx context.Context, olderThan time.Duration) ([]*User, er Find(&users) } -// UserPath returns the path absolute path of user repositories. -func UserPath(userName string) string { //revive:disable-line:exported - return filepath.Join(setting.RepoRootPath, strings.ToLower(userName)) -} - // GetUserByID returns the user object by given ID if exists. func GetUserByID(ctx context.Context, id int64) (*User, error) { u := new(User) diff --git a/modules/context/api.go b/modules/context/api.go index 092ad73f31118..28d95b17e1b81 100644 --- a/modules/context/api.go +++ b/modules/context/api.go @@ -12,11 +12,11 @@ import ( "strings" "code.gitea.io/gitea/models/auth" - repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" mc "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/httpcache" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -291,7 +291,7 @@ func ReferencesGitRepo(allowEmpty ...bool) func(ctx *APIContext) (cancel context // For API calls. if ctx.Repo.GitRepo == nil { - repoPath := repo_model.RepoPath(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name) + repoPath := storage.RepoPath(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name) gitRepo, err := git.OpenRepository(ctx, repoPath) if err != nil { ctx.Error(http.StatusInternalServerError, "RepoRef Invalid repo "+repoPath, err) diff --git a/modules/context/repo.go b/modules/context/repo.go index fd5f20857663e..58e541674bdf0 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -23,6 +23,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" code_indexer "code.gitea.io/gitea/modules/indexer/code" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" @@ -626,7 +627,7 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) { return } - gitRepo, err := git.OpenRepository(ctx, repo_model.RepoPath(userName, repoName)) + gitRepo, err := git.OpenRepository(ctx, storage.RepoPath(userName, repoName)) if err != nil { if strings.Contains(err.Error(), "repository does not exist") || strings.Contains(err.Error(), "no such file or directory") { log.Error("Repository %-v has a broken repository on the file system: %s Error: %v", ctx.Repo.Repository, ctx.Repo.Repository.RepoPath(), err) @@ -638,7 +639,7 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) { } return } - ctx.ServerError("RepoAssignment Invalid repo "+repo_model.RepoPath(userName, repoName), err) + ctx.ServerError("RepoAssignment Invalid repo "+storage.RepoPath(userName, repoName), err) return } if ctx.Repo.GitRepo != nil { @@ -878,7 +879,7 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context ) if ctx.Repo.GitRepo == nil { - repoPath := repo_model.RepoPath(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name) + repoPath := storage.RepoPath(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name) ctx.Repo.GitRepo, err = git.OpenRepository(ctx, repoPath) if err != nil { ctx.ServerError("RepoRef Invalid repo "+repoPath, err) diff --git a/modules/git/storage/legacy.go b/modules/git/storage/legacy.go new file mode 100644 index 0000000000000..aae854f5c8e14 --- /dev/null +++ b/modules/git/storage/legacy.go @@ -0,0 +1,35 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package storage + +import ( + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/setting" +) + +func absPath(path string) string { + return filepath.Join(setting.RepoRootPath, path) +} + +// UserPath returns the path absolute path of user repositories. +func UserPath(userName string) string { //revive:disable-line:exported + return absPath(strings.ToLower(userName)) +} + +// RepoPath returns repository path by given user and repository name. +func RepoPath(userName, repoName string) string { //revive:disable-line:exported + return filepath.Join(UserPath(userName), strings.ToLower(repoName)+".git") +} + +// LocalPath returns local path by given relative path. +func LocalPath(relPath string) string { + return absPath(relPath) +} + +// WikiPath returns wiki data path by given user and repository name. +func WikiPath(userName, repoName string) string { + return filepath.Join(UserPath(userName), strings.ToLower(repoName)+".wiki.git") +} diff --git a/modules/git/storage/storage.go b/modules/git/storage/storage.go new file mode 100644 index 0000000000000..84d4d2999baad --- /dev/null +++ b/modules/git/storage/storage.go @@ -0,0 +1,146 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package storage + +import ( + "io/fs" + "os" + "path" + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +type Storage interface { + IsConfigured() bool + CheckStats() error + IsExist(path string) (bool, error) + IsDir(path string) (bool, error) + // MakeDir(repoRelPath string) error + MakeDir(dir string, perm os.FileMode) error + MakeRepoDir(repoRelPath string) error + RemoveAll(path string) error + ReadDir(owner string) ([]fs.DirEntry, error) + CopyDir(source, target string) error + Rename(oldPath, newPath string) error +} + +type LocalSingleStorage struct { + repoRootPath string +} + +var _ Storage = &LocalSingleStorage{} + +func (l *LocalSingleStorage) absPath(relPath string) string { + return filepath.Join(l.repoRootPath, relPath) +} + +func (l *LocalSingleStorage) IsConfigured() bool { + return len(l.repoRootPath) != 0 +} + +func (l *LocalSingleStorage) CheckStats() error { + _, err := os.Stat(l.repoRootPath) + return err +} + +func (l *LocalSingleStorage) IsExist(path string) (bool, error) { + return util.IsExist(path) +} + +func (l *LocalSingleStorage) IsDir(path string) (bool, error) { + return util.IsDir(path) +} + +func (l *LocalSingleStorage) MakeDir(dir string, perm os.FileMode) error { + return os.MkdirAll(l.absPath(dir), perm) +} + +func (l *LocalSingleStorage) MakeRepoDir(repoRelPath string) error { + _ = os.MkdirAll(filepath.Join(l.absPath(repoRelPath), "objects", "pack"), 0o755) + _ = os.MkdirAll(filepath.Join(l.absPath(repoRelPath), "objects", "info"), 0o755) + _ = os.MkdirAll(filepath.Join(l.absPath(repoRelPath), "refs", "heads"), 0o755) + _ = os.MkdirAll(filepath.Join(l.absPath(repoRelPath), "refs", "tag"), 0o755) + return nil +} + +func (l *LocalSingleStorage) RemoveAll(path string) error { + // TODO: removeAllWithRetry(l.absPath(path)) + return os.RemoveAll(l.absPath(path)) +} + +func (l *LocalSingleStorage) ReadDir(owner string) ([]fs.DirEntry, error) { + return os.ReadDir(l.absPath(owner)) +} + +func (l *LocalSingleStorage) CopyDir(source, target string) error { + return util.CopyDir(source, l.absPath(target)) +} + +func (l *LocalSingleStorage) Rename(oldPath, newPath string) error { + return util.Rename(l.absPath(oldPath), l.absPath(newPath)) +} + +func getStorage() Storage { + return &LocalSingleStorage{ + repoRootPath: setting.RepoRootPath, + } +} + +func IsConfigured() bool { + return getStorage().IsConfigured() +} + +func CheckStats() error { + return getStorage().CheckStats() +} + +func IsExist(path string) (bool, error) { + return getStorage().IsExist(path) +} + +func IsDir(path string) (bool, error) { + return getStorage().IsDir(path) +} + +func MakeDir(dir string, perm os.FileMode) error { + return getStorage().MakeDir(dir, perm) +} + +func MakeRepoDir(repoRelPath string) error { + return getStorage().MakeRepoDir(repoRelPath) +} + +func RemoveAll(p string) error { + return getStorage().RemoveAll(p) +} + +func ReadDir(owner string) ([]fs.DirEntry, error) { + return getStorage().ReadDir(owner) +} + +func CopyDir(source, target string) error { + return getStorage().CopyDir(source, target) +} + +func Rename(oldPath, newPath string) error { + return getStorage().Rename(oldPath, newPath) +} + +// UserRelPath returns the path relative path of user repositories. +func UserRelPath(userName string) string { + return strings.ToLower(userName) +} + +// RepoRelPath returns repository relative path by given user and repository name. +func RepoRelPath(userName, repoName string) string { + return path.Join(strings.ToLower(userName), strings.ToLower(repoName)+".git") +} + +// WikiRelPath returns wiki repository relative path by given user and repository name. +func WikiRelPath(userName, repoName string) string { + return path.Join(strings.ToLower(userName), strings.ToLower(repoName)+".wiki.git") +} diff --git a/modules/repository/create.go b/modules/repository/create.go index e1f0bdcdf995e..1054575cd4e0c 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -23,6 +23,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/models/webhook" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" @@ -45,14 +46,14 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re } } - repoPath := repo_model.RepoPath(u.Name, repo.Name) - isExist, err := util.IsExist(repoPath) + repoRelPath := storage.RepoRelPath(u.Name, repo.Name) + isExist, err := storage.IsExist(repoRelPath) if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) + log.Error("Unable to check if %s exists. Error: %v", repoRelPath, err) return err } if !overwriteOrAdopt && isExist { - log.Error("Files already exist in %s and we are not going to adopt or delete.", repoPath) + log.Error("Files already exist in %s and we are not going to adopt or delete.", repoRelPath) return repo_model.ErrRepoFilesAlreadyExist{ Uname: u.Name, Name: repo.Name, @@ -226,10 +227,10 @@ func CreateRepository(doer, u *user_model.User, opts CreateRepoOptions) (*repo_m return nil } - repoPath := repo_model.RepoPath(u.Name, repo.Name) - isExist, err := util.IsExist(repoPath) + repoRelPath := storage.RepoRelPath(u.Name, repo.Name) + isExist, err := storage.IsExist(repoRelPath) if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) + log.Error("Unable to check if %s exists. Error: %v", repoRelPath, err) return err } if isExist { @@ -240,15 +241,17 @@ func CreateRepository(doer, u *user_model.User, opts CreateRepoOptions) (*repo_m // // Previously Gitea would just delete and start afresh - this was naughty. // So we will now fail and delegate to other functionality to adopt or delete - log.Error("Files already exist in %s and we are not going to adopt or delete.", repoPath) + log.Error("Files already exist in %s and we are not going to adopt or delete.", repoRelPath) return repo_model.ErrRepoFilesAlreadyExist{ Uname: u.Name, Name: repo.Name, } } + repoPath := storage.LocalPath(repoRelPath) + if err = initRepository(ctx, repoPath, doer, repo, opts); err != nil { - if err2 := util.RemoveAll(repoPath); err2 != nil { + if err2 := storage.RemoveAll(repoRelPath); err2 != nil { log.Error("initRepository: %v", err) return fmt.Errorf( "delete repo directory %s/%s failed(2): %v", u.Name, repo.Name, err2) diff --git a/modules/repository/init.go b/modules/repository/init.go index f079f72b77116..15ae684235e2d 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -17,6 +17,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/label" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" @@ -279,10 +280,10 @@ func initRepoCommit(ctx context.Context, tmpPath string, repo *repo_model.Reposi func checkInitRepository(ctx context.Context, owner, name string) (err error) { // Somehow the directory could exist. - repoPath := repo_model.RepoPath(owner, name) - isExist, err := util.IsExist(repoPath) + repoRelPath := storage.RepoRelPath(owner, name) + isExist, err := storage.IsExist(repoRelPath) if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) + log.Error("Unable to check if %s exists. Error: %v", repoRelPath, err) return err } if isExist { @@ -292,6 +293,8 @@ func checkInitRepository(ctx context.Context, owner, name string) (err error) { } } + repoPath := storage.LocalPath(repoRelPath) + // Init git bare new repository. if err = git.InitRepository(ctx, repoPath, true); err != nil { return fmt.Errorf("git.InitRepository: %w", err) diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 3991e9ad995be..bf66999bf8c4c 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -20,12 +20,12 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migration" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" "gopkg.in/ini.v1" //nolint:depguard ) @@ -54,8 +54,6 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, repo *repo_model.Repository, opts migration.MigrateOptions, httpTransport *http.Transport, ) (*repo_model.Repository, error) { - repoPath := repo_model.RepoPath(u.Name, opts.RepoName) - if u.IsOrganization() { t, err := organization.OrgFromUser(u).GetOwnerTeam(ctx) if err != nil { @@ -69,10 +67,12 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second var err error - if err = util.RemoveAll(repoPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %w", repoPath, err) + repoRelPath := storage.RepoRelPath(u.Name, opts.RepoName) + if err = storage.RemoveAll(repoRelPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %w", repoRelPath, err) } + repoPath := storage.LocalPath(repoRelPath) if err = git.Clone(ctx, opts.CloneAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, @@ -90,13 +90,15 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } if opts.Wiki { - wikiPath := repo_model.WikiPath(u.Name, opts.RepoName) + wikiRelPath := storage.WikiRelPath(u.Name, opts.RepoName) wikiRemotePath := WikiRemoteURL(ctx, opts.CloneAddr) if len(wikiRemotePath) > 0 { - if err := util.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) + if err := storage.RemoveAll(wikiRelPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %w", wikiRelPath, err) } + wikiPath := storage.LocalPath(wikiRelPath) + if err := git.Clone(ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, @@ -105,8 +107,8 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, SkipTLSVerify: setting.Migrations.SkipTLSVerify, }); err != nil { log.Warn("Clone wiki: %v", err) - if err := util.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) + if err := storage.RemoveAll(wikiRelPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %w", wikiRelPath, err) } } else { if err := git.WriteCommitGraph(ctx, wikiPath); err != nil { diff --git a/models/unittest/fscopy.go b/modules/util/fscopy.go similarity index 91% rename from models/unittest/fscopy.go rename to modules/util/fscopy.go index 74b12d5057791..69c9e95436d0d 100644 --- a/models/unittest/fscopy.go +++ b/modules/util/fscopy.go @@ -1,7 +1,7 @@ // Copyright 2022 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package unittest +package util import ( "errors" @@ -9,8 +9,6 @@ import ( "os" "path" "strings" - - "code.gitea.io/gitea/modules/util" ) // Copy copies file from source to target path. @@ -64,7 +62,7 @@ func Copy(src, dest string) error { func CopyDir(srcPath, destPath string, filters ...func(filePath string) bool) error { // Check if target directory exists. if _, err := os.Stat(destPath); !errors.Is(err, os.ErrNotExist) { - return util.NewAlreadyExistErrorf("file or directory already exists: %s", destPath) + return NewAlreadyExistErrorf("file or directory already exists: %s", destPath) } err := os.MkdirAll(destPath, os.ModePerm) @@ -73,7 +71,7 @@ func CopyDir(srcPath, destPath string, filters ...func(filePath string) bool) er } // Gather directory info. - infos, err := util.StatDir(srcPath, true) + infos, err := StatDir(srcPath, true) if err != nil { return err } diff --git a/routers/api/v1/admin/adopt.go b/routers/api/v1/admin/adopt.go index ccd8be9171aa0..d126bf87a28bf 100644 --- a/routers/api/v1/admin/adopt.go +++ b/routers/api/v1/admin/adopt.go @@ -9,8 +9,8 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git/storage" repo_module "code.gitea.io/gitea/modules/repository" - "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/routers/api/v1/utils" repo_service "code.gitea.io/gitea/services/repository" ) @@ -100,7 +100,7 @@ func AdoptRepository(ctx *context.APIContext) { ctx.InternalServerError(err) return } - isDir, err := util.IsDir(repo_model.RepoPath(ctxUser.Name, repoName)) + isDir, err := storage.IsDir(storage.RepoRelPath(ctxUser.Name, repoName)) if err != nil { ctx.InternalServerError(err) return @@ -162,7 +162,7 @@ func DeleteUnadoptedRepository(ctx *context.APIContext) { ctx.InternalServerError(err) return } - isDir, err := util.IsDir(repo_model.RepoPath(ctxUser.Name, repoName)) + isDir, err := storage.IsDir(storage.RepoRelPath(ctxUser.Name, repoName)) if err != nil { ctx.InternalServerError(err) return diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index ae0d31c2a6d79..231446da030c1 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -21,6 +21,7 @@ import ( "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" + git_storage "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/httpcache" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" @@ -279,7 +280,7 @@ func GetArchive(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - repoPath := repo_model.RepoPath(ctx.Params(":username"), ctx.Params(":reponame")) + repoPath := git_storage.RepoPath(ctx.Params(":username"), ctx.Params(":reponame")) if ctx.Repo.GitRepo == nil { gitRepo, err := git.OpenRepository(ctx, repoPath) if err != nil { diff --git a/routers/api/v1/repo/pull.go b/routers/api/v1/repo/pull.go index a507c1f44d171..229907e37d272 100644 --- a/routers/api/v1/repo/pull.go +++ b/routers/api/v1/repo/pull.go @@ -23,6 +23,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/setting" @@ -991,7 +992,7 @@ func parseCompareInfo(ctx *context.APIContext, form api.CreatePullRequestOption) headRepo = ctx.Repo.Repository headGitRepo = ctx.Repo.GitRepo } else { - headGitRepo, err = git.OpenRepository(ctx, repo_model.RepoPath(headUser.Name, headRepo.Name)) + headGitRepo, err = git.OpenRepository(ctx, storage.RepoPath(headUser.Name, headRepo.Name)) if err != nil { ctx.Error(http.StatusInternalServerError, "OpenRepository", err) return nil, nil, nil, nil, "", "" @@ -1043,7 +1044,7 @@ func parseCompareInfo(ctx *context.APIContext, form api.CreatePullRequestOption) return nil, nil, nil, nil, "", "" } - compareInfo, err := headGitRepo.GetCompareInfo(repo_model.RepoPath(baseRepo.Owner.Name, baseRepo.Name), baseBranch, headBranch, false, false) + compareInfo, err := headGitRepo.GetCompareInfo(storage.RepoPath(baseRepo.Owner.Name, baseRepo.Name), baseBranch, headBranch, false, false) if err != nil { headGitRepo.Close() ctx.Error(http.StatusInternalServerError, "GetCompareInfo", err) diff --git a/routers/web/admin/repos.go b/routers/web/admin/repos.go index 9a0e467b48712..a7090a49184ac 100644 --- a/routers/web/admin/repos.go +++ b/routers/web/admin/repos.go @@ -13,10 +13,10 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/routers/web/explore" repo_service "code.gitea.io/gitea/services/repository" ) @@ -138,7 +138,7 @@ func AdoptOrDeleteRepository(ctx *context.Context) { ctx.ServerError("IsRepositoryExist", err) return } - isDir, err := util.IsDir(repo_model.RepoPath(ctxUser.Name, repoName)) + isDir, err := storage.IsDir(storage.RepoRelPath(ctxUser.Name, repoName)) if err != nil { ctx.ServerError("IsDir", err) return diff --git a/routers/web/repo/blame.go b/routers/web/repo/blame.go index 0e232c194c2e1..b18dd14d9b828 100644 --- a/routers/web/repo/blame.go +++ b/routers/web/repo/blame.go @@ -10,11 +10,11 @@ import ( "net/url" "strings" - repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/highlight" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/templates" @@ -101,7 +101,7 @@ func RefBlame(ctx *context.Context) { return } - blameReader, err := git.CreateBlameReader(ctx, repo_model.RepoPath(userName, repoName), commitID, fileName) + blameReader, err := git.CreateBlameReader(ctx, storage.RepoPath(userName, repoName), commitID, fileName) if err != nil { ctx.NotFound("CreateBlameReader", err) return diff --git a/routers/web/repo/http.go b/routers/web/repo/http.go index b6ebd25915626..7a08a79f3e845 100644 --- a/routers/web/repo/http.go +++ b/routers/web/repo/http.go @@ -26,6 +26,7 @@ import ( "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" @@ -302,9 +303,9 @@ func httpBase(ctx *context.Context) (h *serviceHandler) { r.URL.Path = strings.ToLower(r.URL.Path) // blue: In case some repo name has upper case name - dir := repo_model.RepoPath(username, reponame) + dir := storage.RepoPath(username, reponame) if isWiki { - dir = repo_model.RepoPath(username, wikiRepoName) + dir = storage.RepoPath(username, wikiRepoName) } return &serviceHandler{cfg, w, r, dir, cfg.Env} diff --git a/routers/web/user/setting/adopt.go b/routers/web/user/setting/adopt.go index 01668c395422f..ca24aa4b03be8 100644 --- a/routers/web/user/setting/adopt.go +++ b/routers/web/user/setting/adopt.go @@ -4,14 +4,11 @@ package setting import ( - "path/filepath" - repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git/storage" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" repo_service "code.gitea.io/gitea/services/repository" ) @@ -28,7 +25,6 @@ func AdoptOrDeleteRepository(ctx *context.Context) { action := ctx.FormString("action") ctxUser := ctx.Doer - root := user_model.UserPath(ctxUser.LowerName) // check not a repo has, err := repo_model.IsRepositoryModelExist(ctx, ctxUser, dir) @@ -37,7 +33,7 @@ func AdoptOrDeleteRepository(ctx *context.Context) { return } - isDir, err := util.IsDir(filepath.Join(root, dir+".git")) + isDir, err := storage.IsDir(storage.RepoRelPath(ctxUser.LowerName, dir)) if err != nil { ctx.ServerError("IsDir", err) return diff --git a/routers/web/user/setting/profile.go b/routers/web/user/setting/profile.go index 47066d5e384ef..294fb8dc674be 100644 --- a/routers/web/user/setting/profile.go +++ b/routers/web/user/setting/profile.go @@ -20,6 +20,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/translation" @@ -255,7 +256,7 @@ func Repos(ctx *context.Context) { repoNames := make([]string, 0, setting.UI.Admin.UserPagingNum) repos := map[string]*repo_model.Repository{} // We're going to iterate by pagesize. - root := user_model.UserPath(ctxUser.Name) + root := storage.UserPath(ctxUser.Name) if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { if err != nil { if os.IsNotExist(err) { diff --git a/services/org/org.go b/services/org/org.go index a62e5b6fc8f6d..8963d60e1ce4f 100644 --- a/services/org/org.go +++ b/services/org/org.go @@ -12,9 +12,8 @@ import ( org_model "code.gitea.io/gitea/models/organization" packages_model "code.gitea.io/gitea/models/packages" repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" + git_storage "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/storage" - "code.gitea.io/gitea/modules/util" user_service "code.gitea.io/gitea/services/user" ) @@ -52,9 +51,8 @@ func DeleteOrganization(org *org_model.Organization) error { // FIXME: system notice // Note: There are something just cannot be roll back, // so just keep error logs of those operations. - path := user_model.UserPath(org.Name) - - if err := util.RemoveAll(path); err != nil { + path := git_storage.UserRelPath(org.Name) + if err := git_storage.RemoveAll(path); err != nil { return fmt.Errorf("failed to RemoveAll %s: %w", path, err) } diff --git a/services/release/release_test.go b/services/release/release_test.go index 805269413d729..655a5063df997 100644 --- a/services/release/release_test.go +++ b/services/release/release_test.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/services/attachment" "github.com/stretchr/testify/assert" @@ -30,7 +31,7 @@ func TestRelease_Create(t *testing.T) { user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) - repoPath := repo_model.RepoPath(user.Name, repo.Name) + repoPath := storage.RepoPath(user.Name, repo.Name) gitRepo, err := git.OpenRepository(git.DefaultContext, repoPath) assert.NoError(t, err) @@ -136,7 +137,7 @@ func TestRelease_Update(t *testing.T) { user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) - repoPath := repo_model.RepoPath(user.Name, repo.Name) + repoPath := storage.RepoPath(user.Name, repo.Name) gitRepo, err := git.OpenRepository(git.DefaultContext, repoPath) assert.NoError(t, err) @@ -279,7 +280,7 @@ func TestRelease_createTag(t *testing.T) { user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) - repoPath := repo_model.RepoPath(user.Name, repo.Name) + repoPath := storage.RepoPath(user.Name, repo.Name) gitRepo, err := git.OpenRepository(git.DefaultContext, repoPath) assert.NoError(t, err) diff --git a/services/repository/adopt.go b/services/repository/adopt.go index 55e77a78a78a9..1c40ec9f18116 100644 --- a/services/repository/adopt.go +++ b/services/repository/adopt.go @@ -16,6 +16,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" repo_module "code.gitea.io/gitea/modules/repository" @@ -54,10 +55,10 @@ func AdoptRepository(ctx context.Context, doer, u *user_model.User, opts repo_mo } if err := db.WithTx(ctx, func(ctx context.Context) error { - repoPath := repo_model.RepoPath(u.Name, repo.Name) - isExist, err := util.IsExist(repoPath) + repoRelPath := storage.RepoRelPath(u.Name, repo.Name) + isExist, err := storage.IsExist(repoRelPath) if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) + log.Error("Unable to check if %s exists. Error: %v", repoRelPath, err) return err } if !isExist { @@ -70,7 +71,7 @@ func AdoptRepository(ctx context.Context, doer, u *user_model.User, opts repo_mo if err := repo_module.CreateRepositoryByExample(ctx, doer, u, repo, true, false); err != nil { return err } - if err := adoptRepository(ctx, repoPath, doer, repo, opts); err != nil { + if err := adoptRepository(ctx, storage.LocalPath(repoRelPath), doer, repo, opts); err != nil { return fmt.Errorf("createDelegateHooks: %w", err) } if err := repo_module.CheckDaemonExportOK(ctx, repo); err != nil { @@ -84,6 +85,7 @@ func AdoptRepository(ctx context.Context, doer, u *user_model.User, opts repo_mo } } + repoPath := storage.LocalPath(repoRelPath) if stdout, _, err := git.NewCommand(ctx, "update-server-info"). SetDescription(fmt.Sprintf("CreateRepository(git update-server-info): %s", repoPath)). RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { @@ -193,10 +195,10 @@ func DeleteUnadoptedRepository(ctx context.Context, doer, u *user_model.User, re return err } - repoPath := repo_model.RepoPath(u.Name, repoName) - isExist, err := util.IsExist(repoPath) + repoRelPath := storage.RepoRelPath(u.Name, repoName) + isExist, err := storage.IsExist(repoRelPath) if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) + log.Error("Unable to check if %s exists. Error: %v", repoRelPath, err) return err } if !isExist { @@ -215,7 +217,7 @@ func DeleteUnadoptedRepository(ctx context.Context, doer, u *user_model.User, re } } - return util.RemoveAll(repoPath) + return storage.RemoveAll(repoRelPath) } type unadoptedRepositories struct { diff --git a/services/repository/adopt_test.go b/services/repository/adopt_test.go index c1520e01c9141..132b9aaf42440 100644 --- a/services/repository/adopt_test.go +++ b/services/repository/adopt_test.go @@ -4,13 +4,12 @@ package repository import ( - "os" "path" "testing" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/git/storage" "github.com/stretchr/testify/assert" ) @@ -68,7 +67,7 @@ func TestListUnadoptedRepositories_ListOptions(t *testing.T) { username := "user2" unadoptedList := []string{path.Join(username, "unadopted1"), path.Join(username, "unadopted2")} for _, unadopted := range unadoptedList { - _ = os.Mkdir(path.Join(setting.RepoRootPath, unadopted+".git"), 0o755) + _ = storage.MakeRepoDir(unadopted) } opts := db.ListOptions{Page: 1, PageSize: 1} diff --git a/services/repository/files/content_test.go b/services/repository/files/content_test.go index a43b71cf31e1a..afb9a82a7c6aa 100644 --- a/services/repository/files/content_test.go +++ b/services/repository/files/content_test.go @@ -7,9 +7,9 @@ import ( "path/filepath" "testing" - repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/test" @@ -236,7 +236,7 @@ func TestGetBlobBySHA(t *testing.T) { ctx.SetParams(":id", "1") ctx.SetParams(":sha", sha) - gitRepo, err := git.OpenRepository(ctx, repo_model.RepoPath(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name)) + gitRepo, err := git.OpenRepository(ctx, storage.RepoPath(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name)) if err != nil { t.Fail() } diff --git a/services/repository/fork.go b/services/repository/fork.go index fb93b10f1c312..cccd9be5a98fb 100644 --- a/services/repository/fork.go +++ b/services/repository/fork.go @@ -14,6 +14,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" repo_module "code.gitea.io/gitea/modules/repository" @@ -92,15 +93,14 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork return } - repoPath := repo_model.RepoPath(owner.Name, repo.Name) - - if exists, _ := util.IsExist(repoPath); !exists { + repoRelPath := storage.RepoRelPath(owner.Name, repo.Name) + if exists, _ := storage.IsExist(repoRelPath); !exists { return } // As the transaction will be failed and hence database changes will be destroyed we only need // to delete the related repository on the filesystem - if errDelete := util.RemoveAll(repoPath); errDelete != nil { + if errDelete := storage.RemoveAll(repoRelPath); errDelete != nil { log.Error("Failed to remove fork repo") } } @@ -134,7 +134,7 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork needsRollback = true - repoPath := repo_model.RepoPath(owner.Name, repo.Name) + repoPath := storage.RepoPath(owner.Name, repo.Name) if stdout, _, err := git.NewCommand(txCtx, "clone", "--bare").AddDynamicArguments(oldRepoPath, repoPath). SetDescription(fmt.Sprintf("ForkRepository(git clone): %s to %s", opts.BaseRepo.FullName(), repo.FullName())). diff --git a/services/repository/transfer_test.go b/services/repository/transfer_test.go index 1299e66be2781..f50053bd817af 100644 --- a/services/repository/transfer_test.go +++ b/services/repository/transfer_test.go @@ -14,9 +14,9 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/notification/action" - "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" ) @@ -42,10 +42,10 @@ func TestTransferOwnership(t *testing.T) { transferredRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) assert.EqualValues(t, 2, transferredRepo.OwnerID) - exist, err := util.IsExist(repo_model.RepoPath("user3", "repo3")) + exist, err := storage.IsExist(storage.RepoRelPath("user3", "repo3")) assert.NoError(t, err) assert.False(t, exist) - exist, err = util.IsExist(repo_model.RepoPath("user2", "repo3")) + exist, err = storage.IsExist(storage.RepoRelPath("user2", "repo3")) assert.NoError(t, err) assert.True(t, exist) unittest.AssertExistsAndLoadBean(t, &activities_model.Action{ diff --git a/services/user/user.go b/services/user/user.go index e0815bd860791..59af46f5e1e3a 100644 --- a/services/user/user.go +++ b/services/user/user.go @@ -19,10 +19,10 @@ import ( system_model "code.gitea.io/gitea/models/system" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/eventsource" + git_storage "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" - "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/agit" "code.gitea.io/gitea/services/packages" container_service "code.gitea.io/gitea/services/packages/container" @@ -101,7 +101,7 @@ func RenameUser(ctx context.Context, u *user_model.User, newUserName string) err } // Do not fail if directory does not exist - if err = util.Rename(user_model.UserPath(oldUserName), user_model.UserPath(newUserName)); err != nil && !os.IsNotExist(err) { + if err = git_storage.Rename(git_storage.UserRelPath(oldUserName), git_storage.UserRelPath(newUserName)); err != nil && !os.IsNotExist(err) { u.Name = oldUserName u.LowerName = strings.ToLower(oldUserName) return fmt.Errorf("rename user directory: %w", err) @@ -110,7 +110,7 @@ func RenameUser(ctx context.Context, u *user_model.User, newUserName string) err if err = committer.Commit(); err != nil { u.Name = oldUserName u.LowerName = strings.ToLower(oldUserName) - if err2 := util.Rename(user_model.UserPath(newUserName), user_model.UserPath(oldUserName)); err2 != nil && !os.IsNotExist(err2) { + if err2 := git_storage.Rename(git_storage.UserRelPath(newUserName), git_storage.UserRelPath(oldUserName)); err2 != nil && !os.IsNotExist(err2) { log.Critical("Unable to rollback directory change during failed username change from: %s to: %s. DB Error: %v. Filesystem Error: %v", oldUserName, newUserName, err, err2) return fmt.Errorf("failed to rollback directory change during failed username change from: %s to: %s. DB Error: %w. Filesystem Error: %v", oldUserName, newUserName, err, err2) } @@ -273,8 +273,8 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) error { // Note: There are something just cannot be roll back, // so just keep error logs of those operations. - path := user_model.UserPath(u.Name) - if err := util.RemoveAll(path); err != nil { + path := git_storage.UserRelPath(u.Name) + if err := git_storage.RemoveAll(path); err != nil { err = fmt.Errorf("Failed to RemoveAll %s: %w", path, err) _ = system_model.CreateNotice(ctx, system_model.NoticeTask, fmt.Sprintf("delete user '%s': %v", u.Name, err)) return err diff --git a/tests/integration/api_repo_lfs_migrate_test.go b/tests/integration/api_repo_lfs_migrate_test.go index e66ca6b1474b4..37c289ea5a2f0 100644 --- a/tests/integration/api_repo_lfs_migrate_test.go +++ b/tests/integration/api_repo_lfs_migrate_test.go @@ -5,12 +5,12 @@ package integration import ( "net/http" - "path" "testing" auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" @@ -34,7 +34,7 @@ func TestAPIRepoLFSMigrateLocal(t *testing.T) { token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeRepo) req := NewRequestWithJSON(t, "POST", "/api/v1/repos/migrate?token="+token, &api.MigrateRepoOptions{ - CloneAddr: path.Join(setting.RepoRootPath, "migration/lfs-test.git"), + CloneAddr: storage.LocalPath("migration/lfs-test.git"), RepoOwnerID: user.ID, RepoName: "lfs-test-local", LFS: true, diff --git a/tests/integration/migration-test/migration_test.go b/tests/integration/migration-test/migration_test.go index 556a54015b4f1..175b602bbf542 100644 --- a/tests/integration/migration-test/migration_test.go +++ b/tests/integration/migration-test/migration_test.go @@ -24,6 +24,7 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/testlogger" @@ -63,10 +64,10 @@ func initMigrationTest(t *testing.T) func() { unittest.InitSettings() - assert.True(t, len(setting.RepoRootPath) != 0) - assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath)) - ownerDirs, err := os.ReadDir(setting.RepoRootPath) + assert.True(t, storage.IsConfigured()) + assert.NoError(t, storage.RemoveAll("")) + assert.NoError(t, storage.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) + ownerDirs, err := storage.ReadDir("") if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } @@ -74,15 +75,12 @@ func initMigrationTest(t *testing.T) func() { if !ownerDir.Type().IsDir() { continue } - repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name())) + repoDirs, err := storage.ReadDir(ownerDir.Name()) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755) + storage.MakeRepoDir(storage.RepoRelPath(ownerDir.Name(), repoDir.Name())) } } diff --git a/tests/integration/mirror_pull_test.go b/tests/integration/mirror_pull_test.go index 1bd91a48b5d4b..51e2319c4b8da 100644 --- a/tests/integration/mirror_pull_test.go +++ b/tests/integration/mirror_pull_test.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/migration" "code.gitea.io/gitea/modules/repository" mirror_service "code.gitea.io/gitea/services/mirror" @@ -26,7 +27,7 @@ func TestMirrorPull(t *testing.T) { user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) - repoPath := repo_model.RepoPath(user.Name, repo.Name) + repoPath := storage.RepoPath(user.Name, repo.Name) opts := migration.MigrateOptions{ RepoName: "test_mirror", diff --git a/tests/integration/pull_merge_test.go b/tests/integration/pull_merge_test.go index f6a36f60af2be..315a068b4e7e2 100644 --- a/tests/integration/pull_merge_test.go +++ b/tests/integration/pull_merge_test.go @@ -25,6 +25,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/models/webhook" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/storage" repo_module "code.gitea.io/gitea/modules/repository" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/test" @@ -242,7 +243,7 @@ func TestCantMergeConflict(t *testing.T) { BaseBranch: "base", }) - gitRepo, err := git.OpenRepository(git.DefaultContext, repo_model.RepoPath(user1.Name, repo1.Name)) + gitRepo, err := git.OpenRepository(git.DefaultContext, storage.RepoPath(user1.Name, repo1.Name)) assert.NoError(t, err) err = pull.Merge(context.Background(), pr, user1, gitRepo, repo_model.MergeStyleMerge, "", "CONFLICT", false) @@ -271,7 +272,7 @@ func TestCantMergeUnrelated(t *testing.T) { OwnerID: user1.ID, Name: "repo1", }) - path := repo_model.RepoPath(user1.Name, repo1.Name) + path := storage.RepoPath(user1.Name, repo1.Name) err := git.NewCommand(git.DefaultContext, "read-tree", "--empty").Run(&git.RunOpts{Dir: path}) assert.NoError(t, err) diff --git a/tests/test_utils.go b/tests/test_utils.go index 6f0fb25560d9b..08d0def837147 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/git" + git_storage "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" @@ -190,9 +191,9 @@ func PrepareTestEnv(t testing.TB, skip ...int) func() { assert.NoError(t, unittest.LoadFixtures()) // load git repo fixtures - assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath)) - ownerDirs, err := os.ReadDir(setting.RepoRootPath) + assert.NoError(t, git_storage.RemoveAll("")) + assert.NoError(t, git_storage.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) + ownerDirs, err := git_storage.ReadDir("") if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } @@ -200,15 +201,12 @@ func PrepareTestEnv(t testing.TB, skip ...int) func() { if !ownerDir.Type().IsDir() { continue } - repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name())) + repoDirs, err := git_storage.ReadDir(ownerDir.Name()) if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755) + assert.NoError(t, git_storage.MakeRepoDir(git_storage.RepoRelPath(ownerDir.Name(), repoDir.Name()))) } } From d4e03c555e5fc80f155ef66312cfc44d698e34b0 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 5 Jun 2023 09:03:41 +0800 Subject: [PATCH 02/10] more improvements --- models/migrations/base/tests.go | 2 +- models/unittest/testdb.go | 5 ++--- services/repository/adopt_test.go | 2 +- services/repository/files/update.go | 8 ++++++++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index 4c828339b01f7..6c5d7a1200aca 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -51,7 +51,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En assert.NoError(t, err, "unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - assert.NoError(t, storage.MakeRepoDir(path.Join(ownerDir.Name(), repoDir.Name()))) + assert.NoError(t, storage.MakeRepoDir(storage.RepoRelPath(ownerDir.Name(), repoDir.Name()))) } } diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index cf23e02335353..6b75b18906891 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -8,7 +8,6 @@ import ( "fmt" "log" "os" - "path" "path/filepath" "strings" "testing" @@ -164,7 +163,7 @@ func MainTest(m *testing.M, testOpts *TestOptions) { fatalTestError("unable to read the new repo root: %v\n", err) } for _, repoDir := range repoDirs { - if err := git_storage.MakeRepoDir(path.Join(ownerDir.Name(), repoDir.Name())); err != nil { + if err := git_storage.MakeRepoDir(git_storage.RepoRelPath(ownerDir.Name(), repoDir.Name())); err != nil { fatalTestError("create directories failed: %v\n", err) } } @@ -243,7 +242,7 @@ func PrepareTestEnv(t testing.TB) { repoDirs, err := git_storage.ReadDir(ownerDir.Name()) assert.NoError(t, err) for _, repoDir := range repoDirs { - err = git_storage.MakeRepoDir(path.Join(ownerDir.Name(), repoDir.Name())) + err = git_storage.MakeRepoDir(git_storage.RepoRelPath(ownerDir.Name(), repoDir.Name())) assert.NoError(t, err) } } diff --git a/services/repository/adopt_test.go b/services/repository/adopt_test.go index 132b9aaf42440..5241008242623 100644 --- a/services/repository/adopt_test.go +++ b/services/repository/adopt_test.go @@ -65,7 +65,7 @@ func TestCheckUnadoptedRepositories(t *testing.T) { func TestListUnadoptedRepositories_ListOptions(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) username := "user2" - unadoptedList := []string{path.Join(username, "unadopted1"), path.Join(username, "unadopted2")} + unadoptedList := []string{path.Join(username, "unadopted1.git"), path.Join(username, "unadopted2.git")} for _, unadopted := range unadoptedList { _ = storage.MakeRepoDir(unadopted) } diff --git a/services/repository/files/update.go b/services/repository/files/update.go index 01bf2ace00933..d69a39047d25a 100644 --- a/services/repository/files/update.go +++ b/services/repository/files/update.go @@ -148,7 +148,10 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use opts.NewBranch = opts.OldBranch } + fmt.Println("1----", repo.RepoPath()) + gitRepo, closer, err := git.RepositoryFromContextOrOpen(ctx, repo.RepoPath()) + fmt.Println("2---", err) if err != nil { return nil, err } @@ -263,6 +266,8 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use } } + fmt.Println("3---") + if hasOldBranch { // Get the commit of the original branch commit, err := t.GetBranchCommit(opts.OldBranch) @@ -294,8 +299,10 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use switch file.Operation { case "create", "update": if err := CreateOrUpdateFile(ctx, t, file, contentStore, repo.ID, hasOldBranch); err != nil { + fmt.Println("4---") return nil, err } + fmt.Println("5---") case "delete": // Remove the file from the index if err := t.RemoveFilesFromIndex(file.TreePath); err != nil { @@ -335,6 +342,7 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use } filesResponse, err := GetFilesResponseFromCommit(ctx, repo, commit, opts.NewBranch, treePaths) + fmt.Println("6---", err) if err != nil { return nil, err } From 69db911d1a8de4935d2e78748e79a43d605e4d0b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 5 Jun 2023 16:07:58 +0800 Subject: [PATCH 03/10] Fix test --- models/migrations/base/tests.go | 2 +- models/migrations/v1_9/v82.go | 16 ++--------- models/unittest/testdb.go | 11 ++++---- modules/git/git.go | 9 ++++--- modules/git/storage/storage.go | 27 +++++++++++-------- modules/test/context_tests.go | 2 ++ services/repository/adopt_test.go | 4 +-- services/repository/files/content_test.go | 7 ----- services/repository/files/main_test.go | 17 ++++++++++++ .../migration-test/migration_test.go | 3 ++- 10 files changed, 53 insertions(+), 45 deletions(-) create mode 100644 services/repository/files/main_test.go diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index 6c5d7a1200aca..b5b157253a81f 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -37,7 +37,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En ourSkip += skip deferFn := testlogger.PrintCurrentTest(t, ourSkip) assert.NoError(t, storage.RemoveAll("")) - assert.NoError(t, storage.CopyDir(filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) + assert.NoError(t, storage.UploadDir(filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) ownerDirs, err := storage.ReadDir("") if err != nil { assert.NoError(t, err, "unable to read the new repo root: %v\n", err) diff --git a/models/migrations/v1_9/v82.go b/models/migrations/v1_9/v82.go index 26806dd64505d..9dd6a3707b931 100644 --- a/models/migrations/v1_9/v82.go +++ b/models/migrations/v1_9/v82.go @@ -5,11 +5,9 @@ package v1_9 //nolint import ( "fmt" - "path/filepath" - "strings" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/git/storage" "xorm.io/xorm" ) @@ -33,16 +31,6 @@ func FixReleaseSha1OnReleaseTable(x *xorm.Engine) error { Name string } - // UserPath returns the path absolute path of user repositories. - UserPath := func(userName string) string { - return filepath.Join(setting.RepoRootPath, strings.ToLower(userName)) - } - - // RepoPath returns repository path by given user and repository name. - RepoPath := func(userName, repoName string) string { - return filepath.Join(UserPath(userName), strings.ToLower(repoName)+".git") - } - // Update release sha1 const batchSize = 100 sess := x.NewSession() @@ -98,7 +86,7 @@ func FixReleaseSha1OnReleaseTable(x *xorm.Engine) error { userCache[repo.OwnerID] = user } - gitRepo, err = git.OpenRepository(git.DefaultContext, RepoPath(user.Name, repo.Name)) + gitRepo, err = git.OpenRepository(git.DefaultContext, storage.RepoPath(user.Name, repo.Name)) if err != nil { return err } diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index 6b75b18906891..02e6da13edeff 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -140,16 +140,17 @@ func MainTest(m *testing.M, testOpts *TestOptions) { fatalTestError("models.Init: %v\n", err) } + if err = git.InitFull(context.Background()); err != nil { + fatalTestError("git.Init: %v\n", err) + } + if err = git_storage.RemoveAll(""); err != nil { fatalTestError("git_storage.RemoveAll: %v\n", err) } - if err = git_storage.CopyDir(filepath.Join(testOpts.GiteaRootPath, "tests", "gitea-repositories-meta"), ""); err != nil { + if err = git_storage.UploadDir(filepath.Join(testOpts.GiteaRootPath, "tests", "gitea-repositories-meta"), ""); err != nil { fatalTestError("git_storage.CopyDir: %v\n", err) } - if err = git.InitFull(context.Background()); err != nil { - fatalTestError("git.Init: %v\n", err) - } ownerDirs, err := git_storage.ReadDir("") if err != nil { fatalTestError("unable to read the new repo root: %v\n", err) @@ -232,7 +233,7 @@ func PrepareTestEnv(t testing.TB) { assert.NoError(t, PrepareTestDatabase()) assert.NoError(t, git_storage.RemoveAll("")) metaPath := filepath.Join(giteaRoot, "tests", "gitea-repositories-meta") - assert.NoError(t, git_storage.CopyDir(metaPath, "")) + assert.NoError(t, git_storage.UploadDir(metaPath, "")) ownerDirs, err := git_storage.ReadDir("") assert.NoError(t, err) for _, ownerDir := range ownerDirs { diff --git a/modules/git/git.go b/modules/git/git.go index f9c0ed669f490..2517dfbbe7ea3 100644 --- a/modules/git/git.go +++ b/modules/git/git.go @@ -16,6 +16,7 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -153,6 +154,10 @@ func InitSimple(ctx context.Context) error { return err } + if err := storage.Init(); err != nil { + return err + } + DefaultContext = ctx globalCommandArgs = nil @@ -166,10 +171,6 @@ func InitSimple(ctx context.Context) error { // InitFull initializes git module with version check and change global variables, sync gitconfig. // It should only be called once at the beginning of the program initialization (TestMain/GlobalInitInstalled) as this code makes unsynchronized changes to variables. func InitFull(ctx context.Context) (err error) { - if err = checkInit(); err != nil { - return err - } - if err = InitSimple(ctx); err != nil { return } diff --git a/modules/git/storage/storage.go b/modules/git/storage/storage.go index 84d4d2999baad..ed3a362fc60a5 100644 --- a/modules/git/storage/storage.go +++ b/modules/git/storage/storage.go @@ -24,7 +24,7 @@ type Storage interface { MakeRepoDir(repoRelPath string) error RemoveAll(path string) error ReadDir(owner string) ([]fs.DirEntry, error) - CopyDir(source, target string) error + UploadDir(localSource, targetRelPath string) error Rename(oldPath, newPath string) error } @@ -32,8 +32,6 @@ type LocalSingleStorage struct { repoRootPath string } -var _ Storage = &LocalSingleStorage{} - func (l *LocalSingleStorage) absPath(relPath string) string { return filepath.Join(l.repoRootPath, relPath) } @@ -48,11 +46,11 @@ func (l *LocalSingleStorage) CheckStats() error { } func (l *LocalSingleStorage) IsExist(path string) (bool, error) { - return util.IsExist(path) + return util.IsExist(l.absPath(path)) } func (l *LocalSingleStorage) IsDir(path string) (bool, error) { - return util.IsDir(path) + return util.IsDir(l.absPath(path)) } func (l *LocalSingleStorage) MakeDir(dir string, perm os.FileMode) error { @@ -76,18 +74,25 @@ func (l *LocalSingleStorage) ReadDir(owner string) ([]fs.DirEntry, error) { return os.ReadDir(l.absPath(owner)) } -func (l *LocalSingleStorage) CopyDir(source, target string) error { - return util.CopyDir(source, l.absPath(target)) +func (l *LocalSingleStorage) UploadDir(localSource, targetRelPath string) error { + return util.CopyDir(localSource, l.absPath(targetRelPath)) } func (l *LocalSingleStorage) Rename(oldPath, newPath string) error { return util.Rename(l.absPath(oldPath), l.absPath(newPath)) } -func getStorage() Storage { - return &LocalSingleStorage{ +var storage Storage + +func Init() error { + storage = &LocalSingleStorage{ repoRootPath: setting.RepoRootPath, } + return nil +} + +func getStorage() Storage { + return storage } func IsConfigured() bool { @@ -122,8 +127,8 @@ func ReadDir(owner string) ([]fs.DirEntry, error) { return getStorage().ReadDir(owner) } -func CopyDir(source, target string) error { - return getStorage().CopyDir(source, target) +func UploadDir(source, target string) error { + return getStorage().UploadDir(source, target) } func Rename(oldPath, newPath string) error { diff --git a/modules/test/context_tests.go b/modules/test/context_tests.go index 349c7e3e8045f..266a14d124c11 100644 --- a/modules/test/context_tests.go +++ b/modules/test/context_tests.go @@ -5,6 +5,7 @@ package test import ( gocontext "context" + "fmt" "io" "net/http" "net/http/httptest" @@ -142,6 +143,7 @@ func LoadUser(t *testing.T, ctx gocontext.Context, userID int64) { func LoadGitRepo(t *testing.T, ctx *context.Context) { assert.NoError(t, ctx.Repo.Repository.LoadOwner(ctx)) var err error + fmt.Printf("----%v\n", ctx.Repo.Repository.RepoPath()) ctx.Repo.GitRepo, err = git.OpenRepository(ctx, ctx.Repo.Repository.RepoPath()) assert.NoError(t, err) } diff --git a/services/repository/adopt_test.go b/services/repository/adopt_test.go index 5241008242623..d6a682a4e90d4 100644 --- a/services/repository/adopt_test.go +++ b/services/repository/adopt_test.go @@ -65,9 +65,9 @@ func TestCheckUnadoptedRepositories(t *testing.T) { func TestListUnadoptedRepositories_ListOptions(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) username := "user2" - unadoptedList := []string{path.Join(username, "unadopted1.git"), path.Join(username, "unadopted2.git")} + unadoptedList := []string{path.Join(username, "unadopted1"), path.Join(username, "unadopted2")} for _, unadopted := range unadoptedList { - _ = storage.MakeRepoDir(unadopted) + _ = storage.MakeRepoDir(unadopted + ".git") } opts := db.ListOptions{Page: 1, PageSize: 1} diff --git a/services/repository/files/content_test.go b/services/repository/files/content_test.go index afb9a82a7c6aa..803bc975cb6ba 100644 --- a/services/repository/files/content_test.go +++ b/services/repository/files/content_test.go @@ -4,7 +4,6 @@ package files import ( - "path/filepath" "testing" "code.gitea.io/gitea/models/unittest" @@ -16,12 +15,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestMain(m *testing.M) { - unittest.MainTest(m, &unittest.TestOptions{ - GiteaRootPath: filepath.Join("..", "..", ".."), - }) -} - func getExpectedReadmeContentsResponse() *api.ContentsResponse { treePath := "README.md" sha := "4b4851ad51df6a7d9f25c979345979eaeb5b349f" diff --git a/services/repository/files/main_test.go b/services/repository/files/main_test.go new file mode 100644 index 0000000000000..cfd6329050820 --- /dev/null +++ b/services/repository/files/main_test.go @@ -0,0 +1,17 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "path/filepath" + "testing" + + "code.gitea.io/gitea/models/unittest" +) + +func TestMain(m *testing.M) { + unittest.MainTest(m, &unittest.TestOptions{ + GiteaRootPath: filepath.Join("..", "..", ".."), + }) +} diff --git a/tests/integration/migration-test/migration_test.go b/tests/integration/migration-test/migration_test.go index 175b602bbf542..beba7a6fa6bc3 100644 --- a/tests/integration/migration-test/migration_test.go +++ b/tests/integration/migration-test/migration_test.go @@ -64,6 +64,8 @@ func initMigrationTest(t *testing.T) func() { unittest.InitSettings() + assert.NoError(t, git.InitFull(context.Background())) + assert.True(t, storage.IsConfigured()) assert.NoError(t, storage.RemoveAll("")) assert.NoError(t, storage.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) @@ -84,7 +86,6 @@ func initMigrationTest(t *testing.T) func() { } } - assert.NoError(t, git.InitFull(context.Background())) setting.LoadDBSetting() setting.InitLoggersForTest() return deferFn From 8e7dcec7d205aff2dcf1bbb70f7cf84c1bdaad3c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 8 Jun 2023 18:50:01 +0800 Subject: [PATCH 04/10] Move more repository root refs --- cmd/serv.go | 11 ------- modules/git/storage/legacy.go | 4 +-- modules/git/storage/storage.go | 32 +++++++++++++------ routers/web/admin/config.go | 3 +- templates/admin/config.tmpl | 2 +- .../migration-test/migration_test.go | 2 +- 6 files changed, 28 insertions(+), 26 deletions(-) diff --git a/cmd/serv.go b/cmd/serv.go index 87bf1cce20e4e..6c9c5796049be 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -66,17 +66,6 @@ func setup(ctx context.Context, debug bool) { setting.RunMode = "dev" } - // Check if setting.RepoRootPath exists. It could be the case that it doesn't exist, this can happen when - // `[repository]` `ROOT` is a relative path and $GITEA_WORK_DIR isn't passed to the SSH connection. - if _, err := os.Stat(setting.RepoRootPath); err != nil { - if os.IsNotExist(err) { - _ = fail(ctx, "Incorrect configuration, no repository directory.", "Directory `[repository].ROOT` %q was not found, please check if $GITEA_WORK_DIR is passed to the SSH connection or make `[repository].ROOT` an absolute value.", setting.RepoRootPath) - } else { - _ = fail(ctx, "Incorrect configuration, repository directory is inaccessible", "Directory `[repository].ROOT` %q is inaccessible. err: %v", setting.RepoRootPath, err) - } - return - } - if err := git.InitSimple(context.Background()); err != nil { _ = fail(ctx, "Failed to init git", "Failed to init git, err: %v", err) } diff --git a/modules/git/storage/legacy.go b/modules/git/storage/legacy.go index aae854f5c8e14..4b83bf85f5531 100644 --- a/modules/git/storage/legacy.go +++ b/modules/git/storage/legacy.go @@ -21,7 +21,7 @@ func UserPath(userName string) string { //revive:disable-line:exported // RepoPath returns repository path by given user and repository name. func RepoPath(userName, repoName string) string { //revive:disable-line:exported - return filepath.Join(UserPath(userName), strings.ToLower(repoName)+".git") + return absPath(filepath.Join(strings.ToLower(userName), strings.ToLower(repoName)+".git")) } // LocalPath returns local path by given relative path. @@ -31,5 +31,5 @@ func LocalPath(relPath string) string { // WikiPath returns wiki data path by given user and repository name. func WikiPath(userName, repoName string) string { - return filepath.Join(UserPath(userName), strings.ToLower(repoName)+".wiki.git") + return absPath(filepath.Join(strings.ToLower(userName), strings.ToLower(repoName)+".wiki.git")) } diff --git a/modules/git/storage/storage.go b/modules/git/storage/storage.go index ed3a362fc60a5..42d3460f8424f 100644 --- a/modules/git/storage/storage.go +++ b/modules/git/storage/storage.go @@ -4,6 +4,7 @@ package storage import ( + "fmt" "io/fs" "os" "path" @@ -15,7 +16,7 @@ import ( ) type Storage interface { - IsConfigured() bool + Configuration() string // information for configuration, json format CheckStats() error IsExist(path string) (bool, error) IsDir(path string) (bool, error) @@ -32,17 +33,27 @@ type LocalSingleStorage struct { repoRootPath string } -func (l *LocalSingleStorage) absPath(relPath string) string { - return filepath.Join(l.repoRootPath, relPath) +func (l *LocalSingleStorage) Configuration() string { + return "{RepoRootPath: " + l.repoRootPath + "}" } -func (l *LocalSingleStorage) IsConfigured() bool { - return len(l.repoRootPath) != 0 +func (l *LocalSingleStorage) absPath(relPath string) string { + return filepath.Join(l.repoRootPath, relPath) } func (l *LocalSingleStorage) CheckStats() error { - _, err := os.Stat(l.repoRootPath) - return err + // Check if l.repoRootPath exists. It could be the case that it doesn't exist, this can happen when + // `[repository]` `ROOT` is a relative path and $GITEA_WORK_DIR isn't passed to the SSH connection. + if _, err := os.Stat(l.repoRootPath); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("incorrect configuration, no repository directory.", "Directory `[repository].ROOT` %q was not found, please check if $GITEA_WORK_DIR is passed to the SSH connection or make `[repository].ROOT` an absolute value", + l.repoRootPath) + } + return fmt.Errorf("incorrect configuration, repository directory is inaccessible", "Directory `[repository].ROOT` %q is inaccessible. err: %v", + l.repoRootPath, err) + } + + return nil } func (l *LocalSingleStorage) IsExist(path string) (bool, error) { @@ -88,15 +99,16 @@ func Init() error { storage = &LocalSingleStorage{ repoRootPath: setting.RepoRootPath, } - return nil + + return storage.CheckStats() } func getStorage() Storage { return storage } -func IsConfigured() bool { - return getStorage().IsConfigured() +func Configuration() string { + return getStorage().Configuration() } func CheckStats() error { diff --git a/routers/web/admin/config.go b/routers/web/admin/config.go index be662c22efdcc..cebc66dbcc11d 100644 --- a/routers/web/admin/config.go +++ b/routers/web/admin/config.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" + git_storage "code.gitea.io/gitea/modules/git/storage" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -121,7 +122,7 @@ func Config(ctx *context.Context) { ctx.Data["RunMode"] = util.ToTitleCase(setting.RunMode) ctx.Data["GitVersion"] = git.VersionInfo() - ctx.Data["RepoRootPath"] = setting.RepoRootPath + ctx.Data["GitStorageConfiguration"] = git_storage.Configuration() ctx.Data["CustomRootPath"] = setting.CustomPath ctx.Data["StaticRootPath"] = setting.StaticRootPath ctx.Data["LogRootPath"] = setting.Log.RootPath diff --git a/templates/admin/config.tmpl b/templates/admin/config.tmpl index 0c52830ab080d..55a1b0ca79bd6 100644 --- a/templates/admin/config.tmpl +++ b/templates/admin/config.tmpl @@ -35,7 +35,7 @@
{{.locale.Tr "admin.config.repo_root_path"}}
-
{{.RepoRootPath}}
+
{{.GitStorageConfiguration}}
{{.locale.Tr "admin.config.static_file_root_path"}}
{{.StaticRootPath}}
{{.locale.Tr "admin.config.custom_file_root_path"}}
diff --git a/tests/integration/migration-test/migration_test.go b/tests/integration/migration-test/migration_test.go index 28663b75cc00e..baaa41f3ce31d 100644 --- a/tests/integration/migration-test/migration_test.go +++ b/tests/integration/migration-test/migration_test.go @@ -66,7 +66,7 @@ func initMigrationTest(t *testing.T) func() { assert.NoError(t, git.InitFull(context.Background())) - assert.True(t, storage.IsConfigured()) + assert.NoError(t, storage.CheckStats()) assert.NoError(t, storage.RemoveAll("")) assert.NoError(t, storage.UploadDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) ownerDirs, err := storage.ReadDir("") From eac4d58c50c2a4fbe9084728751e3ac55c3358fa Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 9 Jun 2023 10:41:06 +0800 Subject: [PATCH 05/10] Fix bug --- modules/git/storage/storage.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/git/storage/storage.go b/modules/git/storage/storage.go index 42d3460f8424f..26f196a754c1e 100644 --- a/modules/git/storage/storage.go +++ b/modules/git/storage/storage.go @@ -46,10 +46,10 @@ func (l *LocalSingleStorage) CheckStats() error { // `[repository]` `ROOT` is a relative path and $GITEA_WORK_DIR isn't passed to the SSH connection. if _, err := os.Stat(l.repoRootPath); err != nil { if os.IsNotExist(err) { - return fmt.Errorf("incorrect configuration, no repository directory.", "Directory `[repository].ROOT` %q was not found, please check if $GITEA_WORK_DIR is passed to the SSH connection or make `[repository].ROOT` an absolute value", + return fmt.Errorf("directory `[repository].ROOT` %q was not found, please check if $GITEA_WORK_DIR is passed to the SSH connection or make `[repository].ROOT` an absolute value", l.repoRootPath) } - return fmt.Errorf("incorrect configuration, repository directory is inaccessible", "Directory `[repository].ROOT` %q is inaccessible. err: %v", + return fmt.Errorf("directory `[repository].ROOT` %q is inaccessible. err: %v", l.repoRootPath, err) } From 5c390c0f2ff27b7877c5fe4823858f4b1dfdf61e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 9 Jun 2023 18:43:41 +0800 Subject: [PATCH 06/10] Fix bug --- models/repo/wiki.go | 5 +++-- .../integration/migration-test/migration_test.go | 16 ++++++---------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/models/repo/wiki.go b/models/repo/wiki.go index 131668fe05225..438cdb4af6f5b 100644 --- a/models/repo/wiki.go +++ b/models/repo/wiki.go @@ -81,9 +81,10 @@ func (repo *Repository) WikiPath() string { // HasWiki returns true if repository has wiki. func (repo *Repository) HasWiki() bool { - isDir, err := storage.IsDir(storage.WikiRelPath(repo.OwnerName, repo.Name)) + relWikiPath := storage.WikiRelPath(repo.OwnerName, repo.Name) + isDir, err := storage.IsDir(relWikiPath) if err != nil { - log.Error("Unable to check if %s is a directory: %v", repo.WikiPath(), err) + log.Error("Unable to check if %s is a directory: %v", relWikiPath, err) } return isDir } diff --git a/tests/integration/migration-test/migration_test.go b/tests/integration/migration-test/migration_test.go index baaa41f3ce31d..f48056abd0865 100644 --- a/tests/integration/migration-test/migration_test.go +++ b/tests/integration/migration-test/migration_test.go @@ -63,31 +63,27 @@ func initMigrationTest(t *testing.T) func() { } unittest.InitSettings() + setting.LoadDBSetting() + setting.InitLoggersForTest() assert.NoError(t, git.InitFull(context.Background())) - - assert.NoError(t, storage.CheckStats()) assert.NoError(t, storage.RemoveAll("")) assert.NoError(t, storage.UploadDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), "")) ownerDirs, err := storage.ReadDir("") - if err != nil { - assert.NoError(t, err, "unable to read the new repo root: %v\n", err) - } + assert.NoError(t, err, "unable to read the new repo root: %v\n", err) + for _, ownerDir := range ownerDirs { if !ownerDir.Type().IsDir() { continue } repoDirs, err := storage.ReadDir(ownerDir.Name()) - if err != nil { - assert.NoError(t, err, "unable to read the new repo root: %v\n", err) - } + assert.NoError(t, err, "unable to read the new repo root: %v\n", err) + for _, repoDir := range repoDirs { storage.MakeRepoDir(storage.RepoRelPath(ownerDir.Name(), repoDir.Name())) } } - setting.LoadDBSetting() - setting.InitLoggersForTest() return deferFn } From 9e74c656963eaba1bed85fdc9e5b8cc12a0904c9 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 24 Jul 2023 13:48:49 +0800 Subject: [PATCH 07/10] Fix empty repository --- .../user30/empty.git/config | 6 +++--- .../user30/empty.git/hooks/post-receive | 16 ++++++++++++++++ .../user30/empty.git/hooks/post-receive.d/gitea | 3 +++ .../user30/empty.git/hooks/pre-receive | 16 ++++++++++++++++ .../user30/empty.git/hooks/pre-receive.d/gitea | 3 +++ .../user30/empty.git/hooks/proc-receive | 3 +++ .../user30/empty.git/hooks/proc-receive.d/gitea | 0 .../user30/empty.git/hooks/update | 15 +++++++++++++++ .../user30/empty.git/hooks/update.d/gitea | 3 +++ .../user30/empty.git/info/refs | 0 .../user30/empty.git/objects/info/packs | 1 + .../user30/renderer.git/description | 1 - 12 files changed, 63 insertions(+), 4 deletions(-) create mode 100755 tests/gitea-repositories-meta/user30/empty.git/hooks/post-receive create mode 100755 tests/gitea-repositories-meta/user30/empty.git/hooks/post-receive.d/gitea create mode 100755 tests/gitea-repositories-meta/user30/empty.git/hooks/pre-receive create mode 100755 tests/gitea-repositories-meta/user30/empty.git/hooks/pre-receive.d/gitea create mode 100755 tests/gitea-repositories-meta/user30/empty.git/hooks/proc-receive create mode 100755 tests/gitea-repositories-meta/user30/empty.git/hooks/proc-receive.d/gitea create mode 100755 tests/gitea-repositories-meta/user30/empty.git/hooks/update create mode 100755 tests/gitea-repositories-meta/user30/empty.git/hooks/update.d/gitea create mode 100644 tests/gitea-repositories-meta/user30/empty.git/info/refs create mode 100644 tests/gitea-repositories-meta/user30/empty.git/objects/info/packs delete mode 100644 tests/gitea-repositories-meta/user30/renderer.git/description diff --git a/tests/gitea-repositories-meta/user30/empty.git/config b/tests/gitea-repositories-meta/user30/empty.git/config index 7c968c3b5a0b8..e6da231579bcc 100644 --- a/tests/gitea-repositories-meta/user30/empty.git/config +++ b/tests/gitea-repositories-meta/user30/empty.git/config @@ -1,6 +1,6 @@ [core] - bare = true repositoryformatversion = 0 - filemode = false - symlinks = false + filemode = true + bare = true ignorecase = true + precomposeunicode = true diff --git a/tests/gitea-repositories-meta/user30/empty.git/hooks/post-receive b/tests/gitea-repositories-meta/user30/empty.git/hooks/post-receive new file mode 100755 index 0000000000000..80ae57095ca0e --- /dev/null +++ b/tests/gitea-repositories-meta/user30/empty.git/hooks/post-receive @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# AUTO GENERATED BY GITEA, DO NOT MODIFY +data=$(cat) +exitcodes="" +hookname=$(basename $0) +GIT_DIR=${GIT_DIR:-$(dirname $0)/..} + +for hook in ${GIT_DIR}/hooks/${hookname}.d/*; do + test -x "${hook}" && test -f "${hook}" || continue + echo "${data}" | "${hook}" + exitcodes="${exitcodes} $?" +done + +for i in ${exitcodes}; do + [ ${i} -eq 0 ] || exit ${i} +done diff --git a/tests/gitea-repositories-meta/user30/empty.git/hooks/post-receive.d/gitea b/tests/gitea-repositories-meta/user30/empty.git/hooks/post-receive.d/gitea new file mode 100755 index 0000000000000..118efffd017ea --- /dev/null +++ b/tests/gitea-repositories-meta/user30/empty.git/hooks/post-receive.d/gitea @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# AUTO GENERATED BY GITEA, DO NOT MODIFY +/Users/lunny/data/Projects/gitea/gitea/gitea hook --config=/Users/lunny/data/Projects/gitea/gitea/custom/conf/app.ini post-receive diff --git a/tests/gitea-repositories-meta/user30/empty.git/hooks/pre-receive b/tests/gitea-repositories-meta/user30/empty.git/hooks/pre-receive new file mode 100755 index 0000000000000..80ae57095ca0e --- /dev/null +++ b/tests/gitea-repositories-meta/user30/empty.git/hooks/pre-receive @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# AUTO GENERATED BY GITEA, DO NOT MODIFY +data=$(cat) +exitcodes="" +hookname=$(basename $0) +GIT_DIR=${GIT_DIR:-$(dirname $0)/..} + +for hook in ${GIT_DIR}/hooks/${hookname}.d/*; do + test -x "${hook}" && test -f "${hook}" || continue + echo "${data}" | "${hook}" + exitcodes="${exitcodes} $?" +done + +for i in ${exitcodes}; do + [ ${i} -eq 0 ] || exit ${i} +done diff --git a/tests/gitea-repositories-meta/user30/empty.git/hooks/pre-receive.d/gitea b/tests/gitea-repositories-meta/user30/empty.git/hooks/pre-receive.d/gitea new file mode 100755 index 0000000000000..7e9e3c25fb246 --- /dev/null +++ b/tests/gitea-repositories-meta/user30/empty.git/hooks/pre-receive.d/gitea @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# AUTO GENERATED BY GITEA, DO NOT MODIFY +/Users/lunny/data/Projects/gitea/gitea/gitea hook --config=/Users/lunny/data/Projects/gitea/gitea/custom/conf/app.ini pre-receive diff --git a/tests/gitea-repositories-meta/user30/empty.git/hooks/proc-receive b/tests/gitea-repositories-meta/user30/empty.git/hooks/proc-receive new file mode 100755 index 0000000000000..d3c140113c60c --- /dev/null +++ b/tests/gitea-repositories-meta/user30/empty.git/hooks/proc-receive @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# AUTO GENERATED BY GITEA, DO NOT MODIFY +/Users/lunny/data/Projects/gitea/gitea/gitea hook --config=/Users/lunny/data/Projects/gitea/gitea/custom/conf/app.ini proc-receive diff --git a/tests/gitea-repositories-meta/user30/empty.git/hooks/proc-receive.d/gitea b/tests/gitea-repositories-meta/user30/empty.git/hooks/proc-receive.d/gitea new file mode 100755 index 0000000000000..e69de29bb2d1d diff --git a/tests/gitea-repositories-meta/user30/empty.git/hooks/update b/tests/gitea-repositories-meta/user30/empty.git/hooks/update new file mode 100755 index 0000000000000..b3570b0004cfa --- /dev/null +++ b/tests/gitea-repositories-meta/user30/empty.git/hooks/update @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# AUTO GENERATED BY GITEA, DO NOT MODIFY +exitcodes="" +hookname=$(basename $0) +GIT_DIR=${GIT_DIR:-$(dirname $0/..)} + +for hook in ${GIT_DIR}/hooks/${hookname}.d/*; do + test -x "${hook}" && test -f "${hook}" || continue + "${hook}" $1 $2 $3 + exitcodes="${exitcodes} $?" +done + +for i in ${exitcodes}; do + [ ${i} -eq 0 ] || exit ${i} +done diff --git a/tests/gitea-repositories-meta/user30/empty.git/hooks/update.d/gitea b/tests/gitea-repositories-meta/user30/empty.git/hooks/update.d/gitea new file mode 100755 index 0000000000000..50fbb928a75a2 --- /dev/null +++ b/tests/gitea-repositories-meta/user30/empty.git/hooks/update.d/gitea @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# AUTO GENERATED BY GITEA, DO NOT MODIFY +/Users/lunny/data/Projects/gitea/gitea/gitea hook --config=/Users/lunny/data/Projects/gitea/gitea/custom/conf/app.ini update $1 $2 $3 diff --git a/tests/gitea-repositories-meta/user30/empty.git/info/refs b/tests/gitea-repositories-meta/user30/empty.git/info/refs new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/gitea-repositories-meta/user30/empty.git/objects/info/packs b/tests/gitea-repositories-meta/user30/empty.git/objects/info/packs new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/tests/gitea-repositories-meta/user30/empty.git/objects/info/packs @@ -0,0 +1 @@ + diff --git a/tests/gitea-repositories-meta/user30/renderer.git/description b/tests/gitea-repositories-meta/user30/renderer.git/description deleted file mode 100644 index 04c23973b8a8a..0000000000000 --- a/tests/gitea-repositories-meta/user30/renderer.git/description +++ /dev/null @@ -1 +0,0 @@ -The repository will be used to test third-party renderer in TestExternalMarkupRenderer From 42ee700f54105b866e892d1c9b49f21af87f19c5 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 24 Jul 2023 15:11:50 +0800 Subject: [PATCH 08/10] Change the sequence --- routers/init.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/routers/init.go b/routers/init.go index ddbabcc397447..4705762fc0010 100644 --- a/routers/init.go +++ b/routers/init.go @@ -101,20 +101,21 @@ func syncAppConfForGit(ctx context.Context) error { } func InitWebInstallPage(ctx context.Context) { - translation.InitLocales(ctx) setting.LoadSettingsForInstall() + + translation.InitLocales(ctx) mustInit(svg.Init) } // InitWebInstalled is for global installed configuration. func InitWebInstalled(ctx context.Context) { - mustInitCtx(ctx, git.InitFull) - log.Info("Git version: %s (home: %s)", git.VersionInfo(), git.HomeDir()) + setting.LoadSettings() // Setup i18n translation.InitLocales(ctx) + mustInitCtx(ctx, git.InitFull) + log.Info("Git version: %s (home: %s)", git.VersionInfo(), git.HomeDir()) - setting.LoadSettings() mustInit(storage.Init) mailer.NewContext(ctx) From aee8eb595d1ca8d03ae05484c0db20296f42b511 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 11 Aug 2023 18:35:09 +0800 Subject: [PATCH 09/10] Fix test --- cmd/admin.go | 2 +- cmd/dump_repo.go | 2 +- cmd/serv.go | 2 +- models/migrations/migrations.go | 2 +- modules/git/git.go | 6 +++--- modules/git/git_test.go | 7 +++++++ modules/git/storage/storage.go | 25 ++++++++++++------------ modules/markup/html_test.go | 2 +- modules/markup/markdown/markdown_test.go | 2 +- modules/setting/repository.go | 5 +++++ 10 files changed, 34 insertions(+), 21 deletions(-) diff --git a/cmd/admin.go b/cmd/admin.go index a4f48b0513008..d8c1061d1f689 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -355,7 +355,7 @@ func runRepoSyncReleases(_ *cli.Context) error { return err } - if err := git.InitSimple(ctx); err != nil { + if err := git.InitSimple(ctx, false); err != nil { return err } diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go index 3a24cf6c5f029..c2cb986d639d9 100644 --- a/cmd/dump_repo.go +++ b/cmd/dump_repo.go @@ -88,7 +88,7 @@ func runDumpRepository(ctx *cli.Context) error { } // migrations.GiteaLocalUploader depends on git module - if err := git.InitSimple(context.Background()); err != nil { + if err := git.InitSimple(context.Background(), false); err != nil { return err } diff --git a/cmd/serv.go b/cmd/serv.go index 9ebfc437762f4..7fda8557d73b6 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -67,7 +67,7 @@ func setup(ctx context.Context, debug bool) { setting.RunMode = "dev" } - if err := git.InitSimple(context.Background()); err != nil { + if err := git.InitSimple(context.Background(), false); err != nil { _ = fail(ctx, "Failed to init git", "Failed to init git, err: %v", err) } } diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index b2140a1eb1327..c5d1828c43610 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -615,7 +615,7 @@ Please try upgrading to a lower version first (suggested v1.6.4), then upgrade t // Some migration tasks depend on the git command if git.DefaultContext == nil { - if err = git.InitSimple(context.Background()); err != nil { + if err = git.InitSimple(context.Background(), true); err != nil { return err } } diff --git a/modules/git/git.go b/modules/git/git.go index 6caed47db13ad..bec92af4bba7d 100644 --- a/modules/git/git.go +++ b/modules/git/git.go @@ -149,12 +149,12 @@ func HomeDir() string { // InitSimple initializes git module with a very simple step, no config changes, no global command arguments. // This method doesn't change anything to filesystem. At the moment, it is only used by some Gitea sub-commands. -func InitSimple(ctx context.Context) error { +func InitSimple(ctx context.Context, allowCreation bool) error { if err := checkInit(); err != nil { return err } - if err := storage.Init(); err != nil { + if err := storage.Init(allowCreation); err != nil { return err } @@ -171,7 +171,7 @@ func InitSimple(ctx context.Context) error { // InitFull initializes git module with version check and change global variables, sync gitconfig. // It should only be called once at the beginning of the program initialization (TestMain/GlobalInitInstalled) as this code makes unsynchronized changes to variables. func InitFull(ctx context.Context) (err error) { - if err = InitSimple(ctx); err != nil { + if err = InitSimple(ctx, true); err != nil { return err } diff --git a/modules/git/git_test.go b/modules/git/git_test.go index 37ab669ea45fb..753be46c2a3b6 100644 --- a/modules/git/git_test.go +++ b/modules/git/git_test.go @@ -24,6 +24,13 @@ func testRun(m *testing.M) error { defer util.RemoveAll(gitHomePath) setting.Git.HomePath = gitHomePath + gitReposPath, err := os.MkdirTemp(os.TempDir(), "git-repos") + if err != nil { + return fmt.Errorf("unable to create temp dir: %w", err) + } + defer util.RemoveAll(gitReposPath) + setting.RepoRootPath = gitReposPath + if err = InitFull(context.Background()); err != nil { return fmt.Errorf("failed to call Init: %w", err) } diff --git a/modules/git/storage/storage.go b/modules/git/storage/storage.go index 26f196a754c1e..bb09e2caae32c 100644 --- a/modules/git/storage/storage.go +++ b/modules/git/storage/storage.go @@ -17,10 +17,9 @@ import ( type Storage interface { Configuration() string // information for configuration, json format - CheckStats() error + Init(allowCreation bool) error IsExist(path string) (bool, error) IsDir(path string) (bool, error) - // MakeDir(repoRelPath string) error MakeDir(dir string, perm os.FileMode) error MakeRepoDir(repoRelPath string) error RemoveAll(path string) error @@ -41,16 +40,22 @@ func (l *LocalSingleStorage) absPath(relPath string) string { return filepath.Join(l.repoRootPath, relPath) } -func (l *LocalSingleStorage) CheckStats() error { +func (l *LocalSingleStorage) Init(allowCreation bool) error { // Check if l.repoRootPath exists. It could be the case that it doesn't exist, this can happen when // `[repository]` `ROOT` is a relative path and $GITEA_WORK_DIR isn't passed to the SSH connection. if _, err := os.Stat(l.repoRootPath); err != nil { - if os.IsNotExist(err) { + if !os.IsNotExist(err) { + return fmt.Errorf("directory `[repository].ROOT` %q is inaccessible. err: %v", + l.repoRootPath, err) + } + + if !allowCreation { return fmt.Errorf("directory `[repository].ROOT` %q was not found, please check if $GITEA_WORK_DIR is passed to the SSH connection or make `[repository].ROOT` an absolute value", l.repoRootPath) } - return fmt.Errorf("directory `[repository].ROOT` %q is inaccessible. err: %v", - l.repoRootPath, err) + if err := os.MkdirAll(l.repoRootPath, os.ModePerm); err != nil { + return err + } } return nil @@ -95,12 +100,12 @@ func (l *LocalSingleStorage) Rename(oldPath, newPath string) error { var storage Storage -func Init() error { +func Init(allowCreation bool) error { storage = &LocalSingleStorage{ repoRootPath: setting.RepoRootPath, } - return storage.CheckStats() + return storage.Init(allowCreation) } func getStorage() Storage { @@ -111,10 +116,6 @@ func Configuration() string { return getStorage().Configuration() } -func CheckStats() error { - return getStorage().CheckStats() -} - func IsExist(path string) (bool, error) { return getStorage().IsExist(path) } diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index 9156bc63312b6..1bae8653f5056 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -30,7 +30,7 @@ var localMetas = map[string]string{ func TestMain(m *testing.M) { unittest.InitSettings() - if err := git.InitSimple(context.Background()); err != nil { + if err := git.InitSimple(context.Background(), true); err != nil { log.Fatal("git init failed, err: %v", err) } os.Exit(m.Run()) diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index f2322b25544bf..07bdfeedef664 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -35,7 +35,7 @@ var localMetas = map[string]string{ func TestMain(m *testing.M) { unittest.InitSettings() - if err := git.InitSimple(context.Background()); err != nil { + if err := git.InitSimple(context.Background(), true); err != nil { log.Fatal("git init failed, err: %v", err) } markup.Init(&markup.ProcessorHelper{ diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 42ffb99138a13..49b429ff7e775 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -4,6 +4,7 @@ package setting import ( + "os" "os/exec" "path" "path/filepath" @@ -282,6 +283,10 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { } else { RepoRootPath = filepath.Clean(RepoRootPath) } + if err := os.MkdirAll(RepoRootPath, os.ModePerm); err != nil { + log.Fatal("Failed to create Repository root path: %v", err) + } + defaultDetectedCharsetsOrder := make([]string, 0, len(Repository.DetectedCharsetsOrder)) for _, charset := range Repository.DetectedCharsetsOrder { defaultDetectedCharsetsOrder = append(defaultDetectedCharsetsOrder, strings.ToLower(strings.TrimSpace(charset))) From a2febb339cf17a8024ae5a9ff88b11deb2efa0e7 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 11 Aug 2023 21:27:06 +0800 Subject: [PATCH 10/10] Fix test --- models/migrations/base/db.go | 13 ------------- models/migrations/base/tests.go | 5 +++-- models/unittest/testdb.go | 6 +++--- modules/git/storage/storage.go | 10 ++++++++-- 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/models/migrations/base/db.go b/models/migrations/base/db.go index 51351cc7d3297..b935b53762a0b 100644 --- a/models/migrations/base/db.go +++ b/models/migrations/base/db.go @@ -13,7 +13,6 @@ import ( "reflect" "regexp" "strings" - "time" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/unittest" @@ -521,18 +520,6 @@ func ModifyColumn(x *xorm.Engine, tableName string, col *schemas.Column) error { return nil } -func removeAllWithRetry(dir string) error { - var err error - for i := 0; i < 20; i++ { - err = os.RemoveAll(dir) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - return err -} - func newXORMEngine() (*xorm.Engine, error) { if err := db.InitEngine(context.Background()); err != nil { return nil, err diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index 349588f6f95ee..d7e2810ac53ab 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/testlogger" + "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" "xorm.io/xorm" @@ -158,10 +159,10 @@ func MainTest(m *testing.M) { exitStatus := m.Run() - if err := storage.RemoveAll(""); err != nil { + if err := storage.ReBuild(); err != nil { fmt.Fprintf(os.Stderr, "os.RemoveAll: %v\n", err) } - if err := removeAllWithRetry(tmpDataPath); err != nil { + if err := util.RemoveAll(tmpDataPath); err != nil { fmt.Fprintf(os.Stderr, "os.RemoveAll: %v\n", err) } os.Exit(exitStatus) diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index f2c06fb4f3bdd..96398490d6354 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -147,7 +147,7 @@ func MainTest(m *testing.M, testOpts *TestOptions) { } if err = git_storage.RemoveAll(""); err != nil { - fatalTestError("git_storage.RemoveAll: %v\n", err) + fatalTestError("git_storage.ReBuild: %v\n", err) } if err = git_storage.UploadDir(filepath.Join(testOpts.GiteaRootPath, "tests", "gitea-repositories-meta"), ""); err != nil { fatalTestError("git_storage.UploadDir: %v\n", err) @@ -186,8 +186,8 @@ func MainTest(m *testing.M, testOpts *TestOptions) { } } - if err = git_storage.RemoveAll(""); err != nil { - fatalTestError("util.RemoveAll: %v\n", err) + if err = git_storage.ReBuild(); err != nil { + fatalTestError("util.ReBuild: %v\n", err) } if err = util.RemoveAll(appDataPath); err != nil { fatalTestError("util.RemoveAll: %v\n", err) diff --git a/modules/git/storage/storage.go b/modules/git/storage/storage.go index bb09e2caae32c..ae0bef402aa05 100644 --- a/modules/git/storage/storage.go +++ b/modules/git/storage/storage.go @@ -82,8 +82,7 @@ func (l *LocalSingleStorage) MakeRepoDir(repoRelPath string) error { } func (l *LocalSingleStorage) RemoveAll(path string) error { - // TODO: removeAllWithRetry(l.absPath(path)) - return os.RemoveAll(l.absPath(path)) + return util.RemoveAll(l.absPath(path)) } func (l *LocalSingleStorage) ReadDir(owner string) ([]fs.DirEntry, error) { @@ -148,6 +147,13 @@ func Rename(oldPath, newPath string) error { return getStorage().Rename(oldPath, newPath) } +func ReBuild() error { + if err := getStorage().RemoveAll(""); err != nil { + return err + } + return getStorage().MakeDir("", os.ModePerm) +} + // UserRelPath returns the path relative path of user repositories. func UserRelPath(userName string) string { return strings.ToLower(userName)