From b92c372464891a6191e9602aa22a2a3071748fe6 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Tue, 5 Jul 2022 22:14:26 -0400 Subject: [PATCH 01/34] Test git-annex Fixes https://github.com/neuropoly/gitea/issues/11 There's always going to be more cases we could cover but this is a solid start. --- integrations/git_annex_test.go | 223 +++++++++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 integrations/git_annex_test.go diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go new file mode 100644 index 0000000000000..616826e2daa75 --- /dev/null +++ b/integrations/git_annex_test.go @@ -0,0 +1,223 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// I should test both: +// - symlink annexing +// - smudge annexing +// - http annexing +// - ssh annexing +// +// and then cross all that with testing different combinations of permissions +// ..yeah? Is that a reasonable thing to do? + +// it would also be good, probably, to test how push-to-create interacts with git-annex + +package integrations + +import ( + "testing" + "github.com/stretchr/testify/require" + + "errors" + "os" + "io/fs" + "math/rand" + "path" + "path/filepath" + "regexp" + "fmt" + "strings" + "net/url" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + /* + "code.gitea.io/gitea/models/perm" + */ + + //"time" // DEBUG +) + +func TestGitAnnex(t *testing.T) { + /* + // TODO: look into how LFS did this + if !setting.Annex.Enabled { + t.Skip() + } + */ + onGiteaRun(t, func(t *testing.T, u *url.URL) { + defer doCleanAnnexLockdown() // workaround https://git-annex.branchable.com/internals/lockdown/ + + API := NewAPITestContext(t, "user2", "annex-repo1") + require.NotNil(t, API) + + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() + //t.Run("CreateRepo", doAPICreateRepository(API, false)) + doAPICreateRepository(API, false)(t) + + // Setup the user's ssh key + withKeyFile(t, "test-key", func(keyFile string) { + //fmt.Printf("ssh key is at %#v\n", keyFile) // DEBUG + os.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set + doAPICreateUserKey(API, "test-key", keyFile)(t) + + repoURL := createSSHUrl(API.GitPath(), u) + + // Setup clone folder + repoPath, err := os.MkdirTemp("", API.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) + doGitClone(repoPath, repoURL)(t) + + //fmt.Printf("So yeah here's the thing: %#v\n", repoPath) // DEBUG + + doInitAnnexRepo(t, repoPath) + + // Upload + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + + // Verify the upload + + // - method 0: check that 'git annex sync' successfully contacted the remote git-annex + remoteAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + remoteAnnexUUID = strings.TrimSpace(remoteAnnexUUID) + require.Regexp(t, + regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$"), + remoteAnnexUUID, + "git annex sync should have been able to download the remote's annex uuid") + + // Verify the upload: Check that the file was uploaded + + // - method 1: 'git annex whereis' + annexWhereis, _, err := git.NewCommand(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + require.True(t, + strings.Contains(annexWhereis, " -- origin\n"), + "git annex whereis should report the file is uploaded to origin") + + // - method 2: look directly into the remote repo to find the file + annexObjectPath, err := AnnexObjectPath(path.Join(setting.RepoRootPath, API.Username, API.Reponame+".git"), "large.bin") + require.NoError(t, err) + _, stat_err := os.Stat(annexObjectPath) + require.NoError(t, stat_err, "Annexed file should exist in remote .git/annex/objects folder") + // TODO: directly diff the source and target files + + //fmt.Printf("Sleeping now. good luck.\n") // give time to allow manually inspecting the test server; the password for all users is 'password'! + //time.Sleep(2 * time.Second) // DEBUG + + }) + + }) + + }) +} + +func doGenerateRandomFile(size int, path string) (err error) { + // Generate random file + bufSize := 4 * 1024 + if bufSize > size { + bufSize = size + } + + buffer := make([]byte, bufSize) + + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + written := 0 + for written < size { + n := size - written + if n > bufSize { + n = bufSize + } + _, err := rand.Read(buffer[:n]) + if err != nil { + return err + } + n, err = f.Write(buffer[:n]) + if err != nil { + return err + } + written += n + } + if err != nil { + return err + } + + return nil +} + +func doCleanAnnexLockdown() { + // do chmod -R +w $REPOS in order to + // handle https://git-annex.branchable.com/internals/lockdown/ + // > (The only bad consequence of this is that rm -rf .git doesn't work unless you first run chmod -R +w .git) + // If this isn't done, the test can only be run once, because it reuses its gitea-repositories/ path + + filepath.WalkDir(setting.RepoRootPath, func(path string, d fs.DirEntry, err error) error { + if err == nil { + // 0200 == u+w, in octal unix permission notation + info, err := d.Info() + if err != nil { + return err + } + + err = os.Chmod(path, info.Mode()|0200) + if err != nil { + return err + } + } + return nil + }) +} + +func AnnexObjectPath(repoPath string, file string) (string, error) { + // given a repo and a file in it + // TODO: handle other branches, e.g. non-HEAD branches etc + annexKey, _, err := git.NewCommand(git.DefaultContext, "show", "HEAD:"+file).RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { return "", err } + + annexKey = strings.TrimSpace(annexKey) + if ! strings.HasPrefix(annexKey, "/annex/objects/") { + return "", errors.New(fmt.Sprintf("%s/%s does not appear to be annexed .", repoPath, file)) + } + annexKey = strings.TrimPrefix(annexKey, "/annex/objects/") + + // we need to know the two-level folder prefix: https://git-annex.branchable.com/internals/hashing/ + keyHashPrefix, _, err := git.NewCommand(git.DefaultContext, "annex", "examinekey", "--format=${hashdirlower}", annexKey).RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { return "", err } + + // TODO: handle non-bare repos + // if ! bare { repoPath += "/.git", and use hashdirmixed instead of hashdirlower } + + return path.Join(repoPath, "annex", "objects", keyHashPrefix, annexKey, annexKey), nil +} + +func doInitAnnexRepo(t *testing.T, repoPath string) { + // initialize a repo with a some annexed and unannexed files + + // set up what files should be annexed + // in this case, all *.bin files will be annexed + // without this, git-annex's default config annexes every file larger than some number of megabytes + f, err := os.Create(path.Join(repoPath, ".gitattributes")) + require.NoError(t, err) + f.WriteString("*.bin filter=annex annex.largefiles=anything") + f.Close() + + require.NoError(t, git.AddChanges(repoPath, false, ".")) + require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Configure git-annex settings"})) + + require.NoError(t, git.NewCommand(git.DefaultContext, "annex", "init", "gitea-annex-test").Run(&git.RunOpts{Dir: repoPath})) + + doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "large.bin")) + require.NoError(t, git.AddChanges(repoPath, false, ".")) + require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) +} From 81d112c6103ac945e54faa9af16062e487fc354b Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Fri, 29 Jul 2022 03:11:47 -0400 Subject: [PATCH 02/34] Test annex repo deletion. --- integrations/git_annex_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 616826e2daa75..b4e0ec198e359 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -111,6 +111,15 @@ func TestGitAnnex(t *testing.T) { //fmt.Printf("Sleeping now. good luck.\n") // give time to allow manually inspecting the test server; the password for all users is 'password'! //time.Sleep(2 * time.Second) // DEBUG + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(API)(t) + + _, stat_err = os.Stat(annexObjectPath) + require.True(t, os.IsNotExist(stat_err), "Annexed file should not exist in remote .git/annex/objects folder") + + _, stat_err = os.Stat(path.Join(setting.RepoRootPath, API.Username, API.Reponame+".git")) + require.True(t, os.IsNotExist(stat_err), "Remote annex repo should be removed from disk") + }) }) From d804ef8ffde7f5ed193d2c0fe2ac56c380af5f45 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Wed, 6 Jul 2022 15:04:38 -0400 Subject: [PATCH 03/34] Run git-annex tests in CI --- .github/workflows/test-git-annex.yml | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/test-git-annex.yml diff --git a/.github/workflows/test-git-annex.yml b/.github/workflows/test-git-annex.yml new file mode 100644 index 0000000000000..8966048f020aa --- /dev/null +++ b/.github/workflows/test-git-annex.yml @@ -0,0 +1,32 @@ +name: Test Git Annex + # Here we just test the git-annex feature, leaving + # the full test suite for upstream to watch over. + +on: + push: + workflow_call: # so release.yml can depend on this (https://stackoverflow.com/a/71489231) + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install toolchain + run: | + sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y make git git-annex + # per README.md, building needs Go 1.17 and Node LTS + - uses: actions/setup-go@v2 + with: + go-version: '^1.17' # The Go version to download (if necessary) and use. + - uses: actions/setup-node@v2 + with: + node-version: 'lts/*' + + - name: Build Release Assets + run: | + TAGS="bindata sqlite sqlite_unlock_notify" make build + + - name: Test + run: | + make 'test-sqlite#TestGitAnnex' From b24da3af3972f9bcf810fe9fb5be78b85cdd3859 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Sun, 31 Jul 2022 15:13:03 -0400 Subject: [PATCH 04/34] Fully reset state at end of withKeyFile. Previously form, the key file would get erased when withKeyFile() ended, but $GIT_SSH_COMMAND instructing git to try to use that file would stay, so following git calls using git+ssh:// or git@localhost URLs/addresses would get confused. Similarly neither was it possible to safely nest withKeyFile()s, because after the inner one was done, the outer one would be stuck with a $GIT_SSH_COMMAND pointing at the inner's key file. This bug was probably hidden since the only tests that try to use ssh clone URLs are all safely tucked away inside of withKeyFile()s, and using multiple accounts via ssh doesn't really seem to be tested yet. --- .../git_helper_for_declarative_test.go | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/integrations/git_helper_for_declarative_test.go b/integrations/git_helper_for_declarative_test.go index 1ea594b739c8b..f6b5e6f0d11d4 100644 --- a/integrations/git_helper_for_declarative_test.go +++ b/integrations/git_helper_for_declarative_test.go @@ -41,6 +41,28 @@ func withKeyFile(t *testing.T, keyname string, callback func(string)) { "ssh -o \"UserKnownHostsFile=/dev/null\" -o \"StrictHostKeyChecking=no\" -o \"IdentitiesOnly=yes\" -i \""+keyFile+"\" \"$@\""), 0o700) assert.NoError(t, err) + // reset ssh wrapper afterwards + _gitSSH, gitSSHExists := os.LookupEnv("GIT_SSH") + defer func() { + if gitSSHExists { + os.Setenv("GIT_SSH", _gitSSH) + } + }() + + _gitSSHCommand, gitSSHCommandExists := os.LookupEnv("GIT_SSH_COMMAND") + defer func() { + if gitSSHCommandExists { + os.Setenv("GIT_SSH_COMMAND", _gitSSHCommand) + } + }() + + _gitSSHVariant, gitSSHVariantExists := os.LookupEnv("GIT_SSH_VARIANT") + defer func() { + if gitSSHVariantExists { + os.Setenv("GIT_SSH_VARIANT", _gitSSHVariant) + } + }() + // Setup ssh wrapper os.Setenv("GIT_SSH", path.Join(tmpDir, "ssh")) os.Setenv("GIT_SSH_COMMAND", From 7f1d99b75b0a5f8817f1b42cc749db3de3e7b072 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Sun, 31 Jul 2022 17:41:40 -0400 Subject: [PATCH 05/34] withCtxKeyFile() Extends withKeyFile() to automatically activate, and delete, an ssh key in a given account. This is more useful than withKeyFile() because activating a key client-side always implies activating it server-side too. --- .../api_helper_for_declarative_test.go | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/integrations/api_helper_for_declarative_test.go b/integrations/api_helper_for_declarative_test.go index 181a6469467cd..75b528fc9c180 100644 --- a/integrations/api_helper_for_declarative_test.go +++ b/integrations/api_helper_for_declarative_test.go @@ -462,3 +462,23 @@ func doAPIAddRepoToOrganizationTeam(ctx APITestContext, teamID int64, orgName, r ctx.Session.MakeRequest(t, req, http.StatusNoContent) } } + +// generate and activate an ssh key for the user attached to the APITestContext +// TODO: pick a better name; golang doesn't do method overriding. +func withCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { + keyName := "One of " + ctx.Username + "'s keys" + withKeyFile(t, keyName, func(keyFile string) { + + var key api.PublicKey + + doAPICreateUserKey(ctx, keyName, keyFile, + func(t *testing.T, _key api.PublicKey) { + // save the key ID so we can delete it at the end + key = _key + })(t) + + defer doAPIDeleteUserKey(ctx, key.ID)(t) + + callback() + }) +} From 7cd4f7f00601ca41b65f6a2912545b904d2d71f0 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Mon, 1 Aug 2022 14:35:01 -0400 Subject: [PATCH 06/34] createHTTPUrl --- integrations/git_helper_for_declarative_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/integrations/git_helper_for_declarative_test.go b/integrations/git_helper_for_declarative_test.go index f6b5e6f0d11d4..2f19bfd8e58c6 100644 --- a/integrations/git_helper_for_declarative_test.go +++ b/integrations/git_helper_for_declarative_test.go @@ -72,6 +72,13 @@ func withKeyFile(t *testing.T, keyname string, callback func(string)) { callback(keyFile) } +func createHTTPUrl(ctx APITestContext, u *url.URL) *url.URL { + u2 := *u + u2.User = url.UserPassword(ctx.Username, userPassword) // userPassword is a module-level const, for the sake of testing + u2.Path = ctx.GitPath() + return &u2 +} + func createSSHUrl(gitPath string, u *url.URL) *url.URL { u2 := *u u2.Scheme = "ssh" From 6cb27e63ae0a7b9e69054cc46b287a4376d56c17 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Mon, 1 Aug 2022 14:35:22 -0400 Subject: [PATCH 07/34] Rearrange test --- integrations/git_annex_test.go | 256 +++++++++++++++++++++++++-------- 1 file changed, 194 insertions(+), 62 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index b4e0ec198e359..ee2b1dd6970fd 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -16,28 +16,29 @@ package integrations import ( - "testing" "github.com/stretchr/testify/require" + "testing" "errors" - "os" + "fmt" "io/fs" "math/rand" + "net/url" + "os" "path" "path/filepath" "regexp" - "fmt" "strings" - "net/url" + repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" + //"code.gitea.io/gitea/models/perm" /* "code.gitea.io/gitea/models/perm" - */ - - //"time" // DEBUG + *///"time" // DEBUG ) func TestGitAnnex(t *testing.T) { @@ -47,83 +48,116 @@ func TestGitAnnex(t *testing.T) { t.Skip() } */ + + trueBool := true // this is silly but there's places it's needed + falseBool := !trueBool + + // Some guidelines: + // a APITestContext is an awkward union of session credential + username + target repo + // which is assumed to be owned by that username; if you want to target a different + // repo, you need to edit its .Reponame or just ignore it and write "username/reponame.git" + onGiteaRun(t, func(t *testing.T, u *url.URL) { defer doCleanAnnexLockdown() // workaround https://git-annex.branchable.com/internals/lockdown/ - API := NewAPITestContext(t, "user2", "annex-repo1") - require.NotNil(t, API) + // Different sessions, so we can test + // We unset Reponame up here at the top, then later add it according to each test + ownerSession := NewAPITestContext(t, "user2", "") + //readCollaboratorSession := NewAPITestContext(t, "user3", "") + //writeCollaboratorSession := NewAPITestContext(t, "user4", "") + //otherSession := NewAPITestContext(t, "user5", "") // a user with no specific access + // Note: there's also full anonymous access, which is only available for public HTTP repos; it should behave the same as 'other' + // but we test it separately below anyway - t.Run("SSH", func(t *testing.T) { + t.Run("Public", func(t *testing.T) { defer PrintCurrentTest(t)() - //t.Run("CreateRepo", doAPICreateRepository(API, false)) - doAPICreateRepository(API, false)(t) - // Setup the user's ssh key - withKeyFile(t, "test-key", func(keyFile string) { - //fmt.Printf("ssh key is at %#v\n", keyFile) // DEBUG - os.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set - doAPICreateUserKey(API, "test-key", keyFile)(t) + // create a public repo + s := ownerSession // copy to prevent cross-contamination + s.Reponame = "annex-public" + doAPICreateRepository(s, false)(t) + doAPIEditRepository(s, &api.EditRepoOption{Private: &falseBool})(t) // make the repo public + // double-check it's public (this should be taken care of by models/fixtures/repository.yml, but better to check) + repo, err := repo_model.GetRepositoryByOwnerAndName(s.Username, s.Reponame) + require.NoError(t, err) + require.True(t, !repo.IsPrivate) + + // set up collaborators + //doAPIAddCollaborator(s, readCollaboratorSession.Username, perm.AccessModeRead)(t) + //doAPIAddCollaborator(s, writeCollaboratorSession.Username, perm.AccessModeWrite)(t) - repoURL := createSSHUrl(API.GitPath(), u) + sshURL := createSSHUrl(s.GitPath(), u) + //httpURL := createSSHUrl(s.GitPath(), u) // XXX this puts username and password into the URL + // anonHTTPUrl := ??? - // Setup clone folder - repoPath, err := os.MkdirTemp("", API.Reponame) - require.NoError(t, err) - defer util.RemoveAll(repoPath) - doGitClone(repoPath, repoURL)(t) + doAPIAnnexInitRepository(t, s, u) // XXX this function always uses ssh, so we don't have a way to test git-annex-push-to-create-over-http; - //fmt.Printf("So yeah here's the thing: %#v\n", repoPath) // DEBUG + t.Run("Owner", func(t *testing.T) { + defer PrintCurrentTest(t)() + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() - doInitAnnexRepo(t, repoPath) + repoURL := sshURL - // Upload - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) + withAnnexCtxKeyFile(t, ownerSession, func() { + repoPath, err := os.MkdirTemp("", s.Reponame) + require.NoError(t, err) + //defer util.RemoveAll(repoPath) + doAnnexClone(t, repoPath, repoURL) - // Verify the upload + t.Run("Contribute", func(t *testing.T) { + defer PrintCurrentTest(t)() - // - method 0: check that 'git annex sync' successfully contacted the remote git-annex - remoteAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) - remoteAnnexUUID = strings.TrimSpace(remoteAnnexUUID) - require.Regexp(t, - regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$"), - remoteAnnexUUID, - "git annex sync should have been able to download the remote's annex uuid") + }) + }) + }) + }) - // Verify the upload: Check that the file was uploaded + /* + t.Run("ReadCollaborator", func(t *testing.T) { + defer PrintCurrentTest(t)() - // - method 1: 'git annex whereis' - annexWhereis, _, err := git.NewCommand(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) - require.True(t, - strings.Contains(annexWhereis, " -- origin\n"), - "git annex whereis should report the file is uploaded to origin") + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() + repoURL := sshURL - // - method 2: look directly into the remote repo to find the file - annexObjectPath, err := AnnexObjectPath(path.Join(setting.RepoRootPath, API.Username, API.Reponame+".git"), "large.bin") - require.NoError(t, err) - _, stat_err := os.Stat(annexObjectPath) - require.NoError(t, stat_err, "Annexed file should exist in remote .git/annex/objects folder") - // TODO: directly diff the source and target files + withAnnexCtxKeyFile(t, readCollaboratorSession, func() { - //fmt.Printf("Sleeping now. good luck.\n") // give time to allow manually inspecting the test server; the password for all users is 'password'! - //time.Sleep(2 * time.Second) // DEBUG + repoPath, err := os.MkdirTemp("", s.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) - // Delete the repo, make sure it's fully gone - doAPIDeleteRepository(API)(t) + doAnnexClone(t, repoPath, repoURL) - _, stat_err = os.Stat(annexObjectPath) - require.True(t, os.IsNotExist(stat_err), "Annexed file should not exist in remote .git/annex/objects folder") - _, stat_err = os.Stat(path.Join(setting.RepoRootPath, API.Username, API.Reponame+".git")) - require.True(t, os.IsNotExist(stat_err), "Remote annex repo should be removed from disk") + // now what? + + // now we try to upload again and see what happens + + t.Run("Contribute", func(t *testing.T) { + defer PrintCurrentTest(t)() + + }) + }) + }) + }) + */ + + t.Run("Delete", func(t *testing.T) { + defer PrintCurrentTest(t)() + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(s)(t) + _, stat_err := os.Stat(path.Join(setting.RepoRootPath, s.GitPath())) + require.True(t, os.IsNotExist(stat_err), "Remote annex repo should be removed from disk") }) }) + //fmt.Printf("Sleeping now. good luck.\n") // give time to allow manually inspecting the test server; the password for all users is 'password'! + //time.Sleep(2 * time.Second) // DEBUG + }) } @@ -192,17 +226,21 @@ func AnnexObjectPath(repoPath string, file string) (string, error) { // given a repo and a file in it // TODO: handle other branches, e.g. non-HEAD branches etc annexKey, _, err := git.NewCommand(git.DefaultContext, "show", "HEAD:"+file).RunStdString(&git.RunOpts{Dir: repoPath}) - if err != nil { return "", err } + if err != nil { + return "", err + } annexKey = strings.TrimSpace(annexKey) - if ! strings.HasPrefix(annexKey, "/annex/objects/") { + if !strings.HasPrefix(annexKey, "/annex/objects/") { return "", errors.New(fmt.Sprintf("%s/%s does not appear to be annexed .", repoPath, file)) } annexKey = strings.TrimPrefix(annexKey, "/annex/objects/") // we need to know the two-level folder prefix: https://git-annex.branchable.com/internals/hashing/ keyHashPrefix, _, err := git.NewCommand(git.DefaultContext, "annex", "examinekey", "--format=${hashdirlower}", annexKey).RunStdString(&git.RunOpts{Dir: repoPath}) - if err != nil { return "", err } + if err != nil { + return "", err + } // TODO: handle non-bare repos // if ! bare { repoPath += "/.git", and use hashdirmixed instead of hashdirlower } @@ -210,8 +248,89 @@ func AnnexObjectPath(repoPath string, file string) (string, error) { return path.Join(repoPath, "annex", "objects", keyHashPrefix, annexKey, annexKey), nil } -func doInitAnnexRepo(t *testing.T, repoPath string) { +func doAnnexClone(t *testing.T, repoPath string, repoURL *url.URL) { + doGitClone(repoPath, repoURL)(t) + + _, _, git_err := git.NewCommand(git.DefaultContext, "annex", "get", ".").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, git_err) + + // Verify the download + + // - method 0: check that 'git annex get' successfully contacted the remote git-annex + remoteAnnexUUID, _, git_err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, git_err) + remoteAnnexUUID = strings.TrimSpace(remoteAnnexUUID) + require.Regexp(t, + regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$"), + remoteAnnexUUID, + "git annex sync should have been able to download the remote's annex uuid") + + // TODO: scan for all annexed files? + + // - method 2: look into .git/annex/objects to find the annexed file + annexObjectPath, err := AnnexObjectPath(repoPath, "large.bin") + require.NoError(t, err) + _, stat_err := os.Stat(annexObjectPath) + require.NoError(t, stat_err, "Annexed file should exist in remote .git/annex/objects folder") + +} + +func doAPIAnnexInitRepository(t *testing.T, ctx APITestContext, u *url.URL) { + + API := ctx // TODO: change the names + + // ohhhh right. I need to install an ssh key here. dammit. + withAnnexCtxKeyFile(t, ctx, func() { + // Setup clone folder + repoPath, err := os.MkdirTemp("", API.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) + + repoURL := createSSHUrl(ctx.GitPath(), u) + doGitClone(repoPath, repoURL)(t) + + //fmt.Printf("So yeah here's the thing: %#v\n", repoPath) // DEBUG + + doAnnexInitRepository(t, repoPath) + + // Upload + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + // Verify the upload + + // - method 0: check that 'git annex sync' successfully contacted the remote git-annex + remoteAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + remoteAnnexUUID = strings.TrimSpace(remoteAnnexUUID) + require.Regexp(t, + regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$"), + remoteAnnexUUID, + "git annex sync should have been able to download the remote's annex uuid") + + // Verify the upload: Check that the file was uploaded + + // - method 1: 'git annex whereis' + annexWhereis, _, err := git.NewCommand(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + require.True(t, + strings.Contains(annexWhereis, " -- origin\n"), + "git annex whereis should report the file is uploaded to origin") + + // - method 2: look directly into the remote repo to find the file + annexObjectPath, err := AnnexObjectPath(path.Join(setting.RepoRootPath, API.Username, API.Reponame+".git"), "large.bin") + require.NoError(t, err) + _, stat_err := os.Stat(annexObjectPath) + require.NoError(t, stat_err, "Annexed file should exist in remote .git/annex/objects folder") + // TODO: directly diff the source and target files + }) +} + +func doAnnexInitRepository(t *testing.T, repoPath string) { // initialize a repo with a some annexed and unannexed files + // TODO: this could be replaced with a fixture repo; see + // integrations/gitea-repositories-meta/ and models/fixtures/repository.yml + // However we reuse this many times. // set up what files should be annexed // in this case, all *.bin files will be annexed @@ -230,3 +349,16 @@ func doInitAnnexRepo(t *testing.T, repoPath string) { require.NoError(t, git.AddChanges(repoPath, false, ".")) require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) } + +func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { + os.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set + + _gitAnnexUseGitSSH, gitAnnexUseGitSSHExists := os.LookupEnv("GIT_ANNEX_USE_GIT_SSH") + defer func() { + if gitAnnexUseGitSSHExists { + os.Setenv("GIT_ANNEX_USE_GIT_SSH", _gitAnnexUseGitSSH) + } + }() + + withCtxKeyFile(t, ctx, callback) +} From a9d829846e7da484ab898457fb47423f63465c91 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Mon, 1 Aug 2022 14:41:13 -0400 Subject: [PATCH 08/34] AnnexObjectPath: handle non-bare repos, too --- integrations/git_annex_test.go | 35 +++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index ee2b1dd6970fd..d579357b14913 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -223,6 +223,24 @@ func doCleanAnnexLockdown() { } func AnnexObjectPath(repoPath string, file string) (string, error) { + + // find the path inside *.git/annex/objects of a given file + // i.e. figure out its two-level hash prefix: https://git-annex.branchable.com/internals/hashing/ + // ASSUMES the target file is checked into HEAD + + var bare bool // whether the repo is bare or not; this changes what the hashing algorithm is, due to backwards compatibility + + bareStr, _, err := git.NewCommand(git.DefaultContext, "config", "core.bare").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { return "", err } + + if bareStr == "true\n" { + bare = true + } else if bareStr == "false\n" { + bare = false + } else { + return "", errors.New(fmt.Sprintf("Could not determine if %s is a bare repo or not; git config core.bare = <%s>", repoPath, bareStr)) + } + // given a repo and a file in it // TODO: handle other branches, e.g. non-HEAD branches etc annexKey, _, err := git.NewCommand(git.DefaultContext, "show", "HEAD:"+file).RunStdString(&git.RunOpts{Dir: repoPath}) @@ -236,15 +254,18 @@ func AnnexObjectPath(repoPath string, file string) (string, error) { } annexKey = strings.TrimPrefix(annexKey, "/annex/objects/") - // we need to know the two-level folder prefix: https://git-annex.branchable.com/internals/hashing/ - keyHashPrefix, _, err := git.NewCommand(git.DefaultContext, "annex", "examinekey", "--format=${hashdirlower}", annexKey).RunStdString(&git.RunOpts{Dir: repoPath}) - if err != nil { - return "", err + var keyformat string + if bare { + keyformat = "hashdirlower" + } else { + keyformat = "hashdirmixed" } + keyHashPrefix, _, err := git.NewCommand(git.DefaultContext, "annex", "examinekey", "--format=${"+keyformat+"}", annexKey).RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { return "", err } - // TODO: handle non-bare repos - // if ! bare { repoPath += "/.git", and use hashdirmixed instead of hashdirlower } - + if !bare { + repoPath = path.Join(repoPath, ".git") + } return path.Join(repoPath, "annex", "objects", keyHashPrefix, annexKey, annexKey), nil } From 0e073c184c60553512a72e32a5055940a6c1c2c4 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Mon, 1 Aug 2022 14:42:03 -0400 Subject: [PATCH 09/34] filecmp --- integrations/git_annex_test.go | 56 ++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index d579357b14913..09ff4194a7e36 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -19,8 +19,10 @@ import ( "github.com/stretchr/testify/require" "testing" + "bytes" "errors" "fmt" + "io" "io/fs" "math/rand" "net/url" @@ -161,8 +163,54 @@ func TestGitAnnex(t *testing.T) { }) } +// https://stackoverflow.com/a/30038571 +func filecmp(file1, file2 string, chunkSize int) (bool, error) { + // Check file size ... + if chunkSize == 0 { + chunkSize = 2 << 12 + } + + f1, err := os.Open(file1) + if err != nil { + return false, err + } + defer f1.Close() + + f2, err := os.Open(file2) + if err != nil { + return false, err + } + defer f2.Close() + + for { + b1 := make([]byte, chunkSize) + _, err1 := f1.Read(b1) + if err1 != nil && err1 != io.EOF { + return false, err1 + } + + b2 := make([]byte, chunkSize) + _, err2 := f2.Read(b2) + if err2 != nil && err2 != io.EOF { + return false, err2 + } + + if err1 == io.EOF && err2 == io.EOF { + return true, nil + } else if err1 != nil || err2 != nil { + return false, nil + } + + if !bytes.Equal(b1, b2) { + return false, nil + } + } +} + func doGenerateRandomFile(size int, path string) (err error) { // Generate random file + + // XXX TODO: maybe this should not be random, but instead a predictable pattern, so that the test is deterministic bufSize := 4 * 1024 if bufSize > size { bufSize = size @@ -231,7 +279,9 @@ func AnnexObjectPath(repoPath string, file string) (string, error) { var bare bool // whether the repo is bare or not; this changes what the hashing algorithm is, due to backwards compatibility bareStr, _, err := git.NewCommand(git.DefaultContext, "config", "core.bare").RunStdString(&git.RunOpts{Dir: repoPath}) - if err != nil { return "", err } + if err != nil { + return "", err + } if bareStr == "true\n" { bare = true @@ -261,7 +311,9 @@ func AnnexObjectPath(repoPath string, file string) (string, error) { keyformat = "hashdirmixed" } keyHashPrefix, _, err := git.NewCommand(git.DefaultContext, "annex", "examinekey", "--format=${"+keyformat+"}", annexKey).RunStdString(&git.RunOpts{Dir: repoPath}) - if err != nil { return "", err } + if err != nil { + return "", err + } if !bare { repoPath = path.Join(repoPath, ".git") From 5828e79188e99df23de62529800a3fde559d8bea Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Sun, 7 Aug 2022 19:47:40 -0400 Subject: [PATCH 10/34] Test contribution by different users also, use filecmp() instead of just checking for target file's existence --- integrations/git_annex_test.go | 157 ++++++++++++++++++++++++++------- 1 file changed, 124 insertions(+), 33 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 09ff4194a7e36..e7bffd014dfb0 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -32,15 +32,14 @@ import ( "regexp" "strings" + "code.gitea.io/gitea/models/perm" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" - //"code.gitea.io/gitea/models/perm" - /* - "code.gitea.io/gitea/models/perm" - *///"time" // DEBUG + + "time" // DEBUG ) func TestGitAnnex(t *testing.T) { @@ -65,9 +64,9 @@ func TestGitAnnex(t *testing.T) { // Different sessions, so we can test // We unset Reponame up here at the top, then later add it according to each test ownerSession := NewAPITestContext(t, "user2", "") - //readCollaboratorSession := NewAPITestContext(t, "user3", "") - //writeCollaboratorSession := NewAPITestContext(t, "user4", "") - //otherSession := NewAPITestContext(t, "user5", "") // a user with no specific access + readCollaboratorSession := NewAPITestContext(t, "user4", "") + writeCollaboratorSession := NewAPITestContext(t, "user5", "") + otherSession := NewAPITestContext(t, "user8", "") // a user with no specific access // Note: there's also full anonymous access, which is only available for public HTTP repos; it should behave the same as 'other' // but we test it separately below anyway @@ -84,14 +83,15 @@ func TestGitAnnex(t *testing.T) { require.NoError(t, err) require.True(t, !repo.IsPrivate) - // set up collaborators - //doAPIAddCollaborator(s, readCollaboratorSession.Username, perm.AccessModeRead)(t) - //doAPIAddCollaborator(s, writeCollaboratorSession.Username, perm.AccessModeWrite)(t) - sshURL := createSSHUrl(s.GitPath(), u) //httpURL := createSSHUrl(s.GitPath(), u) // XXX this puts username and password into the URL // anonHTTPUrl := ??? + // set up collaborators + doAPIAddCollaborator(s, readCollaboratorSession.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(s, writeCollaboratorSession.Username, perm.AccessModeWrite)(t) + + // fill in fixture data doAPIAnnexInitRepository(t, s, u) // XXX this function always uses ssh, so we don't have a way to test git-annex-push-to-create-over-http; t.Run("Owner", func(t *testing.T) { @@ -111,40 +111,129 @@ func TestGitAnnex(t *testing.T) { t.Run("Contribute", func(t *testing.T) { defer PrintCurrentTest(t)() + doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin")) + require.NoError(t, git.AddChanges(repoPath, false, ".")) + require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + // verify the file was uploaded + annexObjectPath, err := AnnexObjectPath(path.Join(setting.RepoRootPath, s.Username, s.Reponame+".git"), "contribution.bin") + require.NoError(t, err) + match, err := filecmp(path.Join(repoPath, "contribution.bin"), annexObjectPath, 0) + require.NoError(t, err, "Annexed file should be readable in both " + repoPath + "/large.bin and " + annexObjectPath) + require.True(t, match, "Annexed files should be the same") + + }) + }) + }) + }) + + t.Run("ReadCollaborator", func(t *testing.T) { + defer PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() + repoURL := sshURL + + withAnnexCtxKeyFile(t, readCollaboratorSession, func() { + + repoPath, err := os.MkdirTemp("", s.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) + + doAnnexClone(t, repoPath, repoURL) + + // now what? + + // now we try to upload again and see what happens + + t.Run("Contribute", func(t *testing.T) { + defer PrintCurrentTest(t)() + + doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin")) + require.NoError(t, git.AddChanges(repoPath, false, ".")) + require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + require.True(t, strings.Contains(err.Error(), "remote: Gitea: User permission denied for writing."), "Uploading should fail due to permissions") }) }) }) }) - /* - t.Run("ReadCollaborator", func(t *testing.T) { + t.Run("WriteCollaborator", func(t *testing.T) { + defer PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { defer PrintCurrentTest(t)() + repoURL := sshURL - t.Run("SSH", func(t *testing.T) { - defer PrintCurrentTest(t)() - repoURL := sshURL + withAnnexCtxKeyFile(t, writeCollaboratorSession, func() { - withAnnexCtxKeyFile(t, readCollaboratorSession, func() { + repoPath, err := os.MkdirTemp("", s.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) - repoPath, err := os.MkdirTemp("", s.Reponame) + doAnnexClone(t, repoPath, repoURL) + + // now what? + + // now we try to upload again and see what happens + + t.Run("Contribute", func(t *testing.T) { + defer PrintCurrentTest(t)() + + doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin")) + require.NoError(t, git.AddChanges(repoPath, false, ".")) + require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + // verify the file was uploaded + annexObjectPath, err := AnnexObjectPath(path.Join(setting.RepoRootPath, s.Username, s.Reponame+".git"), "contribution.bin") require.NoError(t, err) - defer util.RemoveAll(repoPath) + match, err := filecmp(path.Join(repoPath, "contribution.bin"), annexObjectPath, 0) + require.NoError(t, err, "Annexed file should be readable in both " + repoPath + "/large.bin and " + annexObjectPath) + require.True(t, match, "Annexed files should be the same") + + + }) + }) + }) + }) - doAnnexClone(t, repoPath, repoURL) + t.Run("Other", func(t *testing.T) { + defer PrintCurrentTest(t)() + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() + repoURL := sshURL - // now what? + withAnnexCtxKeyFile(t, otherSession, func() { - // now we try to upload again and see what happens + repoPath, err := os.MkdirTemp("", s.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) - t.Run("Contribute", func(t *testing.T) { - defer PrintCurrentTest(t)() + doAnnexClone(t, repoPath, repoURL) + + // now what? + + // now we try to upload again and see what happens + + t.Run("Contribute", func(t *testing.T) { + defer PrintCurrentTest(t)() + + doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin")) + require.NoError(t, git.AddChanges(repoPath, false, ".")) + require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + require.True(t, strings.Contains(err.Error(), "remote: Gitea: User permission denied for writing."), "Uploading should fail due to permissions") - }) }) }) }) - */ + }) t.Run("Delete", func(t *testing.T) { defer PrintCurrentTest(t)() @@ -155,11 +244,10 @@ func TestGitAnnex(t *testing.T) { require.True(t, os.IsNotExist(stat_err), "Remote annex repo should be removed from disk") }) + //fmt.Printf("Sleeping now. good luck.\n") // give time to allow manually inspecting the test server; the password for all users is 'password'! + time.Sleep(0 * time.Second) // DEBUG }) - //fmt.Printf("Sleeping now. good luck.\n") // give time to allow manually inspecting the test server; the password for all users is 'password'! - //time.Sleep(2 * time.Second) // DEBUG - }) } @@ -391,11 +479,14 @@ func doAPIAnnexInitRepository(t *testing.T, ctx APITestContext, u *url.URL) { "git annex whereis should report the file is uploaded to origin") // - method 2: look directly into the remote repo to find the file - annexObjectPath, err := AnnexObjectPath(path.Join(setting.RepoRootPath, API.Username, API.Reponame+".git"), "large.bin") + remoteRepoPath := path.Join(setting.RepoRootPath, API.Username, API.Reponame+".git") + annexObjectPath, err := AnnexObjectPath(remoteRepoPath, "large.bin") require.NoError(t, err) - _, stat_err := os.Stat(annexObjectPath) - require.NoError(t, stat_err, "Annexed file should exist in remote .git/annex/objects folder") - // TODO: directly diff the source and target files + //_, stat_err := os.Stat(annexObjectPath) + //require.NoError(t, stat_err, "Annexed file should exist in remote .git/annex/objects folder") + match, err := filecmp(path.Join(repoPath, "large.bin"), annexObjectPath, 0) + require.NoError(t, err, "Annexed file should be readable in both " + repoPath + " and " + remoteRepoPath) + require.True(t, match, "Annexed files should be the same") }) } From fa396cf37253dea4456f400666cee0f60c91d6bc Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Mon, 8 Aug 2022 00:43:32 -0400 Subject: [PATCH 11/34] Test uploading/downloading Private annex repos While writing this, try rearranging the pattern so that the Private repos are only cloned once, but are written from multiple writers with different permissions. This lets us test that `git annex copy`, which tries to invoke `rsync`, is correctly blocked when it should be. Otherwise the test would be blocked at 'git clone', which wouldn't test the git-annex permissions at all. But: doing this means now not testing that `git annex get` is correctly blocked when it should be. :/ In passing, make sure to catch doGenerateRandomFile() errors. --- integrations/git_annex_test.go | 195 ++++++++++++++++++++++++++++++--- 1 file changed, 179 insertions(+), 16 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index e7bffd014dfb0..6fa5d82aa0657 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -111,17 +111,20 @@ func TestGitAnnex(t *testing.T) { t.Run("Contribute", func(t *testing.T) { defer PrintCurrentTest(t)() - doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin")) + require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) require.NoError(t, git.AddChanges(repoPath, false, ".")) require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) require.NoError(t, err) // verify the file was uploaded annexObjectPath, err := AnnexObjectPath(path.Join(setting.RepoRootPath, s.Username, s.Reponame+".git"), "contribution.bin") require.NoError(t, err) match, err := filecmp(path.Join(repoPath, "contribution.bin"), annexObjectPath, 0) - require.NoError(t, err, "Annexed file should be readable in both " + repoPath + "/large.bin and " + annexObjectPath) + require.NoError(t, err, "Annexed file should be readable in both "+repoPath+"/large.bin and "+annexObjectPath) require.True(t, match, "Annexed files should be the same") }) @@ -151,11 +154,12 @@ func TestGitAnnex(t *testing.T) { t.Run("Contribute", func(t *testing.T) { defer PrintCurrentTest(t)() - doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin")) + require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) require.NoError(t, git.AddChanges(repoPath, false, ".")) require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) - require.True(t, strings.Contains(err.Error(), "remote: Gitea: User permission denied for writing."), "Uploading should fail due to permissions") + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + require.Error(t, err) + //require.True(t, strings.Contains(err.Error(), "Gitea: Unauthorized"), "Uploading should fail due to permissions") }) }) }) @@ -183,20 +187,22 @@ func TestGitAnnex(t *testing.T) { t.Run("Contribute", func(t *testing.T) { defer PrintCurrentTest(t)() - doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin")) + require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) require.NoError(t, git.AddChanges(repoPath, false, ".")) require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) require.NoError(t, err) // verify the file was uploaded annexObjectPath, err := AnnexObjectPath(path.Join(setting.RepoRootPath, s.Username, s.Reponame+".git"), "contribution.bin") require.NoError(t, err) match, err := filecmp(path.Join(repoPath, "contribution.bin"), annexObjectPath, 0) - require.NoError(t, err, "Annexed file should be readable in both " + repoPath + "/large.bin and " + annexObjectPath) + require.NoError(t, err, "Annexed file should be readable in both "+repoPath+"/large.bin and "+annexObjectPath) require.True(t, match, "Annexed files should be the same") - }) }) }) @@ -224,12 +230,14 @@ func TestGitAnnex(t *testing.T) { t.Run("Contribute", func(t *testing.T) { defer PrintCurrentTest(t)() - doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin")) + require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) require.NoError(t, git.AddChanges(repoPath, false, ".")) require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) - require.True(t, strings.Contains(err.Error(), "remote: Gitea: User permission denied for writing."), "Uploading should fail due to permissions") - + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + require.Error(t, err, "Uploading should fail due to permissions") + // XXX this causes a *different* error message than the other cases + // look into why and see if it can be made consistent + //require.True(t, strings.Contains(err.Error(), "Gitea: Unauthorized"), "Uploading should fail due to permissions") }) }) }) @@ -248,6 +256,161 @@ func TestGitAnnex(t *testing.T) { time.Sleep(0 * time.Second) // DEBUG }) + t.Run("Private", func(t *testing.T) { + defer PrintCurrentTest(t)() + + // create a public repo + s := ownerSession // copy to prevent cross-contamination + s.Reponame = "annex-private" + doAPICreateRepository(s, false)(t) + repo, err := repo_model.GetRepositoryByOwnerAndName(s.Username, s.Reponame) + require.NoError(t, err) + require.True(t, repo.IsPrivate) + + sshURL := createSSHUrl(s.GitPath(), u) + //httpURL := createSSHUrl(s.GitPath(), u) // XXX this puts username and password into the URL + // anonHTTPUrl := ??? + + // set up collaborators + doAPIAddCollaborator(s, readCollaboratorSession.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(s, writeCollaboratorSession.Username, perm.AccessModeWrite)(t) + + // fill in fixture data + doAPIAnnexInitRepository(t, s, u) // XXX this function always uses ssh, so we don't have a way to test git-annex-push-to-create-over-http; + + withAnnexCtxKeyFile(t, ownerSession, func() { + + repoPath, err := os.MkdirTemp("", s.Reponame) + require.NoError(t, err) + //defer util.RemoveAll(repoPath) + + doAnnexClone(t, repoPath, sshURL) + + t.Run("Owner", func(t *testing.T) { + defer PrintCurrentTest(t)() + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() + + withAnnexCtxKeyFile(t, ownerSession, func() { + + t.Run("Contribute", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) + require.NoError(t, git.AddChanges(repoPath, false, ".")) + require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + // verify the file was uploaded + annexObjectPath, err := AnnexObjectPath(path.Join(setting.RepoRootPath, s.Username, s.Reponame+".git"), "contribution.bin") + require.NoError(t, err) + match, err := filecmp(path.Join(repoPath, "contribution.bin"), annexObjectPath, 0) + require.NoError(t, err, "Annexed file should be readable in both "+repoPath+"/large.bin and "+annexObjectPath) + require.True(t, match, "Annexed files should be the same") + + }) + }) + }) + }) + + t.Run("ReadCollaborator", func(t *testing.T) { + defer PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() + + withAnnexCtxKeyFile(t, readCollaboratorSession, func() { + + // now we try to upload again and see what happens + + t.Run("Contribute", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) + require.NoError(t, git.AddChanges(repoPath, false, ".")) + require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + require.Error(t, err, "Uploading should fail due to permissions") + // XXX this causes a *different* error message than the other cases + // look into why and see if it can be made consistent + //require.True(t, strings.Contains(err.Error(), "Gitea: Unauthorized"), "Uploading should fail due to permissions") + }) + }) + }) + }) + + t.Run("WriteCollaborator", func(t *testing.T) { + defer PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() + + withAnnexCtxKeyFile(t, writeCollaboratorSession, func() { + + t.Run("Contribute", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) + require.NoError(t, git.AddChanges(repoPath, false, ".")) + require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + // verify the file was uploaded + annexObjectPath, err := AnnexObjectPath(path.Join(setting.RepoRootPath, s.Username, s.Reponame+".git"), "contribution.bin") + require.NoError(t, err) + match, err := filecmp(path.Join(repoPath, "contribution.bin"), annexObjectPath, 0) + require.NoError(t, err, "Annexed file should be readable in both "+repoPath+"/large.bin and "+annexObjectPath) + require.Truef(t, match, "Annexed files %s and %s should be the same", path.Join(repoPath, "contribution.bin"), annexObjectPath) + + }) + }) + }) + }) + + t.Run("Other", func(t *testing.T) { + defer PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() + + withAnnexCtxKeyFile(t, otherSession, func() { + + t.Run("Contribute", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) + require.NoError(t, git.AddChanges(repoPath, false, ".")) + require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + require.Error(t, err, "Uploading should fail due to permissions") + require.True(t, strings.Contains(err.Error(), "Gitea: Unauthorized"), "Uploading should fail due to permissions") + + }) + }) + }) + }) + + t.Run("Delete", func(t *testing.T) { + defer PrintCurrentTest(t)() + + // Delete the repo, make sure it's fully gone + //doAPIDeleteRepository(s)(t) + //_, stat_err := os.Stat(path.Join(setting.RepoRootPath, s.GitPath())) + //require.True(t, os.IsNotExist(stat_err), "Remote annex repo should be removed from disk") + }) + + //fmt.Printf("Sleeping now. good luck.\n") // give time to allow manually inspecting the test server; the password for all users is 'password'! + time.Sleep(0 * time.Second) // DEBUG + }) + }) }) } @@ -485,7 +648,7 @@ func doAPIAnnexInitRepository(t *testing.T, ctx APITestContext, u *url.URL) { //_, stat_err := os.Stat(annexObjectPath) //require.NoError(t, stat_err, "Annexed file should exist in remote .git/annex/objects folder") match, err := filecmp(path.Join(repoPath, "large.bin"), annexObjectPath, 0) - require.NoError(t, err, "Annexed file should be readable in both " + repoPath + " and " + remoteRepoPath) + require.NoError(t, err, "Annexed file should be readable in both "+repoPath+" and "+remoteRepoPath) require.True(t, match, "Annexed files should be the same") }) } @@ -509,7 +672,7 @@ func doAnnexInitRepository(t *testing.T, repoPath string) { require.NoError(t, git.NewCommand(git.DefaultContext, "annex", "init", "gitea-annex-test").Run(&git.RunOpts{Dir: repoPath})) - doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "large.bin")) + require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "large.bin"))) require.NoError(t, git.AddChanges(repoPath, false, ".")) require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) } From 8fc3239ba6e069d332574f84d2bc86b4195fd5b3 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Mon, 8 Aug 2022 00:48:34 -0400 Subject: [PATCH 12/34] Add a UUID to withCtxKeyFile() This allows nesting withCtxKeyFile()s using the *same user*; which, yes, is redundant, but might happen in a complex test easily enough, e.g. if you're switching back and forth between users. TODO: a nicer solution would be to keep a cache. That would also help with the test slowness by avoiding generating unnecessary RSA keys. But withKeyFile() doesn't expose the keys it generates or let us control what they should be so we'd need to patch it first. --- integrations/api_helper_for_declarative_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integrations/api_helper_for_declarative_test.go b/integrations/api_helper_for_declarative_test.go index 75b528fc9c180..3851c1b762118 100644 --- a/integrations/api_helper_for_declarative_test.go +++ b/integrations/api_helper_for_declarative_test.go @@ -14,6 +14,8 @@ import ( "testing" "time" + "github.com/google/uuid" + "code.gitea.io/gitea/models/perm" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/json" @@ -466,7 +468,7 @@ func doAPIAddRepoToOrganizationTeam(ctx APITestContext, teamID int64, orgName, r // generate and activate an ssh key for the user attached to the APITestContext // TODO: pick a better name; golang doesn't do method overriding. func withCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { - keyName := "One of " + ctx.Username + "'s keys" + keyName := "One of " + ctx.Username + "'s keys: #" + uuid.New().String() withKeyFile(t, keyName, func(keyFile string) { var key api.PublicKey From 9318912c3a5b7d463e1a060b597e0439f35d213e Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Wed, 10 Aug 2022 00:17:07 -0400 Subject: [PATCH 13/34] [WIP] Factor permissions tests into reusable subroutines --- integrations/git_annex_test.go | 707 +++++++++++++-------------------- 1 file changed, 271 insertions(+), 436 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 6fa5d82aa0657..502e83bdaf8d9 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -42,6 +42,11 @@ import ( "time" // DEBUG ) +/* + +Tests + +*/ func TestGitAnnex(t *testing.T) { /* // TODO: look into how LFS did this @@ -59,359 +64,293 @@ func TestGitAnnex(t *testing.T) { // repo, you need to edit its .Reponame or just ignore it and write "username/reponame.git" onGiteaRun(t, func(t *testing.T, u *url.URL) { - defer doCleanAnnexLockdown() // workaround https://git-annex.branchable.com/internals/lockdown/ - - // Different sessions, so we can test - // We unset Reponame up here at the top, then later add it according to each test - ownerSession := NewAPITestContext(t, "user2", "") - readCollaboratorSession := NewAPITestContext(t, "user4", "") - writeCollaboratorSession := NewAPITestContext(t, "user5", "") - otherSession := NewAPITestContext(t, "user8", "") // a user with no specific access - // Note: there's also full anonymous access, which is only available for public HTTP repos; it should behave the same as 'other' - // but we test it separately below anyway + defer annexUnlockdown() // workaround https://git-annex.branchable.com/internals/lockdown/ t.Run("Public", func(t *testing.T) { defer PrintCurrentTest(t)() // create a public repo - s := ownerSession // copy to prevent cross-contamination - s.Reponame = "annex-public" - doAPICreateRepository(s, false)(t) - doAPIEditRepository(s, &api.EditRepoOption{Private: &falseBool})(t) // make the repo public - // double-check it's public (this should be taken care of by models/fixtures/repository.yml, but better to check) - repo, err := repo_model.GetRepositoryByOwnerAndName(s.Username, s.Reponame) + ctx := NewAPITestContext(t, "user2", "annex-public") + doAPICreateRepository(ctx, false)(t) + doAPIEditRepository(ctx, &api.EditRepoOption{Private: &falseBool})(t) + + // double-check it's public + repo, err := repo_model.GetRepositoryByOwnerAndName(ctx.Username, ctx.Reponame) require.NoError(t, err) require.True(t, !repo.IsPrivate) - sshURL := createSSHUrl(s.GitPath(), u) - //httpURL := createSSHUrl(s.GitPath(), u) // XXX this puts username and password into the URL - // anonHTTPUrl := ??? - - // set up collaborators - doAPIAddCollaborator(s, readCollaboratorSession.Username, perm.AccessModeRead)(t) - doAPIAddCollaborator(s, writeCollaboratorSession.Username, perm.AccessModeWrite)(t) - - // fill in fixture data - doAPIAnnexInitRepository(t, s, u) // XXX this function always uses ssh, so we don't have a way to test git-annex-push-to-create-over-http; - - t.Run("Owner", func(t *testing.T) { - defer PrintCurrentTest(t)() - t.Run("SSH", func(t *testing.T) { - defer PrintCurrentTest(t)() + // tests + doAnnexPermissionsTests(t, u, ctx) + }) - repoURL := sshURL + t.Run("Private", func(t *testing.T) { + defer PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, ownerSession, func() { - repoPath, err := os.MkdirTemp("", s.Reponame) - require.NoError(t, err) - //defer util.RemoveAll(repoPath) + // create a private repo + ctx := NewAPITestContext(t, "user2", "annex-private") + doAPICreateRepository(ctx, false)(t) - doAnnexClone(t, repoPath, repoURL) + // double-check it's private + repo, err := repo_model.GetRepositoryByOwnerAndName(ctx.Username, ctx.Reponame) + require.NoError(t, err) + require.True(t, repo.IsPrivate) - t.Run("Contribute", func(t *testing.T) { - defer PrintCurrentTest(t)() + // tests + doAnnexPermissionsTests(t, u, ctx) + }) + }) +} - require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) - require.NoError(t, git.AddChanges(repoPath, false, ".")) - require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) +/* Test that reading/writing to the remote git-annex behaves correctly for all permission levels: - // verify the file was uploaded - annexObjectPath, err := AnnexObjectPath(path.Join(setting.RepoRootPath, s.Username, s.Reponame+".git"), "contribution.bin") - require.NoError(t, err) - match, err := filecmp(path.Join(repoPath, "contribution.bin"), annexObjectPath, 0) - require.NoError(t, err, "Annexed file should be readable in both "+repoPath+"/large.bin and "+annexObjectPath) - require.True(t, match, "Annexed files should be the same") +- Owner +- Writer +- Reader +- Other/Anonymous - }) - }) - }) - }) +precondition: the repo attached to ctx (ctx.GitPath()) has been created with desired permissions +*/ +func doAnnexPermissionsTests(t *testing.T, u *url.URL, ctx APITestContext) { - t.Run("ReadCollaborator", func(t *testing.T) { - defer PrintCurrentTest(t)() + // fill in fixture data + // TODO: replace this with a pre-made repo in integrations/gitea-repositories-meta/ ? + withAnnexCtxKeyFile(t, ctx, func() { + // note: this clone is immediately thrown away; + // the tests below reclone it, to test end-to-end. + repoPath, err := os.MkdirTemp("", ctx.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) - t.Run("SSH", func(t *testing.T) { - defer PrintCurrentTest(t)() - repoURL := sshURL + repoURL := createSSHUrl(ctx.GitPath(), u) + doGitClone(repoPath, repoURL)(t) - withAnnexCtxKeyFile(t, readCollaboratorSession, func() { + doInitAnnexRepository(t, repoPath) - repoPath, err := os.MkdirTemp("", s.Reponame) - require.NoError(t, err) - defer util.RemoveAll(repoPath) + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + }) - doAnnexClone(t, repoPath, repoURL) + // Different sessions, so we can test different permissions. + // We leave Reponame blank because we don't actually then later add it according to each case if needed + // + // NB: these usernames need to match appropriate entries in models/fixtures/user.yml + ownerCtx := NewAPITestContext(t, ctx.Username, "") + writerCtx := NewAPITestContext(t, "user5", "") + readerCtx := NewAPITestContext(t, "user4", "") + outsiderCtx := NewAPITestContext(t, "user8", "") // a user with no specific access + // Note: there's also full anonymous access, which is only available for public HTTP repos; + // it should behave the same as 'outsider' but we (will) test it separately below anyway - // now what? + //httpURL := createSSHUrl(ctx.GitPath(), u) // XXX this puts username and password into the URL + // anonHTTPUrl := ??? - // now we try to upload again and see what happens + // set up collaborators + doAPIAddCollaborator(ctx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ctx, writerCtx.Username, perm.AccessModeWrite)(t) - t.Run("Contribute", func(t *testing.T) { - defer PrintCurrentTest(t)() + t.Run("Owner", func(t *testing.T) { + defer PrintCurrentTest(t)() - require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) - require.NoError(t, git.AddChanges(repoPath, false, ".")) - require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) - require.Error(t, err) - //require.True(t, strings.Contains(err.Error(), "Gitea: Unauthorized"), "Uploading should fail due to permissions") - }) - }) - }) - }) + doSSHTests(t, u, ctx, ownerCtx) + }) - t.Run("WriteCollaborator", func(t *testing.T) { - defer PrintCurrentTest(t)() + t.Run("Writer", func(t *testing.T) { + defer PrintCurrentTest(t)() - t.Run("SSH", func(t *testing.T) { - defer PrintCurrentTest(t)() - repoURL := sshURL + doSSHTests(t, u, ctx, writerCtx) + }) - withAnnexCtxKeyFile(t, writeCollaboratorSession, func() { + t.Run("Reader", func(t *testing.T) { + defer PrintCurrentTest(t)() - repoPath, err := os.MkdirTemp("", s.Reponame) - require.NoError(t, err) - defer util.RemoveAll(repoPath) + doSSHTests(t, u, ctx, readerCtx) - doAnnexClone(t, repoPath, repoURL) + // require.Error(t, err, "Uploading should fail due to permissions") + // XXX this causes a *different* error message than the other cases + // look into why and see if it can be made consistent + //require.True(t, strings.Contains(err.Error(), "Gitea: Unauthorized"), "Uploading should fail due to permissions") + }) - // now what? + t.Run("Outsider", func(t *testing.T) { + defer PrintCurrentTest(t)() - // now we try to upload again and see what happens + doSSHTests(t, u, ctx, outsiderCtx) + // require.Error(t, err, "Uploading should fail due to permissions") + // XXX this causes a *different* error message than the other cases + // look into why and see if it can be made consistent + //require.True(t, strings.Contains(err.Error(), "Gitea: Unauthorized"), "Uploading should fail due to permissions") + }) - t.Run("Contribute", func(t *testing.T) { - defer PrintCurrentTest(t)() + t.Run("Delete", func(t *testing.T) { + defer PrintCurrentTest(t)() - require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) - require.NoError(t, git.AddChanges(repoPath, false, ".")) - require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(ctx)(t) + _, stat_err := os.Stat(path.Join(setting.RepoRootPath, ctx.GitPath())) + require.True(t, os.IsNotExist(stat_err), "Remote annex repo should be removed from disk") + }) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) + //fmt.Printf("Sleeping now. good luck.\n") // give time to allow manually inspecting the test server; the password for all users is 'password'! + time.Sleep(0 * time.Second) // DEBUG +} - // verify the file was uploaded - annexObjectPath, err := AnnexObjectPath(path.Join(setting.RepoRootPath, s.Username, s.Reponame+".git"), "contribution.bin") - require.NoError(t, err) - match, err := filecmp(path.Join(repoPath, "contribution.bin"), annexObjectPath, 0) - require.NoError(t, err, "Annexed file should be readable in both "+repoPath+"/large.bin and "+annexObjectPath) - require.True(t, match, "Annexed files should be the same") - }) - }) - }) - }) +/* +Test that downloading/uploading works over SSH, +to the repo encoded in ctx, +*by the user* encoded in user. - t.Run("Other", func(t *testing.T) { - defer PrintCurrentTest(t)() +precondition: user.Username's permissions to ctx.GitPath() have already been granted +*/ +func doSSHTests(t *testing.T, u *url.URL, ctx APITestContext, user APITestContext) { + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() - t.Run("SSH", func(t *testing.T) { - defer PrintCurrentTest(t)() - repoURL := sshURL + repoURL := createSSHUrl(ctx.GitPath(), u) - withAnnexCtxKeyFile(t, otherSession, func() { + repoPath, err := os.MkdirTemp("", ctx.Reponame) + require.NoError(t, err) - repoPath, err := os.MkdirTemp("", s.Reponame) - require.NoError(t, err) - defer util.RemoveAll(repoPath) + // This test is split up into separate withKeyFile()s + // so it can isolate 'git annex copy' from 'git clone': + // + // 'clone' is done as the repo owner, to guarantee it + // works, but 'copy' is done as the user under test. + // + // Otherwise, in cases where permissions block the + // initial 'clone', the test would simply end there + // and never verify if permissions apply properly to + // 'annex copy' -- potentially leaving a security gap. + withAnnexCtxKeyFile(t, ctx, func() { + doGitClone(repoPath, repoURL)(t) + }) - doAnnexClone(t, repoPath, repoURL) + withAnnexCtxKeyFile(t, user, func() { + t.Run("Init", func(t *testing.T) { + defer PrintCurrentTest(t)() - // now what? + _, _, err := git.NewCommand(git.DefaultContext, "annex", "init").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) - // now we try to upload again and see what happens + // - method 0: 'git config remote.origin.annex-uuid'. + // Demonstrates that 'git annex init' successfully contacted + // the remote git-annex and was able to learn its ID number. + remoteAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) - t.Run("Contribute", func(t *testing.T) { - defer PrintCurrentTest(t)() + remoteAnnexUUID = strings.TrimSpace(remoteAnnexUUID) + require.Regexp(t, + regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$"), + remoteAnnexUUID, + "git annex sync should have been able to download the remote's annex uuid") - require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) - require.NoError(t, git.AddChanges(repoPath, false, ".")) - require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) - require.Error(t, err, "Uploading should fail due to permissions") - // XXX this causes a *different* error message than the other cases - // look into why and see if it can be made consistent - //require.True(t, strings.Contains(err.Error(), "Gitea: Unauthorized"), "Uploading should fail due to permissions") - }) - }) - }) + // - method 1: 'git annex whereis'. + // Demonstrates that git-annex understands the annexed file can be found in the remote annex. + annexWhereis, _, err := git.NewCommand(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + require.Regexp(t, + regexp.MustCompile(remoteAnnexUUID + " -- .* \\[origin\\]\n"), + annexWhereis, + "git annex whereis should report the file is uploaded to origin") }) - t.Run("Delete", func(t *testing.T) { + t.Run("Download", func(t *testing.T) { defer PrintCurrentTest(t)() - // Delete the repo, make sure it's fully gone - doAPIDeleteRepository(s)(t) - _, stat_err := os.Stat(path.Join(setting.RepoRootPath, s.GitPath())) - require.True(t, os.IsNotExist(stat_err), "Remote annex repo should be removed from disk") - }) + // NB: this test does something slightly different if run separately from "Init": + // it first runs "git annex init" silently in the background. + // This shouldn't change any results, but be aware in case it does. - //fmt.Printf("Sleeping now. good luck.\n") // give time to allow manually inspecting the test server; the password for all users is 'password'! - time.Sleep(0 * time.Second) // DEBUG - }) - - t.Run("Private", func(t *testing.T) { - defer PrintCurrentTest(t)() - - // create a public repo - s := ownerSession // copy to prevent cross-contamination - s.Reponame = "annex-private" - doAPICreateRepository(s, false)(t) - repo, err := repo_model.GetRepositoryByOwnerAndName(s.Username, s.Reponame) - require.NoError(t, err) - require.True(t, repo.IsPrivate) - - sshURL := createSSHUrl(s.GitPath(), u) - //httpURL := createSSHUrl(s.GitPath(), u) // XXX this puts username and password into the URL - // anonHTTPUrl := ??? - - // set up collaborators - doAPIAddCollaborator(s, readCollaboratorSession.Username, perm.AccessModeRead)(t) - doAPIAddCollaborator(s, writeCollaboratorSession.Username, perm.AccessModeWrite)(t) - - // fill in fixture data - doAPIAnnexInitRepository(t, s, u) // XXX this function always uses ssh, so we don't have a way to test git-annex-push-to-create-over-http; - - withAnnexCtxKeyFile(t, ownerSession, func() { - - repoPath, err := os.MkdirTemp("", s.Reponame) + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) require.NoError(t, err) - //defer util.RemoveAll(repoPath) - - doAnnexClone(t, repoPath, sshURL) - t.Run("Owner", func(t *testing.T) { - defer PrintCurrentTest(t)() - t.Run("SSH", func(t *testing.T) { - defer PrintCurrentTest(t)() - - withAnnexCtxKeyFile(t, ownerSession, func() { - - t.Run("Contribute", func(t *testing.T) { - defer PrintCurrentTest(t)() - - require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) - require.NoError(t, git.AddChanges(repoPath, false, ".")) - require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) - - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) - - // verify the file was uploaded - annexObjectPath, err := AnnexObjectPath(path.Join(setting.RepoRootPath, s.Username, s.Reponame+".git"), "contribution.bin") - require.NoError(t, err) - match, err := filecmp(path.Join(repoPath, "contribution.bin"), annexObjectPath, 0) - require.NoError(t, err, "Annexed file should be readable in both "+repoPath+"/large.bin and "+annexObjectPath) - require.True(t, match, "Annexed files should be the same") - - }) - }) - }) - }) - - t.Run("ReadCollaborator", func(t *testing.T) { - defer PrintCurrentTest(t)() + // verify the file was downloaded + localObjectPath, err := annexObjectPath(repoPath, "large.bin") + require.NoError(t, err) + //localObjectPath := path.Join(repoPath, "large.bin") // or, just compare against the checked-out file - t.Run("SSH", func(t *testing.T) { - defer PrintCurrentTest(t)() + remoteObjectPath, err := annexObjectPath(path.Join(setting.RepoRootPath, ctx.GitPath()), "large.bin") + require.NoError(t, err) - withAnnexCtxKeyFile(t, readCollaboratorSession, func() { + match, err := filecmp(localObjectPath, remoteObjectPath, 0) + require.NoErrorf(t, err, "Annexed file should be readable in both %s and %s", localObjectPath, remoteObjectPath) + require.True(t, match, "Annexed files should be the same") + }) - // now we try to upload again and see what happens + t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() - t.Run("Contribute", func(t *testing.T) { - defer PrintCurrentTest(t)() + // NB: this test does something slightly different if run separately from "Init": + // it first runs "git annex init" silently in the background. + // This shouldn't change any results, but be aware in case it does. - require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) - require.NoError(t, git.AddChanges(repoPath, false, ".")) - require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) - require.Error(t, err, "Uploading should fail due to permissions") - // XXX this causes a *different* error message than the other cases - // look into why and see if it can be made consistent - //require.True(t, strings.Contains(err.Error(), "Gitea: Unauthorized"), "Uploading should fail due to permissions") - }) - }) - }) - }) + require.NoError(t, generateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) + require.NoError(t, git.AddChanges(repoPath, false, ".")) + require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex another file"})) + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) - t.Run("WriteCollaborator", func(t *testing.T) { - defer PrintCurrentTest(t)() + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) - t.Run("SSH", func(t *testing.T) { - defer PrintCurrentTest(t)() + // verify the file was uploaded + localObjectPath, err := annexObjectPath(repoPath, "contribution.bin") + require.NoError(t, err) + //localObjectPath := path.Join(repoPath, "contribution.bin") // or, just compare against the checked-out file - withAnnexCtxKeyFile(t, writeCollaboratorSession, func() { + remoteObjectPath, err := annexObjectPath(path.Join(setting.RepoRootPath, ctx.GitPath()), "contribution.bin") + require.NoError(t, err) - t.Run("Contribute", func(t *testing.T) { - defer PrintCurrentTest(t)() + match, err := filecmp(localObjectPath, remoteObjectPath, 0) + require.NoErrorf(t, err, "Annexed file should be readable in both %s and %s", localObjectPath, remoteObjectPath) + require.True(t, match, "Annexed files should be the same") + }) + }) + }) +} - require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) - require.NoError(t, git.AddChanges(repoPath, false, ".")) - require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) +// ---- Helpers ---- - // verify the file was uploaded - annexObjectPath, err := AnnexObjectPath(path.Join(setting.RepoRootPath, s.Username, s.Reponame+".git"), "contribution.bin") - require.NoError(t, err) - match, err := filecmp(path.Join(repoPath, "contribution.bin"), annexObjectPath, 0) - require.NoError(t, err, "Annexed file should be readable in both "+repoPath+"/large.bin and "+annexObjectPath) - require.Truef(t, match, "Annexed files %s and %s should be the same", path.Join(repoPath, "contribution.bin"), annexObjectPath) +func generateRandomFile(size int, path string) (err error) { + // Generate random file - }) - }) - }) - }) + // XXX TODO: maybe this should not be random, but instead a predictable pattern, so that the test is deterministic + bufSize := 4 * 1024 + if bufSize > size { + bufSize = size + } - t.Run("Other", func(t *testing.T) { - defer PrintCurrentTest(t)() + buffer := make([]byte, bufSize) - t.Run("SSH", func(t *testing.T) { - defer PrintCurrentTest(t)() + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() - withAnnexCtxKeyFile(t, otherSession, func() { + written := 0 + for written < size { + n := size - written + if n > bufSize { + n = bufSize + } + _, err := rand.Read(buffer[:n]) + if err != nil { + return err + } + n, err = f.Write(buffer[:n]) + if err != nil { + return err + } + written += n + } + if err != nil { + return err + } - t.Run("Contribute", func(t *testing.T) { - defer PrintCurrentTest(t)() - - require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) - require.NoError(t, git.AddChanges(repoPath, false, ".")) - require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) - require.Error(t, err, "Uploading should fail due to permissions") - require.True(t, strings.Contains(err.Error(), "Gitea: Unauthorized"), "Uploading should fail due to permissions") - - }) - }) - }) - }) - - t.Run("Delete", func(t *testing.T) { - defer PrintCurrentTest(t)() - - // Delete the repo, make sure it's fully gone - //doAPIDeleteRepository(s)(t) - //_, stat_err := os.Stat(path.Join(setting.RepoRootPath, s.GitPath())) - //require.True(t, os.IsNotExist(stat_err), "Remote annex repo should be removed from disk") - }) - - //fmt.Printf("Sleeping now. good luck.\n") // give time to allow manually inspecting the test server; the password for all users is 'password'! - time.Sleep(0 * time.Second) // DEBUG - }) - }) - }) + return nil } // https://stackoverflow.com/a/30038571 @@ -458,70 +397,41 @@ func filecmp(file1, file2 string, chunkSize int) (bool, error) { } } -func doGenerateRandomFile(size int, path string) (err error) { - // Generate random file - - // XXX TODO: maybe this should not be random, but instead a predictable pattern, so that the test is deterministic - bufSize := 4 * 1024 - if bufSize > size { - bufSize = size - } - - buffer := make([]byte, bufSize) - f, err := os.Create(path) - if err != nil { - return err - } - defer f.Close() +// ---- Annex-specific helpers ---- - written := 0 - for written < size { - n := size - written - if n > bufSize { - n = bufSize - } - _, err := rand.Read(buffer[:n]) - if err != nil { - return err - } - n, err = f.Write(buffer[:n]) - if err != nil { - return err - } - written += n - } - if err != nil { - return err - } +/* Initialize a repo with some baseline annexed and non-annexed files. - return nil -} +TODO: this could be replaced with a fixture repo; +see integrations/gitea-repositories-meta/ and models/fixtures/repository.yml. -func doCleanAnnexLockdown() { - // do chmod -R +w $REPOS in order to - // handle https://git-annex.branchable.com/internals/lockdown/ - // > (The only bad consequence of this is that rm -rf .git doesn't work unless you first run chmod -R +w .git) - // If this isn't done, the test can only be run once, because it reuses its gitea-repositories/ path +However we reuse this template for -different- repos. +*/ +func doInitAnnexRepository(t *testing.T, repoPath string) { + // set up what files should be annexed + // in this case, all *.bin files will be annexed + // without this, git-annex's default config annexes every file larger than some number of megabytes + f, err := os.Create(path.Join(repoPath, ".gitattributes")) + require.NoError(t, err) + f.WriteString("*.bin filter=annex annex.largefiles=anything") + f.Close() + require.NoError(t, git.AddChanges(repoPath, false, ".")) + require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Configure git-annex settings"})) - filepath.WalkDir(setting.RepoRootPath, func(path string, d fs.DirEntry, err error) error { - if err == nil { - // 0200 == u+w, in octal unix permission notation - info, err := d.Info() - if err != nil { - return err - } + // 'git annex init' + // 'gitea-annex-test' is there to avoid the nuisance comment getting stored. + require.NoError(t, git.NewCommand(git.DefaultContext, "annex", "init", "gitea-annex-test").Run(&git.RunOpts{Dir: repoPath})) - err = os.Chmod(path, info.Mode()|0200) - if err != nil { - return err - } - } - return nil - }) + // add a file to the annex + require.NoError(t, generateRandomFile(1024*1024/4, path.Join(repoPath, "large.bin"))) + require.NoError(t, git.AddChanges(repoPath, false, ".")) + require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) } -func AnnexObjectPath(repoPath string, file string) (string, error) { + +/* given a git repo and a path to an annexed file in it (assumed to be committed to its HEAD), + find the path in .git/annex/objects/ that contains its actual contents. */ +func annexObjectPath(repoPath string, file string) (string, error) { // find the path inside *.git/annex/objects of a given file // i.e. figure out its two-level hash prefix: https://git-annex.branchable.com/internals/hashing/ @@ -572,116 +482,41 @@ func AnnexObjectPath(repoPath string, file string) (string, error) { return path.Join(repoPath, "annex", "objects", keyHashPrefix, annexKey, annexKey), nil } -func doAnnexClone(t *testing.T, repoPath string, repoURL *url.URL) { - doGitClone(repoPath, repoURL)(t) - - _, _, git_err := git.NewCommand(git.DefaultContext, "annex", "get", ".").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, git_err) - - // Verify the download - - // - method 0: check that 'git annex get' successfully contacted the remote git-annex - remoteAnnexUUID, _, git_err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, git_err) - remoteAnnexUUID = strings.TrimSpace(remoteAnnexUUID) - require.Regexp(t, - regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$"), - remoteAnnexUUID, - "git annex sync should have been able to download the remote's annex uuid") - // TODO: scan for all annexed files? - - // - method 2: look into .git/annex/objects to find the annexed file - annexObjectPath, err := AnnexObjectPath(repoPath, "large.bin") - require.NoError(t, err) - _, stat_err := os.Stat(annexObjectPath) - require.NoError(t, stat_err, "Annexed file should exist in remote .git/annex/objects folder") - -} +/* +Do chmod -R +w $REPOS in order to handle https://git-annex.branchable.com/internals/lockdown/: -func doAPIAnnexInitRepository(t *testing.T, ctx APITestContext, u *url.URL) { +> (The only bad consequence of this is that rm -rf .git doesn't work unless you first run chmod -R +w .git) - API := ctx // TODO: change the names - - // ohhhh right. I need to install an ssh key here. dammit. - withAnnexCtxKeyFile(t, ctx, func() { - // Setup clone folder - repoPath, err := os.MkdirTemp("", API.Reponame) - require.NoError(t, err) - defer util.RemoveAll(repoPath) - - repoURL := createSSHUrl(ctx.GitPath(), u) - doGitClone(repoPath, repoURL)(t) - - //fmt.Printf("So yeah here's the thing: %#v\n", repoPath) // DEBUG - - doAnnexInitRepository(t, repoPath) - - // Upload - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) - - // Verify the upload - - // - method 0: check that 'git annex sync' successfully contacted the remote git-annex - remoteAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) - remoteAnnexUUID = strings.TrimSpace(remoteAnnexUUID) - require.Regexp(t, - regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$"), - remoteAnnexUUID, - "git annex sync should have been able to download the remote's annex uuid") - - // Verify the upload: Check that the file was uploaded - - // - method 1: 'git annex whereis' - annexWhereis, _, err := git.NewCommand(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) - require.True(t, - strings.Contains(annexWhereis, " -- origin\n"), - "git annex whereis should report the file is uploaded to origin") +Without, these tests can only be run once, because they reuse `gitea-repositories/` +folder and will balk at finding pre-existing partial repos. +*/ +func annexUnlockdown() { + filepath.WalkDir(setting.RepoRootPath, func(path string, d fs.DirEntry, err error) error { + if err == nil { + // 0200 == u+w, in octal unix permission notation + info, err := d.Info() + if err != nil { + return err + } - // - method 2: look directly into the remote repo to find the file - remoteRepoPath := path.Join(setting.RepoRootPath, API.Username, API.Reponame+".git") - annexObjectPath, err := AnnexObjectPath(remoteRepoPath, "large.bin") - require.NoError(t, err) - //_, stat_err := os.Stat(annexObjectPath) - //require.NoError(t, stat_err, "Annexed file should exist in remote .git/annex/objects folder") - match, err := filecmp(path.Join(repoPath, "large.bin"), annexObjectPath, 0) - require.NoError(t, err, "Annexed file should be readable in both "+repoPath+" and "+remoteRepoPath) - require.True(t, match, "Annexed files should be the same") + err = os.Chmod(path, info.Mode()|0200) + if err != nil { + return err + } + } + return nil }) } -func doAnnexInitRepository(t *testing.T, repoPath string) { - // initialize a repo with a some annexed and unannexed files - // TODO: this could be replaced with a fixture repo; see - // integrations/gitea-repositories-meta/ and models/fixtures/repository.yml - // However we reuse this many times. - - // set up what files should be annexed - // in this case, all *.bin files will be annexed - // without this, git-annex's default config annexes every file larger than some number of megabytes - f, err := os.Create(path.Join(repoPath, ".gitattributes")) - require.NoError(t, err) - f.WriteString("*.bin filter=annex annex.largefiles=anything") - f.Close() - - require.NoError(t, git.AddChanges(repoPath, false, ".")) - require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Configure git-annex settings"})) - - require.NoError(t, git.NewCommand(git.DefaultContext, "annex", "init", "gitea-annex-test").Run(&git.RunOpts{Dir: repoPath})) - - require.NoError(t, doGenerateRandomFile(1024*1024/4, path.Join(repoPath, "large.bin"))) - require.NoError(t, git.AddChanges(repoPath, false, ".")) - require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) -} +/* like withKeyFile(), but automatically sets it the account given in ctx for use by git-annex */ func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { os.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set _gitAnnexUseGitSSH, gitAnnexUseGitSSHExists := os.LookupEnv("GIT_ANNEX_USE_GIT_SSH") defer func() { + // reset if gitAnnexUseGitSSHExists { os.Setenv("GIT_ANNEX_USE_GIT_SSH", _gitAnnexUseGitSSH) } From 71b47a5023093fdac6d35d8dfa7da5d5d4fdaa58 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Wed, 10 Aug 2022 15:37:01 -0400 Subject: [PATCH 14/34] Split up annex Init/Download/Upload tests into subroutines that *return errors* --- integrations/git_annex_test.go | 211 ++++++++++++++++++++++----------- 1 file changed, 144 insertions(+), 67 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 502e83bdaf8d9..aee48343c6818 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -42,11 +42,6 @@ import ( "time" // DEBUG ) -/* - -Tests - -*/ func TestGitAnnex(t *testing.T) { /* // TODO: look into how LFS did this @@ -211,6 +206,9 @@ func doSSHTests(t *testing.T, u *url.URL, ctx APITestContext, user APITestContex repoPath, err := os.MkdirTemp("", ctx.Reponame) require.NoError(t, err) + defer util.RemoveAll(repoPath) + + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) // This test is split up into separate withKeyFile()s // so it can isolate 'git annex copy' from 'git clone': @@ -230,86 +228,165 @@ func doSSHTests(t *testing.T, u *url.URL, ctx APITestContext, user APITestContex t.Run("Init", func(t *testing.T) { defer PrintCurrentTest(t)() - _, _, err := git.NewCommand(git.DefaultContext, "annex", "init").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) - - // - method 0: 'git config remote.origin.annex-uuid'. - // Demonstrates that 'git annex init' successfully contacted - // the remote git-annex and was able to learn its ID number. - remoteAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) - - remoteAnnexUUID = strings.TrimSpace(remoteAnnexUUID) - require.Regexp(t, - regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$"), - remoteAnnexUUID, - "git annex sync should have been able to download the remote's annex uuid") - - // - method 1: 'git annex whereis'. - // Demonstrates that git-annex understands the annexed file can be found in the remote annex. - annexWhereis, _, err := git.NewCommand(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) - require.Regexp(t, - regexp.MustCompile(remoteAnnexUUID + " -- .* \\[origin\\]\n"), - annexWhereis, - "git annex whereis should report the file is uploaded to origin") + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) t.Run("Download", func(t *testing.T) { defer PrintCurrentTest(t)() - // NB: this test does something slightly different if run separately from "Init": - // it first runs "git annex init" silently in the background. - // This shouldn't change any results, but be aware in case it does. + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) +} - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) - // verify the file was downloaded - localObjectPath, err := annexObjectPath(repoPath, "large.bin") - require.NoError(t, err) - //localObjectPath := path.Join(repoPath, "large.bin") // or, just compare against the checked-out file +/* test that 'git annex init' works - remoteObjectPath, err := annexObjectPath(path.Join(setting.RepoRootPath, ctx.GitPath()), "large.bin") - require.NoError(t, err) + precondition: repoPath contains a pre-cloned git repo with a git-annex branch - match, err := filecmp(localObjectPath, remoteObjectPath, 0) - require.NoErrorf(t, err, "Annexed file should be readable in both %s and %s", localObjectPath, remoteObjectPath) - require.True(t, match, "Annexed files should be the same") - }) + */ +func doAnnexInitTest(remoteRepoPath string, repoPath string) (err error) { + _, _, err = git.NewCommand(git.DefaultContext, "annex", "init").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } - t.Run("Upload", func(t *testing.T) { - defer PrintCurrentTest(t)() + // - method 0: 'git config remote.origin.annex-uuid'. + // Demonstrates that 'git annex init' successfully contacted + // the remote git-annex and was able to learn its ID number. + readAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + readAnnexUUID = strings.TrimSpace(readAnnexUUID) - // NB: this test does something slightly different if run separately from "Init": - // it first runs "git annex init" silently in the background. - // This shouldn't change any results, but be aware in case it does. - require.NoError(t, generateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin"))) - require.NoError(t, git.AddChanges(repoPath, false, ".")) - require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex another file"})) - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) + match := regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$").MatchString(readAnnexUUID) + if !match { + return errors.New(fmt.Sprintf("'git config remote.origin.annex-uuid' should have been able to download the remote's uuid; but instead read '%s'.", readAnnexUUID)) + } - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) + remoteAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: remoteRepoPath}) + if err != nil { + return err + } - // verify the file was uploaded - localObjectPath, err := annexObjectPath(repoPath, "contribution.bin") - require.NoError(t, err) - //localObjectPath := path.Join(repoPath, "contribution.bin") // or, just compare against the checked-out file + remoteAnnexUUID = strings.TrimSpace(remoteAnnexUUID) + match = regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$").MatchString(remoteAnnexUUID) + if !match { + return errors.New(fmt.Sprintf("'git annex init' should have been able to download the remote's uuid; but instead read '%s'.", remoteAnnexUUID)) + } - remoteObjectPath, err := annexObjectPath(path.Join(setting.RepoRootPath, ctx.GitPath()), "contribution.bin") - require.NoError(t, err) + if readAnnexUUID != remoteAnnexUUID { + return errors.New(fmt.Sprintf("'git annex init' should have read the expected annex UUID '%s', but instead got '%s'", remoteAnnexUUID, readAnnexUUID)) + } - match, err := filecmp(localObjectPath, remoteObjectPath, 0) - require.NoErrorf(t, err, "Annexed file should be readable in both %s and %s", localObjectPath, remoteObjectPath) - require.True(t, match, "Annexed files should be the same") - }) - }) - }) + // - method 1: 'git annex whereis'. + // Demonstrates that git-annex understands the annexed file can be found in the remote annex. + annexWhereis, _, err := git.NewCommand(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + match = regexp.MustCompile(regexp.QuoteMeta(remoteAnnexUUID) + " -- .* \\[origin\\]\n").MatchString(annexWhereis) + if !match { + return errors.New("'git annex whereis' should report large.bin is known to be in [origin]") + } + + return nil +} + +func doAnnexDownloadTest(remoteRepoPath string, repoPath string) (err error) { + // NB: this test does something slightly different if run separately from "doAnnexInitTest()": + // it first runs "git annex init" silently in the background. + // This shouldn't change any results, but be aware in case it does. + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // verify the file was downloaded + localObjectPath, err := annexObjectPath(repoPath, "large.bin") + if err != nil { + return err + } + //localObjectPath := path.Join(repoPath, "large.bin") // or, just compare against the checked-out file + + remoteObjectPath, err := annexObjectPath(remoteRepoPath, "large.bin") + if err != nil { + return err + } + + match, err := filecmp(localObjectPath, remoteObjectPath, 0) + if err != nil { + return err + } + if !match { + return errors.New("Annexed files should be the same") + } + + return nil } +func doAnnexUploadTest(remoteRepoPath string, repoPath string) (err error) { + // NB: this test does something slightly different if run separately from "Init": + // it first runs "git annex init" silently in the background. + // This shouldn't change any results, but be aware in case it does. + + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin")) + if err != nil { + return err + } + + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex another file"}) + if err != nil { + return err + } + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // verify the file was uploaded + localObjectPath, err := annexObjectPath(repoPath, "contribution.bin") + if err != nil { + return err + } + //localObjectPath := path.Join(repoPath, "contribution.bin") // or, just compare against the checked-out file + + remoteObjectPath, err := annexObjectPath(remoteRepoPath, "contribution.bin") + if err != nil { + return err + } + + match, err := filecmp(localObjectPath, remoteObjectPath, 0) + if err != nil { + return err + } + if !match { + return errors.New("Annexed files should be the same") + } + + return nil +} // ---- Helpers ---- From 75139574d7c185d6e467f40b2b00106f2c461fd1 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Wed, 10 Aug 2022 15:51:03 -0400 Subject: [PATCH 15/34] Unroll annex permission tests, to specify the correct outcomes in all cases. This is super frustrating! The tests are nearly identical, but I can't see how to specify a distinct outcome for different cases. --- integrations/git_annex_test.go | 582 ++++++++++++++++++++++++++------- 1 file changed, 459 insertions(+), 123 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index aee48343c6818..2da5833046298 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -74,8 +74,254 @@ func TestGitAnnex(t *testing.T) { require.NoError(t, err) require.True(t, !repo.IsPrivate) + // fill in fixture data + // TODO: replace this with a pre-made repo in integrations/gitea-repositories-meta/ ? + withAnnexCtxKeyFile(t, ctx, func() { + // note: this clone is immediately thrown away; + // the tests below reclone it, to test end-to-end. + repoPath, err := os.MkdirTemp("", ctx.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) + + repoURL := createSSHUrl(ctx.GitPath(), u) + doGitClone(repoPath, repoURL)(t) + + doInitAnnexRepository(t, repoPath) + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + }) + + // Different sessions, so we can test different permissions. + // We leave Reponame blank because we don't actually then later add it according to each case if needed + // + // NB: these usernames need to match appropriate entries in models/fixtures/user.yml + ownerCtx := NewAPITestContext(t, ctx.Username, "") + writerCtx := NewAPITestContext(t, "user5", "") + readerCtx := NewAPITestContext(t, "user4", "") + outsiderCtx := NewAPITestContext(t, "user8", "") // a user with no specific access + // Note: there's also full anonymous access, which is only available for public HTTP repos; + // it should behave the same as 'outsider' but we (will) test it separately below anyway + + //httpURL := createSSHUrl(ctx.GitPath(), u) // XXX this puts username and password into the URL + // anonHTTPUrl := ??? + + // set up collaborators + doAPIAddCollaborator(ctx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ctx, writerCtx.Username, perm.AccessModeWrite)(t) + // tests - doAnnexPermissionsTests(t, u, ctx) + t.Run("Owner", func(t *testing.T) { + defer PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() + + repoURL := createSSHUrl(ctx.GitPath(), u) + + repoPath, err := os.MkdirTemp("", ctx.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) + + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + + // This test is split up into separate withKeyFile()s + // so it can isolate 'git annex copy' from 'git clone': + // + // 'clone' is done as the repo owner, to guarantee it + // works, but 'copy' is done as the user under test. + // + // Otherwise, in cases where permissions block the + // initial 'clone', the test would simply end there + // and never verify if permissions apply properly to + // 'annex copy' -- potentially leaving a security gap. + withAnnexCtxKeyFile(t, ctx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Writer", func(t *testing.T) { + defer PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() + + repoURL := createSSHUrl(ctx.GitPath(), u) + + repoPath, err := os.MkdirTemp("", ctx.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) + + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + + // This test is split up into separate withKeyFile()s + // so it can isolate 'git annex copy' from 'git clone': + // + // 'clone' is done as the repo owner, to guarantee it + // works, but 'copy' is done as the user under test. + // + // Otherwise, in cases where permissions block the + // initial 'clone', the test would simply end there + // and never verify if permissions apply properly to + // 'annex copy' -- potentially leaving a security gap. + withAnnexCtxKeyFile(t, ctx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Reader", func(t *testing.T) { + defer PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() + + repoURL := createSSHUrl(ctx.GitPath(), u) + + repoPath, err := os.MkdirTemp("", ctx.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) + + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + + // This test is split up into separate withKeyFile()s + // so it can isolate 'git annex copy' from 'git clone': + // + // 'clone' is done as the repo owner, to guarantee it + // works, but 'copy' is done as the user under test. + // + // Otherwise, in cases where permissions block the + // initial 'clone', the test would simply end there + // and never verify if permissions apply properly to + // 'annex copy' -- potentially leaving a security gap. + withAnnexCtxKeyFile(t, ctx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + }) + }) + + t.Run("Outsider", func(t *testing.T) { + defer PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() + + repoURL := createSSHUrl(ctx.GitPath(), u) + + repoPath, err := os.MkdirTemp("", ctx.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) + + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + + // This test is split up into separate withKeyFile()s + // so it can isolate 'git annex copy' from 'git clone': + // + // 'clone' is done as the repo owner, to guarantee it + // works, but 'copy' is done as the user under test. + // + // Otherwise, in cases where permissions block the + // initial 'clone', the test would simply end there + // and never verify if permissions apply properly to + // 'annex copy' -- potentially leaving a security gap. + withAnnexCtxKeyFile(t, ctx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + }) + }) + + t.Run("Delete", func(t *testing.T) { + defer PrintCurrentTest(t)() + + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(ctx)(t) + _, stat_err := os.Stat(path.Join(setting.RepoRootPath, ctx.GitPath())) + require.True(t, os.IsNotExist(stat_err), "Remote annex repo should be removed from disk") + }) + + //fmt.Printf("Sleeping now. good luck.\n") // give time to allow manually inspecting the test server; the password for all users is 'password'! + time.Sleep(0 * time.Second) // DEBUG }) t.Run("Private", func(t *testing.T) { @@ -90,168 +336,263 @@ func TestGitAnnex(t *testing.T) { require.NoError(t, err) require.True(t, repo.IsPrivate) - // tests - doAnnexPermissionsTests(t, u, ctx) - }) - }) -} + // fill in fixture data + // TODO: replace this with a pre-made repo in integrations/gitea-repositories-meta/ ? + withAnnexCtxKeyFile(t, ctx, func() { + // note: this clone is immediately thrown away; + // the tests below reclone it, to test end-to-end. + repoPath, err := os.MkdirTemp("", ctx.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) + repoURL := createSSHUrl(ctx.GitPath(), u) + doGitClone(repoPath, repoURL)(t) -/* Test that reading/writing to the remote git-annex behaves correctly for all permission levels: + doInitAnnexRepository(t, repoPath) -- Owner -- Writer -- Reader -- Other/Anonymous + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + }) -precondition: the repo attached to ctx (ctx.GitPath()) has been created with desired permissions -*/ -func doAnnexPermissionsTests(t *testing.T, u *url.URL, ctx APITestContext) { + // Different sessions, so we can test different permissions. + // We leave Reponame blank because we don't actually then later add it according to each case if needed + // + // NB: these usernames need to match appropriate entries in models/fixtures/user.yml + ownerCtx := NewAPITestContext(t, ctx.Username, "") + writerCtx := NewAPITestContext(t, "user5", "") + readerCtx := NewAPITestContext(t, "user4", "") + outsiderCtx := NewAPITestContext(t, "user8", "") // a user with no specific access + // Note: there's also full anonymous access, which is only available for public HTTP repos; + // it should behave the same as 'outsider' but we (will) test it separately below anyway - // fill in fixture data - // TODO: replace this with a pre-made repo in integrations/gitea-repositories-meta/ ? - withAnnexCtxKeyFile(t, ctx, func() { - // note: this clone is immediately thrown away; - // the tests below reclone it, to test end-to-end. - repoPath, err := os.MkdirTemp("", ctx.Reponame) - require.NoError(t, err) - defer util.RemoveAll(repoPath) + //httpURL := createSSHUrl(ctx.GitPath(), u) // XXX this puts username and password into the URL + // anonHTTPUrl := ??? - repoURL := createSSHUrl(ctx.GitPath(), u) - doGitClone(repoPath, repoURL)(t) + // set up collaborators + doAPIAddCollaborator(ctx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ctx, writerCtx.Username, perm.AccessModeWrite)(t) - doInitAnnexRepository(t, repoPath) + // tests + t.Run("Owner", func(t *testing.T) { + defer PrintCurrentTest(t)() - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) - }) + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() - // Different sessions, so we can test different permissions. - // We leave Reponame blank because we don't actually then later add it according to each case if needed - // - // NB: these usernames need to match appropriate entries in models/fixtures/user.yml - ownerCtx := NewAPITestContext(t, ctx.Username, "") - writerCtx := NewAPITestContext(t, "user5", "") - readerCtx := NewAPITestContext(t, "user4", "") - outsiderCtx := NewAPITestContext(t, "user8", "") // a user with no specific access - // Note: there's also full anonymous access, which is only available for public HTTP repos; - // it should behave the same as 'outsider' but we (will) test it separately below anyway + repoURL := createSSHUrl(ctx.GitPath(), u) - //httpURL := createSSHUrl(ctx.GitPath(), u) // XXX this puts username and password into the URL - // anonHTTPUrl := ??? + repoPath, err := os.MkdirTemp("", ctx.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) - // set up collaborators - doAPIAddCollaborator(ctx, readerCtx.Username, perm.AccessModeRead)(t) - doAPIAddCollaborator(ctx, writerCtx.Username, perm.AccessModeWrite)(t) + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) - t.Run("Owner", func(t *testing.T) { - defer PrintCurrentTest(t)() + // This test is split up into separate withKeyFile()s + // so it can isolate 'git annex copy' from 'git clone': + // + // 'clone' is done as the repo owner, to guarantee it + // works, but 'copy' is done as the user under test. + // + // Otherwise, in cases where permissions block the + // initial 'clone', the test would simply end there + // and never verify if permissions apply properly to + // 'annex copy' -- potentially leaving a security gap. + withAnnexCtxKeyFile(t, ctx, func() { + doGitClone(repoPath, repoURL)(t) + }) - doSSHTests(t, u, ctx, ownerCtx) - }) + withAnnexCtxKeyFile(t, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer PrintCurrentTest(t)() - t.Run("Writer", func(t *testing.T) { - defer PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) - doSSHTests(t, u, ctx, writerCtx) - }) + t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() - t.Run("Reader", func(t *testing.T) { - defer PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) - doSSHTests(t, u, ctx, readerCtx) + t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() - // require.Error(t, err, "Uploading should fail due to permissions") - // XXX this causes a *different* error message than the other cases - // look into why and see if it can be made consistent - //require.True(t, strings.Contains(err.Error(), "Gitea: Unauthorized"), "Uploading should fail due to permissions") - }) + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) - t.Run("Outsider", func(t *testing.T) { - defer PrintCurrentTest(t)() + t.Run("Writer", func(t *testing.T) { + defer PrintCurrentTest(t)() - doSSHTests(t, u, ctx, outsiderCtx) - // require.Error(t, err, "Uploading should fail due to permissions") - // XXX this causes a *different* error message than the other cases - // look into why and see if it can be made consistent - //require.True(t, strings.Contains(err.Error(), "Gitea: Unauthorized"), "Uploading should fail due to permissions") - }) + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() - t.Run("Delete", func(t *testing.T) { - defer PrintCurrentTest(t)() + repoURL := createSSHUrl(ctx.GitPath(), u) - // Delete the repo, make sure it's fully gone - doAPIDeleteRepository(ctx)(t) - _, stat_err := os.Stat(path.Join(setting.RepoRootPath, ctx.GitPath())) - require.True(t, os.IsNotExist(stat_err), "Remote annex repo should be removed from disk") - }) + repoPath, err := os.MkdirTemp("", ctx.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) - //fmt.Printf("Sleeping now. good luck.\n") // give time to allow manually inspecting the test server; the password for all users is 'password'! - time.Sleep(0 * time.Second) // DEBUG -} + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + // This test is split up into separate withKeyFile()s + // so it can isolate 'git annex copy' from 'git clone': + // + // 'clone' is done as the repo owner, to guarantee it + // works, but 'copy' is done as the user under test. + // + // Otherwise, in cases where permissions block the + // initial 'clone', the test would simply end there + // and never verify if permissions apply properly to + // 'annex copy' -- potentially leaving a security gap. + withAnnexCtxKeyFile(t, ctx, func() { + doGitClone(repoPath, repoURL)(t) + }) -/* -Test that downloading/uploading works over SSH, -to the repo encoded in ctx, -*by the user* encoded in user. + withAnnexCtxKeyFile(t, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer PrintCurrentTest(t)() -precondition: user.Username's permissions to ctx.GitPath() have already been granted -*/ -func doSSHTests(t *testing.T, u *url.URL, ctx APITestContext, user APITestContext) { - t.Run("SSH", func(t *testing.T) { - defer PrintCurrentTest(t)() - - repoURL := createSSHUrl(ctx.GitPath(), u) - - repoPath, err := os.MkdirTemp("", ctx.Reponame) - require.NoError(t, err) - defer util.RemoveAll(repoPath) - - remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) - - // This test is split up into separate withKeyFile()s - // so it can isolate 'git annex copy' from 'git clone': - // - // 'clone' is done as the repo owner, to guarantee it - // works, but 'copy' is done as the user under test. - // - // Otherwise, in cases where permissions block the - // initial 'clone', the test would simply end there - // and never verify if permissions apply properly to - // 'annex copy' -- potentially leaving a security gap. - withAnnexCtxKeyFile(t, ctx, func() { - doGitClone(repoPath, repoURL)(t) - }) + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, user, func() { - t.Run("Init", func(t *testing.T) { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Reader", func(t *testing.T) { defer PrintCurrentTest(t)() - require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() + + repoURL := createSSHUrl(ctx.GitPath(), u) + + repoPath, err := os.MkdirTemp("", ctx.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) + + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + + // This test is split up into separate withKeyFile()s + // so it can isolate 'git annex copy' from 'git clone': + // + // 'clone' is done as the repo owner, to guarantee it + // works, but 'copy' is done as the user under test. + // + // Otherwise, in cases where permissions block the + // initial 'clone', the test would simply end there + // and never verify if permissions apply properly to + // 'annex copy' -- potentially leaving a security gap. + withAnnexCtxKeyFile(t, ctx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + }) }) - t.Run("Download", func(t *testing.T) { + t.Run("Outsider", func(t *testing.T) { defer PrintCurrentTest(t)() - require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + t.Run("SSH", func(t *testing.T) { + defer PrintCurrentTest(t)() + + repoURL := createSSHUrl(ctx.GitPath(), u) + + repoPath, err := os.MkdirTemp("", ctx.Reponame) + require.NoError(t, err) + defer util.RemoveAll(repoPath) + + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + + // This test is split up into separate withKeyFile()s + // so it can isolate 'git annex copy' from 'git clone': + // + // 'clone' is done as the repo owner, to guarantee it + // works, but 'copy' is done as the user under test. + // + // Otherwise, in cases where permissions block the + // initial 'clone', the test would simply end there + // and never verify if permissions apply properly to + // 'annex copy' -- potentially leaving a security gap. + withAnnexCtxKeyFile(t, ctx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath), "annex init should fail due to permissions") + }) + + t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") + }) + + t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") + }) + }) + }) }) - t.Run("Upload", func(t *testing.T) { + t.Run("Delete", func(t *testing.T) { defer PrintCurrentTest(t)() - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(ctx)(t) + _, stat_err := os.Stat(path.Join(setting.RepoRootPath, ctx.GitPath())) + require.True(t, os.IsNotExist(stat_err), "Remote annex repo should be removed from disk") }) + + //fmt.Printf("Sleeping now. good luck.\n") // give time to allow manually inspecting the test server; the password for all users is 'password'! + time.Sleep(0 * time.Second) // DEBUG }) }) } - /* test that 'git annex init' works - precondition: repoPath contains a pre-cloned git repo with a git-annex branch +precondition: repoPath contains a pre-cloned git repo with a git-annex branch - */ +*/ func doAnnexInitTest(remoteRepoPath string, repoPath string) (err error) { _, _, err = git.NewCommand(git.DefaultContext, "annex", "init").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { @@ -267,7 +608,6 @@ func doAnnexInitTest(remoteRepoPath string, repoPath string) (err error) { } readAnnexUUID = strings.TrimSpace(readAnnexUUID) - match := regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$").MatchString(readAnnexUUID) if !match { return errors.New(fmt.Sprintf("'git config remote.origin.annex-uuid' should have been able to download the remote's uuid; but instead read '%s'.", readAnnexUUID)) @@ -474,7 +814,6 @@ func filecmp(file1, file2 string, chunkSize int) (bool, error) { } } - // ---- Annex-specific helpers ---- /* Initialize a repo with some baseline annexed and non-annexed files. @@ -505,7 +844,6 @@ func doInitAnnexRepository(t *testing.T, repoPath string) { require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) } - /* given a git repo and a path to an annexed file in it (assumed to be committed to its HEAD), find the path in .git/annex/objects/ that contains its actual contents. */ func annexObjectPath(repoPath string, file string) (string, error) { @@ -559,7 +897,6 @@ func annexObjectPath(repoPath string, file string) (string, error) { return path.Join(repoPath, "annex", "objects", keyHashPrefix, annexKey, annexKey), nil } - /* Do chmod -R +w $REPOS in order to handle https://git-annex.branchable.com/internals/lockdown/: @@ -586,7 +923,6 @@ func annexUnlockdown() { }) } - /* like withKeyFile(), but automatically sets it the account given in ctx for use by git-annex */ func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { os.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set From 9cd0c8e97ff801e1897fae6b8a075a34e839a17a Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Wed, 10 Aug 2022 15:56:26 -0400 Subject: [PATCH 16/34] Drop helper --- integrations/git_helper_for_declarative_test.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/integrations/git_helper_for_declarative_test.go b/integrations/git_helper_for_declarative_test.go index 2f19bfd8e58c6..f6b5e6f0d11d4 100644 --- a/integrations/git_helper_for_declarative_test.go +++ b/integrations/git_helper_for_declarative_test.go @@ -72,13 +72,6 @@ func withKeyFile(t *testing.T, keyname string, callback func(string)) { callback(keyFile) } -func createHTTPUrl(ctx APITestContext, u *url.URL) *url.URL { - u2 := *u - u2.User = url.UserPassword(ctx.Username, userPassword) // userPassword is a module-level const, for the sake of testing - u2.Path = ctx.GitPath() - return &u2 -} - func createSSHUrl(gitPath string, u *url.URL) *url.URL { u2 := *u u2.Scheme = "ssh" From bcc00d721c11c495b210d71d3153da3250d4f40c Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Wed, 10 Aug 2022 16:01:02 -0400 Subject: [PATCH 17/34] Title --- .github/workflows/test-git-annex.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-git-annex.yml b/.github/workflows/test-git-annex.yml index 8966048f020aa..60d86364af5b1 100644 --- a/.github/workflows/test-git-annex.yml +++ b/.github/workflows/test-git-annex.yml @@ -23,7 +23,7 @@ jobs: with: node-version: 'lts/*' - - name: Build Release Assets + - name: Build ./gitea run: | TAGS="bindata sqlite sqlite_unlock_notify" make build From 666c16837825c8f0f5df3b2bf3551aaa49ddfd24 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Wed, 10 Aug 2022 16:33:08 -0400 Subject: [PATCH 18/34] explicit is better than implicit --- integrations/git_annex_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 2da5833046298..313cb28757234 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -829,6 +829,7 @@ func doInitAnnexRepository(t *testing.T, repoPath string) { // without this, git-annex's default config annexes every file larger than some number of megabytes f, err := os.Create(path.Join(repoPath, ".gitattributes")) require.NoError(t, err) + f.WriteString("* annex.largefiles=nothing") f.WriteString("*.bin filter=annex annex.largefiles=anything") f.Close() require.NoError(t, git.AddChanges(repoPath, false, ".")) From 852f14d0cb406f56f08f2a3a548dd100c6530db4 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Thu, 18 Aug 2022 21:28:59 -0400 Subject: [PATCH 19/34] Drop redundant code Co-authored-by: Mathieu Guay-Paquet --- integrations/git_annex_test.go | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 313cb28757234..40983ca44c357 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -23,12 +23,10 @@ import ( "errors" "fmt" "io" - "io/fs" "math/rand" "net/url" "os" "path" - "path/filepath" "regexp" "strings" @@ -59,7 +57,6 @@ func TestGitAnnex(t *testing.T) { // repo, you need to edit its .Reponame or just ignore it and write "username/reponame.git" onGiteaRun(t, func(t *testing.T, u *url.URL) { - defer annexUnlockdown() // workaround https://git-annex.branchable.com/internals/lockdown/ t.Run("Public", func(t *testing.T) { defer PrintCurrentTest(t)() @@ -898,32 +895,6 @@ func annexObjectPath(repoPath string, file string) (string, error) { return path.Join(repoPath, "annex", "objects", keyHashPrefix, annexKey, annexKey), nil } -/* -Do chmod -R +w $REPOS in order to handle https://git-annex.branchable.com/internals/lockdown/: - -> (The only bad consequence of this is that rm -rf .git doesn't work unless you first run chmod -R +w .git) - -Without, these tests can only be run once, because they reuse `gitea-repositories/` -folder and will balk at finding pre-existing partial repos. -*/ -func annexUnlockdown() { - filepath.WalkDir(setting.RepoRootPath, func(path string, d fs.DirEntry, err error) error { - if err == nil { - // 0200 == u+w, in octal unix permission notation - info, err := d.Info() - if err != nil { - return err - } - - err = os.Chmod(path, info.Mode()|0200) - if err != nil { - return err - } - } - return nil - }) -} - /* like withKeyFile(), but automatically sets it the account given in ctx for use by git-annex */ func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { os.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set From 1e9aef6cde7903114ff6dbeb8addcb816bb5e418 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Thu, 18 Aug 2022 21:31:10 -0400 Subject: [PATCH 20/34] Logic bug Save then set, not set then save, which does nothing. Co-authored-by: Mathieu Guay-Paquet --- integrations/git_annex_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 40983ca44c357..3ab584d62ebaa 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -897,8 +897,6 @@ func annexObjectPath(repoPath string, file string) (string, error) { /* like withKeyFile(), but automatically sets it the account given in ctx for use by git-annex */ func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { - os.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set - _gitAnnexUseGitSSH, gitAnnexUseGitSSHExists := os.LookupEnv("GIT_ANNEX_USE_GIT_SSH") defer func() { // reset @@ -907,5 +905,7 @@ func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { } }() + os.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set + withCtxKeyFile(t, ctx, callback) } From 040ecd49ce391b1f636b7f5df45cd9bf47e5d515 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Thu, 18 Aug 2022 21:33:34 -0400 Subject: [PATCH 21/34] Clarify Co-authored-by: Mathieu Guay-Paquet --- integrations/git_annex_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 3ab584d62ebaa..56781af98c448 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -48,9 +48,6 @@ func TestGitAnnex(t *testing.T) { } */ - trueBool := true // this is silly but there's places it's needed - falseBool := !trueBool - // Some guidelines: // a APITestContext is an awkward union of session credential + username + target repo // which is assumed to be owned by that username; if you want to target a different @@ -64,7 +61,8 @@ func TestGitAnnex(t *testing.T) { // create a public repo ctx := NewAPITestContext(t, "user2", "annex-public") doAPICreateRepository(ctx, false)(t) - doAPIEditRepository(ctx, &api.EditRepoOption{Private: &falseBool})(t) + private := false // this API takes pointers, so we need a variable + doAPIEditRepository(ctx, &api.EditRepoOption{Private: &private})(t) // double-check it's public repo, err := repo_model.GetRepositoryByOwnerAndName(ctx.Username, ctx.Reponame) From 29a9cae3318bc5a4444fe2359059a4f1fdc0d708 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Thu, 18 Aug 2022 21:35:06 -0400 Subject: [PATCH 22/34] Clarify Co-authored-by: Mathieu Guay-Paquet --- integrations/git_annex_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 56781af98c448..e78366a71f2ee 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -67,7 +67,7 @@ func TestGitAnnex(t *testing.T) { // double-check it's public repo, err := repo_model.GetRepositoryByOwnerAndName(ctx.Username, ctx.Reponame) require.NoError(t, err) - require.True(t, !repo.IsPrivate) + require.False(t, repo.IsPrivate) // fill in fixture data // TODO: replace this with a pre-made repo in integrations/gitea-repositories-meta/ ? From a073a7a8992d5680ede0bd700406d226a2bfd8cc Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Thu, 18 Aug 2022 21:38:22 -0400 Subject: [PATCH 23/34] Remove debugging code Co-authored-by: Mathieu Guay-Paquet --- integrations/git_annex_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index e78366a71f2ee..fce42371d8ac8 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -36,8 +36,6 @@ import ( "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" - - "time" // DEBUG ) func TestGitAnnex(t *testing.T) { @@ -314,9 +312,6 @@ func TestGitAnnex(t *testing.T) { _, stat_err := os.Stat(path.Join(setting.RepoRootPath, ctx.GitPath())) require.True(t, os.IsNotExist(stat_err), "Remote annex repo should be removed from disk") }) - - //fmt.Printf("Sleeping now. good luck.\n") // give time to allow manually inspecting the test server; the password for all users is 'password'! - time.Sleep(0 * time.Second) // DEBUG }) t.Run("Private", func(t *testing.T) { @@ -576,9 +571,6 @@ func TestGitAnnex(t *testing.T) { _, stat_err := os.Stat(path.Join(setting.RepoRootPath, ctx.GitPath())) require.True(t, os.IsNotExist(stat_err), "Remote annex repo should be removed from disk") }) - - //fmt.Printf("Sleeping now. good luck.\n") // give time to allow manually inspecting the test server; the password for all users is 'password'! - time.Sleep(0 * time.Second) // DEBUG }) }) } From 3594864c4405ecd3ddf9e0b9dcfe83902247abf3 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Thu, 18 Aug 2022 21:44:29 -0400 Subject: [PATCH 24/34] Comment --- integrations/git_annex_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index fce42371d8ac8..1fc305c47ae1e 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -621,6 +621,8 @@ func doAnnexInitTest(remoteRepoPath string, repoPath string) (err error) { if err != nil { return err } + // Note: this regex is unanchored because 'whereis' outputs multiple lines containing + // headers and 1+ remotes and we just want to find one of them. match = regexp.MustCompile(regexp.QuoteMeta(remoteAnnexUUID) + " -- .* \\[origin\\]\n").MatchString(annexWhereis) if !match { return errors.New("'git annex whereis' should report large.bin is known to be in [origin]") From 2002fc6737dbfbef2b707138a96e1b6a849bba56 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Thu, 18 Aug 2022 22:00:48 -0400 Subject: [PATCH 25/34] Clarify --- integrations/git_annex_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 1fc305c47ae1e..c3e4188698174 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -633,7 +633,7 @@ func doAnnexInitTest(remoteRepoPath string, repoPath string) (err error) { func doAnnexDownloadTest(remoteRepoPath string, repoPath string) (err error) { // NB: this test does something slightly different if run separately from "doAnnexInitTest()": - // it first runs "git annex init" silently in the background. + // "git annex copy" will notice and run "git annex init", silently. // This shouldn't change any results, but be aware in case it does. _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) From 6bbb4c0a4f01462cc377e0754400a83e11545e09 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Thu, 18 Aug 2022 22:05:37 -0400 Subject: [PATCH 26/34] Clarify --- integrations/git_annex_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index c3e4188698174..143f995572582 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -577,7 +577,8 @@ func TestGitAnnex(t *testing.T) { /* test that 'git annex init' works -precondition: repoPath contains a pre-cloned git repo with a git-annex branch +precondition: repoPath contains a pre-cloned git repo with an annex: a valid git-annex branch, + and a file 'large.bin' in its origin's annex. See doInitAnnexRepository(). */ func doAnnexInitTest(remoteRepoPath string, repoPath string) (err error) { From d0a505a129037793b5d77e144cb3d5059fb826ff Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Thu, 18 Aug 2022 22:15:41 -0400 Subject: [PATCH 27/34] Return error from helper instead of passing in the testing.T --- integrations/git_annex_test.go | 56 ++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 143f995572582..558223ecb1c28 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -79,7 +79,8 @@ func TestGitAnnex(t *testing.T) { repoURL := createSSHUrl(ctx.GitPath(), u) doGitClone(repoPath, repoURL)(t) - doInitAnnexRepository(t, repoPath) + err = doInitAnnexRepository(repoPath) + require.NoError(t, err, "git-annex repository should have been initialized") _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) require.NoError(t, err) @@ -338,7 +339,8 @@ func TestGitAnnex(t *testing.T) { repoURL := createSSHUrl(ctx.GitPath(), u) doGitClone(repoPath, repoURL)(t) - doInitAnnexRepository(t, repoPath) + err = doInitAnnexRepository(repoPath) + require.NoError(t, err, "git-annex repository should have been initialized") _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) require.NoError(t, err) @@ -813,26 +815,56 @@ see integrations/gitea-repositories-meta/ and models/fixtures/repository.yml. However we reuse this template for -different- repos. */ -func doInitAnnexRepository(t *testing.T, repoPath string) { +func doInitAnnexRepository(repoPath string) (error) { // set up what files should be annexed // in this case, all *.bin files will be annexed // without this, git-annex's default config annexes every file larger than some number of megabytes f, err := os.Create(path.Join(repoPath, ".gitattributes")) - require.NoError(t, err) - f.WriteString("* annex.largefiles=nothing") - f.WriteString("*.bin filter=annex annex.largefiles=anything") + if err != nil { + return err + } + + _, err = f.WriteString("* annex.largefiles=nothing") + if err != nil { + return err + } + _, err = f.WriteString("*.bin filter=annex annex.largefiles=anything") + if err != nil { + return err + } f.Close() - require.NoError(t, git.AddChanges(repoPath, false, ".")) - require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Configure git-annex settings"})) + + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Configure git-annex settings"}) + if err != nil { + return err + } // 'git annex init' // 'gitea-annex-test' is there to avoid the nuisance comment getting stored. - require.NoError(t, git.NewCommand(git.DefaultContext, "annex", "init", "gitea-annex-test").Run(&git.RunOpts{Dir: repoPath})) + err = git.NewCommand(git.DefaultContext, "annex", "init", "gitea-annex-test").Run(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } // add a file to the annex - require.NoError(t, generateRandomFile(1024*1024/4, path.Join(repoPath, "large.bin"))) - require.NoError(t, git.AddChanges(repoPath, false, ".")) - require.NoError(t, git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"})) + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "large.bin")) + if err != nil { + return err + } + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"}) + if err != nil { + return err + } + + return nil } /* given a git repo and a path to an annexed file in it (assumed to be committed to its HEAD), From 8eec7fc3d3331e336383f832496d644a56db6db2 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Thu, 18 Aug 2022 23:08:51 -0400 Subject: [PATCH 28/34] Use more obvious spelling --- integrations/git_annex_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 558223ecb1c28..475e8faa710db 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -766,7 +766,7 @@ func generateRandomFile(size int, path string) (err error) { func filecmp(file1, file2 string, chunkSize int) (bool, error) { // Check file size ... if chunkSize == 0 { - chunkSize = 2 << 12 + chunkSize = 4 * 1024 } f1, err := os.Open(file1) From 6628a3097a3979ca9d38127cdf253054c217845e Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Fri, 19 Aug 2022 00:13:53 -0400 Subject: [PATCH 29/34] Rewrite filecmp to be safer. Thanks to io.ReadFull(), this now avoids the rare case where the read streams could get de-synced if one did a short-read without the other, incorrectly reporting false without scanning the complete files. Also moves it to util (and rename it FileCmp so it can be called from elsewhere). Also implements checking the file size before scanning, a feature that seems like it was once in https://stackoverflow.com/a/30038571 judging from the comments, but was since removed. Co-authored-by: Mathieu Guay-Paquet --- integrations/git_annex_test.go | 50 +------------------- modules/util/filecmp.go | 85 ++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 48 deletions(-) create mode 100644 modules/util/filecmp.go diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 475e8faa710db..5c93261160d3b 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -19,10 +19,8 @@ import ( "github.com/stretchr/testify/require" "testing" - "bytes" "errors" "fmt" - "io" "math/rand" "net/url" "os" @@ -656,7 +654,7 @@ func doAnnexDownloadTest(remoteRepoPath string, repoPath string) (err error) { return err } - match, err := filecmp(localObjectPath, remoteObjectPath, 0) + match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) if err != nil { return err } @@ -709,7 +707,7 @@ func doAnnexUploadTest(remoteRepoPath string, repoPath string) (err error) { return err } - match, err := filecmp(localObjectPath, remoteObjectPath, 0) + match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) if err != nil { return err } @@ -762,50 +760,6 @@ func generateRandomFile(size int, path string) (err error) { return nil } -// https://stackoverflow.com/a/30038571 -func filecmp(file1, file2 string, chunkSize int) (bool, error) { - // Check file size ... - if chunkSize == 0 { - chunkSize = 4 * 1024 - } - - f1, err := os.Open(file1) - if err != nil { - return false, err - } - defer f1.Close() - - f2, err := os.Open(file2) - if err != nil { - return false, err - } - defer f2.Close() - - for { - b1 := make([]byte, chunkSize) - _, err1 := f1.Read(b1) - if err1 != nil && err1 != io.EOF { - return false, err1 - } - - b2 := make([]byte, chunkSize) - _, err2 := f2.Read(b2) - if err2 != nil && err2 != io.EOF { - return false, err2 - } - - if err1 == io.EOF && err2 == io.EOF { - return true, nil - } else if err1 != nil || err2 != nil { - return false, nil - } - - if !bytes.Equal(b1, b2) { - return false, nil - } - } -} - // ---- Annex-specific helpers ---- /* Initialize a repo with some baseline annexed and non-annexed files. diff --git a/modules/util/filecmp.go b/modules/util/filecmp.go new file mode 100644 index 0000000000000..49a6656d853ea --- /dev/null +++ b/modules/util/filecmp.go @@ -0,0 +1,85 @@ +package util + +import ( + "bytes" + "io" + "os" +) + +// Decide if two files have the same contents or not. +// chunkSize is the size of the blocks to scan by; pass 0 to get a sensible default. +// *Follows* symlinks. +// +// May return an error if something else goes wrong; in this case, you should ignore the value of 'same'. +// +// derived from https://stackoverflow.com/a/30038571 +// under CC-BY-SA-4.0 by several contributors +func FileCmp(file1, file2 string, chunkSize int) (same bool, err error) { + + if chunkSize == 0 { + chunkSize = 4 * 1024 + } + + // shortcuts: check file metadata + stat1, err := os.Stat(file1) + if err != nil { + return false, err + } + + stat2, err := os.Stat(file1) + if err != nil { + return false, err + } + + // are inputs are literally the same file? + if os.SameFile(stat1, stat2) { + return true, nil + } + + // do inputs at least have the same size? + if stat1.Size() != stat2.Size() { + return false, nil + } + + // long way: compare contents + f1, err := os.Open(file1) + if err != nil { + return false, err + } + defer f1.Close() + + f2, err := os.Open(file2) + if err != nil { + return false, err + } + defer f2.Close() + + b1 := make([]byte, chunkSize) + b2 := make([]byte, chunkSize) + for { + n1, err1 := io.ReadFull(f1, b1) + n2, err2 := io.ReadFull(f2, b2) + + // https://pkg.go.dev/io#Reader + // > Callers should always process the n > 0 bytes returned + // > before considering the error err. Doing so correctly + // > handles I/O errors that happen after reading some bytes + // > and also both of the allowed EOF behaviors. + + if !bytes.Equal(b1[:n1], b2[:n2]) { + return false, nil + } + + if (err1 == io.EOF && err2 == io.EOF) || (err1 == io.ErrUnexpectedEOF && err2 == io.ErrUnexpectedEOF) { + return true, nil + } + + // some other error, like a dropped network connection or a bad transfer + if err1 != nil { + return false, err1 + } + if err2 != nil { + return false, err2 + } + } +} From 65b8706bdcf4e1b6b7ce9bae40be4941067d5ece Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Fri, 19 Aug 2022 00:18:37 -0400 Subject: [PATCH 30/34] gofmt --- integrations/git_annex_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 5c93261160d3b..e2d9a55492f03 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -769,7 +769,7 @@ see integrations/gitea-repositories-meta/ and models/fixtures/repository.yml. However we reuse this template for -different- repos. */ -func doInitAnnexRepository(repoPath string) (error) { +func doInitAnnexRepository(repoPath string) error { // set up what files should be annexed // in this case, all *.bin files will be annexed // without this, git-annex's default config annexes every file larger than some number of megabytes From 0f48015801979962366cf4abb2cf282c6a4d444b Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Fri, 19 Aug 2022 01:02:24 -0400 Subject: [PATCH 31/34] Only build the Go code for testing git-annex There's no need to test the javascript. --- .github/workflows/test-git-annex.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-git-annex.yml b/.github/workflows/test-git-annex.yml index 60d86364af5b1..67b8a3eccda37 100644 --- a/.github/workflows/test-git-annex.yml +++ b/.github/workflows/test-git-annex.yml @@ -25,7 +25,7 @@ jobs: - name: Build ./gitea run: | - TAGS="bindata sqlite sqlite_unlock_notify" make build + TAGS="bindata sqlite sqlite_unlock_notify" make backend - name: Test run: | From de51980659cae2ef8d05c3766427c400933d92bd Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Fri, 19 Aug 2022 01:27:48 -0400 Subject: [PATCH 32/34] Factor, slightly --- integrations/git_annex_test.go | 61 ++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index e2d9a55492f03..52bf0096906e6 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -68,20 +68,9 @@ func TestGitAnnex(t *testing.T) { // fill in fixture data // TODO: replace this with a pre-made repo in integrations/gitea-repositories-meta/ ? withAnnexCtxKeyFile(t, ctx, func() { - // note: this clone is immediately thrown away; - // the tests below reclone it, to test end-to-end. - repoPath, err := os.MkdirTemp("", ctx.Reponame) - require.NoError(t, err) - defer util.RemoveAll(repoPath) - repoURL := createSSHUrl(ctx.GitPath(), u) - doGitClone(repoPath, repoURL)(t) - - err = doInitAnnexRepository(repoPath) + err = doInitRemoteAnnexRepository(t, repoURL) require.NoError(t, err, "git-annex repository should have been initialized") - - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) }) // Different sessions, so we can test different permissions. @@ -121,7 +110,7 @@ func TestGitAnnex(t *testing.T) { // so it can isolate 'git annex copy' from 'git clone': // // 'clone' is done as the repo owner, to guarantee it - // works, but 'copy' is done as the user under test. + // succeeds, but 'copy' is done as the user under test. // // Otherwise, in cases where permissions block the // initial 'clone', the test would simply end there @@ -171,7 +160,7 @@ func TestGitAnnex(t *testing.T) { // so it can isolate 'git annex copy' from 'git clone': // // 'clone' is done as the repo owner, to guarantee it - // works, but 'copy' is done as the user under test. + // succeeds, but 'copy' is done as the user under test. // // Otherwise, in cases where permissions block the // initial 'clone', the test would simply end there @@ -328,20 +317,10 @@ func TestGitAnnex(t *testing.T) { // fill in fixture data // TODO: replace this with a pre-made repo in integrations/gitea-repositories-meta/ ? withAnnexCtxKeyFile(t, ctx, func() { - // note: this clone is immediately thrown away; - // the tests below reclone it, to test end-to-end. - repoPath, err := os.MkdirTemp("", ctx.Reponame) - require.NoError(t, err) - defer util.RemoveAll(repoPath) - repoURL := createSSHUrl(ctx.GitPath(), u) - doGitClone(repoPath, repoURL)(t) - err = doInitAnnexRepository(repoPath) + err = doInitRemoteAnnexRepository(t, repoURL) require.NoError(t, err, "git-annex repository should have been initialized") - - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) - require.NoError(t, err) }) // Different sessions, so we can test different permissions. @@ -821,6 +800,38 @@ func doInitAnnexRepository(repoPath string) error { return nil } +/* Initialize a repo with some baseline annexed and non-annexed files. + +TODO: this could be replaced with a fixture repo; +see integrations/gitea-repositories-meta/ and models/fixtures/repository.yml. + +However we reuse this template for -different- repos. + +TODO: This has to take a testing.T, but only because it reuses a routine + written in the other integration tests which expects it. + It would be cleaner if it didn't have to. +*/ +func doInitRemoteAnnexRepository(t *testing.T, repoURL *url.URL) error { + repoPath, err := os.MkdirTemp("", path.Base(repoURL.Path)) + if err != nil { + return err + } + // This clone is immediately thrown away, which + // helps force the tests to be end-to-end. + defer util.RemoveAll(repoPath) + + doGitClone(repoPath, repoURL)(t) + + err = doInitAnnexRepository(repoPath) + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + return nil +} + /* given a git repo and a path to an annexed file in it (assumed to be committed to its HEAD), find the path in .git/annex/objects/ that contains its actual contents. */ func annexObjectPath(repoPath string, file string) (string, error) { From 01311123f85902efc99d77807b61a4fa0fd42335 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Fri, 19 Aug 2022 01:49:45 -0400 Subject: [PATCH 33/34] tidy --- integrations/git_annex_test.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 52bf0096906e6..5bb20a316d79b 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -2,17 +2,6 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// I should test both: -// - symlink annexing -// - smudge annexing -// - http annexing -// - ssh annexing -// -// and then cross all that with testing different combinations of permissions -// ..yeah? Is that a reasonable thing to do? - -// it would also be good, probably, to test how push-to-create interacts with git-annex - package integrations import ( From 1e1ac78eec9aebd93ea0a549eeae151535f1bcab Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Fri, 19 Aug 2022 02:26:34 -0400 Subject: [PATCH 34/34] Curtail --- integrations/git_annex_test.go | 229 +++++++++------------------------ 1 file changed, 63 insertions(+), 166 deletions(-) diff --git a/integrations/git_annex_test.go b/integrations/git_annex_test.go index 5bb20a316d79b..26becd6595cec 100644 --- a/integrations/git_annex_test.go +++ b/integrations/git_annex_test.go @@ -25,7 +25,19 @@ import ( "code.gitea.io/gitea/modules/util" ) -func TestGitAnnex(t *testing.T) { +// Some guidelines: +// +// * a APITestContext is an awkward union of session credential + username + target repo +// which is assumed to be owned by that username; if you want to target a different +// repo, you need to edit its .Reponame or just ignore it and write "username/reponame.git" + + +/* + Test that permissions are enforced on git-annex-shell commands. + + Along the way, test that uploading, downloading, and deleting all work. +*/ +func TestGitAnnexPermissions(t *testing.T) { /* // TODO: look into how LFS did this if !setting.Annex.Enabled { @@ -33,10 +45,14 @@ func TestGitAnnex(t *testing.T) { } */ - // Some guidelines: - // a APITestContext is an awkward union of session credential + username + target repo - // which is assumed to be owned by that username; if you want to target a different - // repo, you need to edit its .Reponame or just ignore it and write "username/reponame.git" + + // Each case below is split so that 'clone' is done as + // the repo owner, but 'copy' as the user under test. + // + // Otherwise, in cases where permissions block the + // initial 'clone', the test would simply end there + // and never verify if permissions apply properly to + // 'annex copy' -- potentially leaving a security gap. onGiteaRun(t, func(t *testing.T, u *url.URL) { @@ -44,20 +60,22 @@ func TestGitAnnex(t *testing.T) { defer PrintCurrentTest(t)() // create a public repo - ctx := NewAPITestContext(t, "user2", "annex-public") - doAPICreateRepository(ctx, false)(t) + ownerCtx := NewAPITestContext(t, "user2", "annex-public") + doAPICreateRepository(ownerCtx, false)(t) private := false // this API takes pointers, so we need a variable - doAPIEditRepository(ctx, &api.EditRepoOption{Private: &private})(t) + doAPIEditRepository(ownerCtx, &api.EditRepoOption{Private: &private})(t) // double-check it's public - repo, err := repo_model.GetRepositoryByOwnerAndName(ctx.Username, ctx.Reponame) + repo, err := repo_model.GetRepositoryByOwnerAndName(ownerCtx.Username, ownerCtx.Reponame) require.NoError(t, err) require.False(t, repo.IsPrivate) - // fill in fixture data - // TODO: replace this with a pre-made repo in integrations/gitea-repositories-meta/ ? - withAnnexCtxKeyFile(t, ctx, func() { - repoURL := createSSHUrl(ctx.GitPath(), u) + // Remote addresses of the repo + repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL + remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost + + // Fill in fixture data + withAnnexCtxKeyFile(t, ownerCtx, func() { err = doInitRemoteAnnexRepository(t, repoURL) require.NoError(t, err, "git-annex repository should have been initialized") }) @@ -66,19 +84,13 @@ func TestGitAnnex(t *testing.T) { // We leave Reponame blank because we don't actually then later add it according to each case if needed // // NB: these usernames need to match appropriate entries in models/fixtures/user.yml - ownerCtx := NewAPITestContext(t, ctx.Username, "") writerCtx := NewAPITestContext(t, "user5", "") readerCtx := NewAPITestContext(t, "user4", "") outsiderCtx := NewAPITestContext(t, "user8", "") // a user with no specific access - // Note: there's also full anonymous access, which is only available for public HTTP repos; - // it should behave the same as 'outsider' but we (will) test it separately below anyway - - //httpURL := createSSHUrl(ctx.GitPath(), u) // XXX this puts username and password into the URL - // anonHTTPUrl := ??? // set up collaborators - doAPIAddCollaborator(ctx, readerCtx.Username, perm.AccessModeRead)(t) - doAPIAddCollaborator(ctx, writerCtx.Username, perm.AccessModeWrite)(t) + doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) // tests t.Run("Owner", func(t *testing.T) { @@ -87,25 +99,11 @@ func TestGitAnnex(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer PrintCurrentTest(t)() - repoURL := createSSHUrl(ctx.GitPath(), u) - - repoPath, err := os.MkdirTemp("", ctx.Reponame) + repoPath, err := os.MkdirTemp("", ownerCtx.Reponame) require.NoError(t, err) defer util.RemoveAll(repoPath) - remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) - - // This test is split up into separate withKeyFile()s - // so it can isolate 'git annex copy' from 'git clone': - // - // 'clone' is done as the repo owner, to guarantee it - // succeeds, but 'copy' is done as the user under test. - // - // Otherwise, in cases where permissions block the - // initial 'clone', the test would simply end there - // and never verify if permissions apply properly to - // 'annex copy' -- potentially leaving a security gap. - withAnnexCtxKeyFile(t, ctx, func() { + withAnnexCtxKeyFile(t, ownerCtx, func() { doGitClone(repoPath, repoURL)(t) }) @@ -137,25 +135,11 @@ func TestGitAnnex(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer PrintCurrentTest(t)() - repoURL := createSSHUrl(ctx.GitPath(), u) - - repoPath, err := os.MkdirTemp("", ctx.Reponame) + repoPath, err := os.MkdirTemp("", ownerCtx.Reponame) require.NoError(t, err) defer util.RemoveAll(repoPath) - remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) - - // This test is split up into separate withKeyFile()s - // so it can isolate 'git annex copy' from 'git clone': - // - // 'clone' is done as the repo owner, to guarantee it - // succeeds, but 'copy' is done as the user under test. - // - // Otherwise, in cases where permissions block the - // initial 'clone', the test would simply end there - // and never verify if permissions apply properly to - // 'annex copy' -- potentially leaving a security gap. - withAnnexCtxKeyFile(t, ctx, func() { + withAnnexCtxKeyFile(t, ownerCtx, func() { doGitClone(repoPath, repoURL)(t) }) @@ -187,25 +171,11 @@ func TestGitAnnex(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer PrintCurrentTest(t)() - repoURL := createSSHUrl(ctx.GitPath(), u) - - repoPath, err := os.MkdirTemp("", ctx.Reponame) + repoPath, err := os.MkdirTemp("", ownerCtx.Reponame) require.NoError(t, err) defer util.RemoveAll(repoPath) - remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) - - // This test is split up into separate withKeyFile()s - // so it can isolate 'git annex copy' from 'git clone': - // - // 'clone' is done as the repo owner, to guarantee it - // works, but 'copy' is done as the user under test. - // - // Otherwise, in cases where permissions block the - // initial 'clone', the test would simply end there - // and never verify if permissions apply properly to - // 'annex copy' -- potentially leaving a security gap. - withAnnexCtxKeyFile(t, ctx, func() { + withAnnexCtxKeyFile(t, ownerCtx, func() { doGitClone(repoPath, repoURL)(t) }) @@ -237,25 +207,11 @@ func TestGitAnnex(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer PrintCurrentTest(t)() - repoURL := createSSHUrl(ctx.GitPath(), u) - - repoPath, err := os.MkdirTemp("", ctx.Reponame) + repoPath, err := os.MkdirTemp("", ownerCtx.Reponame) require.NoError(t, err) defer util.RemoveAll(repoPath) - remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) - - // This test is split up into separate withKeyFile()s - // so it can isolate 'git annex copy' from 'git clone': - // - // 'clone' is done as the repo owner, to guarantee it - // works, but 'copy' is done as the user under test. - // - // Otherwise, in cases where permissions block the - // initial 'clone', the test would simply end there - // and never verify if permissions apply properly to - // 'annex copy' -- potentially leaving a security gap. - withAnnexCtxKeyFile(t, ctx, func() { + withAnnexCtxKeyFile(t, ownerCtx, func() { doGitClone(repoPath, repoURL)(t) }) @@ -285,8 +241,8 @@ func TestGitAnnex(t *testing.T) { defer PrintCurrentTest(t)() // Delete the repo, make sure it's fully gone - doAPIDeleteRepository(ctx)(t) - _, stat_err := os.Stat(path.Join(setting.RepoRootPath, ctx.GitPath())) + doAPIDeleteRepository(ownerCtx)(t) + _, stat_err := os.Stat(remoteRepoPath) require.True(t, os.IsNotExist(stat_err), "Remote annex repo should be removed from disk") }) }) @@ -295,19 +251,20 @@ func TestGitAnnex(t *testing.T) { defer PrintCurrentTest(t)() // create a private repo - ctx := NewAPITestContext(t, "user2", "annex-private") - doAPICreateRepository(ctx, false)(t) + ownerCtx := NewAPITestContext(t, "user2", "annex-private") + doAPICreateRepository(ownerCtx, false)(t) // double-check it's private - repo, err := repo_model.GetRepositoryByOwnerAndName(ctx.Username, ctx.Reponame) + repo, err := repo_model.GetRepositoryByOwnerAndName(ownerCtx.Username, ownerCtx.Reponame) require.NoError(t, err) require.True(t, repo.IsPrivate) - // fill in fixture data - // TODO: replace this with a pre-made repo in integrations/gitea-repositories-meta/ ? - withAnnexCtxKeyFile(t, ctx, func() { - repoURL := createSSHUrl(ctx.GitPath(), u) + // Remote addresses of the repo + repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL + remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost + // Fill in fixture data + withAnnexCtxKeyFile(t, ownerCtx, func() { err = doInitRemoteAnnexRepository(t, repoURL) require.NoError(t, err, "git-annex repository should have been initialized") }) @@ -316,19 +273,15 @@ func TestGitAnnex(t *testing.T) { // We leave Reponame blank because we don't actually then later add it according to each case if needed // // NB: these usernames need to match appropriate entries in models/fixtures/user.yml - ownerCtx := NewAPITestContext(t, ctx.Username, "") writerCtx := NewAPITestContext(t, "user5", "") readerCtx := NewAPITestContext(t, "user4", "") outsiderCtx := NewAPITestContext(t, "user8", "") // a user with no specific access // Note: there's also full anonymous access, which is only available for public HTTP repos; // it should behave the same as 'outsider' but we (will) test it separately below anyway - //httpURL := createSSHUrl(ctx.GitPath(), u) // XXX this puts username and password into the URL - // anonHTTPUrl := ??? - // set up collaborators - doAPIAddCollaborator(ctx, readerCtx.Username, perm.AccessModeRead)(t) - doAPIAddCollaborator(ctx, writerCtx.Username, perm.AccessModeWrite)(t) + doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) // tests t.Run("Owner", func(t *testing.T) { @@ -337,25 +290,11 @@ func TestGitAnnex(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer PrintCurrentTest(t)() - repoURL := createSSHUrl(ctx.GitPath(), u) - - repoPath, err := os.MkdirTemp("", ctx.Reponame) + repoPath, err := os.MkdirTemp("", ownerCtx.Reponame) require.NoError(t, err) defer util.RemoveAll(repoPath) - remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) - - // This test is split up into separate withKeyFile()s - // so it can isolate 'git annex copy' from 'git clone': - // - // 'clone' is done as the repo owner, to guarantee it - // works, but 'copy' is done as the user under test. - // - // Otherwise, in cases where permissions block the - // initial 'clone', the test would simply end there - // and never verify if permissions apply properly to - // 'annex copy' -- potentially leaving a security gap. - withAnnexCtxKeyFile(t, ctx, func() { + withAnnexCtxKeyFile(t, ownerCtx, func() { doGitClone(repoPath, repoURL)(t) }) @@ -387,25 +326,11 @@ func TestGitAnnex(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer PrintCurrentTest(t)() - repoURL := createSSHUrl(ctx.GitPath(), u) - - repoPath, err := os.MkdirTemp("", ctx.Reponame) + repoPath, err := os.MkdirTemp("", ownerCtx.Reponame) require.NoError(t, err) defer util.RemoveAll(repoPath) - remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) - - // This test is split up into separate withKeyFile()s - // so it can isolate 'git annex copy' from 'git clone': - // - // 'clone' is done as the repo owner, to guarantee it - // works, but 'copy' is done as the user under test. - // - // Otherwise, in cases where permissions block the - // initial 'clone', the test would simply end there - // and never verify if permissions apply properly to - // 'annex copy' -- potentially leaving a security gap. - withAnnexCtxKeyFile(t, ctx, func() { + withAnnexCtxKeyFile(t, ownerCtx, func() { doGitClone(repoPath, repoURL)(t) }) @@ -437,25 +362,11 @@ func TestGitAnnex(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer PrintCurrentTest(t)() - repoURL := createSSHUrl(ctx.GitPath(), u) - - repoPath, err := os.MkdirTemp("", ctx.Reponame) + repoPath, err := os.MkdirTemp("", ownerCtx.Reponame) require.NoError(t, err) defer util.RemoveAll(repoPath) - remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) - - // This test is split up into separate withKeyFile()s - // so it can isolate 'git annex copy' from 'git clone': - // - // 'clone' is done as the repo owner, to guarantee it - // works, but 'copy' is done as the user under test. - // - // Otherwise, in cases where permissions block the - // initial 'clone', the test would simply end there - // and never verify if permissions apply properly to - // 'annex copy' -- potentially leaving a security gap. - withAnnexCtxKeyFile(t, ctx, func() { + withAnnexCtxKeyFile(t, ownerCtx, func() { doGitClone(repoPath, repoURL)(t) }) @@ -487,25 +398,11 @@ func TestGitAnnex(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer PrintCurrentTest(t)() - repoURL := createSSHUrl(ctx.GitPath(), u) - - repoPath, err := os.MkdirTemp("", ctx.Reponame) + repoPath, err := os.MkdirTemp("", ownerCtx.Reponame) require.NoError(t, err) defer util.RemoveAll(repoPath) - remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) - - // This test is split up into separate withKeyFile()s - // so it can isolate 'git annex copy' from 'git clone': - // - // 'clone' is done as the repo owner, to guarantee it - // works, but 'copy' is done as the user under test. - // - // Otherwise, in cases where permissions block the - // initial 'clone', the test would simply end there - // and never verify if permissions apply properly to - // 'annex copy' -- potentially leaving a security gap. - withAnnexCtxKeyFile(t, ctx, func() { + withAnnexCtxKeyFile(t, ownerCtx, func() { doGitClone(repoPath, repoURL)(t) }) @@ -535,8 +432,8 @@ func TestGitAnnex(t *testing.T) { defer PrintCurrentTest(t)() // Delete the repo, make sure it's fully gone - doAPIDeleteRepository(ctx)(t) - _, stat_err := os.Stat(path.Join(setting.RepoRootPath, ctx.GitPath())) + doAPIDeleteRepository(ownerCtx)(t) + _, stat_err := os.Stat(remoteRepoPath) require.True(t, os.IsNotExist(stat_err), "Remote annex repo should be removed from disk") }) })