From 0876aa627186dd7adae4b77e56b710d1e5e1f307 Mon Sep 17 00:00:00 2001 From: kuba-- Date: Mon, 10 Sep 2018 10:10:24 +0200 Subject: [PATCH 1/3] Update go-git Signed-off-by: kuba-- --- Gopkg.lock | 4 +- Gopkg.toml | 2 +- .../gopkg.in/src-d/go-git.v4/common_test.go | 11 +- .../gopkg.in/src-d/go-git.v4/config/config.go | 5 + .../src-d/go-git.v4/config/config_test.go | 2 + .../format/packfile/encoder_advanced_test.go | 7 +- .../plumbing/object/change_adaptor_test.go | 4 +- .../go-git.v4/plumbing/object/change_test.go | 7 +- .../go-git.v4/plumbing/object/commit_test.go | 4 +- .../plumbing/object/difftree_test.go | 4 +- .../go-git.v4/plumbing/object/file_test.go | 17 +- .../src-d/go-git.v4/plumbing/object/object.go | 6 +- .../go-git.v4/plumbing/object/object_test.go | 4 +- .../go-git.v4/plumbing/object/patch_test.go | 5 +- .../go-git.v4/plumbing/object/tag_test.go | 7 +- .../go-git.v4/plumbing/object/tree_test.go | 4 +- .../plumbing/revlist/revlist_test.go | 12 +- .../plumbing/transport/server/loader.go | 3 +- .../plumbing/transport/server/server_test.go | 4 +- .../plumbing/transport/ssh/auth_method.go | 14 +- .../transport/ssh/auth_method_test.go | 62 ++++- vendor/gopkg.in/src-d/go-git.v4/prune_test.go | 4 +- .../gopkg.in/src-d/go-git.v4/remote_test.go | 35 +-- vendor/gopkg.in/src-d/go-git.v4/repository.go | 16 +- .../src-d/go-git.v4/repository_test.go | 30 +-- vendor/gopkg.in/src-d/go-git.v4/status.go | 13 +- .../storage/filesystem/dotgit/dotgit.go | 228 +++++++++++++++++- .../storage/filesystem/dotgit/dotgit_test.go | 63 +++++ .../go-git.v4/storage/filesystem/module.go | 3 +- .../go-git.v4/storage/filesystem/object.go | 33 ++- .../storage/filesystem/object_test.go | 56 +++-- .../go-git.v4/storage/filesystem/storage.go | 38 ++- .../storage/filesystem/storage_test.go | 27 ++- vendor/gopkg.in/src-d/go-git.v4/worktree.go | 56 +++-- .../src-d/go-git.v4/worktree_commit.go | 14 ++ .../src-d/go-git.v4/worktree_commit_test.go | 59 ++++- .../gopkg.in/src-d/go-git.v4/worktree_test.go | 9 + 37 files changed, 692 insertions(+), 180 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 962d47738..2ad68a26e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -574,7 +574,7 @@ version = "v3.1.0" [[projects]] - digest = "1:c8d641ccca2d055cd2cb96dd7c8992478f38034a9da2625bd72adb73aaf22f5b" + digest = "1:8c4de9e369ec9fc3658f7eaa6bcafa8d118c5ea95cee5a5530eef8bd017a6482" name = "gopkg.in/src-d/go-git.v4" packages = [ ".", @@ -619,7 +619,7 @@ "utils/merkletrie/noder", ] pruneopts = "" - revision = "5cc316baa64287c7e56cb7372a5046c30fd955c1" + revision = "208b3c3c32beaab14ebb7adf162fc136c939e99c" source = "github.com/src-d/go-git" [[projects]] diff --git a/Gopkg.toml b/Gopkg.toml index b5045a35e..d043ebf14 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -17,7 +17,7 @@ [[constraint]] name = "gopkg.in/src-d/go-git.v4" source = "github.com/src-d/go-git" - revision = "5cc316baa64287c7e56cb7372a5046c30fd955c1" + revision = "208b3c3c32beaab14ebb7adf162fc136c939e99c" [[constraint]] name = "gopkg.in/src-d/go-git-fixtures.v3" diff --git a/vendor/gopkg.in/src-d/go-git.v4/common_test.go b/vendor/gopkg.in/src-d/go-git.v4/common_test.go index efe1ecc92..dad0a3777 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/common_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/common_test.go @@ -4,6 +4,7 @@ import ( "testing" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -59,10 +60,7 @@ func (s *BaseSuite) NewRepository(f *fixtures.Fixture) *Repository { dotgit = f.DotGit() worktree = memfs.New() - st, err := filesystem.NewStorage(dotgit) - if err != nil { - panic(err) - } + st := filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault()) r, err := Open(st, worktree) if err != nil { @@ -89,10 +87,7 @@ func (s *BaseSuite) NewRepositoryWithEmptyWorktree(f *fixtures.Fixture) *Reposit worktree := memfs.New() - st, err := filesystem.NewStorage(dotgit) - if err != nil { - panic(err) - } + st := filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault()) r, err := Open(st, worktree) if err != nil { diff --git a/vendor/gopkg.in/src-d/go-git.v4/config/config.go b/vendor/gopkg.in/src-d/go-git.v4/config/config.go index ce6506dae..a637f6d70 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/config/config.go +++ b/vendor/gopkg.in/src-d/go-git.v4/config/config.go @@ -40,6 +40,9 @@ type Config struct { IsBare bool // Worktree is the path to the root of the working tree. Worktree string + // CommentChar is the character indicating the start of a + // comment for commands like commit and tag + CommentChar string } Pack struct { @@ -113,6 +116,7 @@ const ( urlKey = "url" bareKey = "bare" worktreeKey = "worktree" + commentCharKey = "commentChar" windowKey = "window" mergeKey = "merge" @@ -151,6 +155,7 @@ func (c *Config) unmarshalCore() { } c.Core.Worktree = s.Options.Get(worktreeKey) + c.Core.CommentChar = s.Options.Get(commentCharKey) } func (c *Config) unmarshalPack() error { diff --git a/vendor/gopkg.in/src-d/go-git.v4/config/config_test.go b/vendor/gopkg.in/src-d/go-git.v4/config/config_test.go index 5cd713e45..fe73de872 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/config/config_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/config/config_test.go @@ -13,6 +13,7 @@ func (s *ConfigSuite) TestUnmarshall(c *C) { input := []byte(`[core] bare = true worktree = foo + commentchar = bar [pack] window = 20 [remote "origin"] @@ -38,6 +39,7 @@ func (s *ConfigSuite) TestUnmarshall(c *C) { c.Assert(cfg.Core.IsBare, Equals, true) c.Assert(cfg.Core.Worktree, Equals, "foo") + c.Assert(cfg.Core.CommentChar, Equals, "bar") c.Assert(cfg.Pack.Window, Equals, uint(20)) c.Assert(cfg.Remotes, HasLen, 2) c.Assert(cfg.Remotes["origin"].Name, Equals, "origin") diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/encoder_advanced_test.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/encoder_advanced_test.go index fc1419eea..e15126e66 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/encoder_advanced_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/format/packfile/encoder_advanced_test.go @@ -8,6 +8,7 @@ import ( "gopkg.in/src-d/go-billy.v4/memfs" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile" . "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -32,8 +33,7 @@ func (s *EncoderAdvancedSuite) TestEncodeDecode(c *C) { fixs = append(fixs, fixtures.ByURL("https://github.com/src-d/go-git.git"). ByTag("packfile").ByTag(".git").One()) fixs.Test(c, func(f *fixtures.Fixture) { - storage, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + storage := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) s.testEncodeDecode(c, storage, 10) }) } @@ -47,8 +47,7 @@ func (s *EncoderAdvancedSuite) TestEncodeDecodeNoDeltaCompression(c *C) { fixs = append(fixs, fixtures.ByURL("https://github.com/src-d/go-git.git"). ByTag("packfile").ByTag(".git").One()) fixs.Test(c, func(f *fixtures.Fixture) { - storage, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + storage := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) s.testEncodeDecode(c, storage, 0) }) } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/change_adaptor_test.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/change_adaptor_test.go index 803c3b89a..c7c003b96 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/change_adaptor_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/change_adaptor_test.go @@ -4,6 +4,7 @@ import ( "sort" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -23,8 +24,7 @@ type ChangeAdaptorSuite struct { func (s *ChangeAdaptorSuite) SetUpSuite(c *C) { s.Suite.SetUpSuite(c) s.Fixture = fixtures.Basic().One() - sto, err := filesystem.NewStorage(s.Fixture.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(s.Fixture.DotGit(), cache.NewObjectLRUDefault()) s.Storer = sto } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/change_test.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/change_test.go index b0e89c711..e2f0a2346 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/change_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/change_test.go @@ -5,6 +5,7 @@ import ( "sort" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/format/diff" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -25,8 +26,7 @@ func (s *ChangeSuite) SetUpSuite(c *C) { s.Suite.SetUpSuite(c) s.Fixture = fixtures.ByURL("https://github.com/src-d/go-git.git"). ByTag(".git").One() - sto, err := filesystem.NewStorage(s.Fixture.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(s.Fixture.DotGit(), cache.NewObjectLRUDefault()) s.Storer = sto } @@ -253,8 +253,7 @@ func (s *ChangeSuite) TestNoFileFilemodes(c *C) { s.Suite.SetUpSuite(c) f := fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) iter, err := sto.IterEncodedObjects(plumbing.AnyObject) c.Assert(err, IsNil) diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/commit_test.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/commit_test.go index e72b703d1..c9acf42f3 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/commit_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/commit_test.go @@ -8,6 +8,7 @@ import ( "time" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" . "gopkg.in/check.v1" "gopkg.in/src-d/go-git-fixtures.v3" @@ -247,8 +248,7 @@ func (s *SuiteCommit) TestStringMultiLine(c *C) { hash := plumbing.NewHash("e7d896db87294e33ca3202e536d4d9bb16023db3") f := fixtures.ByURL("https://github.com/src-d/go-git.git").One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) o, err := sto.EncodedObject(plumbing.CommitObject, hash) c.Assert(err, IsNil) diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/difftree_test.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/difftree_test.go index ff9ecbc3f..4af86840a 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/difftree_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/difftree_test.go @@ -4,6 +4,7 @@ import ( "sort" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -25,8 +26,7 @@ type DiffTreeSuite struct { func (s *DiffTreeSuite) SetUpSuite(c *C) { s.Suite.SetUpSuite(c) s.Fixture = fixtures.Basic().One() - sto, err := filesystem.NewStorage(s.Fixture.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(s.Fixture.DotGit(), cache.NewObjectLRUDefault()) s.Storer = sto s.cache = make(map[string]storer.EncodedObjectStorer) } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/file_test.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/file_test.go index edb82d01c..4b92749cb 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/file_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/file_test.go @@ -4,6 +4,7 @@ import ( "io" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -44,8 +45,7 @@ var fileIterTests = []struct { func (s *FileSuite) TestIter(c *C) { for i, t := range fileIterTests { f := fixtures.ByURL(t.repo).One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) h := plumbing.NewHash(t.commit) commit, err := GetCommit(sto, h) @@ -106,8 +106,7 @@ hs_err_pid* func (s *FileSuite) TestContents(c *C) { for i, t := range contentsTests { f := fixtures.ByURL(t.repo).One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) h := plumbing.NewHash(t.commit) commit, err := GetCommit(sto, h) @@ -160,8 +159,7 @@ var linesTests = []struct { func (s *FileSuite) TestLines(c *C) { for i, t := range linesTests { f := fixtures.ByURL(t.repo).One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) h := plumbing.NewHash(t.commit) commit, err := GetCommit(sto, h) @@ -195,8 +193,7 @@ var ignoreEmptyDirEntriesTests = []struct { func (s *FileSuite) TestIgnoreEmptyDirEntries(c *C) { for i, t := range ignoreEmptyDirEntriesTests { f := fixtures.ByURL(t.repo).One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) h := plumbing.NewHash(t.commit) commit, err := GetCommit(sto, h) @@ -251,9 +248,7 @@ func (s *FileSuite) TestFileIter(c *C) { func (s *FileSuite) TestFileIterSubmodule(c *C) { dotgit := fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One().DotGit() - st, err := filesystem.NewStorage(dotgit) - - c.Assert(err, IsNil) + st := filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault()) hash := plumbing.NewHash("b685400c1f9316f350965a5993d350bc746b0bf4") commit, err := GetCommit(st, hash) diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/object.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/object.go index 4b59aba7a..e960e50c9 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/object.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/object.go @@ -152,7 +152,11 @@ func (s *Signature) decodeTimeAndTimeZone(b []byte) { } func (s *Signature) encodeTimeAndTimeZone(w io.Writer) error { - _, err := fmt.Fprintf(w, "%d %s", s.When.Unix(), s.When.Format("-0700")) + u := s.When.Unix() + if u < 0 { + u = 0 + } + _, err := fmt.Fprintf(w, "%d %s", u, s.When.Format("-0700")) return err } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/object_test.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/object_test.go index 68aa1a13e..8f0eededd 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/object_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/object_test.go @@ -7,6 +7,7 @@ import ( "time" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -26,8 +27,7 @@ type BaseObjectsSuite struct { func (s *BaseObjectsSuite) SetUpSuite(c *C) { s.Suite.SetUpSuite(c) s.Fixture = fixtures.Basic().One() - storer, err := filesystem.NewStorage(s.Fixture.DotGit()) - c.Assert(err, IsNil) + storer := filesystem.NewStorage(s.Fixture.DotGit(), cache.NewObjectLRUDefault()) s.Storer = storer } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/patch_test.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/patch_test.go index 8eb65ec30..47057fba7 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/patch_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/patch_test.go @@ -4,6 +4,7 @@ import ( . "gopkg.in/check.v1" fixtures "gopkg.in/src-d/go-git-fixtures.v3" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem" ) @@ -14,8 +15,8 @@ type PatchSuite struct { var _ = Suite(&PatchSuite{}) func (s *PatchSuite) TestStatsWithSubmodules(c *C) { - storer, err := filesystem.NewStorage( - fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One().DotGit()) + storer := filesystem.NewStorage( + fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One().DotGit(), cache.NewObjectLRUDefault()) commit, err := GetCommit(storer, plumbing.NewHash("b685400c1f9316f350965a5993d350bc746b0bf4")) diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tag_test.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tag_test.go index 9900093e7..59c28b022 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tag_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tag_test.go @@ -7,6 +7,7 @@ import ( "time" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem" "gopkg.in/src-d/go-git.v4/storage/memory" @@ -22,9 +23,7 @@ var _ = Suite(&TagSuite{}) func (s *TagSuite) SetUpSuite(c *C) { s.BaseObjectsSuite.SetUpSuite(c) - storer, err := filesystem.NewStorage( - fixtures.ByURL("https://github.com/git-fixtures/tags.git").One().DotGit()) - c.Assert(err, IsNil) + storer := filesystem.NewStorage(fixtures.ByURL("https://github.com/git-fixtures/tags.git").One().DotGit(), cache.NewObjectLRUDefault()) s.Storer = storer } @@ -265,7 +264,7 @@ func (s *TagSuite) TestStringNonCommit(c *C) { c.Assert(tag.String(), Equals, "tag TAG TWO\n"+ "Tagger: <>\n"+ - "Date: Mon Jan 01 00:00:00 0001 +0000\n"+ + "Date: Thu Jan 01 00:00:00 1970 +0000\n"+ "\n"+ "tag two\n") } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tree_test.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tree_test.go index 59d5d215f..736642186 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tree_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/object/tree_test.go @@ -5,6 +5,7 @@ import ( "io" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/filemode" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -341,8 +342,7 @@ func (s *TreeSuite) TestTreeWalkerNextNonRecursive(c *C) { func (s *TreeSuite) TestTreeWalkerNextSubmodule(c *C) { dotgit := fixtures.ByURL("https://github.com/git-fixtures/submodule.git").One().DotGit() - st, err := filesystem.NewStorage(dotgit) - c.Assert(err, IsNil) + st := filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault()) hash := plumbing.NewHash("b685400c1f9316f350965a5993d350bc746b0bf4") commit, err := GetCommit(st, hash) diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/revlist/revlist_test.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/revlist/revlist_test.go index 55d9bca2a..dea1c73d9 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/revlist/revlist_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/revlist/revlist_test.go @@ -4,6 +4,7 @@ import ( "testing" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -51,8 +52,7 @@ const ( func (s *RevListSuite) SetUpTest(c *C) { s.Suite.SetUpSuite(c) - sto, err := filesystem.NewStorage(fixtures.Basic().One().DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fixtures.Basic().One().DotGit(), cache.NewObjectLRUDefault()) s.Storer = sto } @@ -67,8 +67,7 @@ func (s *RevListSuite) TestRevListObjects_Submodules(c *C) { "6ecf0ef2c2dffb796033e5a02219af86ec6584e5": true, } - sto, err := filesystem.NewStorage(fixtures.ByTag("submodule").One().DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fixtures.ByTag("submodule").One().DotGit(), cache.NewObjectLRUDefault()) ref, err := storer.ResolveReference(sto, plumbing.HEAD) c.Assert(err, IsNil) @@ -109,10 +108,9 @@ func (s *RevListSuite) TestRevListObjects(c *C) { } func (s *RevListSuite) TestRevListObjectsTagObject(c *C) { - sto, err := filesystem.NewStorage( + sto := filesystem.NewStorage( fixtures.ByTag("tags"). - ByURL("https://github.com/git-fixtures/tags.git").One().DotGit()) - c.Assert(err, IsNil) + ByURL("https://github.com/git-fixtures/tags.git").One().DotGit(), cache.NewObjectLRUDefault()) expected := map[string]bool{ "70846e9a10ef7b41064b40f07713d5b8b9a8fc73": true, diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/server/loader.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/server/loader.go index c83752c25..13b35262d 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/server/loader.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/server/loader.go @@ -1,6 +1,7 @@ package server import ( + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -43,7 +44,7 @@ func (l *fsLoader) Load(ep *transport.Endpoint) (storer.Storer, error) { return nil, transport.ErrRepositoryNotFound } - return filesystem.NewStorage(fs) + return filesystem.NewStorage(fs, cache.NewObjectLRUDefault()), nil } // MapLoader is a Loader that uses a lookup map of storer.Storer by diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/server/server_test.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/server/server_test.go index 33d74d19f..302ff486e 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/server/server_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/server/server_test.go @@ -3,6 +3,7 @@ package server_test import ( "testing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/transport" "gopkg.in/src-d/go-git.v4/plumbing/transport/client" "gopkg.in/src-d/go-git.v4/plumbing/transport/server" @@ -53,8 +54,7 @@ func (s *BaseSuite) prepareRepositories(c *C) { fs := fixtures.Basic().One().DotGit() s.Endpoint, err = transport.NewEndpoint(fs.Root()) c.Assert(err, IsNil) - s.loader[s.Endpoint.String()], err = filesystem.NewStorage(fs) - c.Assert(err, IsNil) + s.loader[s.Endpoint.String()] = filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) s.EmptyEndpoint, err = transport.NewEndpoint("/empty.git") c.Assert(err, IsNil) diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/ssh/auth_method.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/ssh/auth_method.go index 84cfab2a6..dbb47c56b 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/ssh/auth_method.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/ssh/auth_method.go @@ -236,7 +236,7 @@ func (a *PublicKeysCallback) ClientConfig() (*ssh.ClientConfig, error) { // NewKnownHostsCallback returns ssh.HostKeyCallback based on a file based on a // known_hosts file. http://man.openbsd.org/sshd#SSH_KNOWN_HOSTS_FILE_FORMAT // -// If files is empty, the list of files will be read from the SSH_KNOWN_HOSTS +// If list of files is empty, then it will be read from the SSH_KNOWN_HOSTS // environment variable, example: // /home/foo/custom_known_hosts_file:/etc/custom_known/hosts_file // @@ -244,13 +244,15 @@ func (a *PublicKeysCallback) ClientConfig() (*ssh.ClientConfig, error) { // ~/.ssh/known_hosts // /etc/ssh/ssh_known_hosts func NewKnownHostsCallback(files ...string) (ssh.HostKeyCallback, error) { - files, err := getDefaultKnownHostsFiles() - if err != nil { - return nil, err + var err error + + if len(files) == 0 { + if files, err = getDefaultKnownHostsFiles(); err != nil { + return nil, err + } } - files, err = filterKnownHostsFiles(files...) - if err != nil { + if files, err = filterKnownHostsFiles(files...); err != nil { return nil, err } diff --git a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/ssh/auth_method_test.go b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/ssh/auth_method_test.go index 00256694f..0cde61eea 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/ssh/auth_method_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/plumbing/transport/ssh/auth_method_test.go @@ -1,16 +1,30 @@ package ssh import ( + "bufio" "fmt" "io/ioutil" "os" + "strings" + "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/testdata" . "gopkg.in/check.v1" ) -type SuiteCommon struct{} +type ( + SuiteCommon struct{} + + mockKnownHosts struct{} +) + +func (mockKnownHosts) host() string { return "github.com" } +func (mockKnownHosts) knownHosts() []byte { + return []byte(`github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==`) +} +func (mockKnownHosts) Network() string { return "tcp" } +func (mockKnownHosts) String() string { return "github.com:22" } var _ = Suite(&SuiteCommon{}) @@ -149,3 +163,49 @@ func (*SuiteCommon) TestNewPublicKeysWithInvalidPEM(c *C) { c.Assert(err, NotNil) c.Assert(auth, IsNil) } + +func (*SuiteCommon) TestNewKnownHostsCallback(c *C) { + var mock = mockKnownHosts{} + + f, err := ioutil.TempFile("", "known-hosts") + c.Assert(err, IsNil) + + _, err = f.Write(mock.knownHosts()) + c.Assert(err, IsNil) + + err = f.Close() + c.Assert(err, IsNil) + + defer os.RemoveAll(f.Name()) + + f, err = os.Open(f.Name()) + c.Assert(err, IsNil) + + defer f.Close() + + var hostKey ssh.PublicKey + scanner := bufio.NewScanner(f) + for scanner.Scan() { + fields := strings.Split(scanner.Text(), " ") + if len(fields) != 3 { + continue + } + if strings.Contains(fields[0], mock.host()) { + var err error + hostKey, _, _, _, err = ssh.ParseAuthorizedKey(scanner.Bytes()) + if err != nil { + c.Fatalf("error parsing %q: %v", fields[2], err) + } + break + } + } + if hostKey == nil { + c.Fatalf("no hostkey for %s", mock.host()) + } + + clb, err := NewKnownHostsCallback(f.Name()) + c.Assert(err, IsNil) + + err = clb(mock.String(), mock, hostKey) + c.Assert(err, IsNil) +} diff --git a/vendor/gopkg.in/src-d/go-git.v4/prune_test.go b/vendor/gopkg.in/src-d/go-git.v4/prune_test.go index 60652ec9f..670cd07bd 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/prune_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/prune_test.go @@ -4,6 +4,7 @@ import ( "time" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage" "gopkg.in/src-d/go-git.v4/storage/filesystem" @@ -22,8 +23,7 @@ func (s *PruneSuite) testPrune(c *C, deleteTime time.Time) { srcFs := fixtures.ByTag("unpacked").One().DotGit() var sto storage.Storer var err error - sto, err = filesystem.NewStorage(srcFs) - c.Assert(err, IsNil) + sto = filesystem.NewStorage(srcFs, cache.NewObjectLRUDefault()) los := sto.(storer.LooseObjectStorer) c.Assert(los, NotNil) diff --git a/vendor/gopkg.in/src-d/go-git.v4/remote_test.go b/vendor/gopkg.in/src-d/go-git.v4/remote_test.go index dd386b083..175faed36 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/remote_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/remote_test.go @@ -9,6 +9,7 @@ import ( "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage" @@ -238,7 +239,7 @@ func (s *RemoteSuite) TestFetchWithPackfileWriter(c *C) { defer os.RemoveAll(dir) // clean up - fss, err := filesystem.NewStorage(osfs.New(dir)) + fss := filesystem.NewStorage(osfs.New(dir), cache.NewObjectLRUDefault()) c.Assert(err, IsNil) mock := &mockPackfileWriter{Storer: fss} @@ -375,8 +376,7 @@ func (s *RemoteSuite) TestFetchFastForwardFS(c *C) { defer os.RemoveAll(dir) // clean up - fss, err := filesystem.NewStorage(osfs.New(dir)) - c.Assert(err, IsNil) + fss := filesystem.NewStorage(osfs.New(dir), cache.NewObjectLRUDefault()) // This exercises `storage.filesystem.Storage.CheckAndSetReference()`. s.testFetchFastForward(c, fss) @@ -400,8 +400,7 @@ func (s *RemoteSuite) TestPushToEmptyRepository(c *C) { c.Assert(err, IsNil) srcFs := fixtures.Basic().One().DotGit() - sto, err := filesystem.NewStorage(srcFs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(srcFs, cache.NewObjectLRUDefault()) r := newRemote(sto, &config.RemoteConfig{ Name: DefaultRemoteName, @@ -438,8 +437,7 @@ func (s *RemoteSuite) TestPushContext(c *C) { c.Assert(err, IsNil) fs := fixtures.ByURL("https://github.com/git-fixtures/tags.git").One().DotGit() - sto, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r := newRemote(sto, &config.RemoteConfig{ Name: DefaultRemoteName, @@ -461,8 +459,7 @@ func (s *RemoteSuite) TestPushTags(c *C) { c.Assert(err, IsNil) fs := fixtures.ByURL("https://github.com/git-fixtures/tags.git").One().DotGit() - sto, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r := newRemote(sto, &config.RemoteConfig{ Name: DefaultRemoteName, @@ -485,15 +482,14 @@ func (s *RemoteSuite) TestPushTags(c *C) { func (s *RemoteSuite) TestPushNoErrAlreadyUpToDate(c *C) { fs := fixtures.Basic().One().DotGit() - sto, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r := newRemote(sto, &config.RemoteConfig{ Name: DefaultRemoteName, URLs: []string{fs.Root()}, }) - err = r.Push(&PushOptions{ + err := r.Push(&PushOptions{ RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*"}, }) c.Assert(err, Equals, NoErrAlreadyUpToDate) @@ -501,8 +497,7 @@ func (s *RemoteSuite) TestPushNoErrAlreadyUpToDate(c *C) { func (s *RemoteSuite) TestPushDeleteReference(c *C) { fs := fixtures.Basic().One().DotGit() - sto, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r, err := PlainClone(c.MkDir(), true, &CloneOptions{ URL: fs.Root(), @@ -526,8 +521,7 @@ func (s *RemoteSuite) TestPushDeleteReference(c *C) { func (s *RemoteSuite) TestPushRejectNonFastForward(c *C) { fs := fixtures.Basic().One().DotGit() - server, err := filesystem.NewStorage(fs) - c.Assert(err, IsNil) + server := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) r, err := PlainClone(c.MkDir(), true, &CloneOptions{ URL: fs.Root(), @@ -554,12 +548,10 @@ func (s *RemoteSuite) TestPushRejectNonFastForward(c *C) { func (s *RemoteSuite) TestPushForce(c *C) { f := fixtures.Basic().One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) dstFs := f.DotGit() - dstSto, err := filesystem.NewStorage(dstFs) - c.Assert(err, IsNil) + dstSto := filesystem.NewStorage(dstFs, cache.NewObjectLRUDefault()) url := dstFs.Root() r := newRemote(sto, &config.RemoteConfig{ @@ -703,8 +695,7 @@ func (s *RemoteSuite) TestPushWrongRemoteName(c *C) { func (s *RemoteSuite) TestGetHaves(c *C) { f := fixtures.Basic().One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) var localRefs = []*plumbing.Reference{ plumbing.NewReferenceFromStrings( diff --git a/vendor/gopkg.in/src-d/go-git.v4/repository.go b/vendor/gopkg.in/src-d/go-git.v4/repository.go index 818cfb3c3..bfe06a369 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/repository.go +++ b/vendor/gopkg.in/src-d/go-git.v4/repository.go @@ -13,6 +13,7 @@ import ( "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/internal/revision" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/format/packfile" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" @@ -220,10 +221,7 @@ func PlainInit(path string, isBare bool) (*Repository, error) { dot, _ = wt.Chroot(GitDirName) } - s, err := filesystem.NewStorage(dot) - if err != nil { - return nil, err - } + s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) return Init(s, wt) } @@ -235,9 +233,8 @@ func PlainOpen(path string) (*Repository, error) { return PlainOpenWithOptions(path, &PlainOpenOptions{}) } -// PlainOpen opens a git repository from the given path. It detects if the -// repository is bare or a normal one. If the path doesn't contain a valid -// repository ErrRepositoryNotExists is returned +// PlainOpenWithOptions opens a git repository from the given path with specific +// options. See PlainOpen for more info. func PlainOpenWithOptions(path string, o *PlainOpenOptions) (*Repository, error) { dot, wt, err := dotGitToOSFilesystems(path, o.DetectDotGit) if err != nil { @@ -252,10 +249,7 @@ func PlainOpenWithOptions(path string, o *PlainOpenOptions) (*Repository, error) return nil, err } - s, err := filesystem.NewStorage(dot) - if err != nil { - return nil, err - } + s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) return Open(s, wt) } diff --git a/vendor/gopkg.in/src-d/go-git.v4/repository_test.go b/vendor/gopkg.in/src-d/go-git.v4/repository_test.go index 261af7a7b..88071cfd5 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/repository_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/repository_test.go @@ -15,6 +15,7 @@ import ( "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage" @@ -51,8 +52,7 @@ func (s *RepositorySuite) TestInitNonStandardDotGit(c *C) { fs := osfs.New(dir) dot, _ := fs.Chroot("storage") - storage, err := filesystem.NewStorage(dot) - c.Assert(err, IsNil) + storage := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) wt, _ := fs.Chroot("worktree") r, err := Init(storage, wt) @@ -78,8 +78,7 @@ func (s *RepositorySuite) TestInitStandardDotGit(c *C) { fs := osfs.New(dir) dot, _ := fs.Chroot(".git") - storage, err := filesystem.NewStorage(dot) - c.Assert(err, IsNil) + storage := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) r, err := Init(storage, fs) c.Assert(err, IsNil) @@ -1058,8 +1057,7 @@ func (s *RepositorySuite) TestPushDepth(c *C) { func (s *RepositorySuite) TestPushNonExistentRemote(c *C) { srcFs := fixtures.Basic().One().DotGit() - sto, err := filesystem.NewStorage(srcFs) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(srcFs, cache.NewObjectLRUDefault()) r, err := Open(sto, srcFs) c.Assert(err, IsNil) @@ -1277,8 +1275,7 @@ func (s *RepositorySuite) TestTags(c *C) { func (s *RepositorySuite) TestBranches(c *C) { f := fixtures.ByURL("https://github.com/git-fixtures/root-references.git").One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) r, err := Open(sto, f.DotGit()) c.Assert(err, IsNil) @@ -1495,8 +1492,7 @@ func (s *RepositorySuite) TestWorktreeBare(c *C) { func (s *RepositorySuite) TestResolveRevision(c *C) { f := fixtures.ByURL("https://github.com/git-fixtures/basic.git").One() - sto, err := filesystem.NewStorage(f.DotGit()) - c.Assert(err, IsNil) + sto := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault()) r, err := Open(sto, f.DotGit()) c.Assert(err, IsNil) @@ -1548,9 +1544,9 @@ func (s *RepositorySuite) TestResolveRevisionWithErrors(c *C) { c.Assert(err, IsNil) datas := map[string]string{ - "efs/heads/master~": "reference not found", - "HEAD^3": `Revision invalid : "3" found must be 0, 1 or 2 after "^"`, - "HEAD^{/whatever}": `No commit message match regexp : "whatever"`, + "efs/heads/master~": "reference not found", + "HEAD^3": `Revision invalid : "3" found must be 0, 1 or 2 after "^"`, + "HEAD^{/whatever}": `No commit message match regexp : "whatever"`, "4e1243bd22c66e76c2ba9eddc1f91394e57f9f83": "reference not found", "918c48b83bd081e863dbe1b80f8998f058cd8294": `refname "918c48b83bd081e863dbe1b80f8998f058cd8294" is ambiguous`, } @@ -1567,8 +1563,7 @@ func (s *RepositorySuite) testRepackObjects( srcFs := fixtures.ByTag("unpacked").One().DotGit() var sto storage.Storer var err error - sto, err = filesystem.NewStorage(srcFs) - c.Assert(err, IsNil) + sto = filesystem.NewStorage(srcFs, cache.NewObjectLRUDefault()) los := sto.(storer.LooseObjectStorer) c.Assert(los, NotNil) @@ -1733,10 +1728,7 @@ func BenchmarkObjects(b *testing.B) { b.Run(f.URL, func(b *testing.B) { fs := f.DotGit() - storer, err := filesystem.NewStorage(fs) - if err != nil { - b.Fatal(err) - } + storer := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) worktree, err := fs.Chroot(filepath.Dir(fs.Root())) if err != nil { diff --git a/vendor/gopkg.in/src-d/go-git.v4/status.go b/vendor/gopkg.in/src-d/go-git.v4/status.go index ef8a500f2..ecbf79350 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/status.go +++ b/vendor/gopkg.in/src-d/go-git.v4/status.go @@ -1,7 +1,10 @@ package git -import "fmt" -import "bytes" +import ( + "bytes" + "fmt" + "path/filepath" +) // Status represents the current status of a Worktree. // The key of the map is the path of the file. @@ -17,6 +20,12 @@ func (s Status) File(path string) *FileStatus { return s[path] } +// IsUntracked checks if file for given path is 'Untracked' +func (s Status) IsUntracked(path string) bool { + stat, ok := (s)[filepath.ToSlash(path)] + return ok && stat.Worktree == Untracked +} + // IsClean returns true if all the files aren't in Unmodified status. func (s Status) IsClean() bool { for _, status := range s { diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit.go index df4f75691..a58c2482a 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit.go +++ b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit.go @@ -57,21 +57,48 @@ var ( ErrSymRefTargetNotFound = errors.New("symbolic reference target not found") ) +// Options holds configuration for the storage. +type Options struct { + // ExclusiveAccess means that the filesystem is not modified externally + // while the repo is open. + ExclusiveAccess bool + // KeepDescriptors makes the file descriptors to be reused but they will + // need to be manually closed calling Close(). + KeepDescriptors bool +} + // The DotGit type represents a local git repository on disk. This // type is not zero-value-safe, use the New function to initialize it. type DotGit struct { - fs billy.Filesystem + options Options + fs billy.Filesystem // incoming object directory information incomingChecked bool incomingDirName string + + objectList []plumbing.Hash + objectMap map[plumbing.Hash]struct{} + packList []plumbing.Hash + packMap map[plumbing.Hash]struct{} + + files map[string]billy.File } // New returns a DotGit value ready to be used. The path argument must // be the absolute path of a git repository directory (e.g. // "/foo/bar/.git"). func New(fs billy.Filesystem) *DotGit { - return &DotGit{fs: fs} + return NewWithOptions(fs, Options{}) +} + +// NewWithOptions sets non default configuration options. +// See New for complete help. +func NewWithOptions(fs billy.Filesystem, o Options) *DotGit { + return &DotGit{ + options: o, + fs: fs, + } } // Initialize creates all the folder scaffolding. @@ -101,6 +128,28 @@ func (d *DotGit) Initialize() error { return nil } +// Close closes all opened files. +func (d *DotGit) Close() error { + var firstError error + if d.files != nil { + for _, f := range d.files { + err := f.Close() + if err != nil && firstError == nil { + firstError = err + continue + } + } + + d.files = nil + } + + if firstError != nil { + return firstError + } + + return nil +} + // ConfigWriter returns a file pointer for write to the config file func (d *DotGit) ConfigWriter() (billy.File, error) { return d.fs.Create(configPath) @@ -143,11 +192,25 @@ func (d *DotGit) Shallow() (billy.File, error) { // NewObjectPack return a writer for a new packfile, it saves the packfile to // disk and also generates and save the index for the given packfile. func (d *DotGit) NewObjectPack() (*PackWriter, error) { + d.cleanPackList() return newPackWrite(d.fs) } // ObjectPacks returns the list of availables packfiles func (d *DotGit) ObjectPacks() ([]plumbing.Hash, error) { + if !d.options.ExclusiveAccess { + return d.objectPacks() + } + + err := d.genPackList() + if err != nil { + return nil, err + } + + return d.packList, nil +} + +func (d *DotGit) objectPacks() ([]plumbing.Hash, error) { packDir := d.fs.Join(objectsPath, packPath) files, err := d.fs.ReadDir(packDir) if err != nil { @@ -181,7 +244,22 @@ func (d *DotGit) objectPackPath(hash plumbing.Hash, extension string) string { } func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.File, error) { - pack, err := d.fs.Open(d.objectPackPath(hash, extension)) + if d.files == nil { + d.files = make(map[string]billy.File) + } + + err := d.hasPack(hash) + if err != nil { + return nil, err + } + + path := d.objectPackPath(hash, extension) + f, ok := d.files[path] + if ok { + return f, nil + } + + pack, err := d.fs.Open(path) if err != nil { if os.IsNotExist(err) { return nil, ErrPackfileNotFound @@ -190,20 +268,36 @@ func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.Fil return nil, err } + if d.options.KeepDescriptors && extension == "pack" { + d.files[path] = pack + } + return pack, nil } // ObjectPack returns a fs.File of the given packfile func (d *DotGit) ObjectPack(hash plumbing.Hash) (billy.File, error) { + err := d.hasPack(hash) + if err != nil { + return nil, err + } + return d.objectPackOpen(hash, `pack`) } // ObjectPackIdx returns a fs.File of the index file for a given packfile func (d *DotGit) ObjectPackIdx(hash plumbing.Hash) (billy.File, error) { + err := d.hasPack(hash) + if err != nil { + return nil, err + } + return d.objectPackOpen(hash, `idx`) } func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) error { + d.cleanPackList() + path := d.objectPackPath(hash, `pack`) if !t.IsZero() { fi, err := d.fs.Stat(path) @@ -224,12 +318,23 @@ func (d *DotGit) DeleteOldObjectPackAndIndex(hash plumbing.Hash, t time.Time) er // NewObject return a writer for a new object file. func (d *DotGit) NewObject() (*ObjectWriter, error) { + d.cleanObjectList() + return newObjectWriter(d.fs) } // Objects returns a slice with the hashes of objects found under the // .git/objects/ directory. func (d *DotGit) Objects() ([]plumbing.Hash, error) { + if d.options.ExclusiveAccess { + err := d.genObjectList() + if err != nil { + return nil, err + } + + return d.objectList, nil + } + var objects []plumbing.Hash err := d.ForEachObjectHash(func(hash plumbing.Hash) error { objects = append(objects, hash) @@ -241,9 +346,29 @@ func (d *DotGit) Objects() ([]plumbing.Hash, error) { return objects, nil } -// Objects returns a slice with the hashes of objects found under the -// .git/objects/ directory. +// ForEachObjectHash iterates over the hashes of objects found under the +// .git/objects/ directory and executes the provided function. func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { + if !d.options.ExclusiveAccess { + return d.forEachObjectHash(fun) + } + + err := d.genObjectList() + if err != nil { + return err + } + + for _, h := range d.objectList { + err := fun(h) + if err != nil { + return err + } + } + + return nil +} + +func (d *DotGit) forEachObjectHash(fun func(plumbing.Hash) error) error { files, err := d.fs.ReadDir(objectsPath) if err != nil { if os.IsNotExist(err) { @@ -278,6 +403,87 @@ func (d *DotGit) ForEachObjectHash(fun func(plumbing.Hash) error) error { return nil } +func (d *DotGit) cleanObjectList() { + d.objectMap = nil + d.objectList = nil +} + +func (d *DotGit) genObjectList() error { + if d.objectMap != nil { + return nil + } + + d.objectMap = make(map[plumbing.Hash]struct{}) + return d.forEachObjectHash(func(h plumbing.Hash) error { + d.objectList = append(d.objectList, h) + d.objectMap[h] = struct{}{} + + return nil + }) +} + +func (d *DotGit) hasObject(h plumbing.Hash) error { + if !d.options.ExclusiveAccess { + return nil + } + + err := d.genObjectList() + if err != nil { + return err + } + + _, ok := d.objectMap[h] + if !ok { + return plumbing.ErrObjectNotFound + } + + return nil +} + +func (d *DotGit) cleanPackList() { + d.packMap = nil + d.packList = nil +} + +func (d *DotGit) genPackList() error { + if d.packMap != nil { + return nil + } + + op, err := d.objectPacks() + if err != nil { + return err + } + + d.packMap = make(map[plumbing.Hash]struct{}) + d.packList = nil + + for _, h := range op { + d.packList = append(d.packList, h) + d.packMap[h] = struct{}{} + } + + return nil +} + +func (d *DotGit) hasPack(h plumbing.Hash) error { + if !d.options.ExclusiveAccess { + return nil + } + + err := d.genPackList() + if err != nil { + return err + } + + _, ok := d.packMap[h] + if !ok { + return ErrPackfileNotFound + } + + return nil +} + func (d *DotGit) objectPath(h plumbing.Hash) string { hash := h.String() return d.fs.Join(objectsPath, hash[0:2], hash[2:40]) @@ -322,6 +528,11 @@ func (d *DotGit) hasIncomingObjects() bool { // Object returns a fs.File pointing the object file, if exists func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { + err := d.hasObject(h) + if err != nil { + return nil, err + } + obj1, err1 := d.fs.Open(d.objectPath(h)) if os.IsNotExist(err1) && d.hasIncomingObjects() { obj2, err2 := d.fs.Open(d.incomingObjectPath(h)) @@ -335,6 +546,11 @@ func (d *DotGit) Object(h plumbing.Hash) (billy.File, error) { // ObjectStat returns a os.FileInfo pointing the object file, if exists func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { + err := d.hasObject(h) + if err != nil { + return nil, err + } + obj1, err1 := d.fs.Stat(d.objectPath(h)) if os.IsNotExist(err1) && d.hasIncomingObjects() { obj2, err2 := d.fs.Stat(d.incomingObjectPath(h)) @@ -348,6 +564,8 @@ func (d *DotGit) ObjectStat(h plumbing.Hash) (os.FileInfo, error) { // ObjectDelete removes the object file, if exists func (d *DotGit) ObjectDelete(h plumbing.Hash) error { + d.cleanObjectList() + err1 := d.fs.Remove(d.objectPath(h)) if os.IsNotExist(err1) && d.hasIncomingObjects() { err2 := d.fs.Remove(d.incomingObjectPath(h)) diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_test.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_test.go index 64c2aeead..308c6b70f 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit/dotgit_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "gopkg.in/src-d/go-billy.v4" "gopkg.in/src-d/go-git.v4/plumbing" . "gopkg.in/check.v1" @@ -424,6 +425,18 @@ func (s *SuiteDotGit) TestObjectPacks(c *C) { fs := f.DotGit() dir := New(fs) + testObjectPacks(c, fs, dir, f) +} + +func (s *SuiteDotGit) TestObjectPacksExclusive(c *C) { + f := fixtures.Basic().ByTag(".git").One() + fs := f.DotGit() + dir := NewWithOptions(fs, Options{ExclusiveAccess: true}) + + testObjectPacks(c, fs, dir, f) +} + +func testObjectPacks(c *C, fs billy.Filesystem, dir *DotGit, f *fixtures.Fixture) { hashes, err := dir.ObjectPacks() c.Assert(err, IsNil) c.Assert(hashes, HasLen, 1) @@ -452,6 +465,45 @@ func (s *SuiteDotGit) TestObjectPack(c *C) { c.Assert(filepath.Ext(pack.Name()), Equals, ".pack") } +func (s *SuiteDotGit) TestObjectPackWithKeepDescriptors(c *C) { + f := fixtures.Basic().ByTag(".git").One() + fs := f.DotGit() + dir := NewWithOptions(fs, Options{KeepDescriptors: true}) + + pack, err := dir.ObjectPack(f.PackfileHash) + c.Assert(err, IsNil) + c.Assert(filepath.Ext(pack.Name()), Equals, ".pack") + + // Move to an specific offset + pack.Seek(42, os.SEEK_SET) + + pack2, err := dir.ObjectPack(f.PackfileHash) + c.Assert(err, IsNil) + + // If the file is the same the offset should be the same + offset, err := pack2.Seek(0, os.SEEK_CUR) + c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(42)) + + err = dir.Close() + c.Assert(err, IsNil) + + pack2, err = dir.ObjectPack(f.PackfileHash) + c.Assert(err, IsNil) + + // If the file is opened again its offset should be 0 + offset, err = pack2.Seek(0, os.SEEK_CUR) + c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(0)) + + err = pack2.Close() + c.Assert(err, IsNil) + + err = dir.Close() + c.Assert(err, NotNil) + +} + func (s *SuiteDotGit) TestObjectPackIdx(c *C) { f := fixtures.Basic().ByTag(".git").One() fs := f.DotGit() @@ -506,6 +558,17 @@ func (s *SuiteDotGit) TestObjects(c *C) { fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() dir := New(fs) + testObjects(c, fs, dir) +} + +func (s *SuiteDotGit) TestObjectsExclusive(c *C) { + fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() + dir := NewWithOptions(fs, Options{ExclusiveAccess: true}) + + testObjects(c, fs, dir) +} + +func testObjects(c *C, fs billy.Filesystem, dir *DotGit) { hashes, err := dir.Objects() c.Assert(err, IsNil) c.Assert(hashes, HasLen, 187) diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/module.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/module.go index 7c8c8d866..927220674 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/module.go +++ b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/module.go @@ -1,6 +1,7 @@ package filesystem import ( + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" ) @@ -15,5 +16,5 @@ func (s *ModuleStorage) Module(name string) (storage.Storer, error) { return nil, err } - return NewStorage(fs) + return NewStorage(fs, cache.NewObjectLRUDefault()), nil } diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/object.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/object.go index 3a3a2bd97..9eb085fd3 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/object.go +++ b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/object.go @@ -18,6 +18,8 @@ import ( ) type ObjectStorage struct { + options Options + // deltaBaseCache is an object cache uses to cache delta's bases when deltaBaseCache cache.Object @@ -25,14 +27,18 @@ type ObjectStorage struct { index map[plumbing.Hash]idxfile.Index } -// NewObjectStorage creates a new ObjectStorage with the given .git directory. -func NewObjectStorage(dir *dotgit.DotGit) (ObjectStorage, error) { - s := ObjectStorage{ - deltaBaseCache: cache.NewObjectLRUDefault(), +// NewObjectStorage creates a new ObjectStorage with the given .git directory and cache. +func NewObjectStorage(dir *dotgit.DotGit, cache cache.Object) *ObjectStorage { + return NewObjectStorageWithOptions(dir, cache, Options{}) +} + +// NewObjectStorageWithOptions creates a new ObjectStorage with the given .git directory, cache and extra options +func NewObjectStorageWithOptions(dir *dotgit.DotGit, cache cache.Object, ops Options) *ObjectStorage { + return &ObjectStorage{ + options: ops, + deltaBaseCache: cache, dir: dir, } - - return s, nil } func (s *ObjectStorage) requireIndex() error { @@ -62,6 +68,7 @@ func (s *ObjectStorage) loadIdxFile(h plumbing.Hash) (err error) { } defer ioutil.CheckClose(f, &err) + idxf := idxfile.NewMemoryIndex() d := idxfile.NewDecoder(f) if err = d.Decode(idxf); err != nil { @@ -169,10 +176,7 @@ func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (p // Create a new object storage with the DotGit(s) and check for the // required hash object. Skip when not found. for _, dg := range dotgits { - o, oe := NewObjectStorage(dg) - if oe != nil { - continue - } + o := NewObjectStorage(dg, s.deltaBaseCache) enobj, enerr := o.EncodedObject(t, h) if enerr != nil { continue @@ -268,7 +272,9 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) ( return nil, err } - defer ioutil.CheckClose(f, &err) + if !s.options.KeepDescriptors { + defer ioutil.CheckClose(f, &err) + } idx := s.index[pack] if canBeDelta { @@ -411,6 +417,11 @@ func (s *ObjectStorage) buildPackfileIters(t plumbing.ObjectType, seen map[plumb }, nil } +// Close closes all opened files. +func (s *ObjectStorage) Close() error { + return s.dir.Close() +} + type lazyPackfilesIter struct { hashes []plumbing.Hash open func(h plumbing.Hash) (storer.EncodedObjectIter, error) diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/object_test.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/object_test.go index b1408b7c2..bd4a94b40 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/object_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/object_test.go @@ -2,9 +2,11 @@ package filesystem import ( "io/ioutil" + "os" "testing" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" . "gopkg.in/check.v1" @@ -26,8 +28,7 @@ var _ = Suite(&FsSuite{}) func (s *FsSuite) TestGetFromObjectFile(c *C) { fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) expected := plumbing.NewHash("f3dfe29d268303fc6e1bbce268605fc99573406e") obj, err := o.EncodedObject(plumbing.AnyObject, expected) @@ -38,20 +39,53 @@ func (s *FsSuite) TestGetFromObjectFile(c *C) { func (s *FsSuite) TestGetFromPackfile(c *C) { fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) { fs := f.DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) + + expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") + obj, err := o.EncodedObject(plumbing.AnyObject, expected) c.Assert(err, IsNil) + c.Assert(obj.Hash(), Equals, expected) + }) +} + +func (s *FsSuite) TestGetFromPackfileKeepDescriptors(c *C) { + fixtures.Basic().ByTag(".git").Test(c, func(f *fixtures.Fixture) { + fs := f.DotGit() + dg := dotgit.NewWithOptions(fs, dotgit.Options{KeepDescriptors: true}) + o := NewObjectStorageWithOptions(dg, cache.NewObjectLRUDefault(), Options{KeepDescriptors: true}) expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") obj, err := o.EncodedObject(plumbing.AnyObject, expected) c.Assert(err, IsNil) c.Assert(obj.Hash(), Equals, expected) + + packfiles, err := dg.ObjectPacks() + c.Assert(err, IsNil) + + pack1, err := dg.ObjectPack(packfiles[0]) + c.Assert(err, IsNil) + + pack1.Seek(42, os.SEEK_SET) + + err = o.Close() + c.Assert(err, IsNil) + + pack2, err := dg.ObjectPack(packfiles[0]) + c.Assert(err, IsNil) + + offset, err := pack2.Seek(0, os.SEEK_CUR) + c.Assert(err, IsNil) + c.Assert(offset, Equals, int64(0)) + + err = o.Close() + c.Assert(err, IsNil) + }) } func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) { fs := fixtures.ByTag(".git").ByTag("multi-packfile").One().DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) expected := plumbing.NewHash("8d45a34641d73851e01d3754320b33bb5be3c4d3") obj, err := o.getFromPackfile(expected, false) @@ -67,8 +101,7 @@ func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) { func (s *FsSuite) TestIter(c *C) { fixtures.ByTag(".git").ByTag("packfile").Test(c, func(f *fixtures.Fixture) { fs := f.DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) iter, err := o.IterEncodedObjects(plumbing.AnyObject) c.Assert(err, IsNil) @@ -88,8 +121,7 @@ func (s *FsSuite) TestIterWithType(c *C) { fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) { for _, t := range objectTypes { fs := f.DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - c.Assert(err, IsNil) + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) iter, err := o.IterEncodedObjects(t) c.Assert(err, IsNil) @@ -271,11 +303,7 @@ func BenchmarkGetObjectFromPackfile(b *testing.B) { for _, f := range fixtures.Basic() { b.Run(f.URL, func(b *testing.B) { fs := f.DotGit() - o, err := NewObjectStorage(dotgit.New(fs)) - if err != nil { - b.Fatal(err) - } - + o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault()) for i := 0; i < b.N; i++ { expected := plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5") obj, err := o.EncodedObject(plumbing.AnyObject, expected) diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/storage.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/storage.go index 622bb4a8d..14a772abe 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/storage.go +++ b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/storage.go @@ -2,6 +2,7 @@ package filesystem import ( + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit" "gopkg.in/src-d/go-billy.v4" @@ -22,25 +23,45 @@ type Storage struct { ModuleStorage } -// NewStorage returns a new Storage backed by a given `fs.Filesystem` -func NewStorage(fs billy.Filesystem) (*Storage, error) { - dir := dotgit.New(fs) - o, err := NewObjectStorage(dir) - if err != nil { - return nil, err +// Options holds configuration for the storage. +type Options struct { + // ExclusiveAccess means that the filesystem is not modified externally + // while the repo is open. + ExclusiveAccess bool + // KeepDescriptors makes the file descriptors to be reused but they will + // need to be manually closed calling Close(). + KeepDescriptors bool +} + +// NewStorage returns a new Storage backed by a given `fs.Filesystem` and cache. +func NewStorage(fs billy.Filesystem, cache cache.Object) *Storage { + return NewStorageWithOptions(fs, cache, Options{}) +} + +// NewStorageWithOptions returns a new Storage with extra options, +// backed by a given `fs.Filesystem` and cache. +func NewStorageWithOptions(fs billy.Filesystem, cache cache.Object, ops Options) *Storage { + dirOps := dotgit.Options{ + ExclusiveAccess: ops.ExclusiveAccess, + KeepDescriptors: ops.KeepDescriptors, } + dir := dotgit.NewWithOptions(fs, dirOps) return &Storage{ fs: fs, dir: dir, - ObjectStorage: o, + ObjectStorage: ObjectStorage{ + options: ops, + deltaBaseCache: cache, + dir: dir, + }, ReferenceStorage: ReferenceStorage{dir: dir}, IndexStorage: IndexStorage{dir: dir}, ShallowStorage: ShallowStorage{dir: dir}, ConfigStorage: ConfigStorage{dir: dir}, ModuleStorage: ModuleStorage{dir: dir}, - }, nil + } } // Filesystem returns the underlying filesystem @@ -48,6 +69,7 @@ func (s *Storage) Filesystem() billy.Filesystem { return s.fs } +// Init initializes .git directory func (s *Storage) Init() error { return s.dir.Initialize() } diff --git a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/storage_test.go b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/storage_test.go index 4d9ba6fec..6fa0d908a 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/storage_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/storage/filesystem/storage_test.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "testing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/storer" "gopkg.in/src-d/go-git.v4/storage/test" @@ -23,9 +24,12 @@ var _ = Suite(&StorageSuite{}) func (s *StorageSuite) SetUpTest(c *C) { s.dir = c.MkDir() - storage, err := NewStorage(osfs.New(s.dir)) - c.Assert(err, IsNil) + storage := NewStorage(osfs.New(s.dir), cache.NewObjectLRUDefault()) + + setUpTest(s, c, storage) +} +func setUpTest(s *StorageSuite, c *C, storage *Storage) { // ensure that right interfaces are implemented var _ storer.EncodedObjectStorer = storage var _ storer.IndexStorer = storage @@ -40,8 +44,7 @@ func (s *StorageSuite) SetUpTest(c *C) { func (s *StorageSuite) TestFilesystem(c *C) { fs := memfs.New() - storage, err := NewStorage(fs) - c.Assert(err, IsNil) + storage := NewStorage(fs, cache.NewObjectLRUDefault()) c.Assert(storage.Filesystem(), Equals, fs) } @@ -51,3 +54,19 @@ func (s *StorageSuite) TestNewStorageShouldNotAddAnyContentsToDir(c *C) { c.Assert(err, IsNil) c.Assert(fis, HasLen, 0) } + +type StorageExclusiveSuite struct { + StorageSuite +} + +var _ = Suite(&StorageExclusiveSuite{}) + +func (s *StorageExclusiveSuite) SetUpTest(c *C) { + s.dir = c.MkDir() + storage := NewStorageWithOptions( + osfs.New(s.dir), + cache.NewObjectLRUDefault(), + Options{ExclusiveAccess: true}) + + setUpTest(&s.StorageSuite, c, storage) +} diff --git a/vendor/gopkg.in/src-d/go-git.v4/worktree.go b/vendor/gopkg.in/src-d/go-git.v4/worktree.go index 99b2cd124..e45d81548 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/worktree.go +++ b/vendor/gopkg.in/src-d/go-git.v4/worktree.go @@ -713,29 +713,54 @@ func (w *Worktree) readGitmodulesFile() (*config.Modules, error) { } // Clean the worktree by removing untracked files. +// An empty dir could be removed - this is what `git clean -f -d .` does. func (w *Worktree) Clean(opts *CleanOptions) error { s, err := w.Status() if err != nil { return err } - // Check Worktree status to be Untracked, obtain absolute path and delete. - for relativePath, status := range s { - // Check if the path contains a directory and if Dir options is false, - // skip the path. - if relativePath != filepath.Base(relativePath) && !opts.Dir { + root := "" + files, err := w.Filesystem.ReadDir(root) + if err != nil { + return err + } + return w.doClean(s, opts, root, files) +} + +func (w *Worktree) doClean(status Status, opts *CleanOptions, dir string, files []os.FileInfo) error { + for _, fi := range files { + if fi.Name() == ".git" { continue } - // Remove the file only if it's an untracked file. - if status.Worktree == Untracked { - absPath := filepath.Join(w.Filesystem.Root(), relativePath) - if err := os.Remove(absPath); err != nil { + // relative path under the root + path := filepath.Join(dir, fi.Name()) + if fi.IsDir() { + if !opts.Dir { + continue + } + + subfiles, err := w.Filesystem.ReadDir(path) + if err != nil { + return err + } + err = w.doClean(status, opts, path, subfiles) + if err != nil { return err } + } else { + if status.IsUntracked(path) { + if err := w.Filesystem.Remove(path); err != nil { + return err + } + } } } + if opts.Dir { + return doCleanDirectories(w.Filesystem, dir) + } return nil } @@ -881,15 +906,18 @@ func rmFileAndDirIfEmpty(fs billy.Filesystem, name string) error { return err } - path := filepath.Dir(name) - files, err := fs.ReadDir(path) + dir := filepath.Dir(name) + return doCleanDirectories(fs, dir) +} + +// doCleanDirectories removes empty subdirs (without files) +func doCleanDirectories(fs billy.Filesystem, dir string) error { + files, err := fs.ReadDir(dir) if err != nil { return err } - if len(files) == 0 { - fs.Remove(path) + return fs.Remove(dir) } - return nil } diff --git a/vendor/gopkg.in/src-d/go-git.v4/worktree_commit.go b/vendor/gopkg.in/src-d/go-git.v4/worktree_commit.go index b83bf0c7a..673eb1678 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/worktree_commit.go +++ b/vendor/gopkg.in/src-d/go-git.v4/worktree_commit.go @@ -3,6 +3,7 @@ package git import ( "bytes" "path" + "sort" "strings" "golang.org/x/crypto/openpgp" @@ -188,7 +189,20 @@ func (h *buildTreeHelper) doBuildTree(e *index.Entry, parent, fullpath string) { h.trees[parent].Entries = append(h.trees[parent].Entries, te) } +type sortableEntries []object.TreeEntry + +func (sortableEntries) sortName(te object.TreeEntry) string { + if te.Mode == filemode.Dir { + return te.Name + "/" + } + return te.Name +} +func (se sortableEntries) Len() int { return len(se) } +func (se sortableEntries) Less(i int, j int) bool { return se.sortName(se[i]) < se.sortName(se[j]) } +func (se sortableEntries) Swap(i int, j int) { se[i], se[j] = se[j], se[i] } + func (h *buildTreeHelper) copyTreeToStorageRecursive(parent string, t *object.Tree) (plumbing.Hash, error) { + sort.Sort(sortableEntries(t.Entries)) for i, e := range t.Entries { if e.Mode != filemode.Dir && !e.Hash.IsZero() { continue diff --git a/vendor/gopkg.in/src-d/go-git.v4/worktree_commit_test.go b/vendor/gopkg.in/src-d/go-git.v4/worktree_commit_test.go index 004926fc5..62aae8a2f 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/worktree_commit_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/worktree_commit_test.go @@ -2,19 +2,25 @@ package git import ( "bytes" + "io/ioutil" + "os" + "os/exec" "strings" "time" - "golang.org/x/crypto/openpgp" - "golang.org/x/crypto/openpgp/armor" - "golang.org/x/crypto/openpgp/errors" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-git.v4/plumbing/storer" + "gopkg.in/src-d/go-git.v4/storage/filesystem" "gopkg.in/src-d/go-git.v4/storage/memory" + "golang.org/x/crypto/openpgp" + "golang.org/x/crypto/openpgp/armor" + "golang.org/x/crypto/openpgp/errors" . "gopkg.in/check.v1" "gopkg.in/src-d/go-billy.v4/memfs" + "gopkg.in/src-d/go-billy.v4/osfs" "gopkg.in/src-d/go-billy.v4/util" ) @@ -196,6 +202,53 @@ func (s *WorktreeSuite) TestCommitSignBadKey(c *C) { c.Assert(err, Equals, errors.InvalidArgumentError("signing key is encrypted")) } +func (s *WorktreeSuite) TestCommitTreeSort(c *C) { + path, err := ioutil.TempDir(os.TempDir(), "test-commit-tree-sort") + c.Assert(err, IsNil) + fs := osfs.New(path) + st := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) + r, err := Init(st, nil) + c.Assert(err, IsNil) + + r, err = Clone(memory.NewStorage(), memfs.New(), &CloneOptions{ + URL: path, + }) + + w, err := r.Worktree() + c.Assert(err, IsNil) + + mfs := w.Filesystem + + err = mfs.MkdirAll("delta", 0755) + c.Assert(err, IsNil) + + for _, p := range []string{"delta_last", "Gamma", "delta/middle", "Beta", "delta-first", "alpha"} { + util.WriteFile(mfs, p, []byte("foo"), 0644) + _, err = w.Add(p) + c.Assert(err, IsNil) + } + + _, err = w.Commit("foo\n", &CommitOptions{ + All: true, + Author: defaultSignature(), + }) + c.Assert(err, IsNil) + + err = r.Push(&PushOptions{}) + c.Assert(err, IsNil) + + cmd := exec.Command("git", "fsck") + cmd.Dir = path + cmd.Env = os.Environ() + buf := &bytes.Buffer{} + cmd.Stderr = buf + cmd.Stdout = buf + + err = cmd.Run() + + c.Assert(err, IsNil, Commentf("%s", buf.Bytes())) +} + func assertStorageStatus( c *C, r *Repository, treesCount, blobCount, commitCount int, head plumbing.Hash, diff --git a/vendor/gopkg.in/src-d/go-git.v4/worktree_test.go b/vendor/gopkg.in/src-d/go-git.v4/worktree_test.go index df191b0a0..c714011c0 100644 --- a/vendor/gopkg.in/src-d/go-git.v4/worktree_test.go +++ b/vendor/gopkg.in/src-d/go-git.v4/worktree_test.go @@ -1591,6 +1591,10 @@ func (s *WorktreeSuite) TestClean(c *C) { c.Assert(len(status), Equals, 1) + fi, err := fs.Lstat("pkgA") + c.Assert(err, IsNil) + c.Assert(fi.IsDir(), Equals, true) + // Clean with Dir: true. err = wt.Clean(&CleanOptions{Dir: true}) c.Assert(err, IsNil) @@ -1599,6 +1603,11 @@ func (s *WorktreeSuite) TestClean(c *C) { c.Assert(err, IsNil) c.Assert(len(status), Equals, 0) + + // An empty dir should be deleted, as well. + _, err = fs.Lstat("pkgA") + c.Assert(err, ErrorMatches, ".*(no such file or directory.*|.*file does not exist)*.") + } func (s *WorktreeSuite) TestAlternatesRepo(c *C) { From b1c8628a3155f72b4082f70c7a0d3838613194af Mon Sep 17 00:00:00 2001 From: kuba-- Date: Mon, 10 Sep 2018 13:04:50 +0200 Subject: [PATCH 2/3] One cache for repository pool. Signed-off-by: kuba-- --- cmd/gitbase/command/server.go | 28 ++++++++++--------- common_test.go | 3 +- fs_error_test.go | 19 +++++++------ integration_test.go | 7 +++-- internal/function/uast_test.go | 3 +- packfiles.go | 11 ++------ packfiles_test.go | 5 ++-- repositories_test.go | 5 ++-- repository_pool.go | 50 ++++++++++++++++++++-------------- repository_pool_test.go | 39 +++++++++++++------------- squash_iterator_test.go | 7 +++-- 11 files changed, 96 insertions(+), 81 deletions(-) diff --git a/cmd/gitbase/command/server.go b/cmd/gitbase/command/server.go index 15265b7d2..3322dfe4a 100644 --- a/cmd/gitbase/command/server.go +++ b/cmd/gitbase/command/server.go @@ -15,6 +15,7 @@ import ( gopilosa "github.com/pilosa/go-pilosa" "github.com/sirupsen/logrus" "github.com/uber/jaeger-client-go/config" + "gopkg.in/src-d/go-git.v4/plumbing/cache" sqle "gopkg.in/src-d/go-mysql-server.v0" "gopkg.in/src-d/go-mysql-server.v0/server" "gopkg.in/src-d/go-mysql-server.v0/sql" @@ -39,18 +40,19 @@ type Server struct { pool *gitbase.RepositoryPool name string - Version string // Version of the application. - Directories []string `short:"d" long:"directories" description:"Path where the git repositories are located (standard and siva), multiple directories can be defined. Accepts globs."` - Depth int `long:"depth" default:"1000" description:"load repositories looking at less than nested subdirectories."` - Host string `long:"host" default:"localhost" description:"Host where the server is going to listen"` - Port int `short:"p" long:"port" default:"3306" description:"Port where the server is going to listen"` - User string `short:"u" long:"user" default:"root" description:"User name used for connection"` - Password string `short:"P" long:"password" default:"" description:"Password used for connection"` - PilosaURL string `long:"pilosa" default:"http://localhost:10101" description:"URL to your pilosa server" env:"PILOSA_ENDPOINT"` - IndexDir string `short:"i" long:"index" default:"/var/lib/gitbase/index" description:"Directory where the gitbase indexes information will be persisted." env:"GITBASE_INDEX_DIR"` - DisableSquash bool `long:"no-squash" description:"Disables the table squashing."` - TraceEnabled bool `long:"trace" env:"GITBASE_TRACE" description:"Enables jaeger tracing"` - ReadOnly bool `short:"r" long:"readonly" description:"Only allow read queries. This disables creating and deleting indexes as well." env:"GITBASE_READONLY"` + Version string // Version of the application. + Directories []string `short:"d" long:"directories" description:"Path where the git repositories are located (standard and siva), multiple directories can be defined. Accepts globs."` + Depth int `long:"depth" default:"1000" description:"load repositories looking at less than nested subdirectories."` + Host string `long:"host" default:"localhost" description:"Host where the server is going to listen."` + Port int `short:"p" long:"port" default:"3306" description:"Port where the server is going to listen."` + User string `short:"u" long:"user" default:"root" description:"User name used for connection."` + Password string `short:"P" long:"password" default:"" description:"Password used for connection."` + PilosaURL string `long:"pilosa" default:"http://localhost:10101" description:"URL to your pilosa server." env:"PILOSA_ENDPOINT"` + IndexDir string `short:"i" long:"index" default:"/var/lib/gitbase/index" description:"Directory where the gitbase indexes information will be persisted." env:"GITBASE_INDEX_DIR"` + CacheSize cache.FileSize `long:"cache" default:"536870912" description:"Object cache size" env:"GITBASE_CACHE_SIZE"` + DisableSquash bool `long:"no-squash" description:"Disables the table squashing."` + TraceEnabled bool `long:"trace" env:"GITBASE_TRACE" description:"Enables jaeger tracing"` + ReadOnly bool `short:"r" long:"readonly" description:"Only allow read queries. This disables creating and deleting indexes as well." env:"GITBASE_READONLY"` SkipGitErrors bool // SkipGitErrors disables failing when Git errors are found. DisableGit bool `long:"no-git" description:"disable the load of git standard repositories."` @@ -159,7 +161,7 @@ func (c *Server) buildDatabase() error { c.engine = NewDatabaseEngine(c.ReadOnly, c.Version) } - c.pool = gitbase.NewRepositoryPool() + c.pool = gitbase.NewRepositoryPool(c.CacheSize) if err := c.addDirectories(); err != nil { return err diff --git a/common_test.go b/common_test.go index 3218997eb..98392c316 100644 --- a/common_test.go +++ b/common_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" fixtures "gopkg.in/src-d/go-git-fixtures.v3" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-mysql-server.v0/sql" ) @@ -35,7 +36,7 @@ func buildSession(t *testing.T, repos fixtures.Fixtures, require.NoError(fixtures.Init()) - pool := NewRepositoryPool() + pool := NewRepositoryPool(cache.DefaultMaxSize) for _, fixture := range repos { path := fixture.Worktree().Root() ok, err := IsGitRepo(path) diff --git a/fs_error_test.go b/fs_error_test.go index 403344dc9..e332e3325 100644 --- a/fs_error_test.go +++ b/fs_error_test.go @@ -13,6 +13,7 @@ import ( billy "gopkg.in/src-d/go-billy.v4" fixtures "gopkg.in/src-d/go-git-fixtures.v3" git "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem" "gopkg.in/src-d/go-mysql-server.v0/sql" ) @@ -56,7 +57,7 @@ func setupErrorRepos(t *testing.T) (*sql.Context, CleanupFunc) { fixture := fixtures.ByTag("worktree").One() baseFS := fixture.Worktree() - pool := NewRepositoryPool() + pool := NewRepositoryPool(cache.DefaultMaxSize) fs, err := brokenFS(brokenPackfile, baseFS) require.NoError(err) @@ -118,12 +119,13 @@ func testTable(t *testing.T, tableName string, number int) { } type billyRepository struct { - id string - fs billy.Filesystem + id string + fs billy.Filesystem + cache cache.Object } func billyRepo(id string, fs billy.Filesystem) repository { - return &billyRepository{id, fs} + return &billyRepository{id, fs, cache.NewObjectLRUDefault()} } func (r *billyRepository) ID() string { @@ -131,10 +133,7 @@ func (r *billyRepository) ID() string { } func (r *billyRepository) Repo() (*Repository, error) { - storage, err := filesystem.NewStorage(r.fs) - if err != nil { - return nil, err - } + storage := filesystem.NewStorage(r.fs, r.cache) repo, err := git.Open(storage, r.fs) if err != nil { @@ -152,6 +151,10 @@ func (r *billyRepository) Path() string { return r.id } +func (r *billyRepository) Cache() cache.Object { + return r.cache +} + type brokenType uint64 const ( diff --git a/integration_test.go b/integration_test.go index 55747085f..ef220f227 100644 --- a/integration_test.go +++ b/integration_test.go @@ -16,6 +16,7 @@ import ( "github.com/src-d/gitbase/internal/function" "github.com/stretchr/testify/require" fixtures "gopkg.in/src-d/go-git-fixtures.v3" + "gopkg.in/src-d/go-git.v4/plumbing/cache" sqle "gopkg.in/src-d/go-mysql-server.v0" "gopkg.in/src-d/go-mysql-server.v0/sql" "gopkg.in/src-d/go-mysql-server.v0/sql/analyzer" @@ -33,7 +34,7 @@ func TestIntegration(t *testing.T) { path := fixtures.ByTag("worktree").One().Worktree().Root() - pool := gitbase.NewRepositoryPool() + pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize) require.NoError(t, pool.AddGitWithID("worktree", path)) testCases := []struct { @@ -443,7 +444,7 @@ func TestMissingHeadRefs(t *testing.T) { "_testdata", ) - pool := gitbase.NewRepositoryPool() + pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize) require.NoError( filepath.Walk(path, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -829,7 +830,7 @@ func setup(t testing.TB) (*sqle.Engine, *gitbase.RepositoryPool, func()) { require.NoError(t, fixtures.Clean()) } - pool := gitbase.NewRepositoryPool() + pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize) for _, f := range fixtures.ByTag("worktree") { pool.AddGitWithID("worktree", f.Worktree().Root()) } diff --git a/internal/function/uast_test.go b/internal/function/uast_test.go index ef54afc89..5d6fbc0a7 100644 --- a/internal/function/uast_test.go +++ b/internal/function/uast_test.go @@ -11,6 +11,7 @@ import ( "gopkg.in/bblfsh/sdk.v1/protocol" "gopkg.in/bblfsh/sdk.v1/uast" fixtures "gopkg.in/src-d/go-git-fixtures.v3" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-mysql-server.v0/sql" "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) @@ -418,7 +419,7 @@ func setup(t *testing.T) (*sql.Context, func()) { t.Helper() require.NoError(t, fixtures.Init()) - pool := gitbase.NewRepositoryPool() + pool := gitbase.NewRepositoryPool(cache.DefaultMaxSize) for _, f := range fixtures.ByTag("worktree") { pool.AddGit(f.Worktree().Root()) } diff --git a/packfiles.go b/packfiles.go index 965810444..db73dff2a 100644 --- a/packfiles.go +++ b/packfiles.go @@ -166,10 +166,7 @@ func getUnpackedObject(repo repository, hash plumbing.Hash) (o object.Object, er defer ioutil.CheckClose(f, &err) - storage, err := filesystem.NewStorage(fs) - if err != nil { - return nil, err - } + storage := filesystem.NewStorage(fs, repo.Cache()) obj := storage.NewEncodedObject() r, err := objfile.NewReader(f) @@ -224,11 +221,7 @@ func newRepoObjectDecoder( return nil, err } - storage, err := filesystem.NewStorage(fs) - if err != nil { - _ = packf.Close() - return nil, err - } + storage := filesystem.NewStorage(fs, repo.Cache()) idx, err := openPackfileIndex(dot, hash) if err != nil { diff --git a/packfiles_test.go b/packfiles_test.go index cb20e63d5..5b2cbec35 100644 --- a/packfiles_test.go +++ b/packfiles_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/cache" ) var testSivaFilePath = filepath.Join("_testdata", "fff7062de8474d10a67d417ccea87ba6f58ca81d.siva") @@ -13,7 +14,7 @@ var testSivaFilePath = filepath.Join("_testdata", "fff7062de8474d10a67d417ccea87 func TestRepositoryPackfiles(t *testing.T) { require := require.New(t) - fs, packfiles, err := repositoryPackfiles(sivaRepo("siva", testSivaFilePath)) + fs, packfiles, err := repositoryPackfiles(sivaRepo("siva", testSivaFilePath, cache.NewObjectLRUDefault())) require.NoError(err) require.Equal([]plumbing.Hash{ @@ -24,7 +25,7 @@ func TestRepositoryPackfiles(t *testing.T) { } func TestRepositoryIndex(t *testing.T) { - idx, err := newRepositoryIndex(sivaRepo("siva", testSivaFilePath)) + idx, err := newRepositoryIndex(sivaRepo("siva", testSivaFilePath, cache.NewObjectLRUDefault())) require.NoError(t, err) testCases := []struct { diff --git a/repositories_test.go b/repositories_test.go index e75d6a7dc..ff0adacd1 100644 --- a/repositories_test.go +++ b/repositories_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stretchr/testify/require" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-mysql-server.v0/sql" "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) @@ -36,10 +37,10 @@ func TestRepositoriesTable_RowIter(t *testing.T) { "seven", "eight", "nine", } - pool := NewRepositoryPool() + pool := NewRepositoryPool(cache.DefaultMaxSize) for _, id := range repoIDs { - pool.Add(gitRepo(id, "")) + pool.Add(gitRepo(id, "", pool.cache)) } session := NewSession(pool) diff --git a/repository_pool.go b/repository_pool.go index d0c13b523..7c31a361f 100644 --- a/repository_pool.go +++ b/repository_pool.go @@ -13,6 +13,7 @@ import ( "gopkg.in/src-d/go-billy.v4/osfs" errors "gopkg.in/src-d/go-errors.v1" "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/storage/filesystem" "gopkg.in/src-d/go-mysql-server.v0/sql" ) @@ -50,7 +51,7 @@ func NewRepositoryFromPath(id, path string) (*Repository, error) { // NewSivaRepositoryFromPath creates and initializes a new Repository structure // and initializes a go-git repository backed by a siva file. -func NewSivaRepositoryFromPath(id, path string) (*Repository, error) { +func NewSivaRepositoryFromPath(id, path string, cache cache.Object) (*Repository, error) { localfs := osfs.New(filepath.Dir(path)) tmpDir, err := ioutil.TempDir(os.TempDir(), "gitbase-siva") @@ -65,11 +66,7 @@ func NewSivaRepositoryFromPath(id, path string) (*Repository, error) { return nil, err } - sto, err := filesystem.NewStorage(fs) - if err != nil { - return nil, err - } - + sto := filesystem.NewStorage(fs, cache) repo, err := git.Open(sto, nil) if err != nil { return nil, err @@ -83,15 +80,17 @@ type repository interface { Repo() (*Repository, error) FS() (billy.Filesystem, error) Path() string + Cache() cache.Object } type gitRepository struct { - id string - path string + id string + path string + cache cache.Object } -func gitRepo(id, path string) repository { - return &gitRepository{id, path} +func gitRepo(id, path string, cache cache.Object) repository { + return &gitRepository{id, path, cache} } func (r *gitRepository) ID() string { @@ -110,13 +109,18 @@ func (r *gitRepository) Path() string { return r.path } +func (r *gitRepository) Cache() cache.Object { + return r.cache +} + type sivaRepository struct { - id string - path string + id string + path string + cache cache.Object } -func sivaRepo(id, path string) repository { - return &sivaRepository{id, path} +func sivaRepo(id, path string, cache cache.Object) repository { + return &sivaRepository{id, path, cache} } func (r *sivaRepository) ID() string { @@ -124,7 +128,7 @@ func (r *sivaRepository) ID() string { } func (r *sivaRepository) Repo() (*Repository, error) { - return NewSivaRepositoryFromPath(r.id, r.path) + return NewSivaRepositoryFromPath(r.id, r.path, r.cache) } func (r *sivaRepository) FS() (billy.Filesystem, error) { @@ -144,17 +148,23 @@ func (r *sivaRepository) Path() string { return r.path } +func (r *sivaRepository) Cache() cache.Object { + return r.cache +} + // RepositoryPool holds a pool git repository paths and // functionality to open and iterate them. type RepositoryPool struct { repositories map[string]repository idOrder []string + cache cache.Object } -// NewRepositoryPool initializes a new RepositoryPool -func NewRepositoryPool() *RepositoryPool { +// NewRepositoryPool initializes a new RepositoryPool with LRU cache. +func NewRepositoryPool(maxCacheSize cache.FileSize) *RepositoryPool { return &RepositoryPool{ repositories: make(map[string]repository), + cache: cache.NewObjectLRU(maxCacheSize), } } @@ -178,17 +188,17 @@ func (p *RepositoryPool) AddGit(path string) error { // AddGitWithID adds a git repository to the pool. ID should be specified. func (p *RepositoryPool) AddGitWithID(id, path string) error { - return p.Add(gitRepo(id, path)) + return p.Add(gitRepo(id, path, p.cache)) } // AddSivaFile adds a siva file to the pool. It also sets its path as ID. func (p *RepositoryPool) AddSivaFile(path string) error { - return p.Add(sivaRepo(path, path)) + return p.Add(sivaRepo(path, path, p.cache)) } // AddSivaFileWithID adds a siva file to the pool. ID should be specified. func (p *RepositoryPool) AddSivaFileWithID(id, path string) error { - return p.Add(sivaRepo(id, path)) + return p.Add(sivaRepo(id, path, p.cache)) } // GetPos retrieves a repository at a given position. If the position is diff --git a/repository_pool_test.go b/repository_pool_test.go index 5d5588177..74fdf8362 100644 --- a/repository_pool_test.go +++ b/repository_pool_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "gopkg.in/src-d/go-git-fixtures.v3" "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-git.v4/plumbing/object" "gopkg.in/src-d/go-mysql-server.v0/sql" ) @@ -36,7 +37,7 @@ func TestRepository(t *testing.T) { func TestRepositoryPoolBasic(t *testing.T) { require := require.New(t) - pool := NewRepositoryPool() + pool := NewRepositoryPool(cache.DefaultMaxSize) repo, err := pool.GetPos(0) require.Nil(repo) @@ -46,7 +47,7 @@ func TestRepositoryPoolBasic(t *testing.T) { require.Nil(repo) require.EqualError(err, ErrPoolRepoNotFound.New("foo").Error()) - pool.Add(gitRepo("0", "/directory/should/not/exist")) + pool.Add(gitRepo("0", "/directory/should/not/exist", pool.cache)) repo, err = pool.GetPos(0) require.Nil(repo) require.EqualError(err, git.ErrRepositoryNotExists.Error()) @@ -56,7 +57,7 @@ func TestRepositoryPoolBasic(t *testing.T) { path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() - err = pool.Add(gitRepo("1", path)) + err = pool.Add(gitRepo("1", path, pool.cache)) require.NoError(err) repo, err = pool.GetPos(1) @@ -69,7 +70,7 @@ func TestRepositoryPoolBasic(t *testing.T) { require.Equal("1", repo.ID) require.NotNil(repo.Repo) - err = pool.Add(gitRepo("1", path)) + err = pool.Add(gitRepo("1", path, pool.cache)) require.Error(err) require.True(errRepoAlreadyRegistered.Is(err)) @@ -85,7 +86,7 @@ func TestRepositoryPoolGit(t *testing.T) { path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() - pool := NewRepositoryPool() + pool := NewRepositoryPool(cache.DefaultMaxSize) require.NoError(pool.AddGit(path)) @@ -118,9 +119,9 @@ func TestRepositoryPoolIterator(t *testing.T) { path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() - pool := NewRepositoryPool() - pool.Add(gitRepo("0", path)) - pool.Add(gitRepo("1", path)) + pool := NewRepositoryPool(cache.DefaultMaxSize) + pool.Add(gitRepo("0", path, pool.cache)) + pool.Add(gitRepo("1", path, pool.cache)) iter, err := pool.RepoIter() require.NoError(err) @@ -204,13 +205,13 @@ func TestRepositoryRowIterator(t *testing.T) { path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() - pool := NewRepositoryPool() + pool := NewRepositoryPool(cache.DefaultMaxSize) session := NewSession(pool) ctx := sql.NewContext(context.TODO(), sql.WithSession(session)) max := 64 for i := 0; i < max; i++ { - pool.Add(gitRepo(strconv.Itoa(i), path)) + pool.Add(gitRepo(strconv.Itoa(i), path, pool.cache)) } testRepoIter(max, require, ctx) @@ -235,7 +236,7 @@ func TestRepositoryPoolSiva(t *testing.T) { expectedRepos := 3 - pool := NewRepositoryPool() + pool := NewRepositoryPool(cache.DefaultMaxSize) path := filepath.Join( os.Getenv("GOPATH"), "src", "github.com", "src-d", "gitbase", @@ -344,16 +345,16 @@ func testCaseRepositoryErrorIter( func TestRepositoryErrorIter(t *testing.T) { path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() - pool := NewRepositoryPool() - pool.Add(gitRepo("one", path)) + pool := NewRepositoryPool(cache.DefaultMaxSize) + pool.Add(gitRepo("one", path, pool.cache)) iter := &testErrorIter{} testCaseRepositoryErrorIter(t, pool, iter, errIter, false) } func TestRepositoryErrorBadRepository(t *testing.T) { - pool := NewRepositoryPool() - pool.Add(gitRepo("one", "badpath")) + pool := NewRepositoryPool(cache.DefaultMaxSize) + pool.Add(gitRepo("one", "badpath", pool.cache)) iter := &testErrorIter{} @@ -381,8 +382,8 @@ func TestRepositoryErrorBadRepository(t *testing.T) { func TestRepositoryErrorBadRow(t *testing.T) { path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() - pool := NewRepositoryPool() - pool.Add(gitRepo("one", path)) + pool := NewRepositoryPool(cache.DefaultMaxSize) + pool.Add(gitRepo("one", path, pool.cache)) iter := &testErrorIter{} @@ -416,8 +417,8 @@ func TestRepositoryErrorBadRow(t *testing.T) { func TestRepositoryIteratorOrder(t *testing.T) { path := fixtures.Basic().ByTag("worktree").One().Worktree().Root() - pool := NewRepositoryPool() - pool.Add(gitRepo("one", path)) + pool := NewRepositoryPool(cache.DefaultMaxSize) + pool.Add(gitRepo("one", path, pool.cache)) timeout, cancel := context.WithTimeout(context.Background(), 5*time.Second) ctx := sql.NewContext(timeout, diff --git a/squash_iterator_test.go b/squash_iterator_test.go index 19ccb91c8..0b0af7192 100644 --- a/squash_iterator_test.go +++ b/squash_iterator_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/require" fixtures "gopkg.in/src-d/go-git-fixtures.v3" + "gopkg.in/src-d/go-git.v4/plumbing/cache" "gopkg.in/src-d/go-mysql-server.v0/sql" "gopkg.in/src-d/go-mysql-server.v0/sql/expression" ) @@ -720,10 +721,10 @@ func setupIter(t *testing.T) (*sql.Context, func()) { func setupIterWithErrors(t *testing.T, badRepo bool, skipErrors bool) (*sql.Context, func()) { require.NoError(t, fixtures.Init()) - pool := NewRepositoryPool() + pool := NewRepositoryPool(cache.DefaultMaxSize) if badRepo { // TODO: add repo with errors - pool.Add(gitRepo("bad_repo", "bad_path")) + pool.Add(gitRepo("bad_repo", "bad_path", pool.cache)) } for _, f := range fixtures.ByTag("worktree") { @@ -932,7 +933,7 @@ func setupWithIndex( func TestRefsIterSiva(t *testing.T) { path := filepath.Join("_testdata", "05893125684f2d3943cd84a7ab2b75e53668fba1.siva") - pool := NewRepositoryPool() + pool := NewRepositoryPool(cache.DefaultMaxSize) require.NoError(t, pool.AddSivaFile(path)) session := NewSession(pool) From 54c3ba0c96152a638c972564ade4c6bb35d6f304 Mon Sep 17 00:00:00 2001 From: kuba-- Date: Thu, 13 Sep 2018 13:22:24 +0200 Subject: [PATCH 3/3] cache in MB Signed-off-by: kuba-- --- cmd/gitbase/command/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/gitbase/command/server.go b/cmd/gitbase/command/server.go index 3322dfe4a..4287216cf 100644 --- a/cmd/gitbase/command/server.go +++ b/cmd/gitbase/command/server.go @@ -49,7 +49,7 @@ type Server struct { Password string `short:"P" long:"password" default:"" description:"Password used for connection."` PilosaURL string `long:"pilosa" default:"http://localhost:10101" description:"URL to your pilosa server." env:"PILOSA_ENDPOINT"` IndexDir string `short:"i" long:"index" default:"/var/lib/gitbase/index" description:"Directory where the gitbase indexes information will be persisted." env:"GITBASE_INDEX_DIR"` - CacheSize cache.FileSize `long:"cache" default:"536870912" description:"Object cache size" env:"GITBASE_CACHE_SIZE"` + CacheSize cache.FileSize `long:"cache" default:"512" description:"Object cache size in megabytes" env:"GITBASE_CACHESIZE_MB"` DisableSquash bool `long:"no-squash" description:"Disables the table squashing."` TraceEnabled bool `long:"trace" env:"GITBASE_TRACE" description:"Enables jaeger tracing"` ReadOnly bool `short:"r" long:"readonly" description:"Only allow read queries. This disables creating and deleting indexes as well." env:"GITBASE_READONLY"` @@ -161,7 +161,7 @@ func (c *Server) buildDatabase() error { c.engine = NewDatabaseEngine(c.ReadOnly, c.Version) } - c.pool = gitbase.NewRepositoryPool(c.CacheSize) + c.pool = gitbase.NewRepositoryPool(c.CacheSize * cache.MiByte) if err := c.addDirectories(); err != nil { return err