Skip to content
This repository was archived by the owner on Sep 11, 2020. It is now read-only.

Commit cfff565

Browse files
committed
storage: keep a certain amount of pack files open by default
The MaxOpenDescriptors option provides a middle ground solution between keeping all pack files open (as offered by the KeepDescriptors option) and keeping none open. By default, 100 pack files can be kept open. Signed-off-by: Arran Walker <[email protected]>
1 parent def8f25 commit cfff565

File tree

5 files changed

+100
-18
lines changed

5 files changed

+100
-18
lines changed

repository.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,9 @@ func PlainInit(path string, isBare bool) (*Repository, error) {
224224
dot, _ = wt.Chroot(GitDirName)
225225
}
226226

227-
s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault())
227+
s := filesystem.NewStorageWithOptions(dot, cache.NewObjectLRUDefault(), filesystem.Options{
228+
MaxOpenDescriptors: 100,
229+
})
228230

229231
return Init(s, wt)
230232
}
@@ -252,7 +254,9 @@ func PlainOpenWithOptions(path string, o *PlainOpenOptions) (*Repository, error)
252254
return nil, err
253255
}
254256

255-
s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault())
257+
s := filesystem.NewStorageWithOptions(dot, cache.NewObjectLRUDefault(), filesystem.Options{
258+
MaxOpenDescriptors: 100,
259+
})
256260

257261
return Open(s, wt)
258262
}

storage/filesystem/dotgit/dotgit.go

Lines changed: 47 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ type Options struct {
6666
// KeepDescriptors makes the file descriptors to be reused but they will
6767
// need to be manually closed calling Close().
6868
KeepDescriptors bool
69+
// MaxOpenDescriptors is the max number of file descriptors to keep
70+
// open. If KeepDescriptors is true, all file descriptors will remain open.
71+
MaxOpenDescriptors int
6972
}
7073

7174
// The DotGit type represents a local git repository on disk. This
@@ -83,7 +86,9 @@ type DotGit struct {
8386
packList []plumbing.Hash
8487
packMap map[plumbing.Hash]struct{}
8588

86-
files map[string]billy.File
89+
fileList []plumbing.Hash
90+
fileListIdx int
91+
fileMap map[plumbing.Hash]billy.File
8792
}
8893

8994
// New returns a DotGit value ready to be used. The path argument must
@@ -132,16 +137,18 @@ func (d *DotGit) Initialize() error {
132137
// Close closes all opened files.
133138
func (d *DotGit) Close() error {
134139
var firstError error
135-
if d.files != nil {
136-
for _, f := range d.files {
140+
if d.fileMap != nil {
141+
for _, f := range d.fileMap {
137142
err := f.Close()
138143
if err != nil && firstError == nil {
139144
firstError = err
140145
continue
141146
}
142147
}
143148

144-
d.files = nil
149+
d.fileMap = nil
150+
d.fileList = nil
151+
d.fileListIdx = 0
145152
}
146153

147154
if firstError != nil {
@@ -245,8 +252,20 @@ func (d *DotGit) objectPackPath(hash plumbing.Hash, extension string) string {
245252
}
246253

247254
func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.File, error) {
248-
if d.files == nil {
249-
d.files = make(map[string]billy.File)
255+
if d.fileMap == nil {
256+
if d.options.MaxOpenDescriptors > 0 {
257+
d.fileList = make([]plumbing.Hash, d.options.MaxOpenDescriptors)
258+
d.fileMap = make(map[plumbing.Hash]billy.File, d.options.MaxOpenDescriptors)
259+
} else {
260+
d.fileMap = make(map[plumbing.Hash]billy.File)
261+
}
262+
}
263+
264+
if extension == "pack" {
265+
f, ok := d.fileMap[hash]
266+
if ok {
267+
return f, nil
268+
}
250269
}
251270

252271
err := d.hasPack(hash)
@@ -255,11 +274,6 @@ func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.Fil
255274
}
256275

257276
path := d.objectPackPath(hash, extension)
258-
f, ok := d.files[path]
259-
if ok {
260-
return f, nil
261-
}
262-
263277
pack, err := d.fs.Open(path)
264278
if err != nil {
265279
if os.IsNotExist(err) {
@@ -269,8 +283,28 @@ func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.Fil
269283
return nil, err
270284
}
271285

272-
if d.options.KeepDescriptors && extension == "pack" {
273-
d.files[path] = pack
286+
if extension == "pack" {
287+
if d.options.KeepDescriptors {
288+
d.fileMap[hash] = pack
289+
} else if d.options.MaxOpenDescriptors > 0 {
290+
if next := d.fileList[d.fileListIdx]; !next.IsZero() {
291+
open := d.fileMap[next]
292+
delete(d.fileMap, next)
293+
if open != nil {
294+
if err = open.Close(); err != nil {
295+
return nil, err
296+
}
297+
}
298+
}
299+
300+
d.fileList[d.fileListIdx] = hash
301+
d.fileMap[hash] = pack
302+
303+
d.fileListIdx++
304+
if d.fileListIdx >= len(d.fileList) {
305+
d.fileListIdx = 0
306+
}
307+
}
274308
}
275309

276310
return pack, nil

storage/filesystem/dotgit/dotgit_test.go

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -519,6 +519,46 @@ func (s *SuiteDotGit) TestObjectPackWithKeepDescriptors(c *C) {
519519

520520
}
521521

522+
func (s *SuiteDotGit) TestObjectPackWithMaxOpenDescriptors(c *C) {
523+
f := fixtures.ByTag("multi-packfile").One()
524+
fs := f.DotGit()
525+
dir := NewWithOptions(fs, Options{MaxOpenDescriptors: 1})
526+
527+
hashes, err := dir.ObjectPacks()
528+
c.Assert(hashes, HasLen, 2)
529+
530+
pack, err := dir.ObjectPack(hashes[0])
531+
c.Assert(err, IsNil)
532+
c.Assert(filepath.Ext(pack.Name()), Equals, ".pack")
533+
534+
// Move to an specific offset
535+
pack.Seek(42, os.SEEK_SET)
536+
537+
pack, err = dir.ObjectPack(hashes[0])
538+
c.Assert(err, IsNil)
539+
540+
// If the file is the same the offset should be the same
541+
offset, err := pack.Seek(0, os.SEEK_CUR)
542+
c.Assert(err, IsNil)
543+
c.Assert(offset, Equals, int64(42))
544+
545+
// Open second pack file, this should close the first
546+
pack2, err := dir.ObjectPack(hashes[1])
547+
c.Assert(err, IsNil)
548+
c.Assert(filepath.Ext(pack.Name()), Equals, ".pack")
549+
550+
// Move second pack file to a specific offset
551+
_, err = pack2.Seek(42, os.SEEK_SET)
552+
c.Assert(err, IsNil)
553+
554+
// Seeking in first pack file (now closed) should error
555+
_, err = pack.Seek(42, os.SEEK_SET)
556+
c.Assert(err, NotNil)
557+
558+
err = dir.Close()
559+
c.Assert(err, IsNil)
560+
}
561+
522562
func (s *SuiteDotGit) TestObjectPackIdx(c *C) {
523563
f := fixtures.Basic().ByTag(".git").One()
524564
fs := f.DotGit()

storage/filesystem/object.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -366,7 +366,7 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) (
366366
return nil, err
367367
}
368368

369-
if !s.options.KeepDescriptors {
369+
if !s.options.KeepDescriptors && s.options.MaxOpenDescriptors == 0 {
370370
defer ioutil.CheckClose(f, &err)
371371
}
372372

storage/filesystem/storage.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ type Options struct {
3131
// KeepDescriptors makes the file descriptors to be reused but they will
3232
// need to be manually closed calling Close().
3333
KeepDescriptors bool
34+
// MaxOpenDescriptors is the max number of file descriptors to keep
35+
// open. If KeepDescriptors is true, all file descriptors will remain open.
36+
MaxOpenDescriptors int
3437
}
3538

3639
// NewStorage returns a new Storage backed by a given `fs.Filesystem` and cache.
@@ -42,8 +45,9 @@ func NewStorage(fs billy.Filesystem, cache cache.Object) *Storage {
4245
// backed by a given `fs.Filesystem` and cache.
4346
func NewStorageWithOptions(fs billy.Filesystem, cache cache.Object, ops Options) *Storage {
4447
dirOps := dotgit.Options{
45-
ExclusiveAccess: ops.ExclusiveAccess,
46-
KeepDescriptors: ops.KeepDescriptors,
48+
ExclusiveAccess: ops.ExclusiveAccess,
49+
KeepDescriptors: ops.KeepDescriptors,
50+
MaxOpenDescriptors: ops.MaxOpenDescriptors,
4751
}
4852
dir := dotgit.NewWithOptions(fs, dirOps)
4953

0 commit comments

Comments
 (0)