Skip to content
This repository was archived by the owner on Sep 11, 2020. It is now read-only.

Commit 42bbcaf

Browse files
committed
filesystem: ObjectStorage, MaxOpenDescriptors option
The MaxOpenDescriptors option provides a middle ground solution between keeping all packfiles open (as offered by the KeepDescriptors option) and keeping none open. Signed-off-by: Arran Walker <[email protected]>
1 parent e5268e9 commit 42bbcaf

File tree

5 files changed

+127
-49
lines changed

5 files changed

+127
-49
lines changed

plumbing/format/packfile/packfile.go

+5
Original file line numberDiff line numberDiff line change
@@ -414,6 +414,11 @@ func (p *Packfile) ID() (plumbing.Hash, error) {
414414
return hash, nil
415415
}
416416

417+
// Scanner returns the packfile's Scanner
418+
func (p *Packfile) Scanner() *Scanner {
419+
return p.s
420+
}
421+
417422
// Close the packfile and its resources.
418423
func (p *Packfile) Close() error {
419424
closer, ok := p.file.(io.Closer)

storage/filesystem/dotgit/dotgit.go

+11-9
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ type DotGit struct {
8383
packList []plumbing.Hash
8484
packMap map[plumbing.Hash]struct{}
8585

86-
files map[string]billy.File
86+
files map[plumbing.Hash]billy.File
8787
}
8888

8989
// New returns a DotGit value ready to be used. The path argument must
@@ -245,8 +245,15 @@ func (d *DotGit) objectPackPath(hash plumbing.Hash, extension string) string {
245245
}
246246

247247
func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.File, error) {
248-
if d.files == nil {
249-
d.files = make(map[string]billy.File)
248+
if d.options.KeepDescriptors && extension == "pack" {
249+
if d.files == nil {
250+
d.files = make(map[plumbing.Hash]billy.File)
251+
}
252+
253+
f, ok := d.files[hash]
254+
if ok {
255+
return f, nil
256+
}
250257
}
251258

252259
err := d.hasPack(hash)
@@ -255,11 +262,6 @@ func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.Fil
255262
}
256263

257264
path := d.objectPackPath(hash, extension)
258-
f, ok := d.files[path]
259-
if ok {
260-
return f, nil
261-
}
262-
263265
pack, err := d.fs.Open(path)
264266
if err != nil {
265267
if os.IsNotExist(err) {
@@ -270,7 +272,7 @@ func (d *DotGit) objectPackOpen(hash plumbing.Hash, extension string) (billy.Fil
270272
}
271273

272274
if d.options.KeepDescriptors && extension == "pack" {
273-
d.files[path] = pack
275+
d.files[hash] = pack
274276
}
275277

276278
return pack, nil

storage/filesystem/object.go

+90-39
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ type ObjectStorage struct {
2626

2727
dir *dotgit.DotGit
2828
index map[plumbing.Hash]idxfile.Index
29+
30+
packList []plumbing.Hash
31+
packListIdx int
32+
packfiles map[plumbing.Hash]*packfile.Packfile
2933
}
3034

3135
// NewObjectStorage creates a new ObjectStorage with the given .git directory and cache.
@@ -187,6 +191,57 @@ func (s *ObjectStorage) encodedObjectSizeFromUnpacked(h plumbing.Hash) (
187191
return size, err
188192
}
189193

194+
func (s *ObjectStorage) packfile(idx idxfile.Index, pack plumbing.Hash) (*packfile.Packfile, error) {
195+
if s.packfiles == nil {
196+
if s.options.MaxOpenDescriptors > 0 {
197+
s.packList = make([]plumbing.Hash, s.options.MaxOpenDescriptors)
198+
s.packfiles = make(map[plumbing.Hash]*packfile.Packfile, s.options.MaxOpenDescriptors)
199+
} else {
200+
s.packfiles = make(map[plumbing.Hash]*packfile.Packfile)
201+
}
202+
}
203+
204+
if p, ok := s.packfiles[pack]; ok {
205+
return p, nil
206+
}
207+
208+
f, err := s.dir.ObjectPack(pack)
209+
if err != nil {
210+
return nil, err
211+
}
212+
213+
var p *packfile.Packfile
214+
if s.objectCache != nil {
215+
p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.objectCache)
216+
} else {
217+
p = packfile.NewPackfile(idx, s.dir.Fs(), f)
218+
}
219+
220+
if s.options.KeepDescriptors {
221+
s.packfiles[pack] = p
222+
} else if s.options.MaxOpenDescriptors > 0 {
223+
if next := s.packList[s.packListIdx]; !next.IsZero() {
224+
open := s.packfiles[next]
225+
delete(s.packfiles, next)
226+
if open != nil {
227+
if err = open.Close(); err != nil {
228+
return nil, err
229+
}
230+
}
231+
}
232+
233+
s.packList[s.packListIdx] = pack
234+
s.packfiles[pack] = p
235+
236+
s.packListIdx++
237+
if s.packListIdx >= len(s.packList) {
238+
s.packListIdx = 0
239+
}
240+
}
241+
242+
return p, err
243+
}
244+
190245
func (s *ObjectStorage) encodedObjectSizeFromPackfile(h plumbing.Hash) (
191246
size int64, err error) {
192247
if err := s.requireIndex(); err != nil {
@@ -198,12 +253,6 @@ func (s *ObjectStorage) encodedObjectSizeFromPackfile(h plumbing.Hash) (
198253
return 0, plumbing.ErrObjectNotFound
199254
}
200255

201-
f, err := s.dir.ObjectPack(pack)
202-
if err != nil {
203-
return 0, err
204-
}
205-
defer ioutil.CheckClose(f, &err)
206-
207256
idx := s.index[pack]
208257
hash, err := idx.FindHash(offset)
209258
if err == nil {
@@ -215,11 +264,13 @@ func (s *ObjectStorage) encodedObjectSizeFromPackfile(h plumbing.Hash) (
215264
return 0, err
216265
}
217266

218-
var p *packfile.Packfile
219-
if s.objectCache != nil {
220-
p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.objectCache)
221-
} else {
222-
p = packfile.NewPackfile(idx, s.dir.Fs(), f)
267+
p, err := s.packfile(idx, pack)
268+
if err != nil {
269+
return 0, err
270+
}
271+
272+
if !s.options.KeepDescriptors && s.options.MaxOpenDescriptors == 0 {
273+
defer ioutil.CheckClose(p, &err)
223274
}
224275

225276
return p.GetSizeByOffset(offset)
@@ -361,29 +412,28 @@ func (s *ObjectStorage) getFromPackfile(h plumbing.Hash, canBeDelta bool) (
361412
return nil, plumbing.ErrObjectNotFound
362413
}
363414

364-
f, err := s.dir.ObjectPack(pack)
415+
idx := s.index[pack]
416+
p, err := s.packfile(idx, pack)
365417
if err != nil {
366418
return nil, err
367419
}
368420

369-
if !s.options.KeepDescriptors {
370-
defer ioutil.CheckClose(f, &err)
421+
if !s.options.KeepDescriptors && s.options.MaxOpenDescriptors == 0 {
422+
defer ioutil.CheckClose(p, &err)
371423
}
372424

373-
idx := s.index[pack]
374425
if canBeDelta {
375-
return s.decodeDeltaObjectAt(f, idx, offset, hash)
426+
return s.decodeDeltaObjectAt(p, offset, hash)
376427
}
377428

378-
return s.decodeObjectAt(f, idx, offset)
429+
return s.decodeObjectAt(p, offset)
379430
}
380431

381432
func (s *ObjectStorage) decodeObjectAt(
382-
f billy.File,
383-
idx idxfile.Index,
433+
p *packfile.Packfile,
384434
offset int64,
385435
) (plumbing.EncodedObject, error) {
386-
hash, err := idx.FindHash(offset)
436+
hash, err := p.FindHash(offset)
387437
if err == nil {
388438
obj, ok := s.objectCache.Get(hash)
389439
if ok {
@@ -395,28 +445,16 @@ func (s *ObjectStorage) decodeObjectAt(
395445
return nil, err
396446
}
397447

398-
var p *packfile.Packfile
399-
if s.objectCache != nil {
400-
p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.objectCache)
401-
} else {
402-
p = packfile.NewPackfile(idx, s.dir.Fs(), f)
403-
}
404-
405448
return p.GetByOffset(offset)
406449
}
407450

408451
func (s *ObjectStorage) decodeDeltaObjectAt(
409-
f billy.File,
410-
idx idxfile.Index,
452+
p *packfile.Packfile,
411453
offset int64,
412454
hash plumbing.Hash,
413455
) (plumbing.EncodedObject, error) {
414-
if _, err := f.Seek(0, io.SeekStart); err != nil {
415-
return nil, err
416-
}
417-
418-
p := packfile.NewScanner(f)
419-
header, err := p.SeekObjectHeader(offset)
456+
scan := p.Scanner()
457+
header, err := scan.SeekObjectHeader(offset)
420458
if err != nil {
421459
return nil, err
422460
}
@@ -429,12 +467,12 @@ func (s *ObjectStorage) decodeDeltaObjectAt(
429467
case plumbing.REFDeltaObject:
430468
base = header.Reference
431469
case plumbing.OFSDeltaObject:
432-
base, err = idx.FindHash(header.OffsetReference)
470+
base, err = p.FindHash(header.OffsetReference)
433471
if err != nil {
434472
return nil, err
435473
}
436474
default:
437-
return s.decodeObjectAt(f, idx, offset)
475+
return s.decodeObjectAt(p, offset)
438476
}
439477

440478
obj := &plumbing.MemoryObject{}
@@ -444,7 +482,7 @@ func (s *ObjectStorage) decodeDeltaObjectAt(
444482
return nil, err
445483
}
446484

447-
if _, _, err := p.NextObject(w); err != nil {
485+
if _, _, err := scan.NextObject(w); err != nil {
448486
return nil, err
449487
}
450488

@@ -515,7 +553,20 @@ func (s *ObjectStorage) buildPackfileIters(
515553

516554
// Close closes all opened files.
517555
func (s *ObjectStorage) Close() error {
518-
return s.dir.Close()
556+
var firstError error
557+
if s.options.KeepDescriptors || s.options.MaxOpenDescriptors > 0 {
558+
for _, packfile := range s.packfiles {
559+
err := packfile.Close()
560+
if firstError == nil && err != nil {
561+
firstError = err
562+
}
563+
}
564+
}
565+
566+
s.packfiles = nil
567+
s.dir.Close()
568+
569+
return firstError
519570
}
520571

521572
type lazyPackfilesIter struct {

storage/filesystem/object_test.go

+18
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,24 @@ func (s *FsSuite) TestGetFromPackfileKeepDescriptors(c *C) {
8686
})
8787
}
8888

89+
func (s *FsSuite) TestGetFromPackfileMaxOpenDescriptors(c *C) {
90+
fs := fixtures.ByTag(".git").ByTag("multi-packfile").One().DotGit()
91+
o := NewObjectStorageWithOptions(dotgit.New(fs), cache.NewObjectLRUDefault(), Options{MaxOpenDescriptors: 1})
92+
93+
expected := plumbing.NewHash("8d45a34641d73851e01d3754320b33bb5be3c4d3")
94+
obj, err := o.getFromPackfile(expected, false)
95+
c.Assert(err, IsNil)
96+
c.Assert(obj.Hash(), Equals, expected)
97+
98+
expected = plumbing.NewHash("e9cfa4c9ca160546efd7e8582ec77952a27b17db")
99+
obj, err = o.getFromPackfile(expected, false)
100+
c.Assert(err, IsNil)
101+
c.Assert(obj.Hash(), Equals, expected)
102+
103+
err = o.Close()
104+
c.Assert(err, IsNil)
105+
}
106+
89107
func (s *FsSuite) TestGetSizeOfObjectFile(c *C) {
90108
fs := fixtures.ByTag(".git").ByTag("unpacked").One().DotGit()
91109
o := NewObjectStorage(dotgit.New(fs), cache.NewObjectLRUDefault())

storage/filesystem/storage.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ type Options struct {
3131
// KeepDescriptors makes the file descriptors to be reused but they will
3232
// need to be manually closed calling Close().
3333
KeepDescriptors bool
34+
// MaxOpenDescriptors is the max number of file descriptors to keep
35+
// open. If KeepDescriptors is true, all file descriptors will remain open.
36+
MaxOpenDescriptors int
3437
}
3538

3639
// NewStorage returns a new Storage backed by a given `fs.Filesystem` and cache.
@@ -43,7 +46,6 @@ func NewStorage(fs billy.Filesystem, cache cache.Object) *Storage {
4346
func NewStorageWithOptions(fs billy.Filesystem, cache cache.Object, ops Options) *Storage {
4447
dirOps := dotgit.Options{
4548
ExclusiveAccess: ops.ExclusiveAccess,
46-
KeepDescriptors: ops.KeepDescriptors,
4749
}
4850
dir := dotgit.NewWithOptions(fs, dirOps)
4951

0 commit comments

Comments
 (0)