From 640029c62b6f52429ed868a782f41079f5b99909 Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Sun, 23 Sep 2018 13:06:09 +0200 Subject: [PATCH 01/12] Avoid unnecessary seek calls that are determining offsets that are already known or about to be thrown away anyway. Signed-off-by: Filip Navara --- plumbing/format/packfile/packfile.go | 32 ++++++++------------- plumbing/format/packfile/parser.go | 6 +--- plumbing/format/packfile/scanner.go | 42 ++++++++++++++++++++++++++-- storage/filesystem/object.go | 6 +--- 4 files changed, 53 insertions(+), 33 deletions(-) diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go index 2166e0aa2..a8bdf9c2f 100644 --- a/plumbing/format/packfile/packfile.go +++ b/plumbing/format/packfile/packfile.go @@ -79,15 +79,7 @@ func (p *Packfile) GetByOffset(o int64) (plumbing.EncodedObject, error) { } } - if _, err := p.s.SeekFromStart(o); err != nil { - if err == io.EOF || isInvalid(err) { - return nil, plumbing.ErrObjectNotFound - } - - return nil, err - } - - return p.nextObject() + return p.objectAtOffset(o) } // GetSizeByOffset retrieves the size of the encoded object from the @@ -108,6 +100,12 @@ func (p *Packfile) GetSizeByOffset(o int64) (size int64, err error) { return h.Length, nil } +func (p *Packfile) objectHeaderAtOffset(offset int64) (*ObjectHeader, error) { + h, err := p.s.SeekObjectHeader(offset) + p.s.pendingObject = nil + return h, err +} + func (p *Packfile) nextObjectHeader() (*ObjectHeader, error) { h, err := p.s.NextObjectHeader() p.s.pendingObject = nil @@ -154,11 +152,7 @@ func (p *Packfile) getObjectType(h *ObjectHeader) (typ plumbing.ObjectType, err if baseType, ok := p.offsetToType[offset]; ok { typ = baseType } else { - if _, err = p.s.SeekFromStart(offset); err != nil { - return - } - - h, err = p.nextObjectHeader() + h, err = p.objectHeaderAtOffset(offset) if err != nil { return } @@ -175,8 +169,8 @@ func (p *Packfile) getObjectType(h *ObjectHeader) (typ plumbing.ObjectType, err return } -func (p *Packfile) nextObject() (plumbing.EncodedObject, error) { - h, err := p.nextObjectHeader() +func (p *Packfile) objectAtOffset(offset int64) (plumbing.EncodedObject, error) { + h, err := p.objectHeaderAtOffset(offset) if err != nil { if err == io.EOF || isInvalid(err) { return nil, plumbing.ErrObjectNotFound @@ -233,11 +227,7 @@ func (p *Packfile) getObjectContent(offset int64) (io.ReadCloser, error) { } } - if _, err := p.s.SeekFromStart(offset); err != nil { - return nil, err - } - - h, err := p.nextObjectHeader() + h, err := p.objectHeaderAtOffset(offset) if err != nil { return nil, err } diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go index 5a62d63bb..71cbba983 100644 --- a/plumbing/format/packfile/parser.go +++ b/plumbing/format/packfile/parser.go @@ -398,11 +398,7 @@ func (p *Parser) readData(o *objectInfo) ([]byte, error) { return data, nil } - if _, err := p.scanner.SeekFromStart(o.Offset); err != nil { - return nil, err - } - - if _, err := p.scanner.NextObjectHeader(); err != nil { + if _, err := p.scanner.SeekObjectHeader(o.Offset); err != nil { return nil, err } diff --git a/plumbing/format/packfile/scanner.go b/plumbing/format/packfile/scanner.go index 6fc183b94..257bf853f 100644 --- a/plumbing/format/packfile/scanner.go +++ b/plumbing/format/packfile/scanner.go @@ -138,14 +138,52 @@ func (s *Scanner) readCount() (uint32, error) { return binary.ReadUint32(s.r) } +// SeekObjectHeader seeks to specified offset and returns the ObjectHeader +// for the next object in the reader +func (s *Scanner) SeekObjectHeader(offset int64) (*ObjectHeader, error) { + // if seeking we assume that you are not interested on the header + if s.version == 0 { + s.version = VersionSupported + } + + if _, err := s.r.Seek(offset, io.SeekStart); err != nil { + return nil, err + } + + h, err := s.nextObjectHeader(); + if err != nil { + return nil, err + } + + h.Offset = offset + return h, nil +} + // NextObjectHeader returns the ObjectHeader for the next object in the reader func (s *Scanner) NextObjectHeader() (*ObjectHeader, error) { - defer s.Flush() - if err := s.doPending(); err != nil { return nil, err } + offset, err := s.r.Seek(0, io.SeekCurrent); + if err != nil { + return nil, err + } + + h, err := s.nextObjectHeader(); + if err != nil { + return nil, err + } + + h.Offset = offset + return h, nil +} + +// nextObjectHeader returns the ObjectHeader for the next object in the reader +// without the Offset field +func (s *Scanner) nextObjectHeader() (*ObjectHeader, error) { + defer s.Flush() + s.crc.Reset() h := &ObjectHeader{} diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 57dcbb43f..6082c4a7d 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -400,11 +400,7 @@ func (s *ObjectStorage) decodeDeltaObjectAt( } p := packfile.NewScanner(f) - if _, err := p.SeekFromStart(offset); err != nil { - return nil, err - } - - header, err := p.NextObjectHeader() + header, err := p.SeekObjectHeader(offset); if err != nil { return nil, err } From a78415e21424eaddf7d07318922940054e8fd65f Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Sun, 23 Sep 2018 14:20:24 +0200 Subject: [PATCH 02/12] Fix bufferedSeeker.Seek for SeekCurrent and non-zero offset. Signed-off-by: Filip Navara --- plumbing/format/packfile/scanner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plumbing/format/packfile/scanner.go b/plumbing/format/packfile/scanner.go index 257bf853f..35017486a 100644 --- a/plumbing/format/packfile/scanner.go +++ b/plumbing/format/packfile/scanner.go @@ -423,7 +423,7 @@ type bufferedSeeker struct { } func (r *bufferedSeeker) Seek(offset int64, whence int) (int64, error) { - if whence == io.SeekCurrent { + if whence == io.SeekCurrent && offset == 0 { current, err := r.r.Seek(offset, whence) if err != nil { return current, err From b6061415fb7669d77d5bf28e33dbbaa44a07e3ac Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Sun, 23 Sep 2018 15:59:42 +0200 Subject: [PATCH 03/12] Add loose object cache. Signed-off-by: Filip Navara --- storage/filesystem/object.go | 21 +++++++++++++++------ storage/filesystem/storage.go | 6 +----- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 6082c4a7d..e49bf37a7 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -23,21 +23,24 @@ type ObjectStorage struct { // deltaBaseCache is an object cache uses to cache delta's bases when deltaBaseCache cache.Object + simpleObjectCache cache.Object + dir *dotgit.DotGit index map[plumbing.Hash]idxfile.Index } // NewObjectStorage creates a new ObjectStorage with the given .git directory and cache. -func NewObjectStorage(dir *dotgit.DotGit, cache cache.Object) *ObjectStorage { - return NewObjectStorageWithOptions(dir, cache, Options{}) +func NewObjectStorage(dir *dotgit.DotGit, deltaBaseCache cache.Object) *ObjectStorage { + return NewObjectStorageWithOptions(dir, deltaBaseCache, Options{}) } // NewObjectStorageWithOptions creates a new ObjectStorage with the given .git directory, cache and extra options -func NewObjectStorageWithOptions(dir *dotgit.DotGit, cache cache.Object, ops Options) *ObjectStorage { +func NewObjectStorageWithOptions(dir *dotgit.DotGit, deltaBaseCache cache.Object, ops Options) *ObjectStorage { return &ObjectStorage{ - options: ops, - deltaBaseCache: cache, - dir: dir, + options: ops, + deltaBaseCache: deltaBaseCache, + simpleObjectCache: cache.NewObjectLRU(cache.MiByte), + dir: dir, } } @@ -296,6 +299,10 @@ func (s *ObjectStorage) DeltaObject(t plumbing.ObjectType, } func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedObject, err error) { + if cacheObj, found := s.simpleObjectCache.Get(h); found { + return cacheObj, nil + } + f, err := s.dir.Object(h) if err != nil { if os.IsNotExist(err) { @@ -327,6 +334,8 @@ func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedOb return nil, err } + s.simpleObjectCache.Put(obj); + _, err = io.Copy(w, r) return obj, err } diff --git a/storage/filesystem/storage.go b/storage/filesystem/storage.go index 14a772abe..370f7bd34 100644 --- a/storage/filesystem/storage.go +++ b/storage/filesystem/storage.go @@ -51,11 +51,7 @@ func NewStorageWithOptions(fs billy.Filesystem, cache cache.Object, ops Options) fs: fs, dir: dir, - ObjectStorage: ObjectStorage{ - options: ops, - deltaBaseCache: cache, - dir: dir, - }, + ObjectStorage: *NewObjectStorageWithOptions(dir, cache, ops), ReferenceStorage: ReferenceStorage{dir: dir}, IndexStorage: IndexStorage{dir: dir}, ShallowStorage: ShallowStorage{dir: dir}, From 25d2f132e43bcaae01ceaba0a264a61ba8d31cf2 Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Sun, 23 Sep 2018 16:14:40 +0200 Subject: [PATCH 04/12] Search packfile indexes first if we already loaded them. Signed-off-by: Filip Navara --- storage/filesystem/object.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index e49bf37a7..2248699c1 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -244,9 +244,19 @@ func (s *ObjectStorage) EncodedObjectSize(h plumbing.Hash) ( // EncodedObject returns the object with the given hash, by searching for it in // the packfile and the git object directories. func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (plumbing.EncodedObject, error) { - obj, err := s.getFromUnpacked(h) - if err == plumbing.ErrObjectNotFound { + var obj plumbing.EncodedObject + var err error + + if s.index != nil { obj, err = s.getFromPackfile(h, false) + if err == plumbing.ErrObjectNotFound { + obj, err = s.getFromUnpacked(h) + } + } else { + obj, err = s.getFromUnpacked(h) + if err == plumbing.ErrObjectNotFound { + obj, err = s.getFromPackfile(h, false) + } } // If the error is still object not found, check if it's a shared object From 4a21431163cb42f605d35884f674304245e99370 Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Sun, 23 Sep 2018 18:45:08 +0200 Subject: [PATCH 05/12] Improve performace for reading many small objects from packfile by letting the I/O buffers be used better. Signed-off-by: Filip Navara --- plumbing/format/packfile/packfile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go index a8bdf9c2f..bd129787c 100644 --- a/plumbing/format/packfile/packfile.go +++ b/plumbing/format/packfile/packfile.go @@ -180,7 +180,7 @@ func (p *Packfile) objectAtOffset(offset int64) (plumbing.EncodedObject, error) // If we have no filesystem, we will return a MemoryObject instead // of an FSObject. - if p.fs == nil { + if p.fs == nil || h.Length <= 16 * 1024 { return p.getNextObject(h) } From 72e361d9714bd6450852b3976bd6b72d9e3ff326 Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Mon, 24 Sep 2018 10:38:30 +0200 Subject: [PATCH 06/12] Fix infinite recursion in FSObject. Signed-off-by: Filip Navara --- plumbing/format/packfile/fsobject.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plumbing/format/packfile/fsobject.go b/plumbing/format/packfile/fsobject.go index 330cb73c9..a268bce7e 100644 --- a/plumbing/format/packfile/fsobject.go +++ b/plumbing/format/packfile/fsobject.go @@ -48,7 +48,7 @@ func NewFSObject( // Reader implements the plumbing.EncodedObject interface. func (o *FSObject) Reader() (io.ReadCloser, error) { obj, ok := o.cache.Get(o.hash) - if ok { + if ok && obj != o { reader, err := obj.Reader() if err != nil { return nil, err From 5e5b9b331101386a9d604c7102ca7d7b80570324 Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Mon, 24 Sep 2018 11:15:06 +0200 Subject: [PATCH 07/12] Remove a cachePut that was adding FSObject to cache, which triggered out-of-memory errors. This code path was never previously triggered for filesystem-based storage. Signed-off-by: Filip Navara --- plumbing/format/packfile/packfile.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go index bd129787c..538a690d5 100644 --- a/plumbing/format/packfile/packfile.go +++ b/plumbing/format/packfile/packfile.go @@ -180,7 +180,7 @@ func (p *Packfile) objectAtOffset(offset int64) (plumbing.EncodedObject, error) // If we have no filesystem, we will return a MemoryObject instead // of an FSObject. - if p.fs == nil || h.Length <= 16 * 1024 { + if p.fs == nil || h.Length <= 16*1024 { return p.getNextObject(h) } @@ -319,8 +319,6 @@ func (p *Packfile) fillOFSDeltaObjectContent(obj plumbing.EncodedObject, offset if err != nil { return err } - - p.cachePut(base) } obj.SetType(base.Type()) From 34375327d34df563652e792c483bbe09c0a5b1af Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Mon, 24 Sep 2018 20:44:12 +0200 Subject: [PATCH 08/12] Combine deltaBaseCache and simpleObjectCache into one objectCache. Signed-off-by: Filip Navara --- storage/filesystem/object.go | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 2248699c1..8ccadd892 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -20,27 +20,25 @@ import ( type ObjectStorage struct { options Options - // deltaBaseCache is an object cache uses to cache delta's bases when - deltaBaseCache cache.Object - - simpleObjectCache cache.Object + // objectCache is an object cache uses to cache delta's bases and also recently + // loaded loose objects + objectCache cache.Object dir *dotgit.DotGit index map[plumbing.Hash]idxfile.Index } // NewObjectStorage creates a new ObjectStorage with the given .git directory and cache. -func NewObjectStorage(dir *dotgit.DotGit, deltaBaseCache cache.Object) *ObjectStorage { - return NewObjectStorageWithOptions(dir, deltaBaseCache, Options{}) +func NewObjectStorage(dir *dotgit.DotGit, objectCache cache.Object) *ObjectStorage { + return NewObjectStorageWithOptions(dir, objectCache, Options{}) } // NewObjectStorageWithOptions creates a new ObjectStorage with the given .git directory, cache and extra options -func NewObjectStorageWithOptions(dir *dotgit.DotGit, deltaBaseCache cache.Object, ops Options) *ObjectStorage { +func NewObjectStorageWithOptions(dir *dotgit.DotGit, objectCache cache.Object, ops Options) *ObjectStorage { return &ObjectStorage{ - options: ops, - deltaBaseCache: deltaBaseCache, - simpleObjectCache: cache.NewObjectLRU(cache.MiByte), - dir: dir, + options: ops, + objectCache: objectCache, + dir: dir, } } @@ -267,7 +265,7 @@ func (s *ObjectStorage) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (p // Create a new object storage with the DotGit(s) and check for the // required hash object. Skip when not found. for _, dg := range dotgits { - o := NewObjectStorage(dg, s.deltaBaseCache) + o := NewObjectStorage(dg, s.objectCache) enobj, enerr := o.EncodedObject(t, h) if enerr != nil { continue @@ -309,7 +307,7 @@ func (s *ObjectStorage) DeltaObject(t plumbing.ObjectType, } func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedObject, err error) { - if cacheObj, found := s.simpleObjectCache.Get(h); found { + if cacheObj, found := s.objectCache.Get(h); found { return cacheObj, nil } @@ -344,7 +342,7 @@ func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedOb return nil, err } - s.simpleObjectCache.Put(obj); + s.objectCache.Put(obj); _, err = io.Copy(w, r) return obj, err @@ -388,7 +386,7 @@ func (s *ObjectStorage) decodeObjectAt( ) (plumbing.EncodedObject, error) { hash, err := idx.FindHash(offset) if err == nil { - obj, ok := s.deltaBaseCache.Get(hash) + obj, ok := s.objectCache.Get(hash) if ok { return obj, nil } @@ -399,8 +397,8 @@ func (s *ObjectStorage) decodeObjectAt( } var p *packfile.Packfile - if s.deltaBaseCache != nil { - p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.deltaBaseCache) + if s.objectCache != nil { + p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.objectCache) } else { p = packfile.NewPackfile(idx, s.dir.Fs(), f) } @@ -510,7 +508,7 @@ func (s *ObjectStorage) buildPackfileIters( } return newPackfileIter( s.dir.Fs(), pack, t, seen, s.index[h], - s.deltaBaseCache, s.options.KeepDescriptors, + s.objectCache, s.options.KeepDescriptors, ) }, }, nil From 344fc952b63644c440d757996a4ffb564ed0c200 Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Wed, 26 Sep 2018 08:56:03 +0200 Subject: [PATCH 09/12] Add smallObjectThreshold constant instead of magic number. Signed-off-by: Filip Navara --- plumbing/format/packfile/packfile.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go index 538a690d5..e40302672 100644 --- a/plumbing/format/packfile/packfile.go +++ b/plumbing/format/packfile/packfile.go @@ -21,6 +21,16 @@ var ( ErrZLib = NewError("zlib reading error") ) +// When reading small objects from packfile it is beneficial to do so at +// once to exploit the buffered I/O. In many cases the objects are so small +// that they were already loaded to memory when the object header was +// loaded from the packfile. Wrapping in FSObject would cause this buffered +// data to be thrown away and then re-read later, with the additional +// seeking causing reloads from disk. Objects smaller than this threshold +// are now always read into memory and stored in cache instead of being +// wrapped in FSObject. +const smallObjectThreshold = 16 * 1024 + // Packfile allows retrieving information from inside a packfile. type Packfile struct { idxfile.Index @@ -180,7 +190,7 @@ func (p *Packfile) objectAtOffset(offset int64) (plumbing.EncodedObject, error) // If we have no filesystem, we will return a MemoryObject instead // of an FSObject. - if p.fs == nil || h.Length <= 16*1024 { + if p.fs == nil || h.Length <= smallObjectThreshold { return p.getNextObject(h) } From dd7c3e280467f40a034677ade3678b4dfd4f0f47 Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Wed, 26 Sep 2018 09:11:59 +0200 Subject: [PATCH 10/12] Add simple tests for SeekObjectHeader. Signed-off-by: Filip Navara --- plumbing/format/packfile/scanner.go | 16 ++++++++-------- plumbing/format/packfile/scanner_test.go | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/plumbing/format/packfile/scanner.go b/plumbing/format/packfile/scanner.go index 35017486a..614b0d1a8 100644 --- a/plumbing/format/packfile/scanner.go +++ b/plumbing/format/packfile/scanner.go @@ -141,7 +141,7 @@ func (s *Scanner) readCount() (uint32, error) { // SeekObjectHeader seeks to specified offset and returns the ObjectHeader // for the next object in the reader func (s *Scanner) SeekObjectHeader(offset int64) (*ObjectHeader, error) { - // if seeking we assume that you are not interested on the header + // if seeking we assume that you are not interested in the header if s.version == 0 { s.version = VersionSupported } @@ -150,8 +150,8 @@ func (s *Scanner) SeekObjectHeader(offset int64) (*ObjectHeader, error) { return nil, err } - h, err := s.nextObjectHeader(); - if err != nil { + h, err := s.nextObjectHeader() + if err != nil { return nil, err } @@ -165,13 +165,13 @@ func (s *Scanner) NextObjectHeader() (*ObjectHeader, error) { return nil, err } - offset, err := s.r.Seek(0, io.SeekCurrent); - if err != nil { + offset, err := s.r.Seek(0, io.SeekCurrent) + if err != nil { return nil, err } - h, err := s.nextObjectHeader(); - if err != nil { + h, err := s.nextObjectHeader() + if err != nil { return nil, err } @@ -346,7 +346,7 @@ var byteSlicePool = sync.Pool{ // SeekFromStart sets a new offset from start, returns the old position before // the change. func (s *Scanner) SeekFromStart(offset int64) (previous int64, err error) { - // if seeking we assume that you are not interested on the header + // if seeking we assume that you are not interested in the header if s.version == 0 { s.version = VersionSupported } diff --git a/plumbing/format/packfile/scanner_test.go b/plumbing/format/packfile/scanner_test.go index 644d0eb1d..091b457c0 100644 --- a/plumbing/format/packfile/scanner_test.go +++ b/plumbing/format/packfile/scanner_test.go @@ -118,6 +118,23 @@ func (s *ScannerSuite) TestNextObjectHeaderWithOutReadObjectNonSeekable(c *C) { c.Assert(n, Equals, f.PackfileHash) } +func (s *ScannerSuite) TestSeekObjectHeader(c *C) { + r := fixtures.Basic().One().Packfile() + p := NewScanner(r) + + h, err := p.SeekObjectHeader(expectedHeadersOFS[4].Offset) + c.Assert(err, IsNil) + c.Assert(h, DeepEquals, &expectedHeadersOFS[4]) +} + +func (s *ScannerSuite) TestSeekObjectHeaderNonSeekable(c *C) { + r := io.MultiReader(fixtures.Basic().One().Packfile()) + p := NewScanner(r) + + _, err := p.SeekObjectHeader(expectedHeadersOFS[4].Offset) + c.Assert(err, Equals, ErrSeekNotSupported) +} + var expectedHeadersOFS = []ObjectHeader{ {Type: plumbing.CommitObject, Offset: 12, Length: 254}, {Type: plumbing.OFSDeltaObject, Offset: 186, Length: 93, OffsetReference: 12}, From 4cb21e21bb4dae1171b5b02111ec7b8555d5be7a Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Mon, 26 Nov 2018 12:33:54 +0100 Subject: [PATCH 11/12] Fix build after rebase. Signed-off-by: Filip Navara --- storage/filesystem/object.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/storage/filesystem/object.go b/storage/filesystem/object.go index 8ccadd892..85bb79865 100644 --- a/storage/filesystem/object.go +++ b/storage/filesystem/object.go @@ -207,7 +207,7 @@ func (s *ObjectStorage) encodedObjectSizeFromPackfile(h plumbing.Hash) ( idx := s.index[pack] hash, err := idx.FindHash(offset) if err == nil { - obj, ok := s.deltaBaseCache.Get(hash) + obj, ok := s.objectCache.Get(hash) if ok { return obj.Size(), nil } @@ -216,8 +216,8 @@ func (s *ObjectStorage) encodedObjectSizeFromPackfile(h plumbing.Hash) ( } var p *packfile.Packfile - if s.deltaBaseCache != nil { - p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.deltaBaseCache) + if s.objectCache != nil { + p = packfile.NewPackfileWithCache(idx, s.dir.Fs(), f, s.objectCache) } else { p = packfile.NewPackfile(idx, s.dir.Fs(), f) } @@ -342,7 +342,7 @@ func (s *ObjectStorage) getFromUnpacked(h plumbing.Hash) (obj plumbing.EncodedOb return nil, err } - s.objectCache.Put(obj); + s.objectCache.Put(obj) _, err = io.Copy(w, r) return obj, err @@ -417,7 +417,7 @@ func (s *ObjectStorage) decodeDeltaObjectAt( } p := packfile.NewScanner(f) - header, err := p.SeekObjectHeader(offset); + header, err := p.SeekObjectHeader(offset) if err != nil { return nil, err } From 3c36f7a11bfea2334133cbc2af786b8ffafa62fc Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Mon, 26 Nov 2018 12:20:19 +0100 Subject: [PATCH 12/12] Do not treat delta references as small objects that can be eagerly loaded. Signed-off-by: Filip Navara --- plumbing/format/packfile/packfile.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plumbing/format/packfile/packfile.go b/plumbing/format/packfile/packfile.go index e40302672..1e7ef2694 100644 --- a/plumbing/format/packfile/packfile.go +++ b/plumbing/format/packfile/packfile.go @@ -190,7 +190,14 @@ func (p *Packfile) objectAtOffset(offset int64) (plumbing.EncodedObject, error) // If we have no filesystem, we will return a MemoryObject instead // of an FSObject. - if p.fs == nil || h.Length <= smallObjectThreshold { + if p.fs == nil { + return p.getNextObject(h) + } + + // If the object is not a delta and it's small enough then read it + // completely into memory now since it is already read from disk + // into buffer anyway. + if h.Length <= smallObjectThreshold && h.Type != plumbing.OFSDeltaObject && h.Type != plumbing.REFDeltaObject { return p.getNextObject(h) }