Skip to content
This repository was archived by the owner on Sep 11, 2020. It is now read-only.

Commit 881bcd3

Browse files
mcuadrosmcuadros
committed
storage: filesystem iter implementation
1 parent 148d586 commit 881bcd3

File tree

4 files changed

+201
-40
lines changed

4 files changed

+201
-40
lines changed

‎fixtures/fixtures.go‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,18 @@ var fixtures = []*Fixture{{
2323
Head: core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"),
2424
PackfileHash: core.NewHash("a3fed42da1e8189a077c0e6846c040dcf73fc9dd"),
2525
DotGitHash: core.NewHash("0a00a25543e6d732dbf4e8e9fec55c8e65fc4e8d"),
26+
ObjectsCount: 31,
2627
}, {
2728
Tags: []string{"ref-delta"},
2829
URL: "https://github.com/git-fixtures/basic",
2930
Head: core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"),
3031
PackfileHash: core.NewHash("c544593473465e6315ad4182d04d366c4592b829"),
32+
ObjectsCount: 31,
3133
}, {
32-
Tags: []string{".git", "unpacked", "multi-packfile"},
33-
URL: "https://github.com/src-d/go-git.git",
34-
DotGitHash: core.NewHash("174be6bd4292c18160542ae6dc6704b877b8a01a"),
34+
Tags: []string{".git", "unpacked", "multi-packfile"},
35+
URL: "https://github.com/src-d/go-git.git",
36+
DotGitHash: core.NewHash("174be6bd4292c18160542ae6dc6704b877b8a01a"),
37+
ObjectsCount: 2133,
3538
}, {
3639
URL: "https://github.com/spinnaker/spinnaker",
3740
Head: core.NewHash("06ce06d0fc49646c4de733c45b7788aabad98a6f"),
@@ -76,6 +79,7 @@ type Fixture struct {
7679
Head core.Hash
7780
PackfileHash core.Hash
7881
DotGitHash core.Hash
82+
ObjectsCount int32
7983
}
8084

8185
func (f *Fixture) Packfile() io.ReadSeeker {

‎formats/packfile/decoder.go‎

Lines changed: 25 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -91,20 +91,17 @@ func (d *Decoder) doDecode() error {
9191

9292
func (d *Decoder) readObjects(count uint32) error {
9393
for i := 0; i < int(count); i++ {
94-
obj, err := d.readObject()
94+
_, err := d.ReadObject()
9595
if err != nil {
9696
return err
9797
}
98-
99-
if _, err := d.tx.Set(obj); err != nil {
100-
return err
101-
}
10298
}
10399

104100
return nil
105101
}
106102

107-
func (d *Decoder) readObject() (core.Object, error) {
103+
// ReadObject reads a object from the stream and return it
104+
func (d *Decoder) ReadObject() (core.Object, error) {
108105
h, err := d.s.NextObjectHeader()
109106
if err != nil {
110107
return nil, err
@@ -130,9 +127,31 @@ func (d *Decoder) readObject() (core.Object, error) {
130127
}
131128

132129
d.remember(obj, h.Offset, crc)
130+
131+
if _, err := d.tx.Set(obj); err != nil {
132+
return nil, err
133+
}
134+
133135
return obj, nil
134136
}
135137

138+
// ReadObjectAt reads an object at the given location
139+
func (d *Decoder) ReadObjectAt(offset int64) (core.Object, error) {
140+
beforeJump, err := d.s.Seek(offset)
141+
if err != nil {
142+
return nil, err
143+
}
144+
145+
defer func() {
146+
_, seekErr := d.s.Seek(beforeJump)
147+
if err == nil {
148+
err = seekErr
149+
}
150+
}()
151+
152+
return d.ReadObject()
153+
}
154+
136155
func (d *Decoder) fillRegularObjectContent(obj core.Object) (uint32, error) {
137156
w, err := obj.Writer()
138157
if err != nil {
@@ -195,23 +214,6 @@ func (d *Decoder) recallByHash(h core.Hash) (core.Object, error) {
195214
return d.tx.Get(core.AnyObject, h)
196215
}
197216

198-
// ReadObjectAt reads an object at the given location
199-
func (d *Decoder) ReadObjectAt(offset int64) (core.Object, error) {
200-
beforeJump, err := d.s.Seek(offset)
201-
if err != nil {
202-
return nil, err
203-
}
204-
205-
defer func() {
206-
_, seekErr := d.s.Seek(beforeJump)
207-
if err == nil {
208-
err = seekErr
209-
}
210-
}()
211-
212-
return d.readObject()
213-
}
214-
215217
// Offsets returns the objects read offset
216218
func (d *Decoder) Offsets() map[core.Hash]int64 {
217219
i := make(map[core.Hash]int64, len(d.offsets))

‎storage/filesystem/object.go‎

Lines changed: 131 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"gopkg.in/src-d/go-git.v4/formats/packfile"
1212
"gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit"
1313
"gopkg.in/src-d/go-git.v4/storage/memory"
14+
"gopkg.in/src-d/go-git.v4/utils/fs"
1415
)
1516

1617
// ObjectStorage is an implementation of core.ObjectStorage that stores
@@ -165,34 +166,50 @@ func (s *ObjectStorage) findObjectInPackfile(h core.Hash) (core.Hash, int64) {
165166
// Iter returns an iterator for all the objects in the packfile with the
166167
// given type.
167168
func (s *ObjectStorage) Iter(t core.ObjectType) (core.ObjectIter, error) {
168-
var objects []core.Object
169+
objects, err := s.dir.Objects()
170+
if err != nil {
171+
return nil, err
172+
}
173+
174+
seen := make(map[core.Hash]bool, 0)
175+
var iters []core.ObjectIter
176+
if len(objects) != 0 {
177+
iters = append(iters, &objectsIter{s: s, t: t, h: objects})
178+
seen = hashListAsMap(objects)
179+
}
169180

170-
hashes, err := s.dir.Objects()
181+
packi, err := s.buildPackfileIters(t, seen)
171182
if err != nil {
172183
return nil, err
173184
}
174185

175-
for _, hash := range hashes {
176-
object, err := s.getFromUnpacked(hash)
186+
iters = append(iters, packi...)
187+
return core.NewMultiObjectIter(iters), nil
188+
}
189+
190+
func (s *ObjectStorage) buildPackfileIters(
191+
t core.ObjectType, seen map[core.Hash]bool) ([]core.ObjectIter, error) {
192+
packs, err := s.dir.ObjectPacks()
193+
if err != nil {
194+
return nil, err
195+
}
196+
197+
var iters []core.ObjectIter
198+
for _, h := range packs {
199+
pack, err := s.dir.ObjectPack(h)
177200
if err != nil {
178201
return nil, err
179202
}
180-
if object.Type() == t {
181-
objects = append(objects, object)
182-
}
183-
}
184203

185-
for hash := range s.index {
186-
object, err := s.getFromPackfile(hash)
204+
iter, err := newPackfileIter(pack, t, seen)
187205
if err != nil {
188206
return nil, err
189207
}
190-
if t == core.AnyObject || object.Type() == t {
191-
objects = append(objects, object)
192-
}
208+
209+
iters = append(iters, iter)
193210
}
194211

195-
return core.NewObjectSliceIter(objects), nil
212+
return iters, nil
196213
}
197214

198215
func (o *ObjectStorage) Begin() core.TxObjectStorage {
@@ -233,3 +250,103 @@ func (i index) Decode(r io.Reader) error {
233250

234251
return nil
235252
}
253+
254+
type packfileIter struct {
255+
f fs.File
256+
d *packfile.Decoder
257+
t core.ObjectType
258+
259+
seen map[core.Hash]bool
260+
position uint32
261+
total uint32
262+
}
263+
264+
func newPackfileIter(
265+
f fs.File,
266+
t core.ObjectType,
267+
seen map[core.Hash]bool,
268+
) (core.ObjectIter, error) {
269+
s := packfile.NewScanner(f)
270+
_, total, err := s.Header()
271+
if err != nil {
272+
return nil, err
273+
}
274+
275+
d := packfile.NewDecoder(s, memory.NewStorage().ObjectStorage())
276+
return &packfileIter{f: f, d: d, t: t, total: total, seen: seen}, nil
277+
}
278+
279+
func (iter *packfileIter) Next() (core.Object, error) {
280+
if iter.position >= iter.total {
281+
return nil, io.EOF
282+
}
283+
284+
obj, err := iter.d.ReadObject()
285+
if err != nil {
286+
return nil, err
287+
}
288+
289+
iter.position++
290+
if iter.seen[obj.Hash()] {
291+
return iter.Next()
292+
}
293+
294+
if iter.t != core.AnyObject && iter.t != obj.Type() {
295+
return iter.Next()
296+
}
297+
298+
return obj, nil
299+
}
300+
301+
// ForEach is never called since is used inside of a MultiObjectIterator
302+
func (iter *packfileIter) ForEach(cb func(core.Object) error) error {
303+
return nil
304+
}
305+
306+
func (iter *packfileIter) Close() {
307+
iter.f.Close()
308+
iter.d.Close()
309+
}
310+
311+
type objectsIter struct {
312+
s *ObjectStorage
313+
t core.ObjectType
314+
h []core.Hash
315+
}
316+
317+
func (iter *objectsIter) Next() (core.Object, error) {
318+
if len(iter.h) == 0 {
319+
return nil, io.EOF
320+
}
321+
322+
obj, err := iter.s.getFromUnpacked(iter.h[0])
323+
iter.h = iter.h[1:]
324+
325+
if err != nil {
326+
return nil, err
327+
}
328+
329+
if iter.t != core.AnyObject && iter.t != obj.Type() {
330+
return iter.Next()
331+
}
332+
333+
return obj, err
334+
}
335+
336+
// ForEach is never called since is used inside of a MultiObjectIterator
337+
func (iter *objectsIter) ForEach(cb func(core.Object) error) error {
338+
return nil
339+
}
340+
341+
func (iter *objectsIter) Close() {
342+
iter.h = []core.Hash{}
343+
}
344+
345+
func hashListAsMap(l []core.Hash) map[core.Hash]bool {
346+
m := make(map[core.Hash]bool, len(l))
347+
for _, h := range l {
348+
m[h] = true
349+
}
350+
351+
return m
352+
}

‎storage/filesystem/object_test.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,3 +53,41 @@ func (s *FsSuite) TestGetFromPackfileMultiplePackfiles(c *C) {
5353
c.Assert(err, IsNil)
5454
c.Assert(obj.Hash(), Equals, expected)
5555
}
56+
57+
func (s *FsSuite) TestIter(c *C) {
58+
fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) {
59+
fs := f.DotGit()
60+
o, err := newObjectStorage(dotgit.New(fs))
61+
c.Assert(err, IsNil)
62+
63+
iter, err := o.Iter(core.AnyObject)
64+
c.Assert(err, IsNil)
65+
66+
var count int32
67+
err = iter.ForEach(func(o core.Object) error {
68+
count++
69+
return nil
70+
})
71+
72+
c.Assert(err, IsNil)
73+
c.Assert(count, Equals, f.ObjectsCount)
74+
})
75+
}
76+
77+
func (s *FsSuite) TestIterWithType(c *C) {
78+
fixtures.ByTag(".git").Test(c, func(f *fixtures.Fixture) {
79+
fs := f.DotGit()
80+
o, err := newObjectStorage(dotgit.New(fs))
81+
c.Assert(err, IsNil)
82+
83+
iter, err := o.Iter(core.CommitObject)
84+
c.Assert(err, IsNil)
85+
86+
err = iter.ForEach(func(o core.Object) error {
87+
c.Assert(o.Type(), Equals, core.CommitObject)
88+
return nil
89+
})
90+
91+
c.Assert(err, IsNil)
92+
})
93+
}

0 commit comments

Comments
 (0)