core: ObjectStorage.Writer

Author: mcuadros

core/storage.go

@@ -1,31 +1,43 @@
 package core
 
-import "errors"
+import (
+	"errors"
+	"io"
+)
 
-//ErrStop is used to stop a ForEach function in an Iter
-var ErrStop = errors.New("stop iter")
+var (
+	//ErrStop is used to stop a ForEach function in an Iter
+	ErrStop           = errors.New("stop iter")
+	ErrNotImplemented = errors.New("method not-implemented")
+)
 
 // ObjectStorage generic storage of objects
 type ObjectStorage interface {
 	// NewObject returns a new Object, the real type of the object can be a
 	// custom implementation or the defaul one, MemoryObject
 	NewObject() Object
+	// Writer retuns a writer for writing a packfile to the Storage, this method
+	// is optional, if not implemented the ObjectStorage should return a
+	// ErrNotImplemented error.
+	//
+	// If the implementation not implements Writer the objects should be written
+	// using the Set method.
+	Writer() (io.WriteCloser, error)
 	// Set save an object into the storage, the object shuld be create with
 	// the NewObject, method, and file if the type is not supported.
 	Set(Object) (Hash, error)
 	// Get an object by hash with the given ObjectType. Implementors should
 	// return (nil, ErrObjectNotFound) if an object doesn't exist with both the
 	// given hash and object type.
 	//
-	// Valid ObjectType values are CommitObject, BlobObject, TagObject, TreeObject
-	// and AnyObject.
+	// Valid ObjectType values are CommitObject, BlobObject, TagObject,
+	// TreeObject and AnyObject.
 	//
 	// If AnyObject is given, the object must be looked up regardless of its type.
 	Get(ObjectType, Hash) (Object, error)
 	// Iter returns a custom ObjectIter over all the object on the storage.
 	//
-	// Valid ObjectType values are CommitObject, BlobObject, TagObject, TreeObject
-	// and AnyObject.
+	// Valid ObjectType values are CommitObject, BlobObject, TagObject,
 	Iter(ObjectType) (ObjectIter, error)
 	// Begin starts a transaction.
 	Begin() TxObjectStorage

examples/storage/aerospike/storage.go

@@ -70,6 +70,11 @@ func (s *ObjectStorage) NewObject() core.Object {
 	return &core.MemoryObject{}
 }
 
+// Writer method not supported, this method is optional to implemented.
+func (s *ObjectStorage) Writer() (io.WriteCloser, error) {
+	return nil, core.ErrNotImplemented
+}
+
 func (s *ObjectStorage) Set(obj core.Object) (core.Hash, error) {
 	key, err := s.buildKey(obj.Hash(), obj.Type())
 	if err != nil {

storage/filesystem/internal/index/index.go

@@ -35,18 +35,26 @@ func NewFromIdx(r io.Reader) (Index, error) {
 
 // NewFrompackfile returns a new index from a packfile reader.
 func NewFromPackfile(rs io.ReadSeeker) (Index, error) {
-	index := make(Index)
+	s := packfile.NewSeekable(rs)
+	return newFromPackfile(rs, s)
+}
 
-	r := packfile.NewSeekable(rs)
-	p := packfile.NewParser(r)
+func NewFromPackfileInMemory(rs io.Reader) (Index, error) {
+	s := packfile.NewStream(rs)
+	return newFromPackfile(rs, s)
+}
 
+func newFromPackfile(r io.Reader, s packfile.ReadRecaller) (Index, error) {
+	index := make(Index)
+
+	p := packfile.NewParser(s)
 	count, err := p.ReadHeader()
 	if err != nil {
 		return nil, err
 	}
 
 	for i := 0; i < int(count); i++ {
-		offset, err := r.Offset()
+		offset, err := s.Offset()
 		if err != nil {
 			return nil, err
 		}
@@ -56,17 +64,19 @@ func NewFromPackfile(rs io.ReadSeeker) (Index, error) {
 			return nil, err
 		}
 
-		err = r.Remember(offset, obj)
+		err = s.Remember(offset, obj)
 		if err != nil {
 			return nil, err
 		}
 
-		err = index.Set(obj.Hash(), offset)
-		if err != nil {
+		if err = index.Set(obj.Hash(), offset); err != nil {
 			return nil, err
 		}
 	}
 
+	//The trailer records 20-byte SHA-1 checksum of all of the above.
+	p.ReadHash()
+
 	return index, nil
 }
 

storage/filesystem/object.go

@@ -1,7 +1,9 @@
 package filesystem
 
 import (
+	"bytes"
 	"fmt"
+	"io"
 	"os"
 
 	"gopkg.in/src-d/go-git.v4/core"
@@ -30,6 +32,58 @@ func (s *ObjectStorage) NewObject() core.Object {
 	return &core.MemoryObject{}
 }
 
+// Writer method not supported on Memory storage
+func (o *ObjectStorage) Writer() (io.WriteCloser, error) {
+	file := bytes.NewBuffer(nil)
+	return newPackWrite(o, file), nil
+}
+
+type packWriter struct {
+	writer     io.Writer
+	pipeReader io.ReadCloser
+	pipeWriter io.WriteCloser
+	file       io.Writer
+	result     chan error
+}
+
+func newPackWrite(o *ObjectStorage, file io.Writer) io.WriteCloser {
+	r, w := io.Pipe()
+
+	ch := make(chan error)
+	go func(r io.ReadCloser) {
+		defer r.Close()
+		index, err := index.NewFromPackfileInMemory(r)
+		o.index = index
+
+		ch <- err
+	}(r)
+
+	return &packWriter{
+		writer:     io.MultiWriter(w, file),
+		pipeReader: r,
+		pipeWriter: w,
+		file:       file,
+		result:     ch,
+	}
+
+}
+
+func (w *packWriter) Write(p []byte) (int, error) {
+	return w.writer.Write(p)
+}
+
+func (w *packWriter) Close() error {
+	defer func() {
+		close(w.result)
+	}()
+
+	if err := w.pipeWriter.Close(); err != nil {
+		return err
+	}
+
+	return <-w.result
+}
+
 // Set adds a new object to the storage. As this functionality is not
 // yet supported, this method always returns a "not implemented yet"
 // error an zero hash.

storage/filesystem/object_test.go

@@ -2,6 +2,7 @@ package filesystem
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"reflect"
@@ -62,6 +63,22 @@ func (s *FsSuite) TearDownSuite(c *C) {
 	}
 }
 
+func (s *FsSuite) TestWriter(c *C) {
+	r, err := os.Open("../../formats/packfile/fixtures/git-fixture.ofs-delta")
+	c.Assert(err, IsNil)
+
+	o := &ObjectStorage{}
+	w, err := o.Writer()
+	c.Assert(err, IsNil)
+
+	n, err := io.Copy(w, r)
+	c.Assert(err, IsNil)
+	c.Check(n, Equals, int64(85300))
+
+	c.Assert(o.index, HasLen, 28)
+	c.Assert(w.Close(), IsNil)
+}
+
 func (s *FsSuite) TestHashNotFound(c *C) {
 	sto := s.newObjectStorage(c, "binary-relations")
 	types := []core.ObjectType{core.AnyObject, core.TagObject, core.CommitObject, core.BlobObject, core.TreeObject}

storage/memory/storage.go

@@ -3,6 +3,7 @@ package memory
 
 import (
 	"fmt"
+	"io"
 
 	"gopkg.in/src-d/go-git.v4/config"
 	"gopkg.in/src-d/go-git.v4/core"
@@ -109,6 +110,11 @@ func (o *ObjectStorage) NewObject() core.Object {
 	return &core.MemoryObject{}
 }
 
+// Writer method not supported on Memory storage
+func (o *ObjectStorage) Writer() (io.WriteCloser, error) {
+	return nil, core.ErrNotImplemented
+}
+
 // Set stores an object, the object should be properly filled before set it.
 func (o *ObjectStorage) Set(obj core.Object) (core.Hash, error) {
 	h := obj.Hash()