Merge pull request #698 from ajnavarro/perf/list-only-reachable-objects

 Perf: Traverse commit history instead of reading the packfile

Author: ajnavarro

Gopkg.lock

@@ -17,7 +17,7 @@
 [[constraint]]
   name = "gopkg.in/src-d/go-git.v4"
   source = "github.com/src-d/go-git"
-  version = "4.9.1"
+  revision = "4d8bd13a73d6efd41b6be0b2eba5130c498fae0d"
 
 [[constraint]]
   name = "gopkg.in/src-d/go-git-fixtures.v3"

Gopkg.toml

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"io"
 
+	git "gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 	"gopkg.in/src-d/go-mysql-server.v0/sql"
@@ -219,7 +220,9 @@ func (i *commitBlobsRowIter) init() error {
 	if len(i.commits) > 0 {
 		i.iter, err = NewCommitsByHashIter(i.repo, i.commits)
 	} else {
-		i.iter, err = i.repo.CommitObjects()
+		i.iter, err = i.repo.Log(&git.LogOptions{
+			All: true,
+		})
 	}
 
 	return err

commit_blobs.go

@@ -5,6 +5,7 @@ import (
 	"io"
 
 	"github.com/sirupsen/logrus"
+	git "gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-git.v4/plumbing/filemode"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
@@ -196,7 +197,10 @@ func (i *commitFilesRowIter) init() error {
 	if len(i.commitHashes) > 0 {
 		i.commits, err = NewCommitsByHashIter(i.repo, i.commitHashes)
 	} else {
-		i.commits, err = i.repo.CommitObjects()
+		i.commits, err = i.repo.
+			Log(&git.LogOptions{
+				All: true,
+			})
 	}
 
 	return err
@@ -433,7 +437,10 @@ func newCommitFilesKeyValueIter(
 		return nil, err
 	}
 
-	commits, err := repo.CommitObjects()
+	commits, err := repo.
+		Log(&git.LogOptions{
+			All: true,
+		})
 	if err != nil {
 		return nil, err
 	}

commit_files.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	git "gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-mysql-server.v0/sql"
 	"gopkg.in/src-d/go-mysql-server.v0/sql/expression"
@@ -35,7 +36,9 @@ func TestCommitFilesTableRowIter(t *testing.T) {
 
 		require.NoError(err)
 
-		commits, err := repo.CommitObjects()
+		commits, err := repo.Log(&git.LogOptions{
+			All: true,
+		})
 		require.NoError(err)
 
 		for {

commit_files_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"io"
 
+	git "gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-git.v4/plumbing"
 
 	"gopkg.in/src-d/go-git.v4/plumbing/filemode"
@@ -215,7 +216,9 @@ func (i *commitTreesRowIter) init() error {
 	if len(i.commitHashes) > 0 {
 		i.commits, err = NewCommitsByHashIter(i.repo, i.commitHashes)
 	} else {
-		i.commits, err = i.repo.CommitObjects()
+		i.commits, err = i.repo.Log(&git.LogOptions{
+			All: true,
+		})
 	}
 
 	return err

commit_trees.go

@@ -23,17 +23,17 @@ func TestCommitTreesRowIter(t *testing.T) {
 	}
 
 	expected := []sql.Row{
-		{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "dbd3641b371024f44d0e469a9c8f5457b0660de1"},
-		{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "a39771a7651f97faf5c72e08224d857fc35133db"},
-		{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
-		{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
-
 		{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "a8d315b2b1c615d43042c3a62402b8a54288cf5c"},
 		{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "a39771a7651f97faf5c72e08224d857fc35133db"},
 		{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
 		{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
 		{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "cf4aa3b38974fb7d81f367c0830f7d78d65ab86b"},
 
+		{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "dbd3641b371024f44d0e469a9c8f5457b0660de1"},
+		{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "a39771a7651f97faf5c72e08224d857fc35133db"},
+		{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
+		{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
+
 		{"918c48b83bd081e863dbe1b80f8998f058cd8294", "fb72698cab7617ac416264415f13224dfd7a165e"},
 		{"918c48b83bd081e863dbe1b80f8998f058cd8294", "a39771a7651f97faf5c72e08224d857fc35133db"},
 		{"918c48b83bd081e863dbe1b80f8998f058cd8294", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
@@ -44,13 +44,13 @@ func TestCommitTreesRowIter(t *testing.T) {
 
 		{"1669dce138d9b841a518c64b10914d88f5e488ea", "eba74343e2f15d62adedfd8c883ee0262b5c8021"},
 
-		{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},
-
-		{"b8e471f58bcbca63b07bda20e428190409c2db47", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},
-
 		{"35e85108805c84807bc66a02d91535e1e24b38b9", "8dcef98b1d52143e1e2dbc458ffe38f925786bf2"},
 
 		{"b029517f6300c2da0f4b651b8642506cd6aaf45d", "aa9b383c260e1d05fbbf6b30a02914555e20c725"},
+
+		{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},
+
+		{"b8e471f58bcbca63b07bda20e428190409c2db47", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},
 	}
 
 	require.Equal(expected, rows)
@@ -90,8 +90,8 @@ func TestCommitTreesPushdown(t *testing.T) {
 				),
 			},
 			[]sql.Row{
-				{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
 				{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
+				{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
 				{"918c48b83bd081e863dbe1b80f8998f058cd8294", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
 			},
 		},

commit_trees_test.go

@@ -3,6 +3,7 @@ package gitbase
 import (
 	"io"
 
+	git "gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-mysql-server.v0/sql"
 
 	"gopkg.in/src-d/go-git.v4/plumbing"
@@ -161,7 +162,10 @@ func (i *commitIter) init() error {
 	if len(i.hashes) > 0 {
 		i.iter, err = NewCommitsByHashIter(i.repo, i.hashes)
 	} else {
-		i.iter, err = i.repo.CommitObjects()
+		i.iter, err =
+			i.repo.Log(&git.LogOptions{
+				All: true,
+			})
 	}
 
 	return err
@@ -245,7 +249,10 @@ func NewCommitsByHashIter(
 	var commitIter object.CommitIter
 	var err error
 	if len(hashes) == 0 {
-		commitIter, err = repo.CommitObjects()
+		commitIter, err =
+			repo.Log(&git.LogOptions{
+				All: true,
+			})
 		if err != nil {
 			return nil, err
 		}
@@ -333,7 +340,10 @@ func newCommitsKeyValueIter(
 		return nil, err
 	}
 
-	commits, err := repo.CommitObjects()
+	commits, err :=
+		repo.Log(&git.LogOptions{
+			All: true,
+		})
 	if err != nil {
 		return nil, err
 	}

commits.go

@@ -101,14 +101,14 @@ func TestCommitsParents(t *testing.T) {
 	}{
 		{
 			name: "test commits parents 1",
-			hash: "e8d3ffab552895c19b9fcf7aa264d277cde33881",
+			hash: "6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
 			parents: []string{
 				"918c48b83bd081e863dbe1b80f8998f058cd8294",
 			},
 		},
 		{
 			name: "test commits parents 2",
-			hash: "6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
+			hash: "e8d3ffab552895c19b9fcf7aa264d277cde33881",
 			parents: []string{
 				"918c48b83bd081e863dbe1b80f8998f058cd8294",
 			},
@@ -137,30 +137,30 @@ func TestCommitsParents(t *testing.T) {
 		},
 		{
 			name: "test commits parents 6",
-			hash: "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
+			hash: "35e85108805c84807bc66a02d91535e1e24b38b9",
 			parents: []string{
 				"b029517f6300c2da0f4b651b8642506cd6aaf45d",
-				"b8e471f58bcbca63b07bda20e428190409c2db47",
 			},
 		},
 		{
-			name: "test commits parents 7",
-			hash: "b8e471f58bcbca63b07bda20e428190409c2db47",
-			parents: []string{
-				"b029517f6300c2da0f4b651b8642506cd6aaf45d",
-			},
+			name:    "test commits parents 7",
+			hash:    "b029517f6300c2da0f4b651b8642506cd6aaf45d",
+			parents: []string{},
 		},
 		{
 			name: "test commits parents 8",
-			hash: "35e85108805c84807bc66a02d91535e1e24b38b9",
+			hash: "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
 			parents: []string{
 				"b029517f6300c2da0f4b651b8642506cd6aaf45d",
+				"b8e471f58bcbca63b07bda20e428190409c2db47",
 			},
 		},
 		{
-			name:    "test commits parents 9",
-			hash:    "b029517f6300c2da0f4b651b8642506cd6aaf45d",
-			parents: []string{},
+			name: "test commits parents 9",
+			hash: "b8e471f58bcbca63b07bda20e428190409c2db47",
+			parents: []string{
+				"b029517f6300c2da0f4b651b8642506cd6aaf45d",
+			},
 		},
 	}
 
@@ -188,65 +188,65 @@ func TestCommitsIndexKeyValueIter(t *testing.T) {
 			assertEncodeKey(t, &packOffsetIndexKey{
 				Repository: path,
 				Packfile:   "323a4b6b5de684f9966953a043bc800154e5dbfa",
-				Offset:     682,
+				Offset:     186,
 			}),
 			[]interface{}{
-				"1669dce138d9b841a518c64b10914d88f5e488ea",
+				"6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
 				"[email protected]",
 			},
 		},
 		{
 			assertEncodeKey(t, &packOffsetIndexKey{
 				Repository: path,
 				Packfile:   "323a4b6b5de684f9966953a043bc800154e5dbfa",
-				Offset:     1292,
+				Offset:     12,
 			}),
 			[]interface{}{
-				"35e85108805c84807bc66a02d91535e1e24b38b9",
+				"e8d3ffab552895c19b9fcf7aa264d277cde33881",
 				"[email protected]",
 			},
 		},
 		{
 			assertEncodeKey(t, &packOffsetIndexKey{
 				Repository: path,
 				Packfile:   "323a4b6b5de684f9966953a043bc800154e5dbfa",
-				Offset:     186,
+				Offset:     353,
 			}),
 			[]interface{}{
-				"6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
+				"918c48b83bd081e863dbe1b80f8998f058cd8294",
 				"[email protected]",
 			},
 		},
 		{
 			assertEncodeKey(t, &packOffsetIndexKey{
 				Repository: path,
 				Packfile:   "323a4b6b5de684f9966953a043bc800154e5dbfa",
-				Offset:     353,
+				Offset:     516,
 			}),
 			[]interface{}{
-				"918c48b83bd081e863dbe1b80f8998f058cd8294",
+				"af2d6a6954d532f8ffb47615169c8fdf9d383a1a",
 				"[email protected]",
 			},
 		},
 		{
 			assertEncodeKey(t, &packOffsetIndexKey{
 				Repository: path,
 				Packfile:   "323a4b6b5de684f9966953a043bc800154e5dbfa",
-				Offset:     905,
+				Offset:     682,
 			}),
 			[]interface{}{
-				"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
+				"1669dce138d9b841a518c64b10914d88f5e488ea",
 				"[email protected]",
 			},
 		},
 		{
 			assertEncodeKey(t, &packOffsetIndexKey{
 				Repository: path,
 				Packfile:   "323a4b6b5de684f9966953a043bc800154e5dbfa",
-				Offset:     516,
+				Offset:     1292,
 			}),
 			[]interface{}{
-				"af2d6a6954d532f8ffb47615169c8fdf9d383a1a",
+				"35e85108805c84807bc66a02d91535e1e24b38b9",
 				"[email protected]",
 			},
 		},
@@ -265,22 +265,22 @@ func TestCommitsIndexKeyValueIter(t *testing.T) {
 			assertEncodeKey(t, &packOffsetIndexKey{
 				Repository: path,
 				Packfile:   "323a4b6b5de684f9966953a043bc800154e5dbfa",
-				Offset:     1130,
+				Offset:     905,
 			}),
 			[]interface{}{
-				"b8e471f58bcbca63b07bda20e428190409c2db47",
-				"[email protected]",
+				"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
+				"[email protected]",
 			},
 		},
 		{
 			assertEncodeKey(t, &packOffsetIndexKey{
 				Repository: path,
 				Packfile:   "323a4b6b5de684f9966953a043bc800154e5dbfa",
-				Offset:     12,
+				Offset:     1130,
 			}),
 			[]interface{}{
-				"e8d3ffab552895c19b9fcf7aa264d277cde33881",
-				"[email protected]",
+				"b8e471f58bcbca63b07bda20e428190409c2db47",
+				"[email protected]",
 			},
 		},
 	}