Skip to content

Perf: Traverse commit history instead of reading the packfile #698

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 18, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Gopkg.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Gopkg.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
[[constraint]]
name = "gopkg.in/src-d/go-git.v4"
source = "github.com/src-d/go-git"
version = "4.9.1"
revision = "4d8bd13a73d6efd41b6be0b2eba5130c498fae0d"

[[constraint]]
name = "gopkg.in/src-d/go-git-fixtures.v3"
Expand Down
5 changes: 4 additions & 1 deletion commit_blobs.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"io"

git "gopkg.in/src-d/go-git.v4"
"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-git.v4/plumbing/object"
"gopkg.in/src-d/go-mysql-server.v0/sql"
Expand Down Expand Up @@ -219,7 +220,9 @@ func (i *commitBlobsRowIter) init() error {
if len(i.commits) > 0 {
i.iter, err = NewCommitsByHashIter(i.repo, i.commits)
} else {
i.iter, err = i.repo.CommitObjects()
i.iter, err = i.repo.Log(&git.LogOptions{
All: true,
})
}

return err
Expand Down
11 changes: 9 additions & 2 deletions commit_files.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"io"

"github.com/sirupsen/logrus"
git "gopkg.in/src-d/go-git.v4"
"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-git.v4/plumbing/filemode"
"gopkg.in/src-d/go-git.v4/plumbing/object"
Expand Down Expand Up @@ -196,7 +197,10 @@ func (i *commitFilesRowIter) init() error {
if len(i.commitHashes) > 0 {
i.commits, err = NewCommitsByHashIter(i.repo, i.commitHashes)
} else {
i.commits, err = i.repo.CommitObjects()
i.commits, err = i.repo.
Log(&git.LogOptions{
All: true,
})
}

return err
Expand Down Expand Up @@ -433,7 +437,10 @@ func newCommitFilesKeyValueIter(
return nil, err
}

commits, err := repo.CommitObjects()
commits, err := repo.
Log(&git.LogOptions{
All: true,
})
if err != nil {
return nil, err
}
Expand Down
5 changes: 4 additions & 1 deletion commit_files_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"testing"

"github.com/stretchr/testify/require"
git "gopkg.in/src-d/go-git.v4"
"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-mysql-server.v0/sql"
"gopkg.in/src-d/go-mysql-server.v0/sql/expression"
Expand Down Expand Up @@ -35,7 +36,9 @@ func TestCommitFilesTableRowIter(t *testing.T) {

require.NoError(err)

commits, err := repo.CommitObjects()
commits, err := repo.Log(&git.LogOptions{
All: true,
})
require.NoError(err)

for {
Expand Down
5 changes: 4 additions & 1 deletion commit_trees.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"io"

git "gopkg.in/src-d/go-git.v4"
"gopkg.in/src-d/go-git.v4/plumbing"

"gopkg.in/src-d/go-git.v4/plumbing/filemode"
Expand Down Expand Up @@ -215,7 +216,9 @@ func (i *commitTreesRowIter) init() error {
if len(i.commitHashes) > 0 {
i.commits, err = NewCommitsByHashIter(i.repo, i.commitHashes)
} else {
i.commits, err = i.repo.CommitObjects()
i.commits, err = i.repo.Log(&git.LogOptions{
All: true,
})
}

return err
Expand Down
20 changes: 10 additions & 10 deletions commit_trees_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,17 @@ func TestCommitTreesRowIter(t *testing.T) {
}

expected := []sql.Row{
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "dbd3641b371024f44d0e469a9c8f5457b0660de1"},
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "a39771a7651f97faf5c72e08224d857fc35133db"},
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},

{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "a8d315b2b1c615d43042c3a62402b8a54288cf5c"},
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "a39771a7651f97faf5c72e08224d857fc35133db"},
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "cf4aa3b38974fb7d81f367c0830f7d78d65ab86b"},

{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "dbd3641b371024f44d0e469a9c8f5457b0660de1"},
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "a39771a7651f97faf5c72e08224d857fc35133db"},
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},

{"918c48b83bd081e863dbe1b80f8998f058cd8294", "fb72698cab7617ac416264415f13224dfd7a165e"},
{"918c48b83bd081e863dbe1b80f8998f058cd8294", "a39771a7651f97faf5c72e08224d857fc35133db"},
{"918c48b83bd081e863dbe1b80f8998f058cd8294", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
Expand All @@ -44,13 +44,13 @@ func TestCommitTreesRowIter(t *testing.T) {

{"1669dce138d9b841a518c64b10914d88f5e488ea", "eba74343e2f15d62adedfd8c883ee0262b5c8021"},

{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},

{"b8e471f58bcbca63b07bda20e428190409c2db47", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},

{"35e85108805c84807bc66a02d91535e1e24b38b9", "8dcef98b1d52143e1e2dbc458ffe38f925786bf2"},

{"b029517f6300c2da0f4b651b8642506cd6aaf45d", "aa9b383c260e1d05fbbf6b30a02914555e20c725"},

{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},

{"b8e471f58bcbca63b07bda20e428190409c2db47", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},
}

require.Equal(expected, rows)
Expand Down Expand Up @@ -90,8 +90,8 @@ func TestCommitTreesPushdown(t *testing.T) {
),
},
[]sql.Row{
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
{"918c48b83bd081e863dbe1b80f8998f058cd8294", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
},
},
Expand Down
16 changes: 13 additions & 3 deletions commits.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package gitbase
import (
"io"

git "gopkg.in/src-d/go-git.v4"
"gopkg.in/src-d/go-mysql-server.v0/sql"

"gopkg.in/src-d/go-git.v4/plumbing"
Expand Down Expand Up @@ -161,7 +162,10 @@ func (i *commitIter) init() error {
if len(i.hashes) > 0 {
i.iter, err = NewCommitsByHashIter(i.repo, i.hashes)
} else {
i.iter, err = i.repo.CommitObjects()
i.iter, err =
i.repo.Log(&git.LogOptions{
All: true,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we should force order by time (WDYT)?
By default pre-order will be used, which I think works slightly different than native git implementation.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

but here we don't want to mimic 100% git log output, we want to reduce unnecessary disk reads. Ordering commits will not give us any performance improvement.

})
}

return err
Expand Down Expand Up @@ -245,7 +249,10 @@ func NewCommitsByHashIter(
var commitIter object.CommitIter
var err error
if len(hashes) == 0 {
commitIter, err = repo.CommitObjects()
commitIter, err =
repo.Log(&git.LogOptions{
All: true,
})
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -333,7 +340,10 @@ func newCommitsKeyValueIter(
return nil, err
}

commits, err := repo.CommitObjects()
commits, err :=
repo.Log(&git.LogOptions{
All: true,
})
if err != nil {
return nil, err
}
Expand Down
62 changes: 31 additions & 31 deletions commits_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,14 +101,14 @@ func TestCommitsParents(t *testing.T) {
}{
{
name: "test commits parents 1",
hash: "e8d3ffab552895c19b9fcf7aa264d277cde33881",
hash: "6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
parents: []string{
"918c48b83bd081e863dbe1b80f8998f058cd8294",
},
},
{
name: "test commits parents 2",
hash: "6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
hash: "e8d3ffab552895c19b9fcf7aa264d277cde33881",
parents: []string{
"918c48b83bd081e863dbe1b80f8998f058cd8294",
},
Expand Down Expand Up @@ -137,30 +137,30 @@ func TestCommitsParents(t *testing.T) {
},
{
name: "test commits parents 6",
hash: "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
hash: "35e85108805c84807bc66a02d91535e1e24b38b9",
parents: []string{
"b029517f6300c2da0f4b651b8642506cd6aaf45d",
"b8e471f58bcbca63b07bda20e428190409c2db47",
},
},
{
name: "test commits parents 7",
hash: "b8e471f58bcbca63b07bda20e428190409c2db47",
parents: []string{
"b029517f6300c2da0f4b651b8642506cd6aaf45d",
},
name: "test commits parents 7",
hash: "b029517f6300c2da0f4b651b8642506cd6aaf45d",
parents: []string{},
},
{
name: "test commits parents 8",
hash: "35e85108805c84807bc66a02d91535e1e24b38b9",
hash: "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
parents: []string{
"b029517f6300c2da0f4b651b8642506cd6aaf45d",
"b8e471f58bcbca63b07bda20e428190409c2db47",
},
},
{
name: "test commits parents 9",
hash: "b029517f6300c2da0f4b651b8642506cd6aaf45d",
parents: []string{},
name: "test commits parents 9",
hash: "b8e471f58bcbca63b07bda20e428190409c2db47",
parents: []string{
"b029517f6300c2da0f4b651b8642506cd6aaf45d",
},
},
}

Expand Down Expand Up @@ -188,65 +188,65 @@ func TestCommitsIndexKeyValueIter(t *testing.T) {
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 682,
Offset: 186,
}),
[]interface{}{
"1669dce138d9b841a518c64b10914d88f5e488ea",
"6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
"[email protected]",
},
},
{
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 1292,
Offset: 12,
}),
[]interface{}{
"35e85108805c84807bc66a02d91535e1e24b38b9",
"e8d3ffab552895c19b9fcf7aa264d277cde33881",
"[email protected]",
},
},
{
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 186,
Offset: 353,
}),
[]interface{}{
"6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
"918c48b83bd081e863dbe1b80f8998f058cd8294",
"[email protected]",
},
},
{
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 353,
Offset: 516,
}),
[]interface{}{
"918c48b83bd081e863dbe1b80f8998f058cd8294",
"af2d6a6954d532f8ffb47615169c8fdf9d383a1a",
"[email protected]",
},
},
{
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 905,
Offset: 682,
}),
[]interface{}{
"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
"1669dce138d9b841a518c64b10914d88f5e488ea",
"[email protected]",
},
},
{
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 516,
Offset: 1292,
}),
[]interface{}{
"af2d6a6954d532f8ffb47615169c8fdf9d383a1a",
"35e85108805c84807bc66a02d91535e1e24b38b9",
"[email protected]",
},
},
Expand All @@ -265,22 +265,22 @@ func TestCommitsIndexKeyValueIter(t *testing.T) {
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 1130,
Offset: 905,
}),
[]interface{}{
"b8e471f58bcbca63b07bda20e428190409c2db47",
"[email protected]",
"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
"[email protected]",
},
},
{
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 12,
Offset: 1130,
}),
[]interface{}{
"e8d3ffab552895c19b9fcf7aa264d277cde33881",
"[email protected]",
"b8e471f58bcbca63b07bda20e428190409c2db47",
"[email protected]",
},
},
}
Expand Down
Loading