Skip to content

Commit 23409f0

Browse files
ashmckenzieIgor Drozdov
ashmckenzie
and
Igor Drozdov
committed
Merge branch 'id-geo-http-push' into 'main'
Perform Git over HTTP request to primary repo See merge request https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/716 Merged-by: Ash McKenzie <[email protected]> Approved-by: Alejandro Rodríguez <[email protected]> Approved-by: Ash McKenzie <[email protected]> Reviewed-by: Valery Sizov <[email protected]> Reviewed-by: Alejandro Rodríguez <[email protected]> Reviewed-by: Igor Drozdov <[email protected]> Reviewed-by: Ash McKenzie <[email protected]> Co-authored-by: Igor Drozdov <[email protected]>
2 parents d893886 + 83a4e8e commit 23409f0

File tree

9 files changed

+431
-7
lines changed

9 files changed

+431
-7
lines changed

‎internal/command/githttp/push.go

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
package githttp
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"fmt"
7+
"io"
8+
9+
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/readwriter"
10+
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/config"
11+
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet/accessverifier"
12+
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet/git"
13+
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/pktline"
14+
)
15+
16+
const service = "git-receive-pack"
17+
18+
var receivePackHttpPrefix = []byte("001f# service=git-receive-pack\n0000")
19+
20+
type PushCommand struct {
21+
Config *config.Config
22+
ReadWriter *readwriter.ReadWriter
23+
Response *accessverifier.Response
24+
}
25+
26+
// See Uploading Data > HTTP(S) section at:
27+
// https://git-scm.com/book/en/v2/Git-Internals-Transfer-Protocols
28+
//
29+
// 1. Perform /info/refs?service=git-receive-pack request
30+
// 2. Remove the header to make it consumable by SSH protocol
31+
// 3. Send the result to the user via SSH (writeToStdout)
32+
// 4. Read the send-pack data provided by user via SSH (stdinReader)
33+
// 5. Perform /git-receive-pack request and send this data
34+
// 6. Return the output to the user
35+
func (c *PushCommand) Execute(ctx context.Context) error {
36+
data := c.Response.Payload.Data
37+
client, err := git.NewClient(c.Config, data.PrimaryRepo, data.RequestHeaders)
38+
if err != nil {
39+
return err
40+
}
41+
42+
if err := c.requestInfoRefs(ctx, client); err != nil {
43+
return err
44+
}
45+
46+
return c.requestReceivePack(ctx, client)
47+
}
48+
49+
func (c *PushCommand) requestInfoRefs(ctx context.Context, client *git.Client) error {
50+
response, err := client.InfoRefs(ctx, service)
51+
if err != nil {
52+
return err
53+
}
54+
defer response.Body.Close()
55+
56+
// Read the first bytes that contain 001f# service=git-receive-pack\n0000 string
57+
// to convert HTTP(S) Git response to the one expected by SSH
58+
p := make([]byte, len(receivePackHttpPrefix))
59+
_, err = response.Body.Read(p)
60+
if err != nil || !bytes.Equal(p, receivePackHttpPrefix) {
61+
return fmt.Errorf("Unexpected git-receive-pack response")
62+
}
63+
64+
_, err = io.Copy(c.ReadWriter.Out, response.Body)
65+
66+
return err
67+
}
68+
69+
func (c *PushCommand) requestReceivePack(ctx context.Context, client *git.Client) error {
70+
pipeReader, pipeWriter := io.Pipe()
71+
go c.readFromStdin(pipeWriter)
72+
73+
response, err := client.ReceivePack(ctx, pipeReader)
74+
if err != nil {
75+
return err
76+
}
77+
defer response.Body.Close()
78+
79+
_, err = io.Copy(c.ReadWriter.Out, response.Body)
80+
81+
return err
82+
}
83+
84+
func (c *PushCommand) readFromStdin(pw *io.PipeWriter) {
85+
var needsPackData bool
86+
87+
scanner := pktline.NewScanner(c.ReadWriter.In)
88+
for scanner.Scan() {
89+
line := scanner.Bytes()
90+
pw.Write(line)
91+
92+
if pktline.IsFlush(line) {
93+
break
94+
}
95+
96+
if !needsPackData && !pktline.IsRefRemoval(line) {
97+
needsPackData = true
98+
}
99+
}
100+
101+
if needsPackData {
102+
io.Copy(pw, c.ReadWriter.In)
103+
}
104+
105+
pw.Close()
106+
}

‎internal/command/githttp/push_test.go

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
package githttp
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"io"
7+
"net/http"
8+
"strings"
9+
"testing"
10+
11+
"github.com/stretchr/testify/require"
12+
13+
"gitlab.com/gitlab-org/gitlab-shell/v14/client/testserver"
14+
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/readwriter"
15+
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/config"
16+
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet/accessverifier"
17+
)
18+
19+
var (
20+
flush = "0000"
21+
infoRefsWithoutPrefix = "00c4e56497bb5f03a90a51293fc6d516788730953899 refs/heads/'test'report-status " +
22+
"report-status-v2 delete-refs side-band-64k quiet atomic ofs-delta push-options object-format=sha1 " +
23+
"agent=git/2.38.3.gl200\n" + flush
24+
)
25+
26+
func TestExecute(t *testing.T) {
27+
url, input := setup(t, http.StatusOK)
28+
output := &bytes.Buffer{}
29+
30+
cmd := &PushCommand{
31+
Config: &config.Config{GitlabUrl: url},
32+
ReadWriter: &readwriter.ReadWriter{Out: output, In: input},
33+
Response: &accessverifier.Response{
34+
Payload: accessverifier.CustomPayload{
35+
Data: accessverifier.CustomPayloadData{PrimaryRepo: url},
36+
},
37+
},
38+
}
39+
40+
require.NoError(t, cmd.Execute(context.Background()))
41+
require.Equal(t, infoRefsWithoutPrefix, output.String())
42+
}
43+
44+
func TestExecuteWithFailedInfoRefs(t *testing.T) {
45+
testCases := []struct {
46+
desc string
47+
statusCode int
48+
responseContent string
49+
expectedErr string
50+
}{
51+
{
52+
desc: "request failed",
53+
statusCode: http.StatusForbidden,
54+
expectedErr: "Internal API error (403)",
55+
}, {
56+
desc: "unexpected response",
57+
statusCode: http.StatusOK,
58+
responseContent: "unexpected response",
59+
expectedErr: "Unexpected git-receive-pack response",
60+
},
61+
}
62+
63+
for _, tc := range testCases {
64+
t.Run(tc.desc, func(t *testing.T) {
65+
requests := []testserver.TestRequestHandler{
66+
{
67+
Path: "/info/refs",
68+
Handler: func(w http.ResponseWriter, r *http.Request) {
69+
require.Equal(t, "git-receive-pack", r.URL.Query().Get("service"))
70+
71+
w.WriteHeader(tc.statusCode)
72+
w.Write([]byte(tc.responseContent))
73+
},
74+
},
75+
}
76+
77+
url := testserver.StartHttpServer(t, requests)
78+
79+
cmd := &PushCommand{
80+
Config: &config.Config{GitlabUrl: url},
81+
Response: &accessverifier.Response{
82+
Payload: accessverifier.CustomPayload{
83+
Data: accessverifier.CustomPayloadData{PrimaryRepo: url},
84+
},
85+
},
86+
}
87+
88+
err := cmd.Execute(context.Background())
89+
require.Error(t, err)
90+
require.Equal(t, tc.expectedErr, err.Error())
91+
})
92+
}
93+
}
94+
95+
func TestExecuteWithFailedReceivePack(t *testing.T) {
96+
url, input := setup(t, http.StatusForbidden)
97+
output := &bytes.Buffer{}
98+
99+
cmd := &PushCommand{
100+
Config: &config.Config{GitlabUrl: url},
101+
ReadWriter: &readwriter.ReadWriter{Out: output, In: input},
102+
Response: &accessverifier.Response{
103+
Payload: accessverifier.CustomPayload{
104+
Data: accessverifier.CustomPayloadData{PrimaryRepo: url},
105+
},
106+
},
107+
}
108+
109+
err := cmd.Execute(context.Background())
110+
require.Error(t, err)
111+
require.Equal(t, "Internal API error (403)", err.Error())
112+
}
113+
114+
func setup(t *testing.T, receivePackStatusCode int) (string, io.Reader) {
115+
infoRefs := "001f# service=git-receive-pack\n" + flush + infoRefsWithoutPrefix
116+
receivePackPrefix := "00ab4c9d98d7750fa65db8ddcc60a89ef919f7a179f9 df505c066e4e63a801268a84627d7e8f7e033c7a " +
117+
"refs/heads/main123 report-status-v2 side-band-64k object-format=sha1 agent=git/2.39.1"
118+
receivePackData := "PACK some data"
119+
120+
// Imitate sending data via multiple packets
121+
input := io.MultiReader(
122+
strings.NewReader(receivePackPrefix),
123+
strings.NewReader(flush),
124+
strings.NewReader(receivePackData),
125+
)
126+
127+
requests := []testserver.TestRequestHandler{
128+
{
129+
Path: "/info/refs",
130+
Handler: func(w http.ResponseWriter, r *http.Request) {
131+
require.Equal(t, "git-receive-pack", r.URL.Query().Get("service"))
132+
133+
w.Write([]byte(infoRefs))
134+
},
135+
},
136+
{
137+
Path: "/git-receive-pack",
138+
Handler: func(w http.ResponseWriter, r *http.Request) {
139+
body, err := io.ReadAll(r.Body)
140+
require.NoError(t, err)
141+
defer r.Body.Close()
142+
143+
require.Equal(t, receivePackPrefix+flush+receivePackData, string(body))
144+
w.WriteHeader(receivePackStatusCode)
145+
},
146+
},
147+
}
148+
149+
return testserver.StartHttpServer(t, requests), input
150+
}

‎internal/command/receivepack/receivepack.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55

66
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/commandargs"
7+
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/githttp"
78
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/readwriter"
89
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/shared/accessverifier"
910
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/command/shared/customaction"
@@ -30,6 +31,20 @@ func (c *Command) Execute(ctx context.Context) error {
3031
}
3132

3233
if response.IsCustomAction() {
34+
// When `geo_proxy_direct_to_primary` feature flag is enabled, a Git over HTTP direct request
35+
// to primary repo is performed instead of proxying the request through Gitlab Rails.
36+
// After the feature flag is enabled by default and removed,
37+
// custom action functionality will be removed along with it.
38+
if response.Payload.Data.GeoProxyDirectToPrimary {
39+
cmd := githttp.PushCommand{
40+
Config: c.Config,
41+
ReadWriter: c.ReadWriter,
42+
Response: response,
43+
}
44+
45+
return cmd.Execute(ctx)
46+
}
47+
3348
customAction := customaction.Command{
3449
Config: c.Config,
3550
ReadWriter: c.ReadWriter,

‎internal/command/shared/customaction/customaction.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@ type Command struct {
3434
EOFSent bool
3535
}
3636

37+
// When `geo_proxy_direct_to_primary` feature flag is enabled, a Git over HTTP direct request
38+
// to primary repo is performed instead of proxying the request through Gitlab Rails.
39+
// After the feature flag is enabled by default and removed, this package will be removed along with it.
3740
func (c *Command) Execute(ctx context.Context, response *accessverifier.Response) error {
3841
data := response.Payload.Data
3942
apiEndpoints := data.ApiEndpoints

‎internal/gitlabnet/accessverifier/client.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,12 @@ type Gitaly struct {
4040
}
4141

4242
type CustomPayloadData struct {
43-
ApiEndpoints []string `json:"api_endpoints"`
44-
Username string `json:"gl_username"`
45-
PrimaryRepo string `json:"primary_repo"`
46-
UserId string `json:"gl_id,omitempty"`
43+
ApiEndpoints []string `json:"api_endpoints"`
44+
Username string `json:"gl_username"`
45+
PrimaryRepo string `json:"primary_repo"`
46+
UserId string `json:"gl_id,omitempty"`
47+
RequestHeaders map[string]string `json:"request_headers"`
48+
GeoProxyDirectToPrimary bool `json:"geo_proxy_direct_to_primary"`
4749
}
4850

4951
type CustomPayload struct {

‎internal/gitlabnet/accessverifier/client_test.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,9 +107,11 @@ func TestGeoPushGetCustomAction(t *testing.T) {
107107
response.Payload = CustomPayload{
108108
Action: "geo_proxy_to_primary",
109109
Data: CustomPayloadData{
110-
ApiEndpoints: []string{"geo/proxy_git_ssh/info_refs_receive_pack", "geo/proxy_git_ssh/receive_pack"},
111-
Username: "custom",
112-
PrimaryRepo: "https://repo/path",
110+
ApiEndpoints: []string{"geo/proxy_git_ssh/info_refs_receive_pack", "geo/proxy_git_ssh/receive_pack"},
111+
GeoProxyDirectToPrimary: true,
112+
RequestHeaders: map[string]string{"Authorization": "Bearer token"},
113+
Username: "custom",
114+
PrimaryRepo: "https://repo/path",
113115
},
114116
}
115117
response.StatusCode = 300

‎internal/gitlabnet/git/client.go

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
package git
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"net/http"
8+
9+
"gitlab.com/gitlab-org/gitlab-shell/v14/client"
10+
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/config"
11+
"gitlab.com/gitlab-org/gitlab-shell/v14/internal/gitlabnet"
12+
)
13+
14+
type Client struct {
15+
url string
16+
headers map[string]string
17+
client *client.GitlabNetClient
18+
}
19+
20+
func NewClient(cfg *config.Config, url string, headers map[string]string) (*Client, error) {
21+
client, err := gitlabnet.GetClient(cfg)
22+
if err != nil {
23+
return nil, fmt.Errorf("Error creating http client: %v", err)
24+
}
25+
26+
return &Client{client: client, headers: headers, url: url}, nil
27+
}
28+
29+
func (c *Client) InfoRefs(ctx context.Context, service string) (*http.Response, error) {
30+
request, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url+"/info/refs?service="+service, nil)
31+
if err != nil {
32+
return nil, err
33+
}
34+
35+
return c.do(request)
36+
}
37+
38+
func (c *Client) ReceivePack(ctx context.Context, body io.Reader) (*http.Response, error) {
39+
request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url+"/git-receive-pack", body)
40+
if err != nil {
41+
return nil, err
42+
}
43+
request.Header.Add("Content-Type", "application/x-git-receive-pack-request")
44+
request.Header.Add("Accept", "application/x-git-receive-pack-result")
45+
46+
return c.do(request)
47+
}
48+
49+
func (c *Client) do(request *http.Request) (*http.Response, error) {
50+
51+
for k, v := range c.headers {
52+
request.Header.Add(k, v)
53+
}
54+
55+
return c.client.Do(request)
56+
}

0 commit comments

Comments
 (0)