Skip to content

Commit ed58d7a

Browse files
committed
Replicate git lfs (pre-)merge hook
This switches from relying on having git-lfs installed on the server, (and in fact .gitattributes being correctly installed.) Instead on merge we walk the merge history and ensure that all lfs objects pointed to in the history are added to the base repository.
1 parent 4a16c7f commit ed58d7a

File tree

1 file changed

+176
-23
lines changed

1 file changed

+176
-23
lines changed

modules/merge/merge.go

Lines changed: 176 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,16 @@ import (
99
"bufio"
1010
"bytes"
1111
"fmt"
12+
"io"
1213
"io/ioutil"
1314
"os"
1415
"path"
1516
"path/filepath"
17+
"strconv"
1618
"strings"
19+
"sync"
20+
21+
"code.gitea.io/gitea/modules/lfs"
1722

1823
"code.gitea.io/gitea/models"
1924
"code.gitea.io/gitea/modules/cache"
@@ -119,34 +124,22 @@ func Merge(pr *models.PullRequest, doer *models.User, baseGitRepo *git.Repositor
119124
return fmt.Errorf("Writing sparse-checkout file to %s: %v", sparseCheckoutListPath, err)
120125
}
121126

122-
if err := git.NewCommand("config", "--local", "core.sparseCheckout", "true").RunInDirPipeline(tmpBasePath, nil, &errbuf); err != nil {
123-
return fmt.Errorf("git config [core.sparsecheckout -> true]: %v", errbuf.String())
124-
}
125-
126-
originLFSURL := setting.LocalURL + pr.HeadRepo.FullName() + ".git/info/lfs"
127-
if err := git.NewCommand("config", "--local", "remote.origin.lfsurl", originLFSURL).RunInDirPipeline(tmpBasePath, nil, &errbuf); err != nil {
128-
return fmt.Errorf("git config [remote.origin.lfsurl]: %v", errbuf.String())
127+
// Switch off LFS process (set required, clean and smudge here also)
128+
if err := git.NewCommand("config", "--local", "filter.lfs.process", "").RunInDirPipeline(tmpBasePath, nil, &errbuf); err != nil {
129+
return fmt.Errorf("git config [filter.lfs.process -> <> ]: %v", errbuf.String())
129130
}
130-
131-
originToken, err := models.GenerateLFSAuthenticationToken(doer, pr.HeadRepo, "upload")
132-
if err != nil {
133-
return fmt.Errorf("Failed to generate authentication token for lfs")
131+
if err := git.NewCommand("config", "--local", "filter.lfs.required", "false").RunInDirPipeline(tmpBasePath, nil, &errbuf); err != nil {
132+
return fmt.Errorf("git config [filter.lfs.required -> <false> ]: %v", errbuf.String())
134133
}
135-
if err := git.NewCommand("config", "--local", "http."+originLFSURL+".extraheader", "Authorization: "+originToken.Header["Authorization"]).RunInDirPipeline(tmpBasePath, nil, &errbuf); err != nil {
136-
return fmt.Errorf("git config [http.origin.extraheader]: %v", errbuf.String())
134+
if err := git.NewCommand("config", "--local", "filter.lfs.clean", "").RunInDirPipeline(tmpBasePath, nil, &errbuf); err != nil {
135+
return fmt.Errorf("git config [filter.lfs.clean -> <> ]: %v", errbuf.String())
137136
}
138-
139-
remoteLFSURL := setting.LocalURL + pr.BaseRepo.FullName() + ".git/info/lfs"
140-
if err := git.NewCommand("config", "--local", "remote."+remoteRepoName+".lfsurl", remoteLFSURL).RunInDirPipeline(tmpBasePath, nil, &errbuf); err != nil {
141-
return fmt.Errorf("git config [remote.remote.lfsurl]: %v", errbuf.String())
137+
if err := git.NewCommand("config", "--local", "filter.lfs.smudge", "").RunInDirPipeline(tmpBasePath, nil, &errbuf); err != nil {
138+
return fmt.Errorf("git config [filter.lfs.smudge -> <> ]: %v", errbuf.String())
142139
}
143140

144-
remoteToken, err := models.GenerateLFSAuthenticationToken(doer, pr.BaseRepo, "upload")
145-
if err != nil {
146-
return fmt.Errorf("Failed to generate authentication token for lfs")
147-
}
148-
if err := git.NewCommand("config", "--local", "http."+remoteLFSURL+".extraheader", "Authorization: "+remoteToken.Header["Authorization"]).RunInDirPipeline(tmpBasePath, nil, &errbuf); err != nil {
149-
return fmt.Errorf("git config [http.remote.extraheader]: %v", errbuf.String())
141+
if err := git.NewCommand("config", "--local", "core.sparseCheckout", "true").RunInDirPipeline(tmpBasePath, nil, &errbuf); err != nil {
142+
return fmt.Errorf("git config [core.sparsecheckout -> true]: %v", errbuf.String())
150143
}
151144

152145
// Read base branch index
@@ -219,6 +212,166 @@ func Merge(pr *models.PullRequest, doer *models.User, baseGitRepo *git.Repositor
219212
return models.ErrInvalidMergeStyle{ID: pr.BaseRepo.ID, Style: mergeStyle}
220213
}
221214

215+
if setting.LFS.StartServer {
216+
// Now we have to implement git lfs pre-push
217+
// git rev-list --objects --filter=blob:limit=1k HEAD --not base
218+
// pass blob shas in to git cat-file --batch-check (possibly unnecessary)
219+
// ensure only blobs and <=1k size then pass in to git cat-file --batch
220+
// to read each sha and check each as a pointer
221+
// Then if they are lfs -> add them to the baseRepo
222+
revListReader, revListWriter := io.Pipe()
223+
shasToCheckReader, shasToCheckWriter := io.Pipe()
224+
catFileCheckReader, catFileCheckWriter := io.Pipe()
225+
shasToBatchReader, shasToBatchWriter := io.Pipe()
226+
catFileBatchReader, catFileBatchWriter := io.Pipe()
227+
errChan := make(chan error, 1)
228+
wg := sync.WaitGroup{}
229+
wg.Add(6)
230+
go func() {
231+
defer wg.Done()
232+
defer catFileBatchReader.Close()
233+
234+
bufferedReader := bufio.NewReader(catFileBatchReader)
235+
buf := make([]byte, 1025)
236+
for {
237+
// File descriptor line
238+
_, err := bufferedReader.ReadString(' ')
239+
if err != nil {
240+
catFileBatchReader.CloseWithError(err)
241+
break
242+
}
243+
// Throw away the blob
244+
if _, err := bufferedReader.ReadString(' '); err != nil {
245+
catFileBatchReader.CloseWithError(err)
246+
break
247+
}
248+
sizeStr, err := bufferedReader.ReadString('\n')
249+
if err != nil {
250+
catFileBatchReader.CloseWithError(err)
251+
break
252+
}
253+
size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1])
254+
if err != nil {
255+
catFileBatchReader.CloseWithError(err)
256+
break
257+
}
258+
pointerBuf := buf[:size+1]
259+
if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil {
260+
catFileBatchReader.CloseWithError(err)
261+
break
262+
}
263+
pointerBuf = pointerBuf[:size]
264+
// now we need to check if the pointerBuf is an LFS pointer
265+
pointer := lfs.IsPointerFile(&pointerBuf)
266+
if pointer == nil {
267+
continue
268+
}
269+
pointer.RepositoryID = pr.BaseRepoID
270+
if _, err := models.NewLFSMetaObject(pointer); err != nil {
271+
catFileBatchReader.CloseWithError(err)
272+
break
273+
}
274+
275+
}
276+
}()
277+
go func() {
278+
defer wg.Done()
279+
defer shasToBatchReader.Close()
280+
defer catFileBatchWriter.Close()
281+
282+
stderr := new(bytes.Buffer)
283+
if err := git.NewCommand("cat-file", "--batch").RunInDirFullPipeline(tmpBasePath, catFileBatchWriter, stderr, shasToBatchReader); err != nil {
284+
shasToBatchReader.CloseWithError(fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String()))
285+
}
286+
}()
287+
go func() {
288+
defer wg.Done()
289+
defer catFileCheckReader.Close()
290+
291+
scanner := bufio.NewScanner(catFileCheckReader)
292+
defer shasToBatchWriter.CloseWithError(scanner.Err())
293+
for scanner.Scan() {
294+
line := scanner.Text()
295+
if len(line) == 0 {
296+
continue
297+
}
298+
fields := strings.Split(line, " ")
299+
if len(fields) < 3 || fields[1] != "blob" {
300+
continue
301+
}
302+
size, _ := strconv.Atoi(string(fields[2]))
303+
if size > 1024 {
304+
continue
305+
}
306+
toWrite := []byte(fields[0] + "\n")
307+
for len(toWrite) > 0 {
308+
n, err := shasToBatchWriter.Write(toWrite)
309+
if err != nil {
310+
catFileCheckReader.CloseWithError(err)
311+
break
312+
}
313+
toWrite = toWrite[n:]
314+
}
315+
}
316+
}()
317+
go func() {
318+
defer wg.Done()
319+
defer shasToCheckReader.Close()
320+
defer catFileCheckWriter.Close()
321+
322+
stderr := new(bytes.Buffer)
323+
cmd := git.NewCommand("cat-file", "--batch-check")
324+
if err := cmd.RunInDirFullPipeline(tmpBasePath, catFileCheckWriter, stderr, shasToCheckReader); err != nil {
325+
shasToCheckWriter.CloseWithError(fmt.Errorf("git cat-file --batch-check [%s]: %v - %s", tmpBasePath, err, errbuf.String()))
326+
}
327+
}()
328+
go func() {
329+
defer wg.Done()
330+
defer revListReader.Close()
331+
defer shasToCheckWriter.Close()
332+
scanner := bufio.NewScanner(revListReader)
333+
for scanner.Scan() {
334+
line := scanner.Text()
335+
if len(line) == 0 {
336+
continue
337+
}
338+
fields := strings.Split(line, " ")
339+
if len(fields) < 2 || len(fields[1]) == 0 {
340+
continue
341+
}
342+
toWrite := []byte(fields[0] + "\n")
343+
for len(toWrite) > 0 {
344+
n, err := shasToCheckWriter.Write(toWrite)
345+
if err != nil {
346+
revListReader.CloseWithError(err)
347+
break
348+
}
349+
toWrite = toWrite[n:]
350+
}
351+
}
352+
shasToCheckWriter.CloseWithError(scanner.Err())
353+
}()
354+
go func() {
355+
defer wg.Done()
356+
defer revListWriter.Close()
357+
stderr := new(bytes.Buffer)
358+
cmd := git.NewCommand("rev-list", "--objects", "--filter=blob:limit=1k", "HEAD", "--not", "origin/"+pr.BaseBranch)
359+
if err := cmd.RunInDirPipeline(tmpBasePath, revListWriter, stderr); err != nil {
360+
log.Error("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String())
361+
errChan <- fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String())
362+
}
363+
}()
364+
365+
wg.Wait()
366+
select {
367+
case err, has := <-errChan:
368+
if has {
369+
return err
370+
}
371+
default:
372+
}
373+
}
374+
222375
env := models.PushingEnvironment(doer, pr.BaseRepo)
223376

224377
// Push back to upstream.

0 commit comments

Comments
 (0)