From d888c9f32469df0b6fa734f0862d1d41f9200202 Mon Sep 17 00:00:00 2001 From: Chris Gavin Date: Thu, 27 Aug 2020 17:29:47 +0100 Subject: [PATCH] Implement reference pruning on push. --- internal/push/push.go | 45 +++++++++++++----- internal/push/push_test.go | 29 +++++++++++ .../action-cache-initial/git/packed-refs | 1 + .../push_test/action-cache-modified/git/HEAD | 1 + .../action-cache-modified/git/config | 4 ++ .../26/936381e619a01122ea33993e3cebc474496805 | Bin 0 -> 527 bytes .../28/509716746a9e77e3f752e9df740cb443bf9960 | Bin 0 -> 66 bytes .../4b/825dc642cb6eb9a060e54bf8d69288fbee4904 | Bin 0 -> 15 bytes .../61/8dc8ffee7af67e4e2dfdbe416c53cca6099fab | Bin 0 -> 61 bytes .../80/8d4d2d6b7aa39089b99ca0d89f1fc6a2bc63ec | Bin 0 -> 45 bytes .../86/e83156c07aea03c86a7cd7c5f6b0ee1453c2d7 | Bin 0 -> 45 bytes .../8e/38374296a8ca208f5e526da24c5b09f3f48624 | Bin 0 -> 39 bytes .../b9/f01aa2c50f49898d4c7845a66be8824499fe9d | Bin 0 -> 560 bytes .../bd/82b85707bc13904e3526517677039d4da4a9bb | Bin 0 -> 564 bytes .../cd/0f803f72414d81157588de0cb622f47f4bb9bc | Bin 0 -> 58 bytes .../d1/231c269d15582e06ded28babdb8c224ba7e6ed | Bin 0 -> 45 bytes .../e5/29a54fad10a936308b2220e05f7f00757f8e7c | Bin 0 -> 595 bytes .../e5/f2bcf5a8815c924448ff3a54455983df118ee4 | Bin 0 -> 57 bytes .../e7/014e2af66d5b702946763455de2dcf9eb93c4f | Bin 0 -> 58 bytes .../action-cache-modified/git/packed-refs | 10 ++++ .../action-cache-modified/git/refs/.gitkeep | 0 .../codeql-bundle-20200101/assets/bundle.bin | 1 + .../codeql-bundle-20200101/metadata.json | 4 ++ .../codeql-bundle-20200630/assets/bundle.bin | 1 + .../codeql-bundle-20200630/metadata.json | 4 ++ 25 files changed, 88 insertions(+), 12 deletions(-) create mode 100644 internal/push/push_test/action-cache-modified/git/HEAD create mode 100644 internal/push/push_test/action-cache-modified/git/config create mode 100644 internal/push/push_test/action-cache-modified/git/objects/26/936381e619a01122ea33993e3cebc474496805 create mode 100644 internal/push/push_test/action-cache-modified/git/objects/28/509716746a9e77e3f752e9df740cb443bf9960 create mode 100644 internal/push/push_test/action-cache-modified/git/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 create mode 100644 internal/push/push_test/action-cache-modified/git/objects/61/8dc8ffee7af67e4e2dfdbe416c53cca6099fab create mode 100644 internal/push/push_test/action-cache-modified/git/objects/80/8d4d2d6b7aa39089b99ca0d89f1fc6a2bc63ec create mode 100644 internal/push/push_test/action-cache-modified/git/objects/86/e83156c07aea03c86a7cd7c5f6b0ee1453c2d7 create mode 100644 internal/push/push_test/action-cache-modified/git/objects/8e/38374296a8ca208f5e526da24c5b09f3f48624 create mode 100644 internal/push/push_test/action-cache-modified/git/objects/b9/f01aa2c50f49898d4c7845a66be8824499fe9d create mode 100644 internal/push/push_test/action-cache-modified/git/objects/bd/82b85707bc13904e3526517677039d4da4a9bb create mode 100644 internal/push/push_test/action-cache-modified/git/objects/cd/0f803f72414d81157588de0cb622f47f4bb9bc create mode 100644 internal/push/push_test/action-cache-modified/git/objects/d1/231c269d15582e06ded28babdb8c224ba7e6ed create mode 100644 internal/push/push_test/action-cache-modified/git/objects/e5/29a54fad10a936308b2220e05f7f00757f8e7c create mode 100644 internal/push/push_test/action-cache-modified/git/objects/e5/f2bcf5a8815c924448ff3a54455983df118ee4 create mode 100644 internal/push/push_test/action-cache-modified/git/objects/e7/014e2af66d5b702946763455de2dcf9eb93c4f create mode 100644 internal/push/push_test/action-cache-modified/git/packed-refs create mode 100644 internal/push/push_test/action-cache-modified/git/refs/.gitkeep create mode 100644 internal/push/push_test/action-cache-modified/releases/codeql-bundle-20200101/assets/bundle.bin create mode 100644 internal/push/push_test/action-cache-modified/releases/codeql-bundle-20200101/metadata.json create mode 100644 internal/push/push_test/action-cache-modified/releases/codeql-bundle-20200630/assets/bundle.bin create mode 100644 internal/push/push_test/action-cache-modified/releases/codeql-bundle-20200630/metadata.json diff --git a/internal/push/push.go b/internal/push/push.go index ee6ee4e..f8a6366 100644 --- a/internal/push/push.go +++ b/internal/push/push.go @@ -14,6 +14,8 @@ import ( "path/filepath" "strings" + "github.com/go-git/go-git/v5/plumbing" + "github.com/github/codeql-action-sync/internal/githubapiutil" log "github.com/sirupsen/logrus" @@ -22,6 +24,7 @@ import ( "github.com/github/codeql-action-sync/internal/version" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing/transport" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/google/go-github/v32/github" "github.com/mitchellh/ioprogress" @@ -142,35 +145,53 @@ func (pushService *pushService) pushGit(repository *github.Repository, initialPu } refSpecBatches := [][]config.RefSpec{} + remoteReferences, err := remote.List(&git.ListOptions{Auth: credentials}) + if err != nil && err != transport.ErrEmptyRemoteRepository { + return errors.Wrap(err, "Error listing remote references.") + } + deleteRefSpecs := []config.RefSpec{} + for _, remoteReference := range remoteReferences { + _, err := gitRepository.Reference(remoteReference.Name(), false) + if err != nil && err != plumbing.ErrReferenceNotFound { + return errors.Wrapf(err, "Error finding local reference %s.", remoteReference.Name()) + } + if err == plumbing.ErrReferenceNotFound { + deleteRefSpecs = append(deleteRefSpecs, config.RefSpec(":"+remoteReference.Name().String())) + } + } + refSpecBatches = append(refSpecBatches, deleteRefSpecs) + if initialPush { releasePathStats, err := ioutil.ReadDir(pushService.cacheDirectory.ReleasesPath()) if err != nil { return errors.Wrap(err, "Error reading releases.") } - refSpecBatches = append(refSpecBatches, []config.RefSpec{}) + initialRefSpecs := []config.RefSpec{} for _, releasePathStat := range releasePathStats { - refSpecBatches[0] = append(refSpecBatches[0], config.RefSpec("+refs/tags/"+releasePathStat.Name()+":refs/tags/"+releasePathStat.Name())) + initialRefSpecs = append(initialRefSpecs, config.RefSpec("+refs/tags/"+releasePathStat.Name()+":refs/tags/"+releasePathStat.Name())) } + refSpecBatches = append(refSpecBatches, initialRefSpecs) } else { // We've got to push `main` on its own, so that it will be made the default branch if the repository has just been created. We then push everything else afterwards. - refSpecBatches = [][]config.RefSpec{ + refSpecBatches = append(refSpecBatches, []config.RefSpec{ config.RefSpec("+refs/heads/main:refs/heads/main"), }, []config.RefSpec{ config.RefSpec("+refs/*:refs/*"), }, - } + ) } for _, refSpecs := range refSpecBatches { - err = remote.PushContext(pushService.ctx, &git.PushOptions{ - RefSpecs: refSpecs, - Auth: credentials, - Progress: os.Stderr, - Force: true, - }) - if err != nil && errors.Cause(err) != git.NoErrAlreadyUpToDate { - return errors.Wrap(err, "Error pushing Action to GitHub Enterprise Server.") + if len(refSpecs) != 0 { + err = remote.PushContext(pushService.ctx, &git.PushOptions{ + RefSpecs: refSpecs, + Auth: credentials, + Progress: os.Stderr, + }) + if err != nil && errors.Cause(err) != git.NoErrAlreadyUpToDate { + return errors.Wrap(err, "Error pushing Action to GitHub Enterprise Server.") + } } } diff --git a/internal/push/push_test.go b/internal/push/push_test.go index aecc753..2487224 100644 --- a/internal/push/push_test.go +++ b/internal/push/push_test.go @@ -150,6 +150,35 @@ func TestPushGit(t *testing.T) { "bd82b85707bc13904e3526517677039d4da4a9bb refs/heads/very-ignored-branch", "bd82b85707bc13904e3526517677039d4da4a9bb refs/tags/an-ignored-tag-too", "26936381e619a01122ea33993e3cebc474496805 refs/tags/v2", + "26936381e619a01122ea33993e3cebc474496805 refs/heads/a-ref-that-will-need-pruning", + }) + + pushService = getTestPushService(t, "./push_test/action-cache-modified/", "") + err = pushService.pushGit(&repository, true) + require.NoError(t, err) + test.CheckExpectedReferencesInRepository(t, destinationPath, []string{ + "26936381e619a01122ea33993e3cebc474496805 refs/tags/codeql-bundle-20200101", + "26936381e619a01122ea33993e3cebc474496805 refs/tags/codeql-bundle-20200630", + "b9f01aa2c50f49898d4c7845a66be8824499fe9d refs/heads/main", + "26936381e619a01122ea33993e3cebc474496805 refs/heads/v1", + "e529a54fad10a936308b2220e05f7f00757f8e7c refs/heads/v3", + "bd82b85707bc13904e3526517677039d4da4a9bb refs/heads/very-ignored-branch", + "bd82b85707bc13904e3526517677039d4da4a9bb refs/tags/an-ignored-tag-too", + "26936381e619a01122ea33993e3cebc474496805 refs/tags/v2", + }) + + err = pushService.pushGit(&repository, false) + require.NoError(t, err) + test.CheckExpectedReferencesInRepository(t, destinationPath, []string{ + "26936381e619a01122ea33993e3cebc474496805 refs/tags/codeql-bundle-20200101", + "26936381e619a01122ea33993e3cebc474496805 refs/tags/codeql-bundle-20200630", + "b9f01aa2c50f49898d4c7845a66be8824499fe9d refs/heads/main", + "26936381e619a01122ea33993e3cebc474496805 refs/heads/v1", + "e529a54fad10a936308b2220e05f7f00757f8e7c refs/heads/v3", + "bd82b85707bc13904e3526517677039d4da4a9bb refs/heads/very-ignored-branch", + "bd82b85707bc13904e3526517677039d4da4a9bb refs/tags/an-ignored-tag-too", + "26936381e619a01122ea33993e3cebc474496805 refs/tags/v2", + "26936381e619a01122ea33993e3cebc474496805 refs/heads/a-ref-that-will-need-pruning/because-it-now-has-this-extra-bit", }) } diff --git a/internal/push/push_test/action-cache-initial/git/packed-refs b/internal/push/push_test/action-cache-initial/git/packed-refs index 6376d60..dc98aa8 100644 --- a/internal/push/push_test/action-cache-initial/git/packed-refs +++ b/internal/push/push_test/action-cache-initial/git/packed-refs @@ -1,4 +1,5 @@ # pack-refs with: peeled fully-peeled sorted +26936381e619a01122ea33993e3cebc474496805 refs/heads/a-ref-that-will-need-pruning b9f01aa2c50f49898d4c7845a66be8824499fe9d refs/heads/main 26936381e619a01122ea33993e3cebc474496805 refs/heads/v1 e529a54fad10a936308b2220e05f7f00757f8e7c refs/heads/v3 diff --git a/internal/push/push_test/action-cache-modified/git/HEAD b/internal/push/push_test/action-cache-modified/git/HEAD new file mode 100644 index 0000000..b870d82 --- /dev/null +++ b/internal/push/push_test/action-cache-modified/git/HEAD @@ -0,0 +1 @@ +ref: refs/heads/main diff --git a/internal/push/push_test/action-cache-modified/git/config b/internal/push/push_test/action-cache-modified/git/config new file mode 100644 index 0000000..07d359d --- /dev/null +++ b/internal/push/push_test/action-cache-modified/git/config @@ -0,0 +1,4 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = true diff --git a/internal/push/push_test/action-cache-modified/git/objects/26/936381e619a01122ea33993e3cebc474496805 b/internal/push/push_test/action-cache-modified/git/objects/26/936381e619a01122ea33993e3cebc474496805 new file mode 100644 index 0000000000000000000000000000000000000000..7fbf639faf11e0cc81843ccf7e36345f79c2d3dc GIT binary patch literal 527 zcmV+q0`UEK0iBUcZ<|0Ag}dfg%pSGo%^)>Zu?^S|NDRgV?|>PG#{dZo`0?wTSn95; z-o=sbr*rRFoMPy@ngPv#-`I#TfC5Acl8~0`Stc6-G;+fzG66xO6w5-(q`c5%TyWXU zs$m41YOE*VWJ}$HpT*bm`=R8E{OIs6AgYG0E4r$wfEPqT;EG2i!~bkToHU0ascc7AW&BU<~2IZ?|mvj(LSA$kw)QqyJ!$7}h4+c!SnVtx0P zW;ceT)yRR)xEps&`zh6=@#?Bc??cx%5`Pr8{Eo&7DvWGhI4)k6;gEDNPf97s5|`Qq zBeB8(i&b)T^w1(k$$;6h>Q-U1N9}CJqu23q*7z?UVJ^!f9c&ACq*s_at|SG9yl!nw z4n#Rr^ybnzRC_*;tU14++rAv%V2;9`JUwyX>D9k{8XxUh!S-LBawbBc?rT7VW?J0D$PsDNexRaD$dN$Q?gP}D$dVM)lJS%NiEFLErST_ Y=I7~_8R{nHrRbI!Db;cT0G&P+4CNpmCjbBd literal 0 HcmV?d00001 diff --git a/internal/push/push_test/action-cache-modified/git/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 b/internal/push/push_test/action-cache-modified/git/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 new file mode 100644 index 0000000000000000000000000000000000000000..adf64119a33d7621aeeaa505d30adb58afaa5559 GIT binary patch literal 15 Wcmb0V^p=O;s>9U@$QN0)^tDWQMc+4faKjzKx=#9rt*)DSfH;-nj<=;U*2U DlKK0V^p=O;s>9U@$QN0)^tDWQOOAep=shqYE_M%1lD<>7Jjr)5aeF<9iKA Dj=~eb literal 0 HcmV?d00001 diff --git a/internal/push/push_test/action-cache-modified/git/objects/8e/38374296a8ca208f5e526da24c5b09f3f48624 b/internal/push/push_test/action-cache-modified/git/objects/8e/38374296a8ca208f5e526da24c5b09f3f48624 new file mode 100644 index 0000000000000000000000000000000000000000..e6fc9ad602b75f4131420eb59df3af03dcbc502a GIT binary patch literal 39 vcmbQWR?+efEa=ED;Cu2BetvrX zLB7@O2aq&^v5FCdUVCQ^t0SJKm z&eJpu)5?u>&K(Dp_2xoihl?{B9RkG^D}OMii{>_SqtbDAG7Kv({l8oQntLnVDVuqb z8cs(E^=x&Bi)FKz21}j6-GqqS$ChU*+3af0G3}q;YWp^=u;xtB6$>DV^d)Ij;%qPd z*$L%R)b9LP*GlWHJF)iO8#HQR6_k#O+!b~=cdrz)xp8ds)n%*L7sI zQE@EuLCmW~TxW;jHVgJ@Yk2uxS>>eL`(0KuKH2yo=S3(-_S^3ZhasHT>P@x>JA6F+ zBG(J&&8y~3f7uQDIO0Qh*d12~Q*-j56kj=M^*A(NpY18~$l@V@+Jb8_G8Pa{;KaPpJkqa7l_>?G6oFDbn7eMSc#rx}6)`b?`CRv`w y{r1k2Ez$gUXuP}#;Dwy6weV+E$F={psSsRK+B1-{j1E=bfs9w|oA3+1ljv(|Bo=}I literal 0 HcmV?d00001 diff --git a/internal/push/push_test/action-cache-modified/git/objects/bd/82b85707bc13904e3526517677039d4da4a9bb b/internal/push/push_test/action-cache-modified/git/objects/bd/82b85707bc13904e3526517677039d4da4a9bb new file mode 100644 index 0000000000000000000000000000000000000000..b443de75db45b99ec376360bf073a3396faf0039 GIT binary patch literal 564 zcmV-40?Yk)0iBY~Zks?9g}de{W{(3>fgqmQa}_qQ*wwfQ8dkfR7QjnXk=+dG9!mj%~G8rN`x-$ zDIYZ7QiL*U7*JI)E+{5KdL2(~5A0LV2H>VQ#=(#LbNTh9 z$mQ)8<^POv6yD)1*ABbbxNG_>g`y;nsy1GW^3meP(K3PrguUlL^s&2Gfx9|?mVG_W-ZxCB`nEEoF z;Suto}UHqOpP}r{Atzk=Ktzc*rn%>ccw*pt;YcrtfqkRTUs;nB>V=vPwMV+ C=oCQ! literal 0 HcmV?d00001 diff --git a/internal/push/push_test/action-cache-modified/git/objects/cd/0f803f72414d81157588de0cb622f47f4bb9bc b/internal/push/push_test/action-cache-modified/git/objects/cd/0f803f72414d81157588de0cb622f47f4bb9bc new file mode 100644 index 0000000000000000000000000000000000000000..8ae695548a8b3f59540f9273a4bc554344a50e97 GIT binary patch literal 58 zcmV-A0LA}!0V^p=O;s>4WH2-^Ff%bxNJ&jgEX^q?*2^l+&tuRCm@Zb5HLv{f_n?>e QOL(?8@1L0f0Fbp40A-XH;{X5v literal 0 HcmV?d00001 diff --git a/internal/push/push_test/action-cache-modified/git/objects/d1/231c269d15582e06ded28babdb8c224ba7e6ed b/internal/push/push_test/action-cache-modified/git/objects/d1/231c269d15582e06ded28babdb8c224ba7e6ed new file mode 100644 index 0000000000000000000000000000000000000000..9dd5dcbdbb207b867faf7b4fa331df4ec0cdff11 GIT binary patch literal 45 zcmb4gBWpzyjFfILyXfXybu~a=v4XCDxffE}khJtmN$86%{ox{^cNvaJlJrUi9m7zws46++w2v7JfiAqv(4w`eH`bJWoW{765J zU!R+Nkf$2|1Y_McO=y}L1hO(#6e<0DQQ-dylN;fbfbSnO@#cO2cJmzw{dwRVPY3Ul zOF#m2?+p{jnYhHsX4LbHvfeay;?a?e;^Rr1jH>;^=^H&@GpDjlUc`<|%Kcw10X=B5 zG0Ao3P;g&+h}6Uh5^2=Q=NNLD?*i5s3prwEG8M6RyH|xHj!3Dy`S)5L=4sv30r%H(1|zw_c_Bh4Jb|GhJ_WGZM`WCE%1i&1+dd z1pC`K*=+e*776ZV!FJT&<;CGdcboNQhz`4`>MB$)rmTi6=6z|bD>bks(A4)ntXz-P z%h1R1v<_v=_FZtEF=MfPjfcPq5%nLPsW4WH2-^Ff%bxNJ&jgEX^q?*2^l+&tvGbFn5}^;*>&vTu|;J PpJ>j{U)odvhH(+QPWBhD literal 0 HcmV?d00001 diff --git a/internal/push/push_test/action-cache-modified/git/objects/e7/014e2af66d5b702946763455de2dcf9eb93c4f b/internal/push/push_test/action-cache-modified/git/objects/e7/014e2af66d5b702946763455de2dcf9eb93c4f new file mode 100644 index 0000000000000000000000000000000000000000..c06b61cb1b3547f9d841ab249afcff3321ab5b9e GIT binary patch literal 58 zcmV-A0LA}!0V^p=O;s>4WH2-^Ff%bxNJ&jgEX^q?*2^l+&tpjJJ@Nlt)weo7-M{-B QbArz-