internal/worker: make tests hermetic

Refactor tests that contact proxy.golang.org to use a local
HTTP server instead. Fixes failures on builders with no
outbound network.

For golang/go#51444.

Change-Id: I7d4e5a0b2dc4b1c0cddd12435e3656307bed4c70
Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/419174
TryBot-Result: Gopher Robot <[email protected]>
Run-TryBot: Damien Neil <[email protected]>
Reviewed-by: Jonathan Amsterdam <[email protected]>

Author: neild

internal/worker/module_proxy.go

@@ -27,8 +27,8 @@ const proxyURL = "https://proxy.golang.org"
 
 // latestVersion returns the version of modulePath provided by the proxy's "@latest"
 // endpoint.
-func latestVersion(ctx context.Context, modulePath string) (string, error) {
-	body, err := proxyRequest(ctx, modulePath, "/@latest")
+func latestVersion(ctx context.Context, proxyURL, modulePath string) (string, error) {
+	body, err := proxyRequest(ctx, proxyURL, modulePath, "/@latest")
 	if err != nil {
 		return "", err
 	}
@@ -46,8 +46,8 @@ func latestVersion(ctx context.Context, modulePath string) (string, error) {
 // latestTaggedVersion returns the latest (largest in the semver sense) tagged
 // version of modulePath, as determined by the module proxy's "list" endpoint.
 // It returns ("", nil) if there are no tagged versions.
-func latestTaggedVersion(ctx context.Context, modulePath string) (string, error) {
-	body, err := proxyRequest(ctx, modulePath, "/@v/list")
+func latestTaggedVersion(ctx context.Context, proxyURL, modulePath string) (string, error) {
+	body, err := proxyRequest(ctx, proxyURL, modulePath, "/@v/list")
 	if err != nil {
 		return "", err
 	}
@@ -59,19 +59,19 @@ func latestTaggedVersion(ctx context.Context, modulePath string) (string, error)
 	return vs[0], nil
 }
 
-func moduleZip(ctx context.Context, modulePath, version string) (*zip.Reader, error) {
+func moduleZip(ctx context.Context, proxyURL, modulePath, version string) (*zip.Reader, error) {
 	ev, err := module.EscapeVersion(version)
 	if err != nil {
 		return nil, err
 	}
-	body, err := proxyRequest(ctx, modulePath, fmt.Sprintf("/@v/%s.zip", ev))
+	body, err := proxyRequest(ctx, proxyURL, modulePath, fmt.Sprintf("/@v/%s.zip", ev))
 	if err != nil {
 		return nil, err
 	}
 	return zip.NewReader(bytes.NewReader(body), int64(len(body)))
 }
 
-func proxyRequest(ctx context.Context, modulePath, suffix string) ([]byte, error) {
+func proxyRequest(ctx context.Context, proxyURL, modulePath, suffix string) ([]byte, error) {
 	ep, err := module.EscapePath(modulePath)
 	if err != nil {
 		return nil, fmt.Errorf("module path %v: %w", modulePath, err)

internal/worker/module_proxy_test.go

@@ -5,14 +5,20 @@
 package worker
 
 import (
+	"archive/zip"
 	"context"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 
 	"golang.org/x/mod/semver"
 )
 
 func TestLatestVersion(t *testing.T) {
-	got, err := latestVersion(context.Background(), "golang.org/x/build")
+	pt := newProxyTest()
+	defer pt.Close()
+
+	got, err := latestVersion(context.Background(), pt.URL, "golang.org/x/build")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -22,15 +28,18 @@ func TestLatestVersion(t *testing.T) {
 }
 
 func TestLatestTaggedVersion(t *testing.T) {
-	got, err := latestTaggedVersion(context.Background(), "golang.org/x/build")
+	pt := newProxyTest()
+	defer pt.Close()
+
+	got, err := latestTaggedVersion(context.Background(), pt.URL, "golang.org/x/build")
 	if err != nil {
 		t.Fatal(err)
 	}
 	if got != "" {
 		t.Errorf(`got %q, wanted ""`, got)
 	}
 
-	got, err = latestTaggedVersion(context.Background(), "golang.org/x/tools")
+	got, err = latestTaggedVersion(context.Background(), pt.URL, "golang.org/x/tools")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -41,14 +50,51 @@ func TestLatestTaggedVersion(t *testing.T) {
 }
 
 func TestModuleZip(t *testing.T) {
+	pt := newProxyTest()
+	defer pt.Close()
+
 	ctx := context.Background()
 	const m = "golang.org/x/time"
-	v, err := latestVersion(ctx, m)
+	v, err := latestVersion(ctx, pt.URL, m)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = moduleZip(ctx, m, v)
+	_, err = moduleZip(ctx, pt.URL, m, v)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
+
+type proxyTest struct {
+	*httptest.Server
+}
+
+func (*proxyTest) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	switch r.URL.Path {
+	case "/golang.org/x/build/@latest":
+		w.Write([]byte(`{"Version":"v0.0.0-20220722180300-9ed544e84dd1","Time":"2022-07-22T18:03:00Z"}`))
+	case "/golang.org/x/build/@v/list":
+	case "/golang.org/x/tools/@v/list":
+		w.Write([]byte("v0.1.1\nv0.1.2\n"))
+	case "/golang.org/x/time/@latest":
+		w.Write([]byte(`{"Version":"v0.0.0-20220722155302-e5dcc9cfc0b9","Time":"2022-07-22T15:53:02Z"}`))
+	case "/golang.org/x/time/@v/v0.0.0-20220722155302-e5dcc9cfc0b9.zip":
+		zw := zip.NewWriter(w)
+		fw, _ := zw.Create("golang.org/x/[email protected]/go.mod")
+		fw.Write([]byte(`module golang.org/x/time`))
+		zw.Close()
+	case "/golang.org/x/mod/@v/v0.5.1.zip":
+		zw := zip.NewWriter(w)
+		fw, _ := zw.Create("golang.org/x/[email protected]/go.mod")
+		fw.Write([]byte(`module golang.org/x/mod`))
+		zw.Close()
+	default:
+		w.WriteHeader(404)
+	}
+}
+
+func newProxyTest() *proxyTest {
+	pt := &proxyTest{}
+	pt.Server = httptest.NewServer(pt)
+	return pt
+}

internal/worker/scan_modules.go

@@ -58,7 +58,7 @@ func ScanModules(ctx context.Context, st store.Store, force bool) error {
 	}
 	for _, modulePath := range modulesToScan {
 		// Scan the latest version, and the latest tagged version (if they differ).
-		latest, err := latestVersion(ctx, modulePath)
+		latest, err := latestVersion(ctx, proxyURL, modulePath)
 		if err != nil {
 			return err
 		}
@@ -68,7 +68,7 @@ func ScanModules(ctx context.Context, st store.Store, force bool) error {
 			}
 			// Otherwise, if the error was in the scanning itself, keep going.
 		}
-		latestTagged, err := latestTaggedVersion(ctx, modulePath)
+		latestTagged, err := latestTaggedVersion(ctx, proxyURL, modulePath)
 		if err != nil {
 			return err
 		}
@@ -99,7 +99,7 @@ func processModule(ctx context.Context, modulePath, version string, dbClient vul
 			return nil
 		}
 	}
-	res, err := scanModule(ctx, modulePath, version, dbClient)
+	res, err := scanModule(ctx, proxyURL, modulePath, version, dbClient)
 	if err2 := createModuleScanRecord(ctx, st, modulePath, version, dbTime, res, err); err2 != nil {
 		return err2
 	}
@@ -142,7 +142,7 @@ func createModuleScanRecord(ctx context.Context, st store.Store, path, version s
 // scanRepo clones the given repo and analyzes it for vulnerabilities. If commit
 // is "HEAD", the head commit is scanned. Otherwise, commit must be a hex string
 // corresponding to a commit, and that commit is checked out and scanned.
-func scanModule(ctx context.Context, modulePath, version string, dbClient vulnc.Client) (_ *vulncheck.Result, err error) {
+func scanModule(ctx context.Context, proxyURL, modulePath, version string, dbClient vulnc.Client) (_ *vulncheck.Result, err error) {
 	defer derrors.Wrap(&err, "scanModule(%q, %q)", modulePath, version)
 
 	start := time.Now()
@@ -160,7 +160,7 @@ func scanModule(ctx context.Context, modulePath, version string, dbClient vulnc.
 		}
 	}()
 
-	zipr, err := moduleZip(ctx, modulePath, version)
+	zipr, err := moduleZip(ctx, proxyURL, modulePath, version)
 	if err != nil {
 		return nil, err
 	}

internal/worker/scan_modules_test.go

@@ -44,13 +44,16 @@ func TestAsScanError(t *testing.T) {
 }
 
 func TestScanModule(t *testing.T) {
+	pt := newProxyTest()
+	defer pt.Close()
+
 	ctx := event.WithExporter(context.Background(),
 		event.NewExporter(log.NewLineHandler(os.Stderr), nil))
 	dbClient, err := vulnc.NewClient([]string{vulnDBURL}, vulnc.Options{})
 	if err != nil {
 		t.Fatal(err)
 	}
-	got, err := scanModule(ctx, "golang.org/x/mod", "v0.5.1", dbClient)
+	got, err := scanModule(ctx, pt.URL, "golang.org/x/mod", "v0.5.1", dbClient)
 	if err != nil {
 		t.Fatal(err)
 	}