internal/postgres,worker: schedule fetching of std@master

In order to ensure that pkg.go.dev/std@master is always showing the
latest version, an endpoint is added which checks for the most recent
version of std@master. If the version in our database is not up to date,
a fetch request is scheduled.

Additionally, since only the most recent pseudoversion of std is ever
shown on pkg.go.dev, older versions are deleted.

For golang/go#44390

Change-Id: Iad56b8653f378fb5a7d72a1e30505da090de279d
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/294350
Trust: Julie Qiu <[email protected]>
Run-TryBot: Julie Qiu <[email protected]>
TryBot-Result: kokoro <[email protected]>
Reviewed-by: Jonathan Amsterdam <[email protected]>

Author: julieqiu

internal/postgres/delete.go

@@ -0,0 +1,41 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/lib/pq"
+	"golang.org/x/pkgsite/internal/database"
+	"golang.org/x/pkgsite/internal/derrors"
+)
+
+// DeletePseudoversionsExcept deletes all pseudoversions for the module except
+// the provided resolvedVersion.
+func (db *DB) DeletePseudoversionsExcept(ctx context.Context, modulePath, resolvedVersion string) (err error) {
+	defer derrors.WrapStack(&err, "DeletePseudoversionsExcept(ctx, db, %q, %q)", modulePath, resolvedVersion)
+	return db.db.Transact(ctx, sql.LevelDefault, func(tx *database.DB) error {
+		var versions []string
+		collect := func(rows *sql.Rows) error {
+			var version string
+			if err := rows.Scan(&version); err != nil {
+				return err
+			}
+			versions = append(versions, version)
+			return nil
+		}
+		const stmt = `
+			DELETE FROM modules
+			WHERE version_type = 'pseudo' AND module_path=$1 AND version != $2
+			RETURNING version`
+		if err := tx.RunQuery(ctx, stmt, collect, modulePath, resolvedVersion); err != nil {
+			return err
+		}
+		_, err := tx.Exec(ctx, `DELETE FROM version_map WHERE module_path = $1 AND resolved_version = ANY($2)`,
+			modulePath, pq.Array(versions))
+		return err
+	})
+}

internal/postgres/delete_test.go

@@ -0,0 +1,51 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package postgres
+
+import (
+	"context"
+	"testing"
+
+	"golang.org/x/pkgsite/internal/testing/sample"
+	"golang.org/x/pkgsite/internal/version"
+)
+
+func TestDeletePseudoversionsExcept(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	testDB, release := acquire(t)
+	defer release()
+
+	pseudo1 := "v0.0.0-20190904010203-89fb59e2e920"
+	versions := []string{
+		sample.VersionString,
+		pseudo1,
+		"v0.0.0-20190904010203-89fb59e2e920",
+		"v0.0.0-20190904010203-89fb59e2e920",
+	}
+	for _, v := range versions {
+		MustInsertModule(ctx, t, testDB, sample.Module(sample.ModulePath, v, ""))
+	}
+	if err := testDB.DeletePseudoversionsExcept(ctx, sample.ModulePath, pseudo1); err != nil {
+		t.Fatal(err)
+	}
+	mods, err := getPathVersions(ctx, testDB, sample.ModulePath, version.TypeRelease)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(mods) != 1 && mods[0].Version != sample.VersionString {
+		t.Errorf("module version %q was not found", sample.VersionString)
+	}
+	mods, err = getPathVersions(ctx, testDB, sample.ModulePath, version.TypePseudo)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(mods) != 1 {
+		t.Fatalf("pseudoversions expected to be deleted were not")
+	}
+	if mods[0].Version != pseudo1 {
+		t.Errorf("got %q; want %q", mods[0].Version, pseudo1)
+	}
+}

internal/worker/server.go

@@ -152,6 +152,11 @@ func (s *Server) Install(handle func(string, http.Handler)) {
 	// Google Cloud Task Queues.
 	handle("/fetch/", http.StripPrefix("/fetch", rmw(http.HandlerFunc(s.handleFetch))))
 
+	// scheduled: fetch-std-master checks if the std@master version in the
+	// database is up to date with the version at HEAD. If not, a fetch request
+	// is queued to refresh the std@master version.
+	handle("/fetch-std-master/", rmw(s.errorHandler(s.handleFetchStdMaster)))
+
 	// scheduled: enqueue queries the module_version_states table for the next
 	// batch of module versions to process, and enqueues them for processing.
 	// Normally this will not cause duplicate processing, because Cloud Tasks
@@ -453,6 +458,23 @@ func (s *Server) handleHTMLPage(f func(w http.ResponseWriter, r *http.Request) e
 	}
 }
 
+func (s *Server) handleFetchStdMaster(w http.ResponseWriter, r *http.Request) error {
+	_, resolvedVersion, _, err := stdlib.Zip("master")
+	if err != nil {
+		return err
+	}
+	vm, err := s.db.GetVersionMap(r.Context(), stdlib.ModulePath, "master")
+	if err != nil {
+		return err
+	}
+	if vm.ResolvedVersion != resolvedVersion {
+		if _, err := s.queue.ScheduleFetch(r.Context(), stdlib.ModulePath, "master", "", false); err != nil {
+			return fmt.Errorf("error scheduling fetch for %s: %w", "master", err)
+		}
+	}
+	return s.db.DeletePseudoversionsExcept(r.Context(), stdlib.ModulePath, vm.ResolvedVersion)
+}
+
 func (s *Server) handlePopulateStdLib(w http.ResponseWriter, r *http.Request) error {
 	msg, err := s.doPopulateStdLib(r.Context(), r.FormValue("suffix"))
 	w.Header().Set("Content-Type", "text/plain; charset=utf-8")