devapp: add ticker to update dashboard data in place of /update

Updates golang/go#20691

Change-Id: I3c691c647860d541d53497c9923383421a5d2298
Reviewed-on: https://go-review.googlesource.com/46837
Reviewed-by: Brad Fitzpatrick <[email protected]>

Author: andybons

devapp/cache.go

@@ -7,7 +7,6 @@ package main
 import (
 	"bytes"
 	"compress/gzip"
-	"context"
 	"encoding/gob"
 	"fmt"
 	"log"
@@ -19,9 +18,8 @@ import (
 // A Cache contains serialized data for dashboards.
 type Cache struct {
 	// Value contains a gzipped gob'd serialization of the object
-	// to be cached. It must be []byte to avail ourselves of the
-	// datastore's 1 MB size limit.
-	Value []byte `datastore:"value,noindex"`
+	// to be cached.
+	Value []byte
 }
 
 var (
@@ -48,15 +46,7 @@ func unpackCache(cache *Cache, data interface{}) error {
 	return nil
 }
 
-func loadCache(ctx context.Context, name string, data interface{}) error {
-	cache, err := getCache(ctx, name)
-	if err != nil {
-		return err
-	}
-	return unpackCache(cache, data)
-}
-
-func writeCache(ctx context.Context, name string, data interface{}) error {
+func writeCache(name string, data interface{}) error {
 	var cache Cache
 	var cacheout bytes.Buffer
 	cachegz := gzip.NewWriter(&cacheout)
@@ -69,17 +59,17 @@ func writeCache(ctx context.Context, name string, data interface{}) error {
 	}
 	cache.Value = cacheout.Bytes()
 	log.Printf("Cache %q update finished; writing %d bytes", name, cacheout.Len())
-	return putCache(ctx, name, &cache)
+	return putCache(name, &cache)
 }
 
-func putCache(_ context.Context, name string, c *Cache) error {
+func putCache(name string, c *Cache) error {
 	dstoreMu.Lock()
 	defer dstoreMu.Unlock()
 	dstore[name] = c
 	return nil
 }
 
-func getCache(_ context.Context, name string) (*Cache, error) {
+func getCache(name string) (*Cache, error) {
 	dstoreMu.Lock()
 	defer dstoreMu.Unlock()
 	cache, ok := dstore[name]

devapp/devapp.go

@@ -13,7 +13,6 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"io"
 	"log"
 	"net"
 	"net/http"
@@ -43,17 +42,19 @@ func init() {
 	http.Handle("/", http.FileServer(http.Dir("./static/")))
 	http.HandleFunc("/favicon.ico", faviconHandler)
 	http.Handle("/release", hstsHandler(func(w http.ResponseWriter, r *http.Request) { servePage(w, r, "release") }))
-	http.Handle("/update", ctxHandler(update))
 }
 
 func main() {
 	var (
 		listen         = flag.String("listen", "localhost:6343", "listen address")
 		devTLSPort     = flag.Int("dev-tls-port", 0, "if non-zero, port number to run localhost self-signed TLS server")
 		autocertBucket = flag.String("autocert-bucket", "", "if non-empty, listen on port 443 and serve a LetsEncrypt TLS cert using this Google Cloud Storage bucket as a cache")
+		updateInterval = flag.Duration("update-interval", 5*time.Minute, "how often to update the dashboard data")
 	)
 	flag.Parse()
 
+	go updateLoop(*updateInterval)
+
 	ln, err := net.Listen("tcp", *listen)
 	if err != nil {
 		log.Fatalf("Error listening on %s: %v\n", *listen, err)
@@ -74,6 +75,16 @@ func main() {
 	log.Fatal(<-errc)
 }
 
+func updateLoop(interval time.Duration) {
+	ticker := time.NewTicker(interval)
+	for {
+		if err := update(); err != nil {
+			log.Printf("update: %v", err)
+		}
+		<-ticker.C
+	}
+}
+
 func serveDevTLS(port int) error {
 	ln, err := net.Listen("tcp", "localhost:"+strconv.Itoa(port))
 	if err != nil {
@@ -152,43 +163,27 @@ func hstsHandler(fn http.HandlerFunc) http.Handler {
 	})
 }
 
-func ctxHandler(fn func(ctx context.Context, w http.ResponseWriter, r *http.Request) error) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		ctx := r.Context()
-		if err := fn(ctx, w, r); err != nil {
-			log.Printf("handler failed: %v", err)
-			http.Error(w, err.Error(), 500)
-		}
-	})
-}
-
-func logFn(ctx context.Context, w io.Writer) func(string, ...interface{}) {
-	return func(format string, args ...interface{}) {
-		log.Printf(format, args...)
-	}
-}
-
 type page struct {
 	// Content is the complete HTML of the page.
-	Content []byte `datastore:"content,noindex"`
+	Content []byte
 }
 
 var (
 	pageStore   = map[string]*page{}
 	pageStoreMu sync.Mutex
 )
 
-func getPage(ctx context.Context, name string) (*page, error) {
+func getPage(name string) (*page, error) {
 	pageStoreMu.Lock()
 	defer pageStoreMu.Unlock()
 	p, ok := pageStore[name]
 	if ok {
 		return p, nil
 	}
-	return &page{}, fmt.Errorf("page key %s not found", name)
+	return nil, fmt.Errorf("page key %s not found", name)
 }
 
-func writePage(ctx context.Context, pageStr string, content []byte) error {
+func writePage(pageStr string, content []byte) error {
 	pageStoreMu.Lock()
 	defer pageStoreMu.Unlock()
 	entity := &page{
@@ -199,11 +194,10 @@ func writePage(ctx context.Context, pageStr string, content []byte) error {
 }
 
 func servePage(w http.ResponseWriter, r *http.Request, pageStr string) {
-	ctx := r.Context()
-	entity, err := getPage(ctx, pageStr)
+	entity, err := getPage(pageStr)
 	if err != nil {
-		log.Printf("getPage(ctx, %s) = %v", pageStr, err)
-		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+		log.Printf("getPage(%s) = %v", pageStr, err)
+		http.NotFound(w, r)
 		return
 	}
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
@@ -224,36 +218,33 @@ func (ct *countTransport) Count() int64 {
 	return atomic.LoadInt64(&ct.count)
 }
 
-func update(ctx context.Context, w http.ResponseWriter, _ *http.Request) error {
-	token, err := getToken(ctx)
+func update() error {
+	log.Printf("Updating dashboard data...")
+	token, err := getToken()
 	if err != nil {
 		return err
 	}
-	gzdata, _ := getCache(ctx, "gzdata")
+	gzdata, _ := getCache("gzdata")
+
+	ctx := context.Background()
 	ct := &countTransport{newTransport(ctx), 0}
 	gh := godash.NewGitHubClient("golang/go", token, ct)
 	defer func() {
 		log.Printf("Sent %d requests to GitHub", ct.Count())
 	}()
-	ger := gerrit.NewClient("https://go-review.googlesource.com", gerrit.NoAuth)
-	// Without a deadline, urlfetch will use a 5s timeout which is too slow for Gerrit.
-	gerctx, cancel := context.WithTimeout(ctx, 9*time.Minute)
-	defer cancel()
-	ger.HTTPClient = &http.Client{Transport: newTransport(gerctx)}
 
 	data, err := parseData(gzdata)
 	if err != nil {
 		return err
 	}
 
 	if err := data.Reviewers.LoadGithub(ctx, gh); err != nil {
-		log.Printf("failed to load reviewers: %v", err)
-		return err
+		return fmt.Errorf("failed to load reviewers: %v", err)
 	}
-	l := logFn(ctx, w)
-	if err := data.FetchData(gerctx, gh, ger, l, 7, false, false); err != nil {
-		log.Printf("failed to fetch data: %v", err)
-		return err
+	ger := gerrit.NewClient("https://go-review.googlesource.com", gerrit.NoAuth)
+
+	if err := data.FetchData(ctx, gh, ger, log.Printf, 7, false, false); err != nil {
+		return fmt.Errorf("failed to fetch data: %v", err)
 	}
 
 	var output bytes.Buffer
@@ -265,10 +256,10 @@ func update(ctx context.Context, w http.ResponseWriter, _ *http.Request) error {
 	var html bytes.Buffer
 	godash.PrintHTML(&html, output.String())
 
-	if err := writePage(ctx, kind, html.Bytes()); err != nil {
+	if err := writePage(kind, html.Bytes()); err != nil {
 		return err
 	}
-	return writeCache(ctx, "gzdata", &data)
+	return writeCache("gzdata", &data)
 }
 
 func newTransport(ctx context.Context) http.RoundTripper {

devapp/github.go

@@ -5,7 +5,6 @@
 package main
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -25,7 +24,7 @@ var (
 	githubOnce    sync.Once
 )
 
-func getTokenFromFile(ctx context.Context) (string, error) {
+func getTokenFromFile() (string, error) {
 	githubOnce.Do(func() {
 		const short = ".github-issue-token"
 		filename := filepath.Clean(os.Getenv("HOME") + "/" + short)
@@ -62,13 +61,13 @@ it at any time.`
 	return githubToken, githubOnceErr
 }
 
-func getToken(ctx context.Context) (string, error) {
+func getToken() (string, error) {
 	if metadata.OnGCE() {
 		// Re-use maintner-github-token until this is migrated to using the maintner API.
 		token, err := metadata.ProjectAttributeValue("maintner-github-token")
 		if len(token) > 0 && err == nil {
 			return token, nil
 		}
 	}
-	return getTokenFromFile(ctx)
+	return getTokenFromFile()
 }