sandbox: add container creation metrics

This change measures the latency and success of container creation.
These metrics will help capacity planning and investigating production
issues.

Updates golang/go#25224
Updates golang/go#38530

Change-Id: Id7f373acb8741d4465c6e632badb188b6e855787
Reviewed-on: https://go-review.googlesource.com/c/playground/+/229980
Run-TryBot: Alexander Rakoczy <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>

Author: toothrot

sandbox/metrics.go

@@ -21,13 +21,12 @@ import (
 	mrpb "google.golang.org/genproto/googleapis/api/monitoredres"
 )
 
-// Customizations of ochttp views. Views are updated as follows:
-//  * The views are prefixed with go-playground-sandbox.
-//  * ochttp.KeyServerRoute is added as a tag to label metrics per-route.
 var (
-	mContainers         = stats.Int64("go-playground/sandbox/container_count", "number of sandbox containers", stats.UnitDimensionless)
-	mUnwantedContainers = stats.Int64("go-playground/sandbox/unwanted_container_count", "number of sandbox containers that are unexpectedly running", stats.UnitDimensionless)
-	mMaxContainers      = stats.Int64("go-playground/sandbox/max_container_count", "target number of sandbox containers", stats.UnitDimensionless)
+	kContainerCreateSuccess = tag.MustNewKey("go-playground/sandbox/container_create_success")
+	mContainers             = stats.Int64("go-playground/sandbox/container_count", "number of sandbox containers", stats.UnitDimensionless)
+	mUnwantedContainers     = stats.Int64("go-playground/sandbox/unwanted_container_count", "number of sandbox containers that are unexpectedly running", stats.UnitDimensionless)
+	mMaxContainers          = stats.Int64("go-playground/sandbox/max_container_count", "target number of sandbox containers", stats.UnitDimensionless)
+	mContainerCreateLatency = stats.Float64("go-playground/sandbox/container_create_latency", "", stats.UnitMilliseconds)
 
 	containerCount = &view.View{
 		Name:        "go-playground/sandbox/container_count",
@@ -50,7 +49,25 @@ var (
 		Measure:     mMaxContainers,
 		Aggregation: view.LastValue(),
 	}
+	containerCreateCount = &view.View{
+		Name:        "go-playground/sandbox/container_create_count",
+		Description: "Number of containers created",
+		Measure:     mContainerCreateLatency,
+		TagKeys:     []tag.Key{kContainerCreateSuccess},
+		Aggregation: view.Count(),
+	}
+	containerCreationLatency = &view.View{
+		Name:        "go-playground/sandbox/container_create_latency",
+		Description: "Latency distribution of container creation",
+		Measure:     mContainerCreateLatency,
+		Aggregation: ochttp.DefaultLatencyDistribution,
+	}
+)
 
+// Customizations of ochttp views. Views are updated as follows:
+//  * The views are prefixed with go-playground-sandbox.
+//  * ochttp.KeyServerRoute is added as a tag to label metrics per-route.
+var (
 	ServerRequestCountView = &view.View{
 		Name:        "go-playground-sandbox/http/server/request_count",
 		Description: "Count of HTTP requests started",
@@ -104,6 +121,8 @@ func newMetricService() (*metricService, error) {
 		containerCount,
 		unwantedContainerCount,
 		maxContainerCount,
+		containerCreateCount,
+		containerCreationLatency,
 		ServerRequestCountView,
 		ServerRequestBytesView,
 		ServerResponseBytesView,

sandbox/sandbox.go

@@ -33,6 +33,7 @@ import (
 
 	"go.opencensus.io/plugin/ochttp"
 	"go.opencensus.io/stats"
+	"go.opencensus.io/tag"
 	"go.opencensus.io/trace"
 	"golang.org/x/playground/internal"
 	"golang.org/x/playground/sandbox/sandboxtypes"
@@ -419,6 +420,17 @@ func getContainer(ctx context.Context) (*Container, error) {
 }
 
 func startContainer(ctx context.Context) (c *Container, err error) {
+	start := time.Now()
+	defer func() {
+		status := "success"
+		if err != nil {
+			status = "error"
+		}
+		// Ignore error. The only error can be invalid tag key or value length, which we know are safe.
+		_ = stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(kContainerCreateSuccess, status)},
+			mContainerCreateLatency.M(float64(time.Since(start))/float64(time.Millisecond)))
+	}()
+
 	name := "play_run_" + randHex(8)
 	setContainerWanted(name, true)
 	cmd := exec.Command("docker", "run",