[usage] Upload usage reports to cloud storage

components/usage/pkg/contentservice/client.go

@@ -5,16 +5,22 @@
 package contentservice
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
+	"encoding/json"
 	"fmt"
+	"net/http"
 
+	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/gitpod-io/gitpod/content-service/api"
+	"github.com/gitpod-io/gitpod/usage/pkg/db"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 )
 
 type Interface interface {
-	GetSignedUploadUrl(ctx context.Context) (string, error)
+	UploadUsageReport(ctx context.Context, filename string, report map[db.AttributionID][]db.WorkspaceInstanceForUsage) error
 }
 
 type Client struct {
@@ -25,7 +31,44 @@ func New(url string) *Client {
 	return &Client{url: url}
 }
 
-func (c *Client) GetSignedUploadUrl(ctx context.Context) (string, error) {
+func (c *Client) UploadUsageReport(ctx context.Context, filename string, report map[db.AttributionID][]db.WorkspaceInstanceForUsage) error {
+	url, err := c.getSignedUploadUrl(ctx, filename)
+	if err != nil {
+		return fmt.Errorf("failed to obtain signed upload URL: %w", err)
+	}
+
+	reportBytes := &bytes.Buffer{}
+	gz := gzip.NewWriter(reportBytes)
+	err = json.NewEncoder(gz).Encode(report)
+	if err != nil {
+		return fmt.Errorf("failed to marshal report to JSON: %w", err)
+	}
+	err = gz.Close()
+	if err != nil {
+		return fmt.Errorf("failed to compress usage report: %w", err)
+	}
+
+	req, err := http.NewRequest(http.MethodPut, url, reportBytes)
+	if err != nil {
+		return fmt.Errorf("failed to construct http request: %w", err)
+	}
+
+	req.Header.Set("Content-Encoding", "gzip")
+
+	log.Infof("Uploading %q to object storage...", filename)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to make http request: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("unexpected http response code: %s", resp.Status)
+	}
+	log.Info("Upload complete")
+
+	return nil
+}
+
+func (c *Client) getSignedUploadUrl(ctx context.Context, key string) (string, error) {
 	conn, err := grpc.Dial(c.url, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		return "", fmt.Errorf("failed to dial content-service gRPC server: %w", err)
@@ -34,9 +77,9 @@ func (c *Client) GetSignedUploadUrl(ctx context.Context) (string, error) {
 
 	uc := api.NewUsageReportServiceClient(conn)
 
-	resp, err := uc.UploadURL(ctx, &api.UsageReportUploadURLRequest{Name: "some-name"})
+	resp, err := uc.UploadURL(ctx, &api.UsageReportUploadURLRequest{Name: key})
 	if err != nil {
-		return "", fmt.Errorf("failed to obtain signed upload URL: %w", err)
+		return "", fmt.Errorf("failed RPC to content service: %w", err)
 	}
 
 	return resp.Url, nil

components/usage/pkg/contentservice/noop.go

@@ -4,8 +4,14 @@
 
 package contentservice
 
-import "context"
+import (
+	"context"
+
+	"github.com/gitpod-io/gitpod/usage/pkg/db"
+)
 
 type NoOpClient struct{}
 
-func (c *NoOpClient) GetSignedUploadUrl(ctx context.Context) (string, error) { return "", nil }
+func (c *NoOpClient) UploadUsageReport(ctx context.Context, filename string, report map[db.AttributionID][]db.WorkspaceInstanceForUsage) error {
+	return nil
+}

components/usage/pkg/controller/reconciler.go

@@ -7,12 +7,8 @@ package controller
 import (
 	"context"
 	"database/sql"
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"math"
-	"os"
-	"path/filepath"
 	"time"
 
 	"github.com/gitpod-io/gitpod/common-go/log"
@@ -76,36 +72,15 @@ func (u *UsageReconciler) Reconcile() (err error) {
 	}
 	log.WithField("usage_reconcile_status", status).Info("Reconcile completed.")
 
-	// For now, write the report to a temp directory so we can manually retrieve it
-	dir := os.TempDir()
-	f, err := ioutil.TempFile(dir, fmt.Sprintf("%s-*", now.Format(time.RFC3339)))
-	if err != nil {
-		return fmt.Errorf("failed to create temporary file: %w", err)
-	}
-	defer f.Close()
-
-	enc := json.NewEncoder(f)
-	err = enc.Encode(report)
-	if err != nil {
-		return fmt.Errorf("failed to marshal report to JSON: %w", err)
-	}
-
-	stat, err := f.Stat()
-	filePath := filepath.Join(dir, stat.Name())
-	if err != nil {
-		return fmt.Errorf("failed to get file stats: %w", err)
-	}
-	log.Infof("Wrote usage report into %s", filePath)
-
-	uploadURL, err := u.contentService.GetSignedUploadUrl(ctx)
+	err = db.CreateUsageRecords(ctx, u.conn, usageReportToUsageRecords(report, u.pricer, u.nowFunc().UTC()))
 	if err != nil {
-		return fmt.Errorf("failed to obtain signed upload URL: %w", err)
+		return fmt.Errorf("failed to write usage records to database: %s", err)
 	}
-	log.Infof("signed upload url: %s", uploadURL)
 
-	err = db.CreateUsageRecords(ctx, u.conn, usageReportToUsageRecords(report, u.pricer, u.nowFunc().UTC()))
+	filename := fmt.Sprintf("%s.gz", now.Format(time.RFC3339))
+	err = u.contentService.UploadUsageReport(ctx, filename, report)
 	if err != nil {
-		return fmt.Errorf("failed to write usage records to database: %s", err)
+		return fmt.Errorf("failed to upload usage report: %w", err)
 	}
 
 	return nil