cmd/relui: add Docker and Postgres configuration

This adds a Dockerfile for relui, as well as a file for running
integration tests against the database. It also adds a configuration for
a development database, as well as a test environment.

For golang/go#47401

Change-Id: I668c99d028265c865baa7e15dfe627423815af94
Reviewed-on: https://go-review.googlesource.com/c/build/+/347289
Trust: Alexander Rakoczy <[email protected]>
Run-TryBot: Alexander Rakoczy <[email protected]>
TryBot-Result: Go Bot <[email protected]>
Reviewed-by: Heschi Kreinick <[email protected]>

Author: toothrot

cmd/relui/Dockerfile

@@ -0,0 +1,30 @@
+# Copyright 2021 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+FROM golang:1.17 AS build
+
+COPY go.mod /app/go.mod
+COPY go.sum /app/go.sum
+
+WORKDIR /app
+
+RUN go mod download
+
+COPY . /app
+
+RUN go build golang.org/x/build/cmd/relui
+
+FROM marketplace.gcr.io/google/debian10:latest
+
+RUN apt-get update && apt-get install -y \
+	--no-install-recommends \
+	tini
+
+ARG PORT=8080
+ENV PORT=${PORT}
+EXPOSE ${PORT}
+
+COPY --from=build /app/relui /app/relui
+WORKDIR /app
+ENTRYPOINT ["/usr/bin/tini", "--", "./relui"]

cmd/relui/Dockerfile.test

@@ -0,0 +1,18 @@
+# Copyright 2021 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+FROM golang:1.17 AS build
+
+COPY go.mod /app/go.mod
+COPY go.sum /app/go.sum
+
+WORKDIR /app
+
+RUN go mod download
+
+COPY . /app
+
+RUN go build golang.org/x/build/cmd/relui
+
+ENTRYPOINT go test -v ./cmd/relui/...

cmd/relui/Makefile

@@ -0,0 +1,31 @@
+# Copyright 2021 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+VERSION := $(shell ../coordinator/version.sh)
+
+.PHONY: dev
+dev: postgres-dev docker
+	docker run --rm --name=relui-dev -v postgres-run-dev:/var/run/postgresql -p 8080:8080 golang/relui:$(VERSION)
+
+RUNNING := $(shell docker ps -q -f Name=postgres-dev)
+.PHONY: postgres-dev
+postgres-dev: network-dev
+	docker stop postgres-dev || true
+	docker run -d --rm --name=postgres-dev --network=relui-dev -v postgres-data-dev:/var/lib/postgresql/data -v postgres-run-dev:/var/run/postgresql -e POSTGRES_HOST_AUTH_METHOD=trust postgres:13
+
+.PHONY: network-dev
+network-dev:
+	docker network inspect relui-dev -f "{{.Name}}" || docker network create -d bridge relui-dev
+
+.PHONY: test
+test: postgres-dev docker-test
+	docker run --rm --name=relui-test -v postgres-run-dev:/var/run/postgresql -e RELUI_TEST_DATABASE="host=/var/run/postgresql user=postgres database=relui-test" golang/relui-test:$(VERSION)
+
+.PHONY: docker
+docker:
+	docker build -f Dockerfile -t golang/relui:$(VERSION) ../..
+
+.PHONY: docker-test
+docker-test:
+	docker build -f Dockerfile.test -t golang/relui-test:$(VERSION) ../..

cmd/relui/main.go

@@ -9,15 +9,24 @@
 package main
 
 import (
+	"context"
+	"flag"
 	"log"
 	"net/http"
 	"os"
+)
 
-	"golang.org/x/build/internal/datastore/fake"
+var (
+	pgConnect = flag.String("pg-connect", "host=/var/run/postgresql user=postgres database=relui-dev", "Postgres connection string or URI")
 )
 
 func main() {
-	d := &dsStore{client: &fake.Client{}}
+	ctx := context.Background()
+	d := new(pgStore)
+	if err := d.Connect(ctx, *pgConnect); err != nil {
+		log.Fatalf("d.Connect() = %v", err)
+	}
+	defer d.Close()
 	s := &server{
 		store: d,
 	}

cmd/relui/store.go

@@ -9,13 +9,19 @@ package main
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"log"
 
 	"cloud.google.com/go/datastore"
 	"github.com/googleapis/google-cloud-go-testing/datastore/dsiface"
+	"github.com/jackc/pgx/v4"
+	"github.com/jackc/pgx/v4/pgxpool"
 	reluipb "golang.org/x/build/cmd/relui/protos"
 )
 
+var errDBNotExist = errors.New("database does not exist")
+
 // store is a persistence interface for saving data.
 type store interface {
 	AddWorkflow(workflow *reluipb.Workflow) error
@@ -24,6 +30,115 @@ type store interface {
 	Workflows() []*reluipb.Workflow
 }
 
+// pgStore is a store backed by a Postgres database.
+type pgStore struct {
+	db *pgxpool.Pool
+}
+
+// Connect connects to the database using the credentials supplied in
+// the provided connString.
+//
+// Any key/value or URI string compatible with libpq is valid. If the
+// database does not exist, one will be created using the credentials
+// provided.
+func (p *pgStore) Connect(ctx context.Context, connString string) error {
+	cfg, err := pgx.ParseConfig(connString)
+	if err != nil {
+		return fmt.Errorf("pgx.ParseConfig() = %w", err)
+	}
+	if err := CreateDBIfNotExists(ctx, cfg); err != nil {
+		return err
+	}
+	pool, err := pgxpool.Connect(ctx, connString)
+	if err != nil {
+		return err
+	}
+	p.db = pool
+	return nil
+}
+
+// Close closes the pgxpool.Pool.
+func (p *pgStore) Close() {
+	p.db.Close()
+}
+
+// ConnectMaintenanceDB connects to the maintenance database using the
+// credentials from cfg. If maintDB is an empty string, the database
+// with the name cfg.User will be used.
+func ConnectMaintenanceDB(ctx context.Context, cfg *pgx.ConnConfig, maintDB string) (*pgx.Conn, error) {
+	cfg = cfg.Copy()
+	cfg.Database = maintDB
+	return pgx.ConnectConfig(ctx, cfg)
+}
+
+// CreateDBIfNotExists checks whether the given dbName is an existing
+// database, and creates one if not.
+func CreateDBIfNotExists(ctx context.Context, cfg *pgx.ConnConfig) error {
+	exists, err := checkIfDBExists(ctx, cfg)
+	if err != nil || exists {
+		return err
+	}
+	conn, err := ConnectMaintenanceDB(ctx, cfg, "")
+	if err != nil {
+		return fmt.Errorf("ConnectMaintenanceDB = %w", err)
+	}
+	createSQL := fmt.Sprintf("CREATE DATABASE %s", pgx.Identifier{cfg.Database}.Sanitize())
+	if _, err := conn.Exec(ctx, createSQL); err != nil {
+		return fmt.Errorf("conn.Exec(%q) = %w", createSQL, err)
+	}
+	return nil
+}
+
+// DropDB drops the database specified in cfg. An error returned if
+// the database does not exist.
+func DropDB(ctx context.Context, cfg *pgx.ConnConfig) error {
+	exists, err := checkIfDBExists(ctx, cfg)
+	if err != nil {
+		return fmt.Errorf("p.checkIfDBExists() = %w", err)
+	}
+	if !exists {
+		return errDBNotExist
+	}
+	conn, err := ConnectMaintenanceDB(ctx, cfg, "")
+	if err != nil {
+		return fmt.Errorf("ConnectMaintenanceDB = %w", err)
+	}
+	dropSQL := fmt.Sprintf("DROP DATABASE %s", pgx.Identifier{cfg.Database}.Sanitize())
+	if _, err := conn.Exec(ctx, dropSQL); err != nil {
+		return fmt.Errorf("conn.Exec(%q) = %w", dropSQL, err)
+	}
+	return nil
+}
+
+func checkIfDBExists(ctx context.Context, cfg *pgx.ConnConfig) (bool, error) {
+	conn, err := ConnectMaintenanceDB(ctx, cfg, "")
+	if err != nil {
+		return false, fmt.Errorf("ConnectMaintenanceDB = %w", err)
+	}
+	row := conn.QueryRow(ctx, "SELECT 1 from pg_database WHERE datname=$1 LIMIT 1", cfg.Database)
+	var exists int
+	if err := row.Scan(&exists); err != nil && err != pgx.ErrNoRows {
+		return false, fmt.Errorf("row.Scan() = %w", err)
+	}
+	return exists == 1, nil
+}
+
+func (*pgStore) AddWorkflow(workflow *reluipb.Workflow) error {
+	return nil
+}
+
+func (*pgStore) BuildableTask(workflowId, id string) *reluipb.BuildableTask {
+	return nil
+}
+
+func (*pgStore) Workflow(id string) *reluipb.Workflow {
+	return nil
+}
+
+func (*pgStore) Workflows() []*reluipb.Workflow {
+	return nil
+}
+
 var _ store = (*dsStore)(nil)
 
 // dsStore is a store backed by Google Cloud Datastore.

cmd/relui/store_test.go

@@ -0,0 +1,57 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+	"testing"
+
+	"github.com/jackc/pgx/v4"
+)
+
+const dbEnvKey = "RELUI_TEST_DATABASE"
+
+func TestCreateDBIfNotExists(t *testing.T) {
+	dbEnv := os.Getenv(dbEnvKey)
+	if dbEnv == "" {
+		t.Skipf("%q is not set. Skipping.", dbEnvKey)
+	}
+	if testing.Short() {
+		t.Skip("Skipping database tests in short mode.")
+	}
+
+	ctx := context.Background()
+	cfg, err := pgx.ParseConfig(dbEnv)
+	if err != nil {
+		t.Fatalf("pgx.ParseConfig(os.Getenv(%q)) = %v, wanted no error", dbEnvKey, err)
+	}
+	testCfg := cfg.Copy()
+	testCfg.Database = "relui-test-nonexistent"
+	if err := DropDB(ctx, testCfg); err != nil && !errors.Is(err, errDBNotExist) {
+		t.Fatalf("p.DropDB() = %v, wanted %q or nil", err, errDBNotExist)
+	}
+	exists, err := checkIfDBExists(ctx, testCfg)
+	if exists || err != nil {
+		t.Fatalf("p.checkIfDBExists() = %t, %v, wanted %t, nil", exists, err, false)
+	}
+	if err := CreateDBIfNotExists(ctx, testCfg); err != nil {
+		t.Errorf("p.CreateDBIfNotExists() = %v, wanted no error", err)
+	}
+	exists, err = checkIfDBExists(ctx, testCfg)
+	if !exists || err != nil {
+		t.Fatalf("p.checkIfDBExists() = %t, %v, wanted %t, nil", exists, err, true)
+	}
+	defer DropDB(ctx, testCfg)
+	// Create again with the same name.
+	if err := CreateDBIfNotExists(ctx, testCfg); err != nil {
+		t.Errorf("p.CreateDBIfNotExists() = %v, wanted no error", err)
+	}
+	exists, err = checkIfDBExists(ctx, testCfg)
+	if !exists || err != nil {
+		t.Fatalf("p.checkIfDBExists() = %t, %v, wanted %t, nil", exists, err, true)
+	}
+}

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/aws/aws-sdk-go v1.30.15
 	github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625
-	github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d
+	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
 	github.com/davecgh/go-spew v1.1.1
 	github.com/gliderlabs/ssh v0.3.3
 	github.com/golang/protobuf v1.4.3
@@ -23,8 +23,9 @@ require (
 	github.com/googleapis/gax-go/v2 v2.0.5
 	github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7
+	github.com/jackc/pgx/v4 v4.13.0
 	github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1
-	github.com/kr/pty v1.1.3
+	github.com/kr/pty v1.1.8
 	github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07
 	go.opencensus.io v0.23.0
 	go4.org v0.0.0-20180809161055-417644f6feb5