[installer]: allow use of external container registry

Simon Emms · roboquat · commit 2d731655b511 · 2021-11-11T09:57:08.000+01:00
diff --git a/installer/pkg/components/blobserve/configmap.go b/installer/pkg/components/blobserve/configmap.go
@@ -78,7 +78,6 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 				MaxSize:  MaxSizeBytes,
 			},
 		},
-		// todo(sje): make conditional on the workspace having a pull secret
 		AuthCfg:        "/mnt/pull-secret.json",
 		PProfAddr:      ":6060",
 		PrometheusAddr: "127.0.0.1:9500",
diff --git a/installer/pkg/components/blobserve/deployment.go b/installer/pkg/components/blobserve/deployment.go
@@ -5,6 +5,7 @@
 package blobserve
 
 import (
+	"fmt"
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
 	dockerregistry "github.com/gitpod-io/gitpod/installer/pkg/components/docker-registry"
 	appsv1 "k8s.io/api/apps/v1"
@@ -18,55 +19,27 @@ import (
 func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 	labels := common.DefaultLabels(Component)
 
+	volumeName := "pull-secret"
+	var secretName string
+	if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) {
+		secretName = dockerregistry.BuiltInRegistryAuth
+	} else if ctx.Config.ContainerRegistry.External != nil {
+		secretName = ctx.Config.ContainerRegistry.External.Certificate.Name
+	} else {
+		return nil, fmt.Errorf("%s: invalid container registry config", Component)
+	}
+
 	var hashObj []runtime.Object
 	if objs, err := configmap(ctx); err != nil {
 		return nil, err
 	} else {
 		hashObj = append(hashObj, objs...)
 	}
 
-	volumes := []corev1.Volume{{
-		Name:         "cache",
-		VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}},
-	}, {
-		Name: "config",
-		VolumeSource: corev1.VolumeSource{
-			ConfigMap: &corev1.ConfigMapVolumeSource{
-				LocalObjectReference: corev1.LocalObjectReference{Name: Component},
-			},
-		},
-	}}
-
-	volumeMounts := []corev1.VolumeMount{
-		{
-			Name:      "config",
-			MountPath: "/mnt/config",
-			ReadOnly:  true,
-		}, {
-			Name:      "cache",
-			MountPath: "/mnt/cache",
-		},
-	}
-
-	if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) {
-		volumeName := "pull-secret"
-		volumes = append(volumes, corev1.Volume{
-			Name: volumeName,
-			VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{
-				SecretName: dockerregistry.BuiltInRegistryAuth,
-			}},
-		})
-		volumeMounts = append(volumeMounts, corev1.VolumeMount{
-			Name:      volumeName,
-			MountPath: "/mnt/pull-secret.json",
-			SubPath:   ".dockerconfigjson",
-		})
-
-		if objs, err := common.DockerRegistryHash(ctx); err != nil {
-			return nil, err
-		} else {
-			hashObj = append(hashObj, objs...)
-		}
+	if objs, err := common.DockerRegistryHash(ctx); err != nil {
+		return nil, err
+	} else {
+		hashObj = append(hashObj, objs...)
 	}
 
 	configHash, err := common.ObjectHash(hashObj, nil)
@@ -99,7 +72,22 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 						Affinity:           &corev1.Affinity{},
 						ServiceAccountName: Component,
 						EnableServiceLinks: pointer.Bool(false),
-						Volumes:            volumes,
+						Volumes: []corev1.Volume{{
+							Name:         "cache",
+							VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}},
+						}, {
+							Name: "config",
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									LocalObjectReference: corev1.LocalObjectReference{Name: Component},
+								},
+							},
+						}, {
+							Name: volumeName,
+							VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{
+								SecretName: secretName,
+							}},
+						}},
 						Containers: []corev1.Container{{
 							Name:            Component,
 							Args:            []string{"run", "-v", "/mnt/config/config.json"},
@@ -123,7 +111,18 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 								common.DefaultEnv(&ctx.Config),
 								common.TracingEnv(&ctx.Config),
 							),
-							VolumeMounts: volumeMounts,
+							VolumeMounts: []corev1.VolumeMount{{
+								Name:      "config",
+								MountPath: "/mnt/config",
+								ReadOnly:  true,
+							}, {
+								Name:      "cache",
+								MountPath: "/mnt/cache",
+							}, {
+								Name:      volumeName,
+								MountPath: "/mnt/pull-secret.json",
+								SubPath:   ".dockerconfigjson",
+							}},
 						}, *common.KubeRBACProxyContainer()},
 					},
 				},
diff --git a/installer/pkg/components/image-builder-mk3/configmap.go b/installer/pkg/components/image-builder-mk3/configmap.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	dockerregistry "github.com/gitpod-io/gitpod/installer/pkg/components/docker-registry"
 	"k8s.io/utils/pointer"
+	"strings"
 	"time"
 
 	"github.com/gitpod-io/gitpod/common-go/util"
@@ -23,6 +24,15 @@ import (
 )
 
 func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
+	var registryName string
+	if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) {
+		registryName = fmt.Sprintf("%s.%s", dockerregistry.RegistryName, ctx.Config.Domain)
+	} else if ctx.Config.ContainerRegistry.External != nil {
+		registryName = strings.TrimSuffix(ctx.Config.ContainerRegistry.External.URL, "/")
+	} else {
+		return nil, fmt.Errorf("%s: invalid container registry config", Component)
+	}
+
 	orchestrator := config.Configuration{
 		WorkspaceManager: config.WorkspaceManagerConfig{
 			Address: fmt.Sprintf("%s:%d", wsmanager.Component, wsmanager.RPCPort),
@@ -32,28 +42,13 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 				PrivateKey:  "/wsman-certs/tls.key",
 			},
 		},
-		BuilderImage:       common.ImageName(ctx.Config.Repository, BuilderImage, ctx.VersionManifest.Components.ImageBuilderMk3.BuilderImage.Version),
-		BuilderAuthKeyFile: "/config/authkey",
+		AuthFile:                 PullSecretFile,
+		BaseImageRepository:      fmt.Sprintf("%s/base-images", registryName),
+		BuilderImage:             common.ImageName(ctx.Config.Repository, BuilderImage, ctx.VersionManifest.Components.ImageBuilderMk3.BuilderImage.Version),
+		BuilderAuthKeyFile:       "/config/authkey",
+		WorkspaceImageRepository: fmt.Sprintf("%s/workspace-images", registryName),
 	}
 
-	var baseImageRepo string
-	var workspaceImgRepo string
-	if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) {
-		// todo(sje): handle external registry
-		registryName := fmt.Sprintf("%s.%s", dockerregistry.RegistryName, ctx.Config.Domain)
-
-		baseImageRepo = fmt.Sprintf("%s/base-images", registryName)
-		workspaceImgRepo = fmt.Sprintf("%s/workspace-images", registryName)
-
-		orchestrator.AuthFile = PullSecretFile
-	} else {
-		// todo(sje): handle outside cluster values for image builder mk3
-		return nil, fmt.Errorf("in cluster container currently only supported option")
-	}
-
-	orchestrator.BaseImageRepository = baseImageRepo
-	orchestrator.WorkspaceImageRepository = workspaceImgRepo
-
 	imgcfg := config.ServiceConfig{
 		Orchestrator: orchestrator,
 		RefCache: config.RefCacheConfig{
diff --git a/installer/pkg/components/image-builder-mk3/deployment.go b/installer/pkg/components/image-builder-mk3/deployment.go
@@ -34,27 +34,21 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 		hashObj = append(hashObj, objs...)
 	}
 
-	var volumes []corev1.Volume
-	var volumeMounts []corev1.VolumeMount
-
+	var secretName string
 	if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) {
-		volumeMounts = append(volumeMounts, corev1.VolumeMount{
-			Name:      "pull-secret",
-			MountPath: PullSecretFile,
-			SubPath:   ".dockerconfigjson",
-		})
-		volumes = append(volumes, corev1.Volume{
-			Name: "pull-secret",
-			VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{
-				SecretName: dockerregistry.BuiltInRegistryAuth,
-			}},
-		})
-		if objs, err := common.DockerRegistryHash(ctx); err != nil {
-			return nil, err
-		} else {
-			hashObj = append(hashObj, objs...)
-		}
+		secretName = dockerregistry.BuiltInRegistryAuth
+	} else if ctx.Config.ContainerRegistry.External != nil {
+		secretName = ctx.Config.ContainerRegistry.External.Certificate.Name
+	} else {
+		return nil, fmt.Errorf("%s: invalid container registry config", Component)
+	}
+
+	if objs, err := common.DockerRegistryHash(ctx); err != nil {
+		return nil, err
+	} else {
+		hashObj = append(hashObj, objs...)
 	}
+
 	configHash, err := common.ObjectHash(hashObj, nil)
 	if err != nil {
 		return nil, err
@@ -87,7 +81,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 					DNSPolicy:                     "ClusterFirst",
 					RestartPolicy:                 "Always",
 					TerminationGracePeriodSeconds: pointer.Int64(30),
-					Volumes: append([]corev1.Volume{{
+					Volumes: []corev1.Volume{{
 						Name: "configuration",
 						VolumeSource: corev1.VolumeSource{
 							ConfigMap: &corev1.ConfigMapVolumeSource{
@@ -108,7 +102,12 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 								SecretName: wsmanager.TLSSecretNameClient,
 							},
 						},
-					}}, volumes...),
+					}, {
+						Name: "pull-secret",
+						VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{
+							SecretName: secretName,
+						}},
+					}},
 					Containers: []corev1.Container{{
 						Name:            Component,
 						Image:           common.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.ImageBuilderMk3.Version),
@@ -136,7 +135,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 							Privileged: pointer.Bool(false),
 							RunAsUser:  pointer.Int64(33333),
 						},
-						VolumeMounts: append([]corev1.VolumeMount{{
+						VolumeMounts: []corev1.VolumeMount{{
 							Name:      "configuration",
 							MountPath: "/config/image-builder.json",
 							SubPath:   "image-builder.json",
@@ -148,7 +147,11 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 							Name:      "wsman-tls-certs",
 							MountPath: "/wsman-certs",
 							ReadOnly:  true,
-						}}, volumeMounts...),
+						}, {
+							Name:      "pull-secret",
+							MountPath: PullSecretFile,
+							SubPath:   ".dockerconfigjson",
+						}},
 					}, *common.KubeRBACProxyContainer()},
 				},
 			},
diff --git a/installer/pkg/components/proxy/configmap.go b/installer/pkg/components/proxy/configmap.go
@@ -10,6 +10,7 @@ import (
 	"encoding/base64"
 	"fmt"
 	minioComponent "github.com/gitpod-io/gitpod/installer/pkg/components/minio"
+	openvsxproxy "github.com/gitpod-io/gitpod/installer/pkg/components/openvsx-proxy"
 	"text/template"
 
 	"github.com/gitpod-io/gitpod/installer/pkg/common"
@@ -18,6 +19,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/utils/pointer"
 )
 
 //go:embed templates/configmap/vhost.docker-registry.tpl
@@ -88,36 +90,9 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 		return nil, err
 	}
 
-	// todo(sje) make conditional
-	// todo(sje): allow value to be set via config
-	username := ctx.Values.InternalRegistryUsername
-	if username == "" {
-		return nil, fmt.Errorf("unknown value: internal registry username")
-	}
-
-	password := ctx.Values.InternalRegistryPassword
-	if password == "" {
-		return nil, fmt.Errorf("unknown value: internal registry password")
-	}
-
-	hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
-	if err != nil {
-		return nil, err
-	}
-
-	dockerRegistry, err := renderTemplate(vhostDockerRegistry, dockerRegistryTpl{
-		Domain:       ctx.Config.Domain,
-		ReverseProxy: fmt.Sprintf("https://%s.%s.%s", common.DockerRegistryName, ctx.Namespace, kubeDomain),
-		Username:     username,
-		Password:     base64.StdEncoding.EncodeToString(hashedPassword),
-	})
-	if err != nil {
-		return nil, err
-	}
-
 	openVSX, err := renderTemplate(vhostOpenVSXTmpl, openVSXTpl{
 		Domain:  ctx.Config.Domain,
-		RepoURL: fmt.Sprintf("openvsx-proxy.%s.%s:%d", ctx.Namespace, kubeDomain, 8080), // todo(sje): get port from (future) config
+		RepoURL: fmt.Sprintf("openvsx-proxy.%s.%s:%d", ctx.Namespace, kubeDomain, openvsxproxy.ServicePort),
 	})
 	if err != nil {
 		return nil, err
@@ -140,6 +115,43 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 		return nil, err
 	}
 
+	data := map[string]string{
+		"vhost.empty":            *empty,
+		"vhost.minio":            *minio,
+		"vhost.open-vsx":         *openVSX,
+		"vhost.payment-endpoint": *paymentEndpoint,
+		"vhost.kedge":            *kedge,
+	}
+
+	if pointer.BoolDeref(ctx.Config.ContainerRegistry.InCluster, false) {
+		username := ctx.Values.InternalRegistryUsername
+		if username == "" {
+			return nil, fmt.Errorf("unknown value: internal registry username")
+		}
+
+		password := ctx.Values.InternalRegistryPassword
+		if password == "" {
+			return nil, fmt.Errorf("unknown value: internal registry password")
+		}
+
+		hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
+		if err != nil {
+			return nil, err
+		}
+
+		dockerRegistry, err := renderTemplate(vhostDockerRegistry, dockerRegistryTpl{
+			Domain:       ctx.Config.Domain,
+			ReverseProxy: fmt.Sprintf("https://%s.%s.%s", common.DockerRegistryName, ctx.Namespace, kubeDomain),
+			Username:     username,
+			Password:     base64.StdEncoding.EncodeToString(hashedPassword),
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		data["vhost.docker-registry"] = *dockerRegistry
+	}
+
 	return []runtime.Object{
 		&corev1.ConfigMap{
 			TypeMeta: common.TypeMetaConfigmap,
@@ -148,14 +160,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 				Namespace: ctx.Namespace,
 				Labels:    common.DefaultLabels(Component),
 			},
-			Data: map[string]string{
-				"vhost.empty":            *empty,
-				"vhost.minio":            *minio,
-				"vhost.docker-registry":  *dockerRegistry,
-				"vhost.open-vsx":         *openVSX,
-				"vhost.payment-endpoint": *paymentEndpoint,
-				"vhost.kedge":            *kedge,
-			},
+			Data: data,
 		},
 	}, nil
 }
diff --git a/installer/pkg/components/registry-facade/configmap.go b/installer/pkg/components/registry-facade/configmap.go
@@ -48,7 +48,6 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 				Type: "image",
 			}},
 		},
-		// todo(sje): only enabled if the pullSecret is not nil in daemonset
 		AuthCfg:        "/mnt/pull-secret.json",
 		PProfAddr:      ":6060",
 		PrometheusAddr: "127.0.0.1:9500",
diff --git a/installer/pkg/components/registry-facade/daemonset.go b/installer/pkg/components/registry-facade/daemonset.go
diff --git a/installer/pkg/components/ws-manager/configmap_test.go b/installer/pkg/components/ws-manager/configmap_test.go
