⚠️ Serve catalog over HTTPS (#263)

* make catalog server serve catalog contents over HTTPS

adds cert-manager as a dependency again to create self-signed
certs for the catalog server

Signed-off-by: everettraven <[email protected]>

* fix e2e

Signed-off-by: everettraven <[email protected]>

* Reorganize manifests for cert-manager overlay

This allows the use of alternate certificate managers.

Signed-off-by: Tayler Geiger <[email protected]>

* Reconfigure TLS functionality to use Listener

Fix a few manifest issues as well.

Signed-off-by: Tayler Geiger <[email protected]>

* Add certwatcher for TLS cert and key from controller-runtime

- Add error for missing either tls-key or tls-cert arguments.
- Move server creation and configuration to serverutil

Signed-off-by: Tayler Geiger <[email protected]>

* Update README and docs for HTTPS

---------

Signed-off-by: everettraven <[email protected]>
Signed-off-by: Tayler Geiger <[email protected]>
Co-authored-by: everettraven <[email protected]>

Author: trgeiger

.goreleaser.yml

@@ -66,6 +66,8 @@ release:
   header: |
     ## Installation
     ```bash
+    kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml
+    kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-webhook --timeout=60s
     kubectl apply -f https://github.com/operator-framework/catalogd/releases/download/{{ .Tag }}/catalogd.yaml
     kubectl wait --for=condition=Available --namespace=catalogd-system deployment/catalogd-controller-manager --timeout=60s
     ```

Makefile

@@ -48,7 +48,7 @@ clean: ## Remove binaries and test artifacts
 .PHONY: generate
 generate: $(CONTROLLER_GEN) ## Generate code and manifests.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
-	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/base/crd/bases
 
 .PHONY: fmt
 fmt: ## Run go fmt against code.
@@ -152,20 +152,24 @@ kind-load: $(KIND) ## Load the built images onto the local cluster
 	$(KIND) load docker-image $(IMAGE) --name $(KIND_CLUSTER_NAME)
 
 .PHONY: install
-install: build-container kind-load deploy wait ## Install local catalogd
+install: build-container kind-load cert-manager deploy wait ## Install local catalogd
 
 .PHONY: deploy
 deploy: $(KUSTOMIZE) ## Deploy Catalogd to the K8s cluster specified in ~/.kube/config.
-	cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMAGE)
-	$(KUSTOMIZE) build config/default | kubectl apply -f -
+	cd config/base/manager && $(KUSTOMIZE) edit set image controller=$(IMAGE)
+	$(KUSTOMIZE) build config/overlays/cert-manager | kubectl apply -f -
 
 .PHONY: undeploy
 undeploy: $(KUSTOMIZE) ## Undeploy Catalogd from the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=true -f -
+	$(KUSTOMIZE) build config/overlays/cert-manager | kubectl delete --ignore-not-found=true -f -
 
 wait:
 	kubectl wait --for=condition=Available --namespace=$(CATALOGD_NAMESPACE) deployment/catalogd-controller-manager --timeout=60s
 
+.PHONY: cert-manager
+cert-manager:
+	kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/${CERT_MGR_VERSION}/cert-manager.yaml
+	kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-webhook --timeout=60s
 ##@ Release
 
 export ENABLE_RELEASE_PIPELINE ?= false
@@ -175,7 +179,7 @@ release: $(GORELEASER) ## Runs goreleaser for catalogd. By default, this will ru
 	$(GORELEASER) $(GORELEASER_ARGS)
 
 quickstart: $(KUSTOMIZE) generate ## Generate the installation release manifests and scripts
-	$(KUSTOMIZE) build config/default | sed "s/:devel/:$(GIT_VERSION)/g" > catalogd.yaml
+	$(KUSTOMIZE) build config/overlays/cert-manager | sed "s/:devel/:$(GIT_VERSION)/g" > catalogd.yaml
 
 .PHONY: demo-update
 demo-update:

README.md

@@ -8,7 +8,9 @@ Catalogd helps customers discover installable content by hosting catalog metadat
 [![asciicast](https://asciinema.org/a/624043.svg)](https://asciinema.org/a/624043)
 
 ## Quickstart Steps
-**NOTE:** Procedure steps marked with an asterisk (`*`) are likely to change with future API updates.
+Procedure steps marked with an asterisk (`*`) are likely to change with future API updates.
+
+**NOTE:** The examples below use the `-k` flag in curl to skip validating the TLS certificates. This is for demonstration purposes only.
 
 1. To install catalogd, navigate to the [releases](https://github.com/operator-framework/catalogd/releases/) page, and follow the install instructions included in the release you want to install.
 
@@ -95,13 +97,13 @@ Catalogd helps customers discover installable content by hosting catalog metadat
 
 1. Port forward the `catalogd-catalogserver` service in the `catalogd-system` namespace:
     ```sh
-    $ kubectl -n catalogd-system port-forward svc/catalogd-catalogserver 8080:80
+    $ kubectl -n catalogd-system port-forward svc/catalogd-catalogserver 8080:443
     ```
 
 1. Run the following command to get a list of packages:
 
     ```sh
-    $ curl http://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.package") | .name'
+    $ curl -k https://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.package") | .name'
     ```
 
     *Example output*
@@ -128,7 +130,7 @@ Catalogd helps customers discover installable content by hosting catalog metadat
 1. Run the following command to get a list of channels for the `ack-acm-controller` package:
 
     ```sh
-    $ curl http://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.channel") | select(.package == "ack-acm-controller") | .name'
+    $ curl -k https://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.channel") | select(.package == "ack-acm-controller") | .name'
     ```
 
     *Example output*
@@ -142,7 +144,7 @@ Catalogd helps customers discover installable content by hosting catalog metadat
 1. Run the following command to get a list of bundles belonging to the `ack-acm-controller` package:
 
     ```sh
-    $ curl http://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.bundle") | select(.package == "ack-acm-controller") | .name'
+    $ curl -k https://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.bundle") | select(.package == "ack-acm-controller") | .name'
     ```
     
     *Example output*

Tiltfile

@@ -5,7 +5,7 @@ load('../tilt-support/Tiltfile', 'deploy_repo')
 
 repo = {
     'image': 'quay.io/operator-framework/catalogd',
-    'yaml': 'config/default',
+    'yaml': 'config/overlays/cert-manager',
     'binaries': {
         'manager': 'catalogd-controller-manager',
     },

api/core/v1alpha1/catalog_types_test.go

@@ -20,7 +20,7 @@ import (
 )
 
 func TestPollIntervalCELValidationRules(t *testing.T) {
-	validators := fieldValidatorsFromFile(t, "../../../config/crd/bases/catalogd.operatorframework.io_catalogs.yaml")
+	validators := fieldValidatorsFromFile(t, "../../../config/base/crd/bases/catalogd.operatorframework.io_catalogs.yaml")
 	pth := "openAPIV3Schema.properties.spec"
 	validator, found := validators["v1alpha1"][pth]
 	assert.True(t, found)

cmd/manager/main.go

@@ -19,7 +19,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -39,8 +38,8 @@ import (
 
 	"github.com/operator-framework/catalogd/api/core/v1alpha1"
 	"github.com/operator-framework/catalogd/internal/garbagecollection"
+	"github.com/operator-framework/catalogd/internal/serverutil"
 	"github.com/operator-framework/catalogd/internal/source"
-	"github.com/operator-framework/catalogd/internal/third_party/server"
 	"github.com/operator-framework/catalogd/internal/version"
 	corecontrollers "github.com/operator-framework/catalogd/pkg/controllers/core"
 	"github.com/operator-framework/catalogd/pkg/features"
@@ -71,9 +70,11 @@ func main() {
 		catalogdVersion      bool
 		systemNamespace      string
 		catalogServerAddr    string
-		httpExternalAddr     string
+		externalAddr         string
 		cacheDir             string
 		gcInterval           time.Duration
+		certFile             string
+		keyFile              string
 	)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
@@ -83,10 +84,12 @@ func main() {
 			"Enabling this will ensure there is only one active controller manager.")
 	flag.StringVar(&systemNamespace, "system-namespace", "", "The namespace catalogd uses for internal state, configuration, and workloads")
 	flag.StringVar(&catalogServerAddr, "catalogs-server-addr", ":8083", "The address where the unpacked catalogs' content will be accessible")
-	flag.StringVar(&httpExternalAddr, "http-external-address", "http://catalogd-catalogserver.catalogd-system.svc", "The external address at which the http server is reachable.")
+	flag.StringVar(&externalAddr, "external-address", "catalogd-catalogserver.catalogd-system.svc", "The external address at which the http(s) server is reachable.")
 	flag.StringVar(&cacheDir, "cache-dir", "/var/cache/", "The directory in the filesystem that catalogd will use for file based caching")
 	flag.BoolVar(&catalogdVersion, "version", false, "print the catalogd version and exit")
 	flag.DurationVar(&gcInterval, "gc-interval", 12*time.Hour, "interval in which garbage collection should be run against the catalog content cache")
+	flag.StringVar(&certFile, "tls-cert", "", "The certificate file used for serving catalog contents over HTTPS. Requires tls-key.")
+	flag.StringVar(&keyFile, "tls-key", "", "The key file used for serving catalog contents over HTTPS. Requires tls-cert.")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -103,6 +106,18 @@ func main() {
 	}
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+
+	if (certFile != "" && keyFile == "") || (certFile == "" && keyFile != "") {
+		setupLog.Error(nil, "unable to configure TLS certificates: tls-cert and tls-key flags must be used together")
+		os.Exit(1)
+	}
+
+	protocol := "http://"
+	if certFile != "" && keyFile != "" {
+		protocol = "https://"
+	}
+	externalAddr = protocol + externalAddr
+
 	cfg := ctrl.GetConfigOrDie()
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
 		Scheme: scheme,
@@ -143,29 +158,25 @@ func main() {
 		os.Exit(1)
 	}
 
-	baseStorageURL, err := url.Parse(fmt.Sprintf("%s/catalogs/", httpExternalAddr))
+	baseStorageURL, err := url.Parse(fmt.Sprintf("%s/catalogs/", externalAddr))
 	if err != nil {
 		setupLog.Error(err, "unable to create base storage URL")
 		os.Exit(1)
 	}
 
 	localStorage = storage.LocalDir{RootDir: storeDir, BaseURL: baseStorageURL}
-	shutdownTimeout := 30 * time.Second
-	catalogServer := server.Server{
-		Kind: "catalogs",
-		Server: &http.Server{
-			Addr:        catalogServerAddr,
-			Handler:     catalogdmetrics.AddMetricsToHandler(localStorage.StorageServerHandler()),
-			ReadTimeout: 5 * time.Second,
-			// TODO: Revert this to 10 seconds if/when the API
-			// evolves to have significantly smaller responses
-			WriteTimeout: 5 * time.Minute,
-		},
-		ShutdownTimeout: &shutdownTimeout,
+
+	catalogServerConfig := serverutil.CatalogServerConfig{
+		ExternalAddr: externalAddr,
+		CatalogAddr:  catalogServerAddr,
+		CertFile:     certFile,
+		KeyFile:      keyFile,
+		LocalStorage: localStorage,
 	}
 
-	if err := mgr.Add(&catalogServer); err != nil {
-		setupLog.Error(err, "unable to start catalog server")
+	err = serverutil.AddCatalogServerToManager(mgr, catalogServerConfig)
+	if err != nil {
+		setupLog.Error(err, "unable to configure catalog server")
 		os.Exit(1)
 	}
 
@@ -202,7 +213,7 @@ func main() {
 		Interval:       gcInterval,
 	}
 	if err := mgr.Add(gc); err != nil {
-		setupLog.Error(err, "problem adding garbage collector to manager")
+		setupLog.Error(err, "unable to add garbage collector to manager")
 		os.Exit(1)
 	}
 

config/crd/bases/catalogd.operatorframework.io_catalogs.yaml renamed to config/base/crd/bases/catalogd.operatorframework.io_catalogs.yaml

@@ -15,4 +15,3 @@ resources:
 - ../crd
 - ../rbac
 - ../manager
-

config/crd/kustomization.yaml renamed to config/base/crd/kustomization.yaml

@@ -76,7 +76,7 @@ spec:
         args:
         - --leader-elect
         - --metrics-bind-address=127.0.0.1:8080
-        - --http-external-address=http://catalogd-catalogserver.catalogd-system.svc
+        - --external-address=catalogd-catalogserver.catalogd-system.svc
         image: controller:latest
         name: manager
         volumeMounts:

config/crd/patches/catalog_validation.yaml renamed to config/base/crd/patches/catalog_validation.yaml

@@ -0,0 +1,65 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: manager-role
+rules:
+- apiGroups:
+  - catalogd.operatorframework.io
+  resources:
+  - catalogs
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - catalogd.operatorframework.io
+  resources:
+  - catalogs/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - catalogd.operatorframework.io
+  resources:
+  - catalogs/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/log
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: manager-role
+  namespace: system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get

config/default/kustomization.yaml renamed to config/base/default/kustomization.yaml

@@ -0,0 +1,28 @@
+# Adds namespace to all resources.
+namespace: catalogd-system
+
+# Value of this field is prepended to the
+# names of all resources, e.g. a deployment named
+# "wordpress" becomes "alices-wordpress".
+# Note that it should also match with the prefix (text before '-') of the namespace
+# field above.
+namePrefix: catalogd-
+
+# the following config is for teaching kustomize how to do var substitution
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../base/crd
+- ../../base/rbac
+- ../../base/manager
+- resources
+
+patches:
+- target:
+    kind: Service
+    name: catalogserver
+  path: patches/catalogserver_service_port.yaml
+- target:
+    kind: Deployment
+    name: controller-manager
+  path: patches/manager_deployment_certs.yaml

config/manager/catalogserver_service.yaml renamed to config/base/manager/catalogserver_service.yaml

@@ -0,0 +1,6 @@
+- op: replace
+  path: /spec/ports/0/port
+  value: 443
+- op: replace
+  path: /spec/ports/0/name
+  value: https

config/manager/kustomization.yaml renamed to config/base/manager/kustomization.yaml

@@ -0,0 +1,12 @@
+- op: add
+  path: /spec/template/spec/volumes/-
+  value: {"name":"catalogserver-certs", "secret":{"secretName":"catalogd-catalogserver-cert"}}
+- op: add
+  path: /spec/template/spec/containers/1/volumeMounts/-
+  value: {"name":"catalogserver-certs", "mountPath":"/var/certs"}
+- op: add
+  path: /spec/template/spec/containers/1/args/-
+  value: "--tls-cert=/var/certs/tls.crt"
+- op: add
+  path: /spec/template/spec/containers/1/args/-
+  value: "--tls-key=/var/certs/tls.key"

config/manager/manager.yaml renamed to config/base/manager/manager.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: catalogserver-cert
+  namespace: system
+spec:
+  secretName: catalogd-catalogserver-cert
+  dnsNames:
+    - localhost
+    - catalogd-catalogserver.catalogd-system.svc
+  issuerRef:
+    kind: Issuer
+    name: catalogd-catalogserver-ca-issuer