(e2e) Use custom built test catalog for e2e testing

This PR
* Stands up a local image registry in the cluster
* Builds bundles and catalog images and uploads them to the image registry
* Uses the custom images in the e2e test suite

* Also introduces a `pullSecret` field for the Operator API's Spec struct,
to allow installation of operators on cluster whose bundles require
imagePullSecret to be provisioned. This is required because the bundle
images built and uploaded to the local registry above requires a pull
secret for the local registry.

closes #215

Author: anik120

Makefile

@@ -11,10 +11,17 @@ export GORELEASER_VERSION ?= v1.16.2
 export RUKPAK_VERSION=$(shell go list -mod=mod -m -f "{{.Version}}" github.com/operator-framework/rukpak)
 export WAIT_TIMEOUT ?= 60s
 IMG?=$(IMAGE_REPO):$(IMAGE_TAG)
+TESTDATA_DIR := testdata
 
 OPERATOR_CONTROLLER_NAMESPACE ?= operator-controller-system
 KIND_CLUSTER_NAME ?= operator-controller
 
+REGISTRY_NAME="docker-registry"
+REGISTRY_NAMESPACE=operator-controller-e2e
+DNS_NAME=$(REGISTRY_NAME).$(REGISTRY_NAMESPACE).svc.cluster.local
+
+CONTAINER_RUNTIME ?= docker
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -82,7 +89,7 @@ test-unit: envtest ## Run the unit tests
 	eval $$($(ENVTEST) use -p env $(ENVTEST_VERSION)) && go test -tags $(GO_BUILD_TAGS) -count=1 -short $(UNIT_TEST_DIRS) -coverprofile cover.out
 
 e2e: KIND_CLUSTER_NAME=operator-controller-e2e
-e2e: run test-e2e kind-cluster-cleanup ## Run e2e test suite on local kind cluster
+e2e: run image-registry kind-load-test-artifacts registry-load-test-artifacts test-e2e kind-cluster-cleanup ## Run e2e test suite on local kind cluster
 
 kind-load: kind ## Loads the currently constructed image onto the cluster
 	$(KIND) load docker-image $(IMG) --name $(KIND_CLUSTER_NAME)
@@ -94,6 +101,20 @@ kind-cluster: kind kind-cluster-cleanup ## Standup a kind cluster
 kind-cluster-cleanup: kind ## Delete the kind cluster
 	$(KIND) delete cluster --name ${KIND_CLUSTER_NAME}
 
+image-registry: ## Setup in-cluster image registry
+	./test/tools/imageregistry/setup_imageregistry.sh ${KIND_CLUSTER_NAME}
+
+kind-load-test-artifacts: kind ## Load the e2e testdata container images into a kind cluster
+	$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.47.0 -t localhost/testdata/bundles/registry-v1/prometheus-operator.v0.47.0
+	$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/catalogs -f $(TESTDATA_DIR)/catalogs/test-catalog.Dockerfile -t localhost/testdata/catalogs/test-catalog
+	$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator.v0.47.0 --name $(KIND_CLUSTER_NAME)
+	$(KIND) load docker-image localhost/testdata/catalogs/test-catalog --name $(KIND_CLUSTER_NAME)
+
+registry-load-test-artifacts: ## Load e2e testdata container images created in kind-load-test-artifacts into registry
+	$(CONTAINER_RUNTIME) tag localhost/testdata/bundles/registry-v1/prometheus-operator.v0.47.0 $(DNS_NAME):5000/bundles/registry-v1/prometheus-operator.v0.47.0
+	$(CONTAINER_RUNTIME) tag localhost/testdata/catalogs/test-catalog $(DNS_NAME):5000/catalogs/test-catalog
+	./test/tools/imageregistry/load_test_image.sh $(KIND) $(KIND_CLUSTER_NAME)
+
 ##@ Build
 
 BUILDCMD = sh -c 'mkdir -p $(BUILDBIN) && ${GORELEASER} build ${GORELEASER_ARGS} --single-target -o $(BUILDBIN)/manager'

api/v1alpha1/operator_types.go

@@ -42,6 +42,11 @@ type OperatorSpec struct {
 	//+kubebuilder:validation:Pattern:=^[a-z0-9]+([\.-][a-z0-9]+)*$
 	// Channel constraint defintion
 	Channel string `json:"channel,omitempty"`
+
+	//+kubebuilder:validation:MaxLength:=48
+	//+kubebuilder:validation:Pattern:=^[a-z0-9]+(-[a-z0-9]+)*$
+	// ImagePullSecretName contains the name of the image pull secret that will be used to provision the operator bundle, in the rukpak-system namespace.
+	ImagePullSecretName string `json:"pullSecret,omitempty"` // TODO: Find a better place for this field. This is only valid if the package bundles are OCI images.
 }
 
 const (

config/crd/bases/operators.operatorframework.io_operators.yaml

@@ -44,6 +44,13 @@ spec:
                 maxLength: 48
                 pattern: ^[a-z0-9]+(-[a-z0-9]+)*$
                 type: string
+              pullSecret:
+                description: ImagePullSecretName contains the name of the image pull
+                  secret that will be used to provision the operator bundle, in the
+                  rukpak-system namespace.
+                maxLength: 48
+                pattern: ^[a-z0-9]+(-[a-z0-9]+)*$
+                type: string
               version:
                 description: "Version is an optional semver constraint on the package
                   version. If not specified, the latest version available of the package

internal/controllers/operator_controller.go

@@ -241,6 +241,12 @@ func (r *OperatorReconciler) getBundleEntityFromSolution(solution *solver.Soluti
 }
 
 func (r *OperatorReconciler) generateExpectedBundleDeployment(o operatorsv1alpha1.Operator, bundlePath string) *unstructured.Unstructured {
+	bdImage := map[string]interface{}{
+		"ref": bundlePath,
+	}
+	if o.Spec.ImagePullSecretName != "" {
+		bdImage["pullSecret"] = o.Spec.ImagePullSecretName
+	}
 	// We use unstructured here to avoid problems of serializing default values when sending patches to the apiserver.
 	// If you use a typed object, any default values from that struct get serialized into the JSON patch, which could
 	// cause unrelated fields to be patched back to the default value even though that isn't the intention. Using an
@@ -261,10 +267,8 @@ func (r *OperatorReconciler) generateExpectedBundleDeployment(o operatorsv1alpha
 					"provisionerClassName": "core-rukpak-io-registry",
 					"source": map[string]interface{}{
 						// TODO: Don't assume image type
-						"type": string(rukpakv1alpha1.SourceTypeImage),
-						"image": map[string]interface{}{
-							"ref": bundlePath,
-						},
+						"type":  string(rukpakv1alpha1.SourceTypeImage),
+						"image": bdImage,
 					},
 				},
 			},

test/e2e/install_test.go

@@ -17,8 +17,10 @@ import (
 )
 
 const (
-	defaultTimeout = 30 * time.Second
-	defaultPoll    = 1 * time.Second
+	defaultTimeout      = 30 * time.Second
+	defaultPoll         = 1 * time.Second
+	testCatalogRef      = "docker-registry.operator-controller-e2e.svc.cluster.local:5000/catalogs/test-catalog:latest"
+	localRegistrySecret = "registrysecret"
 )
 
 var _ = Describe("Operator Install", func() {
@@ -32,14 +34,15 @@ var _ = Describe("Operator Install", func() {
 	When("An operator is installed from an operator catalog", func() {
 		BeforeEach(func() {
 			ctx = context.Background()
-			pkgName = "argocd-operator"
+			pkgName = "prometheus"
 			operatorName = fmt.Sprintf("operator-%s", rand.String(8))
 			operator = &operatorv1alpha1.Operator{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: operatorName,
 				},
 				Spec: operatorv1alpha1.OperatorSpec{
-					PackageName: pkgName,
+					PackageName:         pkgName,
+					ImagePullSecretName: localRegistrySecret,
 				},
 			}
 			operatorCatalog = &catalogd.Catalog{
@@ -50,9 +53,8 @@ var _ = Describe("Operator Install", func() {
 					Source: catalogd.CatalogSource{
 						Type: catalogd.SourceTypeImage,
 						Image: &catalogd.ImageSource{
-							// (TODO): Set up a local image registry, and build and store a test catalog in it
-							// to use in the test suite
-							Ref: "quay.io/operatorhubio/catalog:latest",
+							Ref:        testCatalogRef,
+							PullSecret: localRegistrySecret,
 						},
 					},
 				},
@@ -64,7 +66,7 @@ var _ = Describe("Operator Install", func() {
 				g.Expect(err).ToNot(HaveOccurred())
 				g.Expect(len(operatorCatalog.Status.Conditions)).To(Equal(1))
 				g.Expect(operatorCatalog.Status.Conditions[0].Message).To(ContainSubstring("successfully unpacked the catalog image"))
-			}).WithTimeout(5 * time.Minute).WithPolling(defaultPoll).Should(Succeed())
+			}).WithTimeout(1 * time.Minute).WithPolling(defaultPoll).Should(Succeed())
 		})
 		It("resolves the specified package with correct bundle path", func() {
 			By("creating the Operator resource")

test/tools/imageregistry/bundle_local_image.yaml

@@ -0,0 +1,11 @@
+apiVersion: core.rukpak.io/v1alpha1
+kind: Bundle
+metadata:
+  name: prometheus.v0.47.0
+spec:
+  source:
+    type: image
+    image:
+      ref: docker-registry.operator-controller-e2e.svc.cluster.local:5000/bundles/prometheus:0.47.0
+      pullSecret: registrysecret
+  provisionerClassName: core-rukpak-io-registry

test/tools/imageregistry/daemonset.yaml

@@ -0,0 +1,46 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: setup-script
+data:
+  setup.sh: |
+    echo "$TRUSTED_CERT" > /usr/local/share/ca-certificates/ca.crt && update-ca-certificates && systemctl restart containerd
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: node-custom-setup
+  labels:
+    k8s-app: node-custom-setup
+spec:
+  selector:
+    matchLabels:
+      k8s-app: node-custom-setup
+  template:
+    metadata:
+      labels:
+        k8s-app: node-custom-setup
+    spec:
+      hostPID: true
+      hostNetwork: true
+      initContainers:
+      - name: init-node
+        command: ["nsenter"]
+        args: ["--mount=/proc/1/ns/mnt", "--", "sh", "-c", "$(SETUP_SCRIPT)"]
+        image: debian
+        env:
+        - name: TRUSTED_CERT
+          valueFrom:
+            configMapKeyRef:
+              name: trusted-ca
+              key: ca.crt
+        - name: SETUP_SCRIPT
+          valueFrom:
+            configMapKeyRef:
+              name: setup-script
+              key: setup.sh
+        securityContext:
+          privileged: true
+      containers:
+      - name: wait
+        image: registry.k8s.io/pause:3.1

test/tools/imageregistry/load_test_image.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+export REGISTRY_NAME="docker-registry"
+export REGISTRY_NAMESPACE=operator-controller-e2e
+export DNS_NAME=$REGISTRY_NAME.$REGISTRY_NAMESPACE.svc.cluster.local
+KIND=$1
+KIND_CLUSTER_NAME=$2
+
+# push test bundle image into in-cluster docker registry
+kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl login -u myuser -p mypasswd $DNS_NAME:5000 --insecure-registry"
+
+for x in $(docker images --format "{{.Repository}}:{{.Tag}}" | grep $DNS_NAME); do
+echo $x
+    $KIND load docker-image $x --name $KIND_CLUSTER_NAME
+    kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl -n k8s.io push $x --insecure-registry"
+    kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl -n k8s.io rmi $x --insecure-registry"
+done
+

test/tools/imageregistry/nerdctl.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nerdctl
+spec:
+  containers:
+  - command:
+    - sleep
+    - infinity
+    image: ghcr.io/containerd/nerdctl
+    imagePullPolicy: Always
+    name: nerdctl
+    volumeMounts:
+    - mountPath: /run/containerd
+      name: run-containerd
+    - mountPath: /var/lib/containerd
+      name: var-lib-containerd
+  volumes:
+  - name: run-containerd
+    hostPath:
+      path: /run/containerd
+  - name: var-lib-containerd
+    hostPath:
+      path: /var/lib/containerd

test/tools/imageregistry/registry.yaml

@@ -0,0 +1,49 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: docker-registry-pod
+  labels:
+    app: registry
+spec:
+  initContainers:
+  - name: auth
+    image: registry:2.6.2
+    command:
+    - "sh"
+    - "-c"
+    - "htpasswd -Bbn myuser mypasswd >> /auth/htpasswd"
+    volumeMounts:
+    - name: auth-vol
+      mountPath: "/auth"
+  containers:
+  - name: registry
+    image: registry:2.6.2
+    volumeMounts:
+    - name: repo-vol
+      mountPath: "/var/lib/registry"
+    - name: certs-vol
+      mountPath: "/certs"
+      readOnly: true
+    - name: auth-vol
+      mountPath: "/auth"
+      readOnly: true
+    env:
+    - name: REGISTRY_AUTH
+      value: "htpasswd"
+    - name: REGISTRY_AUTH_HTPASSWD_REALM
+      value: "Registry Realm"
+    - name: REGISTRY_AUTH_HTPASSWD_PATH
+      value: "/auth/htpasswd"
+    - name: REGISTRY_HTTP_TLS_CERTIFICATE
+      value: "/certs/tls.crt"
+    - name: REGISTRY_HTTP_TLS_KEY
+      value: "/certs/tls.key"
+  volumes:
+  - name: repo-vol
+    emptyDir: {}
+  - name: certs-vol
+    secret:
+      secretName: certs-secret
+  - name: auth-vol
+    emptyDir: {}
+

test/tools/imageregistry/service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: docker-registry
+spec:
+  selector:
+    app: registry
+  ports:
+  - port: 5000
+    targetPort: 5000

test/tools/imageregistry/setup_imageregistry.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+export REGISTRY_NAME="docker-registry"
+export REGISTRY_NAMESPACE=operator-controller-e2e
+export DNS_NAME=$REGISTRY_NAME.$REGISTRY_NAMESPACE.svc.cluster.local
+export KIND_CLUSTER_NAME=$1
+
+kubectl create ns $REGISTRY_NAMESPACE || true
+
+# create self-signed certificate for registry server
+mkdir -p /tmp/var/imageregistry/certs
+openssl req -x509 -newkey rsa:4096 -days 365 -nodes -sha256 -keyout /tmp/var/imageregistry/certs/tls.key -out /tmp/var/imageregistry/certs/tls.crt -subj "/CN=$DNS_NAME" -addext "subjectAltName = DNS:$DNS_NAME"
+kubectl create secret tls certs-secret --cert=/tmp/var/imageregistry/certs/tls.crt --key=/tmp/var/imageregistry/certs/tls.key -n $REGISTRY_NAMESPACE
+kubectl create configmap trusted-ca -n $REGISTRY_NAMESPACE --from-file=ca.crt=/tmp/var/imageregistry/certs/tls.crt
+
+# create image registry service
+kubectl apply -f test/tools/imageregistry/service.yaml -n $REGISTRY_NAMESPACE
+
+# set local variables
+export REGISTRY_IP=$(kubectl get service $REGISTRY_NAME -n $REGISTRY_NAMESPACE -o jsonpath='{ .spec.clusterIP }')
+export REGISTRY_PORT=5000
+
+# Add ca certificate to Node
+kubectl apply -f test/tools/imageregistry/daemonset.yaml -n $REGISTRY_NAMESPACE
+
+# Add an entry in /etc/hosts of Node
+docker exec $(docker ps | grep $KIND_CLUSTER_NAME'-control-plane' | cut -c 1-12) sh -c "/usr/bin/echo $REGISTRY_IP $DNS_NAME >>/etc/hosts"
+
+sleep 5
+# create image registry pod
+kubectl apply -f test/tools/imageregistry/registry.yaml -n $REGISTRY_NAMESPACE
+
+# create image upload  pod
+kubectl apply -f test/tools/imageregistry/nerdctl.yaml -n $REGISTRY_NAMESPACE
+
+# create imagePull secret for provisioner
+export IMAGE_PULL_RECRET="registrysecret"
+kubectl create secret docker-registry $IMAGE_PULL_RECRET --docker-server=$DNS_NAME:5000 --docker-username="myuser" --docker-password="mypasswd" --docker-email="[email protected]" -n rukpak-system
+kubectl create secret docker-registry $IMAGE_PULL_RECRET --docker-server=$DNS_NAME:5000 --docker-username="myuser" --docker-password="mypasswd" --docker-email="[email protected]" -n catalogd-system
+
+echo #### Valiables ####
+echo
+echo REGISTRY_NAME     $REGISTRY_NAME
+echo REGISTRY_IP       $REGISTRY_IP
+echo REGISTRY_PORT     $REGISTRY_PORT
+echo IMAGE_PULL_RECRET $IMAGE_PULL_RECRET
+
+# clean up 
+rm -rf /tmp/var/imageregistry/certs
+kubectl wait --for=condition=ContainersReady --namespace=$REGISTRY_NAMESPACE pod/docker-registry-pod --timeout=60s
+kubectl wait --for=condition=ContainersReady --namespace=$REGISTRY_NAMESPACE pod/nerdctl --timeout=60s
+

test/tools/imageregistry/sniff.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+export REGISTRY_NAME="docker-registry"
+export REGISTRY_NAMESPACE=operator-controller-e2e
+export DNS_NAME=$REGISTRY_NAME.$REGISTRY_NAMESPACE.svc.cluster.local
+export KIND_CLUSTER_NAME=$1
+
+# push test bundle image into in-cluster docker registry
+kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl login -u myuser -p mypasswd $DNS_NAME:5000 --insecure-registry"
+
+docker build testdata/bundles/registry-v1/prometheus-operator.v0.47.0 -t localhost/testdata/bundles/registry-v1:prometheus-operator:v0.47.0
+kind load docker-image localhost/testdata/bundles/registry-v1:prometheus-operator:v0.47.0 --name $KIND_CLUSTER_NAME
+kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl -n k8s.io tag localhost/testdata/bundles/registry-v1:prometheus-operator:v0.47.0 $DNS_NAME:5000/bundles/registry-v1:prometheus-operator:v0.47.0"
+kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl -n k8s.io push $DNS_NAME:5000/bundles/registry-v1:prometheus-operator:v0.47.0 --insecure-registry"
+kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl -n k8s.io rmi $DNS_NAME:5000/bundles/registry-v1:prometheus-operator:v0.47.0 --insecure-registry"
+
+# create bundle
+kubectl apply -f tools/imageregistry/bundle_local_image.yaml

testdata/bundles/registry-v1/prometheus-operator.v0.47.0/Dockerfile

@@ -0,0 +1,3 @@
+FROM scratch
+COPY manifests /manifests
+COPY metadata /metadata