From 3e18957d5189e57f757c6e782f19b3aa90070b70 Mon Sep 17 00:00:00 2001 From: dtfranz Date: Tue, 3 Jan 2023 17:00:03 -0800 Subject: [PATCH 1/5] Adding e2e make target, makefile adjustments, and e2e test framework Signed-off-by: dtfranz --- Makefile | 55 ++++++++++++++++++++++++---- config/manager/kustomization.yaml | 6 +++ config/manager/manager.yaml | 1 + test/e2e/e2e_suite_test.go | 61 +++++++++++++++++++++++++++++++ test/e2e/reconcile_test.go | 44 ++++++++++++++++++++++ 5 files changed, 160 insertions(+), 7 deletions(-) create mode 100644 test/e2e/e2e_suite_test.go create mode 100644 test/e2e/reconcile_test.go diff --git a/Makefile b/Makefile index 2f361a625..fc3effda3 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,15 @@ - +########################### +# Configuration Variables # +########################### # Image URL to use all building/pushing image targets -IMG ?= controller:latest +export IMAGE_REPO ?= quay.io/operator-framework/operator-controller +export IMAGE_TAG ?= latest +export GO_BUILD_TAGS ?= upstream +IMG?=$(IMAGE_REPO):$(IMAGE_TAG) + +OPERATOR_CONTROLLER_NAMESPACE ?= operator-controller-system +KIND_CLUSTER_NAME ?= operator-controller + # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.25.0 @@ -54,10 +63,27 @@ fmt: ## Run go fmt against code. vet: ## Run go vet against code. go vet ./... -.PHONY: test -test: manifests generate fmt vet envtest ## Run tests. +.PHONY: test test-e2e e2e kind-load kind-cluster kind-cluster-cleanup +test: manifests generate fmt vet envtest test-e2e ## Run all tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out +FOCUS := $(if $(TEST),-v -focus "$(TEST)") +E2E_FLAGS ?= "" +test-e2e: ginkgo ## Run the e2e tests + $(GINKGO) --tags $(GO_BUILD_TAGS) $(E2E_FLAGS) -trace -progress $(FOCUS) test/e2e + +e2e: run test-e2e kind-cluster-cleanup ## Run e2e test suite on local kind cluster + +kind-load: kind ## Loads the currently constructed image onto the cluster + $(KIND) load docker-image $(IMG) --name $(KIND_CLUSTER_NAME) + +kind-cluster: kind kind-cluster-cleanup ## Standup a kind cluster + $(KIND) create cluster --name ${KIND_CLUSTER_NAME} + $(KIND) export kubeconfig --name ${KIND_CLUSTER_NAME} + +kind-cluster-cleanup: kind ## Delete the kind cluster + $(KIND) delete cluster --name ${KIND_CLUSTER_NAME} + ##@ Build .PHONY: build @@ -65,14 +91,17 @@ build: manifests generate fmt vet ## Build manager binary. go build -o bin/manager main.go .PHONY: run -run: manifests generate fmt vet ## Run a controller from your host. - go run ./main.go +run: docker-build kind-cluster kind-load install deploy wait ## Build the operator-controller then deploy it into a new kind cluster. + +.PHONY: wait +wait: + kubectl wait --for=condition=Available --namespace=$(OPERATOR_CONTROLLER_NAMESPACE) deployment/operator-controller-controller-manager --timeout=60s # If you wish built the manager image targeting other platforms you can use the --platform flag. # (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build -docker-build: test ## Build docker image with the manager. +docker-build: generate ## Build docker image with the operator-controller. docker build -t ${IMG} . .PHONY: docker-push @@ -129,12 +158,24 @@ $(LOCALBIN): ## Tool Binaries KUSTOMIZE ?= $(LOCALBIN)/kustomize CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +KIND ?= $(LOCALBIN)/kind +GINKGO ?= $(LOCALBIN)/ginkgo ENVTEST ?= $(LOCALBIN)/setup-envtest ## Tool Versions KUSTOMIZE_VERSION ?= v4.5.7 CONTROLLER_TOOLS_VERSION ?= v0.10.0 +.PHONY: kind +kind: $(KIND) ## Download kind locally if necessary. +$(KIND): $(LOCALBIN) + test -s $(LOCALBIN)/kind || GOBIN=$(LOCALBIN) go install sigs.k8s.io/kind@v0.15.0 + +.PHONY: ginkgo +ginkgo: $(GINKGO) ## Download ginkgo locally if necessary. +$(GINKGO): $(LOCALBIN) + test -s $(LOCALBIN)/ginkgo || GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo@v2.1.4 + KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. If wrong version is installed, it will be removed before downloading. diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5c5f0b84c..0e79562ad 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: quay.io/operator-framework/operator-controller + newTag: latest diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index ff064e5d3..42a435060 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -71,6 +71,7 @@ spec: args: - --leader-elect image: controller:latest + imagePullPolicy: IfNotPresent name: manager securityContext: allowPrivilegeEscalation: false diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go new file mode 100644 index 000000000..c895a62a5 --- /dev/null +++ b/test/e2e/e2e_suite_test.go @@ -0,0 +1,61 @@ +package e2e + +import ( + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1" +) + +var ( + cfg *rest.Config + c client.Client + kubeClient kubernetes.Interface +) + +func TestE2E(t *testing.T) { + RegisterFailHandler(Fail) + SetDefaultEventuallyTimeout(1 * time.Minute) + SetDefaultEventuallyPollingInterval(1 * time.Second) + RunSpecs(t, "E2E Suite") +} + +var _ = BeforeSuite(func() { + cfg = ctrl.GetConfigOrDie() + + scheme := runtime.NewScheme() + err := operatorv1alpha1.AddToScheme(scheme) + Expect(err).To(BeNil()) + err = rbacv1.AddToScheme(scheme) + Expect(err).To(BeNil()) + err = batchv1.AddToScheme(scheme) + Expect(err).To(BeNil()) + + err = corev1.AddToScheme(scheme) + Expect(err).To(BeNil()) + + err = appsv1.AddToScheme(scheme) + Expect(err).To(BeNil()) + + err = apiextensionsv1.AddToScheme(scheme) + Expect(err).To(BeNil()) + + c, err = client.New(cfg, client.Options{Scheme: scheme}) + Expect(err).To(BeNil()) + + kubeClient, err = kubernetes.NewForConfig(cfg) + Expect(err).To(BeNil()) +}) diff --git a/test/e2e/reconcile_test.go b/test/e2e/reconcile_test.go new file mode 100644 index 000000000..582f86234 --- /dev/null +++ b/test/e2e/reconcile_test.go @@ -0,0 +1,44 @@ +package e2e + +import ( + "context" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + operatorv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1" +) + +var _ = Describe("Basic Reconciliation", func() { + When("an Operator CR is created", func() { + var ( + bundle *operatorv1alpha1.Operator + ctx context.Context + err error + ) + BeforeEach(func() { + ctx = context.Background() + + bundle = &operatorv1alpha1.Operator{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "operator", + Namespace: "operator-controller-system", + }, + Spec: operatorv1alpha1.OperatorSpec{ + PackageName: "my-cool-package", + }, + } + err = c.Create(ctx, bundle) + Expect(err).To(Not(HaveOccurred())) + }) + AfterEach(func() { + By("deleting the testing Operator resource") + err = c.Delete(ctx, bundle) + Expect(err).To(Not(HaveOccurred())) + }) + It("is reconciled", func() { + // Note that we don't actually test anything since the reconciler currently does nothing + }) + }) +}) From 1d621bab45e6b87feb95c67a9f1fe1760ce41035 Mon Sep 17 00:00:00 2001 From: dtfranz Date: Wed, 4 Jan 2023 12:12:29 -0800 Subject: [PATCH 2/5] addressing comments Signed-off-by: dtfranz --- Makefile | 3 +- api/v1alpha1/operator_types.go | 1 + ...rators.operatorframework.io_operators.yaml | 2 +- config/manager/kustomization.yaml | 2 +- test/e2e/e2e_suite_test.go | 31 +++---------------- test/e2e/reconcile_test.go | 26 +++++++++------- 6 files changed, 23 insertions(+), 42 deletions(-) diff --git a/Makefile b/Makefile index fc3effda3..c0b00b688 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ ########################### # Image URL to use all building/pushing image targets export IMAGE_REPO ?= quay.io/operator-framework/operator-controller -export IMAGE_TAG ?= latest +export IMAGE_TAG ?= devel export GO_BUILD_TAGS ?= upstream IMG?=$(IMAGE_REPO):$(IMAGE_TAG) @@ -72,6 +72,7 @@ E2E_FLAGS ?= "" test-e2e: ginkgo ## Run the e2e tests $(GINKGO) --tags $(GO_BUILD_TAGS) $(E2E_FLAGS) -trace -progress $(FOCUS) test/e2e +e2e: KIND_CLUSTER_NAME=operator-controller-e2e e2e: run test-e2e kind-cluster-cleanup ## Run e2e test suite on local kind cluster kind-load: kind ## Loads the currently constructed image onto the cluster diff --git a/api/v1alpha1/operator_types.go b/api/v1alpha1/operator_types.go index e75cf80cf..64b023bda 100644 --- a/api/v1alpha1/operator_types.go +++ b/api/v1alpha1/operator_types.go @@ -32,6 +32,7 @@ type OperatorStatus struct{} //+kubebuilder:object:root=true //+kubebuilder:subresource:status +//+kubebuilder:resource:scope=Cluster // Operator is the Schema for the operators API type Operator struct { diff --git a/config/crd/bases/operators.operatorframework.io_operators.yaml b/config/crd/bases/operators.operatorframework.io_operators.yaml index d48b64322..e6584b095 100644 --- a/config/crd/bases/operators.operatorframework.io_operators.yaml +++ b/config/crd/bases/operators.operatorframework.io_operators.yaml @@ -13,7 +13,7 @@ spec: listKind: OperatorList plural: operators singular: operator - scope: Namespaced + scope: Cluster versions: - name: v1alpha1 schema: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 0e79562ad..754433c31 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: quay.io/operator-framework/operator-controller - newTag: latest + newTag: devel diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index c895a62a5..428c0b25f 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -6,13 +6,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -21,9 +15,8 @@ import ( ) var ( - cfg *rest.Config - c client.Client - kubeClient kubernetes.Interface + cfg *rest.Config + c client.Client ) func TestE2E(t *testing.T) { @@ -38,24 +31,8 @@ var _ = BeforeSuite(func() { scheme := runtime.NewScheme() err := operatorv1alpha1.AddToScheme(scheme) - Expect(err).To(BeNil()) - err = rbacv1.AddToScheme(scheme) - Expect(err).To(BeNil()) - err = batchv1.AddToScheme(scheme) - Expect(err).To(BeNil()) - - err = corev1.AddToScheme(scheme) - Expect(err).To(BeNil()) - - err = appsv1.AddToScheme(scheme) - Expect(err).To(BeNil()) - - err = apiextensionsv1.AddToScheme(scheme) - Expect(err).To(BeNil()) + Expect(err).To(Not(HaveOccurred())) c, err = client.New(cfg, client.Options{Scheme: scheme}) - Expect(err).To(BeNil()) - - kubeClient, err = kubernetes.NewForConfig(cfg) - Expect(err).To(BeNil()) + Expect(err).To(Not(HaveOccurred())) }) diff --git a/test/e2e/reconcile_test.go b/test/e2e/reconcile_test.go index 582f86234..91545d796 100644 --- a/test/e2e/reconcile_test.go +++ b/test/e2e/reconcile_test.go @@ -2,43 +2,45 @@ package e2e import ( "context" + "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" operatorv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1" ) -var _ = Describe("Basic Reconciliation", func() { - When("an Operator CR is created", func() { +var _ = Describe("Operator Install", func() { + When("a valid Operator CR specifying a package", func() { var ( - bundle *operatorv1alpha1.Operator - ctx context.Context - err error + operator *operatorv1alpha1.Operator + ctx context.Context + err error ) BeforeEach(func() { ctx = context.Background() - bundle = &operatorv1alpha1.Operator{ + operator = &operatorv1alpha1.Operator{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: "operator", - Namespace: "operator-controller-system", + Name: fmt.Sprintf("operator-%s", rand.String(8)), }, Spec: operatorv1alpha1.OperatorSpec{ PackageName: "my-cool-package", }, } - err = c.Create(ctx, bundle) + err = c.Create(ctx, operator) Expect(err).To(Not(HaveOccurred())) }) AfterEach(func() { By("deleting the testing Operator resource") - err = c.Delete(ctx, bundle) + err = c.Delete(ctx, operator) Expect(err).To(Not(HaveOccurred())) }) - It("is reconciled", func() { - // Note that we don't actually test anything since the reconciler currently does nothing + It("installs the specified package", func() { + // Pending until we actually have some code to test + // Expect that a CRD and Deployment were successfully installed by rukpak }) }) }) From 6858e1dd776eb7aef7c7773ff34e4a6965c414dd Mon Sep 17 00:00:00 2001 From: dtfranz Date: Wed, 4 Jan 2023 13:50:18 -0800 Subject: [PATCH 3/5] added e2e github action and added pending tag to test Signed-off-by: dtfranz --- .github/workflows/e2e.yaml | 20 ++++++++++++++++++++ test/e2e/reconcile_test.go | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/e2e.yaml diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 000000000..ab880ed26 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,20 @@ +name: e2e + +on: + workflow_dispatch: + pull_request: + push: + branches: + - main + +jobs: + e2e-kind: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-go@v3 + with: + go-version-file: "go.mod" + - name: Run e2e tests + run: | + make e2e diff --git a/test/e2e/reconcile_test.go b/test/e2e/reconcile_test.go index 91545d796..d2b993d55 100644 --- a/test/e2e/reconcile_test.go +++ b/test/e2e/reconcile_test.go @@ -38,7 +38,7 @@ var _ = Describe("Operator Install", func() { err = c.Delete(ctx, operator) Expect(err).To(Not(HaveOccurred())) }) - It("installs the specified package", func() { + PIt("installs the specified package", func() { // Pending until we actually have some code to test // Expect that a CRD and Deployment were successfully installed by rukpak }) From e60478f7085ea2cbcf1b7957dfaf1010e01c1d66 Mon Sep 17 00:00:00 2001 From: dtfranz Date: Thu, 5 Jan 2023 09:05:02 -0800 Subject: [PATCH 4/5] back-out scoping change --- api/v1alpha1/operator_types.go | 1 - config/crd/bases/operators.operatorframework.io_operators.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/api/v1alpha1/operator_types.go b/api/v1alpha1/operator_types.go index 64b023bda..e75cf80cf 100644 --- a/api/v1alpha1/operator_types.go +++ b/api/v1alpha1/operator_types.go @@ -32,7 +32,6 @@ type OperatorStatus struct{} //+kubebuilder:object:root=true //+kubebuilder:subresource:status -//+kubebuilder:resource:scope=Cluster // Operator is the Schema for the operators API type Operator struct { diff --git a/config/crd/bases/operators.operatorframework.io_operators.yaml b/config/crd/bases/operators.operatorframework.io_operators.yaml index e6584b095..d48b64322 100644 --- a/config/crd/bases/operators.operatorframework.io_operators.yaml +++ b/config/crd/bases/operators.operatorframework.io_operators.yaml @@ -13,7 +13,7 @@ spec: listKind: OperatorList plural: operators singular: operator - scope: Cluster + scope: Namespaced versions: - name: v1alpha1 schema: From dce6b5a8c906d13ff95eea91dd0d8a0f55a9d19e Mon Sep 17 00:00:00 2001 From: dtfranz Date: Thu, 5 Jan 2023 11:34:36 -0800 Subject: [PATCH 5/5] disable -j flag for make Signed-off-by: dtfranz --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index c0b00b688..15215f9ee 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,9 @@ endif SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec +# Disable -j flag for make +.NOTPARALLEL: + .PHONY: all all: build