From ef60cd0c5cfae0b8fecc201fc365c50c0ee8b05a Mon Sep 17 00:00:00 2001 From: Lennart Jern Date: Tue, 25 Oct 2022 14:52:09 +0300 Subject: [PATCH 1/2] Add clusterctl upgrade tests These tests make use of the CAPI e2e framework. The test spec creates a secondary management cluster with older versions of the controllers. A workload cluster is created to test the functionality of the old controllers before they are upgraded. Then clusterctl upgrade is used to upgrade them and the workload cluster is scaled to check that things are working also after the upgrade. --- .gitignore | 3 ++ Makefile | 3 +- test/e2e/data/e2e_conf.yaml | 27 +++++++++- .../kustomize/v1alpha5/kustomization.yaml | 6 +++ test/e2e/shared/defaults.go | 1 + .../e2e/suites/e2e/clusterctl_upgrade_test.go | 53 +++++++++++++++++++ 6 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 test/e2e/data/kustomize/v1alpha5/kustomization.yaml create mode 100644 test/e2e/suites/e2e/clusterctl_upgrade_test.go diff --git a/.gitignore b/.gitignore index 778c8394df..0a4079efc6 100644 --- a/.gitignore +++ b/.gitignore @@ -173,3 +173,6 @@ sshuttle.pid # Book docs/book/book/ + +# venv +.venv diff --git a/Makefile b/Makefile index a8efca9b23..62a27af8dc 100644 --- a/Makefile +++ b/Makefile @@ -120,7 +120,7 @@ LDFLAGS := $(shell source ./hack/version.sh; version::ldflags) ## -------------------------------------- # The number of ginkgo tests to run concurrently -E2E_GINKGO_PARALLEL=2 +E2E_GINKGO_PARALLEL ?= 2 E2E_ARGS ?= @@ -143,6 +143,7 @@ E2E_KUSTOMIZE_DIR=test/e2e/data/kustomize .PHONY: e2e-templates e2e-templates: ## Generate cluster templates for e2e tests e2e-templates: $(addprefix $(E2E_TEMPLATES_DIR)/, \ + cluster-template-v1alpha5.yaml \ cluster-template-external-cloud-provider.yaml \ cluster-template-multi-az.yaml \ cluster-template-multi-network.yaml \ diff --git a/test/e2e/data/e2e_conf.yaml b/test/e2e/data/e2e_conf.yaml index 4c8b68157c..51cd7f12d4 100644 --- a/test/e2e/data/e2e_conf.yaml +++ b/test/e2e/data/e2e_conf.yaml @@ -35,6 +35,7 @@ providers: - name: v1.2.4 value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.4/core-components.yaml" type: url + contract: v1beta1 files: - sourcePath: "../data/shared/v1beta1/metadata.yaml" replacements: @@ -48,6 +49,7 @@ providers: - name: v1.2.4 value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.4/bootstrap-components.yaml" type: url + contract: v1beta1 files: - sourcePath: "../data/shared/v1beta1/metadata.yaml" replacements: @@ -61,6 +63,7 @@ providers: - name: v1.2.4 value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.4/control-plane-components.yaml" type: url + contract: v1beta1 files: - sourcePath: "../data/shared/v1beta1/metadata.yaml" replacements: @@ -71,17 +74,36 @@ providers: - name: openstack type: InfrastructureProvider versions: + # This is only for clusterctl upgrade tests + - name: v0.6.3 + value: "https://github.com/kubernetes-sigs/cluster-api-provider-openstack/releases/download/v0.6.3/infrastructure-components.yaml" + type: url + contract: v1beta1 + files: + - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml" + - sourcePath: "./infrastructure-openstack/cluster-template.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: "--v=2" + new: "--v=4" + - old: "--leader-elect" + new: "--leader-elect=false\n - --sync-period=1m" - name: v0.6.99 value: ../../../config/default - contract: v1beta1 + # This is the upcoming version. + # Specify no contract so that upgrade tests that start from a specific contract won't pick it up. + # contract: v1beta1 files: - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml" - sourcePath: "./infrastructure-openstack/cluster-template.yaml" - sourcePath: "./infrastructure-openstack/cluster-template-external-cloud-provider.yaml" - sourcePath: "./infrastructure-openstack/cluster-template-without-lb.yaml" replacements: + # TODO: We should use e2e here instead of main, but we need a way to get it into the workload cluster + # for upgrade tests for that to work. - old: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:dev - new: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e + new: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:main - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" - old: "--v=2" @@ -116,6 +138,7 @@ variables: OPENSTACK_VOLUME_TYPE_ALT: "test-volume-type" CONFORMANCE_WORKER_MACHINE_COUNT: "5" CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT: "1" + INIT_WITH_KUBERNETES_VERSION: "v1.25.0" intervals: conformance/wait-control-plane: ["30m", "10s"] diff --git a/test/e2e/data/kustomize/v1alpha5/kustomization.yaml b/test/e2e/data/kustomize/v1alpha5/kustomization.yaml new file mode 100644 index 0000000000..4a4b95323c --- /dev/null +++ b/test/e2e/data/kustomize/v1alpha5/kustomization.yaml @@ -0,0 +1,6 @@ +--- +resources: +- ../../../../../kustomize/v1alpha5/default + +components: +- ../common-patches diff --git a/test/e2e/shared/defaults.go b/test/e2e/shared/defaults.go index b83556c6ac..b189e7cf2a 100644 --- a/test/e2e/shared/defaults.go +++ b/test/e2e/shared/defaults.go @@ -48,6 +48,7 @@ const ( FlavorExternalCloudProvider = "external-cloud-provider-ci-artifacts" FlavorMultiNetwork = "multi-network-ci-artifacts" FlavorMultiAZ = "multi-az-ci-artifacts" + FlavorV1alpha5 = "v1alpha5-ci-artifacts" ) // DefaultScheme returns the default scheme to use for testing. diff --git a/test/e2e/suites/e2e/clusterctl_upgrade_test.go b/test/e2e/suites/e2e/clusterctl_upgrade_test.go new file mode 100644 index 0000000000..c51cebd6ca --- /dev/null +++ b/test/e2e/suites/e2e/clusterctl_upgrade_test.go @@ -0,0 +1,53 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo" + capi_e2e "sigs.k8s.io/cluster-api/test/e2e" + + "sigs.k8s.io/cluster-api-provider-openstack/test/e2e/shared" +) + +// Note: There isn't really anything fixing this to the v0.6 release. The test will +// simply pick the latest release with the correct contract from the E2EConfig, as seen here +// https://github.com/kubernetes-sigs/cluster-api/blob/3abb9089485f51d46054096c17ccb8b59f0f7331/test/e2e/clusterctl_upgrade.go#L265 +// When new minor releases are added (with the same contract) we will need to work on this +// if we want to continue testing v0.6. +var _ = Describe("When testing clusterctl upgrades (v0.6=>current) [clusterctl-upgrade]", func() { + ctx := context.TODO() + shared.SetEnvVar("USE_CI_ARTIFACTS", "true", false) + + capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput { + return capi_e2e.ClusterctlUpgradeSpecInput{ + E2EConfig: e2eCtx.E2EConfig, + ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath, + BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy, + ArtifactFolder: e2eCtx.Settings.ArtifactFolder, + SkipCleanup: false, + InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.4/clusterctl-{OS}-{ARCH}", + InitWithProvidersContract: "v1beta1", + MgmtFlavor: shared.FlavorDefault, + WorkloadFlavor: shared.FlavorV1alpha5, + } + }) +}) From 40f1a333735f6b09cfa9f4d28de2e5782046d847 Mon Sep 17 00:00:00 2001 From: Lennart Jern Date: Thu, 27 Oct 2022 08:50:13 +0300 Subject: [PATCH 2/2] Add CI image to target cluster for clusterctl upgrade tests --- scripts/ci-e2e.sh | 18 +++++++++-- test/e2e/data/e2e_conf.yaml | 4 +-- .../data/kustomize/default/kustomization.yaml | 1 + .../upgrade-patches/ci-hack-kcp.yaml | 31 +++++++++++++++++++ .../upgrade-patches/ci-hack-kct.yaml | 31 +++++++++++++++++++ .../upgrade-patches/kustomization.yaml | 14 +++++++++ .../e2e/suites/e2e/clusterctl_upgrade_test.go | 2 ++ 7 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 test/e2e/data/kustomize/upgrade-patches/ci-hack-kcp.yaml create mode 100644 test/e2e/data/kustomize/upgrade-patches/ci-hack-kct.yaml create mode 100644 test/e2e/data/kustomize/upgrade-patches/kustomization.yaml diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index 46c4e062d9..0df8ce99d3 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -15,8 +15,8 @@ # limitations under the License. ################################################################################ -# usage: ci-conformance.sh -# This program runs the clusterctl conformance e2e tests. +# usage: ci-e2e.sh +# This program runs the e2e tests. ################################################################################ set -x @@ -84,6 +84,20 @@ fi "hack/ci/create_devstack.sh" +# Upload image for e2e clusterctl upgrade tests +source "${REPO_ROOT}/hack/ci/${RESOURCE_TYPE}.sh" +CONTAINER_ARCHIVE="${ARTIFACTS}/capo-e2e-image.tar" +SSH_KEY="$(get_ssh_private_key_file)" +SSH_ARGS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o PasswordAuthentication=no" +CONTROLLER_IP=${CONTROLLER_IP:-"10.0.3.15"} + +make e2e-image +docker save -o "${CONTAINER_ARCHIVE}" gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e +scp -i "${SSH_KEY}" ${SSH_ARGS} "${CONTAINER_ARCHIVE}" "cloud@${CONTROLLER_IP}:capo-e2e-image.tar" +ssh -i "${SSH_KEY}" ${SSH_ARGS} "cloud@${CONTROLLER_IP}" -- sudo chown root:root capo-e2e-image.tar +ssh -i "${SSH_KEY}" ${SSH_ARGS} "cloud@${CONTROLLER_IP}" -- sudo chmod u=rw,g=r,o=r capo-e2e-image.tar +ssh -i "${SSH_KEY}" ${SSH_ARGS} "cloud@${CONTROLLER_IP}" -- sudo mv capo-e2e-image.tar /var/www/html/capo-e2e-image.tar + export OPENSTACK_CLOUD_YAML_FILE OPENSTACK_CLOUD_YAML_FILE="$(pwd)/clouds.yaml" make test-e2e diff --git a/test/e2e/data/e2e_conf.yaml b/test/e2e/data/e2e_conf.yaml index 51cd7f12d4..8478183865 100644 --- a/test/e2e/data/e2e_conf.yaml +++ b/test/e2e/data/e2e_conf.yaml @@ -100,10 +100,8 @@ providers: - sourcePath: "./infrastructure-openstack/cluster-template-external-cloud-provider.yaml" - sourcePath: "./infrastructure-openstack/cluster-template-without-lb.yaml" replacements: - # TODO: We should use e2e here instead of main, but we need a way to get it into the workload cluster - # for upgrade tests for that to work. - old: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:dev - new: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:main + new: gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" - old: "--v=2" diff --git a/test/e2e/data/kustomize/default/kustomization.yaml b/test/e2e/data/kustomize/default/kustomization.yaml index 8609199ac8..02be0b4243 100644 --- a/test/e2e/data/kustomize/default/kustomization.yaml +++ b/test/e2e/data/kustomize/default/kustomization.yaml @@ -4,3 +4,4 @@ resources: components: - ../common-patches +- ../upgrade-patches diff --git a/test/e2e/data/kustomize/upgrade-patches/ci-hack-kcp.yaml b/test/e2e/data/kustomize/upgrade-patches/ci-hack-kcp.yaml new file mode 100644 index 0000000000..45fe9a3990 --- /dev/null +++ b/test/e2e/data/kustomize/upgrade-patches/ci-hack-kcp.yaml @@ -0,0 +1,31 @@ +--- +# Workaround for https://github.com/kubernetes-sigs/cluster-api/issues/7457 +# There is a small but important difference between these two: +# path: /a/b/c +# *creates* the c array, overwriting anything that was there before +# path: /a/b/c/- +# *adds* to the c array and does not work if the array is missing +# +# We add to the postKubeadmCommands (instead of pre*) since we need the CI artifacts +# script to run first. Without this, the container images are not imported properly. +- op: add + path: /spec/kubeadmConfigSpec/postKubeadmCommands + value: + - /usr/local/bin/ci-artifacts-openstack.sh +- op: add + path: /spec/kubeadmConfigSpec/files/- + value: + content: | + #!/bin/bash + DOWNLOAD_E2E_IMAGE=${DOWNLOAD_E2E_IMAGE:=false} + if [ ! "${DOWNLOAD_E2E_IMAGE}" = true ]; then + echo "Not downloading E2E image, exiting" + exit 0 + fi + # Download the locally built CAPO controller image + echo "Downloading ${E2E_IMAGE_URL}" + wget "${E2E_IMAGE_URL}" -O "/tmp/capo-controller-manager.tar" + sudo ctr -n k8s.io images import "/tmp/capo-controller-manager.tar" || echo "* ignoring expected 'ctr images import' result" + owner: root:root + path: /usr/local/bin/ci-artifacts-openstack.sh + permissions: "0750" diff --git a/test/e2e/data/kustomize/upgrade-patches/ci-hack-kct.yaml b/test/e2e/data/kustomize/upgrade-patches/ci-hack-kct.yaml new file mode 100644 index 0000000000..2429b30a58 --- /dev/null +++ b/test/e2e/data/kustomize/upgrade-patches/ci-hack-kct.yaml @@ -0,0 +1,31 @@ +--- +# Workaround for https://github.com/kubernetes-sigs/cluster-api/issues/7457 +# There is a small but important difference between these two: +# path: /a/b/c +# *creates* the c array, overwriting anything that was there before +# path: /a/b/c/- +# *adds* to the c array and does not work if the array is missing +# +# We add to the postKubeadmCommands (instead of pre*) since we need the CI artifacts +# script to run first. Without this, the container images are not imported properly. +- op: add + path: /spec/template/spec/postKubeadmCommands + value: + - /usr/local/bin/ci-artifacts-openstack.sh +- op: add + path: /spec/template/spec/files/- + value: + content: | + #!/bin/bash + DOWNLOAD_E2E_IMAGE=${DOWNLOAD_E2E_IMAGE:=false} + if [ ! "${DOWNLOAD_E2E_IMAGE}" = true ]; then + echo "Not downloading E2E image, exiting" + exit 0 + fi + # Download the locally built CAPO controller image + echo "Downloading ${E2E_IMAGE_URL}" + wget "${E2E_IMAGE_URL}" -O "/tmp/capo-controller-manager.tar" + sudo ctr -n k8s.io images import "/tmp/capo-controller-manager.tar" || echo "* ignoring expected 'ctr images import' result" + owner: root:root + path: /usr/local/bin/ci-artifacts-openstack.sh + permissions: "0750" diff --git a/test/e2e/data/kustomize/upgrade-patches/kustomization.yaml b/test/e2e/data/kustomize/upgrade-patches/kustomization.yaml new file mode 100644 index 0000000000..e2e0b835f8 --- /dev/null +++ b/test/e2e/data/kustomize/upgrade-patches/kustomization.yaml @@ -0,0 +1,14 @@ +# Modifications to release templates for clusterctl upgrade scenarios +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +patches: +- target: + kind: KubeadmControlPlane + name: \${CLUSTER_NAME}-control-plane + path: ci-hack-kcp.yaml +- target: + kind: KubeadmConfigTemplate + name: \${CLUSTER_NAME}-md-0 + path: ci-hack-kct.yaml diff --git a/test/e2e/suites/e2e/clusterctl_upgrade_test.go b/test/e2e/suites/e2e/clusterctl_upgrade_test.go index c51cebd6ca..cb73bafa99 100644 --- a/test/e2e/suites/e2e/clusterctl_upgrade_test.go +++ b/test/e2e/suites/e2e/clusterctl_upgrade_test.go @@ -36,6 +36,8 @@ import ( var _ = Describe("When testing clusterctl upgrades (v0.6=>current) [clusterctl-upgrade]", func() { ctx := context.TODO() shared.SetEnvVar("USE_CI_ARTIFACTS", "true", false) + shared.SetEnvVar("DOWNLOAD_E2E_IMAGE", "true", false) + shared.SetEnvVar("E2E_IMAGE_URL", "http://10.0.3.15/capo-e2e-image.tar", false) capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput { return capi_e2e.ClusterctlUpgradeSpecInput{