Support new control plane label and taint

Signed-off-by: Stefan Büringer [email protected]

Author: sbueringer

bootstrap/kubeadm/config/manager/manager.yaml

@@ -41,3 +41,5 @@ spec:
       tolerations:
         - effect: NoSchedule
           key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane

config/manager/manager.yaml

@@ -42,3 +42,5 @@ spec:
       tolerations:
         - effect: NoSchedule
           key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane

controlplane/kubeadm/config/manager/manager.yaml

@@ -41,3 +41,5 @@ spec:
       tolerations:
         - effect: NoSchedule
           key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane

controlplane/kubeadm/controllers/controller_test.go

@@ -1516,7 +1516,7 @@ func createMachineNodePair(name string, cluster *clusterv1.Cluster, kcp *control
 	node := &corev1.Node{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   name,
-			Labels: map[string]string{"node-role.kubernetes.io/master": ""},
+			Labels: map[string]string{"node-role.kubernetes.io/control-plane": ""},
 		},
 	}
 

controlplane/kubeadm/internal/workload_cluster.go

@@ -51,13 +51,14 @@ import (
 )
 
 const (
-	kubeProxyKey              = "kube-proxy"
-	kubeadmConfigKey          = "kubeadm-config"
-	kubeletConfigKey          = "kubelet"
-	cgroupDriverKey           = "cgroupDriver"
-	labelNodeRoleControlPlane = "node-role.kubernetes.io/master"
-	clusterStatusKey          = "ClusterStatus"
-	clusterConfigurationKey   = "ClusterConfiguration"
+	kubeProxyKey                 = "kube-proxy"
+	kubeadmConfigKey             = "kubeadm-config"
+	kubeletConfigKey             = "kubelet"
+	cgroupDriverKey              = "cgroupDriver"
+	labelNodeRoleOldControlPlane = "node-role.kubernetes.io/master" // Deprecated: https://github.com/kubernetes/kubeadm/issues/2200
+	labelNodeRoleControlPlane    = "node-role.kubernetes.io/control-plane"
+	clusterStatusKey             = "ClusterStatus"
+	clusterConfigurationKey      = "ClusterConfiguration"
 )
 
 var (
@@ -121,16 +122,28 @@ type Workload struct {
 var _ WorkloadCluster = &Workload{}
 
 func (w *Workload) getControlPlaneNodes(ctx context.Context) (*corev1.NodeList, error) {
-	nodes := &corev1.NodeList{}
-	labels := map[string]string{
-		labelNodeRoleControlPlane: "",
-	}
-	if err := w.Client.List(ctx, nodes, ctrlclient.MatchingLabels(labels)); err != nil {
+	allNodes := &corev1.NodeList{}
+	if err := w.Client.List(ctx, allNodes); err != nil {
 		return nil, err
 	}
+
+	nodes := &corev1.NodeList{}
+	for i := range allNodes.Items {
+		node := allNodes.Items[i]
+		if isControlPlaneNode(node) {
+			nodes.Items = append(nodes.Items, node)
+		}
+	}
+
 	return nodes, nil
 }
 
+func isControlPlaneNode(node corev1.Node) bool {
+	_, hasLabelNodeRoleControlPlane := node.Labels[labelNodeRoleControlPlane]
+	_, hasLabelNodeRoleOldControlPlane := node.Labels[labelNodeRoleOldControlPlane]
+	return hasLabelNodeRoleControlPlane || hasLabelNodeRoleOldControlPlane
+}
+
 func (w *Workload) getConfigMap(ctx context.Context, configMap ctrlclient.ObjectKey) (*corev1.ConfigMap, error) {
 	original := &corev1.ConfigMap{}
 	if err := w.Client.Get(ctx, configMap, original); err != nil {

controlplane/kubeadm/internal/workload_cluster_conditions_test.go

@@ -980,6 +980,9 @@ func fakeNode(name string, options ...fakeNodeOption) *corev1.Node {
 	p := &corev1.Node{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: name,
+			Labels: map[string]string{
+				labelNodeRoleControlPlane: "",
+			},
 		},
 	}
 	for _, opt := range options {

controlplane/kubeadm/internal/workload_cluster_test.go

@@ -38,6 +38,66 @@ import (
 	"sigs.k8s.io/cluster-api/util/yaml"
 )
 
+func TestIsControlPlaneNode(t *testing.T) {
+	tests := []struct {
+		name               string
+		node               corev1.Node
+		isControlPlaneNode bool
+	}{
+		{
+			name: "control plane node with old label",
+			node: corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						labelNodeRoleOldControlPlane: "",
+					},
+				},
+			},
+			isControlPlaneNode: true,
+		},
+		{
+			name: "control plane node with new label",
+			node: corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						labelNodeRoleControlPlane: "",
+					},
+				},
+			},
+			isControlPlaneNode: true,
+		},
+		{
+			name: "control plane node with both label",
+			node: corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						labelNodeRoleControlPlane:    "",
+						labelNodeRoleOldControlPlane: "",
+					},
+				},
+			},
+			isControlPlaneNode: true,
+		},
+		{
+			name: "worker node",
+			node: corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{},
+				},
+			},
+			isControlPlaneNode: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			g.Expect(isControlPlaneNode(tt.node)).To(Equal(tt.isControlPlaneNode))
+		})
+	}
+}
+
 func TestUpdateKubeProxyImageInfo(t *testing.T) {
 	tests := []struct {
 		name        string

test/framework/deployment_helpers.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/api/policy/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	utilversion "k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -42,6 +43,11 @@ import (
 	"sigs.k8s.io/cluster-api/test/framework/internal/log"
 )
 
+const (
+	nodeRoleOldControlPlane = "node-role.kubernetes.io/master" // Deprecated: https://github.com/kubernetes/kubeadm/issues/2200
+	nodeRoleControlPlane    = "node-role.kubernetes.io/control-plane"
+)
+
 // WaitForDeploymentsAvailableInput is the input for WaitForDeploymentsAvailable.
 type WaitForDeploymentsAvailableInput struct {
 	Getter     Getter
@@ -306,10 +312,23 @@ func DeployUnevictablePod(ctx context.Context, input DeployUnevictablePodInput)
 		},
 	}
 	if input.ControlPlane != nil {
-		workloadDeployment.Spec.Template.Spec.NodeSelector = map[string]string{"node-role.kubernetes.io/master": ""}
+		serverVersion, err := workloadClient.ServerVersion()
+		Expect(err).ToNot(HaveOccurred())
+
+		// Use the control-plane label for Kubernetes version >= v1.20.0.
+		if utilversion.MustParseGeneric(serverVersion.String()).AtLeast(utilversion.MustParseGeneric("v1.20.0")) {
+			workloadDeployment.Spec.Template.Spec.NodeSelector = map[string]string{nodeRoleControlPlane: ""}
+		} else {
+			workloadDeployment.Spec.Template.Spec.NodeSelector = map[string]string{nodeRoleOldControlPlane: ""}
+		}
+
 		workloadDeployment.Spec.Template.Spec.Tolerations = []corev1.Toleration{
 			{
-				Key:    "node-role.kubernetes.io/master",
+				Key:    nodeRoleOldControlPlane,
+				Effect: "NoSchedule",
+			},
+			{
+				Key:    nodeRoleControlPlane,
 				Effect: "NoSchedule",
 			},
 		}

test/infrastructure/docker/config/manager/manager.yaml

@@ -44,6 +44,8 @@ spec:
       tolerations:
       - effect: NoSchedule
         key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
       volumes:
         - name: dockersock
           hostPath: