Skip to content

Commit 5756be1

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #5919 from sbueringer/pr-controlplane-label-taint
🌱 Support new control plane label and taint
2 parents 236e6da + 2497fcc commit 5756be1

File tree

10 files changed

+141
-17
lines changed

10 files changed

+141
-17
lines changed

‎bootstrap/kubeadm/config/manager/manager.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,5 @@ spec:
4141
tolerations:
4242
- effect: NoSchedule
4343
key: node-role.kubernetes.io/master
44+
- effect: NoSchedule
45+
key: node-role.kubernetes.io/control-plane

‎config/manager/manager.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,3 +42,5 @@ spec:
4242
tolerations:
4343
- effect: NoSchedule
4444
key: node-role.kubernetes.io/master
45+
- effect: NoSchedule
46+
key: node-role.kubernetes.io/control-plane

‎controlplane/kubeadm/config/manager/manager.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,5 @@ spec:
4141
tolerations:
4242
- effect: NoSchedule
4343
key: node-role.kubernetes.io/master
44+
- effect: NoSchedule
45+
key: node-role.kubernetes.io/control-plane

‎controlplane/kubeadm/internal/controllers/controller_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1516,7 +1516,7 @@ func createMachineNodePair(name string, cluster *clusterv1.Cluster, kcp *control
15161516
node := &corev1.Node{
15171517
ObjectMeta: metav1.ObjectMeta{
15181518
Name: name,
1519-
Labels: map[string]string{"node-role.kubernetes.io/master": ""},
1519+
Labels: map[string]string{"node-role.kubernetes.io/control-plane": ""},
15201520
},
15211521
}
15221522

‎controlplane/kubeadm/internal/workload_cluster.go‎

Lines changed: 33 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import (
3636
apierrors "k8s.io/apimachinery/pkg/api/errors"
3737
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3838
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
39+
"k8s.io/apimachinery/pkg/util/sets"
3940
"k8s.io/client-go/util/retry"
4041
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
4142
"sigs.k8s.io/yaml"
@@ -51,13 +52,14 @@ import (
5152
)
5253

5354
const (
54-
kubeProxyKey = "kube-proxy"
55-
kubeadmConfigKey = "kubeadm-config"
56-
kubeletConfigKey = "kubelet"
57-
cgroupDriverKey = "cgroupDriver"
58-
labelNodeRoleControlPlane = "node-role.kubernetes.io/master"
59-
clusterStatusKey = "ClusterStatus"
60-
clusterConfigurationKey = "ClusterConfiguration"
55+
kubeProxyKey = "kube-proxy"
56+
kubeadmConfigKey = "kubeadm-config"
57+
kubeletConfigKey = "kubelet"
58+
cgroupDriverKey = "cgroupDriver"
59+
labelNodeRoleOldControlPlane = "node-role.kubernetes.io/master" // Deprecated: https://github.com/kubernetes/kubeadm/issues/2200
60+
labelNodeRoleControlPlane = "node-role.kubernetes.io/control-plane"
61+
clusterStatusKey = "ClusterStatus"
62+
clusterConfigurationKey = "ClusterConfiguration"
6163
)
6264

6365
var (
@@ -121,14 +123,31 @@ type Workload struct {
121123
var _ WorkloadCluster = &Workload{}
122124

123125
func (w *Workload) getControlPlaneNodes(ctx context.Context) (*corev1.NodeList, error) {
124-
nodes := &corev1.NodeList{}
125-
labels := map[string]string{
126-
labelNodeRoleControlPlane: "",
127-
}
128-
if err := w.Client.List(ctx, nodes, ctrlclient.MatchingLabels(labels)); err != nil {
129-
return nil, err
126+
controlPlaneNodes := &corev1.NodeList{}
127+
controlPlaneNodeNames := sets.NewString()
128+
129+
for _, label := range []string{labelNodeRoleOldControlPlane, labelNodeRoleControlPlane} {
130+
nodes := &corev1.NodeList{}
131+
if err := w.Client.List(ctx, nodes, ctrlclient.MatchingLabels(map[string]string{
132+
label: "",
133+
})); err != nil {
134+
return nil, err
135+
}
136+
137+
for i := range nodes.Items {
138+
node := nodes.Items[i]
139+
140+
// Continue if we already added that node.
141+
if controlPlaneNodeNames.Has(node.Name) {
142+
continue
143+
}
144+
145+
controlPlaneNodeNames.Insert(node.Name)
146+
controlPlaneNodes.Items = append(controlPlaneNodes.Items, node)
147+
}
130148
}
131-
return nodes, nil
149+
150+
return controlPlaneNodes, nil
132151
}
133152

134153
func (w *Workload) getConfigMap(ctx context.Context, configMap ctrlclient.ObjectKey) (*corev1.ConfigMap, error) {

‎controlplane/kubeadm/internal/workload_cluster_conditions_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -980,6 +980,9 @@ func fakeNode(name string, options ...fakeNodeOption) *corev1.Node {
980980
p := &corev1.Node{
981981
ObjectMeta: metav1.ObjectMeta{
982982
Name: name,
983+
Labels: map[string]string{
984+
labelNodeRoleControlPlane: "",
985+
},
983986
},
984987
}
985988
for _, opt := range options {

‎controlplane/kubeadm/internal/workload_cluster_test.go‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,78 @@ import (
3838
"sigs.k8s.io/cluster-api/util/yaml"
3939
)
4040

41+
func TestGetControlPlaneNodes(t *testing.T) {
42+
tests := []struct {
43+
name string
44+
nodes []corev1.Node
45+
expectedNodes []string
46+
}{
47+
{
48+
name: "Return control plane nodes",
49+
nodes: []corev1.Node{
50+
{
51+
ObjectMeta: metav1.ObjectMeta{
52+
Name: "control-plane-node-with-old-label",
53+
Labels: map[string]string{
54+
labelNodeRoleOldControlPlane: "",
55+
},
56+
},
57+
},
58+
{
59+
ObjectMeta: metav1.ObjectMeta{
60+
Name: "control-plane-node-with-both-labels",
61+
Labels: map[string]string{
62+
labelNodeRoleOldControlPlane: "",
63+
labelNodeRoleControlPlane: "",
64+
},
65+
},
66+
},
67+
{
68+
ObjectMeta: metav1.ObjectMeta{
69+
Name: "control-plane-node-with-new-label",
70+
Labels: map[string]string{
71+
labelNodeRoleControlPlane: "",
72+
},
73+
},
74+
},
75+
{
76+
ObjectMeta: metav1.ObjectMeta{
77+
Name: "worker-node",
78+
Labels: map[string]string{},
79+
},
80+
},
81+
},
82+
expectedNodes: []string{
83+
"control-plane-node-with-both-labels",
84+
"control-plane-node-with-old-label",
85+
"control-plane-node-with-new-label",
86+
},
87+
},
88+
}
89+
90+
for _, tt := range tests {
91+
t.Run(tt.name, func(t *testing.T) {
92+
g := NewWithT(t)
93+
objs := []client.Object{}
94+
for i := range tt.nodes {
95+
objs = append(objs, &tt.nodes[i])
96+
}
97+
fakeClient := fake.NewClientBuilder().WithObjects(objs...).Build()
98+
99+
w := &Workload{
100+
Client: fakeClient,
101+
}
102+
nodes, err := w.getControlPlaneNodes(ctx)
103+
g.Expect(err).ToNot(HaveOccurred())
104+
var actualNodes []string
105+
for _, n := range nodes.Items {
106+
actualNodes = append(actualNodes, n.Name)
107+
}
108+
g.Expect(actualNodes).To(Equal(tt.expectedNodes))
109+
})
110+
}
111+
}
112+
41113
func TestUpdateKubeProxyImageInfo(t *testing.T) {
42114
tests := []struct {
43115
name string

‎docs/book/src/user/troubleshooting.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,10 @@ kubectl label nodes <name> node-role.kubernetes.io/worker=""
3535
For convenience, here is an example one-liner to do this post installation
3636

3737
```
38+
# Kubernetes 1.19 (kubeadm 1.19 sets only the node-role.kubernetes.io/master label)
3839
kubectl get nodes --no-headers -l '!node-role.kubernetes.io/master' -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}' | xargs -I{} kubectl label node {} node-role.kubernetes.io/worker=''
40+
# Kubernetes >= 1.20 (kubeadm >= 1.20 sets the node-role.kubernetes.io/control-plane label)
41+
kubectl get nodes --no-headers -l '!node-role.kubernetes.io/control-plane' -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}' | xargs -I{} kubectl label node {} node-role.kubernetes.io/worker=''
3942
```
4043

4144
## Cluster API with Docker

‎test/framework/deployment_helpers.go‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import (
3434
"k8s.io/api/policy/v1beta1"
3535
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3636
"k8s.io/apimachinery/pkg/util/intstr"
37+
utilversion "k8s.io/apimachinery/pkg/util/version"
3738
"k8s.io/client-go/kubernetes"
3839
"k8s.io/utils/pointer"
3940
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -42,6 +43,11 @@ import (
4243
"sigs.k8s.io/cluster-api/test/framework/internal/log"
4344
)
4445

46+
const (
47+
nodeRoleOldControlPlane = "node-role.kubernetes.io/master" // Deprecated: https://github.com/kubernetes/kubeadm/issues/2200
48+
nodeRoleControlPlane = "node-role.kubernetes.io/control-plane"
49+
)
50+
4551
// WaitForDeploymentsAvailableInput is the input for WaitForDeploymentsAvailable.
4652
type WaitForDeploymentsAvailableInput struct {
4753
Getter Getter
@@ -306,10 +312,23 @@ func DeployUnevictablePod(ctx context.Context, input DeployUnevictablePodInput)
306312
},
307313
}
308314
if input.ControlPlane != nil {
309-
workloadDeployment.Spec.Template.Spec.NodeSelector = map[string]string{"node-role.kubernetes.io/master": ""}
315+
serverVersion, err := workloadClient.ServerVersion()
316+
Expect(err).ToNot(HaveOccurred())
317+
318+
// Use the control-plane label for Kubernetes version >= v1.20.0.
319+
if utilversion.MustParseGeneric(serverVersion.String()).AtLeast(utilversion.MustParseGeneric("v1.20.0")) {
320+
workloadDeployment.Spec.Template.Spec.NodeSelector = map[string]string{nodeRoleControlPlane: ""}
321+
} else {
322+
workloadDeployment.Spec.Template.Spec.NodeSelector = map[string]string{nodeRoleOldControlPlane: ""}
323+
}
324+
310325
workloadDeployment.Spec.Template.Spec.Tolerations = []corev1.Toleration{
311326
{
312-
Key: "node-role.kubernetes.io/master",
327+
Key: nodeRoleOldControlPlane,
328+
Effect: "NoSchedule",
329+
},
330+
{
331+
Key: nodeRoleControlPlane,
313332
Effect: "NoSchedule",
314333
},
315334
}

‎test/infrastructure/docker/config/manager/manager.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ spec:
4444
tolerations:
4545
- effect: NoSchedule
4646
key: node-role.kubernetes.io/master
47+
- effect: NoSchedule
48+
key: node-role.kubernetes.io/control-plane
4749
volumes:
4850
- name: dockersock
4951
hostPath:

0 commit comments

Comments
 (0)