Skip to content

Commit 1dd0d2e

Browse files
tzifudzitzifudzi
committed
Add Windows secondary IP mode configurable options (#443)
#443
1 parent 4a70259 commit 1dd0d2e

File tree

18 files changed

+1408
-184
lines changed

18 files changed

+1408
-184
lines changed

‎controllers/core/configmap_controller.go‎

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ type ConfigMapReconciler struct {
4646
Condition condition.Conditions
4747
curWinIPAMEnabledCond bool
4848
curWinPrefixDelegationEnabledCond bool
49-
curWinPDWarmIPTarget int
50-
curWinPDMinIPTarget int
49+
curWinWarmIPTarget int
50+
curWinMinIPTarget int
5151
curWinPDWarmPrefixTarget int
5252
Context context.Context
5353
}
@@ -116,21 +116,33 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
116116
isPrefixFlagUpdated = true
117117
}
118118

119-
// Check if configurations for Windows prefix delegation have changed
120-
var isPDConfigUpdated bool
121-
warmIPTarget, minIPTarget, warmPrefixTarget := config.ParseWinPDTargets(r.Log, configmap)
122-
if r.curWinPDWarmIPTarget != warmIPTarget || r.curWinPDMinIPTarget != minIPTarget || r.curWinPDWarmPrefixTarget != warmPrefixTarget {
123-
r.curWinPDWarmIPTarget = warmIPTarget
124-
r.curWinPDMinIPTarget = minIPTarget
119+
// Check if Windows IP target configurations in ConfigMap have changed
120+
var isWinIPConfigsUpdated bool
121+
122+
warmIPTarget, minIPTarget, warmPrefixTarget, isPDEnabled := config.ParseWinIPTargetConfigs(r.Log, configmap)
123+
var winMinIPTargetUpdated = r.curWinMinIPTarget != minIPTarget
124+
var winWarmIPTargetUpdated = r.curWinWarmIPTarget != warmIPTarget
125+
var winPDWarmPrefixTargetUpdated = r.curWinPDWarmPrefixTarget != warmPrefixTarget
126+
if winWarmIPTargetUpdated || winMinIPTargetUpdated {
127+
r.curWinWarmIPTarget = warmIPTarget
128+
r.curWinMinIPTarget = minIPTarget
129+
isWinIPConfigsUpdated = true
130+
}
131+
if isPDEnabled && winPDWarmPrefixTargetUpdated {
125132
r.curWinPDWarmPrefixTarget = warmPrefixTarget
126-
logger.Info("updated PD configs from configmap", config.WarmIPTarget, r.curWinPDWarmIPTarget,
127-
config.MinimumIPTarget, r.curWinPDMinIPTarget, config.WarmPrefixTarget, r.curWinPDWarmPrefixTarget)
128-
129-
isPDConfigUpdated = true
133+
isWinIPConfigsUpdated = true
134+
}
135+
if isWinIPConfigsUpdated {
136+
logger.Info(
137+
"Detected update in Windows IP configuration parameter values in ConfigMap",
138+
config.WinWarmIPTarget, r.curWinWarmIPTarget,
139+
config.WinMinimumIPTarget, r.curWinMinIPTarget,
140+
config.WinWarmPrefixTarget, r.curWinPDWarmPrefixTarget,
141+
config.EnableWindowsPrefixDelegationKey, isPDEnabled,
142+
)
130143
}
131144

132-
// Flag is updated, update all nodes
133-
if isIPAMFlagUpdated || isPrefixFlagUpdated || isPDConfigUpdated {
145+
if isIPAMFlagUpdated || isPrefixFlagUpdated || isWinIPConfigsUpdated {
134146
err := UpdateNodesOnConfigMapChanges(r.K8sAPI, r.NodeManager)
135147
if err != nil {
136148
// Error in updating nodes

‎controllers/core/configmap_controller_test.go‎

Lines changed: 49 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,9 @@ package controllers
1616
import (
1717
"context"
1818
"errors"
19+
"strconv"
1920
"testing"
2021

21-
mock_condition "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/condition"
22-
mock_k8s "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/k8s"
23-
mock_node "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node"
24-
mock_manager "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node/manager"
25-
"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config"
26-
cooldown "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown"
2722
"github.com/golang/mock/gomock"
2823
"github.com/stretchr/testify/assert"
2924
corev1 "k8s.io/api/core/v1"
@@ -35,18 +30,36 @@ import (
3530
fakeClient "sigs.k8s.io/controller-runtime/pkg/client/fake"
3631
"sigs.k8s.io/controller-runtime/pkg/log/zap"
3732
"sigs.k8s.io/controller-runtime/pkg/reconcile"
33+
34+
mock_condition "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/condition"
35+
mock_k8s "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/k8s"
36+
mock_node "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node"
37+
mock_manager "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node/manager"
38+
"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config"
39+
cooldown "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown"
3840
)
3941

4042
var (
4143
mockConfigMap = &corev1.ConfigMap{
4244
TypeMeta: metav1.TypeMeta{},
4345
ObjectMeta: metav1.ObjectMeta{Name: config.VpcCniConfigMapName, Namespace: config.KubeSystemNamespace},
44-
Data: map[string]string{config.EnableWindowsIPAMKey: "true", config.EnableWindowsPrefixDelegationKey: "true"},
46+
Data: map[string]string{
47+
config.EnableWindowsIPAMKey: "true",
48+
config.EnableWindowsPrefixDelegationKey: "false",
49+
config.WinMinimumIPTarget: strconv.Itoa(config.IPv4DefaultWinMinIPTarget),
50+
config.WinWarmIPTarget: strconv.Itoa(config.IPv4DefaultWinWarmIPTarget),
51+
},
4552
}
4653
mockConfigMapPD = &corev1.ConfigMap{
4754
TypeMeta: metav1.TypeMeta{},
4855
ObjectMeta: metav1.ObjectMeta{Name: config.VpcCniConfigMapName, Namespace: config.KubeSystemNamespace},
49-
Data: map[string]string{config.EnableWindowsIPAMKey: "false", config.EnableWindowsPrefixDelegationKey: "true"},
56+
Data: map[string]string{
57+
config.EnableWindowsIPAMKey: "true",
58+
config.EnableWindowsPrefixDelegationKey: "true",
59+
config.WinMinimumIPTarget: strconv.Itoa(config.IPv4PDDefaultMinIPTargetSize),
60+
config.WinWarmIPTarget: strconv.Itoa(config.IPv4PDDefaultWarmIPTargetSize),
61+
config.WinWarmPrefixTarget: strconv.Itoa(config.IPv4PDDefaultWarmPrefixTargetSize),
62+
},
5063
}
5164
mockConfigMapReq = reconcile.Request{
5265
NamespacedName: types.NamespacedName{
@@ -89,11 +102,13 @@ func NewConfigMapMock(ctrl *gomock.Controller, mockObjects ...client.Object) Con
89102
return ConfigMapMock{
90103
MockNodeManager: mockNodeManager,
91104
ConfigMapReconciler: &ConfigMapReconciler{
92-
Client: client,
93-
Log: zap.New(),
94-
NodeManager: mockNodeManager,
95-
K8sAPI: mockK8sWrapper,
96-
Condition: mockCondition,
105+
Client: client,
106+
Log: zap.New(),
107+
NodeManager: mockNodeManager,
108+
K8sAPI: mockK8sWrapper,
109+
Condition: mockCondition,
110+
curWinMinIPTarget: config.IPv4DefaultWinMinIPTarget,
111+
curWinWarmIPTarget: config.IPv4DefaultWinWarmIPTarget,
97112
},
98113
MockNode: mockNode,
99114
MockK8sAPI: mockK8sWrapper,
@@ -103,13 +118,13 @@ func NewConfigMapMock(ctrl *gomock.Controller, mockObjects ...client.Object) Con
103118
}
104119
}
105120

106-
func Test_Reconcile_ConfigMap_Updated(t *testing.T) {
121+
func Test_Reconcile_ConfigMap_Updated_Secondary_IP(t *testing.T) {
107122
ctrl := gomock.NewController(t)
108123
defer ctrl.Finish()
109124

110125
mock := NewConfigMapMock(ctrl, mockConfigMap)
111126
mock.MockCondition.EXPECT().IsWindowsIPAMEnabled().Return(true)
112-
mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(true)
127+
mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(false)
113128
mock.MockK8sAPI.EXPECT().ListNodes().Return(nodeList, nil)
114129
mock.MockNodeManager.EXPECT().GetNode(mockNodeName).Return(mock.MockNode, true)
115130
mock.MockNodeManager.EXPECT().UpdateNode(mockNodeName).Return(nil)
@@ -120,14 +135,32 @@ func Test_Reconcile_ConfigMap_Updated(t *testing.T) {
120135
res, err := mock.ConfigMapReconciler.Reconcile(context.TODO(), mockConfigMapReq)
121136
assert.NoError(t, err)
122137
assert.Equal(t, res, reconcile.Result{})
138+
}
123139

140+
func Test_Reconcile_ConfigMap_Updated_PD(t *testing.T) {
141+
ctrl := gomock.NewController(t)
142+
defer ctrl.Finish()
143+
144+
mock := NewConfigMapMock(ctrl, mockConfigMapPD)
145+
mock.MockCondition.EXPECT().IsWindowsIPAMEnabled().Return(true)
146+
mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(true)
147+
mock.MockK8sAPI.EXPECT().ListNodes().Return(nodeList, nil)
148+
mock.MockNodeManager.EXPECT().GetNode(mockNodeName).Return(mock.MockNode, true)
149+
mock.MockNodeManager.EXPECT().UpdateNode(mockNodeName).Return(nil)
150+
151+
mock.MockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("30"), nil).AnyTimes()
152+
153+
cooldown.InitCoolDownPeriod(mock.MockK8sAPI, zap.New(zap.UseDevMode(true)).WithName("cooldown"))
154+
res, err := mock.ConfigMapReconciler.Reconcile(context.TODO(), mockConfigMapReq)
155+
assert.NoError(t, err)
156+
assert.Equal(t, res, reconcile.Result{})
124157
}
125158

126159
func Test_Reconcile_ConfigMap_PD_Disabled_If_IPAM_Disabled(t *testing.T) {
127160
ctrl := gomock.NewController(t)
128161
defer ctrl.Finish()
129162

130-
mock := NewConfigMapMock(ctrl, mockConfigMapPD)
163+
mock := NewConfigMapMock(ctrl, mockConfigMap)
131164
mock.MockCondition.EXPECT().IsWindowsIPAMEnabled().Return(false)
132165
mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(false)
133166
mock.MockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("30"), nil).AnyTimes()

‎docs/windows/secondary_ip_mode_config_options.md‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Configuration options when using secondary IP addresses Windows
2+
3+
We provide multiple configuration options that allow you to fine-tune the IP address allocation behavior on Windows
4+
nodes using the secondary IP address mode. These configuration options can be set in the `amazon-vpc-cni` ConfigMap in
5+
the `kube-system` namespace.
6+
7+
- `windows-warm-ip-target` → The total number of IP addresses that should be allocated to each Windows node in excess of
8+
the current need at any given time. The excess IPs can be used by newly launched pods, which aids in faster pod
9+
startup times since there is no wait time for additional IP addresses to be allocated. The VPC Resource Controller
10+
will attempt to ensure that this excess desired threshold is always met.
11+
12+
Defaults to 3 if unspecified or invalid. Must be greater than or equal to 1.
13+
14+
For example, if no pods were running on a given Windows node, and if you set `windows-warm-ip-target` to 5, the VPC
15+
Resource Controller will aim to ensure that each Windows node always has at least 5 IP addresses in excess, ready for
16+
use, allocated to its ENI. If 2 pods are scheduled on the node, the controller will allocate 2 additional IP addresses
17+
to the ENI, maintaining the 5 warm IP address target.
18+
19+
- `windows-minimum-ip-target` → Defaults to 3 if unspecified or invalid. The minimum number of IP addresses, both in use
20+
by running pods and available as warm IPs, that should be allocated to each Windows node at any given time. The
21+
controller will attempt to ensure that this minimum threshold is always met.
22+
23+
Defaults to 3 if unspecified or invalid. Must be greater than or equal to 0.
24+
25+
For example, if no pods were running on a given Windows node, and if you set `windows-minimum-ip-target` to 10, the
26+
VPC Resource Controller will aim to ensure that the total number of IP addresses on the Windows node should be at
27+
least 10. Therefore, before pods are scheduled, there should be at least 10 IP addresses available. If 5 pods are
28+
scheduled on a given node, they will consume 5 of the 10 available IPs. The VPC Resource Controller will keep 5 the
29+
remaining available IPs available in addition to the 5 already in use to meet the target of 10.
30+
31+
### Considerations while using the above configuration options
32+
33+
- These configuration options only apply when the VPC Resource Controller is operating in the secondary IP mode. They do
34+
not affect the prefix delegation mode. More explicitly, if `enable-windows-prefix-delegation` is set to false, or is
35+
not specified, then the VPC Resource Controller operates in secondary IP mode.
36+
- Setting either `windows-warm-ip-target` or `windows-minimum-ip-target` to a negative value will result in the
37+
respective default value being used.
38+
- If the values of `windows-warm-ip-target` or `windows-minimum-ip-target` are set such that the maximum node IP
39+
capacity would be exceeded, the controller will limit the allocation to the maximum capacity possible.
40+
- The `warm-prefix-target` configuration option will be ignored when using the secondary IP mode, as it only applies to
41+
the prefix delegation mode.
42+
- If `windows-warm-ip-target` is set to 0, the system will implicitly set `windows-warm-ip-target` to 1. This is
43+
because on-demand IP allocation whereby an IP is allocated on the Windows node as the pods are scheduled is currently
44+
not supported. Implicitly Setting `windows-warm-ip-target` to 1 ensures the minimum acceptable non-zero value is set
45+
since the `windows-warm-ip-target` should always be at least 1.
46+
- The configuration options `warm-ip-target` and `minimum-ip-target` are deprecated in favor of the new
47+
options `windows-warm-ip-target` and `windows-minimum-ip-target`.
48+
49+
### Examples
50+
51+
| `windows-warm-ip-target` | `windows-minimum-ip-target` | Running Pods | Total Allocated IPs | Warm IPs |
52+
|--------------------------|-----------------------------|--------------|---------------------|----------|
53+
| 1 | 0 | 0 | 1 | 1 |
54+
| 1 | 0 | 5 | 6 | 1 |
55+
| 5 | 0 | 0 | 5 | 5 |
56+
| 1 | 1 | 0 | 1 | 1 |
57+
| 1 | 1 | 1 | 2 | 1 |
58+
| 1 | 3 | 3 | 4 | 1 |
59+
| 1 | 3 | 5 | 6 | 1 |
60+
| 5 | 10 | 0 | 10 | 10 |
61+
| 10 | 10 | 0 | 10 | 10 |
62+
| 10 | 10 | 10 | 20 | 10 |
63+
| 15 | 10 | 10 | 25 | 15 |

0 commit comments

Comments
 (0)