Skip to content
This repository was archived by the owner on Apr 17, 2025. It is now read-only.

Commit 72605fb

Browse files
cPu1cPu1
authored
Merge pull request eksctl-io#7218 from cPu1/cluster-subnets-security-groups
Allow mutating control plane subnets and security groups
2 parents dd6426c + c2c91ab commit 72605fb

35 files changed

+582
-46
lines changed

‎examples/38-cluster-subnets-sgs.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# An example config for updating API server endpoint access, public access CIDRs, and control plane subnets and security groups.
2+
# To perform the update, run `eksctl utils update-cluster-vpc-config -f 38-cluster-subnets-sgs.yaml`
3+
4+
apiVersion: eksctl.io/v1alpha5
5+
kind: ClusterConfig
6+
metadata:
7+
name: cluster-38
8+
region: us-west-2
9+
10+
iam:
11+
withOIDC: true
12+
13+
vpc:
14+
controlPlaneSubnetIDs: [subnet-1234, subnet-5678]
15+
controlPlaneSecurityGroupIDs: [sg-1234, sg-5678]
16+
clusterEndpoints:
17+
publicAccess: true
18+
privateAccess: true
19+
publicAccessCIDRs: ["1.1.1.1/32"]
20+
21+
managedNodeGroups:
22+
- name: mng1

‎integration/tests/managed/managed_nodegroup_test.go‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,13 @@ package managed
66
import (
77
"context"
88
"fmt"
9+
"strings"
910
"testing"
1011
"time"
1112

1213
"github.com/aws/aws-sdk-go-v2/aws"
14+
"github.com/aws/aws-sdk-go-v2/service/ec2"
15+
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
1316
awseks "github.com/aws/aws-sdk-go-v2/service/eks"
1417

1518
harness "github.com/dlespiau/kube-test-harness"
@@ -27,6 +30,7 @@ import (
2730
clusterutils "github.com/weaveworks/eksctl/integration/utilities/cluster"
2831
"github.com/weaveworks/eksctl/integration/utilities/kube"
2932
api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
33+
"github.com/weaveworks/eksctl/pkg/awsapi"
3034
"github.com/weaveworks/eksctl/pkg/eks"
3135
"github.com/weaveworks/eksctl/pkg/testutils"
3236
)
@@ -521,6 +525,107 @@ var _ = Describe("(Integration) Create Managed Nodegroups", func() {
521525
Expect(cmd).To(RunSuccessfully())
522526
})
523527
})
528+
529+
Context("eksctl utils update-cluster-vpc-config", Serial, func() {
530+
makeAWSProvider := func(ctx context.Context, clusterConfig *api.ClusterConfig) api.ClusterProvider {
531+
clusterProvider, err := eks.New(ctx, &api.ProviderConfig{Region: params.Region}, clusterConfig)
532+
Expect(err).NotTo(HaveOccurred())
533+
return clusterProvider.AWSProvider
534+
}
535+
getPrivateSubnetIDs := func(ctx context.Context, ec2API awsapi.EC2, vpcID string) []string {
536+
out, err := ec2API.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{
537+
Filters: []ec2types.Filter{
538+
{
539+
Name: aws.String("vpc-id"),
540+
Values: []string{vpcID},
541+
},
542+
},
543+
})
544+
Expect(err).NotTo(HaveOccurred())
545+
var subnetIDs []string
546+
for _, s := range out.Subnets {
547+
if !*s.MapPublicIpOnLaunch {
548+
subnetIDs = append(subnetIDs, *s.SubnetId)
549+
}
550+
}
551+
return subnetIDs
552+
}
553+
It("should update the VPC config", func() {
554+
clusterConfig := makeClusterConfig()
555+
ctx := context.Background()
556+
awsProvider := makeAWSProvider(ctx, clusterConfig)
557+
cluster, err := awsProvider.EKS().DescribeCluster(ctx, &awseks.DescribeClusterInput{
558+
Name: aws.String(params.ClusterName),
559+
})
560+
Expect(err).NotTo(HaveOccurred(), "error describing cluster")
561+
clusterSubnetIDs := getPrivateSubnetIDs(ctx, awsProvider.EC2(), *cluster.Cluster.ResourcesVpcConfig.VpcId)
562+
Expect(len(cluster.Cluster.ResourcesVpcConfig.SecurityGroupIds) > 0).To(BeTrue(), "at least one security group ID must be associated with the cluster")
563+
564+
clusterVPC := &api.ClusterVPC{
565+
ClusterEndpoints: &api.ClusterEndpoints{
566+
PrivateAccess: api.Enabled(),
567+
PublicAccess: api.Enabled(),
568+
},
569+
PublicAccessCIDRs: []string{"127.0.0.1/32"},
570+
ControlPlaneSubnetIDs: clusterSubnetIDs,
571+
ControlPlaneSecurityGroupIDs: []string{cluster.Cluster.ResourcesVpcConfig.SecurityGroupIds[0]},
572+
}
573+
By("accepting CLI options")
574+
cmd := params.EksctlUtilsCmd.WithArgs(
575+
"update-cluster-vpc-config",
576+
"--cluster", params.ClusterName,
577+
"--private-access",
578+
"--public-access",
579+
"--public-access-cidrs", strings.Join(clusterVPC.PublicAccessCIDRs, ","),
580+
"--control-plane-subnet-ids", strings.Join(clusterVPC.ControlPlaneSubnetIDs, ","),
581+
"--control-plane-security-group-ids", strings.Join(clusterVPC.ControlPlaneSecurityGroupIDs, ","),
582+
"-v4",
583+
"--approve",
584+
).
585+
WithTimeout(45 * time.Minute)
586+
session := cmd.Run()
587+
Expect(session.ExitCode()).To(Equal(0))
588+
589+
formatWithClusterAndRegion := func(format string, values ...any) string {
590+
return fmt.Sprintf(format, append([]any{params.ClusterName, params.Region}, values...)...)
591+
}
592+
Expect(strings.Split(string(session.Buffer().Contents()), "\n")).To(ContainElements(
593+
ContainSubstring(formatWithClusterAndRegion("control plane subnets and security groups for cluster %q in %q have been updated to: "+
594+
"controlPlaneSubnetIDs=%v, controlPlaneSecurityGroupIDs=%v", clusterVPC.ControlPlaneSubnetIDs, clusterVPC.ControlPlaneSecurityGroupIDs)),
595+
ContainSubstring(formatWithClusterAndRegion("Kubernetes API endpoint access for cluster %q in %q has been updated to: privateAccess=%v, publicAccess=%v",
596+
*clusterVPC.ClusterEndpoints.PrivateAccess, *clusterVPC.ClusterEndpoints.PublicAccess)),
597+
ContainSubstring(formatWithClusterAndRegion("public access CIDRs for cluster %q in %q have been updated to: %v", clusterVPC.PublicAccessCIDRs)),
598+
))
599+
600+
By("accepting a config file")
601+
clusterConfig.VPC = clusterVPC
602+
cmd = params.EksctlUtilsCmd.WithArgs(
603+
"update-cluster-vpc-config",
604+
"--config-file", "-",
605+
"-v4",
606+
"--approve",
607+
).
608+
WithoutArg("--region", params.Region).
609+
WithStdin(clusterutils.Reader(clusterConfig))
610+
session = cmd.Run()
611+
Expect(session.ExitCode()).To(Equal(0))
612+
Expect(strings.Split(string(session.Buffer().Contents()), "\n")).To(ContainElements(
613+
ContainSubstring(formatWithClusterAndRegion("Kubernetes API endpoint access for cluster %q in %q is already up-to-date")),
614+
ContainSubstring(formatWithClusterAndRegion("control plane subnet IDs for cluster %q in %q are already up-to-date")),
615+
ContainSubstring(formatWithClusterAndRegion("control plane security group IDs for cluster %q in %q are already up-to-date")),
616+
))
617+
618+
By("resetting public access CIDRs")
619+
cmd = params.EksctlUtilsCmd.WithArgs(
620+
"update-cluster-vpc-config",
621+
"--cluster", params.ClusterName,
622+
"--public-access-cidrs", "0.0.0.0/0",
623+
"-v4",
624+
"--approve",
625+
)
626+
Expect(cmd).To(RunSuccessfully())
627+
})
628+
})
524629
})
525630

526631
var _ = SynchronizedAfterSuite(func() {}, func() {

‎pkg/apis/eksctl.io/v1alpha5/assets/schema.json‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -655,6 +655,22 @@
655655
"description": "See [managing access to API](/usage/vpc-networking/#managing-access-to-the-kubernetes-api-server-endpoints)",
656656
"x-intellij-html-description": "See <a href=\"/usage/vpc-networking/#managing-access-to-the-kubernetes-api-server-endpoints\">managing access to API</a>"
657657
},
658+
"controlPlaneSecurityGroupIDs": {
659+
"items": {
660+
"type": "string"
661+
},
662+
"type": "array",
663+
"description": "configures the security groups for the control plane.",
664+
"x-intellij-html-description": "configures the security groups for the control plane."
665+
},
666+
"controlPlaneSubnetIDs": {
667+
"items": {
668+
"type": "string"
669+
},
670+
"type": "array",
671+
"description": "configures the subnets for the control plane.",
672+
"x-intellij-html-description": "configures the subnets for the control plane."
673+
},
658674
"extraCIDRs": {
659675
"items": {
660676
"type": "string"
@@ -733,7 +749,9 @@
733749
"autoAllocateIPv6",
734750
"nat",
735751
"clusterEndpoints",
736-
"publicAccessCIDRs"
752+
"publicAccessCIDRs",
753+
"controlPlaneSubnetIDs",
754+
"controlPlaneSecurityGroupIDs"
737755
],
738756
"additionalProperties": false,
739757
"description": "holds global subnet and all child subnets",

‎pkg/apis/eksctl.io/v1alpha5/validation.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,10 @@ func (c *ClusterConfig) ValidateVPCConfig() error {
318318
c.VPC.ExtraIPv6CIDRs = cidrs
319319
}
320320

321+
if c.VPC.SecurityGroup != "" && len(c.VPC.ControlPlaneSecurityGroupIDs) > 0 {
322+
return errors.New("only one of vpc.securityGroup and vpc.controlPlaneSecurityGroupIDs can be specified")
323+
}
324+
321325
if (c.VPC.IPv6Cidr != "" || c.VPC.IPv6Pool != "") && !c.IPv6Enabled() {
322326
return fmt.Errorf("Ipv6Cidr and Ipv6CidrPool are only supported when IPFamily is set to IPv6")
323327
}

‎pkg/apis/eksctl.io/v1alpha5/validation_test.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1332,6 +1332,41 @@ var _ = Describe("ClusterConfig validation", func() {
13321332
})
13331333

13341334
})
1335+
1336+
type vpcSecurityGroupEntry struct {
1337+
updateVPC func(*api.ClusterVPC)
1338+
expectedErr string
1339+
}
1340+
DescribeTable("vpc.securityGroup and vpc.controlPlaneSecurityGroupIDs", func(e vpcSecurityGroupEntry) {
1341+
e.updateVPC(cfg.VPC)
1342+
err := cfg.ValidateVPCConfig()
1343+
if e.expectedErr != "" {
1344+
Expect(err).To(MatchError(ContainSubstring(e.expectedErr)))
1345+
} else {
1346+
Expect(err).NotTo(HaveOccurred())
1347+
}
1348+
},
1349+
Entry("both set", vpcSecurityGroupEntry{
1350+
updateVPC: func(v *api.ClusterVPC) {
1351+
v.SecurityGroup = "sg-1234"
1352+
v.ControlPlaneSecurityGroupIDs = []string{"sg-1234"}
1353+
},
1354+
expectedErr: "only one of vpc.securityGroup and vpc.controlPlaneSecurityGroupIDs can be specified",
1355+
}),
1356+
Entry("vpc.securityGroup set", vpcSecurityGroupEntry{
1357+
updateVPC: func(v *api.ClusterVPC) {
1358+
v.SecurityGroup = "sg-1234"
1359+
},
1360+
}),
1361+
Entry("vpc.controlPlaneSecurityGroupIDs set", vpcSecurityGroupEntry{
1362+
updateVPC: func(v *api.ClusterVPC) {
1363+
v.ControlPlaneSecurityGroupIDs = []string{"sg-1234"}
1364+
},
1365+
}),
1366+
Entry("neither set", vpcSecurityGroupEntry{
1367+
updateVPC: func(v *api.ClusterVPC) {},
1368+
}),
1369+
)
13351370
})
13361371

13371372
Describe("ValidatePrivateCluster", func() {

‎pkg/apis/eksctl.io/v1alpha5/vpc.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,12 @@ type (
175175
// k8s API endpoint
176176
// +optional
177177
PublicAccessCIDRs []string `json:"publicAccessCIDRs,omitempty"`
178+
// ControlPlaneSubnetIDs configures the subnets for the control plane.
179+
// +optional
180+
ControlPlaneSubnetIDs []string `json:"controlPlaneSubnetIDs,omitempty"`
181+
// ControlPlaneSecurityGroupIDs configures the security groups for the control plane.
182+
// +optional
183+
ControlPlaneSecurityGroupIDs []string `json:"controlPlaneSecurityGroupIDs,omitempty"`
178184
}
179185
// ClusterSubnets holds private and public subnets
180186
ClusterSubnets struct {

‎pkg/apis/eksctl.io/v1alpha5/zz_generated.deepcopy.go‎

Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/cfn/builder/cluster.go‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ type ClusterResourceSet struct {
2727
ec2API awsapi.EC2
2828
region string
2929
vpcResourceSet VPCResourceSet
30-
securityGroups []*gfnt.Value
30+
securityGroups *gfnt.Value
3131
}
3232

3333
// NewClusterResourceSet returns a resource set for the new cluster.
@@ -115,7 +115,13 @@ func (c *ClusterResourceSet) AddAllResources(ctx context.Context) error {
115115
func (c *ClusterResourceSet) addResourcesForSecurityGroups(vpcID *gfnt.Value) *clusterSecurityGroup {
116116
var refControlPlaneSG, refClusterSharedNodeSG *gfnt.Value
117117

118-
if c.spec.VPC.SecurityGroup == "" {
118+
if sg := c.spec.VPC.SecurityGroup; sg != "" {
119+
refControlPlaneSG = gfnt.NewString(sg)
120+
c.securityGroups = gfnt.NewStringSlice(sg)
121+
} else if securityGroupIDs := c.spec.VPC.ControlPlaneSecurityGroupIDs; len(securityGroupIDs) > 0 {
122+
refControlPlaneSG = gfnt.NewString(securityGroupIDs[0])
123+
c.securityGroups = gfnt.NewStringSlice(securityGroupIDs...)
124+
} else {
119125
refControlPlaneSG = c.newResource(cfnControlPlaneSGResource, &gfnec2.SecurityGroup{
120126
GroupDescription: gfnt.NewString("Communication between the control plane and worker nodegroups"),
121127
VpcId: vpcID,
@@ -146,10 +152,8 @@ func (c *ClusterResourceSet) addResourcesForSecurityGroups(vpcID *gfnt.Value) *c
146152
})
147153
}
148154
}
149-
} else {
150-
refControlPlaneSG = gfnt.NewString(c.spec.VPC.SecurityGroup)
155+
c.securityGroups = gfnt.NewSlice(refControlPlaneSG)
151156
}
152-
c.securityGroups = []*gfnt.Value{refControlPlaneSG} // only this one SG is passed to EKS API, nodes are isolated
153157

154158
if c.spec.VPC.SharedNodeSecurityGroup == "" {
155159
refClusterSharedNodeSG = c.newResource(cfnSharedNodeSGResource, &gfnec2.SecurityGroup{
@@ -263,12 +267,16 @@ func (c *ClusterResourceSet) newResource(name string, resource gfn.Resource) *gf
263267

264268
func (c *ClusterResourceSet) addResourcesForControlPlane(subnetDetails *SubnetDetails) {
265269
clusterVPC := &gfneks.Cluster_ResourcesVpcConfig{
266-
SubnetIds: gfnt.NewSlice(subnetDetails.ControlPlaneSubnetRefs()...),
267270
EndpointPublicAccess: gfnt.NewBoolean(*c.spec.VPC.ClusterEndpoints.PublicAccess),
268271
EndpointPrivateAccess: gfnt.NewBoolean(*c.spec.VPC.ClusterEndpoints.PrivateAccess),
269-
SecurityGroupIds: gfnt.NewSlice(c.securityGroups...),
272+
SecurityGroupIds: c.securityGroups,
270273
PublicAccessCidrs: gfnt.NewStringSlice(c.spec.VPC.PublicAccessCIDRs...),
271274
}
275+
if subnetIDs := c.spec.VPC.ControlPlaneSubnetIDs; len(subnetIDs) > 0 {
276+
clusterVPC.SubnetIds = gfnt.NewStringSlice(subnetIDs...)
277+
} else {
278+
clusterVPC.SubnetIds = gfnt.NewSlice(subnetDetails.ControlPlaneSubnetRefs()...)
279+
}
272280

273281
serviceRoleARN := gfnt.MakeFnGetAttString("ServiceRole", "Arn")
274282
if api.IsSetAndNonEmptyString(c.spec.IAM.ServiceRoleARN) {

0 commit comments

Comments
 (0)