diff --git a/bootstrap/eks/api/v1alpha3/zz_generated.conversion.go b/bootstrap/eks/api/v1alpha3/zz_generated.conversion.go index 08c2db670d..a6f40d8afb 100644 --- a/bootstrap/eks/api/v1alpha3/zz_generated.conversion.go +++ b/bootstrap/eks/api/v1alpha3/zz_generated.conversion.go @@ -213,6 +213,9 @@ func autoConvert_v1beta1_EKSConfigSpec_To_v1alpha3_EKSConfigSpec(in *v1beta1.EKS // WARNING: in.APIRetryAttempts requires manual conversion: does not exist in peer-type // WARNING: in.PauseContainer requires manual conversion: does not exist in peer-type // WARNING: in.UseMaxPods requires manual conversion: does not exist in peer-type + // WARNING: in.PreBootstrapCommands requires manual conversion: does not exist in peer-type + // WARNING: in.PostBootstrapCommands requires manual conversion: does not exist in peer-type + // WARNING: in.BootstrapCommandOverride requires manual conversion: does not exist in peer-type return nil } diff --git a/bootstrap/eks/api/v1alpha4/zz_generated.conversion.go b/bootstrap/eks/api/v1alpha4/zz_generated.conversion.go index 224a6e3afd..89b5b08a8e 100644 --- a/bootstrap/eks/api/v1alpha4/zz_generated.conversion.go +++ b/bootstrap/eks/api/v1alpha4/zz_generated.conversion.go @@ -213,6 +213,9 @@ func autoConvert_v1beta1_EKSConfigSpec_To_v1alpha4_EKSConfigSpec(in *v1beta1.EKS // WARNING: in.APIRetryAttempts requires manual conversion: does not exist in peer-type // WARNING: in.PauseContainer requires manual conversion: does not exist in peer-type // WARNING: in.UseMaxPods requires manual conversion: does not exist in peer-type + // WARNING: in.PreBootstrapCommands requires manual conversion: does not exist in peer-type + // WARNING: in.PostBootstrapCommands requires manual conversion: does not exist in peer-type + // WARNING: in.BootstrapCommandOverride requires manual conversion: does not exist in peer-type return nil } diff --git a/bootstrap/eks/api/v1beta1/eksconfig_types.go b/bootstrap/eks/api/v1beta1/eksconfig_types.go index 066696b178..605e2e43a0 100644 --- a/bootstrap/eks/api/v1beta1/eksconfig_types.go +++ b/bootstrap/eks/api/v1beta1/eksconfig_types.go @@ -52,6 +52,16 @@ type EKSConfigSpec struct { // the ip family will be set to ipv6. // +optional // ServiceIPV6Cidr *string `json:"serviceIPV6Cidr,omitempty"` + + // PreBootstrapCommands specifies extra commands to run before bootstrapping nodes to the cluster + // +optional + PreBootstrapCommands []string `json:"preBootstrapCommands,omitempty"` + // PostBootstrapCommands specifies extra commands to run after bootstrapping nodes to the cluster + // +optional + PostBootstrapCommands []string `json:"postBootstrapCommands,omitempty"` + // BootstrapCommandOverride allows you to override the bootstrap command to use for EKS nodes. + // +optional + BootstrapCommandOverride *string `json:"boostrapCommandOverride,omitempty"` } // PauseContainer contains details of pause container. diff --git a/bootstrap/eks/api/v1beta1/zz_generated.deepcopy.go b/bootstrap/eks/api/v1beta1/zz_generated.deepcopy.go index c05693098f..dfd2e1764d 100644 --- a/bootstrap/eks/api/v1beta1/zz_generated.deepcopy.go +++ b/bootstrap/eks/api/v1beta1/zz_generated.deepcopy.go @@ -125,6 +125,21 @@ func (in *EKSConfigSpec) DeepCopyInto(out *EKSConfigSpec) { *out = new(bool) **out = **in } + if in.PreBootstrapCommands != nil { + in, out := &in.PreBootstrapCommands, &out.PreBootstrapCommands + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PostBootstrapCommands != nil { + in, out := &in.PostBootstrapCommands, &out.PostBootstrapCommands + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.BootstrapCommandOverride != nil { + in, out := &in.BootstrapCommandOverride, &out.BootstrapCommandOverride + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EKSConfigSpec. diff --git a/bootstrap/eks/controllers/eksconfig_controller.go b/bootstrap/eks/controllers/eksconfig_controller.go index c5f83c050b..b5cfdc78af 100644 --- a/bootstrap/eks/controllers/eksconfig_controller.go +++ b/bootstrap/eks/controllers/eksconfig_controller.go @@ -188,13 +188,16 @@ func (r *EKSConfigReconciler) joinWorker(ctx context.Context, cluster *clusterv1 nodeInput := &userdata.NodeInput{ // AWSManagedControlPlane webhooks default and validate EKSClusterName - ClusterName: controlPlane.Spec.EKSClusterName, - KubeletExtraArgs: config.Spec.KubeletExtraArgs, - ContainerRuntime: config.Spec.ContainerRuntime, - DNSClusterIP: config.Spec.DNSClusterIP, - DockerConfigJSON: config.Spec.DockerConfigJSON, - APIRetryAttempts: config.Spec.APIRetryAttempts, - UseMaxPods: config.Spec.UseMaxPods, + ClusterName: controlPlane.Spec.EKSClusterName, + KubeletExtraArgs: config.Spec.KubeletExtraArgs, + ContainerRuntime: config.Spec.ContainerRuntime, + DNSClusterIP: config.Spec.DNSClusterIP, + DockerConfigJSON: config.Spec.DockerConfigJSON, + APIRetryAttempts: config.Spec.APIRetryAttempts, + UseMaxPods: config.Spec.UseMaxPods, + PreBootstrapCommands: config.Spec.PreBootstrapCommands, + PostBootstrapCommands: config.Spec.PostBootstrapCommands, + BootstrapCommandOverride: config.Spec.BootstrapCommandOverride, } if config.Spec.PauseContainer != nil { nodeInput.PauseContainerAccount = &config.Spec.PauseContainer.AccountNumber diff --git a/bootstrap/eks/internal/userdata/node.go b/bootstrap/eks/internal/userdata/node.go index f7ac0d5a8f..c618553565 100644 --- a/bootstrap/eks/internal/userdata/node.go +++ b/bootstrap/eks/internal/userdata/node.go @@ -25,8 +25,19 @@ import ( ) const ( + defaultBootstrapCommand = "/etc/eks/bootstrap.sh" + nodeUserData = `#!/bin/bash -/etc/eks/bootstrap.sh {{.ClusterName}} {{- template "args" . }} +set -o errexit; set -o pipefail; set -o nounset; +{{- template "commands" .PreBootstrapCommands }} +{{ .BootstrapCommand }} {{.ClusterName}} {{- template "args" . }} +{{- template "commands" .PostBootstrapCommands }} +` + commandsTemplate = `{{- define "commands" -}} +{{ range . }} +{{.}} +{{- end -}} +{{- end -}} ` ) @@ -43,8 +54,11 @@ type NodeInput struct { UseMaxPods *bool // NOTE: currently the IPFamily/ServiceIPV6Cidr isn't exposed to the user. // TODO (richardcase): remove the above comment when IPV6 / dual stack is implemented. - IPFamily *string - ServiceIPV6Cidr *string + IPFamily *string + ServiceIPV6Cidr *string + PreBootstrapCommands []string + PostBootstrapCommands []string + BootstrapCommandOverride *string } func (ni *NodeInput) DockerConfigJSONEscaped() string { @@ -55,6 +69,14 @@ func (ni *NodeInput) DockerConfigJSONEscaped() string { return shellescape.Quote(*ni.DockerConfigJSON) } +func (ni *NodeInput) BootstrapCommand() string { + if ni.BootstrapCommandOverride != nil && *ni.BootstrapCommandOverride != "" { + return *ni.BootstrapCommandOverride + } + + return defaultBootstrapCommand +} + // NewNode returns the user data string to be used on a node instance. func NewNode(input *NodeInput) ([]byte, error) { tm := template.New("Node") @@ -67,6 +89,10 @@ func NewNode(input *NodeInput) ([]byte, error) { return nil, fmt.Errorf("failed to parse kubeletExtraArgs template: %w", err) } + if _, err := tm.Parse(commandsTemplate); err != nil { + return nil, fmt.Errorf("failed to parse commandsTemplate template: %w", err) + } + t, err := tm.Parse(nodeUserData) if err != nil { return nil, fmt.Errorf("failed to parse Node template: %w", err) diff --git a/bootstrap/eks/internal/userdata/node_test.go b/bootstrap/eks/internal/userdata/node_test.go index 99263236fd..85094f11fe 100644 --- a/bootstrap/eks/internal/userdata/node_test.go +++ b/bootstrap/eks/internal/userdata/node_test.go @@ -46,6 +46,7 @@ func TestNewNode(t *testing.T) { }, }, expectedBytes: []byte(`#!/bin/bash +set -o errexit; set -o pipefail; set -o nounset; /etc/eks/bootstrap.sh test-cluster `), expectErr: false, @@ -62,6 +63,7 @@ func TestNewNode(t *testing.T) { }, }, expectedBytes: []byte(`#!/bin/bash +set -o errexit; set -o pipefail; set -o nounset; /etc/eks/bootstrap.sh test-cluster --kubelet-extra-args '--node-labels=node-role.undistro.io/infra=true --register-with-taints=dedicated=infra:NoSchedule' `), }, @@ -74,6 +76,7 @@ func TestNewNode(t *testing.T) { }, }, expectedBytes: []byte(`#!/bin/bash +set -o errexit; set -o pipefail; set -o nounset; /etc/eks/bootstrap.sh test-cluster --container-runtime containerd `), }, @@ -90,6 +93,7 @@ func TestNewNode(t *testing.T) { }, }, expectedBytes: []byte(`#!/bin/bash +set -o errexit; set -o pipefail; set -o nounset; /etc/eks/bootstrap.sh test-cluster --kubelet-extra-args '--node-labels=node-role.undistro.io/infra=true --register-with-taints=dedicated=infra:NoSchedule' --container-runtime containerd `), }, @@ -103,6 +107,7 @@ func TestNewNode(t *testing.T) { }, }, expectedBytes: []byte(`#!/bin/bash +set -o errexit; set -o pipefail; set -o nounset; /etc/eks/bootstrap.sh test-cluster --ip-family ipv6 --service-ipv6-cidr fe80:0000:0000:0000:0204:61ff:fe9d:f156/24 `), }, @@ -115,6 +120,7 @@ func TestNewNode(t *testing.T) { }, }, expectedBytes: []byte(`#!/bin/bash +set -o errexit; set -o pipefail; set -o nounset; /etc/eks/bootstrap.sh test-cluster --use-max-pods false `), }, @@ -127,6 +133,7 @@ func TestNewNode(t *testing.T) { }, }, expectedBytes: []byte(`#!/bin/bash +set -o errexit; set -o pipefail; set -o nounset; /etc/eks/bootstrap.sh test-cluster --aws-api-retry-attempts 5 `), }, @@ -140,6 +147,7 @@ func TestNewNode(t *testing.T) { }, }, expectedBytes: []byte(`#!/bin/bash +set -o errexit; set -o pipefail; set -o nounset; /etc/eks/bootstrap.sh test-cluster --pause-container-account 12345678 --pause-container-version v1 `), }, @@ -152,6 +160,7 @@ func TestNewNode(t *testing.T) { }, }, expectedBytes: []byte(`#!/bin/bash +set -o errexit; set -o pipefail; set -o nounset; /etc/eks/bootstrap.sh test-cluster --dns-cluster-ip 192.168.0.1 `), }, @@ -164,7 +173,67 @@ func TestNewNode(t *testing.T) { }, }, expectedBytes: []byte(`#!/bin/bash +set -o errexit; set -o pipefail; set -o nounset; /etc/eks/bootstrap.sh test-cluster --docker-config-json '{"debug":true}' +`), + }, + { + name: "with pre-bootstrap command", + args: args{ + input: &NodeInput{ + ClusterName: "test-cluster", + PreBootstrapCommands: []string{"date", "echo \"testing\""}, + }, + }, + expectedBytes: []byte(`#!/bin/bash +set -o errexit; set -o pipefail; set -o nounset; +date +echo "testing" +/etc/eks/bootstrap.sh test-cluster +`), + }, + { + name: "with post-bootstrap command", + args: args{ + input: &NodeInput{ + ClusterName: "test-cluster", + PostBootstrapCommands: []string{"date", "echo \"testing\""}, + }, + }, + expectedBytes: []byte(`#!/bin/bash +set -o errexit; set -o pipefail; set -o nounset; +/etc/eks/bootstrap.sh test-cluster +date +echo "testing" +`), + }, + { + name: "with pre & post-bootstrap command", + args: args{ + input: &NodeInput{ + ClusterName: "test-cluster", + PreBootstrapCommands: []string{"echo \"testing pre\""}, + PostBootstrapCommands: []string{"echo \"testing post\""}, + }, + }, + expectedBytes: []byte(`#!/bin/bash +set -o errexit; set -o pipefail; set -o nounset; +echo "testing pre" +/etc/eks/bootstrap.sh test-cluster +echo "testing post" +`), + }, + { + name: "with bootstrap override command", + args: args{ + input: &NodeInput{ + ClusterName: "test-cluster", + BootstrapCommandOverride: pointer.String("/custom/mybootstrap.sh"), + }, + }, + expectedBytes: []byte(`#!/bin/bash +set -o errexit; set -o pipefail; set -o nounset; +/custom/mybootstrap.sh test-cluster `), }, } diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigs.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigs.yaml index 239714544f..8ec93af8af 100644 --- a/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigs.yaml +++ b/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigs.yaml @@ -268,6 +268,10 @@ spec: description: APIRetryAttempts is the number of retry attempts for AWS API call. type: integer + boostrapCommandOverride: + description: BootstrapCommandOverride allows you to override the bootstrap + command to use for EKS nodes. + type: string containerRuntime: description: ContainerRuntime specify the container runtime to use when bootstrapping EKS. @@ -302,6 +306,18 @@ spec: - accountNumber - version type: object + postBootstrapCommands: + description: PostBootstrapCommands specifies extra commands to run + after bootstrapping nodes to the cluster + items: + type: string + type: array + preBootstrapCommands: + description: PreBootstrapCommands specifies extra commands to run + before bootstrapping nodes to the cluster + items: + type: string + type: array useMaxPods: description: UseMaxPods sets --max-pods for the kubelet when true. type: boolean diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigtemplates.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigtemplates.yaml index 345bdc072d..2b542c0ddd 100644 --- a/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigtemplates.yaml +++ b/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigtemplates.yaml @@ -132,6 +132,10 @@ spec: description: APIRetryAttempts is the number of retry attempts for AWS API call. type: integer + boostrapCommandOverride: + description: BootstrapCommandOverride allows you to override + the bootstrap command to use for EKS nodes. + type: string containerRuntime: description: ContainerRuntime specify the container runtime to use when bootstrapping EKS. @@ -168,6 +172,18 @@ spec: - accountNumber - version type: object + postBootstrapCommands: + description: PostBootstrapCommands specifies extra commands + to run after bootstrapping nodes to the cluster + items: + type: string + type: array + preBootstrapCommands: + description: PreBootstrapCommands specifies extra commands + to run before bootstrapping nodes to the cluster + items: + type: string + type: array useMaxPods: description: UseMaxPods sets --max-pods for the kubelet when true.