wip

Signed-off-by: Richard Case <[email protected]>

Author: richardcase

bootstrap/eks/api/v1alpha3/zz_generated.conversion.go

@@ -52,6 +52,16 @@ type EKSConfigSpec struct {
 	// the ip family will be set to ipv6.
 	// +optional
 	// ServiceIPV6Cidr *string `json:"serviceIPV6Cidr,omitempty"`
+
+	// PreBootstrapCommands specifies extra commands to run before bootstrapping nodes to the cluster
+	// +optional
+	PreBootstrapCommands []string `json:"preBootstrapCommands,omitempty"`
+	// PostBootstrapCommands specifies extra commands to run after bootstrapping nodes to the cluster
+	// +optional
+	PostBootstrapCommands []string `json:"postBootstrapCommands,omitempty"`
+	// BootstrapCommandOverride allows you to override the bootstrap command to use for EKS nodes.
+	// +optional
+	BootstrapCommandOverride *string `json:"boostrapCommandOverride,omitempty"`
 }
 
 // PauseContainer contains details of pause container.

bootstrap/eks/api/v1alpha4/zz_generated.conversion.go

@@ -188,13 +188,16 @@ func (r *EKSConfigReconciler) joinWorker(ctx context.Context, cluster *clusterv1
 
 	nodeInput := &userdata.NodeInput{
 		// AWSManagedControlPlane webhooks default and validate EKSClusterName
-		ClusterName:      controlPlane.Spec.EKSClusterName,
-		KubeletExtraArgs: config.Spec.KubeletExtraArgs,
-		ContainerRuntime: config.Spec.ContainerRuntime,
-		DNSClusterIP:     config.Spec.DNSClusterIP,
-		DockerConfigJSON: config.Spec.DockerConfigJSON,
-		APIRetryAttempts: config.Spec.APIRetryAttempts,
-		UseMaxPods:       config.Spec.UseMaxPods,
+		ClusterName:              controlPlane.Spec.EKSClusterName,
+		KubeletExtraArgs:         config.Spec.KubeletExtraArgs,
+		ContainerRuntime:         config.Spec.ContainerRuntime,
+		DNSClusterIP:             config.Spec.DNSClusterIP,
+		DockerConfigJSON:         config.Spec.DockerConfigJSON,
+		APIRetryAttempts:         config.Spec.APIRetryAttempts,
+		UseMaxPods:               config.Spec.UseMaxPods,
+		PreBootstrapCommands:     config.Spec.PreBootstrapCommands,
+		PostBootstrapCommands:    config.Spec.PostBootstrapCommands,
+		BootstrapCommandOverride: config.Spec.BootstrapCommandOverride,
 	}
 	if config.Spec.PauseContainer != nil {
 		nodeInput.PauseContainerAccount = &config.Spec.PauseContainer.AccountNumber

bootstrap/eks/api/v1beta1/eksconfig_types.go

@@ -25,8 +25,19 @@ import (
 )
 
 const (
+	defaultBootstrapCommand = "/etc/eks/bootstrap.sh"
+
 	nodeUserData = `#!/bin/bash
-/etc/eks/bootstrap.sh {{.ClusterName}} {{- template "args" . }}
+set -o errexit; set -o pipefail; set -o nounset;
+{{- template "commands" .PreBootstrapCommands }}
+{{ .BootstrapCommand }} {{.ClusterName}} {{- template "args" . }}
+{{- template "commands" .PostBootstrapCommands }}
+`
+	commandsTemplate = `{{- define "commands" -}}
+{{ range . }}
+{{.}}
+{{- end -}}
+{{- end -}}
 `
 )
 
@@ -43,8 +54,11 @@ type NodeInput struct {
 	UseMaxPods            *bool
 	// NOTE: currently the IPFamily/ServiceIPV6Cidr isn't exposed to the user.
 	// TODO (richardcase): remove the above comment when IPV6 / dual stack is implemented.
-	IPFamily        *string
-	ServiceIPV6Cidr *string
+	IPFamily                 *string
+	ServiceIPV6Cidr          *string
+	PreBootstrapCommands     []string
+	PostBootstrapCommands    []string
+	BootstrapCommandOverride *string
 }
 
 func (ni *NodeInput) DockerConfigJSONEscaped() string {
@@ -55,6 +69,14 @@ func (ni *NodeInput) DockerConfigJSONEscaped() string {
 	return shellescape.Quote(*ni.DockerConfigJSON)
 }
 
+func (ni *NodeInput) BootstrapCommand() string {
+	if ni.BootstrapCommandOverride != nil && *ni.BootstrapCommandOverride != "" {
+		return *ni.BootstrapCommandOverride
+	}
+
+	return defaultBootstrapCommand
+}
+
 // NewNode returns the user data string to be used on a node instance.
 func NewNode(input *NodeInput) ([]byte, error) {
 	tm := template.New("Node")
@@ -67,6 +89,10 @@ func NewNode(input *NodeInput) ([]byte, error) {
 		return nil, fmt.Errorf("failed to parse kubeletExtraArgs template: %w", err)
 	}
 
+	if _, err := tm.Parse(commandsTemplate); err != nil {
+		return nil, fmt.Errorf("failed to parse commandsTemplate template: %w", err)
+	}
+
 	t, err := tm.Parse(nodeUserData)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse Node template: %w", err)

bootstrap/eks/api/v1beta1/zz_generated.deepcopy.go

@@ -46,6 +46,7 @@ func TestNewNode(t *testing.T) {
 				},
 			},
 			expectedBytes: []byte(`#!/bin/bash
+set -o errexit; set -o pipefail; set -o nounset;
 /etc/eks/bootstrap.sh test-cluster
 `),
 			expectErr: false,
@@ -62,6 +63,7 @@ func TestNewNode(t *testing.T) {
 				},
 			},
 			expectedBytes: []byte(`#!/bin/bash
+set -o errexit; set -o pipefail; set -o nounset;
 /etc/eks/bootstrap.sh test-cluster --kubelet-extra-args '--node-labels=node-role.undistro.io/infra=true --register-with-taints=dedicated=infra:NoSchedule'
 `),
 		},
@@ -74,6 +76,7 @@ func TestNewNode(t *testing.T) {
 				},
 			},
 			expectedBytes: []byte(`#!/bin/bash
+set -o errexit; set -o pipefail; set -o nounset;
 /etc/eks/bootstrap.sh test-cluster --container-runtime containerd
 `),
 		},
@@ -90,6 +93,7 @@ func TestNewNode(t *testing.T) {
 				},
 			},
 			expectedBytes: []byte(`#!/bin/bash
+set -o errexit; set -o pipefail; set -o nounset;
 /etc/eks/bootstrap.sh test-cluster --kubelet-extra-args '--node-labels=node-role.undistro.io/infra=true --register-with-taints=dedicated=infra:NoSchedule' --container-runtime containerd
 `),
 		},
@@ -103,6 +107,7 @@ func TestNewNode(t *testing.T) {
 				},
 			},
 			expectedBytes: []byte(`#!/bin/bash
+set -o errexit; set -o pipefail; set -o nounset;
 /etc/eks/bootstrap.sh test-cluster --ip-family ipv6 --service-ipv6-cidr fe80:0000:0000:0000:0204:61ff:fe9d:f156/24
 `),
 		},
@@ -115,6 +120,7 @@ func TestNewNode(t *testing.T) {
 				},
 			},
 			expectedBytes: []byte(`#!/bin/bash
+set -o errexit; set -o pipefail; set -o nounset;
 /etc/eks/bootstrap.sh test-cluster --use-max-pods false
 `),
 		},
@@ -127,6 +133,7 @@ func TestNewNode(t *testing.T) {
 				},
 			},
 			expectedBytes: []byte(`#!/bin/bash
+set -o errexit; set -o pipefail; set -o nounset;
 /etc/eks/bootstrap.sh test-cluster --aws-api-retry-attempts 5
 `),
 		},
@@ -140,6 +147,7 @@ func TestNewNode(t *testing.T) {
 				},
 			},
 			expectedBytes: []byte(`#!/bin/bash
+set -o errexit; set -o pipefail; set -o nounset;
 /etc/eks/bootstrap.sh test-cluster --pause-container-account 12345678 --pause-container-version v1
 `),
 		},
@@ -152,6 +160,7 @@ func TestNewNode(t *testing.T) {
 				},
 			},
 			expectedBytes: []byte(`#!/bin/bash
+set -o errexit; set -o pipefail; set -o nounset;
 /etc/eks/bootstrap.sh test-cluster --dns-cluster-ip 192.168.0.1
 `),
 		},
@@ -164,7 +173,67 @@ func TestNewNode(t *testing.T) {
 				},
 			},
 			expectedBytes: []byte(`#!/bin/bash
+set -o errexit; set -o pipefail; set -o nounset;
 /etc/eks/bootstrap.sh test-cluster --docker-config-json '{"debug":true}'
+`),
+		},
+		{
+			name: "with pre-bootstrap command",
+			args: args{
+				input: &NodeInput{
+					ClusterName:          "test-cluster",
+					PreBootstrapCommands: []string{"date", "echo \"testing\""},
+				},
+			},
+			expectedBytes: []byte(`#!/bin/bash
+set -o errexit; set -o pipefail; set -o nounset;
+date
+echo "testing"
+/etc/eks/bootstrap.sh test-cluster
+`),
+		},
+		{
+			name: "with post-bootstrap command",
+			args: args{
+				input: &NodeInput{
+					ClusterName:           "test-cluster",
+					PostBootstrapCommands: []string{"date", "echo \"testing\""},
+				},
+			},
+			expectedBytes: []byte(`#!/bin/bash
+set -o errexit; set -o pipefail; set -o nounset;
+/etc/eks/bootstrap.sh test-cluster
+date
+echo "testing"
+`),
+		},
+		{
+			name: "with pre & post-bootstrap command",
+			args: args{
+				input: &NodeInput{
+					ClusterName:           "test-cluster",
+					PreBootstrapCommands:  []string{"echo \"testing pre\""},
+					PostBootstrapCommands: []string{"echo \"testing post\""},
+				},
+			},
+			expectedBytes: []byte(`#!/bin/bash
+set -o errexit; set -o pipefail; set -o nounset;
+echo "testing pre"
+/etc/eks/bootstrap.sh test-cluster
+echo "testing post"
+`),
+		},
+		{
+			name: "with bootstrap override command",
+			args: args{
+				input: &NodeInput{
+					ClusterName:              "test-cluster",
+					BootstrapCommandOverride: pointer.String("/custom/mybootstrap.sh"),
+				},
+			},
+			expectedBytes: []byte(`#!/bin/bash
+set -o errexit; set -o pipefail; set -o nounset;
+/custom/mybootstrap.sh test-cluster
 `),
 		},
 	}

bootstrap/eks/controllers/eksconfig_controller.go

@@ -268,6 +268,10 @@ spec:
                 description: APIRetryAttempts is the number of retry attempts for
                   AWS API call.
                 type: integer
+              boostrapCommandOverride:
+                description: BootstrapCommandOverride allows you to override the bootstrap
+                  command to use for EKS nodes.
+                type: string
               containerRuntime:
                 description: ContainerRuntime specify the container runtime to use
                   when bootstrapping EKS.
@@ -302,6 +306,18 @@ spec:
                 - accountNumber
                 - version
                 type: object
+              postBootstrapCommands:
+                description: PostBootstrapCommands specifies extra commands to run
+                  after bootstrapping nodes to the cluster
+                items:
+                  type: string
+                type: array
+              preBootstrapCommands:
+                description: PreBootstrapCommands specifies extra commands to run
+                  before bootstrapping nodes to the cluster
+                items:
+                  type: string
+                type: array
               useMaxPods:
                 description: UseMaxPods  sets --max-pods for the kubelet when true.
                 type: boolean

bootstrap/eks/internal/userdata/node.go

@@ -132,6 +132,10 @@ spec:
                         description: APIRetryAttempts is the number of retry attempts
                           for AWS API call.
                         type: integer
+                      boostrapCommandOverride:
+                        description: BootstrapCommandOverride allows you to override
+                          the bootstrap command to use for EKS nodes.
+                        type: string
                       containerRuntime:
                         description: ContainerRuntime specify the container runtime
                           to use when bootstrapping EKS.
@@ -168,6 +172,18 @@ spec:
                         - accountNumber
                         - version
                         type: object
+                      postBootstrapCommands:
+                        description: PostBootstrapCommands specifies extra commands
+                          to run after bootstrapping nodes to the cluster
+                        items:
+                          type: string
+                        type: array
+                      preBootstrapCommands:
+                        description: PreBootstrapCommands specifies extra commands
+                          to run before bootstrapping nodes to the cluster
+                        items:
+                          type: string
+                        type: array
                       useMaxPods:
                         description: UseMaxPods  sets --max-pods for the kubelet when
                           true.