Skip to content

WIP feat: add support for pre/post commands on EKS node bootstrap #3341

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
richardcase wants to merge 1 commit into from
Closed

WIP feat: add support for pre/post commands on EKS node bootstrap #3341

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions bootstrap/eks/api/v1alpha3/zz_generated.conversion.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions bootstrap/eks/api/v1alpha4/zz_generated.conversion.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 10 additions & 0 deletions bootstrap/eks/api/v1beta1/eksconfig_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,16 @@ type EKSConfigSpec struct {
// the ip family will be set to ipv6.
// +optional
// ServiceIPV6Cidr *string `json:"serviceIPV6Cidr,omitempty"`

// PreBootstrapCommands specifies extra commands to run before bootstrapping nodes to the cluster
// +optional
PreBootstrapCommands []string `json:"preBootstrapCommands,omitempty"`
// PostBootstrapCommands specifies extra commands to run after bootstrapping nodes to the cluster
// +optional
PostBootstrapCommands []string `json:"postBootstrapCommands,omitempty"`
// BootstrapCommandOverride allows you to override the bootstrap command to use for EKS nodes.
// +optional
BootstrapCommandOverride *string `json:"boostrapCommandOverride,omitempty"`
}

// PauseContainer contains details of pause container.
Expand Down
15 changes: 15 additions & 0 deletions bootstrap/eks/api/v1beta1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

17 changes: 10 additions & 7 deletions bootstrap/eks/controllers/eksconfig_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,13 +188,16 @@ func (r *EKSConfigReconciler) joinWorker(ctx context.Context, cluster *clusterv1

nodeInput := &userdata.NodeInput{
// AWSManagedControlPlane webhooks default and validate EKSClusterName
ClusterName: controlPlane.Spec.EKSClusterName,
KubeletExtraArgs: config.Spec.KubeletExtraArgs,
ContainerRuntime: config.Spec.ContainerRuntime,
DNSClusterIP: config.Spec.DNSClusterIP,
DockerConfigJSON: config.Spec.DockerConfigJSON,
APIRetryAttempts: config.Spec.APIRetryAttempts,
UseMaxPods: config.Spec.UseMaxPods,
ClusterName: controlPlane.Spec.EKSClusterName,
KubeletExtraArgs: config.Spec.KubeletExtraArgs,
ContainerRuntime: config.Spec.ContainerRuntime,
DNSClusterIP: config.Spec.DNSClusterIP,
DockerConfigJSON: config.Spec.DockerConfigJSON,
APIRetryAttempts: config.Spec.APIRetryAttempts,
UseMaxPods: config.Spec.UseMaxPods,
PreBootstrapCommands: config.Spec.PreBootstrapCommands,
PostBootstrapCommands: config.Spec.PostBootstrapCommands,
BootstrapCommandOverride: config.Spec.BootstrapCommandOverride,
}
if config.Spec.PauseContainer != nil {
nodeInput.PauseContainerAccount = &config.Spec.PauseContainer.AccountNumber
Expand Down
32 changes: 29 additions & 3 deletions bootstrap/eks/internal/userdata/node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,19 @@ import (
)

const (
defaultBootstrapCommand = "/etc/eks/bootstrap.sh"

nodeUserData = `#!/bin/bash
/etc/eks/bootstrap.sh {{.ClusterName}} {{- template "args" . }}
set -o errexit; set -o pipefail; set -o nounset;
{{- template "commands" .PreBootstrapCommands }}
{{ .BootstrapCommand }} {{.ClusterName}} {{- template "args" . }}
{{- template "commands" .PostBootstrapCommands }}
`
commandsTemplate = `{{- define "commands" -}}
{{ range . }}
{{.}}
{{- end -}}
{{- end -}}
`
)

Expand All @@ -43,8 +54,11 @@ type NodeInput struct {
UseMaxPods *bool
// NOTE: currently the IPFamily/ServiceIPV6Cidr isn't exposed to the user.
// TODO (richardcase): remove the above comment when IPV6 / dual stack is implemented.
IPFamily *string
ServiceIPV6Cidr *string
IPFamily *string
ServiceIPV6Cidr *string
PreBootstrapCommands []string
PostBootstrapCommands []string
BootstrapCommandOverride *string
}

func (ni *NodeInput) DockerConfigJSONEscaped() string {
Expand All @@ -55,6 +69,14 @@ func (ni *NodeInput) DockerConfigJSONEscaped() string {
return shellescape.Quote(*ni.DockerConfigJSON)
}

func (ni *NodeInput) BootstrapCommand() string {
if ni.BootstrapCommandOverride != nil && *ni.BootstrapCommandOverride != "" {
return *ni.BootstrapCommandOverride
}

return defaultBootstrapCommand
}

// NewNode returns the user data string to be used on a node instance.
func NewNode(input *NodeInput) ([]byte, error) {
tm := template.New("Node")
Expand All @@ -67,6 +89,10 @@ func NewNode(input *NodeInput) ([]byte, error) {
return nil, fmt.Errorf("failed to parse kubeletExtraArgs template: %w", err)
}

if _, err := tm.Parse(commandsTemplate); err != nil {
return nil, fmt.Errorf("failed to parse commandsTemplate template: %w", err)
}

t, err := tm.Parse(nodeUserData)
if err != nil {
return nil, fmt.Errorf("failed to parse Node template: %w", err)
Expand Down
69 changes: 69 additions & 0 deletions bootstrap/eks/internal/userdata/node_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ func TestNewNode(t *testing.T) {
},
},
expectedBytes: []byte(`#!/bin/bash
set -o errexit; set -o pipefail; set -o nounset;
/etc/eks/bootstrap.sh test-cluster
`),
expectErr: false,
Expand All @@ -62,6 +63,7 @@ func TestNewNode(t *testing.T) {
},
},
expectedBytes: []byte(`#!/bin/bash
set -o errexit; set -o pipefail; set -o nounset;
/etc/eks/bootstrap.sh test-cluster --kubelet-extra-args '--node-labels=node-role.undistro.io/infra=true --register-with-taints=dedicated=infra:NoSchedule'
`),
},
Expand All @@ -74,6 +76,7 @@ func TestNewNode(t *testing.T) {
},
},
expectedBytes: []byte(`#!/bin/bash
set -o errexit; set -o pipefail; set -o nounset;
/etc/eks/bootstrap.sh test-cluster --container-runtime containerd
`),
},
Expand All @@ -90,6 +93,7 @@ func TestNewNode(t *testing.T) {
},
},
expectedBytes: []byte(`#!/bin/bash
set -o errexit; set -o pipefail; set -o nounset;
/etc/eks/bootstrap.sh test-cluster --kubelet-extra-args '--node-labels=node-role.undistro.io/infra=true --register-with-taints=dedicated=infra:NoSchedule' --container-runtime containerd
`),
},
Expand All @@ -103,6 +107,7 @@ func TestNewNode(t *testing.T) {
},
},
expectedBytes: []byte(`#!/bin/bash
set -o errexit; set -o pipefail; set -o nounset;
/etc/eks/bootstrap.sh test-cluster --ip-family ipv6 --service-ipv6-cidr fe80:0000:0000:0000:0204:61ff:fe9d:f156/24
`),
},
Expand All @@ -115,6 +120,7 @@ func TestNewNode(t *testing.T) {
},
},
expectedBytes: []byte(`#!/bin/bash
set -o errexit; set -o pipefail; set -o nounset;
/etc/eks/bootstrap.sh test-cluster --use-max-pods false
`),
},
Expand All @@ -127,6 +133,7 @@ func TestNewNode(t *testing.T) {
},
},
expectedBytes: []byte(`#!/bin/bash
set -o errexit; set -o pipefail; set -o nounset;
/etc/eks/bootstrap.sh test-cluster --aws-api-retry-attempts 5
`),
},
Expand All @@ -140,6 +147,7 @@ func TestNewNode(t *testing.T) {
},
},
expectedBytes: []byte(`#!/bin/bash
set -o errexit; set -o pipefail; set -o nounset;
/etc/eks/bootstrap.sh test-cluster --pause-container-account 12345678 --pause-container-version v1
`),
},
Expand All @@ -152,6 +160,7 @@ func TestNewNode(t *testing.T) {
},
},
expectedBytes: []byte(`#!/bin/bash
set -o errexit; set -o pipefail; set -o nounset;
/etc/eks/bootstrap.sh test-cluster --dns-cluster-ip 192.168.0.1
`),
},
Expand All @@ -164,7 +173,67 @@ func TestNewNode(t *testing.T) {
},
},
expectedBytes: []byte(`#!/bin/bash
set -o errexit; set -o pipefail; set -o nounset;
/etc/eks/bootstrap.sh test-cluster --docker-config-json '{"debug":true}'
`),
},
{
name: "with pre-bootstrap command",
args: args{
input: &NodeInput{
ClusterName: "test-cluster",
PreBootstrapCommands: []string{"date", "echo \"testing\""},
},
},
expectedBytes: []byte(`#!/bin/bash
set -o errexit; set -o pipefail; set -o nounset;
date
echo "testing"
/etc/eks/bootstrap.sh test-cluster
`),
},
{
name: "with post-bootstrap command",
args: args{
input: &NodeInput{
ClusterName: "test-cluster",
PostBootstrapCommands: []string{"date", "echo \"testing\""},
},
},
expectedBytes: []byte(`#!/bin/bash
set -o errexit; set -o pipefail; set -o nounset;
/etc/eks/bootstrap.sh test-cluster
date
echo "testing"
`),
},
{
name: "with pre & post-bootstrap command",
args: args{
input: &NodeInput{
ClusterName: "test-cluster",
PreBootstrapCommands: []string{"echo \"testing pre\""},
PostBootstrapCommands: []string{"echo \"testing post\""},
},
},
expectedBytes: []byte(`#!/bin/bash
set -o errexit; set -o pipefail; set -o nounset;
echo "testing pre"
/etc/eks/bootstrap.sh test-cluster
echo "testing post"
`),
},
{
name: "with bootstrap override command",
args: args{
input: &NodeInput{
ClusterName: "test-cluster",
BootstrapCommandOverride: pointer.String("/custom/mybootstrap.sh"),
},
},
expectedBytes: []byte(`#!/bin/bash
set -o errexit; set -o pipefail; set -o nounset;
/custom/mybootstrap.sh test-cluster
`),
},
}
Expand Down
16 changes: 16 additions & 0 deletions config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,10 @@ spec:
description: APIRetryAttempts is the number of retry attempts for
AWS API call.
type: integer
boostrapCommandOverride:
description: BootstrapCommandOverride allows you to override the bootstrap
command to use for EKS nodes.
type: string
containerRuntime:
description: ContainerRuntime specify the container runtime to use
when bootstrapping EKS.
Expand Down Expand Up @@ -302,6 +306,18 @@ spec:
- accountNumber
- version
type: object
postBootstrapCommands:
description: PostBootstrapCommands specifies extra commands to run
after bootstrapping nodes to the cluster
items:
type: string
type: array
preBootstrapCommands:
description: PreBootstrapCommands specifies extra commands to run
before bootstrapping nodes to the cluster
items:
type: string
type: array
useMaxPods:
description: UseMaxPods sets --max-pods for the kubelet when true.
type: boolean
Expand Down
16 changes: 16 additions & 0 deletions config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigtemplates.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,10 @@ spec:
description: APIRetryAttempts is the number of retry attempts
for AWS API call.
type: integer
boostrapCommandOverride:
description: BootstrapCommandOverride allows you to override
the bootstrap command to use for EKS nodes.
type: string
containerRuntime:
description: ContainerRuntime specify the container runtime
to use when bootstrapping EKS.
Expand Down Expand Up @@ -168,6 +172,18 @@ spec:
- accountNumber
- version
type: object
postBootstrapCommands:
description: PostBootstrapCommands specifies extra commands
to run after bootstrapping nodes to the cluster
items:
type: string
type: array
preBootstrapCommands:
description: PreBootstrapCommands specifies extra commands
to run before bootstrapping nodes to the cluster
items:
type: string
type: array
useMaxPods:
description: UseMaxPods sets --max-pods for the kubelet when
true.
Expand Down