Skip to content
This repository was archived by the owner on Jan 9, 2023. It is now read-only.

Commit 9b9dc89

Browse files
author
Luke Addison
committed
Increase kubelet resource reservations
1 parent 1f97e08 commit 9b9dc89

File tree

5 files changed

+113
-7
lines changed

5 files changed

+113
-7
lines changed

‎puppet/modules/kubernetes/manifests/kubelet.pp‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,21 @@
1010
String $role = 'worker',
1111
String $container_runtime = 'docker',
1212
String $kubelet_dir = '/var/lib/kubelet',
13-
String $hard_eviction_memory_threshold =
14-
five_percent_of_total_ram(dig44($facts, ['memory', 'system', 'total_bytes'], 1)),
13+
Optional[String] $eviction_hard_memory_available_threshold = '5%',
14+
Optional[String] $eviction_hard_nodefs_available_threshold = '10%',
15+
Optional[String] $eviction_hard_nodefs_inodes_free_threshold = '5%',
16+
Boolean $eviction_soft_enabled = true,
17+
Optional[String] $eviction_soft_memory_available_threshold = '10%',
18+
Optional[String] $eviction_soft_nodefs_available_threshold = '15%',
19+
Optional[String] $eviction_soft_nodefs_inodes_free_threshold = '10%',
20+
Optional[String] $eviction_soft_memory_available_grace_period = '0m',
21+
Optional[String] $eviction_soft_nodefs_available_grace_period = '0m',
22+
Optional[String] $eviction_soft_nodefs_inodes_free_grace_period = '0m',
23+
String $eviction_max_pod_grace_period = '-1',
24+
String $eviction_pressure_transition_period = '2m',
25+
Optional[String] $eviction_minimum_reclaim_memory_available = '100Mi',
26+
Optional[String] $eviction_minimum_reclaim_nodefs_available = '1Gi',
27+
Optional[String] $eviction_minimum_reclaim_nodefs_inodes_free = undef,
1528
Optional[String] $network_plugin = undef,
1629
Integer $network_plugin_mtu = 1460,
1730
Boolean $allow_privileged = true,
@@ -26,7 +39,7 @@
2639
$pod_cidr = undef,
2740
$hostname_override = undef,
2841
Enum['systemd', 'cgroupfs'] $cgroup_driver = $::osfamily ? {
29-
'RedHat' => 'systemd',
42+
'RedHat' => 'cgroupfs',
3043
default => 'cgroupfs',
3144
},
3245
String $cgroup_root = '/',
@@ -43,6 +56,30 @@
4356
){
4457
require ::kubernetes
4558

59+
if ! $eviction_soft_memory_available_threshold or ! $eviction_soft_memory_available_grace_period {
60+
$_eviction_soft_memory_available_threshold = undef
61+
$_eviction_soft_memory_available_grace_period = undef
62+
} else {
63+
$_eviction_soft_memory_available_threshold = $eviction_soft_memory_available_threshold
64+
$_eviction_soft_memory_available_grace_period = $eviction_soft_memory_available_grace_period
65+
}
66+
67+
if ! $eviction_soft_nodefs_available_threshold or ! $eviction_soft_nodefs_available_grace_period {
68+
$_eviction_soft_nodefs_available_threshold = undef
69+
$_eviction_soft_nodefs_available_grace_period = undef
70+
} else {
71+
$_eviction_soft_nodefs_available_threshold = $eviction_soft_nodefs_available_threshold
72+
$_eviction_soft_nodefs_available_grace_period = $eviction_soft_nodefs_available_grace_period
73+
}
74+
75+
if ! $eviction_soft_nodefs_inodes_free_threshold or ! $eviction_soft_nodefs_inodes_free_grace_period {
76+
$_eviction_soft_nodefs_inodes_free_threshold = undef
77+
$_eviction_soft_nodefs_inodes_free_grace_period = undef
78+
} else {
79+
$_eviction_soft_nodefs_inodes_free_threshold = $eviction_soft_nodefs_inodes_free_threshold
80+
$_eviction_soft_nodefs_inodes_free_grace_period = $eviction_soft_nodefs_inodes_free_grace_period
81+
}
82+
4683
$_systemd_wants = $systemd_wants
4784
if $container_runtime == 'docker' {
4885
$_systemd_after = ['docker.service'] + $systemd_after

‎puppet/modules/kubernetes/spec/classes/kubelet_spec.rb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
should_not contain_file(service_file).with_content(/--network-plugin/)
2121
should contain_file(service_file).with_content(/--container-runtime=docker/)
2222
should contain_file(service_file).with_content(%r{--kubeconfig=/etc/kubernetes/kubeconfig-kubelet})
23-
should contain_file(service_file).with_content(%r{--eviction-hard=memory.available<191Mi})
23+
should contain_file(service_file).with_content(%r{--eviction-hard=memory.available<5%})
2424
end
2525
end
2626

‎puppet/modules/kubernetes/templates/kubelet.service.erb‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ ExecStart=<%= scope['kubernetes::_dest_dir'] %>/kubelet \
6868
<% if not @kernelversion.nil? and scope.function_versioncmp([scope['kubernetes::version'], '1.6.0']) >= 0 and scope.function_versioncmp([scope['kubernetes::version'], '1.7.0']) < 0 and scope.function_versioncmp([@kernelversion, '4.9']) >= 0 -%>
6969
--cgroups-per-qos=false \
7070
--enforce-node-allocatable= \
71+
<% else -%>
72+
--enforce-node-allocatable=pods \
7173
<% end -%>
7274
<% if scope.function_versioncmp([scope['kubernetes::version'], '1.6.0']) >= 0 -%>
7375
--cgroup-driver=<%= @cgroup_driver %> \
@@ -103,7 +105,53 @@ ExecStart=<%= scope['kubernetes::_dest_dir'] %>/kubelet \
103105
"--tls-cert-file=<%= @cert_file %>" \
104106
"--tls-private-key-file=<%= @key_file %>" \
105107
<% end -%>
106-
"--eviction-hard=memory.available<<%= @hard_eviction_memory_threshold %>" \
108+
<%
109+
# build eviction hard command line
110+
@eviction_hard = []
111+
@eviction_hard << "memory.available<#{@eviction_hard_memory_available_threshold}" unless @eviction_hard_memory_available_threshold.nil? or @eviction_hard_memory_available_threshold == 'nil'
112+
@eviction_hard << "nodefs.available<#{@eviction_hard_nodefs_available_threshold}" unless @eviction_hard_nodefs_available_threshold.nil? or @eviction_hard_nodefs_available_threshold == 'nil'
113+
@eviction_hard << "nodefs.inodesFree<#{@eviction_hard_nodefs_inodes_free_threshold}" unless @eviction_hard_nodefs_inodes_free_threshold.nil? or @eviction_hard_nodefs_inodes_free_threshold == 'nil'
114+
if @eviction_hard.length > 0
115+
-%>
116+
"--eviction-hard=<%= @eviction_hard.join(',') %>" \
117+
<% end -%>
118+
<% if @eviction_soft_enabled -%>
119+
<%
120+
# build eviction soft command line
121+
@eviction_soft = []
122+
123+
@eviction_soft << "memory.available<#{@_eviction_soft_memory_available_threshold}" unless @_eviction_soft_memory_available_threshold.nil? or @_eviction_soft_memory_available_threshold == 'nil'
124+
@eviction_soft << "nodefs.available<#{@_eviction_soft_nodefs_available_threshold}" unless @_eviction_soft_nodefs_available_threshold.nil? or @_eviction_soft_nodefs_available_threshold == 'nil'
125+
@eviction_soft << "nodefs.inodesFree<#{@_eviction_soft_nodefs_inodes_free_threshold}" unless @_eviction_soft_nodefs_inodes_free_threshold.nil? or @_eviction_soft_nodefs_inodes_free_threshold == 'nil'
126+
if @eviction_soft.length > 0
127+
-%>
128+
<%
129+
# build eviction soft grace period command line
130+
@eviction_soft_grace_period = []
131+
132+
@eviction_soft_grace_period << "memory.available=#{@_eviction_soft_memory_available_grace_period}" unless @_eviction_soft_memory_available_grace_period.nil? or @_eviction_soft_memory_available_grace_period == 'nil'
133+
@eviction_soft_grace_period << "nodefs.available=#{@_eviction_soft_nodefs_available_grace_period}" unless @_eviction_soft_nodefs_available_grace_period.nil? or @_eviction_soft_nodefs_available_grace_period == 'nil'
134+
@eviction_soft_grace_period << "nodefs.inodesFree=#{@_eviction_soft_nodefs_inodes_free_grace_period}" unless @_eviction_soft_nodefs_inodes_free_grace_period.nil? or @_eviction_soft_nodefs_inodes_free_grace_period == 'nil'
135+
if @eviction_soft_grace_period.length > 0
136+
-%>
137+
--eviction-soft=<%= @eviction_soft.join(',') %> \
138+
--eviction-soft-grace-period=<%= @eviction_soft_grace_period.join(',') %> \
139+
--eviction-max-pod-grace-period=<%= @eviction_max_pod_grace_period %> \
140+
--eviction-pressure-transition-period=<%= @eviction_pressure_transition_period %> \
141+
<% end -%>
142+
<% end -%>
143+
<% end -%>
144+
<%
145+
# build minumum reclaim command line
146+
@eviction_minimum_reclaim = []
147+
148+
@eviction_minimum_reclaim << "memory.available=#{@eviction_minimum_reclaim_memory_available}" unless @eviction_minimum_reclaim_memory_available.nil? or @eviction_minimum_reclaim_memory_available == 'nil'
149+
@eviction_minimum_reclaim << "nodefs.available=#{@eviction_minimum_reclaim_nodefs_available}" unless @eviction_minimum_reclaim_nodefs_available.nil? or @eviction_minimum_reclaim_nodefs_available == 'nil'
150+
@eviction_minimum_reclaim << "nodefs.inodesFree=#{@eviction_minimum_reclaim_nodefs_inodes_free}" unless @eviction_minimum_reclaim_nodefs_inodes_free.nil? or @eviction_minimum_reclaim_nodefs_inodes_free == 'nil'
151+
if @eviction_minimum_reclaim.length > 0
152+
-%>
153+
"--eviction-minimum-reclaim=<%= @eviction_minimum_reclaim.join(',') %>" \
154+
<% end -%>
107155
--logtostderr=true
108156

109157
Restart=on-failure

‎puppet/modules/site_module/files/20-cgroupfs.conf‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
[Service]
2+
ExecStart=
3+
ExecStart=/usr/bin/dockerd-current \
4+
--add-runtime docker-runc=/usr/libexec/docker/docker-runc-current \
5+
--default-runtime=docker-runc \
6+
--exec-opt native.cgroupdriver=cgroupfs \
7+
--userland-proxy-path=/usr/libexec/docker/docker-proxy-current \
8+
--init-path=/usr/libexec/docker/docker-init-current \
9+
--seccomp-profile=/etc/docker/seccomp.json \
10+
$OPTIONS \
11+
$DOCKER_STORAGE_OPTIONS \
12+
$DOCKER_NETWORK_OPTIONS \
13+
$ADD_REGISTRY \
14+
$BLOCK_REGISTRY \
15+
$INSECURE_REGISTRY \
16+
$REGISTRIES

‎puppet/modules/site_module/manifests/docker_config.pp‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,15 @@
33
ensure => file,
44
content => template('site_module/docker.erb'),
55
}
6+
67
file { '/etc/systemd/system/docker.service.d':
78
ensure => directory,
89
} -> file { '/etc/systemd/system/docker.service.d/10-slice.conf':
9-
ensure => directory,
10-
content => '[Service]\nSlice=podruntime.slice\n',
10+
ensure => file,
11+
content => "[Service]\nSlice=podruntime.slice\n",
12+
} -> file { '/etc/systemd/system/docker.service.d/20-cgroupfs.conf':
13+
ensure => file,
14+
content => file('site_module/20-cgroupfs.conf'),
1115
}
16+
1217
}

0 commit comments

Comments
 (0)