From 2749415be3185e0a0d208768b96016b6a90f1800 Mon Sep 17 00:00:00 2001
From: Luke Addison <luke.addison@jetstack.io>
Date: Fri, 8 Jun 2018 20:42:23 +0100
Subject: [PATCH] Run kube services under /system.slice

---
 puppet/modules/kubernetes/manifests/kubelet.pp  |  2 +-
 .../kubernetes/spec/classes/kubelet_spec.rb     |  4 ++++
 .../templates/kube-apiserver.service.erb        |  7 ++++++-
 .../kube-controller-manager.service.erb         |  7 ++++++-
 .../kubernetes/templates/kube-proxy.service.erb | 12 +++++++++++-
 .../templates/kube-scheduler.service.erb        |  7 ++++++-
 .../kubernetes/templates/kubelet.service.erb    | 13 +++++++++----
 .../site_module/manifests/docker_config.pp      | 17 ++++++++++++-----
 8 files changed, 55 insertions(+), 14 deletions(-)

diff --git a/puppet/modules/kubernetes/manifests/kubelet.pp b/puppet/modules/kubernetes/manifests/kubelet.pp
index 6f4f1ebbfb..61d5446205 100644
--- a/puppet/modules/kubernetes/manifests/kubelet.pp
+++ b/puppet/modules/kubernetes/manifests/kubelet.pp
@@ -30,7 +30,7 @@
     default  => 'cgroupfs',
   },
   String $cgroup_root = '/',
-  Optional[String] $cgroup_kube_name = '/podruntime.slice',
+  Optional[String] $cgroup_kube_name = undef,
   Optional[String] $cgroup_kube_reserved_memory = '256Mi',
   Optional[String] $cgroup_kube_reserved_cpu = '10m',
   Optional[String] $cgroup_system_name = '/system.slice',
diff --git a/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb b/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb
index 7bd1d51e30..ee2d7150e3 100644
--- a/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb
+++ b/puppet/modules/kubernetes/spec/classes/kubelet_spec.rb
@@ -211,6 +211,7 @@
 
       context 'with both cpu and memory a supplied' do
         let(:params) { {
+          "cgroup_kube_name" => "/podruntime.slice",
           "cgroup_#{cgroup_type}_reserved_cpu"    => '100m',
           "cgroup_#{cgroup_type}_reserved_memory" => '128Mi',
         }}
@@ -221,6 +222,7 @@
 
       context 'with only cpu supplied' do
         let(:params) { {
+          "cgroup_kube_name" => "/podruntime.slice",
           "cgroup_#{cgroup_type}_reserved_cpu"    => '100m',
           "cgroup_#{cgroup_type}_reserved_memory" => nil,
         }}
@@ -231,6 +233,7 @@
 
       context 'with only memory supplied' do
         let(:params) { {
+          "cgroup_kube_name" => "/podruntime.slice",
           "cgroup_#{cgroup_type}_reserved_cpu"    => nil,
           "cgroup_#{cgroup_type}_reserved_memory" => '128Mi',
         }}
@@ -241,6 +244,7 @@
 
       context 'with nothing supplied' do
         let(:params) { {
+          "cgroup_kube_name" => "/podruntime.slice",
           "cgroup_#{cgroup_type}_reserved_cpu"    => nil,
           "cgroup_#{cgroup_type}_reserved_memory" => nil,
         }}
diff --git a/puppet/modules/kubernetes/templates/kube-apiserver.service.erb b/puppet/modules/kubernetes/templates/kube-apiserver.service.erb
index 3c327836c4..c45450a125 100644
--- a/puppet/modules/kubernetes/templates/kube-apiserver.service.erb
+++ b/puppet/modules/kubernetes/templates/kube-apiserver.service.erb
@@ -4,7 +4,12 @@ Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 <%= scope.function_template(['kubernetes/_systemd_unit.erb']) %>
 
 [Service]
-Slice=podruntime.slice
+<%
+  if scope['kubernetes::kubelet::cgroup_kube_name']
+  @cgroup_kube_basename = scope.call_function('regsubst', [scope['kubernetes::kubelet::cgroup_kube_name'], '^\/', ''])
+-%>
+  Slice=<%= @cgroup_kube_basename %>
+<% end -%>
 User=<%= scope['kubernetes::user'] %>
 Group=<%= scope['kubernetes::group'] %>
 <%- if scope['kubernetes::_service_account_key_file'] and scope['kubernetes::service_account_key_generate'] -%>
diff --git a/puppet/modules/kubernetes/templates/kube-controller-manager.service.erb b/puppet/modules/kubernetes/templates/kube-controller-manager.service.erb
index cd735fc5b4..7dcaad2ea6 100644
--- a/puppet/modules/kubernetes/templates/kube-controller-manager.service.erb
+++ b/puppet/modules/kubernetes/templates/kube-controller-manager.service.erb
@@ -4,7 +4,12 @@ Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 <%= scope.function_template(['kubernetes/_systemd_unit.erb']) %>
 
 [Service]
-Slice=podruntime.slice
+<%
+  if scope['kubernetes::kubelet::cgroup_kube_name']
+  @cgroup_kube_basename = scope.call_function('regsubst', [scope['kubernetes::kubelet::cgroup_kube_name'], '^\/', ''])
+-%>
+  Slice=<%= @cgroup_kube_basename %>
+<% end -%>
 User=<%= scope['kubernetes::user'] %>
 Group=<%= scope['kubernetes::group'] %>
 <%- if scope['kubernetes::_service_account_key_file'] and scope['kubernetes::service_account_key_generate'] -%>
diff --git a/puppet/modules/kubernetes/templates/kube-proxy.service.erb b/puppet/modules/kubernetes/templates/kube-proxy.service.erb
index a114a7fa6a..d42fb6c811 100644
--- a/puppet/modules/kubernetes/templates/kube-proxy.service.erb
+++ b/puppet/modules/kubernetes/templates/kube-proxy.service.erb
@@ -10,7 +10,17 @@ ExecStartPre=/sbin/sysctl -w net.bridge.bridge-nf-call-ip6tables=1
 ExecStart=<%= scope['kubernetes::_dest_dir'] %>/proxy \
   --v=<%= scope['kubernetes::log_level'] %> \
   --cluster-cidr=<%= scope['kubernetes::pod_network'] %> \
-  --resource-container=podruntime.slice \
+<%
+  if scope['kubernetes::kubelet::cgroup_kube_name']
+  @cgroup_kube_basename = scope.call_function('regsubst', [scope['kubernetes::kubelet::cgroup_kube_name'], '^\/', ''])
+-%>
+  --resource-container=<%= @cgroup_kube_basename %> \
+<%
+  elsif scope['kubernetes::kubelet::cgroup_system_name']
+  @cgroup_system_basename = scope.call_function('regsubst', [scope['kubernetes::kubelet::cgroup_system_name'], '^\/', ''])
+-%>
+  --resource-container=<%= @cgroup_system_basename %> \
+<% end -%>
 <% if @kubeconfig_path -%>
   --kubeconfig=<%= @kubeconfig_path %> \
 <% end -%>
diff --git a/puppet/modules/kubernetes/templates/kube-scheduler.service.erb b/puppet/modules/kubernetes/templates/kube-scheduler.service.erb
index 98e82302d9..108f255722 100644
--- a/puppet/modules/kubernetes/templates/kube-scheduler.service.erb
+++ b/puppet/modules/kubernetes/templates/kube-scheduler.service.erb
@@ -4,7 +4,12 @@ Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 <%= scope.function_template(['kubernetes/_systemd_unit.erb']) %>
 
 [Service]
-Slice=podruntime.slice
+<%
+  if scope['kubernetes::kubelet::cgroup_kube_name']
+  @cgroup_kube_basename = scope.call_function('regsubst', [scope['kubernetes::kubelet::cgroup_kube_name'], '^\/', ''])
+-%>
+  Slice=<%= @cgroup_kube_basename %>
+<% end -%>
 User=<%= scope['kubernetes::user'] %>
 Group=<%= scope['kubernetes::group'] %>
 ExecStart=<%= scope['kubernetes::_dest_dir'] %>/scheduler \
diff --git a/puppet/modules/kubernetes/templates/kubelet.service.erb b/puppet/modules/kubernetes/templates/kubelet.service.erb
index 6d12406db8..d3f71b2cb4 100644
--- a/puppet/modules/kubernetes/templates/kubelet.service.erb
+++ b/puppet/modules/kubernetes/templates/kubelet.service.erb
@@ -4,7 +4,12 @@ Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 <%= scope.function_template(['kubernetes/_systemd_unit.erb']) %>
 
 [Service]
-Slice=podruntime.slice
+<%
+  if @cgroup_kube_name
+  @cgroup_kube_basename = scope.call_function('regsubst', [@cgroup_kube_name, '^\/', ''])
+-%>
+Slice=<%= @cgroup_kube_basename %>
+<% end -%>
 WorkingDirectory=<%= @kubelet_dir %>
 <% if @cloud_provider == 'aws' -%>
 # prevent metadata service access on AWS
@@ -73,9 +78,6 @@ ExecStart=<%= scope['kubernetes::_dest_dir'] %>/kubelet \
   --cgroup-driver=<%= @cgroup_driver %> \
   --cgroup-root=<%= @cgroup_root %> \
 <% if @cgroup_kube_name -%>
-  --kube-reserved-cgroup=<%= @cgroup_kube_name %> \
-  --runtime-cgroups=<%= @cgroup_kube_name %> \
-  --kubelet-cgroups=<%= @cgroup_kube_name %> \
 <%
     # build kube reserved command line
     @cgroup_kube_reserved = []
@@ -83,6 +85,9 @@ ExecStart=<%= scope['kubernetes::_dest_dir'] %>/kubelet \
     @cgroup_kube_reserved << "memory=#{@cgroup_kube_reserved_memory}" unless @cgroup_kube_reserved_memory.nil? or @cgroup_kube_reserved_memory == 'nil'
     if @cgroup_kube_reserved.length > 0
 -%>
+  --kube-reserved-cgroup=<%= @cgroup_kube_name %> \
+  --runtime-cgroups=<%= @cgroup_kube_name %> \
+  --kubelet-cgroups=<%= @cgroup_kube_name %> \
   "--kube-reserved=<%= @cgroup_kube_reserved.join(',') %>" \
 <% end -%>
 <% end -%>
diff --git a/puppet/modules/site_module/manifests/docker_config.pp b/puppet/modules/site_module/manifests/docker_config.pp
index ff285d2a50..58f2568d66 100644
--- a/puppet/modules/site_module/manifests/docker_config.pp
+++ b/puppet/modules/site_module/manifests/docker_config.pp
@@ -3,10 +3,17 @@
     ensure  => file,
     content => template('site_module/docker.erb'),
   }
-  file { '/etc/systemd/system/docker.service.d':
-    ensure  => directory,
-  } -> file { '/etc/systemd/system/docker.service.d/10-slice.conf':
-    ensure  => directory,
-    content => '[Service]\nSlice=podruntime.slice\n',
+
+  if $kubernetes::kubelet::cgroup_kube_name {
+
+    $cgroup_kube_basename = regsubst( $kubernetes::kubelet::cgroup_kube_name, '^\/', '')
+
+    file { '/etc/systemd/system/docker.service.d':
+      ensure  => directory,
+    } -> file { '/etc/systemd/system/docker.service.d/10-slice.conf':
+      ensure  => directory,
+      content => "[Service]\nSlice=${cgroup_kube_basename}\n",
+    }
+
   }
 }