From 3d215b3cef92ff7d1b450a70c04db80cc5ea4db2 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 20 Jul 2020 12:56:21 +0200 Subject: [PATCH 1/5] Increate leader election lease time for vsphere The machine-api-controller components are refreshing their lease more than all other components combined. Bringing this to 90s each, will decrease etcd writes at idle. --- cmd/vsphere/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/vsphere/main.go b/cmd/vsphere/main.go index 5a9758739b..89d3e767f6 100644 --- a/cmd/vsphere/main.go +++ b/cmd/vsphere/main.go @@ -45,7 +45,7 @@ func main() { leaderElectLeaseDuration := flag.Duration( "leader-elect-lease-duration", - 15*time.Second, + 90*time.Second, "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", ) From ef1c68b7482e47e94593494e9c62f31a42a14512 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 20 Jul 2020 12:57:14 +0200 Subject: [PATCH 2/5] Increate leader election lease time for nodelink The machine-api-controller components are refreshing their lease more than all other components combined. Bringing this to 90s each, will decrease etcd writes at idle. --- cmd/nodelink-controller/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/nodelink-controller/main.go b/cmd/nodelink-controller/main.go index 73bae14dac..99572c5a54 100644 --- a/cmd/nodelink-controller/main.go +++ b/cmd/nodelink-controller/main.go @@ -44,7 +44,7 @@ func main() { leaderElectLeaseDuration := flag.Duration( "leader-elect-lease-duration", - 15*time.Second, + 90*time.Second, "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", ) From aa7bd363b719c3f1f8213a57cd0277638b15f444 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 20 Jul 2020 12:57:45 +0200 Subject: [PATCH 3/5] Increate leader election lease time for machineset The machine-api-controller components are refreshing their lease more than all other components combined. Bringing this to 90s each, will decrease etcd writes at idle. --- cmd/machineset/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/machineset/main.go b/cmd/machineset/main.go index 7ed5dddbee..109b306bbb 100644 --- a/cmd/machineset/main.go +++ b/cmd/machineset/main.go @@ -75,7 +75,7 @@ func main() { leaderElectLeaseDuration := flag.Duration( "leader-elect-lease-duration", - 15*time.Second, + 90*time.Second, "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", ) From f4f091cd3764664732a2285178113b58a5d9e465 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 20 Jul 2020 12:58:11 +0200 Subject: [PATCH 4/5] Increate leader election lease time for machine-healthcheck The machine-api-controller components are refreshing their lease more than all other components combined. Bringing this to 90s each, will decrease etcd writes at idle. --- cmd/machine-healthcheck/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/machine-healthcheck/main.go b/cmd/machine-healthcheck/main.go index c42467dbb7..812f8b61fb 100644 --- a/cmd/machine-healthcheck/main.go +++ b/cmd/machine-healthcheck/main.go @@ -59,7 +59,7 @@ func main() { leaderElectLeaseDuration := flag.Duration( "leader-elect-lease-duration", - 15*time.Second, + 90*time.Second, "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", ) From 761bcd55511284668c11bb43808b27169da337ad Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Mon, 20 Jul 2020 13:18:22 +0200 Subject: [PATCH 5/5] Enforce default leader elect lease duration to 90s for all controllers Enforce this value in MAO deployment will ensure the value won't be overritten by changed CLI defaults in the future. --- pkg/operator/sync.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/operator/sync.go b/pkg/operator/sync.go index 7b7c48d537..7c2b51d742 100644 --- a/pkg/operator/sync.go +++ b/pkg/operator/sync.go @@ -423,6 +423,7 @@ func newContainers(config *OperatorConfig, features map[string]bool) []corev1.Co "--logtostderr=true", "--v=3", "--leader-elect=true", + "--leader-elect-lease-duration=90s", fmt.Sprintf("--namespace=%s", config.TargetNamespace), }