s/kube_node_info/kube_node_role

Use kube_node_role{role="control-plane"} (see [1] and [2]) to estimate if the cluster is HA or not.
* [1]: https://github.com/search?q=repo%3Akubernetes%2Fkube-state-metrics%20kube_node_role&type=code
* [2]: https://kubernetes.io/docs/reference/labels-annotations-taints/#node-role-kubernetes-io-control-plane

Also drop any thresholds as they would lead to false positives.

Author: rexagod

alerts/resource_alerts.libsonnet

@@ -37,9 +37,9 @@ local utils = import '../lib/utils.libsonnet';
           if $._config.showMultiCluster then {
             expr: |||
               (sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) -
-              0.85 * sum(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s) > 0
+              sum(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s) > 0
               and
-              count by (cluster) (max by (cluster, node) (kube_node_info)) == 1)
+              count by (cluster) (max by (cluster, node) (kube_node_role{role="control-plane"})) < 3)
               or
               (sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) -
               (sum(kube_node_status_allocatable{%(kubeStateMetricsSelector)s,resource="cpu"}) by (%(clusterLabel)s) -
@@ -54,9 +54,9 @@ local utils = import '../lib/utils.libsonnet';
           } else {
             expr: |||
               (sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) -
-              0.85 * sum(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s}) > 0
+              sum(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s}) > 0
               and
-              count(max by (node) (kube_node_info)) == 1)
+              count(max by (node) (kube_node_role{role="control-plane"})) < 3)
               or
               (sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) -
               (sum(kube_node_status_allocatable{resource="cpu", %(kubeStateMetricsSelector)s}) -
@@ -82,9 +82,9 @@ local utils = import '../lib/utils.libsonnet';
           if $._config.showMultiCluster then {
             expr: |||
               (sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) -
-              0.85 * sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s) > 0
+              sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s) > 0
               and
-              count by (cluster) (max by (cluster, node) (kube_node_info)) == 1)
+              count by (cluster) (max by (cluster, node) (kube_node_role{role="control-plane"})) < 3)
               or
               (sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) by (%(clusterLabel)s) -
               (sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) by (%(clusterLabel)s) -
@@ -99,9 +99,9 @@ local utils = import '../lib/utils.libsonnet';
           } else {
             expr: |||
               (sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) -
-              0.85 * sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) > 0
+              sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) > 0
               and
-              count(max by (node) (kube_node_info)) == 1)
+              count(max by (node) (kube_node_role{role="control-plane"})) < 3)
               or
               (sum(namespace_memory:kube_pod_container_resource_requests:sum{%(ignoringOverprovisionedWorkloadSelector)s}) -
               (sum(kube_node_status_allocatable{resource="memory", %(kubeStateMetricsSelector)s}) -

tests/tests.yaml

@@ -1425,15 +1425,15 @@ tests:
         summary: "StatefulSet has not matched the expected number of replicas."
 
 - name: KubeCPUOvercommit alert (single-node)
-- interval: 1m
+  interval: 1m
   input_series:
   - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="kubernetes", namespace="default"}'
     values: '1x10'
   - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="kubernetes", namespace="kube-system"}'
     values: '1x10'
   - series: 'kube_node_status_allocatable{cluster="kubernetes", node="n1", resource="cpu", job="kube-state-metrics"}'
     values: '1.9x10' # This value was seen on a 2x vCPU node
-  - series: 'kube_node_info{cluster="kubernetes", node="n1", job="kube-state-metrics"}'
+  - series: 'kube_node_role{ cluster="kubernetes", node="n1", role="control-plane"}'
     values: '1x10'
   alert_rule_test:
   - eval_time: 9m
@@ -1444,12 +1444,12 @@ tests:
     - exp_labels:
         severity: warning
       exp_annotations:
-        description: Cluster has overcommitted CPU resource requests for Pods by 0.385 CPU shares and cannot tolerate node failure.
+        description: Cluster has overcommitted CPU resource requests for Pods by 0.10000000000000009 CPU shares and cannot tolerate node failure.
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit
         summary: Cluster has overcommitted CPU resource requests.
 
 - name: KubeCPUOvercommit alert (multi-node)
-- interval: 1m
+  interval: 1m
   input_series:
   - series: 'namespace_cpu:kube_pod_container_resource_requests:sum{cluster="kubernetes", namespace="default"}'
     values: '2x10'
@@ -1459,9 +1459,9 @@ tests:
     values: '1.9x10' # This value was seen on a 2x vCPU node
   - series: 'kube_node_status_allocatable{cluster="kubernetes", node="n2", resource="cpu", job="kube-state-metrics"}'
     values: '1.9x10'
-  - series: 'kube_node_info{cluster="kubernetes", node="n1", job="kube-state-metrics"}'
+  - series: 'kube_node_role{ cluster="kubernetes", node="n1", role="control-plane"}'
     values: '1x10'
-  - series: 'kube_node_info{cluster="kubernetes", node="n2", job="kube-state-metrics"}'
+  - series: 'kube_node_role{ cluster="kubernetes", node="n2", role="control-plane"}'
     values: '1x10'
   alert_rule_test:
   - eval_time: 9m
@@ -1472,20 +1472,20 @@ tests:
     - exp_labels:
         severity: warning
       exp_annotations:
-        description: Cluster has overcommitted CPU resource requests for Pods by 2.1 CPU shares and cannot tolerate node failure.
+        description: Cluster has overcommitted CPU resource requests for Pods by 0.20000000000000018 CPU shares and cannot tolerate node failure.
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit
         summary: Cluster has overcommitted CPU resource requests.
 
 - name: KubeMemoryOvercommit alert (single-node)
-- interval: 1m
+  interval: 1m
   input_series:
   - series: 'namespace_memory:kube_pod_container_resource_requests:sum{cluster="kubernetes", namespace="default"}'
     values: '1000000000x10' # 1 GB
   - series: 'namespace_memory:kube_pod_container_resource_requests:sum{cluster="kubernetes", namespace="kube-system"}'
     values: '1000000000x10'
   - series: 'kube_node_status_allocatable{cluster="kubernetes", node="n1", resource="memory", job="kube-state-metrics"}'
     values: '1000000000x10'
-  - series: 'kube_node_info{cluster="kubernetes", node="n1", job="kube-state-metrics"}'
+  - series: 'kube_node_role{cluster="kubernetes", node="n1", role="control-plane"}'
     values: '1x10'
   alert_rule_test:
   - eval_time: 9m
@@ -1496,12 +1496,12 @@ tests:
     - exp_labels:
         severity: warning
       exp_annotations:
-        description: Cluster has overcommitted memory resource requests for Pods by 1.15G bytes and cannot tolerate node failure.
+        description: Cluster has overcommitted memory resource requests for Pods by 1G bytes and cannot tolerate node failure.
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryovercommit
         summary: Cluster has overcommitted memory resource requests.
 
 - name: KubeMemoryOvercommit alert (multi-node)
-- interval: 1m
+  interval: 1m
   input_series:
   - series: 'namespace_memory:kube_pod_container_resource_requests:sum{cluster="kubernetes", namespace="default"}'
     values: '2000000000x10' # 2 GB
@@ -1511,9 +1511,9 @@ tests:
     values: '1000000000x10'
   - series: 'kube_node_status_allocatable{cluster="kubernetes", node="n2", resource="memory", job="kube-state-metrics"}'
     values: '1000000000x10'
-  - series: 'kube_node_info{cluster="kubernetes", node="n1", job="kube-state-metrics"}'
+  - series: 'kube_node_role{cluster="kubernetes", node="n1", role="control-plane"}'
     values: '1x10'
-  - series: 'kube_node_info{cluster="kubernetes", node="n2", job="kube-state-metrics"}'
+  - series: 'kube_node_role{cluster="kubernetes", node="n2", role="control-plane"}'
     values: '1x10'
   alert_rule_test:
   - eval_time: 9m
@@ -1524,6 +1524,6 @@ tests:
     - exp_labels:
         severity: warning
       exp_annotations:
-        description: Cluster has overcommitted memory resource requests for Pods by 3G bytes and cannot tolerate node failure.
+        description: Cluster has overcommitted memory resource requests for Pods by 2G bytes and cannot tolerate node failure.
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryovercommit
         summary: Cluster has overcommitted memory resource requests.