observability: Ring the phone only when a data loss occurs with GitpodWsDaemonCrashLoopingg

Author: utam0k

operations/observability/mixins/workspace/rules/components/ws-daemon/alerts.libsonnet

@@ -12,7 +12,7 @@
           {
             alert: 'GitpodWsDaemonCrashLooping',
             labels: {
-              severity: 'critical',
+              severity: 'warning',
             },
             annotations: {
               runbook_url: 'https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWsDaemonCrashLooping.md',
@@ -23,6 +23,20 @@
               increase(kube_pod_container_status_restarts_total{container="ws-daemon"}[10m]) > 0
             |||,
           },
+          {
+            alert: 'BackupFailureBecauseOfGitpodWsDaemonCrash',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              runbook_url: 'https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWsDaemonCrashLooping.md',
+              summary: 'Increase the number of backup failure because of ws-daemon is crashlooping.',
+              description: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is restarting {{ printf "%.2f" $value }} times / 10 minutes.',
+            },
+            expr: |||
+              sum(increase(kube_pod_container_status_restarts_total{container="ws-daemon"}[10m])) > 0 AND sum(increase(gitpod_ws_manager_workspace_backups_failure_total{type="REGULAR"}[10m])) > 0
+            |||,
+          },
           {
             alert: 'GitpodWsDaemonExcessiveGC',
             labels: {