grab 'status' label for cortex_alertmanager_alerts_received_total metric (#4907)

* grab 'status' label for cortex_alertmanager_alerts_received_total metric

Signed-off-by: Alvin Lin <[email protected]>

* Update CHANGELOG

Signed-off-by: Alvin Lin <[email protected]>

Signed-off-by: Alvin Lin <[email protected]>

Author: alvinlin123

CHANGELOG.md

@@ -44,6 +44,7 @@
 * [CHANGE] Compactor: Added `cortex_compactor_runs_interrupted_total` to separate compaction interruptions from failures
 * [CHANGE] Enable PromQL `@` modifier, negative offset always. #4927
 * [CHANGE] Store-gateway: Add user label to `cortex_bucket_store_blocks_loaded` metric. #4918
+* [CHANGE] AlertManager: include `status` label in `cortex_alertmanager_alerts_received_total`. #4907
 * [ENHANCEMENT] AlertManager: Retrying AlertManager Get Requests (Get Alertmanager status, Get Alertmanager Receivers) on next replica on error #4840
 * [ENHANCEMENT] Querier/Ruler: Retry store-gateway in case of unexpected failure, instead of failing the query. #4532 #4839
 * [ENHANCEMENT] Ring: DoBatch prioritize 4xx errors when failing. #4783

pkg/alertmanager/alertmanager_metrics.go

@@ -72,7 +72,7 @@ func newAlertmanagerMetrics() *alertmanagerMetrics {
 		alertsReceived: prometheus.NewDesc(
 			"cortex_alertmanager_alerts_received_total",
 			"The total number of received alerts.",
-			[]string{"user"}, nil),
+			[]string{"user", "status"}, nil),
 		alertsInvalid: prometheus.NewDesc(
 			"cortex_alertmanager_alerts_invalid_total",
 			"The total number of received alerts that were invalid.",
@@ -288,7 +288,7 @@ func (m *alertmanagerMetrics) Describe(out chan<- *prometheus.Desc) {
 func (m *alertmanagerMetrics) Collect(out chan<- prometheus.Metric) {
 	data := m.regs.BuildMetricFamiliesPerUser()
 
-	data.SendSumOfCountersPerUser(out, m.alertsReceived, "alertmanager_alerts_received_total")
+	data.SendSumOfCountersPerUserWithLabels(out, m.alertsReceived, "alertmanager_alerts_received_total", "status")
 	data.SendSumOfCountersPerUser(out, m.alertsInvalid, "alertmanager_alerts_invalid_total")
 
 	data.SendSumOfCountersPerUserWithLabels(out, m.numNotifications, "alertmanager_notifications_total", "integration")

pkg/alertmanager/alertmanager_metrics_test.go

@@ -48,9 +48,12 @@ func TestAlertmanagerMetricsStore(t *testing.T) {
 		cortex_alertmanager_alerts_invalid_total{user="user3"} 200
 		# HELP cortex_alertmanager_alerts_received_total The total number of received alerts.
 		# TYPE cortex_alertmanager_alerts_received_total counter
-		cortex_alertmanager_alerts_received_total{user="user1"} 10
-		cortex_alertmanager_alerts_received_total{user="user2"} 100
-		cortex_alertmanager_alerts_received_total{user="user3"} 1000
+		cortex_alertmanager_alerts_received_total{status="firing",user="user1"} 4
+		cortex_alertmanager_alerts_received_total{status="resolved",user="user1"} 6
+		cortex_alertmanager_alerts_received_total{status="firing",user="user2"} 40
+		cortex_alertmanager_alerts_received_total{status="resolved",user="user2"} 60
+		cortex_alertmanager_alerts_received_total{status="firing",user="user3"} 400
+		cortex_alertmanager_alerts_received_total{status="resolved",user="user3"} 600
 		# HELP cortex_alertmanager_config_hash Hash of the currently loaded alertmanager configuration.
 		# TYPE cortex_alertmanager_config_hash gauge
 		cortex_alertmanager_config_hash{user="user1"} 0
@@ -327,9 +330,12 @@ func TestAlertmanagerMetricsRemoval(t *testing.T) {
 
         	            # HELP cortex_alertmanager_alerts_received_total The total number of received alerts.
         	            # TYPE cortex_alertmanager_alerts_received_total counter
-        	            cortex_alertmanager_alerts_received_total{user="user1"} 10
-        	            cortex_alertmanager_alerts_received_total{user="user2"} 100
-        	            cortex_alertmanager_alerts_received_total{user="user3"} 1000
+        	            cortex_alertmanager_alerts_received_total{status="firing",user="user1"} 4
+						cortex_alertmanager_alerts_received_total{status="resolved",user="user1"} 6
+						cortex_alertmanager_alerts_received_total{status="firing",user="user2"} 40
+						cortex_alertmanager_alerts_received_total{status="resolved",user="user2"} 60
+						cortex_alertmanager_alerts_received_total{status="firing",user="user3"} 400
+						cortex_alertmanager_alerts_received_total{status="resolved",user="user3"} 600
 
         	            # HELP cortex_alertmanager_config_hash Hash of the currently loaded alertmanager configuration.
         	            # TYPE cortex_alertmanager_config_hash gauge
@@ -611,8 +617,10 @@ func TestAlertmanagerMetricsRemoval(t *testing.T) {
 
     		# HELP cortex_alertmanager_alerts_received_total The total number of received alerts.
     		# TYPE cortex_alertmanager_alerts_received_total counter
-    		cortex_alertmanager_alerts_received_total{user="user1"} 10
-    		cortex_alertmanager_alerts_received_total{user="user2"} 100
+    		cortex_alertmanager_alerts_received_total{status="firing",user="user1"} 4
+			cortex_alertmanager_alerts_received_total{status="resolved",user="user1"} 6
+			cortex_alertmanager_alerts_received_total{status="firing",user="user2"} 40
+			cortex_alertmanager_alerts_received_total{status="resolved",user="user2"} 60
 
     		# HELP cortex_alertmanager_config_hash Hash of the currently loaded alertmanager configuration.
     		# TYPE cortex_alertmanager_config_hash gauge