add alert name in error log (#5456)

* add alert name in error log

Signed-off-by: Yijie Qin <[email protected]>

* add changelog

Signed-off-by: Yijie Qin <[email protected]>

* address comment

Signed-off-by: Yijie Qin <[email protected]>

---------

Signed-off-by: Yijie Qin <[email protected]>

Author: qinxx108

CHANGELOG.md

@@ -40,6 +40,7 @@
 * [ENHANCEMENT] Distributor/Ingester: Add experimental `-distributor.sign_write_requests` flag to sign the write requests. #5430
 * [ENHANCEMENT] Store Gateway/Querier/Compactor: Handling CMK Access Denied errors. #5420 #5442 #5446
 * [ENHANCEMENT] Store Gateway: Implementing multi level index cache. #5451
+* [ENHANCEMENT] Alertmanager: Add the alert name in error log when it get throttled. #5456
 * [BUGFIX] Ruler: Validate if rule group can be safely converted back to rule group yaml from protobuf message #5265
 * [BUGFIX] Querier: Convert gRPC `ResourceExhausted` status code from store gateway to 422 limit error. #5286
 * [BUGFIX] Alertmanager: Route web-ui requests to the alertmanager distributor when sharding is enabled. #5293

pkg/alertmanager/alertmanager.go

@@ -604,7 +604,7 @@ func (g *dispatcherLimits) MaxNumberOfAggregationGroups() int {
 }
 
 var (
-	errTooManyAlerts = "too many alerts, limit: %d"
+	errTooManyAlerts = "too many alerts, limit: %d, alert name: %s"
 	errAlertsTooBig  = "alerts too big, total size limit: %d bytes"
 )
 
@@ -670,7 +670,7 @@ func (a *alertsLimiter) PreStore(alert *types.Alert, existing bool) error {
 
 	if !existing && countLimit > 0 && (a.count+1) > countLimit {
 		a.failureCounter.Inc()
-		return fmt.Errorf(errTooManyAlerts, countLimit)
+		return fmt.Errorf(errTooManyAlerts, countLimit, alert.Name())
 	}
 
 	if existing {

pkg/alertmanager/alertmanager_test.go

@@ -114,7 +114,7 @@ route:
 
 var (
 	alert1 = model.Alert{
-		Labels:       model.LabelSet{"alert": "first"},
+		Labels:       model.LabelSet{"alert": "first", "alertname": "alert1"},
 		Annotations:  model.LabelSet{"job": "test"},
 		StartsAt:     time.Now(),
 		EndsAt:       time.Now(),
@@ -123,7 +123,7 @@ var (
 	alert1Size = alertSize(alert1)
 
 	alert2 = model.Alert{
-		Labels:       model.LabelSet{"alert": "second"},
+		Labels:       model.LabelSet{"alert": "second", "alertname": "alert2"},
 		Annotations:  model.LabelSet{"job": "test", "cluster": "prod"},
 		StartsAt:     time.Now(),
 		EndsAt:       time.Now(),
@@ -161,7 +161,7 @@ func TestAlertsLimiterWithCountLimit(t *testing.T) {
 
 	ops := []callbackOp{
 		{alert: &types.Alert{Alert: alert1}, existing: false, expectedCount: 1, expectedTotalSize: alert1Size},
-		{alert: &types.Alert{Alert: alert2}, existing: false, expectedInsertError: fmt.Errorf(errTooManyAlerts, 1), expectedCount: 1, expectedTotalSize: alert1Size},
+		{alert: &types.Alert{Alert: alert2}, existing: false, expectedInsertError: fmt.Errorf(errTooManyAlerts, 1, alert2.Name()), expectedCount: 1, expectedTotalSize: alert1Size},
 		{alert: &types.Alert{Alert: alert1}, delete: true, expectedCount: 0, expectedTotalSize: 0},
 
 		{alert: &types.Alert{Alert: alert2}, existing: false, expectedCount: 1, expectedTotalSize: alert2Size},