diff --git a/CHANGELOG.md b/CHANGELOG.md index 87b3153f2ee..fbe7d9be528 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ * [ENHANCEMENT] Distributor/Ingester: Add experimental `-distributor.sign_write_requests` flag to sign the write requests. #5430 * [ENHANCEMENT] Store Gateway/Querier/Compactor: Handling CMK Access Denied errors. #5420 #5442 #5446 * [ENHANCEMENT] Store Gateway: Implementing multi level index cache. #5451 +* [ENHANCEMENT] Alertmanager: Add the alert name in error log when it get throttled. #5456 * [BUGFIX] Ruler: Validate if rule group can be safely converted back to rule group yaml from protobuf message #5265 * [BUGFIX] Querier: Convert gRPC `ResourceExhausted` status code from store gateway to 422 limit error. #5286 * [BUGFIX] Alertmanager: Route web-ui requests to the alertmanager distributor when sharding is enabled. #5293 diff --git a/pkg/alertmanager/alertmanager.go b/pkg/alertmanager/alertmanager.go index 6ac39ddfbb6..d5a2aa616b9 100644 --- a/pkg/alertmanager/alertmanager.go +++ b/pkg/alertmanager/alertmanager.go @@ -604,7 +604,7 @@ func (g *dispatcherLimits) MaxNumberOfAggregationGroups() int { } var ( - errTooManyAlerts = "too many alerts, limit: %d" + errTooManyAlerts = "too many alerts, limit: %d, alert name: %s" errAlertsTooBig = "alerts too big, total size limit: %d bytes" ) @@ -670,7 +670,7 @@ func (a *alertsLimiter) PreStore(alert *types.Alert, existing bool) error { if !existing && countLimit > 0 && (a.count+1) > countLimit { a.failureCounter.Inc() - return fmt.Errorf(errTooManyAlerts, countLimit) + return fmt.Errorf(errTooManyAlerts, countLimit, alert.Name()) } if existing { diff --git a/pkg/alertmanager/alertmanager_test.go b/pkg/alertmanager/alertmanager_test.go index 23a309f47eb..52aa186c391 100644 --- a/pkg/alertmanager/alertmanager_test.go +++ b/pkg/alertmanager/alertmanager_test.go @@ -114,7 +114,7 @@ route: var ( alert1 = model.Alert{ - Labels: model.LabelSet{"alert": "first"}, + Labels: model.LabelSet{"alert": "first", "alertname": "alert1"}, Annotations: model.LabelSet{"job": "test"}, StartsAt: time.Now(), EndsAt: time.Now(), @@ -123,7 +123,7 @@ var ( alert1Size = alertSize(alert1) alert2 = model.Alert{ - Labels: model.LabelSet{"alert": "second"}, + Labels: model.LabelSet{"alert": "second", "alertname": "alert2"}, Annotations: model.LabelSet{"job": "test", "cluster": "prod"}, StartsAt: time.Now(), EndsAt: time.Now(), @@ -161,7 +161,7 @@ func TestAlertsLimiterWithCountLimit(t *testing.T) { ops := []callbackOp{ {alert: &types.Alert{Alert: alert1}, existing: false, expectedCount: 1, expectedTotalSize: alert1Size}, - {alert: &types.Alert{Alert: alert2}, existing: false, expectedInsertError: fmt.Errorf(errTooManyAlerts, 1), expectedCount: 1, expectedTotalSize: alert1Size}, + {alert: &types.Alert{Alert: alert2}, existing: false, expectedInsertError: fmt.Errorf(errTooManyAlerts, 1, alert2.Name()), expectedCount: 1, expectedTotalSize: alert1Size}, {alert: &types.Alert{Alert: alert1}, delete: true, expectedCount: 0, expectedTotalSize: 0}, {alert: &types.Alert{Alert: alert2}, existing: false, expectedCount: 1, expectedTotalSize: alert2Size},