diff --git a/CHANGELOG.md b/CHANGELOG.md index 56c85033fd7..8782e0e1b68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## master / unreleased +* [FEATURE] Ingester: added `-admin-limit-message` to customize the message contained in limit errors.#5460 * [CHANGE] AlertManager: include reason label in cortex_alertmanager_notifications_failed_total.#5409 * [CHANGE] Query: Set CORS Origin headers for Query API #5388 * [CHANGE] Updating prometheus/alertmanager from v0.25.0 to v0.25.1-0.20230505130626-263ca5c9438e. This includes the below changes. #5276 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 2b28e3adc5b..e84d16a611f 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -2620,6 +2620,10 @@ instance_limits: # max-global-series-per-metric limits. # CLI flag: -ingester.ignore-series-limit-for-metric-names [ignore_series_limit_for_metric_names: | default = ""] + +# Customize the message contained in limit errors +# CLI flag: -ingester.admin-limit-message +[admin_limit_message: | default = "please contact administrator to raise it"] ``` ### `ingester_client_config` diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 19cf30097f3..4956e0a98dc 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -124,6 +124,9 @@ type Config struct { // For testing, you can override the address and ID of this ingester. ingesterClientFactory func(addr string, cfg client.Config) (client.HealthAndIngesterClient, error) + + // For admin contact details + AdminLimitMessage string `yaml:"admin_limit_message"` } // RegisterFlags adds the flags required to config this to the given FlagSet @@ -144,6 +147,9 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.Int64Var(&cfg.DefaultLimits.MaxInflightPushRequests, "ingester.instance-limits.max-inflight-push-requests", 0, "Max inflight push requests that this ingester can handle (across all tenants). Additional requests will be rejected. 0 = unlimited.") f.StringVar(&cfg.IgnoreSeriesLimitForMetricNames, "ingester.ignore-series-limit-for-metric-names", "", "Comma-separated list of metric names, for which -ingester.max-series-per-metric and -ingester.max-global-series-per-metric limits will be ignored. Does not affect max-series-per-user or max-global-series-per-metric limits.") + + f.StringVar(&cfg.AdminLimitMessage, "ingester.admin-limit-message", "please contact administrator to raise it", "Customize the message contained in limit errors") + } func (cfg *Config) getIgnoreSeriesLimitForMetricNamesMap() map[string]struct{} { @@ -653,7 +659,9 @@ func New(cfg Config, limits *validation.Overrides, registerer prometheus.Registe cfg.DistributorShardingStrategy, cfg.DistributorShardByAllLabels, cfg.LifecyclerConfig.RingConfig.ReplicationFactor, - cfg.LifecyclerConfig.RingConfig.ZoneAwarenessEnabled) + cfg.LifecyclerConfig.RingConfig.ZoneAwarenessEnabled, + cfg.AdminLimitMessage, + ) i.TSDBState.shipperIngesterID = i.lifecycler.ID diff --git a/pkg/ingester/limiter.go b/pkg/ingester/limiter.go index c2293cceaf6..5b837801a2f 100644 --- a/pkg/ingester/limiter.go +++ b/pkg/ingester/limiter.go @@ -34,6 +34,7 @@ type Limiter struct { shuffleShardingEnabled bool shardByAllLabels bool zoneAwarenessEnabled bool + AdminLimitMessage string } // NewLimiter makes a new in-memory series limiter @@ -44,6 +45,7 @@ func NewLimiter( shardByAllLabels bool, replicationFactor int, zoneAwarenessEnabled bool, + AdminLimitMessage string, ) *Limiter { return &Limiter{ limits: limits, @@ -52,6 +54,7 @@ func NewLimiter( shuffleShardingEnabled: shardingStrategy == util.ShardingStrategyShuffle, shardByAllLabels: shardByAllLabels, zoneAwarenessEnabled: zoneAwarenessEnabled, + AdminLimitMessage: AdminLimitMessage, } } @@ -122,8 +125,8 @@ func (l *Limiter) formatMaxSeriesPerUserError(userID string) error { localLimit := l.limits.MaxLocalSeriesPerUser(userID) globalLimit := l.limits.MaxGlobalSeriesPerUser(userID) - return fmt.Errorf("per-user series limit of %d exceeded, please contact administrator to raise it (local limit: %d global limit: %d actual local limit: %d)", - minNonZero(localLimit, globalLimit), localLimit, globalLimit, actualLimit) + return fmt.Errorf("per-user series limit of %d exceeded, %s (local limit: %d global limit: %d actual local limit: %d)", + minNonZero(localLimit, globalLimit), l.AdminLimitMessage, localLimit, globalLimit, actualLimit) } func (l *Limiter) formatMaxSeriesPerMetricError(userID string) error { @@ -131,8 +134,8 @@ func (l *Limiter) formatMaxSeriesPerMetricError(userID string) error { localLimit := l.limits.MaxLocalSeriesPerMetric(userID) globalLimit := l.limits.MaxGlobalSeriesPerMetric(userID) - return fmt.Errorf("per-metric series limit of %d exceeded, please contact administrator to raise it (local limit: %d global limit: %d actual local limit: %d)", - minNonZero(localLimit, globalLimit), localLimit, globalLimit, actualLimit) + return fmt.Errorf("per-metric series limit of %d exceeded, %s (local limit: %d global limit: %d actual local limit: %d)", + minNonZero(localLimit, globalLimit), l.AdminLimitMessage, localLimit, globalLimit, actualLimit) } func (l *Limiter) formatMaxMetadataPerUserError(userID string) error { @@ -140,8 +143,8 @@ func (l *Limiter) formatMaxMetadataPerUserError(userID string) error { localLimit := l.limits.MaxLocalMetricsWithMetadataPerUser(userID) globalLimit := l.limits.MaxGlobalMetricsWithMetadataPerUser(userID) - return fmt.Errorf("per-user metric metadata limit of %d exceeded, please contact administrator to raise it (local limit: %d global limit: %d actual local limit: %d)", - minNonZero(localLimit, globalLimit), localLimit, globalLimit, actualLimit) + return fmt.Errorf("per-user metric metadata limit of %d exceeded, %s (local limit: %d global limit: %d actual local limit: %d)", + minNonZero(localLimit, globalLimit), l.AdminLimitMessage, localLimit, globalLimit, actualLimit) } func (l *Limiter) formatMaxMetadataPerMetricError(userID string) error { @@ -149,8 +152,8 @@ func (l *Limiter) formatMaxMetadataPerMetricError(userID string) error { localLimit := l.limits.MaxLocalMetadataPerMetric(userID) globalLimit := l.limits.MaxGlobalMetadataPerMetric(userID) - return fmt.Errorf("per-metric metadata limit of %d exceeded, please contact administrator to raise it (local limit: %d global limit: %d actual local limit: %d)", - minNonZero(localLimit, globalLimit), localLimit, globalLimit, actualLimit) + return fmt.Errorf("per-metric metadata limit of %d exceeded, %s (local limit: %d global limit: %d actual local limit: %d)", + minNonZero(localLimit, globalLimit), l.AdminLimitMessage, localLimit, globalLimit, actualLimit) } func (l *Limiter) maxSeriesPerMetric(userID string) int { diff --git a/pkg/ingester/limiter_test.go b/pkg/ingester/limiter_test.go index 4c736372fb4..1b65e388ea7 100644 --- a/pkg/ingester/limiter_test.go +++ b/pkg/ingester/limiter_test.go @@ -223,12 +223,12 @@ func runLimiterMaxFunctionTest( require.NoError(t, err) // Assert on default sharding strategy. - limiter := NewLimiter(overrides, ring, util.ShardingStrategyDefault, testData.shardByAllLabels, testData.ringReplicationFactor, testData.ringZoneAwarenessEnabled) + limiter := NewLimiter(overrides, ring, util.ShardingStrategyDefault, testData.shardByAllLabels, testData.ringReplicationFactor, testData.ringZoneAwarenessEnabled, "") actual := runMaxFn(limiter) assert.Equal(t, testData.expectedDefaultSharding, actual) // Assert on shuffle sharding strategy. - limiter = NewLimiter(overrides, ring, util.ShardingStrategyShuffle, testData.shardByAllLabels, testData.ringReplicationFactor, testData.ringZoneAwarenessEnabled) + limiter = NewLimiter(overrides, ring, util.ShardingStrategyShuffle, testData.shardByAllLabels, testData.ringReplicationFactor, testData.ringZoneAwarenessEnabled, "") actual = runMaxFn(limiter) assert.Equal(t, testData.expectedShuffleSharding, actual) }) @@ -290,7 +290,7 @@ func TestLimiter_AssertMaxSeriesPerMetric(t *testing.T) { }, nil) require.NoError(t, err) - limiter := NewLimiter(limits, ring, util.ShardingStrategyDefault, testData.shardByAllLabels, testData.ringReplicationFactor, false) + limiter := NewLimiter(limits, ring, util.ShardingStrategyDefault, testData.shardByAllLabels, testData.ringReplicationFactor, false, "") actual := limiter.AssertMaxSeriesPerMetric("test", testData.series) assert.Equal(t, testData.expected, actual) @@ -352,7 +352,7 @@ func TestLimiter_AssertMaxMetadataPerMetric(t *testing.T) { }, nil) require.NoError(t, err) - limiter := NewLimiter(limits, ring, util.ShardingStrategyDefault, testData.shardByAllLabels, testData.ringReplicationFactor, false) + limiter := NewLimiter(limits, ring, util.ShardingStrategyDefault, testData.shardByAllLabels, testData.ringReplicationFactor, false, "") actual := limiter.AssertMaxMetadataPerMetric("test", testData.metadata) assert.Equal(t, testData.expected, actual) @@ -415,7 +415,7 @@ func TestLimiter_AssertMaxSeriesPerUser(t *testing.T) { }, nil) require.NoError(t, err) - limiter := NewLimiter(limits, ring, util.ShardingStrategyDefault, testData.shardByAllLabels, testData.ringReplicationFactor, false) + limiter := NewLimiter(limits, ring, util.ShardingStrategyDefault, testData.shardByAllLabels, testData.ringReplicationFactor, false, "") actual := limiter.AssertMaxSeriesPerUser("test", testData.series) assert.Equal(t, testData.expected, actual) @@ -478,7 +478,7 @@ func TestLimiter_AssertMaxMetricsWithMetadataPerUser(t *testing.T) { }, nil) require.NoError(t, err) - limiter := NewLimiter(limits, ring, util.ShardingStrategyDefault, testData.shardByAllLabels, testData.ringReplicationFactor, false) + limiter := NewLimiter(limits, ring, util.ShardingStrategyDefault, testData.shardByAllLabels, testData.ringReplicationFactor, false, "") actual := limiter.AssertMaxMetricsWithMetadataPerUser("test", testData.metadata) assert.Equal(t, testData.expected, actual) @@ -501,7 +501,7 @@ func TestLimiter_FormatError(t *testing.T) { }, nil) require.NoError(t, err) - limiter := NewLimiter(limits, ring, util.ShardingStrategyDefault, true, 3, false) + limiter := NewLimiter(limits, ring, util.ShardingStrategyDefault, true, 3, false, "please contact administrator to raise it") actual := limiter.FormatError("user-1", errMaxSeriesPerUserLimitExceeded) assert.EqualError(t, actual, "per-user series limit of 100 exceeded, please contact administrator to raise it (local limit: 0 global limit: 100 actual local limit: 100)") diff --git a/pkg/ingester/user_state_test.go b/pkg/ingester/user_state_test.go index ec6234e25a3..c3aae474dae 100644 --- a/pkg/ingester/user_state_test.go +++ b/pkg/ingester/user_state_test.go @@ -77,7 +77,7 @@ func TestMetricCounter(t *testing.T) { // We're testing code that's not dependant on sharding strategy, replication factor, etc. To simplify the test, // we use local limit only. - limiter := NewLimiter(overrides, nil, util.ShardingStrategyDefault, true, 3, false) + limiter := NewLimiter(overrides, nil, util.ShardingStrategyDefault, true, 3, false, "") mc := newMetricCounter(limiter, ignored) for i := 0; i < tc.series; i++ {