From bc94c0f6906dd9153c16cb1a4993df4ab9e0743a Mon Sep 17 00:00:00 2001 From: alanprot Date: Mon, 9 Dec 2024 16:08:59 -0800 Subject: [PATCH 1/3] Configure labelset limits on the TestPushRace Signed-off-by: alanprot --- pkg/ingester/ingester_test.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index ab2c9ad0b81..f64368877fa 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -407,14 +407,26 @@ func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { func TestPushRace(t *testing.T) { cfg := defaultIngesterTestConfig(t) + l := defaultLimitsTestConfig() cfg.LabelsStringInterningEnabled = true cfg.LifecyclerConfig.JoinAfter = 0 + + l.LimitsPerLabelSet = []validation.LimitsPerLabelSet{ + { + LabelSet: labels.FromMap(map[string]string{ + labels.MetricName: "foo", + }), + Limits: validation.LimitsPerLabelSetEntry{ + MaxSeries: 10e10, + }, + }, + } + dir := t.TempDir() blocksDir := filepath.Join(dir, "blocks") - require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, defaultLimitsTestConfig(), nil, blocksDir, prometheus.NewRegistry(), true) + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, l, nil, blocksDir, prometheus.NewRegistry(), true) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) From e18cd83e9be2fce766103e2716533bb8d001c164 Mon Sep 17 00:00:00 2001 From: alanprot Date: Mon, 9 Dec 2024 16:13:23 -0800 Subject: [PATCH 2/3] Fix race Signed-off-by: alanprot --- pkg/ingester/user_state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ingester/user_state.go b/pkg/ingester/user_state.go index 60ca5c1eb6d..7b0dec69204 100644 --- a/pkg/ingester/user_state.go +++ b/pkg/ingester/user_state.go @@ -118,7 +118,7 @@ func (m *labelSetCounter) canAddSeriesForLabelSet(ctx context.Context, u *userTS s := m.shards[util.HashFP(model.Fingerprint(set.Hash))%numMetricCounterShards] s.RLock() if r, ok := s.valuesCounter[set.Hash]; ok { - s.RUnlock() + defer s.RUnlock() return r.count, nil } s.RUnlock() From 854b99f98ada2de0d82dadbd91add0961d27fcab Mon Sep 17 00:00:00 2001 From: alanprot Date: Mon, 9 Dec 2024 16:17:19 -0800 Subject: [PATCH 3/3] Changelog Signed-off-by: alanprot --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9c4d8c67ca..b5c3fc5b7f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,11 +45,12 @@ * [ENHANCEMENT] Distributor: Expose `cortex_label_size_bytes` native histogram metric. #6372 * [ENHANCEMENT] Add new option `-server.grpc_server-num-stream-workers` to configure the number of worker goroutines that should be used to process incoming streams. #6386 * [ENHANCEMENT] Distributor: Return HTTP 5XX instead of HTTP 4XX when instance limits are hit. #6358 -* [ENHANCEMENT] Ingester: Make sure unregistered ingester joining the ring after WAL replay #6277 +* [ENHANCEMENT] Ingester: Make sure unregistered ingester joining the ring after WAL replay. #6277 * [BUGFIX] Runtime-config: Handle absolute file paths when working directory is not / #6224 * [BUGFIX] Ruler: Allow rule evaluation to complete during shutdown. #6326 -* [BUGFIX] Ring: update ring with new ip address when instance is lost, rejoins, but heartbeat is disabled #6271 -* [BUGFIX] ingester: Fix regression on usage of cortex_ingester_queried_chunks #6398 +* [BUGFIX] Ring: update ring with new ip address when instance is lost, rejoins, but heartbeat is disabled. #6271 +* [BUGFIX] Ingester: Fix regression on usage of cortex_ingester_queried_chunks. #6398 +* [BUGFIX] Ingester: Fix possible race condition when `active series per LabelSet` is configured. #6409 ## 1.18.1 2024-10-14