Allowing to disabled detailed ring metrics

CHANGELOG.md

@@ -6,6 +6,7 @@
 * [ENHANCEMENT] Ingester: Allowing to configure `-blocks-storage.tsdb.head-compaction-interval` flag up to 30 min and add a jitter on the first head compaction. #5919 #5928
 * [ENHANCEMENT] Distributor: Added `max_inflight_push_requests` config to ingester client to protect distributor from OOMKilled. #5917
 * [ENHANCEMENT] Distributor/Querier: Clean stale per-ingester metrics after ingester restarts. #5930
+* [ENHANCEMENT] Distributor/Ring: Allow disabling detailed ring metrics by ring member. #5931
 * [CHANGE] Upgrade Dockerfile Node version from 14x to 18x. #5906
 * [BUGFIX] Configsdb: Fix endline issue in db password. #5920
 

docs/configuration/config-file-reference.md

@@ -2830,6 +2830,13 @@ lifecycler:
     # CLI flag: -distributor.excluded-zones
     [excluded_zones: <string> | default = ""]
 
+    # Set to true to enable ring detailed metrics. These metrics provide
+    # detailed information, such as token count and ownership per tenant.
+    # Disabling them can significantly decrease the number of metrics emitted by
+    # the distributors.
+    # CLI flag: -ring.detailed-metrics-enabled
+    [detailed_metrics_enabled: <boolean> | default = true]
+
   # Number of tokens for each ingester.
   # CLI flag: -ingester.num-tokens
   [num_tokens: <int> | default = 128]

pkg/ring/ring.go

@@ -134,11 +134,12 @@ var (
 
 // Config for a Ring
 type Config struct {
-	KVStore              kv.Config              `yaml:"kvstore"`
-	HeartbeatTimeout     time.Duration          `yaml:"heartbeat_timeout"`
-	ReplicationFactor    int                    `yaml:"replication_factor"`
-	ZoneAwarenessEnabled bool                   `yaml:"zone_awareness_enabled"`
-	ExcludedZones        flagext.StringSliceCSV `yaml:"excluded_zones"`
+	KVStore                kv.Config              `yaml:"kvstore"`
+	HeartbeatTimeout       time.Duration          `yaml:"heartbeat_timeout"`
+	ReplicationFactor      int                    `yaml:"replication_factor"`
+	ZoneAwarenessEnabled   bool                   `yaml:"zone_awareness_enabled"`
+	ExcludedZones          flagext.StringSliceCSV `yaml:"excluded_zones"`
+	DetailedMetricsEnabled bool                   `yaml:"detailed_metrics_enabled"`
 
 	// Whether the shuffle-sharding subring cache is disabled. This option is set
 	// internally and never exposed to the user.
@@ -155,6 +156,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	cfg.KVStore.RegisterFlagsWithPrefix(prefix, "collectors/", f)
 
 	f.DurationVar(&cfg.HeartbeatTimeout, prefix+"ring.heartbeat-timeout", time.Minute, "The heartbeat timeout after which ingesters are skipped for reads/writes. 0 = never (timeout disabled).")
+	f.BoolVar(&cfg.DetailedMetricsEnabled, prefix+"ring.detailed-metrics-enabled", true, "Set to true to enable ring detailed metrics. These metrics provide detailed information, such as token count and ownership per tenant. Disabling them can significantly decrease the number of metrics emitted by the distributors.")
 	f.IntVar(&cfg.ReplicationFactor, prefix+"distributor.replication-factor", 3, "The number of ingesters to write to and read from.")
 	f.BoolVar(&cfg.ZoneAwarenessEnabled, prefix+"distributor.zone-awareness-enabled", false, "True to enable the zone-awareness and replicate ingested samples across different availability zones.")
 	f.Var(&cfg.ExcludedZones, prefix+"distributor.excluded-zones", "Comma-separated list of zones to exclude from the ring. Instances in excluded zones will be filtered out from the ring.")
@@ -678,19 +680,21 @@ func (r *Ring) updateRingMetrics(compareResult CompareResult) {
 		return
 	}
 
-	prevOwners := r.reportedOwners
-	r.reportedOwners = make(map[string]struct{})
-	numTokens, ownedRange := r.countTokens()
-	for id, totalOwned := range ownedRange {
-		r.memberOwnershipGaugeVec.WithLabelValues(id).Set(float64(totalOwned) / float64(math.MaxUint32+1))
-		r.numTokensGaugeVec.WithLabelValues(id).Set(float64(numTokens[id]))
-		delete(prevOwners, id)
-		r.reportedOwners[id] = struct{}{}
-	}
+	if r.cfg.DetailedMetricsEnabled {
+		prevOwners := r.reportedOwners
+		r.reportedOwners = make(map[string]struct{})
+		numTokens, ownedRange := r.countTokens()
+		for id, totalOwned := range ownedRange {
+			r.memberOwnershipGaugeVec.WithLabelValues(id).Set(float64(totalOwned) / float64(math.MaxUint32+1))
+			r.numTokensGaugeVec.WithLabelValues(id).Set(float64(numTokens[id]))
+			delete(prevOwners, id)
+			r.reportedOwners[id] = struct{}{}
+		}
 
-	for k := range prevOwners {
-		r.memberOwnershipGaugeVec.DeleteLabelValues(k)
-		r.numTokensGaugeVec.DeleteLabelValues(k)
+		for k := range prevOwners {
+			r.memberOwnershipGaugeVec.DeleteLabelValues(k)
+			r.numTokensGaugeVec.DeleteLabelValues(k)
+		}
 	}
 
 	r.totalTokensGauge.Set(float64(len(r.ringTokens)))

pkg/ring/ring_test.go

@@ -2984,28 +2984,13 @@ func userToken(user, zone string, skip int) uint32 {
 }
 
 func TestUpdateMetrics(t *testing.T) {
-	cfg := Config{
-		KVStore:              kv.Config{},
-		HeartbeatTimeout:     0, // get healthy stats
-		ReplicationFactor:    3,
-		ZoneAwarenessEnabled: true,
-	}
-
-	registry := prometheus.NewRegistry()
-
-	// create the ring to set up metrics, but do not start
-	ring, err := NewWithStoreClientAndStrategy(cfg, testRingName, testRingKey, &MockClient{}, NewDefaultReplicationStrategy(), registry, log.NewNopLogger())
-	require.NoError(t, err)
-
-	ringDesc := Desc{
-		Ingesters: map[string]InstanceDesc{
-			"A": {Addr: "127.0.0.1", Timestamp: 22, Tokens: []uint32{math.MaxUint32 / 4, (math.MaxUint32 / 4) * 3}},
-			"B": {Addr: "127.0.0.2", Timestamp: 11, Tokens: []uint32{(math.MaxUint32 / 4) * 2, math.MaxUint32}},
-		},
-	}
-	ring.updateRingState(&ringDesc)
-
-	err = testutil.GatherAndCompare(registry, bytes.NewBufferString(`
+	testCase := []struct {
+		DetailedMetricsEnabled bool
+		Expected               string
+	}{
+		{
+			DetailedMetricsEnabled: true,
+			Expected: `
 		# HELP ring_member_ownership_percent The percent ownership of the ring by member
 		# TYPE ring_member_ownership_percent gauge
 		ring_member_ownership_percent{member="A",name="test"} 0.49999999976716936
@@ -3031,16 +3016,68 @@ func TestUpdateMetrics(t *testing.T) {
 		# HELP ring_tokens_total Number of tokens in the ring
 		# TYPE ring_tokens_total gauge
 		ring_tokens_total{name="test"} 4
-	`))
-	assert.NoError(t, err)
+	`,
+		},
+		{
+			DetailedMetricsEnabled: false,
+			Expected: `
+		# HELP ring_members Number of members in the ring
+		# TYPE ring_members gauge
+		ring_members{name="test",state="ACTIVE"} 2
+		ring_members{name="test",state="JOINING"} 0
+		ring_members{name="test",state="LEAVING"} 0
+		ring_members{name="test",state="PENDING"} 0
+		ring_members{name="test",state="Unhealthy"} 0
+		# HELP ring_oldest_member_timestamp Timestamp of the oldest member in the ring.
+		# TYPE ring_oldest_member_timestamp gauge
+		ring_oldest_member_timestamp{name="test",state="ACTIVE"} 11
+		ring_oldest_member_timestamp{name="test",state="JOINING"} 0
+		ring_oldest_member_timestamp{name="test",state="LEAVING"} 0
+		ring_oldest_member_timestamp{name="test",state="PENDING"} 0
+		ring_oldest_member_timestamp{name="test",state="Unhealthy"} 0
+		# HELP ring_tokens_total Number of tokens in the ring
+		# TYPE ring_tokens_total gauge
+		ring_tokens_total{name="test"} 4
+	`,
+		},
+	}
+
+	for _, tc := range testCase {
+		t.Run(fmt.Sprintf("DetailedMetricsEnabled=%v", tc.DetailedMetricsEnabled), func(t *testing.T) {
+			cfg := Config{
+				KVStore:                kv.Config{},
+				HeartbeatTimeout:       0, // get healthy stats
+				ReplicationFactor:      3,
+				ZoneAwarenessEnabled:   true,
+				DetailedMetricsEnabled: tc.DetailedMetricsEnabled,
+			}
+			registry := prometheus.NewRegistry()
+
+			// create the ring to set up metrics, but do not start
+			ring, err := NewWithStoreClientAndStrategy(cfg, testRingName, testRingKey, &MockClient{}, NewDefaultReplicationStrategy(), registry, log.NewNopLogger())
+			require.NoError(t, err)
+
+			ringDesc := Desc{
+				Ingesters: map[string]InstanceDesc{
+					"A": {Addr: "127.0.0.1", Timestamp: 22, Tokens: []uint32{math.MaxUint32 / 4, (math.MaxUint32 / 4) * 3}},
+					"B": {Addr: "127.0.0.2", Timestamp: 11, Tokens: []uint32{(math.MaxUint32 / 4) * 2, math.MaxUint32}},
+				},
+			}
+			ring.updateRingState(&ringDesc)
+
+			err = testutil.GatherAndCompare(registry, bytes.NewBufferString(tc.Expected))
+			assert.NoError(t, err)
+		})
+	}
 }
 
 func TestUpdateMetricsWithRemoval(t *testing.T) {
 	cfg := Config{
-		KVStore:              kv.Config{},
-		HeartbeatTimeout:     0, // get healthy stats
-		ReplicationFactor:    3,
-		ZoneAwarenessEnabled: true,
+		KVStore:                kv.Config{},
+		HeartbeatTimeout:       0, // get healthy stats
+		ReplicationFactor:      3,
+		ZoneAwarenessEnabled:   true,
+		DetailedMetricsEnabled: true,
 	}
 
 	registry := prometheus.NewRegistry()