diff --git a/CHANGELOG.md b/CHANGELOG.md index 79ed405041e..879682273f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ * [ENHANCEMENT] Ingester/Store Gateway Clients: Introduce an experimental HealthCheck handler to quickly fail requests directed to unhealthy targets. #6225 #6257 * [ENHANCEMENT] Upgrade build image and Go version to 1.23.2. #6261 #6262 * [ENHANCEMENT] Querier/Ruler: Expose `store_gateway_consistency_check_max_attempts` for max retries when querying store gateway in consistency check. #6276 +* [ENHANCEMENT] StoreGateway: Add new `cortex_bucket_store_chunk_pool_inuse_bytes` metric to track the usage in chunk pool. #6310 * [BUGFIX] Runtime-config: Handle absolute file paths when working directory is not / #6224 ## 1.18.1 2024-10-14 diff --git a/pkg/storegateway/chunk_bytes_pool.go b/pkg/storegateway/chunk_bytes_pool.go index 7b416a8a538..1a0e82d12ce 100644 --- a/pkg/storegateway/chunk_bytes_pool.go +++ b/pkg/storegateway/chunk_bytes_pool.go @@ -10,7 +10,8 @@ type chunkBytesPool struct { pool *pool.BucketedPool[byte] // Metrics. - poolByteStats *prometheus.CounterVec + poolByteStats *prometheus.CounterVec + poolInUseBytes prometheus.GaugeFunc } func newChunkBytesPool(minBucketSize, maxBucketSize int, maxChunkPoolBytes uint64, reg prometheus.Registerer) (*chunkBytesPool, error) { @@ -25,6 +26,12 @@ func newChunkBytesPool(minBucketSize, maxBucketSize int, maxChunkPoolBytes uint6 Name: "cortex_bucket_store_chunk_pool_operation_bytes_total", Help: "Total bytes number of bytes pooled by operation.", }, []string{"operation", "stats"}), + poolInUseBytes: promauto.With(reg).NewGaugeFunc(prometheus.GaugeOpts{ + Name: "cortex_bucket_store_chunk_pool_inuse_bytes", + Help: "Total bytes in use in the chunk pool.", + }, func() float64 { + return float64(upstream.UsedBytes()) + }), }, nil } diff --git a/pkg/storegateway/chunk_bytes_pool_test.go b/pkg/storegateway/chunk_bytes_pool_test.go index dc8f1e317c3..3dab2bc49b2 100644 --- a/pkg/storegateway/chunk_bytes_pool_test.go +++ b/pkg/storegateway/chunk_bytes_pool_test.go @@ -20,9 +20,15 @@ func TestChunkBytesPool_Get(t *testing.T) { p, err := newChunkBytesPool(cortex_tsdb.ChunkPoolDefaultMinBucketSize, cortex_tsdb.ChunkPoolDefaultMaxBucketSize, 0, reg) require.NoError(t, err) testBytes := []byte("test") - _, err = p.Get(store.EstimatedMaxChunkSize - 1) + b0, err := p.Get(store.EstimatedMaxChunkSize - 1) require.NoError(t, err) + assert.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(fmt.Sprintf(` + # HELP cortex_bucket_store_chunk_pool_inuse_bytes Total bytes in use in the chunk pool. + # TYPE cortex_bucket_store_chunk_pool_inuse_bytes gauge + cortex_bucket_store_chunk_pool_inuse_bytes %d + `, 16000)), "cortex_bucket_store_chunk_pool_inuse_bytes")) + b, err := p.Get(store.EstimatedMaxChunkSize + 1) require.NoError(t, err) @@ -31,11 +37,21 @@ func TestChunkBytesPool_Get(t *testing.T) { p.Put(b) assert.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(fmt.Sprintf(` + # HELP cortex_bucket_store_chunk_pool_inuse_bytes Total bytes in use in the chunk pool. + # TYPE cortex_bucket_store_chunk_pool_inuse_bytes gauge + cortex_bucket_store_chunk_pool_inuse_bytes %d # HELP cortex_bucket_store_chunk_pool_operation_bytes_total Total bytes number of bytes pooled by operation. # TYPE cortex_bucket_store_chunk_pool_operation_bytes_total counter cortex_bucket_store_chunk_pool_operation_bytes_total{operation="get",stats="cap"} %d cortex_bucket_store_chunk_pool_operation_bytes_total{operation="get",stats="requested"} %d cortex_bucket_store_chunk_pool_operation_bytes_total{operation="put",stats="cap"} %d cortex_bucket_store_chunk_pool_operation_bytes_total{operation="put",stats="len"} %d - `, store.EstimatedMaxChunkSize*3, store.EstimatedMaxChunkSize*2, store.EstimatedMaxChunkSize*2, len(testBytes))))) + `, 16000, store.EstimatedMaxChunkSize*3, store.EstimatedMaxChunkSize*2, store.EstimatedMaxChunkSize*2, len(testBytes))))) + + p.Put(b0) + assert.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(fmt.Sprintf(` + # HELP cortex_bucket_store_chunk_pool_inuse_bytes Total bytes in use in the chunk pool. + # TYPE cortex_bucket_store_chunk_pool_inuse_bytes gauge + cortex_bucket_store_chunk_pool_inuse_bytes %d + `, 0)), "cortex_bucket_store_chunk_pool_inuse_bytes")) }