From 70dd2538bd96c4648adc9ea163b62fefd025ad8c Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 24 Aug 2023 16:18:00 -0700 Subject: [PATCH 1/3] add new metric cortex_bucket_store_chunk_refetches_total for store gateway chunk refetches Signed-off-by: Ben Ye --- pkg/storegateway/bucket_store_metrics.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/storegateway/bucket_store_metrics.go b/pkg/storegateway/bucket_store_metrics.go index 76bf3e8f883..e0f728ca8ad 100644 --- a/pkg/storegateway/bucket_store_metrics.go +++ b/pkg/storegateway/bucket_store_metrics.go @@ -25,6 +25,7 @@ type BucketStoreMetrics struct { seriesGetAllDuration *prometheus.Desc seriesMergeDuration *prometheus.Desc seriesRefetches *prometheus.Desc + chunkRefetches *prometheus.Desc resultSeriesCount *prometheus.Desc queriesDropped *prometheus.Desc chunkSizeBytes *prometheus.Desc @@ -105,6 +106,10 @@ func NewBucketStoreMetrics() *BucketStoreMetrics { "cortex_bucket_store_series_refetches_total", "Total number of cases where the built-in max series size was not enough to fetch series from index, resulting in refetch.", nil, nil), + chunkRefetches: prometheus.NewDesc( + "cortex_bucket_store_chunk_refetches_total", + "Total number of cases where configured estimated chunk bytes was not enough was to fetch chunks from object store, resulting in refetch.", + nil, nil), resultSeriesCount: prometheus.NewDesc( "cortex_bucket_store_series_result_series", "Number of series observed in the final result of a query.", @@ -205,6 +210,7 @@ func (m *BucketStoreMetrics) Describe(out chan<- *prometheus.Desc) { out <- m.seriesGetAllDuration out <- m.seriesMergeDuration out <- m.seriesRefetches + out <- m.chunkRefetches out <- m.resultSeriesCount out <- m.queriesDropped out <- m.chunkSizeBytes @@ -247,6 +253,7 @@ func (m *BucketStoreMetrics) Collect(out chan<- prometheus.Metric) { data.SendSumOfHistograms(out, m.seriesGetAllDuration, "thanos_bucket_store_series_get_all_duration_seconds") data.SendSumOfHistograms(out, m.seriesMergeDuration, "thanos_bucket_store_series_merge_duration_seconds") data.SendSumOfCounters(out, m.seriesRefetches, "thanos_bucket_store_series_refetches_total") + data.SendSumOfCounters(out, m.chunkRefetches, "thanos_bucket_store_chunk_refetches_total") data.SendSumOfHistograms(out, m.resultSeriesCount, "thanos_bucket_store_series_result_series") data.SendSumOfCounters(out, m.queriesDropped, "thanos_bucket_store_queries_dropped_total") data.SendSumOfHistograms(out, m.chunkSizeBytes, "thanos_bucket_store_sent_chunk_size_bytes") From cd62bc5148ebbd1636b7e87eb6ff30112086594d Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 24 Aug 2023 16:27:46 -0700 Subject: [PATCH 2/3] add new cortex_bucket_store_chunk_refetches_total Signed-off-by: Ben Ye --- pkg/storegateway/bucket_store_metrics_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/storegateway/bucket_store_metrics_test.go b/pkg/storegateway/bucket_store_metrics_test.go index 04fb3def042..e0d4bc32431 100644 --- a/pkg/storegateway/bucket_store_metrics_test.go +++ b/pkg/storegateway/bucket_store_metrics_test.go @@ -393,6 +393,10 @@ func TestBucketStoreMetrics(t *testing.T) { # TYPE cortex_bucket_store_cached_postings_original_size_bytes_total counter cortex_bucket_store_cached_postings_original_size_bytes_total 1261064 + # HELP cortex_bucket_store_chunk_refetches_total Total number of cases where configured estimated chunk bytes was not enough was to fetch chunks from object store, resulting in refetch. + # TYPE cortex_bucket_store_chunk_refetches_total counter + cortex_bucket_store_chunk_refetches_total 0 + # HELP cortex_bucket_store_cached_postings_compressed_size_bytes_total Compressed size of postings stored into cache. # TYPE cortex_bucket_store_cached_postings_compressed_size_bytes_total counter cortex_bucket_store_cached_postings_compressed_size_bytes_total 1283583 @@ -634,6 +638,7 @@ type mockedBucketStoreMetrics struct { seriesGetAllDuration prometheus.Histogram seriesMergeDuration prometheus.Histogram seriesRefetches prometheus.Counter + chunkRefetches prometheus.Counter resultSeriesCount prometheus.Histogram chunkSizeBytes prometheus.Histogram postingsSizeBytes prometheus.Histogram @@ -748,6 +753,10 @@ func newMockedBucketStoreMetrics(reg prometheus.Registerer) *mockedBucketStoreMe Name: "thanos_bucket_store_series_refetches_total", Help: fmt.Sprintf("Total number of cases where %v bytes was not enough was to fetch series from index, resulting in refetch.", 64*1024), }) + m.chunkRefetches = promauto.With(reg).NewCounter(prometheus.CounterOpts{ + Name: "thanos_bucket_store_chunk_refetches_total", + Help: "Total number of cases where configured estimated chunk bytes was not enough was to fetch chunks from object store, resulting in refetch", + }) m.cachedPostingsCompressions = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ Name: "thanos_bucket_store_cached_postings_compressions_total", From 1c82c36fccc45f236b7cbf8b8b96f83972312a3f Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 24 Aug 2023 16:28:57 -0700 Subject: [PATCH 3/3] update changelog Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 117ac06c8d2..f3367ee6590 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ * [ENHANCEMENT] Querier: Retry store gateway on different zones when zone awareness is enabled. #5476 * [ENHANCEMENT] DDBKV: Change metric name from dynamodb_kv_read_capacity_total to dynamodb_kv_consumed_capacity_total and include Delete, Put, Batch dimension. #5481 * [ENHANCEMENT] Compactor: allow unregisteronshutdown to be configurable. #5503 +* [ENHANCEMENT] Store Gateway: add metric `cortex_bucket_store_chunk_refetches_total` for number of chunk refetches. #5532 * [BUGFIX] Ruler: Validate if rule group can be safely converted back to rule group yaml from protobuf message #5265 * [BUGFIX] Querier: Convert gRPC `ResourceExhausted` status code from store gateway to 422 limit error. #5286 * [BUGFIX] Alertmanager: Route web-ui requests to the alertmanager distributor when sharding is enabled. #5293