From 9a1a8ce7c990250e8b69c20b4f17c59f8de5809e Mon Sep 17 00:00:00 2001 From: alanprot Date: Mon, 23 Dec 2024 11:38:25 -0800 Subject: [PATCH 1/2] Create metric Signed-off-by: alanprot --- pkg/storage/tsdb/expanded_postings_cache.go | 8 +++++++- pkg/storage/tsdb/expanded_postings_cache_test.go | 13 +++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/pkg/storage/tsdb/expanded_postings_cache.go b/pkg/storage/tsdb/expanded_postings_cache.go index 57027cf2a42..aeae9fb37ea 100644 --- a/pkg/storage/tsdb/expanded_postings_cache.go +++ b/pkg/storage/tsdb/expanded_postings_cache.go @@ -40,6 +40,7 @@ type ExpandedPostingsCacheMetrics struct { CacheRequests *prometheus.CounterVec CacheHits *prometheus.CounterVec CacheEvicts *prometheus.CounterVec + CacheMiss *prometheus.CounterVec NonCacheableQueries *prometheus.CounterVec } @@ -53,6 +54,10 @@ func NewPostingCacheMetrics(r prometheus.Registerer) *ExpandedPostingsCacheMetri Name: "cortex_ingester_expanded_postings_cache_hits", Help: "Total number of hit requests to the cache.", }, []string{"cache"}), + CacheMiss: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Name: "cortex_ingester_expanded_postings_cache_miss", + Help: "Total number of miss requests to the cache.", + }, []string{"cache", "reason"}), CacheEvicts: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ Name: "cortex_ingester_expanded_postings_cache_evicts", Help: "Total number of evictions in the cache, excluding items that got evicted due to TTL.", @@ -374,6 +379,7 @@ func (c *fifoCache[V]) getPromiseForKey(k string, fetch func() (V, int64, error) loaded, ok := c.cachedValues.LoadOrStore(k, r) if !ok { + c.metrics.CacheMiss.WithLabelValues(c.name, "miss").Inc() r.v, r.sizeBytes, r.err = fetch() r.sizeBytes += int64(len(k)) r.ts = c.timeNow() @@ -387,7 +393,7 @@ func (c *fifoCache[V]) getPromiseForKey(k string, fetch func() (V, int64, error) // If is cached but is expired, lets try to replace the cache value. if loaded.(*cacheEntryPromise[V]).isExpired(c.cfg.Ttl, c.timeNow()) && c.cachedValues.CompareAndSwap(k, loaded, r) { - c.metrics.CacheEvicts.WithLabelValues(c.name, "expired").Inc() + c.metrics.CacheMiss.WithLabelValues(c.name, "expired").Inc() r.v, r.sizeBytes, r.err = fetch() r.sizeBytes += int64(len(k)) c.updateSize(loaded.(*cacheEntryPromise[V]).sizeBytes, r.sizeBytes) diff --git a/pkg/storage/tsdb/expanded_postings_cache_test.go b/pkg/storage/tsdb/expanded_postings_cache_test.go index 3517ec6e9ca..7909ce0ff24 100644 --- a/pkg/storage/tsdb/expanded_postings_cache_test.go +++ b/pkg/storage/tsdb/expanded_postings_cache_test.go @@ -181,10 +181,11 @@ func TestFifoCacheExpire(t *testing.T) { } err := testutil.GatherAndCompare(r, bytes.NewBufferString(fmt.Sprintf(` - # HELP cortex_ingester_expanded_postings_cache_evicts Total number of evictions in the cache, excluding items that got evicted due to TTL. - # TYPE cortex_ingester_expanded_postings_cache_evicts counter - cortex_ingester_expanded_postings_cache_evicts{cache="test",reason="expired"} %v -`, numberOfKeys)), "cortex_ingester_expanded_postings_cache_evicts") + # HELP cortex_ingester_expanded_postings_cache_miss Total number of miss requests to the cache. + # TYPE cortex_ingester_expanded_postings_cache_miss counter + cortex_ingester_expanded_postings_cache_miss{cache="test",reason="expired"} %v + cortex_ingester_expanded_postings_cache_miss{cache="test",reason="miss"} %v +`, numberOfKeys, numberOfKeys)), "cortex_ingester_expanded_postings_cache_miss") require.NoError(t, err) cache.timeNow = func() time.Time { @@ -195,12 +196,12 @@ func TestFifoCacheExpire(t *testing.T) { return 2, 18, nil }) - // Should expire all keys again as ttl is expired + // Should expire all keys expired keys err = testutil.GatherAndCompare(r, bytes.NewBufferString(fmt.Sprintf(` # HELP cortex_ingester_expanded_postings_cache_evicts Total number of evictions in the cache, excluding items that got evicted due to TTL. # TYPE cortex_ingester_expanded_postings_cache_evicts counter cortex_ingester_expanded_postings_cache_evicts{cache="test",reason="expired"} %v -`, numberOfKeys*2)), "cortex_ingester_expanded_postings_cache_evicts") +`, numberOfKeys)), "cortex_ingester_expanded_postings_cache_evicts") require.NoError(t, err) } }) From 5fe7db781378faeef4d7bad4b21e22ab3fdc3479 Mon Sep 17 00:00:00 2001 From: alanprot Date: Mon, 23 Dec 2024 11:53:33 -0800 Subject: [PATCH 2/2] Adding total on all metric Signed-off-by: alanprot --- pkg/ingester/ingester_test.go | 28 +++++++++---------- pkg/storage/tsdb/expanded_postings_cache.go | 10 +++---- .../tsdb/expanded_postings_cache_test.go | 26 ++++++++--------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index d4d05fef9a0..ad84c57be4c 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5553,22 +5553,22 @@ func TestExpendedPostingsCache(t *testing.T) { if c.expectedHeadPostingCall > 0 || c.expectedBlockPostingCall > 0 { metric := ` - # HELP cortex_ingester_expanded_postings_cache_requests Total number of requests to the cache. - # TYPE cortex_ingester_expanded_postings_cache_requests counter + # HELP cortex_ingester_expanded_postings_cache_requests_total Total number of requests to the cache. + # TYPE cortex_ingester_expanded_postings_cache_requests_total counter ` if c.expectedBlockPostingCall > 0 { metric += ` - cortex_ingester_expanded_postings_cache_requests{cache="block"} 4 + cortex_ingester_expanded_postings_cache_requests_total{cache="block"} 4 ` } if c.expectedHeadPostingCall > 0 { metric += ` - cortex_ingester_expanded_postings_cache_requests{cache="head"} 4 + cortex_ingester_expanded_postings_cache_requests_total{cache="head"} 4 ` } - err = testutil.GatherAndCompare(r, bytes.NewBufferString(metric), "cortex_ingester_expanded_postings_cache_requests") + err = testutil.GatherAndCompare(r, bytes.NewBufferString(metric), "cortex_ingester_expanded_postings_cache_requests_total") require.NoError(t, err) } @@ -5583,22 +5583,22 @@ func TestExpendedPostingsCache(t *testing.T) { if c.expectedHeadPostingCall > 0 || c.expectedBlockPostingCall > 0 { metric := ` - # HELP cortex_ingester_expanded_postings_cache_hits Total number of hit requests to the cache. - # TYPE cortex_ingester_expanded_postings_cache_hits counter + # HELP cortex_ingester_expanded_postings_cache_hits_total Total number of hit requests to the cache. + # TYPE cortex_ingester_expanded_postings_cache_hits_total counter ` if c.expectedBlockPostingCall > 0 { metric += ` - cortex_ingester_expanded_postings_cache_hits{cache="block"} 4 + cortex_ingester_expanded_postings_cache_hits_total{cache="block"} 4 ` } if c.expectedHeadPostingCall > 0 { metric += ` - cortex_ingester_expanded_postings_cache_hits{cache="head"} 4 + cortex_ingester_expanded_postings_cache_hits_total{cache="head"} 4 ` } - err = testutil.GatherAndCompare(r, bytes.NewBufferString(metric), "cortex_ingester_expanded_postings_cache_hits") + err = testutil.GatherAndCompare(r, bytes.NewBufferString(metric), "cortex_ingester_expanded_postings_cache_hits_total") require.NoError(t, err) } @@ -5644,10 +5644,10 @@ func TestExpendedPostingsCache(t *testing.T) { require.Equal(t, postingsForMatchersCalls.Load(), int64(c.expectedBlockPostingCall)) if c.cacheConfig.Head.Enabled { err = testutil.GatherAndCompare(r, bytes.NewBufferString(` - # HELP cortex_ingester_expanded_postings_non_cacheable_queries Total number of non cacheable queries. - # TYPE cortex_ingester_expanded_postings_non_cacheable_queries counter - cortex_ingester_expanded_postings_non_cacheable_queries{cache="head"} 1 -`), "cortex_ingester_expanded_postings_non_cacheable_queries") + # HELP cortex_ingester_expanded_postings_non_cacheable_queries_total Total number of non cacheable queries. + # TYPE cortex_ingester_expanded_postings_non_cacheable_queries_total counter + cortex_ingester_expanded_postings_non_cacheable_queries_total{cache="head"} 1 +`), "cortex_ingester_expanded_postings_non_cacheable_queries_total") require.NoError(t, err) } diff --git a/pkg/storage/tsdb/expanded_postings_cache.go b/pkg/storage/tsdb/expanded_postings_cache.go index aeae9fb37ea..3ea8da709ec 100644 --- a/pkg/storage/tsdb/expanded_postings_cache.go +++ b/pkg/storage/tsdb/expanded_postings_cache.go @@ -47,23 +47,23 @@ type ExpandedPostingsCacheMetrics struct { func NewPostingCacheMetrics(r prometheus.Registerer) *ExpandedPostingsCacheMetrics { return &ExpandedPostingsCacheMetrics{ CacheRequests: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Name: "cortex_ingester_expanded_postings_cache_requests", + Name: "cortex_ingester_expanded_postings_cache_requests_total", Help: "Total number of requests to the cache.", }, []string{"cache"}), CacheHits: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Name: "cortex_ingester_expanded_postings_cache_hits", + Name: "cortex_ingester_expanded_postings_cache_hits_total", Help: "Total number of hit requests to the cache.", }, []string{"cache"}), CacheMiss: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Name: "cortex_ingester_expanded_postings_cache_miss", + Name: "cortex_ingester_expanded_postings_cache_miss_total", Help: "Total number of miss requests to the cache.", }, []string{"cache", "reason"}), CacheEvicts: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Name: "cortex_ingester_expanded_postings_cache_evicts", + Name: "cortex_ingester_expanded_postings_cache_evicts_total", Help: "Total number of evictions in the cache, excluding items that got evicted due to TTL.", }, []string{"cache", "reason"}), NonCacheableQueries: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Name: "cortex_ingester_expanded_postings_non_cacheable_queries", + Name: "cortex_ingester_expanded_postings_non_cacheable_queries_total", Help: "Total number of non cacheable queries.", }, []string{"cache"}), } diff --git a/pkg/storage/tsdb/expanded_postings_cache_test.go b/pkg/storage/tsdb/expanded_postings_cache_test.go index 7909ce0ff24..19272c00e72 100644 --- a/pkg/storage/tsdb/expanded_postings_cache_test.go +++ b/pkg/storage/tsdb/expanded_postings_cache_test.go @@ -154,10 +154,10 @@ func TestFifoCacheExpire(t *testing.T) { if c.expectedFinalItems != numberOfKeys { err := testutil.GatherAndCompare(r, bytes.NewBufferString(fmt.Sprintf(` - # HELP cortex_ingester_expanded_postings_cache_evicts Total number of evictions in the cache, excluding items that got evicted due to TTL. - # TYPE cortex_ingester_expanded_postings_cache_evicts counter - cortex_ingester_expanded_postings_cache_evicts{cache="test",reason="full"} %v -`, numberOfKeys-c.expectedFinalItems)), "cortex_ingester_expanded_postings_cache_evicts") + # HELP cortex_ingester_expanded_postings_cache_evicts_total Total number of evictions in the cache, excluding items that got evicted due to TTL. + # TYPE cortex_ingester_expanded_postings_cache_evicts_total counter + cortex_ingester_expanded_postings_cache_evicts_total{cache="test",reason="full"} %v +`, numberOfKeys-c.expectedFinalItems)), "cortex_ingester_expanded_postings_cache_evicts_total") require.NoError(t, err) } @@ -181,11 +181,11 @@ func TestFifoCacheExpire(t *testing.T) { } err := testutil.GatherAndCompare(r, bytes.NewBufferString(fmt.Sprintf(` - # HELP cortex_ingester_expanded_postings_cache_miss Total number of miss requests to the cache. - # TYPE cortex_ingester_expanded_postings_cache_miss counter - cortex_ingester_expanded_postings_cache_miss{cache="test",reason="expired"} %v - cortex_ingester_expanded_postings_cache_miss{cache="test",reason="miss"} %v -`, numberOfKeys, numberOfKeys)), "cortex_ingester_expanded_postings_cache_miss") + # HELP cortex_ingester_expanded_postings_cache_miss_total Total number of miss requests to the cache. + # TYPE cortex_ingester_expanded_postings_cache_miss_total counter + cortex_ingester_expanded_postings_cache_miss_total{cache="test",reason="expired"} %v + cortex_ingester_expanded_postings_cache_miss_total{cache="test",reason="miss"} %v +`, numberOfKeys, numberOfKeys)), "cortex_ingester_expanded_postings_cache_miss_total") require.NoError(t, err) cache.timeNow = func() time.Time { @@ -198,10 +198,10 @@ func TestFifoCacheExpire(t *testing.T) { // Should expire all keys expired keys err = testutil.GatherAndCompare(r, bytes.NewBufferString(fmt.Sprintf(` - # HELP cortex_ingester_expanded_postings_cache_evicts Total number of evictions in the cache, excluding items that got evicted due to TTL. - # TYPE cortex_ingester_expanded_postings_cache_evicts counter - cortex_ingester_expanded_postings_cache_evicts{cache="test",reason="expired"} %v -`, numberOfKeys)), "cortex_ingester_expanded_postings_cache_evicts") + # HELP cortex_ingester_expanded_postings_cache_evicts_total Total number of evictions in the cache, excluding items that got evicted due to TTL. + # TYPE cortex_ingester_expanded_postings_cache_evicts_total counter + cortex_ingester_expanded_postings_cache_evicts_total{cache="test",reason="expired"} %v +`, numberOfKeys)), "cortex_ingester_expanded_postings_cache_evicts_total") require.NoError(t, err) } })