Expose TSDB checkpoint metrics from ingesters (#2589)

* Expose TSDB checkpoint metrics from ingesters

Signed-off-by: Marco Pracucci <[email protected]>

* Updated CHANGELOG

Signed-off-by: Marco Pracucci <[email protected]>

* Renamed checkpoint metrics

Signed-off-by: Marco Pracucci <[email protected]>

Author: pracucci

CHANGELOG.md

@@ -56,7 +56,7 @@
 * [ENHANCEMENT] Redis Cache: Added `idle_timeout`, `wait_on_pool_exhaustion` and `max_conn_lifetime` options to redis cache configuration. #2550
 * [ENHANCEMENT] WAL: the experimental tag has been removed on the WAL in ingesters.
 * [ENHANCEMENT] Use newer AWS API for paginated queries - removes 'Deprecated' message from logfiles. #2452
-* [ENHANCEMENT] Experimental TSDB: added the following metrics to the ingester: #2580 #2583
+* [ENHANCEMENT] Experimental TSDB: added the following metrics to the ingester: #2580 #2583 #2589
   * `cortex_ingester_tsdb_appender_add_duration_seconds`
   * `cortex_ingester_tsdb_appender_commit_duration_seconds`
   * `cortex_ingester_tsdb_refcache_purge_duration_seconds`
@@ -68,6 +68,10 @@
   * `cortex_ingester_tsdb_wal_truncations_failed_total`
   * `cortex_ingester_tsdb_wal_truncations_total`
   * `cortex_ingester_tsdb_wal_writes_failed_total`
+  * `cortex_ingester_tsdb_checkpoint_deletions_failed_total`
+  * `cortex_ingester_tsdb_checkpoint_deletions_total`
+  * `cortex_ingester_tsdb_checkpoint_creations_failed_total`
+  * `cortex_ingester_tsdb_checkpoint_creations_total`
 * [ENHANCEMENT] Experimental TSDB: added metrics useful to alert on critical conditions of the blocks storage: #2573
   * `cortex_compactor_last_successful_run_timestamp_seconds`
   * `cortex_querier_blocks_last_successful_sync_timestamp_seconds` (when store-gateway is disabled)

pkg/ingester/metrics.go

@@ -225,14 +225,18 @@ type tsdbMetrics struct {
 	uploadFailures  *prometheus.Desc // sum(thanos_shipper_upload_failures_total)
 
 	// Metrics aggregated from TSDB.
-	tsdbCompactionsTotal   *prometheus.Desc
-	tsdbCompactionDuration *prometheus.Desc
-	tsdbFsyncDuration      *prometheus.Desc
-	tsdbPageFlushes        *prometheus.Desc
-	tsdbPageCompletions    *prometheus.Desc
-	tsdbTruncateFail       *prometheus.Desc
-	tsdbTruncateTotal      *prometheus.Desc
-	tsdbWritesFailed       *prometheus.Desc
+	tsdbCompactionsTotal    *prometheus.Desc
+	tsdbCompactionDuration  *prometheus.Desc
+	tsdbFsyncDuration       *prometheus.Desc
+	tsdbPageFlushes         *prometheus.Desc
+	tsdbPageCompletions     *prometheus.Desc
+	tsdbTruncateFail        *prometheus.Desc
+	tsdbTruncateTotal       *prometheus.Desc
+	tsdbWritesFailed        *prometheus.Desc
+	checkpointDeleteFail    *prometheus.Desc
+	checkpointDeleteTotal   *prometheus.Desc
+	checkpointCreationFail  *prometheus.Desc
+	checkpointCreationTotal *prometheus.Desc
 
 	// These two metrics replace metrics in ingesterMetrics, as we count them differently
 	memSeriesCreatedTotal *prometheus.Desc
@@ -248,19 +252,19 @@ func newTSDBMetrics(r prometheus.Registerer) *tsdbMetrics {
 
 		dirSyncs: prometheus.NewDesc(
 			"cortex_ingester_shipper_dir_syncs_total",
-			"TSDB: Total number of dir syncs",
+			"Total number of TSDB dir syncs",
 			nil, nil),
 		dirSyncFailures: prometheus.NewDesc(
 			"cortex_ingester_shipper_dir_sync_failures_total",
-			"TSDB: Total number of failed dir syncs",
+			"Total number of failed TSDB dir syncs",
 			nil, nil),
 		uploads: prometheus.NewDesc(
 			"cortex_ingester_shipper_uploads_total",
-			"TSDB: Total number of uploaded blocks",
+			"Total number of uploaded TSDB blocks",
 			nil, nil),
 		uploadFailures: prometheus.NewDesc(
 			"cortex_ingester_shipper_upload_failures_total",
-			"TSDB: Total number of block upload failures",
+			"Total number of TSDB block upload failures",
 			nil, nil),
 		tsdbCompactionsTotal: prometheus.NewDesc(
 			"cortex_ingester_tsdb_compactions_total",
@@ -294,6 +298,22 @@ func newTSDBMetrics(r prometheus.Registerer) *tsdbMetrics {
 			"cortex_ingester_tsdb_wal_writes_failed_total",
 			"Total number of TSDB WAL writes that failed.",
 			nil, nil),
+		checkpointDeleteFail: prometheus.NewDesc(
+			"cortex_ingester_tsdb_checkpoint_deletions_failed_total",
+			"Total number of TSDB checkpoint deletions that failed.",
+			nil, nil),
+		checkpointDeleteTotal: prometheus.NewDesc(
+			"cortex_ingester_tsdb_checkpoint_deletions_total",
+			"Total number of TSDB checkpoint deletions attempted.",
+			nil, nil),
+		checkpointCreationFail: prometheus.NewDesc(
+			"cortex_ingester_tsdb_checkpoint_creations_failed_total",
+			"Total number of TSDB checkpoint creations that failed.",
+			nil, nil),
+		checkpointCreationTotal: prometheus.NewDesc(
+			"cortex_ingester_tsdb_checkpoint_creations_total",
+			"Total number of TSDB checkpoint creations attempted.",
+			nil, nil),
 
 		memSeriesCreatedTotal: prometheus.NewDesc(memSeriesCreatedTotalName, memSeriesCreatedTotalHelp, []string{"user"}, nil),
 		memSeriesRemovedTotal: prometheus.NewDesc(memSeriesRemovedTotalName, memSeriesRemovedTotalHelp, []string{"user"}, nil),
@@ -319,6 +339,10 @@ func (sm *tsdbMetrics) Describe(out chan<- *prometheus.Desc) {
 	out <- sm.tsdbTruncateFail
 	out <- sm.tsdbTruncateTotal
 	out <- sm.tsdbWritesFailed
+	out <- sm.checkpointDeleteFail
+	out <- sm.checkpointDeleteTotal
+	out <- sm.checkpointCreationFail
+	out <- sm.checkpointCreationTotal
 
 	out <- sm.memSeriesCreatedTotal
 	out <- sm.memSeriesRemovedTotal
@@ -341,6 +365,10 @@ func (sm *tsdbMetrics) Collect(out chan<- prometheus.Metric) {
 	data.SendSumOfCounters(out, sm.tsdbTruncateFail, "prometheus_tsdb_wal_truncations_failed_total")
 	data.SendSumOfCounters(out, sm.tsdbTruncateTotal, "prometheus_tsdb_wal_truncations_total")
 	data.SendSumOfCounters(out, sm.tsdbWritesFailed, "prometheus_tsdb_wal_writes_failed_total")
+	data.SendSumOfCounters(out, sm.checkpointDeleteFail, "prometheus_tsdb_checkpoint_deletions_failed_total")
+	data.SendSumOfCounters(out, sm.checkpointDeleteTotal, "prometheus_tsdb_checkpoint_deletions_total")
+	data.SendSumOfCounters(out, sm.checkpointCreationFail, "prometheus_tsdb_checkpoint_creations_failed_total")
+	data.SendSumOfCounters(out, sm.checkpointCreationTotal, "prometheus_tsdb_checkpoint_creations_total")
 
 	data.SendSumOfCountersPerUser(out, sm.memSeriesCreatedTotal, "prometheus_tsdb_head_series_created_total")
 	data.SendSumOfCountersPerUser(out, sm.memSeriesRemovedTotal, "prometheus_tsdb_head_series_removed_total")

pkg/ingester/metrics_test.go

@@ -20,22 +20,22 @@ func TestTSDBMetrics(t *testing.T) {
 	tsdbMetrics.setRegistryForUser("user3", populateTSDBMetrics(999))
 
 	err := testutil.GatherAndCompare(mainReg, bytes.NewBufferString(`
-			# HELP cortex_ingester_shipper_dir_syncs_total TSDB: Total number of dir syncs
+			# HELP cortex_ingester_shipper_dir_syncs_total Total number of TSDB dir syncs
 			# TYPE cortex_ingester_shipper_dir_syncs_total counter
 			# 12345 + 85787 + 999
 			cortex_ingester_shipper_dir_syncs_total 99131
 
-			# HELP cortex_ingester_shipper_dir_sync_failures_total TSDB: Total number of failed dir syncs
+			# HELP cortex_ingester_shipper_dir_sync_failures_total Total number of failed TSDB dir syncs
 			# TYPE cortex_ingester_shipper_dir_sync_failures_total counter
 			# 2*(12345 + 85787 + 999)
 			cortex_ingester_shipper_dir_sync_failures_total 198262
 
-			# HELP cortex_ingester_shipper_uploads_total TSDB: Total number of uploaded blocks
+			# HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks
 			# TYPE cortex_ingester_shipper_uploads_total counter
 			# 3*(12345 + 85787 + 999)
 			cortex_ingester_shipper_uploads_total 297393
 
-			# HELP cortex_ingester_shipper_upload_failures_total TSDB: Total number of block upload failures
+			# HELP cortex_ingester_shipper_upload_failures_total Total number of TSDB block upload failures
 			# TYPE cortex_ingester_shipper_upload_failures_total counter
 			# 4*(12345 + 85787 + 999)
 			cortex_ingester_shipper_upload_failures_total 396524
@@ -88,6 +88,22 @@ func TestTSDBMetrics(t *testing.T) {
 			# TYPE cortex_ingester_tsdb_wal_writes_failed_total counter
 			cortex_ingester_tsdb_wal_writes_failed_total 1486965
 
+			# HELP cortex_ingester_tsdb_checkpoint_deletions_failed_total Total number of TSDB checkpoint deletions that failed.
+			# TYPE cortex_ingester_tsdb_checkpoint_deletions_failed_total counter
+			cortex_ingester_tsdb_checkpoint_deletions_failed_total 1586096
+
+			# HELP cortex_ingester_tsdb_checkpoint_deletions_total Total number of TSDB checkpoint deletions attempted.
+			# TYPE cortex_ingester_tsdb_checkpoint_deletions_total counter
+			cortex_ingester_tsdb_checkpoint_deletions_total 1685227
+
+			# HELP cortex_ingester_tsdb_checkpoint_creations_failed_total Total number of TSDB checkpoint creations that failed.
+			# TYPE cortex_ingester_tsdb_checkpoint_creations_failed_total counter
+			cortex_ingester_tsdb_checkpoint_creations_failed_total 1784358
+
+			# HELP cortex_ingester_tsdb_checkpoint_creations_total Total number of TSDB checkpoint creations attempted.
+			# TYPE cortex_ingester_tsdb_checkpoint_creations_total counter
+			cortex_ingester_tsdb_checkpoint_creations_total 1883489
+
 			# HELP cortex_ingester_memory_series_created_total The total number of series that were created per user.
 			# TYPE cortex_ingester_memory_series_created_total counter
 			# 5 * (12345, 85787 and 999 respectively)
@@ -194,5 +210,29 @@ func populateTSDBMetrics(base float64) *prometheus.Registry {
 	})
 	writesFailed.Add(15 * base)
 
+	checkpointDeleteFail := promauto.With(r).NewCounter(prometheus.CounterOpts{
+		Name: "prometheus_tsdb_checkpoint_deletions_failed_total",
+		Help: "Total number of checkpoint deletions that failed.",
+	})
+	checkpointDeleteFail.Add(16 * base)
+
+	checkpointDeleteTotal := promauto.With(r).NewCounter(prometheus.CounterOpts{
+		Name: "prometheus_tsdb_checkpoint_deletions_total",
+		Help: "Total number of checkpoint deletions attempted.",
+	})
+	checkpointDeleteTotal.Add(17 * base)
+
+	checkpointCreationFail := promauto.With(r).NewCounter(prometheus.CounterOpts{
+		Name: "prometheus_tsdb_checkpoint_creations_failed_total",
+		Help: "Total number of checkpoint creations that failed.",
+	})
+	checkpointCreationFail.Add(18 * base)
+
+	checkpointCreationTotal := promauto.With(r).NewCounter(prometheus.CounterOpts{
+		Name: "prometheus_tsdb_checkpoint_creations_total",
+		Help: "Total number of checkpoint creations attempted.",
+	})
+	checkpointCreationTotal.Add(19 * base)
+
 	return r
 }