Add a flag to enable TSDB memorry snapshot on shutdown

CHANGELOG.md

@@ -8,6 +8,7 @@
 * [FEATURE] Ingester: Add active series to all_user_stats page. #4972
 * [FEATURE] Ingester: Added `-blocks-storage.tsdb.head-chunks-write-queue-size` allowing to configure the size of the in-memory queue used before flushing chunks to the disk . #5000
 * [FEATURE] Query Frontend: Log query params in query frontend even if error happens. #5005
+* [FEATURE] Ingester: Enable snapshotting of In-memory TSDB on disk during shutdown via `-blocks-storage.tsdb.memory-snapshot-on-shutdown`. #5011
 * [BUGFIX] Updated `golang.org/x/net` dependency to fix CVE-2022-27664. #5008
 
 ## 1.14.0 in progress

docs/blocks-storage/querier.md

@@ -882,4 +882,9 @@ blocks_storage:
     # will be stored. 0 or less means disabled.
     # CLI flag: -blocks-storage.tsdb.max-exemplars
     [max_exemplars: <int> | default = 0]
+
+    # True to enable snapshotting of in-memory TSDB data on disk when shutting
+    # down.
+    # CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown
+    [memory_snapshot_on_shutdown: <boolean> | default = false]
 ```

docs/blocks-storage/store-gateway.md

@@ -946,4 +946,9 @@ blocks_storage:
     # will be stored. 0 or less means disabled.
     # CLI flag: -blocks-storage.tsdb.max-exemplars
     [max_exemplars: <int> | default = 0]
+
+    # True to enable snapshotting of in-memory TSDB data on disk when shutting
+    # down.
+    # CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown
+    [memory_snapshot_on_shutdown: <boolean> | default = false]
 ```

docs/configuration/config-file-reference.md

@@ -3725,6 +3725,11 @@ tsdb:
   # be stored. 0 or less means disabled.
   # CLI flag: -blocks-storage.tsdb.max-exemplars
   [max_exemplars: <int> | default = 0]
+
+  # True to enable snapshotting of in-memory TSDB data on disk when shutting
+  # down.
+  # CLI flag: -blocks-storage.tsdb.memory-snapshot-on-shutdown
+  [memory_snapshot_on_shutdown: <boolean> | default = false]
 ```
 
 ### `compactor_config`

docs/configuration/v1-guarantees.md

@@ -98,4 +98,6 @@ Currently experimental features are:
   - Enabled via `-compactor.sharding-enabled=true`, `-compactor.sharding-strategy=shuffle-sharding`, and `-compactor.tenant-shard-size` set to a value larger than 0.
 - Vertical sharding at query frontend for range/instant queries
   - `-frontend.query-vertical-shard-size` (int) CLI flag
-  - `query_vertical_shard_size` (int) field in runtime config file
+  - `query_vertical_shard_size` (int) field in runtime config file
+- Snapshotting of in-memory TSDB on disk during shutdown
+  - `-blocks-storage.tsdb.memory-snapshot-on-shutdown` (boolean) CLI flag

pkg/ingester/ingester.go

@@ -1833,20 +1833,21 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) {
 	}
 	// Create a new user database
 	db, err := tsdb.Open(udir, userLogger, tsdbPromReg, &tsdb.Options{
-		RetentionDuration:         i.cfg.BlocksStorageConfig.TSDB.Retention.Milliseconds(),
-		MinBlockDuration:          blockRanges[0],
-		MaxBlockDuration:          blockRanges[len(blockRanges)-1],
-		NoLockfile:                true,
-		StripeSize:                i.cfg.BlocksStorageConfig.TSDB.StripeSize,
-		HeadChunksWriteBufferSize: i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteBufferSize,
-		WALCompression:            i.cfg.BlocksStorageConfig.TSDB.WALCompressionEnabled,
-		WALSegmentSize:            i.cfg.BlocksStorageConfig.TSDB.WALSegmentSizeBytes,
-		SeriesLifecycleCallback:   userDB,
-		BlocksToDelete:            userDB.blocksToDelete,
-		EnableExemplarStorage:     enableExemplars,
-		IsolationDisabled:         true,
-		MaxExemplars:              int64(i.cfg.BlocksStorageConfig.TSDB.MaxExemplars),
-		HeadChunksWriteQueueSize:  i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteQueueSize,
+		RetentionDuration:              i.cfg.BlocksStorageConfig.TSDB.Retention.Milliseconds(),
+		MinBlockDuration:               blockRanges[0],
+		MaxBlockDuration:               blockRanges[len(blockRanges)-1],
+		NoLockfile:                     true,
+		StripeSize:                     i.cfg.BlocksStorageConfig.TSDB.StripeSize,
+		HeadChunksWriteBufferSize:      i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteBufferSize,
+		WALCompression:                 i.cfg.BlocksStorageConfig.TSDB.WALCompressionEnabled,
+		WALSegmentSize:                 i.cfg.BlocksStorageConfig.TSDB.WALSegmentSizeBytes,
+		SeriesLifecycleCallback:        userDB,
+		BlocksToDelete:                 userDB.blocksToDelete,
+		EnableExemplarStorage:          enableExemplars,
+		IsolationDisabled:              true,
+		MaxExemplars:                   int64(i.cfg.BlocksStorageConfig.TSDB.MaxExemplars),
+		HeadChunksWriteQueueSize:       i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteQueueSize,
+		EnableMemorySnapshotOnShutdown: i.cfg.BlocksStorageConfig.TSDB.MemorySnapshotOnShutdown,
 	}, nil)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to open TSDB: %s", udir)

pkg/storage/tsdb/config.go

@@ -151,6 +151,9 @@ type TSDBConfig struct {
 
 	// Positive value enables experiemental support for exemplars. 0 or less to disable.
 	MaxExemplars int `yaml:"max_exemplars"`
+
+	// Enable snapshotting of in-memory TSDB data on disk when shutting down.
+	MemorySnapshotOnShutdown bool `yaml:"memory_snapshot_on_shutdown"`
 }
 
 // RegisterFlags registers the TSDBConfig flags.
@@ -176,6 +179,7 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) {
 	f.DurationVar(&cfg.CloseIdleTSDBTimeout, "blocks-storage.tsdb.close-idle-tsdb-timeout", 0, "If TSDB has not received any data for this duration, and all blocks from TSDB have been shipped, TSDB is closed and deleted from local disk. If set to positive value, this value should be equal or higher than -querier.query-ingesters-within flag to make sure that TSDB is not closed prematurely, which could cause partial query results. 0 or negative value disables closing of idle TSDB.")
 	f.IntVar(&cfg.MaxExemplars, "blocks-storage.tsdb.max-exemplars", 0, "Enables support for exemplars in TSDB and sets the maximum number that will be stored. 0 or less means disabled.")
 	f.IntVar(&cfg.HeadChunksWriteQueueSize, "blocks-storage.tsdb.head-chunks-write-queue-size", chunks.DefaultWriteQueueSize, "The size of the in-memory queue used before flushing chunks to the disk.")
+	f.BoolVar(&cfg.MemorySnapshotOnShutdown, "blocks-storage.tsdb.memory-snapshot-on-shutdown", false, "True to enable snapshotting of in-memory TSDB data on disk when shutting down.")
 }
 
 // Validate the config.