Experimental TSDB: Add support for local filesystem backend (#2245)

* Experimental TSDB: Add support for local filesystem object store

Signed-off-by: Edward Welch <[email protected]>

* Updated doc

Signed-off-by: Marco Pracucci <[email protected]>

* Renamed blocks storage filesystem directory config option name to keep it more consistent with the rest of the config

Signed-off-by: Marco Pracucci <[email protected]>

* Removed unused params from filesystem.NewBucketClient()

Signed-off-by: Marco Pracucci <[email protected]>

Co-authored-by: Marco Pracucci <[email protected]>

Author: slim-bean

CHANGELOG.md

@@ -8,6 +8,7 @@
   * `-flusher.wal-dir` for the WAL directory to recover from.
   * `-flusher.concurrent-flushes` for number of concurrent flushes.
   * `-flusher.flush-op-timeout` is duration after which a flush should timeout.
+* [ENHANCEMENT] Experimental TSDB: Add support for local `filesystem` backend. #2245
 
 ## 0.7.0-rc.0 / 2020-03-09
 

docs/architecture.md

@@ -61,6 +61,7 @@ The blocks storage doesn't require a dedicated storage backend for the index. Th
 * [Amazon S3](https://aws.amazon.com/s3)
 * [Google Cloud Storage](https://cloud.google.com/storage/)
 * [Microsoft Azure Storage](https://azure.microsoft.com/en-us/services/storage/)
+* [Local Filesystem](https://thanos.io/storage.md/#filesystem) (single node only)
 
 For more information, please check out the [Blocks storage](operations/blocks-storage.md) documentation.
 

docs/configuration/config-file-reference.md

@@ -2164,7 +2164,7 @@ The `tsdb_config` configures the experimental blocks storage.
 # CLI flag: -experimental.tsdb.ship-concurrency
 [ship_concurrency: <int> | default = 10]
 
-# Backend storage to use. Either "s3" or "gcs".
+# Backend storage to use. Supported backends are: s3, gcs, azure, filesystem.
 # CLI flag: -experimental.tsdb.backend
 [backend: <string> | default = "s3"]
 
@@ -2295,6 +2295,11 @@ azure:
   # Number of retries for recoverable errors
   # CLI flag: -experimental.tsdb.azure.max-retries
   [max_retries: <int> | default = 20]
+
+filesystem:
+  # Local filesystem storage directory.
+  # CLI flag: -experimental.tsdb.filesystem.dir
+  [dir: <string> | default = ""]
 ```
 
 ### `compactor_config`

docs/operations/blocks-storage.md

@@ -12,6 +12,7 @@ The supported backends for the blocks storage are:
 * [Amazon S3](https://aws.amazon.com/s3)
 * [Google Cloud Storage](https://cloud.google.com/storage/)
 * [Microsoft Azure Storage](https://azure.microsoft.com/en-us/services/storage/)
+* [Local Filesystem](https://thanos.io/storage.md/#filesystem) (single node only)
 
 _Internally, this storage engine is based on [Thanos](https://thanos.io), but no Thanos knowledge is required in order to run it._
 
@@ -119,7 +120,7 @@ tsdb:
   # CLI flag: -experimental.tsdb.ship-concurrency
   [ship_concurrency: <int> | default = 10]
 
-  # Backend storage to use. Either "s3" or "gcs".
+  # Backend storage to use. Supported backends are: s3, gcs, azure, filesystem.
   # CLI flag: -experimental.tsdb.backend
   [backend: <string> | default = "s3"]
 
@@ -252,6 +253,11 @@ tsdb:
     # Number of retries for recoverable errors
     # CLI flag: -experimental.tsdb.azure.max-retries
     [max_retries: <int> | default = 20]
+
+  filesystem:
+    # Local filesystem storage directory.
+    # CLI flag: -experimental.tsdb.filesystem.dir
+    [dir: <string> | default = ""]
 ```
 ### `compactor_config`
 

docs/operations/blocks-storage.template

@@ -12,6 +12,7 @@ The supported backends for the blocks storage are:
 * [Amazon S3](https://aws.amazon.com/s3)
 * [Google Cloud Storage](https://cloud.google.com/storage/)
 * [Microsoft Azure Storage](https://azure.microsoft.com/en-us/services/storage/)
+* [Local Filesystem](https://thanos.io/storage.md/#filesystem) (single node only)
 
 _Internally, this storage engine is based on [Thanos](https://thanos.io), but no Thanos knowledge is required in order to run it._
 

pkg/storage/tsdb/backend/filesystem/bucket_client.go

@@ -0,0 +1,11 @@
+package filesystem
+
+import (
+	"github.com/thanos-io/thanos/pkg/objstore"
+	"github.com/thanos-io/thanos/pkg/objstore/filesystem"
+)
+
+// NewBucketClient creates a new filesystem bucket client
+func NewBucketClient(cfg Config) (objstore.Bucket, error) {
+	return filesystem.NewBucket(cfg.Directory)
+}

pkg/storage/tsdb/backend/filesystem/config.go

@@ -0,0 +1,13 @@
+package filesystem
+
+import "flag"
+
+// Config stores the configuration for storing and accessing objects in the local filesystem.
+type Config struct {
+	Directory string `yaml:"dir"`
+}
+
+// RegisterFlags registers the flags for TSDB filesystem storage
+func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
+	f.StringVar(&cfg.Directory, "experimental.tsdb.filesystem.dir", "", "Local filesystem storage directory.")
+}

pkg/storage/tsdb/bucket_client.go

@@ -7,6 +7,7 @@ import (
 	"github.com/thanos-io/thanos/pkg/objstore"
 
 	"github.com/cortexproject/cortex/pkg/storage/tsdb/backend/azure"
+	"github.com/cortexproject/cortex/pkg/storage/tsdb/backend/filesystem"
 	"github.com/cortexproject/cortex/pkg/storage/tsdb/backend/gcs"
 	"github.com/cortexproject/cortex/pkg/storage/tsdb/backend/s3"
 )
@@ -20,6 +21,8 @@ func NewBucketClient(ctx context.Context, cfg Config, name string, logger log.Lo
 		return gcs.NewBucketClient(ctx, cfg.GCS, name, logger)
 	case BackendAzure:
 		return azure.NewBucketClient(cfg.Azure, name, logger)
+	case BackendFilesystem:
+		return filesystem.NewBucketClient(cfg.Filesystem)
 	default:
 		return nil, errUnsupportedBackend
 	}

pkg/storage/tsdb/config.go

@@ -3,13 +3,15 @@ package tsdb
 import (
 	"errors"
 	"flag"
+	"fmt"
 	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/alecthomas/units"
 
 	"github.com/cortexproject/cortex/pkg/storage/tsdb/backend/azure"
+	"github.com/cortexproject/cortex/pkg/storage/tsdb/backend/filesystem"
 	"github.com/cortexproject/cortex/pkg/storage/tsdb/backend/gcs"
 	"github.com/cortexproject/cortex/pkg/storage/tsdb/backend/s3"
 )
@@ -24,12 +26,17 @@ const (
 	// BackendAzure is the value for the Azure storage backend
 	BackendAzure = "azure"
 
+	// BackendFilesystem is the value for the filesystem storge backend
+	BackendFilesystem = "filesystem"
+
 	// TenantIDExternalLabel is the external label set when shipping blocks to the storage
 	TenantIDExternalLabel = "__org_id__"
 )
 
 // Validation errors
 var (
+	supportedBackends = []string{BackendS3, BackendGCS, BackendAzure, BackendFilesystem}
+
 	errUnsupportedBackend           = errors.New("unsupported TSDB storage backend")
 	errInvalidShipConcurrency       = errors.New("invalid TSDB ship concurrency")
 	errInvalidCompactionInterval    = errors.New("invalid TSDB compaction interval")
@@ -54,9 +61,10 @@ type Config struct {
 	MaxTSDBOpeningConcurrencyOnStartup int `yaml:"max_tsdb_opening_concurrency_on_startup"`
 
 	// Backends
-	S3    s3.Config    `yaml:"s3"`
-	GCS   gcs.Config   `yaml:"gcs"`
-	Azure azure.Config `yaml:"azure"`
+	S3         s3.Config         `yaml:"s3"`
+	GCS        gcs.Config        `yaml:"gcs"`
+	Azure      azure.Config      `yaml:"azure"`
+	Filesystem filesystem.Config `yaml:"filesystem"`
 }
 
 // DurationList is the block ranges for a tsdb
@@ -102,6 +110,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	cfg.GCS.RegisterFlags(f)
 	cfg.Azure.RegisterFlags(f)
 	cfg.BucketStore.RegisterFlags(f)
+	cfg.Filesystem.RegisterFlags(f)
 
 	if len(cfg.BlockRanges) == 0 {
 		cfg.BlockRanges = []time.Duration{2 * time.Hour} // Default 2h block
@@ -112,7 +121,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.DurationVar(&cfg.Retention, "experimental.tsdb.retention-period", 6*time.Hour, "TSDB blocks retention in the ingester before a block is removed. This should be larger than the block_ranges_period and large enough to give queriers enough time to discover newly uploaded blocks.")
 	f.DurationVar(&cfg.ShipInterval, "experimental.tsdb.ship-interval", 1*time.Minute, "How frequently the TSDB blocks are scanned and new ones are shipped to the storage. 0 means shipping is disabled.")
 	f.IntVar(&cfg.ShipConcurrency, "experimental.tsdb.ship-concurrency", 10, "Maximum number of tenants concurrently shipping blocks to the storage.")
-	f.StringVar(&cfg.Backend, "experimental.tsdb.backend", "s3", `Backend storage to use. Either "s3" or "gcs".`)
+	f.StringVar(&cfg.Backend, "experimental.tsdb.backend", "s3", fmt.Sprintf("Backend storage to use. Supported backends are: %s.", strings.Join(supportedBackends, ", ")))
 	f.IntVar(&cfg.MaxTSDBOpeningConcurrencyOnStartup, "experimental.tsdb.max-tsdb-opening-concurrency-on-startup", 10, "limit the number of concurrently opening TSDB's on startup")
 	f.DurationVar(&cfg.HeadCompactionInterval, "experimental.tsdb.head-compaction-interval", 1*time.Minute, "How frequently does Cortex try to compact TSDB head. Block is only created if data covers smallest block range. Must be greater than 0 and max 5 minutes.")
 	f.IntVar(&cfg.HeadCompactionConcurrency, "experimental.tsdb.head-compaction-concurrency", 5, "Maximum number of tenants concurrently compacting TSDB head into a new block")
@@ -121,7 +130,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 
 // Validate the config
 func (cfg *Config) Validate() error {
-	if cfg.Backend != BackendS3 && cfg.Backend != BackendGCS && cfg.Backend != BackendAzure {
+	if cfg.Backend != BackendS3 && cfg.Backend != BackendGCS && cfg.Backend != BackendAzure && cfg.Backend != BackendFilesystem {
 		return errUnsupportedBackend
 	}