Update Thanos version to master (#2335)

* Updated Thanos to master
Signed-off-by: Peter Štibraný <[email protected]>

* Integrate Compactor and Store changes

- Deletion delay, blocks cleaner
- Postings compression

Signed-off-by: Peter Štibraný <[email protected]>

* Expose max item size for memcached client.

Signed-off-by: Peter Štibraný <[email protected]>

* Added new metrics related to postings compression to block_store_metrics

Signed-off-by: Peter Štibraný <[email protected]>

* Lint

Signed-off-by: Peter Štibraný <[email protected]>

* Lint

Signed-off-by: Peter Štibraný <[email protected]>

* CHANGELOG.md, documentation

Signed-off-by: Peter Štibraný <[email protected]>

* Added prefix.

Signed-off-by: Peter Štibraný <[email protected]>

* Removed unused option.

Signed-off-by: Peter Štibraný <[email protected]>

* Remap thanos_memcached_operation_skipped_total metric.

Signed-off-by: Peter Štibraný <[email protected]>

* Comment that blocks_meta_modified is ignored.

Signed-off-by: Peter Štibraný <[email protected]>

* Use promauto.With(registerer) to create metrics.

Signed-off-by: Peter Štibraný <[email protected]>

Author: pstibrany

CHANGELOG.md

@@ -64,6 +64,10 @@
   - `-<prefix>.memcached.consistent-hash`: Old default: false, new default: true. This allows for better cache hits when the memcaches are scaled up and down.
   - `-querier.batch-iterators`: Old default: false, new default: true.
   - `-querier.ingester-streaming`: Old default: false, new default: true.
+* [CHANGE] Experimental TSDB: Added `-experimental.tsdb.bucket-store.postings-cache-compression-enabled` to enable postings compression when storing to cache. #2335
+* [CHANGE] Experimental TSDB: Added `-compactor.deletion-delay`, which is time before a block marked for deletion is deleted from bucket. If not 0, blocks will be marked for deletion and compactor component will delete blocks marked for deletion from the bucket. If delete-delay is 0, blocks will be deleted straight away. Note that deleting blocks immediately can cause query failures, if store gateway / querier still has the block loaded, or compactor is ignoring the deletion because it's compacting the block at the same time. Default value is 48h. #2335
+* [CHANGE] Experimental TSDB: Added `-experimental.tsdb.bucket-store.index-cache.postings-compression-enabled`, to set duration after which the blocks marked for deletion will be filtered out while fetching blocks used for querying. This option allows querier to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet. Default is 24h, half of the default value for `-compactor.deletion-delay`. #2335
+* [CHANGE] Experimental TSDB: Added `-experimental.tsdb.bucket-store.index-cache.memcached.max-item-size` to control maximum size of item that is stored to memcached. Defaults to 1 MiB. #2335
 * [FEATURE] Added experimental storage API to the ruler service that is enabled when the `-experimental.ruler.enable-api` is set to true #2269
   * `-ruler.storage.type` flag now allows `s3`,`gcs`, and `azure` values
   * `-ruler.storage.(s3|gcs|azure)` flags exist to allow the configuration of object clients set for rule storage

docs/configuration/config-file-reference.md

@@ -2453,6 +2453,24 @@ bucket_store:
       # CLI flag: -experimental.tsdb.bucket-store.index-cache.memcached.max-get-multi-batch-size
       [max_get_multi_batch_size: <int> | default = 0]
 
+      # The maximum size of an item stored in memcached. Bigger items are not
+      # stored. If set to 0, no maximum size is enforced.
+      # CLI flag: -experimental.tsdb.bucket-store.index-cache.memcached.max-item-size
+      [max_item_size: <int> | default = 1048576]
+
+    # Compress postings before storing them to postings cache.
+    # CLI flag: -experimental.tsdb.bucket-store.index-cache.postings-compression-enabled
+    [postings_compression_enabled: <boolean> | default = false]
+
+  # Duration after which the blocks marked for deletion will be filtered out
+  # while fetching blocks. The idea of ignore-deletion-marks-delay is to ignore
+  # blocks that are marked for deletion with some delay. This ensures store can
+  # still serve blocks that are meant to be deleted but do not have a
+  # replacement yet.Default is 24h, half of the default value for
+  # -compactor.deletion-delay.
+  # CLI flag: -experimental.tsdb.bucket-store.ignore-deletion-marks-delay
+  [ignore_deletion_mark_delay: <duration> | default = 24h0m0s]
+
 # How frequently does Cortex try to compact TSDB head. Block is only created if
 # data covers smallest block range. Must be greater than 0 and max 5 minutes.
 # CLI flag: -experimental.tsdb.head-compaction-interval
@@ -2571,6 +2589,15 @@ The `compactor_config` configures the compactor for the experimental blocks stor
 # CLI flag: -compactor.compaction-retries
 [compaction_retries: <int> | default = 3]
 
+# Time before a block marked for deletion is deleted from bucket. If not 0,
+# blocks will be marked for deletion and compactor component will delete blocks
+# marked for deletion from the bucket. If delete-delay is 0, blocks will be
+# deleted straight away. Note that deleting blocks immediately can cause query
+# failures, if store gateway still has the block loaded, or compactor is
+# ignoring the deletion because it's compacting the block at the same time.
+# CLI flag: -compactor.deletion-delay
+[deletion_delay: <duration> | default = 48h0m0s]
+
 # Shard tenants across multiple compactor instances. Sharding is required if you
 # run multiple compactor instances, in order to coordinate compactions and avoid
 # race conditions leading to the same tenant blocks simultaneously compacted by

docs/operations/blocks-storage.md

@@ -252,6 +252,24 @@ tsdb:
         # CLI flag: -experimental.tsdb.bucket-store.index-cache.memcached.max-get-multi-batch-size
         [max_get_multi_batch_size: <int> | default = 0]
 
+        # The maximum size of an item stored in memcached. Bigger items are not
+        # stored. If set to 0, no maximum size is enforced.
+        # CLI flag: -experimental.tsdb.bucket-store.index-cache.memcached.max-item-size
+        [max_item_size: <int> | default = 1048576]
+
+      # Compress postings before storing them to postings cache.
+      # CLI flag: -experimental.tsdb.bucket-store.index-cache.postings-compression-enabled
+      [postings_compression_enabled: <boolean> | default = false]
+
+    # Duration after which the blocks marked for deletion will be filtered out
+    # while fetching blocks. The idea of ignore-deletion-marks-delay is to
+    # ignore blocks that are marked for deletion with some delay. This ensures
+    # store can still serve blocks that are meant to be deleted but do not have
+    # a replacement yet.Default is 24h, half of the default value for
+    # -compactor.deletion-delay.
+    # CLI flag: -experimental.tsdb.bucket-store.ignore-deletion-marks-delay
+    [ignore_deletion_mark_delay: <duration> | default = 24h0m0s]
+
   # How frequently does Cortex try to compact TSDB head. Block is only created
   # if data covers smallest block range. Must be greater than 0 and max 5
   # minutes.
@@ -372,6 +390,16 @@ compactor:
   # CLI flag: -compactor.compaction-retries
   [compaction_retries: <int> | default = 3]
 
+  # Time before a block marked for deletion is deleted from bucket. If not 0,
+  # blocks will be marked for deletion and compactor component will delete
+  # blocks marked for deletion from the bucket. If delete-delay is 0, blocks
+  # will be deleted straight away. Note that deleting blocks immediately can
+  # cause query failures, if store gateway still has the block loaded, or
+  # compactor is ignoring the deletion because it's compacting the block at the
+  # same time.
+  # CLI flag: -compactor.deletion-delay
+  [deletion_delay: <duration> | default = 48h0m0s]
+
   # Shard tenants across multiple compactor instances. Sharding is required if
   # you run multiple compactor instances, in order to coordinate compactions and
   # avoid race conditions leading to the same tenant blocks simultaneously

go.mod

@@ -14,15 +14,10 @@ require (
 	github.com/aws/aws-sdk-go v1.25.48
 	github.com/blang/semver v3.5.0+incompatible
 	github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b
-	github.com/cenkalti/backoff v1.0.0 // indirect
 	github.com/cespare/xxhash v1.1.0
-	github.com/coreos/go-semver v0.3.0 // indirect
-	github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d // indirect
-	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
 	github.com/fsouza/fake-gcs-server v1.7.0
 	github.com/go-kit/kit v0.9.0
-	github.com/go-openapi/analysis v0.19.4 // indirect
 	github.com/gocql/gocql v0.0.0-20200121121104-95d072f1b5bb
 	github.com/gogo/protobuf v1.3.1
 	github.com/gogo/status v1.0.3
@@ -31,24 +26,19 @@ require (
 	github.com/golang/snappy v0.0.1
 	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/gorilla/mux v1.7.1
-	github.com/gorilla/websocket v1.4.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
 	github.com/hashicorp/consul/api v1.3.0
 	github.com/hashicorp/go-cleanhttp v0.5.1
 	github.com/hashicorp/go-sockaddr v1.0.2
 	github.com/hashicorp/memberlist v0.1.5
 	github.com/json-iterator/go v1.1.9
-	github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect
-	github.com/lann/builder v0.0.0-20150808151131-f22ce00fd939 // indirect
-	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/lib/pq v1.3.0
-	github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect
 	github.com/mitchellh/go-wordwrap v1.0.0
 	github.com/oklog/ulid v1.3.1
 	github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02
 	github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9
 	github.com/opentracing/opentracing-go v1.1.1-0.20200124165624-2876d2018785
-	github.com/pkg/errors v0.8.1
+	github.com/pkg/errors v0.9.1
 	github.com/prometheus/alertmanager v0.20.0
 	github.com/prometheus/client_golang v1.5.0
 	github.com/prometheus/client_model v0.2.0
@@ -58,19 +48,17 @@ require (
 	github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e
 	github.com/spf13/afero v1.2.2
 	github.com/stretchr/testify v1.4.0
-	github.com/thanos-io/thanos v0.11.0
-	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
+	github.com/thanos-io/thanos v0.8.1-0.20200326105947-214ff4480e93
 	github.com/uber/jaeger-client-go v2.20.1+incompatible
 	github.com/weaveworks/common v0.0.0-20200310113808-2708ba4e60a4
 	go.etcd.io/bbolt v1.3.3
 	go.etcd.io/etcd v0.0.0-20190709142735-eb7dd97135a5
 	go.uber.org/atomic v1.5.0
-	golang.org/x/net v0.0.0-20191112182307-2180aed22343
+	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	google.golang.org/api v0.14.0
 	google.golang.org/grpc v1.25.1
 	gopkg.in/yaml.v2 v2.2.7
-	k8s.io/client-go v12.0.0+incompatible // indirect
 	sigs.k8s.io/yaml v1.1.0
 )