From a9f0fb226546422953b0f5c7fad10b0959adbf06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Mon, 4 Nov 2019 10:19:37 +0100 Subject: [PATCH 1/3] If ingester.max-transfer-retries is set to 0, hand-over attempts are disabled. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Due to interaction with BackOffConfig.MaxRetries, zero value meant infinite number of attempts, which is hardly ever a desired behaviour (with no ingester waiting to accept a transfer, this would never end). Fixed #1771. Signed-off-by: Peter Štibraný --- CHANGELOG.md | 1 + docs/arguments.md | 2 +- pkg/ingester/ingester.go | 4 ++-- pkg/ingester/lifecycle_test.go | 2 +- pkg/ingester/transfer.go | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2153972918c..1c5abf27370 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * [CHANGE] The frontend component has been refactored to be easier to re-use. When upgrading the frontend, cache entries will be discarded and re-created with the new protobuf schema. #1734 * [CHANGE] Remove direct DB/API access from the ruler * [CHANGE] Removed `Delta` encoding. Any old chunks with `Delta` encoding cannot be read anymore. If `ingester.chunk-encoding` is set to `Delta` the ingester will fail to start. #1706 +* [CHANGE] Setting `-ingester.ingester.max-transfer-retries` to 0 now disables hand-over when ingester is shuttind down. Previously, zero meant infinite number of attempts. #1771 * [FEATURE] Global limit on the max series per user and metric #1760 * `-ingester.max-global-series-per-user` * `-ingester.max-global-series-per-metric` diff --git a/docs/arguments.md b/docs/arguments.md index 25f00396594..d419b2b4827 100644 --- a/docs/arguments.md +++ b/docs/arguments.md @@ -230,7 +230,7 @@ It also talks to a KVStore and has it's own copies of the same flags used by the - `-ingester.ingester.max-transfer-retries` - How many times a LEAVING ingester tries to find a PENDING ingester during the [hand-over process](ingester-handover.md). Each attempt takes a second or so. (default 10) + How many times a LEAVING ingester tries to find a PENDING ingester during the [hand-over process](ingester-handover.md). Each attempt takes a second or so. Negative value or zero disables hand-over process completely. (default 10) - `-ingester.normalise-tokens` diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index d22890013f1..1034fcfeb3e 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -102,7 +102,7 @@ func newIngesterMetrics(r prometheus.Registerer) *ingesterMetrics { type Config struct { LifecyclerConfig ring.LifecyclerConfig `yaml:"lifecycler,omitempty"` - // Config for transferring chunks. + // Config for transferring chunks. Zero or negative = no retries. MaxTransferRetries int `yaml:"max_transfer_retries,omitempty"` // Config for chunk flushing. @@ -134,7 +134,7 @@ type Config struct { func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.LifecyclerConfig.RegisterFlags(f) - f.IntVar(&cfg.MaxTransferRetries, "ingester.max-transfer-retries", 10, "Number of times to try and transfer chunks before falling back to flushing.") + f.IntVar(&cfg.MaxTransferRetries, "ingester.max-transfer-retries", 10, "Number of times to try and transfer chunks before falling back to flushing. Negative value or zero disables hand-over.") f.DurationVar(&cfg.FlushCheckPeriod, "ingester.flush-period", 1*time.Minute, "Period with which to attempt to flush chunks.") f.DurationVar(&cfg.RetainPeriod, "ingester.retain-period", 5*time.Minute, "Period chunks will remain in memory after flushing.") f.DurationVar(&cfg.FlushOpTimeout, "ingester.flush-op-timeout", 1*time.Minute, "Timeout for individual flush operations.") diff --git a/pkg/ingester/lifecycle_test.go b/pkg/ingester/lifecycle_test.go index 1c1ca391d52..6d7677d52df 100644 --- a/pkg/ingester/lifecycle_test.go +++ b/pkg/ingester/lifecycle_test.go @@ -44,7 +44,7 @@ func defaultIngesterTestConfig() Config { cfg.LifecyclerConfig.Addr = "localhost" cfg.LifecyclerConfig.ID = "localhost" cfg.LifecyclerConfig.FinalSleep = 0 - cfg.MaxTransferRetries = -1 + cfg.MaxTransferRetries = 0 return cfg } diff --git a/pkg/ingester/transfer.go b/pkg/ingester/transfer.go index 93cda02f665..dd6e5f1db79 100644 --- a/pkg/ingester/transfer.go +++ b/pkg/ingester/transfer.go @@ -349,7 +349,7 @@ func fromWireChunks(wireChunks []client.Chunk) ([]*desc, error) { // TransferOut finds an ingester in PENDING state and transfers our chunks to it. // Called as part of the ingester shutdown process. func (i *Ingester) TransferOut(ctx context.Context) error { - if i.cfg.MaxTransferRetries < 0 { + if i.cfg.MaxTransferRetries <= 0 { return fmt.Errorf("transfers disabled") } backoff := util.NewBackoff(ctx, util.BackoffConfig{ From fd3dc7d4004338434b79970b7ac280ea94e5d725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Mon, 4 Nov 2019 11:55:11 +0100 Subject: [PATCH 2/3] =?UTF-8?q?Fixed=20typo=20Signed-off-by:=20Peter=20S?= =?UTF-8?q?=CC=8Ctibrany=CC=81=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c5abf27370..f99533c9d46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ * [CHANGE] The frontend component has been refactored to be easier to re-use. When upgrading the frontend, cache entries will be discarded and re-created with the new protobuf schema. #1734 * [CHANGE] Remove direct DB/API access from the ruler * [CHANGE] Removed `Delta` encoding. Any old chunks with `Delta` encoding cannot be read anymore. If `ingester.chunk-encoding` is set to `Delta` the ingester will fail to start. #1706 -* [CHANGE] Setting `-ingester.ingester.max-transfer-retries` to 0 now disables hand-over when ingester is shuttind down. Previously, zero meant infinite number of attempts. #1771 +* [CHANGE] Setting `-ingester.ingester.max-transfer-retries` to 0 now disables hand-over when ingester is shutting down. Previously, zero meant infinite number of attempts. #1771 * [FEATURE] Global limit on the max series per user and metric #1760 * `-ingester.max-global-series-per-user` * `-ingester.max-global-series-per-metric` From a05eeb210b698d3cd61d5c658a834142007731fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20S=CC=8Ctibrany=CC=81?= Date: Mon, 4 Nov 2019 13:53:30 +0100 Subject: [PATCH 3/3] Fix ingester.max-transfer-retries argument name. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- CHANGELOG.md | 2 +- docs/arguments.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f99533c9d46..5209d2eb285 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ * [CHANGE] The frontend component has been refactored to be easier to re-use. When upgrading the frontend, cache entries will be discarded and re-created with the new protobuf schema. #1734 * [CHANGE] Remove direct DB/API access from the ruler * [CHANGE] Removed `Delta` encoding. Any old chunks with `Delta` encoding cannot be read anymore. If `ingester.chunk-encoding` is set to `Delta` the ingester will fail to start. #1706 -* [CHANGE] Setting `-ingester.ingester.max-transfer-retries` to 0 now disables hand-over when ingester is shutting down. Previously, zero meant infinite number of attempts. #1771 +* [CHANGE] Setting `-ingester.max-transfer-retries` to 0 now disables hand-over when ingester is shutting down. Previously, zero meant infinite number of attempts. #1771 * [FEATURE] Global limit on the max series per user and metric #1760 * `-ingester.max-global-series-per-user` * `-ingester.max-global-series-per-metric` diff --git a/docs/arguments.md b/docs/arguments.md index d419b2b4827..d7e4e1c72b8 100644 --- a/docs/arguments.md +++ b/docs/arguments.md @@ -228,7 +228,7 @@ It also talks to a KVStore and has it's own copies of the same flags used by the How long to wait in PENDING state during the [hand-over process](ingester-handover.md). (default 0s) -- `-ingester.ingester.max-transfer-retries` +- `-ingester.max-transfer-retries` How many times a LEAVING ingester tries to find a PENDING ingester during the [hand-over process](ingester-handover.md). Each attempt takes a second or so. Negative value or zero disables hand-over process completely. (default 10)