From 017be2e57322d6e03f74da422e1881f9946635da Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Mon, 21 Oct 2019 18:42:08 +0200 Subject: [PATCH 1/5] HTTP handler for ingester shutdown Signed-off-by: Ganesh Vernekar --- pkg/ingester/ingester.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 428f05c2ad4..29dfc199bdb 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -244,6 +244,15 @@ func (i *Ingester) Shutdown() { i.lifecycler.Shutdown() } +// ShutdownHandler triggers the following set of operations in order: +// * Change the state of ring to stop accepting writes. +// * Flush all the chunks. +func (i *Ingester) ShutdownHandler(w http.ResponseWriter, r *http.Request) { + i.Shutdown() + i.Flush() + w.WriteHeader(http.StatusNoContent) +} + // StopIncomingRequests is called during the shutdown process. func (i *Ingester) StopIncomingRequests() { i.userStatesMtx.Lock() From 073ed6affbd907ab95d94707428f0f32de5c8ff1 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Tue, 22 Oct 2019 11:32:28 +0200 Subject: [PATCH 2/5] Add /shutdown endpoint Signed-off-by: Ganesh Vernekar --- pkg/cortex/modules.go | 1 + pkg/ingester/ingester.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 08f47d86dea..1ea0249d4ec 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -237,6 +237,7 @@ func (t *Cortex) initIngester(cfg *Config) (err error) { grpc_health_v1.RegisterHealthServer(t.server.GRPC, t.ingester) t.server.HTTP.Path("/ready").Handler(http.HandlerFunc(t.ingester.ReadinessHandler)) t.server.HTTP.Path("/flush").Handler(http.HandlerFunc(t.ingester.FlushHandler)) + t.server.HTTP.Path("/shutdown").Handler(http.HandlerFunc(t.ingester.ShutdownHandler)) return } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 29dfc199bdb..78dad226bf3 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -249,7 +249,6 @@ func (i *Ingester) Shutdown() { // * Flush all the chunks. func (i *Ingester) ShutdownHandler(w http.ResponseWriter, r *http.Request) { i.Shutdown() - i.Flush() w.WriteHeader(http.StatusNoContent) } From 974e779d771b6d7f789b8dfdc4bfbbba6fbc8d43 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Thu, 24 Oct 2019 16:37:18 +0200 Subject: [PATCH 3/5] Avoid panic on multiple shutdown calls to ingester Signed-off-by: Ganesh Vernekar --- pkg/ingester/ingester.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 78dad226bf3..676249bdc2d 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -236,12 +236,18 @@ func (i *Ingester) loop() { // Shutdown beings the process to stop this ingester. func (i *Ingester) Shutdown() { - // First wait for our flush loop to stop. - close(i.quit) - i.done.Wait() + select { + case <-i.quit: + // Ingester was already shutdown. + return + default: + // First wait for our flush loop to stop. + close(i.quit) + i.done.Wait() - // Next initiate our graceful exit from the ring. - i.lifecycler.Shutdown() + // Next initiate our graceful exit from the ring. + i.lifecycler.Shutdown() + } } // ShutdownHandler triggers the following set of operations in order: From 0a4365cd3651e4ebed0ffe9f5327f5b249560194 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Mon, 4 Nov 2019 14:43:37 +0100 Subject: [PATCH 4/5] Add entry in CHANGELOG and docs/apis.md Signed-off-by: Ganesh Vernekar --- CHANGELOG.md | 1 + docs/apis.md | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b5459e9224..7c1c8b29afd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## master / unreleased * [CHANGE] Removed `Delta` encoding. Any old chunks with `Delta` encoding cannot be read anymore. If `ingester.chunk-encoding` is set to `Delta` the ingester will fail to start. #1706 +* [FEATURE] Added `/shutdown` endpoint for ingester to shutdown all operations of the ingester. #1746 * [ENHANCEMENT] Allocation improvements in adding samples to Chunk. #1706 ## 0.3.0 / 2019-10-11 diff --git a/docs/apis.md b/docs/apis.md index a2d6f0f015d..ec3b383c725 100644 --- a/docs/apis.md +++ b/docs/apis.md @@ -127,3 +127,10 @@ Error Response: BadRequest(400) These API endpoints will disable/enable the current Rule and Alertmanager configuration for a tenant. Note that setting a new config will effectively "re-enable" the Rules and Alertmanager configuration for a tenant. + +#### Ingester Shutdown + +`POST /shutdown` - Shutdown all operations of an ingester. Shutdown operations performed are similar to when an ingester is gracefully shutting down, including flushing of chunks if no other ingester is in `PENDING` state. Ingester does not terminate after calling this endpoint. + +- Normal Response Codes: NoContent(204) +- Error Response Codes: Unauthorized(401), NotFound(404) From 6d291419bf9eb68cd27375044254b847191cd492 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Mon, 11 Nov 2019 13:12:56 +0100 Subject: [PATCH 5/5] Fix review comments Signed-off-by: Ganesh Vernekar --- docs/apis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apis.md b/docs/apis.md index ec3b383c725..815b11d5a54 100644 --- a/docs/apis.md +++ b/docs/apis.md @@ -133,4 +133,4 @@ Note that setting a new config will effectively "re-enable" the Rules and Alertm `POST /shutdown` - Shutdown all operations of an ingester. Shutdown operations performed are similar to when an ingester is gracefully shutting down, including flushing of chunks if no other ingester is in `PENDING` state. Ingester does not terminate after calling this endpoint. - Normal Response Codes: NoContent(204) -- Error Response Codes: Unauthorized(401), NotFound(404) +- Error Response Codes: Unauthorized(401)