From cb1550d7fac4dd824531551ccc28d5aa38b8be85 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 9 Oct 2020 16:16:54 +0200 Subject: [PATCH] Fixed gRPC connections leaking in rulers when rulers sharding is enabled and APIs called Signed-off-by: Marco Pracucci --- CHANGELOG.md | 3 ++- pkg/ruler/ruler.go | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c3d70f723e..c2f013cabc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,7 +85,8 @@ * [BUGFIX] Reduce failures to update heartbeat when using Consul. #3259 * [BUGFIX] When using ruler sharding, moving all user rule groups from ruler to a different one and then back could end up with some user groups not being evaluated at all. #3235 * [BUGFIX] Use a valid grpc header when logging IP addresses. #3307 -* [BUGFIX] Fixes the metric `cortex_prometheus_rule_group_duration_seconds` in the Ruler, it wouldn't report any values. #3310 +* [BUGFIX] Fixed the metric `cortex_prometheus_rule_group_duration_seconds` in the Ruler, it wouldn't report any values. #3310 +* [BUGFIX] Fixed gRPC connections leaking in rulers when rulers sharding is enabled and APIs called. #3314 ## 1.4.0 / 2020-10-02 diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 427ee78549f..9b3e7c5c545 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -707,6 +707,12 @@ func (r *Ruler) getShardedRules(ctx context.Context) ([]*GroupStateDesc, error) } cc := NewRulerClient(conn) newGrps, err := cc.Rules(ctx, nil) + + // Close the gRPC connection regardless the RPC was successful or not. + if closeErr := conn.Close(); closeErr != nil { + level.Warn(r.logger).Log("msg", "failed to close gRPC connection to ruler", "remote", rlr.Addr, "err", closeErr) + } + if err != nil { return nil, fmt.Errorf("unable to retrieve rules from other rulers, %v", err) }