disable rule groups

Signed-off-by: Anand Rajagopal <[email protected]>

Author: rajagopalanand

pkg/ruler/ruler.go

@@ -71,6 +71,14 @@ const (
 	recordingRuleFilter string = "record"
 )
 
+type DisabledRuleGroupErr struct {
+	Message string
+}
+
+func (e *DisabledRuleGroupErr) Error() string {
+	return e.Message
+}
+
 // Config is the configuration for the recording rules server.
 type Config struct {
 	// This is used for template expansion in alerts; must be a valid URL.
@@ -126,6 +134,8 @@ type Config struct {
 
 	EnableQueryStats      bool `yaml:"query_stats_enabled"`
 	DisableRuleGroupLabel bool `yaml:"disable_rule_group_label"`
+
+	DisabledRuleGroups flagext.IntSliceCSV `yaml:"disabled_rule_groups"`
 }
 
 // Validate config and returns error on failure
@@ -187,6 +197,8 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.BoolVar(&cfg.EnableQueryStats, "ruler.query-stats-enabled", false, "Report the wall time for ruler queries to complete as a per user metric and as an info level log message.")
 	f.BoolVar(&cfg.DisableRuleGroupLabel, "ruler.disable-rule-group-label", false, "Disable the rule_group label on exported metrics")
 
+	f.Var(&cfg.DisabledRuleGroups, "ruler.disabled-rule-groups", "Comma separated list of keys of rule groups this ruler cannot evaluate. If specified, a ruler that would normally pick the specified rule groups for processing will ignore them instead.")
+
 	cfg.RingCheckPeriod = 5 * time.Second
 }
 
@@ -415,9 +427,14 @@ func tokenForGroup(g *rulespb.RuleGroupDesc) uint32 {
 	return ringHasher.Sum32()
 }
 
-func instanceOwnsRuleGroup(r ring.ReadRing, g *rulespb.RuleGroupDesc, instanceAddr string) (bool, error) {
+func instanceOwnsRuleGroup(r ring.ReadRing, g *rulespb.RuleGroupDesc, disabledRuleGroups flagext.IntSliceCSV, instanceAddr string) (bool, error) {
 	hash := tokenForGroup(g)
 
+	for _, disabledGroupToken := range disabledRuleGroups {
+		if hash == uint32(disabledGroupToken) {
+			return false, &DisabledRuleGroupErr{Message: fmt.Sprintf("Skipping rule group %s, within namespace %s, owned by %s", g.Name, g.Namespace, g.User)}
+		}
+	}
 	rlrs, err := r.Get(hash, RingOp, nil, nil, nil)
 	if err != nil {
 		return false, errors.Wrap(err, "error reading ring to verify rule group ownership")
@@ -544,7 +561,7 @@ func (r *Ruler) listRulesShardingDefault(ctx context.Context) (map[string]rulesp
 
 	filteredConfigs := make(map[string]rulespb.RuleGroupList)
 	for userID, groups := range configs {
-		filtered := filterRuleGroups(userID, groups, r.ring, r.lifecycler.GetInstanceAddr(), r.logger, r.ringCheckErrors)
+		filtered := filterRuleGroups(userID, groups, r.cfg.DisabledRuleGroups, r.ring, r.lifecycler.GetInstanceAddr(), r.logger, r.ringCheckErrors)
 		if len(filtered) > 0 {
 			filteredConfigs[userID] = filtered
 		}
@@ -602,7 +619,7 @@ func (r *Ruler) listRulesShuffleSharding(ctx context.Context) (map[string]rulesp
 					return errors.Wrapf(err, "failed to fetch rule groups for user %s", userID)
 				}
 
-				filtered := filterRuleGroups(userID, groups, userRings[userID], r.lifecycler.GetInstanceAddr(), r.logger, r.ringCheckErrors)
+				filtered := filterRuleGroups(userID, groups, r.cfg.DisabledRuleGroups, userRings[userID], r.lifecycler.GetInstanceAddr(), r.logger, r.ringCheckErrors)
 				if len(filtered) == 0 {
 					continue
 				}
@@ -624,15 +641,20 @@ func (r *Ruler) listRulesShuffleSharding(ctx context.Context) (map[string]rulesp
 //
 // Reason why this function is not a method on Ruler is to make sure we don't accidentally use r.ring,
 // but only ring passed as parameter.
-func filterRuleGroups(userID string, ruleGroups []*rulespb.RuleGroupDesc, ring ring.ReadRing, instanceAddr string, log log.Logger, ringCheckErrors prometheus.Counter) []*rulespb.RuleGroupDesc {
+func filterRuleGroups(userID string, ruleGroups []*rulespb.RuleGroupDesc, disabledRuleGroups flagext.IntSliceCSV, ring ring.ReadRing, instanceAddr string, log log.Logger, ringCheckErrors prometheus.Counter) []*rulespb.RuleGroupDesc {
 	// Prune the rule group to only contain rules that this ruler is responsible for, based on ring.
 	var result []*rulespb.RuleGroupDesc
 	for _, g := range ruleGroups {
-		owned, err := instanceOwnsRuleGroup(ring, g, instanceAddr)
+		owned, err := instanceOwnsRuleGroup(ring, g, disabledRuleGroups, instanceAddr)
 		if err != nil {
-			ringCheckErrors.Inc()
-			level.Error(log).Log("msg", "failed to check if the ruler replica owns the rule group", "user", userID, "namespace", g.Namespace, "group", g.Name, "err", err)
-			continue
+			switch e := err.(type) {
+			case *DisabledRuleGroupErr:
+				level.Info(log).Log("msg", e.Message)
+			default:
+				ringCheckErrors.Inc()
+				level.Error(log).Log("msg", "failed to check if the ruler replica owns the rule group", "user", userID, "namespace", g.Namespace, "group", g.Name, "err", err)
+				continue
+			}
 		}
 
 		if owned {

pkg/util/flagext/intslicecsv.go

@@ -0,0 +1,53 @@
+package flagext
+
+import (
+	"strconv"
+	"strings"
+)
+
+// IntSliceCSV is a slice of ints that is parsed from a comma-separated string
+// It implements flag.Value and yaml Marshalers
+type IntSliceCSV []int
+
+// String implements flag.Value
+func (v IntSliceCSV) String() string {
+	var result strings.Builder
+	for i, value := range v {
+		if i != 0 {
+			result.WriteString(",")
+		}
+		result.WriteString(strconv.Itoa(value))
+	}
+	return result.String()
+}
+
+// Set implements flag.Value
+func (v *IntSliceCSV) Set(s string) error {
+	if len(strings.TrimSpace(s)) == 0 {
+		return nil
+	}
+	str := strings.Split(s, ",")
+	for _, sv := range str {
+		val, err := strconv.Atoi(strings.TrimSpace(sv))
+		if err != nil {
+			return err
+		}
+		*v = append(*v, val)
+	}
+	return nil
+}
+
+// UnmarshalYAML implements yaml.Unmarshaler.
+func (v *IntSliceCSV) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+	return v.Set(s)
+}
+
+// MarshalYAML implements yaml.Marshaler.
+func (v IntSliceCSV) MarshalYAML() (interface{}, error) {
+	result := v.String()
+	return result, nil
+}

pkg/util/flagext/intslicecsv_test.go

@@ -0,0 +1,34 @@
+package flagext
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v2"
+)
+
+func Test_IntSliceCSV(t *testing.T) {
+	type TestStruct struct {
+		CSV IntSliceCSV `yaml:"csv"`
+	}
+
+	var testStruct TestStruct
+	s := "4,5,6,7"
+	assert.Nil(t, testStruct.CSV.Set(s))
+
+	assert.Equal(t, []int{4, 5, 6, 7}, []int(testStruct.CSV))
+	assert.Equal(t, s, testStruct.CSV.String())
+
+	expected := []byte(`csv: 4,5,6,7
+`)
+
+	actual, err := yaml.Marshal(testStruct)
+	assert.Nil(t, err)
+	assert.Equal(t, expected, actual)
+
+	var testStruct2 TestStruct
+
+	err = yaml.Unmarshal(expected, &testStruct2)
+	assert.Nil(t, err)
+	assert.Equal(t, testStruct, testStruct2)
+}