Skip to content

Commit 3b5f3ce

Browse files
authored
feat: support customization of sharding labels (#24)
## What type of PR is this? /kind feature ## What this PR does / why we need it: 1. Support label redefinition through flags: - `ctrlmesh.kusionstack.io/control`, the sharding hash label. - `ctrlmesh.kusionstack.io/shard-hash`, the controller-mesh control label. - `ctrlmesh.kusionstack.io/namespace`, the namespace label. 2. Support disable patch runnable. ## Which issue(s) this PR fixes: <!-- *Automatically closes linked issue when PR is merged. Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`. _If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_* --> ## Special notes for your reviewer: ### Does this PR introduce a user-facing change? <!-- If no, just write "NONE" in the release-note block below. If yes, a release note is required: Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required". --> ```release-note ``` ### Additional documentation e.g., design docs, usage docs, etc.: <!-- Please use the following format for linking documentation: - [Design]: <link> - [Usage]: <link> - [Other doc]: <link> --> ```docs ```
2 parents 8101267 + ba4702d commit 3b5f3ce

File tree

7 files changed

+96
-54
lines changed

7 files changed

+96
-54
lines changed

pkg/apis/ctrlmesh/types.go

+42-6
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@ limitations under the License.
1616

1717
package ctrlmesh
1818

19+
import (
20+
"flag"
21+
)
22+
1923
// Environments
2024
const (
2125
EnvEnableWebhookServer = "ENABLE_WEBHOOK_SERVER"
@@ -28,9 +32,6 @@ const (
2832
// Labels
2933
const (
3034
CtrlmeshControlPrefix = "ctrlmesh.kusionstack.io/"
31-
CtrlmeshShardHashKey = "ctrlmesh.kusionstack.io/shard-hash"
32-
CtrlmeshControlKey = "ctrlmesh.kusionstack.io/control"
33-
CtrlmeshNamespaceKey = "ctrlmesh.kusionstack.io/namespace"
3435
CtrlmeshIgnoreWebhookLabel = "ctrlmesh.kusionstack.io/ignore-webhook"
3536
CtrlmeshIgnoreValidateLabel = "ctrlmesh.kusionstack.io/ignore-validate"
3637
CtrlmeshDefaultReplicasLabel = "ctrlmesh.kusionstack.io/default-replicas"
@@ -62,7 +63,42 @@ const (
6263
ProtectFinalizer = "finalizer.ctrlmesh.kusionstack.io/protected"
6364
)
6465

65-
// Name
66-
const (
67-
ShardingConfigMapName = "ctrlmesh-sharding-config"
66+
var (
67+
shardHashLabel string
68+
meshControlLabel string
69+
namespaceLabel string
6870
)
71+
72+
func init() {
73+
flag.StringVar(&shardHashLabel, "label-shard-hash", "ctrlmesh.kusionstack.io/shard-hash", "The sharding hash label.")
74+
flag.StringVar(&meshControlLabel, "label-mesh-control", "ctrlmesh.kusionstack.io/control", "The controller mesh control label.")
75+
flag.StringVar(&namespaceLabel, "label-namespace", "ctrlmesh.kusionstack.io/namespace", "The namespace label.")
76+
}
77+
78+
func ShardHashLabel() string {
79+
return shardHashLabel
80+
}
81+
82+
func MeshControlLabel() string {
83+
return meshControlLabel
84+
}
85+
86+
func NamespaceShardLabel() string {
87+
return namespaceLabel
88+
}
89+
90+
func IsControlledByMesh(labels map[string]string) bool {
91+
if labels == nil {
92+
return false
93+
}
94+
val, ok := labels[MeshControlLabel()]
95+
return ok && val == "true"
96+
}
97+
98+
func ShouldSyncLabels() []string {
99+
return []string{
100+
ShardHashLabel(),
101+
MeshControlLabel(),
102+
NamespaceShardLabel(),
103+
}
104+
}

pkg/cmd/manager/main.go

+8-6
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,9 @@ var (
5252
scheme = runtime.NewScheme()
5353
setupLog = ctrl.Log.WithName("setup")
5454

55-
restConfigQPS = flag.Int("rest-config-qps", 30, "QPS of rest config.")
56-
restConfigBurst = flag.Int("rest-config-burst", 50, "Burst of rest config.")
55+
restConfigQPS = flag.Int("rest-config-qps", 30, "QPS of rest config.")
56+
restConfigBurst = flag.Int("rest-config-burst", 50, "Burst of rest config.")
57+
enablePatchRunnable = flag.Bool("enable-patch-runnable", true, "")
5758
)
5859

5960
func init() {
@@ -161,10 +162,11 @@ func main() {
161162
setupLog.Error(err, "unable to create controller", "controller", "CircuitBreaker")
162163
os.Exit(1)
163164
}
164-
165-
if err = patchrunnable.SetupWithManager(mgr); err != nil {
166-
setupLog.Error(err, "unable to create runnable", "runnable", "PatchRunnable")
167-
os.Exit(1)
165+
if *enablePatchRunnable {
166+
if err = patchrunnable.SetupWithManager(mgr); err != nil {
167+
setupLog.Error(err, "unable to create runnable", "runnable", "PatchRunnable")
168+
os.Exit(1)
169+
}
168170
}
169171
}()
170172

pkg/manager/controllers/patchrunnable/labelpatch_runnable.go

+13-13
Original file line numberDiff line numberDiff line change
@@ -295,25 +295,25 @@ func updateLabel(ns *v1.Namespace) (bool, error) {
295295

296296
nsHash := strconv.Itoa(rand.Hash(ns.Name, constants.DefaultShardingSize))
297297

298-
if _, ok := ns.Labels[ctrlmesh.CtrlmeshControlKey]; !ok {
299-
if _, exist := ns.Labels[ctrlmesh.CtrlmeshShardHashKey]; exist {
300-
return false, fmt.Errorf("label %s already exist but can not find %s", ctrlmesh.CtrlmeshShardHashKey, ctrlmesh.CtrlmeshControlKey)
298+
if _, ok := ns.Labels[ctrlmesh.MeshControlLabel()]; !ok {
299+
if _, exist := ns.Labels[ctrlmesh.ShardHashLabel()]; exist {
300+
return false, fmt.Errorf("label %s already exist but can not find %s", ctrlmesh.ShardHashLabel(), ctrlmesh.MeshControlLabel())
301301
}
302-
if _, exist := ns.Labels[ctrlmesh.CtrlmeshNamespaceKey]; exist {
303-
return false, fmt.Errorf("label %s already exist but can not find %s", ctrlmesh.CtrlmeshNamespaceKey, ctrlmesh.CtrlmeshControlKey)
302+
if _, exist := ns.Labels[ctrlmesh.NamespaceShardLabel()]; exist {
303+
return false, fmt.Errorf("label %s already exist but can not find %s", ctrlmesh.NamespaceShardLabel(), ctrlmesh.MeshControlLabel())
304304
}
305-
ns.Labels[ctrlmesh.CtrlmeshControlKey] = "true"
306-
ns.Labels[ctrlmesh.CtrlmeshNamespaceKey] = ns.Name
307-
ns.Labels[ctrlmesh.CtrlmeshShardHashKey] = nsHash
305+
ns.Labels[ctrlmesh.MeshControlLabel()] = "true"
306+
ns.Labels[ctrlmesh.NamespaceShardLabel()] = ns.Name
307+
ns.Labels[ctrlmesh.ShardHashLabel()] = nsHash
308308
return true, nil
309309
} else {
310-
if val, exist := ns.Labels[ctrlmesh.CtrlmeshShardHashKey]; !exist || nsHash != val {
311-
ns.Labels[ctrlmesh.CtrlmeshNamespaceKey] = ns.Name
312-
ns.Labels[ctrlmesh.CtrlmeshShardHashKey] = nsHash
310+
if val, exist := ns.Labels[ctrlmesh.ShardHashLabel()]; !exist || nsHash != val {
311+
ns.Labels[ctrlmesh.NamespaceShardLabel()] = ns.Name
312+
ns.Labels[ctrlmesh.ShardHashLabel()] = nsHash
313313
return true, nil
314314
}
315-
if val, exist := ns.Labels[ctrlmesh.CtrlmeshNamespaceKey]; !exist || val != ns.Name {
316-
ns.Labels[ctrlmesh.CtrlmeshNamespaceKey] = ns.Name
315+
if val, exist := ns.Labels[ctrlmesh.NamespaceShardLabel()]; !exist || val != ns.Name {
316+
ns.Labels[ctrlmesh.NamespaceShardLabel()] = ns.Name
317317
return true, nil
318318
}
319319
}

pkg/manager/controllers/patchrunnable/resource.go

+7-6
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import (
2020
"context"
2121
"flag"
2222
"reflect"
23-
"strings"
2423

2524
v1 "k8s.io/api/core/v1"
2625
"k8s.io/apimachinery/pkg/api/errors"
@@ -101,12 +100,13 @@ type ResourceConfig struct {
101100

102101
func getShardingLabel(obj client.Object) map[string]string {
103102
shardingLabels := map[string]string{}
104-
if beNil(obj) || obj.GetLabels() == nil {
103+
if beNil(obj) || obj.GetLabels() == nil || !ctrlmesh.IsControlledByMesh(obj.GetLabels()) {
105104
return shardingLabels
106105
}
107-
for k, v := range obj.GetLabels() {
108-
if strings.HasPrefix(k, ctrlmesh.CtrlmeshControlPrefix) {
109-
shardingLabels[k] = v
106+
lbs := obj.GetLabels()
107+
for _, k := range ctrlmesh.ShouldSyncLabels() {
108+
if val, ok := lbs[k]; ok {
109+
shardingLabels[k] = val
110110
}
111111
}
112112
return shardingLabels
@@ -119,6 +119,7 @@ func beNil(a interface{}) bool {
119119
switch reflect.TypeOf(a).Kind() {
120120
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
121121
return reflect.ValueOf(a).IsNil()
122+
default:
123+
return false
122124
}
123-
return false
124125
}

pkg/manager/controllers/shardingconfigserver/auto_sharding.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -185,14 +185,14 @@ func genCanaryLimits(canaryConfig *ctrlmeshv1alpha1.CanaryConfig, originLimiter
185185
}
186186
if len(canaryConfig.InNamespaces) > 0 {
187187
newSel.Selector.MatchExpressions = append(newSel.Selector.MatchExpressions, metav1.LabelSelectorRequirement{
188-
Key: ctrlmesh.CtrlmeshNamespaceKey,
188+
Key: ctrlmesh.NamespaceShardLabel(),
189189
Operator: metav1.LabelSelectorOpIn,
190190
Values: canaryConfig.InNamespaces,
191191
})
192192
}
193193
if len(canaryConfig.InShardHash) > 0 {
194194
newSel.Selector.MatchExpressions = append(newSel.Selector.MatchExpressions, metav1.LabelSelectorRequirement{
195-
Key: ctrlmesh.CtrlmeshShardHashKey,
195+
Key: ctrlmesh.ShardHashLabel(),
196196
Operator: metav1.LabelSelectorOpIn,
197197
Values: canaryConfig.InShardHash,
198198
})
@@ -214,20 +214,20 @@ func genShardingGlobalLimits(root *ctrlmeshv1alpha1.ShardingConfig, batch []stri
214214
}
215215
if canaryConfig != nil && len(canaryConfig.InNamespaces) > 0 && *root.Spec.Root.Canary.Replicas > 0 {
216216
newSel.Selector.MatchExpressions = append(newSel.Selector.MatchExpressions, metav1.LabelSelectorRequirement{
217-
Key: ctrlmesh.CtrlmeshNamespaceKey,
217+
Key: ctrlmesh.NamespaceShardLabel(),
218218
Operator: metav1.LabelSelectorOpNotIn,
219219
Values: canaryConfig.InNamespaces,
220220
})
221221
}
222222
if canaryConfig != nil && len(canaryConfig.InShardHash) > 0 && *root.Spec.Root.Canary.Replicas > 0 {
223223
newSel.Selector.MatchExpressions = append(newSel.Selector.MatchExpressions, metav1.LabelSelectorRequirement{
224-
Key: ctrlmesh.CtrlmeshShardHashKey,
224+
Key: ctrlmesh.ShardHashLabel(),
225225
Operator: metav1.LabelSelectorOpNotIn,
226226
Values: canaryConfig.InShardHash,
227227
})
228228
}
229229
newSel.Selector.MatchExpressions = append(newSel.Selector.MatchExpressions, metav1.LabelSelectorRequirement{
230-
Key: ctrlmesh.CtrlmeshShardHashKey,
230+
Key: ctrlmesh.ShardHashLabel(),
231231
Operator: metav1.LabelSelectorOpIn,
232232
Values: batch,
233233
})

pkg/webhook/ns/namespace_mutating_handler.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -67,17 +67,17 @@ func (h *MutatingHandler) Handle(ctx context.Context, req admission.Request) adm
6767

6868
func (h *MutatingHandler) shouldUpdateNs(ns *v1.Namespace) (shouldUpdate bool) {
6969
shouldUpdate = false
70-
if _, exist := ns.Labels[ctrlmesh.CtrlmeshControlKey]; !exist {
71-
ns.Labels[ctrlmesh.CtrlmeshControlKey] = "true"
70+
if _, exist := ns.Labels[ctrlmesh.MeshControlLabel()]; !exist {
71+
ns.Labels[ctrlmesh.MeshControlLabel()] = "true"
7272
shouldUpdate = true
7373
}
74-
if val, exist := ns.Labels[ctrlmesh.CtrlmeshNamespaceKey]; !exist || val != ns.Name {
75-
ns.Labels[ctrlmesh.CtrlmeshNamespaceKey] = ns.Name
74+
if val, exist := ns.Labels[ctrlmesh.NamespaceShardLabel()]; !exist || val != ns.Name {
75+
ns.Labels[ctrlmesh.NamespaceShardLabel()] = ns.Name
7676
shouldUpdate = true
7777
}
7878
nsHash := strconv.Itoa(rand.Hash(ns.Name, constants.DefaultShardingSize))
79-
if val, exist := ns.Labels[ctrlmesh.CtrlmeshShardHashKey]; !exist || nsHash != val {
80-
ns.Labels[ctrlmesh.CtrlmeshShardHashKey] = nsHash
79+
if val, exist := ns.Labels[ctrlmesh.ShardHashLabel()]; !exist || nsHash != val {
80+
ns.Labels[ctrlmesh.ShardHashLabel()] = nsHash
8181
shouldUpdate = true
8282
}
8383
return shouldUpdate

pkg/webhook/resources/resource_mutating_handler.go

+15-12
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import (
2020
"context"
2121
"encoding/json"
2222
"net/http"
23-
"strings"
2423

2524
"gomodules.xyz/jsonpatch/v2"
2625
admissionv1 "k8s.io/api/admission/v1"
@@ -34,6 +33,8 @@ import (
3433
"sigs.k8s.io/controller-runtime/pkg/client"
3534
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
3635
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
36+
37+
"github.com/KusionStack/controller-mesh/pkg/apis/ctrlmesh"
3738
)
3839

3940
var (
@@ -75,18 +76,10 @@ func (r *MutatingHandler) Handle(ctx context.Context, req admission.Request) adm
7576
return admission.Errored(http.StatusBadRequest, err)
7677
}
7778
}
78-
patchLabel := map[string]string{}
79-
if ns.Labels != nil {
80-
for key, val := range ns.Labels {
81-
if r.isSelecterKey(key) {
82-
patchLabel[key] = val
83-
}
84-
}
85-
}
79+
patchLabel := r.getSyncLabels(ns.Labels)
8680
if len(patchLabel) == 0 {
8781
return admission.Allowed("")
8882
}
89-
9083
marshalled, err := json.Marshal(&PatchMeta{&metav1.ObjectMeta{Labels: patchLabel}})
9184
if err != nil {
9285
klog.Errorf("meta marshal error, %s", err)
@@ -119,8 +112,18 @@ func (r *MutatingHandler) getNamespaceWithRetry(c client.Client, ctx context.Con
119112
return nil
120113
}
121114

122-
func (r *MutatingHandler) isSelecterKey(key string) bool {
123-
return strings.HasPrefix(key, SelectorPrefix)
115+
func (r *MutatingHandler) getSyncLabels(labels map[string]string) map[string]string {
116+
syncLabels := ctrlmesh.ShouldSyncLabels()
117+
res := map[string]string{}
118+
if !ctrlmesh.IsControlledByMesh(labels) {
119+
return res
120+
}
121+
for _, l := range syncLabels {
122+
if val, ok := labels[l]; ok {
123+
res[l] = val
124+
}
125+
}
126+
return res
124127
}
125128

126129
func (r *MutatingHandler) InjectDecoder(d *admission.Decoder) error {

0 commit comments

Comments
 (0)