Skip to content

Commit aede84b

Browse files
alvaroalemanalvaroaleman
committed
Add multicluster controller example
1 parent 26660e7 commit aede84b

File tree

4 files changed

+338
-1
lines changed

4 files changed

+338
-1
lines changed

‎examples/README.md‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,15 @@ This example implements a *new* Kubernetes resource, ChaosPod, and creates a cus
3131
5. Adds ChaosPod webhooks to manager
3232
6. Starts the manager
3333

34+
### multicluster
35+
36+
This example implements a simplistic controller that watches pods in one cluster (`referencecluster`) and creates
37+
an identical pod for each pod observed in a different cluster (`mirrorcluster`).
38+
39+
* `main.go`: Initialization code
40+
* `main_test.go`: Tests that verify the reconciliation
41+
* `reconciler/reconciler.go`: The actual reconciliation logic
42+
3443
## Deploying and Running
3544

36-
To install and run the provided examples, see the Kubebuilder [Quick Start](https://book.kubebuilder.io/quick-start.html).
45+
To install and run the provided examples, see the Kubebuilder [Quick Start](https://book.kubebuilder.io/quick-start.html).

‎examples/multicluster/main.go‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
Copyright 2020 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package main
18+
19+
import (
20+
"fmt"
21+
"os"
22+
23+
"k8s.io/client-go/rest"
24+
25+
ctrl "sigs.k8s.io/controller-runtime"
26+
"sigs.k8s.io/controller-runtime/examples/multicluster/reconciler"
27+
"sigs.k8s.io/controller-runtime/pkg/client/config"
28+
"sigs.k8s.io/controller-runtime/pkg/clusterconnector"
29+
)
30+
31+
func main() {
32+
log := ctrl.Log.WithName("pod-mirror-controller")
33+
34+
referenceClusterCfg, err := config.GetConfigWithContext("reference-cluster")
35+
if err != nil {
36+
log.Error(err, "failed to get reference cluster config")
37+
os.Exit(1)
38+
}
39+
40+
mirrorClusterCfg, err := config.GetConfigWithContext("mirror-cluster")
41+
if err != nil {
42+
log.Error(err, "failed to get mirror cluster config")
43+
os.Exit(1)
44+
}
45+
ctrl.SetupSignalHandler()
46+
47+
if err := run(referenceClusterCfg, mirrorClusterCfg, ctrl.SetupSignalHandler()); err != nil {
48+
log.Error(err, "failed to run controller")
49+
os.Exit(1)
50+
}
51+
52+
log.Info("Finished gracefully")
53+
}
54+
55+
func run(referenceClusterConfig, mirrorClusterConfig *rest.Config, stop <-chan struct{}) error {
56+
mgr, err := ctrl.NewManager(referenceClusterConfig, ctrl.Options{})
57+
if err != nil {
58+
return fmt.Errorf("failed to construct manager: %w", err)
59+
}
60+
clusterConnector, err := clusterconnector.New(mirrorClusterConfig, mgr, "mirror_cluster")
61+
if err != nil {
62+
return fmt.Errorf("failed to construct clusterconnector: %w", err)
63+
}
64+
65+
if err := reconciler.Add(mgr, clusterConnector); err != nil {
66+
return fmt.Errorf("failed to construct reconciler: %w", err)
67+
}
68+
69+
if err := mgr.Start(stop); err != nil {
70+
return fmt.Errorf("failed to start manager: %w", err)
71+
}
72+
73+
return nil
74+
}

‎examples/multicluster/main_test.go‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"testing"
7+
8+
. "github.com/onsi/ginkgo"
9+
. "github.com/onsi/gomega"
10+
corev1 "k8s.io/api/core/v1"
11+
apierrors "k8s.io/apimachinery/pkg/api/errors"
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"k8s.io/apimachinery/pkg/types"
14+
"k8s.io/client-go/rest"
15+
16+
"sigs.k8s.io/controller-runtime/pkg/client"
17+
"sigs.k8s.io/controller-runtime/pkg/envtest"
18+
"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
19+
)
20+
21+
func TestRun(t *testing.T) {
22+
RegisterFailHandler(Fail)
23+
RunSpecsWithDefaultAndCustomReporters(t, "MirrorPod reconciler Integration Suite", []Reporter{printer.NewlineReporter{}})
24+
}
25+
26+
var _ = Describe("clusterconnector.ClusterConnector", func() {
27+
var stop chan struct{}
28+
var referenceClusterCfg *rest.Config
29+
var mirrorClusterCfg *rest.Config
30+
var referenceClusterTestEnv *envtest.Environment
31+
var mirrorClusterTestEnv *envtest.Environment
32+
var referenceClusterClient client.Client
33+
var mirrorClusterClient client.Client
34+
35+
Describe("multi-cluster-controller", func() {
36+
BeforeEach(func() {
37+
stop = make(chan struct{})
38+
referenceClusterTestEnv = &envtest.Environment{}
39+
mirrorClusterTestEnv = &envtest.Environment{}
40+
41+
var err error
42+
referenceClusterCfg, err = referenceClusterTestEnv.Start()
43+
Expect(err).NotTo(HaveOccurred())
44+
45+
mirrorClusterCfg, err = mirrorClusterTestEnv.Start()
46+
Expect(err).NotTo(HaveOccurred())
47+
48+
referenceClusterClient, err = client.New(referenceClusterCfg, client.Options{})
49+
Expect(err).NotTo(HaveOccurred())
50+
51+
mirrorClusterClient, err = client.New(mirrorClusterCfg, client.Options{})
52+
Expect(err).NotTo(HaveOccurred())
53+
54+
go func() {
55+
run(referenceClusterCfg, mirrorClusterCfg, stop)
56+
}()
57+
})
58+
59+
AfterEach(func() {
60+
close(stop)
61+
Expect(referenceClusterTestEnv.Stop()).To(Succeed())
62+
Expect(mirrorClusterTestEnv.Stop()).To(Succeed())
63+
})
64+
65+
It("Should reconcile pods", func() {
66+
ctx := context.Background()
67+
referencePod := &corev1.Pod{
68+
ObjectMeta: metav1.ObjectMeta{
69+
Namespace: "default",
70+
Name: "satan",
71+
},
72+
Spec: corev1.PodSpec{
73+
Containers: []corev1.Container{{
74+
Name: "fancy-one",
75+
Image: "nginx",
76+
}},
77+
},
78+
}
79+
Expect(referenceClusterClient.Create(ctx, referencePod)).NotTo(HaveOccurred())
80+
name := types.NamespacedName{Namespace: referencePod.Namespace, Name: referencePod.Name}
81+
82+
By("Setting a finalizer", func() {
83+
Eventually(func() error {
84+
updatedPod := &corev1.Pod{}
85+
if err := referenceClusterClient.Get(ctx, name, updatedPod); err != nil {
86+
return err
87+
}
88+
if n := len(updatedPod.Finalizers); n != 1 {
89+
return fmt.Errorf("expected exactly one finalizer, got %d", n)
90+
}
91+
return nil
92+
}).Should(Succeed())
93+
94+
})
95+
96+
By("Creating a pod in the mirror cluster", func() {
97+
Eventually(func() error {
98+
return mirrorClusterClient.Get(ctx, name, &corev1.Pod{})
99+
}).Should(Succeed())
100+
101+
})
102+
103+
By("Recreating a manually deleted pod in the mirror cluster", func() {
104+
Expect(mirrorClusterClient.Delete(ctx,
105+
&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: name.Namespace, Name: name.Name}}),
106+
).NotTo(HaveOccurred())
107+
108+
Eventually(func() error {
109+
return mirrorClusterClient.Get(ctx, name, &corev1.Pod{})
110+
}).Should(Succeed())
111+
112+
})
113+
114+
By("Cleaning up after the reference pod got deleted", func() {
115+
Expect(referenceClusterClient.Delete(ctx, referencePod)).NotTo(HaveOccurred())
116+
117+
Eventually(func() bool {
118+
return apierrors.IsNotFound(mirrorClusterClient.Get(ctx, name, &corev1.Pod{}))
119+
}).Should(BeTrue())
120+
121+
Eventually(func() bool {
122+
return apierrors.IsNotFound(referenceClusterClient.Get(ctx, name, &corev1.Pod{}))
123+
}).Should(BeTrue())
124+
})
125+
})
126+
127+
})
128+
129+
})

‎examples/multicluster/reconciler/reconciler.go‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
/*
2+
Copyright 2020 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package reconciler
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"time"
23+
24+
corev1 "k8s.io/api/core/v1"
25+
apierrors "k8s.io/apimachinery/pkg/api/errors"
26+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
"k8s.io/apimachinery/pkg/types"
28+
"k8s.io/apimachinery/pkg/util/sets"
29+
30+
ctrl "sigs.k8s.io/controller-runtime"
31+
"sigs.k8s.io/controller-runtime/pkg/client"
32+
"sigs.k8s.io/controller-runtime/pkg/clusterconnector"
33+
"sigs.k8s.io/controller-runtime/pkg/handler"
34+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
35+
"sigs.k8s.io/controller-runtime/pkg/source"
36+
)
37+
38+
func Add(mgr ctrl.Manager, mirrorCluster clusterconnector.ClusterConnector) error {
39+
return ctrl.NewControllerManagedBy(mgr).
40+
// Watch Pods in the reference cluster
41+
For(&corev1.Pod{}).
42+
// Watch pods in the mirror cluster
43+
Watches(
44+
source.NewKindWithCache(&corev1.Pod{}, mirrorCluster.GetCache()),
45+
&handler.EnqueueRequestForObject{},
46+
).
47+
Complete(&reconciler{
48+
referenceClusterClient: mgr.GetClient(),
49+
mirrorClusterClient: mirrorCluster.GetClient(),
50+
})
51+
}
52+
53+
type reconciler struct {
54+
referenceClusterClient client.Client
55+
mirrorClusterClient client.Client
56+
}
57+
58+
func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) {
59+
return reconcile.Result{}, r.reconcile(req)
60+
}
61+
62+
const podFinalizerName = "pod-finalzer.mirror.org/v1"
63+
64+
func (r *reconciler) reconcile(req reconcile.Request) error {
65+
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
66+
defer cancel()
67+
68+
referencePod := &corev1.Pod{}
69+
if err := r.referenceClusterClient.Get(ctx, req.NamespacedName, referencePod); err != nil {
70+
if apierrors.IsNotFound(err) {
71+
return nil
72+
}
73+
return fmt.Errorf("failed to get pod from reference clsuster: %w", err)
74+
}
75+
76+
if referencePod.DeletionTimestamp != nil && sets.NewString(referencePod.Finalizers...).Has(podFinalizerName) {
77+
mirrorClusterPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{
78+
Namespace: referencePod.Namespace,
79+
Name: referencePod.Name,
80+
}}
81+
if err := r.mirrorClusterClient.Delete(ctx, mirrorClusterPod); err != nil && !apierrors.IsNotFound(err) {
82+
return fmt.Errorf("failed to delete pod in mirror cluster: %w", err)
83+
}
84+
85+
referencePod.Finalizers = sets.NewString(referencePod.Finalizers...).Delete(podFinalizerName).UnsortedList()
86+
if err := r.referenceClusterClient.Update(ctx, referencePod); err != nil {
87+
return fmt.Errorf("failed to update pod in refernce cluster after removing finalizer: %w", err)
88+
}
89+
90+
return nil
91+
}
92+
93+
if !sets.NewString(referencePod.Finalizers...).Has(podFinalizerName) {
94+
referencePod.Finalizers = append(referencePod.Finalizers, podFinalizerName)
95+
if err := r.referenceClusterClient.Update(ctx, referencePod); err != nil {
96+
return fmt.Errorf("failed to update pod after adding finalizer: %w", err)
97+
}
98+
}
99+
100+
// Check if pod already exists
101+
podName := types.NamespacedName{Namespace: referencePod.Namespace, Name: referencePod.Name}
102+
podExists := true
103+
if err := r.mirrorClusterClient.Get(ctx, podName, &corev1.Pod{}); err != nil {
104+
if !apierrors.IsNotFound(err) {
105+
return fmt.Errorf("failed to check in mirror cluster if pod exists: %w", err)
106+
}
107+
podExists = false
108+
}
109+
if podExists {
110+
return nil
111+
}
112+
113+
mirrorPod := &corev1.Pod{
114+
ObjectMeta: metav1.ObjectMeta{
115+
Namespace: referencePod.Namespace,
116+
Name: referencePod.Name,
117+
},
118+
Spec: *referencePod.Spec.DeepCopy(),
119+
}
120+
if err := r.mirrorClusterClient.Create(ctx, mirrorPod); err != nil {
121+
return fmt.Errorf("failed to create mirror pod: %w", err)
122+
}
123+
124+
return nil
125+
}

0 commit comments

Comments
 (0)