Skip to content

Commit 7dbaea2

Browse files
yastijyastij
committed
add loadbalancing controller for capv
Signed-off-by: Yassine TIJANI <[email protected]>
1 parent 92fe9d0 commit 7dbaea2

File tree

6 files changed

+241
-53
lines changed

6 files changed

+241
-53
lines changed

‎controllers/loadbalancer_controller.go‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
package controllers
2+
3+
import (
4+
goctx "context"
5+
6+
"github.com/go-logr/logr"
7+
"k8s.io/client-go/tools/record"
8+
infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/api/v1alpha2"
9+
"sigs.k8s.io/cluster-api-provider-vsphere/api/v1alpha2/cloud"
10+
controllerutil "sigs.k8s.io/cluster-api-provider-vsphere/controllers/util"
11+
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/config"
12+
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/services"
13+
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/services/loadbalancer/aws"
14+
infrautilv1 "sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/util"
15+
16+
clusterutilv1 "sigs.k8s.io/cluster-api/util"
17+
ctrl "sigs.k8s.io/controller-runtime"
18+
"sigs.k8s.io/controller-runtime/pkg/client"
19+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
20+
)
21+
22+
type providerFunc func(config *cloud.LoadBalancerConfig) (services.LoadBalancerService, error)
23+
24+
var providerFactory map[string]providerFunc
25+
26+
func init() {
27+
providerFactory = map[string]providerFunc{
28+
cloud.AwsProvider: aws.NewProvider,
29+
}
30+
}
31+
32+
// LoadBalancerReconciler reconciles load balancers
33+
type LoadBalancerReconciler struct {
34+
client.Client
35+
ProviderName string
36+
Recorder record.EventRecorder
37+
Log logr.Logger
38+
}
39+
40+
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=vsphereclusters,verbs=get;list;watch;
41+
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=vsphereclusters/status,verbs=get;update;patch
42+
43+
// Reconcile todo
44+
func (r *LoadBalancerReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, reterr error) {
45+
logger := r.Log.WithName("loadBalancer")
46+
47+
ctx, result, reterr := controllerutil.GetClusterContext(req, logger, r)
48+
if ctx == nil {
49+
return result, reterr
50+
}
51+
vsphereCluster := ctx.VSphereCluster
52+
// check if the cluster does not have Load balancing config
53+
if vsphereCluster.Spec.LoadBalancerConfiguration == nil {
54+
return ctrl.Result{}, reterr
55+
}
56+
// Always close the context when exiting this function so we can persist any VSphereCluster changes.
57+
defer func() {
58+
if err := ctx.Patch(); err != nil && reterr == nil {
59+
reterr = err
60+
}
61+
}()
62+
63+
// Handle deleted clusters
64+
if !vsphereCluster.DeletionTimestamp.IsZero() {
65+
return r.reconcileDelete(vsphereCluster)
66+
}
67+
68+
// Handle non-deleted clusters
69+
return r.reconcileNormal(ctx, vsphereCluster)
70+
}
71+
72+
func (r *LoadBalancerReconciler) reconcileDelete(vsphereCluster *infrav1.VSphereCluster) (_ ctrl.Result, reterr error) {
73+
newProvider := providerFactory[r.ProviderName]
74+
loadBalancerProvider, reterr := newProvider(vsphereCluster.Spec.LoadBalancerConfiguration)
75+
if reterr != nil {
76+
return ctrl.Result{}, reterr
77+
}
78+
if reterr = loadBalancerProvider.Delete(vsphereCluster); reterr != nil {
79+
return reconcile.Result{RequeueAfter: config.DefaultRequeue}, reterr
80+
}
81+
vsphereCluster.Finalizers = clusterutilv1.Filter(vsphereCluster.Finalizers, infrav1.LoadBalancerFinalizer)
82+
return ctrl.Result{}, nil
83+
}
84+
85+
func (r *LoadBalancerReconciler) reconcileNormal(ctx goctx.Context, vsphereCluster *infrav1.VSphereCluster) (_ ctrl.Result, reterr error) {
86+
newProvider, ok := providerFactory[r.ProviderName]
87+
if !ok {
88+
r.Log.V(3).Info("unable to initialize the provider")
89+
return ctrl.Result{}, reterr
90+
}
91+
loadBalancerProvider, reterr := newProvider(vsphereCluster.Spec.LoadBalancerConfiguration)
92+
if reterr != nil {
93+
return ctrl.Result{}, reterr
94+
}
95+
r.Log.V(3).Info("reconciling loadbalancer")
96+
97+
machines, reterr := infrautilv1.GetMachinesInCluster(ctx, r.Client, vsphereCluster.Namespace, vsphereCluster.Name)
98+
if reterr != nil {
99+
return ctrl.Result{}, reterr
100+
}
101+
r.Log.V(6).Info("got machines", "machines", machines)
102+
103+
machines = clusterutilv1.GetControlPlaneMachines(machines)
104+
vsphereMachines, reterr := infrautilv1.GetVSphereMachinesInCluster(ctx, r.Client, vsphereCluster.Namespace, vsphereCluster.Name)
105+
if reterr != nil {
106+
return ctrl.Result{}, reterr
107+
}
108+
r.Log.V(3).Info("got vsphere machines", "vsphere-machines", vsphereMachines)
109+
110+
var controlPlaneIPs []string
111+
for _, controlPlane := range machines {
112+
vsphereMachine, ok := vsphereMachines[controlPlane.Name]
113+
if !ok {
114+
r.Log.V(6).Info("machine not yet linked to the cluster", "machine-name", controlPlane.Name)
115+
continue
116+
}
117+
ip, reterr := infrautilv1.GetMachinePreferredIPAddress(vsphereMachine)
118+
if reterr != nil && reterr == infrautilv1.ErrParseCIDR {
119+
return ctrl.Result{}, reterr
120+
}
121+
if len(ip) != 0 {
122+
controlPlaneIPs = append(controlPlaneIPs, ip)
123+
}
124+
}
125+
r.Log.V(3).Info("gathered controlplane IPs", "controlplane-ips", controlPlaneIPs)
126+
127+
apiEndpoint, reterr := loadBalancerProvider.Reconcile(vsphereCluster, controlPlaneIPs)
128+
if reterr != nil {
129+
return ctrl.Result{}, reterr
130+
}
131+
vsphereCluster.Finalizers = append(vsphereCluster.Finalizers, infrav1.LoadBalancerFinalizer)
132+
vsphereCluster.Status.APIEndpoints = append(vsphereCluster.Status.APIEndpoints, apiEndpoint)
133+
vsphereCluster.Status.Ready = true
134+
return ctrl.Result{}, nil
135+
}
136+
137+
// SetupWithManager adds this controller to the provided manager.
138+
func (r *LoadBalancerReconciler) SetupWithManager(mgr ctrl.Manager) error {
139+
return ctrl.NewControllerManagedBy(mgr).
140+
For(&infrav1.VSphereCluster{}).
141+
Complete(r)
142+
}

‎controllers/util/cluster.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
package util
2+
3+
import (
4+
goctx "context"
5+
"fmt"
6+
7+
"github.com/go-logr/logr"
8+
"github.com/pkg/errors"
9+
apierrors "k8s.io/apimachinery/pkg/api/errors"
10+
clusterutilv1 "sigs.k8s.io/cluster-api/util"
11+
ctrl "sigs.k8s.io/controller-runtime"
12+
"sigs.k8s.io/controller-runtime/pkg/client"
13+
"sigs.k8s.io/controller-runtime/pkg/reconcile"
14+
15+
infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/api/v1alpha2"
16+
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/config"
17+
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/context"
18+
)
19+
20+
// GetClusterContext creates clusterContext wrapped with logs
21+
func GetClusterContext(req ctrl.Request, logger logr.Logger, client client.Client) (ctx *context.ClusterContext, _ ctrl.Result, reterr error) {
22+
parentContext := goctx.Background()
23+
24+
logger = logger.WithName(fmt.Sprintf("namespace=%s", req.Namespace)).
25+
WithName(fmt.Sprintf("vsphereCluster=%s", req.Name))
26+
27+
// Fetch the VSphereCluster instance
28+
vsphereCluster := &infrav1.VSphereCluster{}
29+
reterr = client.Get(parentContext, req.NamespacedName, vsphereCluster)
30+
if reterr != nil {
31+
if apierrors.IsNotFound(reterr) {
32+
return nil, reconcile.Result{}, nil
33+
}
34+
return nil, reconcile.Result{}, reterr
35+
}
36+
37+
logger = logger.WithName(vsphereCluster.APIVersion)
38+
39+
// Fetch the Cluster.
40+
cluster, reterr := clusterutilv1.GetOwnerCluster(parentContext, client, vsphereCluster.ObjectMeta)
41+
if reterr != nil {
42+
return nil, reconcile.Result{}, reterr
43+
}
44+
if cluster == nil {
45+
logger.Info("Waiting for Cluster Controller to set OwnerRef on VSphereCluster")
46+
return nil, reconcile.Result{RequeueAfter: config.DefaultRequeue}, nil
47+
}
48+
49+
logger = logger.WithName(fmt.Sprintf("cluster=%s", cluster.Name))
50+
51+
// Create the context.
52+
ctx, reterr = context.NewClusterContext(&context.ClusterContextParams{
53+
Context: parentContext,
54+
Cluster: cluster,
55+
VSphereCluster: vsphereCluster,
56+
Client: client,
57+
Logger: logger,
58+
})
59+
if reterr != nil {
60+
return nil, reconcile.Result{}, errors.Errorf("failed to create cluster context: %+v", reterr)
61+
}
62+
return ctx, reconcile.Result{}, nil
63+
}

‎controllers/vspherecluster_controller.go‎

Lines changed: 14 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ limitations under the License.
1717
package controllers
1818

1919
import (
20-
goctx "context"
2120
"fmt"
2221

2322
"github.com/go-logr/logr"
@@ -26,13 +25,13 @@ import (
2625
apierrors "k8s.io/apimachinery/pkg/api/errors"
2726
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2827
"k8s.io/client-go/tools/record"
28+
controllerutil "sigs.k8s.io/cluster-api-provider-vsphere/controllers/util"
2929
clusterutilv1 "sigs.k8s.io/cluster-api/util"
3030
ctrl "sigs.k8s.io/controller-runtime"
3131
"sigs.k8s.io/controller-runtime/pkg/client"
3232
"sigs.k8s.io/controller-runtime/pkg/reconcile"
3333

3434
infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/api/v1alpha2"
35-
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/config"
3635
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/context"
3736
infrautilv1 "sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/util"
3837
)
@@ -56,48 +55,13 @@ type VSphereClusterReconciler struct {
5655

5756
// Reconcile ensures the back-end state reflects the Kubernetes resource state intent.
5857
func (r *VSphereClusterReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, reterr error) {
59-
parentContext := goctx.Background()
58+
logger := r.Log.WithName(controllerName)
6059

61-
logger := r.Log.WithName(controllerName).
62-
WithName(fmt.Sprintf("namespace=%s", req.Namespace)).
63-
WithName(fmt.Sprintf("vsphereCluster=%s", req.Name))
64-
65-
// Fetch the VSphereCluster instance
66-
vsphereCluster := &infrav1.VSphereCluster{}
67-
err := r.Get(parentContext, req.NamespacedName, vsphereCluster)
68-
if err != nil {
69-
if apierrors.IsNotFound(err) {
70-
return reconcile.Result{}, nil
71-
}
72-
return reconcile.Result{}, err
60+
ctx, result, reterr := controllerutil.GetClusterContext(req, logger, r)
61+
if ctx == nil {
62+
return result, reterr
7363
}
74-
75-
logger = logger.WithName(vsphereCluster.APIVersion)
76-
77-
// Fetch the Cluster.
78-
cluster, err := clusterutilv1.GetOwnerCluster(parentContext, r.Client, vsphereCluster.ObjectMeta)
79-
if err != nil {
80-
return reconcile.Result{}, err
81-
}
82-
if cluster == nil {
83-
logger.Info("Waiting for Cluster Controller to set OwnerRef on VSphereCluster")
84-
return reconcile.Result{RequeueAfter: config.DefaultRequeue}, nil
85-
}
86-
87-
logger = logger.WithName(fmt.Sprintf("cluster=%s", cluster.Name))
88-
89-
// Create the context.
90-
ctx, err := context.NewClusterContext(&context.ClusterContextParams{
91-
Context: parentContext,
92-
Cluster: cluster,
93-
VSphereCluster: vsphereCluster,
94-
Client: r.Client,
95-
Logger: logger,
96-
})
97-
if err != nil {
98-
return reconcile.Result{}, errors.Errorf("failed to create cluster context: %+v", err)
99-
}
100-
64+
vsphereCluster := ctx.VSphereCluster
10165
// Always close the context when exiting this function so we can persist any VSphereCluster changes.
10266
defer func() {
10367
if err := ctx.Patch(); err != nil && reterr == nil {
@@ -130,7 +94,7 @@ func (r *VSphereClusterReconciler) reconcileNormal(ctx *context.ClusterContext)
13094
// * Downloading OVAs into the content library for any machines that
13195
// use them.
13296
// * Create any load balancers for VMC on AWS, etc.
133-
ctx.VSphereCluster.Status.Ready = true
97+
ctx.VSphereCluster.Status.Ready = ctx.VSphereCluster.Spec.LoadBalancerConfiguration == nil
13498
ctx.Logger.V(6).Info("VSphereCluster is infrastructure-ready")
13599

136100
// If the VSphereCluster doesn't have our finalizer, add it.
@@ -141,12 +105,13 @@ func (r *VSphereClusterReconciler) reconcileNormal(ctx *context.ClusterContext)
141105
"cluster-namespace", ctx.VSphereCluster.Namespace,
142106
"cluster-name", ctx.VSphereCluster.Name)
143107
}
144-
145-
// Update the VSphereCluster resource with its API enpoints.
146-
if err := r.reconcileAPIEndpoints(ctx); err != nil {
147-
return reconcile.Result{}, errors.Wrapf(err,
148-
"failed to reconcile API endpoints for VSphereCluster %s/%s",
149-
ctx.VSphereCluster.Namespace, ctx.VSphereCluster.Name)
108+
if ctx.VSphereCluster.Spec.LoadBalancerConfiguration == nil {
109+
// Update the VSphereCluster resource with its API enpoints.
110+
if err := r.reconcileAPIEndpoints(ctx); err != nil {
111+
return reconcile.Result{}, errors.Wrapf(err,
112+
"failed to reconcile API endpoints for VSphereCluster %s/%s",
113+
ctx.VSphereCluster.Namespace, ctx.VSphereCluster.Name)
114+
}
150115
}
151116

152117
// Create the cloud config secret for the target cluster.

‎controllers/vspheremachine_controller.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ func (r *VSphereMachineReconciler) reconcileNormal(ctx *context.MachineContext)
196196

197197
if !ctx.Cluster.Status.InfrastructureReady {
198198
ctx.Logger.Info("Cluster infrastructure is not ready yet, requeuing machine")
199-
return reconcile.Result{RequeueAfter: config.DefaultRequeue}, nil
199+
return reconcile.Result{}, nil
200200
}
201201

202202
// Make sure bootstrap data is available and populated.

‎main.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,9 @@ func main() {
8383
"Bind address to expose the pprof profiler")
8484
syncPeriod := flag.Duration("sync-period", defaultSyncPeriod,
8585
"The interval at which cluster-api objects are synchronized")
86+
loadBalancerProvider := flag.String(
87+
"loadbalancer-provider", "",
88+
"the name of the load balancer provider")
8689
flag.DurationVar(&config.DefaultRequeue, "requeue-period", defaultRequeuePeriod,
8790
"The default amount of time to wait before an operation is requeued.")
8891
flag.Parse()
@@ -129,6 +132,17 @@ func main() {
129132
setupLog.Error(err, "unable to create controller", "controller", "VSphereCluster")
130133
os.Exit(1)
131134
}
135+
if *loadBalancerProvider != "" {
136+
if err = (&controllers.LoadBalancerReconciler{
137+
Client: mgr.GetClient(),
138+
ProviderName: *loadBalancerProvider,
139+
Log: ctrl.Log.WithName("controllers").WithName("loadBalancer"),
140+
Recorder: mgr.GetEventRecorderFor("loadBalancer-controller"),
141+
}).SetupWithManager(mgr); err != nil {
142+
setupLog.Error(err, "unable to create controller", "controller", "loadBalancer")
143+
os.Exit(1)
144+
}
145+
}
132146
// +kubebuilder:scaffold:builder
133147

134148
setupLog.Info("starting manager")

‎pkg/cloud/vsphere/util/machines.go‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ func GetMachinesInCluster(
6363
func GetVSphereMachinesInCluster(
6464
ctx context.Context,
6565
controllerClient client.Client,
66-
namespace, clusterName string) ([]*infrav1.VSphereMachine, error) {
66+
namespace, clusterName string) (map[string]*infrav1.VSphereMachine, error) {
6767

6868
labels := map[string]string{clusterv1.MachineClusterLabelName: clusterName}
6969
machineList := &infrav1.VSphereMachineList{}
@@ -75,9 +75,10 @@ func GetVSphereMachinesInCluster(
7575
return nil, err
7676
}
7777

78-
machines := make([]*infrav1.VSphereMachine, len(machineList.Items))
78+
machines := map[string]*infrav1.VSphereMachine{}
7979
for i := range machineList.Items {
80-
machines[i] = &machineList.Items[i]
80+
machineName := machineList.Items[i].Name
81+
machines[machineName] = &machineList.Items[i]
8182
}
8283

8384
return machines, nil
@@ -112,6 +113,9 @@ func GetMachineManagedObjectReference(machine *infrav1.VSphereMachine) vim25type
112113
// ErrNoMachineIPAddr indicates that no valid IP addresses were found in a machine context
113114
var ErrNoMachineIPAddr = errors.New("no IP addresses found for machine")
114115

116+
// ErrParseCIDR indicates that we cannot parse the API server CIDR
117+
var ErrParseCIDR = errors.New("error parsing preferred API server CIDR")
118+
115119
// GetMachinePreferredIPAddress returns the preferred IP address for a
116120
// VSphereMachine resource.
117121
func GetMachinePreferredIPAddress(machine *infrav1.VSphereMachine) (string, error) {

0 commit comments

Comments
 (0)