Skip to content

Commit 62aa488

Browse files
yastijyastij
committed
add vmc logic
Signed-off-by: Yassine TIJANI <[email protected]>
1 parent cc649b0 commit 62aa488

File tree

3 files changed

+194
-102
lines changed

3 files changed

+194
-102
lines changed

‎pkg/cloud/vsphere/actuators/cluster/actuator.go‎

Lines changed: 181 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"fmt"
2222
"net/url"
2323
"strconv"
24+
"strings"
2425

2526
"github.com/pkg/errors"
2627

@@ -29,6 +30,7 @@ import (
2930
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3031
corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
3132
"k8s.io/klog/klogr"
33+
elb "sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/services/aws"
3234
clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
3335
clientv1 "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1"
3436
clusterErr "sigs.k8s.io/cluster-api/pkg/controller/error"
@@ -41,6 +43,7 @@ import (
4143
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/context"
4244
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/services/certificates"
4345
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/services/kubeclient"
46+
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/services/kubeconfig"
4447
)
4548

4649
//+kubebuilder:rbac:groups=vsphere.cluster.k8s.io,resources=vsphereclusterproviderspecs;vsphereclusterproviderstatuses,verbs=get;list;watch;create;update;patch;delete
@@ -91,11 +94,22 @@ func (a *Actuator) Reconcile(cluster *clusterv1.Cluster) (opErr error) {
9194
return err
9295
}
9396

94-
if err := a.reconcileCloudConfigSecret(ctx); err != nil {
97+
isVMwareCloud := ctx.ClusterConfig.VmwareCloud != nil
98+
if isVMwareCloud {
99+
if err = a.reconcileLoadBalancers(ctx); err != nil {
100+
return err
101+
}
102+
}
103+
104+
if err := a.reconcileKubeConfig(ctx); err != nil {
95105
return err
96106
}
97107

98-
if err := a.reconcileReadyState(ctx); err != nil {
108+
if err := a.reconcileReadyState(ctx, isVMwareCloud); err != nil {
109+
return err
110+
}
111+
112+
if err := a.reconcileCloudConfigSecret(ctx); err != nil {
99113
return err
100114
}
101115

@@ -130,6 +144,13 @@ func (a *Actuator) Delete(cluster *clusterv1.Cluster) (opErr error) {
130144
return err
131145
}
132146

147+
// Delete the Load balancer if we are using VMC
148+
if ctx.ClusterConfig.VmwareCloud != nil {
149+
if err := a.deleteLoadBalancer(ctx); err != nil {
150+
return err
151+
}
152+
}
153+
133154
return nil
134155
}
135156

@@ -140,7 +161,7 @@ func (a *Actuator) reconcilePKI(ctx *context.ClusterContext) error {
140161
return nil
141162
}
142163

143-
func (a *Actuator) reconcileReadyState(ctx *context.ClusterContext) error {
164+
func (a *Actuator) reconcileReadyState(ctx *context.ClusterContext, isVMwareCloud bool) error {
144165

145166
// Always recalculate the API Endpoints.
146167
ctx.Cluster.Status.APIEndpoints = []clusterv1.APIEndpoint{}
@@ -163,45 +184,45 @@ func (a *Actuator) reconcileReadyState(ctx *context.ClusterContext) error {
163184
ctx.Logger.Error(err, "unable to list nodes for target cluster")
164185
return &clusterErr.RequeueAfterError{RequeueAfter: config.DefaultRequeue}
165186
}
187+
if isVMwareCloud {
188+
// Get the RESTConfig in order to parse its Host to use as the control plane
189+
// endpoint to add to the Cluster's API endpoints.
190+
restConfig := client.RESTConfig()
191+
if restConfig == nil {
192+
ctx.Logger.Error(errors.New("restConfig == nil"), "error getting RESTConfig for kube client")
193+
return &clusterErr.RequeueAfterError{RequeueAfter: config.DefaultRequeue}
194+
}
166195

167-
// Get the RESTConfig in order to parse its Host to use as the control plane
168-
// endpoint to add to the Cluster's API endpoints.
169-
restConfig := client.RESTConfig()
170-
if restConfig == nil {
171-
ctx.Logger.Error(errors.New("restConfig == nil"), "error getting RESTConfig for kube client")
172-
return &clusterErr.RequeueAfterError{RequeueAfter: config.DefaultRequeue}
173-
}
174-
175-
// Calculate the API endpoint for the cluster.
176-
controlPlaneEndpointURL, err := url.Parse(restConfig.Host)
177-
if err != nil {
178-
return errors.Wrapf(err, "unable to parse cluster's restConifg host value: %v", restConfig.Host)
179-
}
196+
// Calculate the API endpoint for the cluster.
197+
controlPlaneEndpointURL, err := url.Parse(restConfig.Host)
198+
if err != nil {
199+
return errors.Wrapf(err, "unable to parse cluster's restConifg host value: %v", restConfig.Host)
200+
}
180201

181-
// The API endpoint may just have a host.
182-
apiEndpoint := clusterv1.APIEndpoint{
183-
Host: controlPlaneEndpointURL.Hostname(),
184-
}
202+
// The API endpoint may just have a host.
203+
apiEndpoint := clusterv1.APIEndpoint{
204+
Host: controlPlaneEndpointURL.Hostname(),
205+
}
185206

186-
// Check to see if there is also a port.
187-
if szPort := controlPlaneEndpointURL.Port(); szPort != "" {
188-
port, err := strconv.Atoi(szPort)
189-
if err != nil {
190-
return errors.Wrapf(err, "unable to get parse host and port for control plane endpoint %q for %q", controlPlaneEndpointURL.Host, ctx)
207+
// Check to see if there is also a port.
208+
if szPort := controlPlaneEndpointURL.Port(); szPort != "" {
209+
port, err := strconv.Atoi(szPort)
210+
if err != nil {
211+
return errors.Wrapf(err, "unable to get parse host and port for control plane endpoint %q for %q", controlPlaneEndpointURL.Host, ctx)
212+
}
213+
apiEndpoint.Port = port
191214
}
192-
apiEndpoint.Port = port
193-
}
194215

195-
// Update the API endpoints.
196-
ctx.Cluster.Status.APIEndpoints = []clusterv1.APIEndpoint{apiEndpoint}
197-
ctx.Logger.V(6).Info("calculated API endpoint for target cluster", "api-endpoint-host", apiEndpoint.Host, "api-endpoint-port", apiEndpoint.Port)
216+
// Update the API endpoints.
217+
ctx.Cluster.Status.APIEndpoints = []clusterv1.APIEndpoint{apiEndpoint}
218+
ctx.Logger.V(6).Info("calculated API endpoint for target cluster", "api-endpoint-host", apiEndpoint.Host, "api-endpoint-port", apiEndpoint.Port)
198219

199-
// Update the kubeadm control plane endpoint with the one from the kubeconfig.
200-
if ctx.ClusterConfig.ClusterConfiguration.ControlPlaneEndpoint != controlPlaneEndpointURL.Host {
201-
ctx.ClusterConfig.ClusterConfiguration.ControlPlaneEndpoint = controlPlaneEndpointURL.Host
202-
ctx.Logger.V(6).Info("stored control plane endpoint in kubeadm cluster config", "control-plane-endpoint", controlPlaneEndpointURL.Host)
220+
// Update the kubeadm control plane endpoint with the one from the kubeconfig.
221+
if ctx.ClusterConfig.ClusterConfiguration.ControlPlaneEndpoint != controlPlaneEndpointURL.Host {
222+
ctx.ClusterConfig.ClusterConfiguration.ControlPlaneEndpoint = controlPlaneEndpointURL.Host
223+
ctx.Logger.V(6).Info("stored control plane endpoint in kubeadm cluster config", "control-plane-endpoint", controlPlaneEndpointURL.Host)
224+
}
203225
}
204-
205226
// Update the ready status.
206227
ctx.ClusterStatus.Ready = true
207228

@@ -262,3 +283,128 @@ func (a *Actuator) deleteControlPlaneConfigMap(ctx *context.ClusterContext) erro
262283
}
263284
return nil
264285
}
286+
287+
func (a *Actuator) deleteLoadBalancer(ctx *context.ClusterContext) error {
288+
elbSvc := elb.New(ctx.ClusterConfig.VmwareCloud.AwsProvider.Region)
289+
clusterName := ctx.ClusterName()
290+
291+
if err := elbSvc.Delete(clusterName); err != nil {
292+
ctx.Logger.Error(err, "cannot delete load balancers")
293+
return &clusterErr.RequeueAfterError{RequeueAfter: config.DefaultRequeue}
294+
}
295+
return nil
296+
}
297+
298+
func (a *Actuator) reconcileLoadBalancers(ctx *context.ClusterContext) error {
299+
300+
ctx.Logger.V(2).Info("Reconciling load balancers")
301+
302+
clusterName := ctx.ClusterName()
303+
controlPlaneMachines, err := ctx.GetControlPlaneMachines()
304+
if err != nil {
305+
ctx.Logger.Error(err, "error getting control plane machines")
306+
return &clusterErr.RequeueAfterError{RequeueAfter: config.DefaultRequeue}
307+
}
308+
controlPlaneIPs := []string{}
309+
for _, controlPlaneMachine := range controlPlaneMachines {
310+
for _, nodeAddress := range controlPlaneMachine.Status.Addresses {
311+
if nodeAddress.Type == apiv1.NodeExternalIP || nodeAddress.Type == apiv1.NodeInternalIP {
312+
controlPlaneIPs = append(controlPlaneIPs, nodeAddress.Address)
313+
break
314+
}
315+
}
316+
}
317+
318+
awsProviderInfo := ctx.ClusterConfig.VmwareCloud.AwsProvider
319+
elbSvc := elb.New(awsProviderInfo.Region)
320+
vpcID := awsProviderInfo.VpcID
321+
subnets := awsProviderInfo.Subnets
322+
loadBalancerDNS, loadBalancerPort, err := elbSvc.Reconcile(vpcID, controlPlaneIPs, clusterName, subnets)
323+
if err != nil {
324+
ctx.Logger.Error(err, "cannot reconcile load balancer")
325+
return &clusterErr.RequeueAfterError{RequeueAfter: config.DefaultRequeue}
326+
}
327+
328+
apiEndpoint := clusterv1.APIEndpoint{
329+
Host: loadBalancerDNS,
330+
Port: loadBalancerPort,
331+
}
332+
333+
// Update the API endpoints.
334+
ctx.Cluster.Status.APIEndpoints = []clusterv1.APIEndpoint{apiEndpoint}
335+
ctx.Logger.V(6).Info("calculated API endpoint for target cluster", "api-endpoint-host", apiEndpoint.Host, "api-endpoint-port", apiEndpoint.Port)
336+
controlPlaneEndpointURL := strings.Join([]string{apiEndpoint.Host, strconv.Itoa(apiEndpoint.Port)}, ":")
337+
// Update the kubeadm control plane endpoint with the one from the kubeconfig.
338+
if ctx.ClusterConfig.ClusterConfiguration.ControlPlaneEndpoint != controlPlaneEndpointURL {
339+
ctx.ClusterConfig.ClusterConfiguration.ControlPlaneEndpoint = controlPlaneEndpointURL
340+
ctx.Logger.V(6).Info("stored control plane endpoint in kubeadm cluster config", "control-plane-endpoint", controlPlaneEndpointURL)
341+
}
342+
343+
return nil
344+
345+
}
346+
347+
// reconcileKubeConfig creates a secret on the management cluster with
348+
// the kubeconfig for target cluster.
349+
func (a *Actuator) reconcileKubeConfig(ctx *context.ClusterContext) error {
350+
351+
// Get the control plane endpoint.
352+
controlPlaneEndpoint, err := ctx.ControlPlaneEndpoint()
353+
if err != nil {
354+
ctx.Logger.Error(err, "requeueing until control plane endpoint is available")
355+
return &clusterErr.RequeueAfterError{RequeueAfter: config.DefaultRequeue}
356+
}
357+
358+
// Create a new kubeconfig for the target cluster.
359+
ctx.Logger.V(6).Info("generating kubeconfig secret", "controlPlaneEndpoint", controlPlaneEndpoint)
360+
kubeConfig, err := kubeconfig.New(ctx.Cluster.Name, controlPlaneEndpoint, ctx.ClusterConfig.CAKeyPair)
361+
if err != nil {
362+
return errors.Wrapf(err, "error generating kubeconfig for %q", ctx)
363+
}
364+
365+
// Define the kubeconfig secret for the target cluster.
366+
secret := &apiv1.Secret{
367+
ObjectMeta: metav1.ObjectMeta{
368+
Namespace: ctx.Cluster.Namespace,
369+
Name: remotev1.KubeConfigSecretName(ctx.Cluster.Name),
370+
OwnerReferences: []metav1.OwnerReference{
371+
{
372+
APIVersion: ctx.Cluster.APIVersion,
373+
Kind: ctx.Cluster.Kind,
374+
Name: ctx.Cluster.Name,
375+
UID: ctx.Cluster.UID,
376+
},
377+
},
378+
},
379+
StringData: map[string]string{
380+
"value": kubeConfig,
381+
},
382+
}
383+
ctx.Logger.V(6).Info("computed kubeconfig", "kubeconfig", kubeConfig)
384+
if exstingSecret, err := a.coreClient.Secrets(ctx.Cluster.Namespace).Get(secret.Name, metav1.GetOptions{}); err != nil {
385+
if apierrors.IsNotFound(err) {
386+
if _, err := a.coreClient.Secrets(ctx.Cluster.Namespace).Create(secret); err != nil {
387+
if !apierrors.IsAlreadyExists(err) {
388+
ctx.Logger.Error(err, "error creating kubeconfig secret")
389+
return &clusterErr.RequeueAfterError{RequeueAfter: config.DefaultRequeue}
390+
}
391+
ctx.Logger.V(6).Info("kubeconfig secret already exists")
392+
} else {
393+
ctx.Logger.V(4).Info("created kubeconfig secret")
394+
}
395+
} else {
396+
ctx.Logger.Error(err, "cannot get the kubeconfig secret")
397+
return &clusterErr.RequeueAfterError{RequeueAfter: config.DefaultRequeue}
398+
}
399+
} else {
400+
secret.ResourceVersion = exstingSecret.ResourceVersion
401+
updatedKubeConfig, err := a.coreClient.Secrets(ctx.Cluster.Namespace).Update(secret)
402+
if err != nil {
403+
ctx.Logger.Error(err, "cannot update the existing kubeconfig")
404+
return &clusterErr.RequeueAfterError{RequeueAfter: config.DefaultRequeue}
405+
}
406+
ctx.Logger.V(6).Info("updated kubeconfig", "kubeconfig", updatedKubeConfig)
407+
}
408+
// Create the kubeconfig secret.
409+
return nil
410+
}

‎pkg/cloud/vsphere/actuators/machine/actuator.go‎

Lines changed: 10 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ import (
3131
clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
3232
clientv1 "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1"
3333
clusterErr "sigs.k8s.io/cluster-api/pkg/controller/error"
34-
remotev1 "sigs.k8s.io/cluster-api/pkg/controller/remote"
3534
controllerClient "sigs.k8s.io/controller-runtime/pkg/client"
3635

3736
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/actuators"
@@ -40,7 +39,6 @@ import (
4039
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/context"
4140
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/services/govmomi"
4241
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/services/kubeclient"
43-
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/cloud/vsphere/services/kubeconfig"
4442
"sigs.k8s.io/cluster-api-provider-vsphere/pkg/tokens"
4543
)
4644

@@ -107,8 +105,8 @@ func (a *Actuator) Create(
107105
if err := a.reconcilePKI(ctx); err != nil {
108106
return err
109107
}
110-
111-
return a.doInitOrJoin(ctx)
108+
dependsOnEndpoint := ctx.ClusterConfig.VmwareCloud != nil
109+
return a.doInitOrJoin(ctx, dependsOnEndpoint)
112110
}
113111

114112
// Delete removes a machine.
@@ -177,10 +175,6 @@ func (a *Actuator) Update(
177175
return err
178176
}
179177

180-
if err := a.reconcileKubeConfig(ctx); err != nil {
181-
return err
182-
}
183-
184178
if err := a.reconcileReadyState(ctx); err != nil {
185179
return err
186180
}
@@ -229,7 +223,14 @@ func (a *Actuator) reconcilePKI(ctx *context.MachineContext) error {
229223
return nil
230224
}
231225

232-
func (a *Actuator) doInitOrJoin(ctx *context.MachineContext) error {
226+
func (a *Actuator) doInitOrJoin(ctx *context.MachineContext, dependsOnEndpoint bool) error {
227+
if dependsOnEndpoint {
228+
_, err := ctx.ControlPlaneEndpoint()
229+
if err != nil {
230+
ctx.Logger.Error(err, "control plane endpoint not set")
231+
return &clusterErr.RequeueAfterError{RequeueAfter: config.DefaultRequeue}
232+
}
233+
}
233234

234235
// Determine whether or not to initialize the control plane, join an
235236
// existing control plane, or join the cluster as a worker node.
@@ -264,60 +265,6 @@ func (a *Actuator) doInitOrJoin(ctx *context.MachineContext) error {
264265
return govmomi.Create(ctx, token)
265266
}
266267

267-
// reconcileKubeConfig creates a secret on the management cluster with
268-
// the kubeconfig for target cluster.
269-
func (a *Actuator) reconcileKubeConfig(ctx *context.MachineContext) error {
270-
if !ctx.HasControlPlaneRole() {
271-
return nil
272-
}
273-
274-
// Get the control plane endpoint.
275-
controlPlaneEndpoint, err := ctx.ControlPlaneEndpoint()
276-
if err != nil {
277-
ctx.Logger.Error(err, "requeueing until control plane endpoint is available")
278-
return &clusterErr.RequeueAfterError{RequeueAfter: config.DefaultRequeue}
279-
}
280-
281-
// Create a new kubeconfig for the target cluster.
282-
ctx.Logger.V(6).Info("generating kubeconfig secret")
283-
kubeConfig, err := kubeconfig.New(ctx.Cluster.Name, controlPlaneEndpoint, ctx.ClusterConfig.CAKeyPair)
284-
if err != nil {
285-
return errors.Wrapf(err, "error generating kubeconfig for %q", ctx)
286-
}
287-
288-
// Define the kubeconfig secret for the target cluster.
289-
secret := &apiv1.Secret{
290-
ObjectMeta: metav1.ObjectMeta{
291-
Namespace: ctx.Cluster.Namespace,
292-
Name: remotev1.KubeConfigSecretName(ctx.Cluster.Name),
293-
OwnerReferences: []metav1.OwnerReference{
294-
{
295-
APIVersion: ctx.Cluster.APIVersion,
296-
Kind: ctx.Cluster.Kind,
297-
Name: ctx.Cluster.Name,
298-
UID: ctx.Cluster.UID,
299-
},
300-
},
301-
},
302-
StringData: map[string]string{
303-
"value": kubeConfig,
304-
},
305-
}
306-
307-
// Create the kubeconfig secret.
308-
if _, err := a.coreClient.Secrets(ctx.Cluster.Namespace).Create(secret); err != nil {
309-
if !apierrors.IsAlreadyExists(err) {
310-
ctx.Logger.Error(err, "error creating kubeconfig secret")
311-
return &clusterErr.RequeueAfterError{RequeueAfter: config.DefaultRequeue}
312-
}
313-
ctx.Logger.V(6).Info("kubeconfig secret already exists")
314-
} else {
315-
ctx.Logger.V(4).Info("created kubeconfig secret")
316-
}
317-
318-
return nil
319-
}
320-
321268
// reconcileReadyState returns a requeue error until the machine appears
322269
// in the target cluster's list of nodes.
323270
func (a *Actuator) reconcileReadyState(ctx *context.MachineContext) error {

0 commit comments

Comments
 (0)