Adds Managed APIServer Loadbalancer (kubernetes-sigs#401)

* Adds Managed APIServer Loadbalancer

This commit adds the optional feature manage APIServer Loadbalancer. If
the feature is enabled a LoadBalancer for the control plane node(s) is created.
The cluster actuator creates the LoadBalancers and all corresponding objects.
The machine actuator creates and removes LoadBalancer members for control plane
nodes.

* add check for lb member

* fix panic

Author: sbueringer

Gopkg.lock

@@ -14,6 +14,23 @@ spec:
   validation:
     openAPIV3Schema:
       properties:
+        apiServerLoadBalancerAdditionalPorts:
+          description: APIServerLoadBalancerAdditionalPorts adds additional ports
+            to the APIServerLoadBalancer
+          items:
+            format: int64
+            type: integer
+          type: array
+        apiServerLoadBalancerFloatingIP:
+          description: APIServerLoadBalancerFloatingIP is the floatingIP which will
+            be associated to the APIServer loadbalancer. The floatingIP will be created
+            if it not already exists.
+          type: string
+        apiServerLoadBalancerPort:
+          description: APIServerLoadBalancerPort is the port on which the listener
+            on the APIServer loadbalancer will be created
+          format: int64
+          type: integer
         apiVersion:
           description: 'APIVersion defines the versioned schema of this representation
             of an object. Servers should convert recognized schemas to the latest
@@ -87,6 +104,10 @@ spec:
                       ipVersion:
                         format: int64
                         type: integer
+                      ipv6AddressMode:
+                        type: string
+                      ipv6RaMode:
+                        type: string
                       limit:
                         format: int64
                         type: integer
@@ -140,6 +161,11 @@ spec:
             object represents. Servers may infer this from the endpoint the client
             submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
           type: string
+        managedAPIServerLoadBalancer:
+          description: 'ManagedAPIServerLoadBalancer defines whether a LoadBalancer
+            for the APIServer should be created. If set to true the following properties
+            are mandatory: APIServerLoadBalancerFloatingIP, APIServerLoadBalancerPort'
+          type: boolean
         managedSecurityGroups:
           description: ManagedSecurityGroups defines that kubernetes manages the OpenStack
             security groups for now, that means that we'll create two security groups,
@@ -177,6 +203,7 @@ spec:
       required:
       - cloudsSecret
       - cloudName
+      - managedAPIServerLoadBalancer
       - managedSecurityGroups
   version: v1alpha1
 status:

config/crds/openstackproviderconfig_v1alpha1_openstackclusterproviderspec.yaml

@@ -129,6 +129,24 @@ spec:
           description: Network contains all information about the created OpenStack
             Network. It includes Subnets and Router.
           properties:
+            apiServerLoadBalancer:
+              description: Be careful when using APIServerLoadBalancer, because this
+                field is optional and therefore not set in all cases
+              properties:
+                id:
+                  type: string
+                internalIP:
+                  type: string
+                ip:
+                  type: string
+                name:
+                  type: string
+              required:
+              - name
+              - id
+              - ip
+              - internalIP
+              type: object
             id:
               type: string
             name:

config/crds/openstackproviderconfig_v1alpha1_openstackclusterproviderstatus.yaml

@@ -203,6 +203,23 @@ type OpenstackClusterProviderSpec struct {
 	// to get public internet to the VMs.
 	ExternalNetworkID string `json:"externalNetworkId,omitempty"`
 
+	// ManagedAPIServerLoadBalancer defines whether a LoadBalancer for the
+	// APIServer should be created. If set to true the following properties are
+	// mandatory: APIServerLoadBalancerFloatingIP, APIServerLoadBalancerPort
+	ManagedAPIServerLoadBalancer bool `json:"managedAPIServerLoadBalancer"`
+
+	// APIServerLoadBalancerFloatingIP is the floatingIP which will be associated
+	// to the APIServer loadbalancer. The floatingIP will be created if it not
+	// already exists.
+	APIServerLoadBalancerFloatingIP string `json:"apiServerLoadBalancerFloatingIP,omitempty"`
+
+	// APIServerLoadBalancerPort is the port on which the listener on the APIServer
+	// loadbalancer will be created
+	APIServerLoadBalancerPort int `json:"apiServerLoadBalancerPort,omitempty"`
+
+	// APIServerLoadBalancerAdditionalPorts adds additional ports to the APIServerLoadBalancer
+	APIServerLoadBalancerAdditionalPorts []int `json:"apiServerLoadBalancerAdditionalPorts,omitempty"`
+
 	// ManagedSecurityGroups defines that kubernetes manages the OpenStack security groups
 	// for now, that means that we'll create two security groups, one allowing SSH
 	// and API access from everywhere, and another one that allows all traffic to/from
@@ -282,6 +299,10 @@ type Network struct {
 
 	Subnet *Subnet `json:"subnet,omitempty"`
 	Router *Router `json:"router,omitempty"`
+
+	// Be careful when using APIServerLoadBalancer, because this field is optional and therefore not
+	// set in all cases
+	APIServerLoadBalancer *LoadBalancer `json:"apiServerLoadBalancer,omitempty"`
 }
 
 // Subnet represents basic information about the associated OpenStack Neutron Subnet
@@ -298,6 +319,14 @@ type Router struct {
 	ID   string `json:"id"`
 }
 
+// LoadBalancer represents basic information about the associated OpenStack LoadBalancer
+type LoadBalancer struct {
+	Name       string `json:"name"`
+	ID         string `json:"id"`
+	IP         string `json:"ip"`
+	InternalIP string `json:"internalIP"`
+}
+
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 

pkg/apis/openstackproviderconfig/v1alpha1/types.go

@@ -81,7 +81,7 @@ func (a *Actuator) Reconcile(cluster *clusterv1.Cluster) error {
 
 	defer func() {
 		if err := a.storeCluster(cluster, clusterCopy, clusterProviderSpec, clusterProviderStatus); err != nil {
-			klog.Errorf("failed to store cluster %q in namespace %q: %v", cluster.Name, cluster.Namespace, err)
+			klog.Errorf("Failed to store cluster %q in namespace %q: %v", cluster.Name, cluster.Namespace, err)
 		}
 	}()
 
@@ -107,6 +107,12 @@ func (a *Actuator) Reconcile(cluster *clusterv1.Cluster) error {
 		if err != nil {
 			return errors.Errorf("failed to reconcile router: %v", err)
 		}
+		if clusterProviderSpec.ManagedAPIServerLoadBalancer {
+			err = networkingService.ReconcileLoadBalancer(clusterName, clusterProviderSpec, clusterProviderStatus)
+			if err != nil {
+				return errors.Errorf("failed to reconcile load balancer: %v", err)
+			}
+		}
 	}
 
 	err = networkingService.ReconcileSecurityGroups(clusterName, *clusterProviderSpec, clusterProviderStatus)
@@ -138,6 +144,7 @@ func (a *Actuator) Reconcile(cluster *clusterv1.Cluster) error {
 		return errors.Wrapf(err, "failed to get kubeconfig secret for cluster %q", cluster.Name)
 	}
 
+	klog.Infof("Reconciled Cluster %s/%s successfully", cluster.Namespace, cluster.Name)
 	return nil
 }
 
@@ -180,9 +187,9 @@ func (a *Actuator) Delete(cluster *clusterv1.Cluster) error {
 	return nil
 }
 
-func (a *Actuator) storeCluster(cluster *clusterv1.Cluster, clusterCopy *clusterv1.Cluster, spec *providerv1.OpenstackClusterProviderSpec, status *providerv1.OpenstackClusterProviderStatus) error {
+func (a *Actuator) storeCluster(cluster *clusterv1.Cluster, clusterCopy *clusterv1.Cluster, clusterProviderSpec *providerv1.OpenstackClusterProviderSpec, clusterProviderStatus *providerv1.OpenstackClusterProviderStatus) error {
 
-	rawSpec, rawStatus, err := providerv1.EncodeClusterSpecAndStatus(cluster, spec, status)
+	rawSpec, rawStatus, err := providerv1.EncodeClusterSpecAndStatus(cluster, clusterProviderSpec, clusterProviderStatus)
 	if err != nil {
 		return err
 	}
@@ -212,6 +219,28 @@ func (a *Actuator) storeCluster(cluster *clusterv1.Cluster, clusterCopy *cluster
 		cluster.ResourceVersion = result.ResourceVersion
 	}
 
+	// set to APIServerLoadBalancer IP & Port for now. With generated kubeadm
+	// this can be changed to clusterConfiguration.controlPlaneEndpoint for the
+	// non-LoadBalancer case. For now it doesn't matter because the Status is not
+	// used yet.
+	apiServerHost := clusterProviderSpec.APIServerLoadBalancerFloatingIP
+	apiServerPort := clusterProviderSpec.APIServerLoadBalancerPort
+	if clusterProviderSpec.ManagedAPIServerLoadBalancer {
+		if clusterProviderStatus.Network != nil && clusterProviderStatus.Network.APIServerLoadBalancer != nil &&
+			apiServerHost != clusterProviderStatus.Network.APIServerLoadBalancer.IP {
+			return fmt.Errorf("APIServerLoadBalancer has IP %s instead of IP %s", clusterProviderStatus.Network.APIServerLoadBalancer.IP, apiServerHost)
+		}
+	}
+	// Check if API endpoints is not set or has changed.
+	if len(cluster.Status.APIEndpoints) == 0 || cluster.Status.APIEndpoints[0].Host != apiServerHost {
+		cluster.Status.APIEndpoints = []clusterv1.APIEndpoint{
+			{
+				Host: apiServerHost,
+				Port: apiServerPort,
+			},
+		}
+	}
+
 	cluster.Status.ProviderStatus = rawStatus
 	if !reflect.DeepEqual(cluster.Status, clusterCopy.Status) {
 		klog.Infof("Updating cluster status %s", cluster.Name)