From e78f0b83b0297590043826138b33f7c050f1ba53 Mon Sep 17 00:00:00 2001 From: Mario Constanti Date: Mon, 9 May 2022 13:18:33 +0200 Subject: [PATCH] Feature: restrict API Server LB access * Access to API-Server can be restricted by setting allowedCidrs on spec.APIServerLoadBalancer.AllowedCIDRs field * Default LB provider is set to amphora if multiple providers exist. Features like access restriction via IPs is only supported in amphora. * Outgoing NAT Router-IPs are now part of the OpenStackCluster status. * To support the entire feature set of conversion, the cluster.x-k8s.io/conversion-data annotation got introduced if quering objects in a previous api version. * make generate now regenerate the mock loadbalancer_service client if needed Signed-off-by: Mario Constanti --- api/v1alpha3/conversion.go | 58 +++++++- api/v1alpha3/conversion_test.go | 123 ++++++++++++----- api/v1alpha3/zz_generated.conversion.go | 72 ++++++---- api/v1alpha4/conversion.go | 58 +++++++- api/v1alpha4/conversion_test.go | 112 ++++++++++++++-- api/v1alpha4/zz_generated.conversion.go | 72 ++++++---- api/v1alpha5/openstackcluster_webhook.go | 6 + api/v1alpha5/openstackcluster_webhook_test.go | 29 ++++ api/v1alpha5/types.go | 10 +- api/v1alpha5/zz_generated.deepcopy.go | 17 ++- ...re.cluster.x-k8s.io_openstackclusters.yaml | 46 +++++-- ...er.x-k8s.io_openstackclustertemplates.yaml | 10 +- ...re.cluster.x-k8s.io_openstackmachines.yaml | 8 +- ...er.x-k8s.io_openstackmachinetemplates.yaml | 8 +- .../src/clusteropenstack/configuration.md | 76 +++++++++-- go.mod | 53 ++++---- go.sum | 103 ++++++++++----- main.go | 4 + pkg/cloud/services/loadbalancer/client.go | 21 +++ .../services/loadbalancer/loadbalancer.go | 124 ++++++++++++++++-- .../loadbalancer/loadbalancer_test.go | 17 +++ .../loadbalancer/mock_loadbalancer/doc.go | 20 +++ .../loadbalancer_service_mock.go | 31 +++++ pkg/cloud/services/networking/router.go | 6 + pkg/utils/openstack/loadbalancer.go | 81 ++++++++++++ pkg/utils/strings/strings.go | 29 ++++ 26 files changed, 1000 insertions(+), 194 deletions(-) create mode 100644 pkg/cloud/services/loadbalancer/mock_loadbalancer/doc.go create mode 100644 pkg/utils/openstack/loadbalancer.go create mode 100644 pkg/utils/strings/strings.go diff --git a/api/v1alpha3/conversion.go b/api/v1alpha3/conversion.go index 1cc50d7b21..ddf54195fc 100644 --- a/api/v1alpha3/conversion.go +++ b/api/v1alpha3/conversion.go @@ -21,6 +21,7 @@ import ( corev1 "k8s.io/api/core/v1" conversion "k8s.io/apimachinery/pkg/conversion" + utilconversion "sigs.k8s.io/cluster-api/util/conversion" ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha5" @@ -31,13 +32,36 @@ var _ ctrlconversion.Convertible = &OpenStackCluster{} func (r *OpenStackCluster) ConvertTo(dstRaw ctrlconversion.Hub) error { dst := dstRaw.(*infrav1.OpenStackCluster) - return Convert_v1alpha3_OpenStackCluster_To_v1alpha5_OpenStackCluster(r, dst, nil) + if err := Convert_v1alpha3_OpenStackCluster_To_v1alpha5_OpenStackCluster(r, dst, nil); err != nil { + return err + } + + // Manually restore data. + restored := &infrav1.OpenStackCluster{} + if ok, err := utilconversion.UnmarshalData(r, restored); err != nil || !ok { + return err + } + + if restored.Spec.APIServerLoadBalancer.AllowedCIDRs != nil { + dst.Spec.APIServerLoadBalancer.AllowedCIDRs = restored.Spec.APIServerLoadBalancer.AllowedCIDRs + } + + return nil } func (r *OpenStackCluster) ConvertFrom(srcRaw ctrlconversion.Hub) error { src := srcRaw.(*infrav1.OpenStackCluster) - return Convert_v1alpha5_OpenStackCluster_To_v1alpha3_OpenStackCluster(src, r, nil) + if err := Convert_v1alpha5_OpenStackCluster_To_v1alpha3_OpenStackCluster(src, r, nil); err != nil { + return err + } + + // Preserve Hub data on down-conversion except for metadata + if err := utilconversion.MarshalData(src, r); err != nil { + return err + } + + return nil } var _ ctrlconversion.Convertible = &OpenStackClusterList{} @@ -65,7 +89,16 @@ func (r *OpenStackMachine) ConvertTo(dstRaw ctrlconversion.Hub) error { func (r *OpenStackMachine) ConvertFrom(srcRaw ctrlconversion.Hub) error { src := srcRaw.(*infrav1.OpenStackMachine) - return Convert_v1alpha5_OpenStackMachine_To_v1alpha3_OpenStackMachine(src, r, nil) + if err := Convert_v1alpha5_OpenStackMachine_To_v1alpha3_OpenStackMachine(src, r, nil); err != nil { + return err + } + + // Preserve Hub data on down-conversion except for metadata + if err := utilconversion.MarshalData(src, r); err != nil { + return err + } + + return nil } var _ ctrlconversion.Convertible = &OpenStackMachineList{} @@ -93,7 +126,16 @@ func (r *OpenStackMachineTemplate) ConvertTo(dstRaw ctrlconversion.Hub) error { func (r *OpenStackMachineTemplate) ConvertFrom(srcRaw ctrlconversion.Hub) error { src := srcRaw.(*infrav1.OpenStackMachineTemplate) - return Convert_v1alpha5_OpenStackMachineTemplate_To_v1alpha3_OpenStackMachineTemplate(src, r, nil) + if err := Convert_v1alpha5_OpenStackMachineTemplate_To_v1alpha3_OpenStackMachineTemplate(src, r, nil); err != nil { + return err + } + + // Preserve Hub data on down-conversion except for metadata + if err := utilconversion.MarshalData(src, r); err != nil { + return err + } + + return nil } var _ ctrlconversion.Convertible = &OpenStackMachineTemplateList{} @@ -318,3 +360,11 @@ func Convert_v1alpha5_Instance_To_v1alpha3_Instance(in *infrav1.Instance, out *I } return nil } + +func Convert_v1alpha5_Router_To_v1alpha3_Router(in *infrav1.Router, out *Router, s conversion.Scope) error { + return autoConvert_v1alpha5_Router_To_v1alpha3_Router(in, out, s) +} + +func Convert_v1alpha5_LoadBalancer_To_v1alpha3_LoadBalancer(in *infrav1.LoadBalancer, out *LoadBalancer, s conversion.Scope) error { + return autoConvert_v1alpha5_LoadBalancer_To_v1alpha3_LoadBalancer(in, out, s) +} diff --git a/api/v1alpha3/conversion_test.go b/api/v1alpha3/conversion_test.go index 8a9b4932f6..096ae4e804 100644 --- a/api/v1alpha3/conversion_test.go +++ b/api/v1alpha3/conversion_test.go @@ -22,6 +22,7 @@ import ( fuzz "github.com/google/gofuzz" "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/api/apitesting/fuzzer" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" utilconversion "sigs.k8s.io/cluster-api/util/conversion" @@ -99,6 +100,11 @@ func TestConvertFrom(t *testing.T) { ManagedAPIServerLoadBalancer: true, APIServerLoadBalancerAdditionalPorts: []int{80, 443}, }, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "cluster.x-k8s.io/conversion-data": "{\"spec\":{\"allowAllInClusterTraffic\":false,\"apiServerLoadBalancer\":{\"additionalPorts\":[80,443],\"enabled\":true},\"cloudName\":\"\",\"controlPlaneEndpoint\":{\"host\":\"\",\"port\":0},\"disableAPIServerFloatingIP\":false,\"managedSecurityGroups\":false,\"network\":{},\"subnet\":{}},\"status\":{\"ready\":false}}", + }, + }, }, }, } @@ -121,15 +127,24 @@ func TestFuzzyConversion(t *testing.T) { fuzzerFuncs := func(_ runtimeserializer.CodecFactory) []interface{} { return []interface{}{ // Don't test spoke-hub-spoke conversion of v1alpha3 fields which are not in v1alpha5 - func(v1alpha3ClusterSpec *OpenStackClusterSpec, c fuzz.Continue) { - c.FuzzNoCustom(v1alpha3ClusterSpec) + func(v1alpha3Cluster *OpenStackCluster, c fuzz.Continue) { + c.FuzzNoCustom(v1alpha3Cluster) - v1alpha3ClusterSpec.UseOctavia = false + v1alpha3Cluster.ObjectMeta.Annotations = map[string]string{} - if v1alpha3ClusterSpec.CloudsSecret != nil { + v1alpha3Cluster.Spec.UseOctavia = false + if v1alpha3Cluster.Spec.CloudsSecret != nil { // In switching to IdentityRef, fetching the cloud secret // from a different namespace is no longer supported - v1alpha3ClusterSpec.CloudsSecret.Namespace = "" + v1alpha3Cluster.Spec.CloudsSecret.Namespace = "" + } + + if v1alpha3Cluster.Spec.Bastion != nil { + v1alpha3Cluster.Spec.Bastion.Instance.Image = "" + v1alpha3Cluster.Spec.Bastion.Instance.UserDataSecret = nil + if v1alpha3Cluster.Spec.Bastion.Instance.CloudsSecret != nil { + v1alpha3Cluster.Spec.Bastion.Instance.CloudsSecret.Namespace = "" + } } }, func(v1alpha3RootVolume *RootVolume, c fuzz.Continue) { @@ -139,20 +154,34 @@ func TestFuzzyConversion(t *testing.T) { v1alpha3RootVolume.DeviceType = "disk" v1alpha3RootVolume.SourceType = "image" }, - func(v1alpha3MachineSpec *OpenStackMachineSpec, c fuzz.Continue) { - c.FuzzNoCustom(v1alpha3MachineSpec) + func(v1alpha3Machine *OpenStackMachine, c fuzz.Continue) { + c.FuzzNoCustom(v1alpha3Machine) + + v1alpha3Machine.ObjectMeta.Annotations = map[string]string{} - v1alpha3MachineSpec.UserDataSecret = nil + v1alpha3Machine.Spec.UserDataSecret = nil - if v1alpha3MachineSpec.CloudsSecret != nil { + if v1alpha3Machine.Spec.CloudsSecret != nil { // In switching to IdentityRef, fetching the cloud secret // from a different namespace is no longer supported - v1alpha3MachineSpec.CloudsSecret.Namespace = "" + v1alpha3Machine.Spec.CloudsSecret.Namespace = "" } - if v1alpha3MachineSpec.RootVolume != nil { + if v1alpha3Machine.Spec.RootVolume != nil { // OpenStackMachineSpec.Image is ignored in v1alpha3 if RootVolume is set - v1alpha3MachineSpec.Image = "" + v1alpha3Machine.Spec.Image = "" + } + }, + func(v1alpha3MachineTemplate *OpenStackMachineTemplate, c fuzz.Continue) { + c.FuzzNoCustom(v1alpha3MachineTemplate) + + v1alpha3MachineTemplate.ObjectMeta.Annotations = map[string]string{} + + v1alpha3MachineTemplate.Spec.Template.Spec.Image = "" + v1alpha3MachineTemplate.Spec.Template.Spec.UserDataSecret = nil + + if v1alpha3MachineTemplate.Spec.Template.Spec.CloudsSecret != nil { + v1alpha3MachineTemplate.Spec.Template.Spec.CloudsSecret.Namespace = "" } }, func(v1alpha3Instance *Instance, c fuzz.Continue) { @@ -193,33 +222,67 @@ func TestFuzzyConversion(t *testing.T) { }, // Don't test hub-spoke-hub conversion of v1alpha5 fields which are not in v1alpha3 - func(v1alpha5ClusterSpec *infrav1.OpenStackClusterSpec, c fuzz.Continue) { - c.FuzzNoCustom(v1alpha5ClusterSpec) + func(v1alpha5Cluster *infrav1.OpenStackCluster, c fuzz.Continue) { + c.FuzzNoCustom(v1alpha5Cluster) + + v1alpha5Cluster.ObjectMeta.Annotations = map[string]string{} + + v1alpha5Cluster.Spec.APIServerFixedIP = "" + v1alpha5Cluster.Spec.AllowAllInClusterTraffic = false + v1alpha5Cluster.Spec.DisableAPIServerFloatingIP = false + v1alpha5Cluster.Spec.APIServerLoadBalancer.AllowedCIDRs = nil + if v1alpha5Cluster.Spec.Bastion != nil { + v1alpha5Cluster.Spec.Bastion.Instance.ImageUUID = "" + v1alpha5Cluster.Spec.Bastion.Instance.Ports = nil + } + + v1alpha5Cluster.Status.FailureMessage = nil + v1alpha5Cluster.Status.FailureReason = nil - v1alpha5ClusterSpec.APIServerFixedIP = "" - v1alpha5ClusterSpec.AllowAllInClusterTraffic = false - v1alpha5ClusterSpec.DisableAPIServerFloatingIP = false + if v1alpha5Cluster.Status.Bastion != nil { + v1alpha5Cluster.Status.Bastion.ImageUUID = "" + v1alpha5Cluster.Status.Bastion.Networks = nil + } + + if v1alpha5Cluster.Status.Network != nil { + if v1alpha5Cluster.Status.Network.APIServerLoadBalancer != nil { + v1alpha5Cluster.Status.Network.APIServerLoadBalancer.AllowedCIDRs = nil + } + if v1alpha5Cluster.Status.Network.Router != nil { + v1alpha5Cluster.Status.Network.Router.IPs = []string{} + } + } + + if v1alpha5Cluster.Status.ExternalNetwork != nil { + if v1alpha5Cluster.Status.ExternalNetwork.APIServerLoadBalancer != nil { + v1alpha5Cluster.Status.ExternalNetwork.APIServerLoadBalancer.AllowedCIDRs = nil + } + if v1alpha5Cluster.Status.ExternalNetwork.Router != nil { + v1alpha5Cluster.Status.ExternalNetwork.Router.IPs = []string{} + } + } }, - func(v1alpha5MachineSpec *infrav1.OpenStackMachineSpec, c fuzz.Continue) { - c.FuzzNoCustom(v1alpha5MachineSpec) + func(v1alpha5Machine *infrav1.OpenStackMachine, c fuzz.Continue) { + c.FuzzNoCustom(v1alpha5Machine) - v1alpha5MachineSpec.Ports = nil - v1alpha5MachineSpec.ImageUUID = "" + v1alpha5Machine.ObjectMeta.Annotations = map[string]string{} + v1alpha5Machine.Spec.Ports = nil + v1alpha5Machine.Spec.ImageUUID = "" + }, + func(v1alpha5MachineTemplate *infrav1.OpenStackMachineTemplate, c fuzz.Continue) { + c.FuzzNoCustom(v1alpha5MachineTemplate) + + v1alpha5MachineTemplate.ObjectMeta.Annotations = map[string]string{} + + v1alpha5MachineTemplate.Spec.Template.Spec.Image = "" + v1alpha5MachineTemplate.Spec.Template.Spec.ImageUUID = "" + v1alpha5MachineTemplate.Spec.Template.Spec.Ports = nil }, func(v1alpha5Network *infrav1.Network, c fuzz.Continue) { c.FuzzNoCustom(v1alpha5Network) v1alpha5Network.PortOpts = nil }, - func(v1alpha5ClusterStatus *infrav1.OpenStackClusterStatus, c fuzz.Continue) { - c.FuzzNoCustom(v1alpha5ClusterStatus) - - v1alpha5ClusterStatus.FailureMessage = nil - v1alpha5ClusterStatus.FailureReason = nil - if v1alpha5ClusterStatus.Bastion != nil { - v1alpha5ClusterStatus.Bastion.ImageUUID = "" - } - }, func(v1alpha5OpenStackIdentityRef *infrav1.OpenStackIdentityReference, c fuzz.Continue) { c.FuzzNoCustom(v1alpha5OpenStackIdentityRef) diff --git a/api/v1alpha3/zz_generated.conversion.go b/api/v1alpha3/zz_generated.conversion.go index 0fe68a9881..3d2ad5e8c4 100644 --- a/api/v1alpha3/zz_generated.conversion.go +++ b/api/v1alpha3/zz_generated.conversion.go @@ -65,11 +65,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha5.LoadBalancer)(nil), (*LoadBalancer)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha5_LoadBalancer_To_v1alpha3_LoadBalancer(a.(*v1alpha5.LoadBalancer), b.(*LoadBalancer), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*Network)(nil), (*v1alpha5.Network)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_Network_To_v1alpha5_Network(a.(*Network), b.(*v1alpha5.Network), scope) }); err != nil { @@ -185,11 +180,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha5.Router)(nil), (*Router)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha5_Router_To_v1alpha3_Router(a.(*v1alpha5.Router), b.(*Router), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*SecurityGroup)(nil), (*v1alpha5.SecurityGroup)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_SecurityGroup_To_v1alpha5_SecurityGroup(a.(*SecurityGroup), b.(*v1alpha5.SecurityGroup), scope) }); err != nil { @@ -300,6 +290,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha5.LoadBalancer)(nil), (*LoadBalancer)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha5_LoadBalancer_To_v1alpha3_LoadBalancer(a.(*v1alpha5.LoadBalancer), b.(*LoadBalancer), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha5.NetworkFilter)(nil), (*Filter)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha5_NetworkFilter_To_v1alpha3_Filter(a.(*v1alpha5.NetworkFilter), b.(*Filter), scope) }); err != nil { @@ -330,6 +325,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha5.Router)(nil), (*Router)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha5_Router_To_v1alpha3_Router(a.(*v1alpha5.Router), b.(*Router), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha5.SubnetFilter)(nil), (*SubnetFilter)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha5_SubnetFilter_To_v1alpha3_SubnetFilter(a.(*v1alpha5.SubnetFilter), b.(*SubnetFilter), scope) }); err != nil { @@ -494,21 +494,33 @@ func autoConvert_v1alpha5_LoadBalancer_To_v1alpha3_LoadBalancer(in *v1alpha5.Loa out.ID = in.ID out.IP = in.IP out.InternalIP = in.InternalIP + // WARNING: in.AllowedCIDRs requires manual conversion: does not exist in peer-type return nil } -// Convert_v1alpha5_LoadBalancer_To_v1alpha3_LoadBalancer is an autogenerated conversion function. -func Convert_v1alpha5_LoadBalancer_To_v1alpha3_LoadBalancer(in *v1alpha5.LoadBalancer, out *LoadBalancer, s conversion.Scope) error { - return autoConvert_v1alpha5_LoadBalancer_To_v1alpha3_LoadBalancer(in, out, s) -} - func autoConvert_v1alpha3_Network_To_v1alpha5_Network(in *Network, out *v1alpha5.Network, s conversion.Scope) error { out.Name = in.Name out.ID = in.ID out.Tags = *(*[]string)(unsafe.Pointer(&in.Tags)) out.Subnet = (*v1alpha5.Subnet)(unsafe.Pointer(in.Subnet)) - out.Router = (*v1alpha5.Router)(unsafe.Pointer(in.Router)) - out.APIServerLoadBalancer = (*v1alpha5.LoadBalancer)(unsafe.Pointer(in.APIServerLoadBalancer)) + if in.Router != nil { + in, out := &in.Router, &out.Router + *out = new(v1alpha5.Router) + if err := Convert_v1alpha3_Router_To_v1alpha5_Router(*in, *out, s); err != nil { + return err + } + } else { + out.Router = nil + } + if in.APIServerLoadBalancer != nil { + in, out := &in.APIServerLoadBalancer, &out.APIServerLoadBalancer + *out = new(v1alpha5.LoadBalancer) + if err := Convert_v1alpha3_LoadBalancer_To_v1alpha5_LoadBalancer(*in, *out, s); err != nil { + return err + } + } else { + out.APIServerLoadBalancer = nil + } return nil } @@ -523,8 +535,24 @@ func autoConvert_v1alpha5_Network_To_v1alpha3_Network(in *v1alpha5.Network, out out.Tags = *(*[]string)(unsafe.Pointer(&in.Tags)) out.Subnet = (*Subnet)(unsafe.Pointer(in.Subnet)) // WARNING: in.PortOpts requires manual conversion: does not exist in peer-type - out.Router = (*Router)(unsafe.Pointer(in.Router)) - out.APIServerLoadBalancer = (*LoadBalancer)(unsafe.Pointer(in.APIServerLoadBalancer)) + if in.Router != nil { + in, out := &in.Router, &out.Router + *out = new(Router) + if err := Convert_v1alpha5_Router_To_v1alpha3_Router(*in, *out, s); err != nil { + return err + } + } else { + out.Router = nil + } + if in.APIServerLoadBalancer != nil { + in, out := &in.APIServerLoadBalancer, &out.APIServerLoadBalancer + *out = new(LoadBalancer) + if err := Convert_v1alpha5_LoadBalancer_To_v1alpha3_LoadBalancer(*in, *out, s); err != nil { + return err + } + } else { + out.APIServerLoadBalancer = nil + } return nil } @@ -1156,14 +1184,10 @@ func autoConvert_v1alpha5_Router_To_v1alpha3_Router(in *v1alpha5.Router, out *Ro out.Name = in.Name out.ID = in.ID out.Tags = *(*[]string)(unsafe.Pointer(&in.Tags)) + // WARNING: in.IPs requires manual conversion: does not exist in peer-type return nil } -// Convert_v1alpha5_Router_To_v1alpha3_Router is an autogenerated conversion function. -func Convert_v1alpha5_Router_To_v1alpha3_Router(in *v1alpha5.Router, out *Router, s conversion.Scope) error { - return autoConvert_v1alpha5_Router_To_v1alpha3_Router(in, out, s) -} - func autoConvert_v1alpha3_SecurityGroup_To_v1alpha5_SecurityGroup(in *SecurityGroup, out *v1alpha5.SecurityGroup, s conversion.Scope) error { out.Name = in.Name out.ID = in.ID diff --git a/api/v1alpha4/conversion.go b/api/v1alpha4/conversion.go index f5068c4ad5..68105d9a8b 100644 --- a/api/v1alpha4/conversion.go +++ b/api/v1alpha4/conversion.go @@ -18,6 +18,7 @@ package v1alpha4 import ( conversion "k8s.io/apimachinery/pkg/conversion" + utilconversion "sigs.k8s.io/cluster-api/util/conversion" ctrlconversion "sigs.k8s.io/controller-runtime/pkg/conversion" infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha5" @@ -28,13 +29,36 @@ var _ ctrlconversion.Convertible = &OpenStackCluster{} func (r *OpenStackCluster) ConvertTo(dstRaw ctrlconversion.Hub) error { dst := dstRaw.(*infrav1.OpenStackCluster) - return Convert_v1alpha4_OpenStackCluster_To_v1alpha5_OpenStackCluster(r, dst, nil) + if err := Convert_v1alpha4_OpenStackCluster_To_v1alpha5_OpenStackCluster(r, dst, nil); err != nil { + return err + } + + // Manually restore data. + restored := &infrav1.OpenStackCluster{} + if ok, err := utilconversion.UnmarshalData(r, restored); err != nil || !ok { + return err + } + + if restored.Spec.APIServerLoadBalancer.AllowedCIDRs != nil { + dst.Spec.APIServerLoadBalancer.AllowedCIDRs = restored.Spec.APIServerLoadBalancer.AllowedCIDRs + } + + return nil } func (r *OpenStackCluster) ConvertFrom(srcRaw ctrlconversion.Hub) error { src := srcRaw.(*infrav1.OpenStackCluster) - return Convert_v1alpha5_OpenStackCluster_To_v1alpha4_OpenStackCluster(src, r, nil) + if err := Convert_v1alpha5_OpenStackCluster_To_v1alpha4_OpenStackCluster(src, r, nil); err != nil { + return err + } + + // Preserve Hub data on down-conversion except for metadata + if err := utilconversion.MarshalData(src, r); err != nil { + return err + } + + return nil } var _ ctrlconversion.Convertible = &OpenStackClusterList{} @@ -90,7 +114,16 @@ func (r *OpenStackMachine) ConvertTo(dstRaw ctrlconversion.Hub) error { func (r *OpenStackMachine) ConvertFrom(srcRaw ctrlconversion.Hub) error { src := srcRaw.(*infrav1.OpenStackMachine) - return Convert_v1alpha5_OpenStackMachine_To_v1alpha4_OpenStackMachine(src, r, nil) + if err := Convert_v1alpha5_OpenStackMachine_To_v1alpha4_OpenStackMachine(src, r, nil); err != nil { + return err + } + + // Preserve Hub data on down-conversion except for metadata + if err := utilconversion.MarshalData(src, r); err != nil { + return err + } + + return nil } var _ ctrlconversion.Convertible = &OpenStackMachineList{} @@ -118,7 +151,16 @@ func (r *OpenStackMachineTemplate) ConvertTo(dstRaw ctrlconversion.Hub) error { func (r *OpenStackMachineTemplate) ConvertFrom(srcRaw ctrlconversion.Hub) error { src := srcRaw.(*infrav1.OpenStackMachineTemplate) - return Convert_v1alpha5_OpenStackMachineTemplate_To_v1alpha4_OpenStackMachineTemplate(src, r, nil) + if err := Convert_v1alpha5_OpenStackMachineTemplate_To_v1alpha4_OpenStackMachineTemplate(src, r, nil); err != nil { + return err + } + + // Preserve Hub data on down-conversion except for metadata + if err := utilconversion.MarshalData(src, r); err != nil { + return err + } + + return nil } var _ ctrlconversion.Convertible = &OpenStackMachineTemplateList{} @@ -314,3 +356,11 @@ func Convert_v1alpha5_OpenStackMachineSpec_To_v1alpha4_OpenStackMachineSpec(in * } return nil } + +func Convert_v1alpha5_Router_To_v1alpha4_Router(in *infrav1.Router, out *Router, s conversion.Scope) error { + return autoConvert_v1alpha5_Router_To_v1alpha4_Router(in, out, s) +} + +func Convert_v1alpha5_LoadBalancer_To_v1alpha4_LoadBalancer(in *infrav1.LoadBalancer, out *LoadBalancer, s conversion.Scope) error { + return autoConvert_v1alpha5_LoadBalancer_To_v1alpha4_LoadBalancer(in, out, s) +} diff --git a/api/v1alpha4/conversion_test.go b/api/v1alpha4/conversion_test.go index dd502b5aaa..baed6d41c9 100644 --- a/api/v1alpha4/conversion_test.go +++ b/api/v1alpha4/conversion_test.go @@ -22,6 +22,7 @@ import ( fuzz "github.com/google/gofuzz" "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/api/apitesting/fuzzer" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" utilconversion "sigs.k8s.io/cluster-api/util/conversion" @@ -147,6 +148,11 @@ func TestConvertFrom(t *testing.T) { }, }, }, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "cluster.x-k8s.io/conversion-data": "{\"spec\":{\"cloudName\":\"\",\"flavor\":\"\",\"ports\":[{\"fixedIPs\":[{\"subnet\":{\"id\":\"986f5848-127f-4357-944e-5dd75472def8\"}}]}]},\"status\":{\"ready\":false}}", + }, + }, }, }, { @@ -165,6 +171,11 @@ func TestConvertFrom(t *testing.T) { ManagedAPIServerLoadBalancer: true, APIServerLoadBalancerAdditionalPorts: []int{80, 443}, }, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "cluster.x-k8s.io/conversion-data": "{\"spec\":{\"allowAllInClusterTraffic\":false,\"apiServerLoadBalancer\":{\"additionalPorts\":[80,443],\"enabled\":true},\"cloudName\":\"\",\"controlPlaneEndpoint\":{\"host\":\"\",\"port\":0},\"disableAPIServerFloatingIP\":false,\"managedSecurityGroups\":false,\"network\":{},\"subnet\":{}},\"status\":{\"ready\":false}}", + }, + }, }, }, { @@ -197,6 +208,15 @@ func TestFuzzyConversion(t *testing.T) { fuzzerFuncs := func(_ runtimeserializer.CodecFactory) []interface{} { return []interface{}{ // Don't test spoke-hub-spoke conversion of v1alpha4 fields which are not in infrav1 + func(v1alpha4Cluster *OpenStackCluster, c fuzz.Continue) { + c.FuzzNoCustom(v1alpha4Cluster) + + v1alpha4Cluster.ObjectMeta.Annotations = map[string]string{} + + if v1alpha4Cluster.Spec.Bastion != nil { + v1alpha4Cluster.Spec.Bastion.Instance.Image = "" + } + }, func(v1alpha3SubnetFilter *SubnetFilter, c fuzz.Continue) { c.FuzzNoCustom(v1alpha3SubnetFilter) v1alpha3SubnetFilter.EnableDHCP = nil @@ -232,14 +252,23 @@ func TestFuzzyConversion(t *testing.T) { v1alpha4RootVolume.DeviceType = "disk" v1alpha4RootVolume.SourceType = "image" }, - func(v1alpha4MachineSpec *OpenStackMachineSpec, c fuzz.Continue) { - c.FuzzNoCustom(v1alpha4MachineSpec) + func(v1alpha4Machine *OpenStackMachine, c fuzz.Continue) { + c.FuzzNoCustom(v1alpha4Machine) - if v1alpha4MachineSpec.RootVolume != nil { + v1alpha4Machine.ObjectMeta.Annotations = map[string]string{} + + if v1alpha4Machine.Spec.RootVolume != nil { // OpenStackMachineSpec.Image is ignored in v1alpha4 if RootVolume is set - v1alpha4MachineSpec.Image = "" + v1alpha4Machine.Spec.Image = "" } }, + func(v1alpha4MachineTemplate *OpenStackMachineTemplate, c fuzz.Continue) { + c.FuzzNoCustom(v1alpha4MachineTemplate) + + v1alpha4MachineTemplate.ObjectMeta.Annotations = map[string]string{} + + v1alpha4MachineTemplate.Spec.Template.Spec.Image = "" + }, func(v1alpha4Instance *Instance, c fuzz.Continue) { c.FuzzNoCustom(v1alpha4Instance) @@ -248,6 +277,15 @@ func TestFuzzyConversion(t *testing.T) { v1alpha4Instance.Image = "" } }, + func(v1alpha4ClusterTemplate *OpenStackClusterTemplate, c fuzz.Continue) { + c.FuzzNoCustom(v1alpha4ClusterTemplate) + + v1alpha4ClusterTemplate.ObjectMeta.Annotations = map[string]string{} + + if v1alpha4ClusterTemplate.Spec.Template.Spec.Bastion != nil { + v1alpha4ClusterTemplate.Spec.Template.Spec.Bastion.Instance.Image = "" + } + }, // Don't test hub-spoke-hub conversion of infrav1 fields which are not in v1alpha4 func(v1alpha5PortOpts *infrav1.PortOpts, c fuzz.Continue) { @@ -281,25 +319,62 @@ func TestFuzzyConversion(t *testing.T) { } } }, - func(v1alpha5ClusterStatus *infrav1.OpenStackClusterStatus, c fuzz.Continue) { - c.FuzzNoCustom(v1alpha5ClusterStatus) + func(v1alpha5Cluster *infrav1.OpenStackCluster, c fuzz.Continue) { + c.FuzzNoCustom(v1alpha5Cluster) + + v1alpha5Cluster.ObjectMeta.Annotations = map[string]string{} - if v1alpha5ClusterStatus.Bastion != nil { - v1alpha5ClusterStatus.Bastion.ImageUUID = "" + v1alpha5Cluster.Spec.APIServerLoadBalancer.AllowedCIDRs = nil + + if v1alpha5Cluster.Spec.Bastion != nil { + v1alpha5Cluster.Spec.Bastion.Instance.Image = "" + } + + if v1alpha5Cluster.Status.Bastion != nil { + v1alpha5Cluster.Status.Bastion.ImageUUID = "" + v1alpha5Cluster.Status.Bastion.Image = "" + v1alpha5Cluster.Status.Bastion.Networks = nil + } + + if v1alpha5Cluster.Status.Network != nil { + if v1alpha5Cluster.Status.Network.APIServerLoadBalancer != nil { + v1alpha5Cluster.Status.Network.APIServerLoadBalancer.AllowedCIDRs = nil + } + if v1alpha5Cluster.Status.Network.Router != nil { + v1alpha5Cluster.Status.Network.Router.IPs = []string{} + } + } + + if v1alpha5Cluster.Status.ExternalNetwork != nil { + if v1alpha5Cluster.Status.ExternalNetwork.APIServerLoadBalancer != nil { + v1alpha5Cluster.Status.ExternalNetwork.APIServerLoadBalancer.AllowedCIDRs = nil + } + if v1alpha5Cluster.Status.ExternalNetwork.Router != nil { + v1alpha5Cluster.Status.ExternalNetwork.Router.IPs = []string{} + } } }, - func(v1alpha5MachineSpec *infrav1.OpenStackMachineSpec, c fuzz.Continue) { - c.FuzzNoCustom(v1alpha5MachineSpec) + func(v1alpha5Machine *infrav1.OpenStackMachine, c fuzz.Continue) { + c.FuzzNoCustom(v1alpha5Machine) + + v1alpha5Machine.ObjectMeta.Annotations = map[string]string{} // In v1alpha4 boot from volume only supports // image by UUID, and boot from local only // suppots image by name - if v1alpha5MachineSpec.RootVolume != nil && v1alpha5MachineSpec.RootVolume.Size > 0 { - v1alpha5MachineSpec.Image = "" + if v1alpha5Machine.Spec.RootVolume != nil && v1alpha5Machine.Spec.RootVolume.Size > 0 { + v1alpha5Machine.Spec.Image = "" } else { - v1alpha5MachineSpec.ImageUUID = "" + v1alpha5Machine.Spec.ImageUUID = "" } }, + func(v1alpha5MachineTemplate *infrav1.OpenStackMachineTemplate, c fuzz.Continue) { + c.FuzzNoCustom(v1alpha5MachineTemplate) + + v1alpha5MachineTemplate.ObjectMeta.Annotations = map[string]string{} + + v1alpha5MachineTemplate.Spec.Template.Spec.Image = "" + }, func(v1alpha5Instance *infrav1.Instance, c fuzz.Continue) { c.FuzzNoCustom(v1alpha5Instance) @@ -318,6 +393,17 @@ func TestFuzzyConversion(t *testing.T) { v1alpha5RootVolume.VolumeType = "" v1alpha5RootVolume.AvailabilityZone = "" }, + func(v1alpha5ClusterTemplate *infrav1.OpenStackClusterTemplate, c fuzz.Continue) { + c.FuzzNoCustom(v1alpha5ClusterTemplate) + + v1alpha5ClusterTemplate.ObjectMeta.Annotations = map[string]string{} + + v1alpha5ClusterTemplate.Spec.Template.Spec.APIServerLoadBalancer.AllowedCIDRs = nil + + if v1alpha5ClusterTemplate.Spec.Template.Spec.Bastion != nil { + v1alpha5ClusterTemplate.Spec.Template.Spec.Bastion.Instance.Image = "" + } + }, } } diff --git a/api/v1alpha4/zz_generated.conversion.go b/api/v1alpha4/zz_generated.conversion.go index 5e684eca67..071829385d 100644 --- a/api/v1alpha4/zz_generated.conversion.go +++ b/api/v1alpha4/zz_generated.conversion.go @@ -75,11 +75,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha5.LoadBalancer)(nil), (*LoadBalancer)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha5_LoadBalancer_To_v1alpha4_LoadBalancer(a.(*v1alpha5.LoadBalancer), b.(*LoadBalancer), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*Network)(nil), (*v1alpha5.Network)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_Network_To_v1alpha5_Network(a.(*Network), b.(*v1alpha5.Network), scope) }); err != nil { @@ -255,11 +250,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha5.Router)(nil), (*Router)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha5_Router_To_v1alpha4_Router(a.(*v1alpha5.Router), b.(*Router), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*SecurityGroup)(nil), (*v1alpha5.SecurityGroup)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_SecurityGroup_To_v1alpha5_SecurityGroup(a.(*SecurityGroup), b.(*v1alpha5.SecurityGroup), scope) }); err != nil { @@ -385,6 +375,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha5.LoadBalancer)(nil), (*LoadBalancer)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha5_LoadBalancer_To_v1alpha4_LoadBalancer(a.(*v1alpha5.LoadBalancer), b.(*LoadBalancer), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha5.NetworkFilter)(nil), (*Filter)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha5_NetworkFilter_To_v1alpha4_Filter(a.(*v1alpha5.NetworkFilter), b.(*Filter), scope) }); err != nil { @@ -410,6 +405,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1alpha5.Router)(nil), (*Router)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha5_Router_To_v1alpha4_Router(a.(*v1alpha5.Router), b.(*Router), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1alpha5.SubnetFilter)(nil), (*SubnetFilter)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha5_SubnetFilter_To_v1alpha4_SubnetFilter(a.(*v1alpha5.SubnetFilter), b.(*SubnetFilter), scope) }); err != nil { @@ -608,14 +608,10 @@ func autoConvert_v1alpha5_LoadBalancer_To_v1alpha4_LoadBalancer(in *v1alpha5.Loa out.ID = in.ID out.IP = in.IP out.InternalIP = in.InternalIP + // WARNING: in.AllowedCIDRs requires manual conversion: does not exist in peer-type return nil } -// Convert_v1alpha5_LoadBalancer_To_v1alpha4_LoadBalancer is an autogenerated conversion function. -func Convert_v1alpha5_LoadBalancer_To_v1alpha4_LoadBalancer(in *v1alpha5.LoadBalancer, out *LoadBalancer, s conversion.Scope) error { - return autoConvert_v1alpha5_LoadBalancer_To_v1alpha4_LoadBalancer(in, out, s) -} - func autoConvert_v1alpha4_Network_To_v1alpha5_Network(in *Network, out *v1alpha5.Network, s conversion.Scope) error { out.Name = in.Name out.ID = in.ID @@ -630,8 +626,24 @@ func autoConvert_v1alpha4_Network_To_v1alpha5_Network(in *Network, out *v1alpha5 } else { out.PortOpts = nil } - out.Router = (*v1alpha5.Router)(unsafe.Pointer(in.Router)) - out.APIServerLoadBalancer = (*v1alpha5.LoadBalancer)(unsafe.Pointer(in.APIServerLoadBalancer)) + if in.Router != nil { + in, out := &in.Router, &out.Router + *out = new(v1alpha5.Router) + if err := Convert_v1alpha4_Router_To_v1alpha5_Router(*in, *out, s); err != nil { + return err + } + } else { + out.Router = nil + } + if in.APIServerLoadBalancer != nil { + in, out := &in.APIServerLoadBalancer, &out.APIServerLoadBalancer + *out = new(v1alpha5.LoadBalancer) + if err := Convert_v1alpha4_LoadBalancer_To_v1alpha5_LoadBalancer(*in, *out, s); err != nil { + return err + } + } else { + out.APIServerLoadBalancer = nil + } return nil } @@ -654,8 +666,24 @@ func autoConvert_v1alpha5_Network_To_v1alpha4_Network(in *v1alpha5.Network, out } else { out.PortOpts = nil } - out.Router = (*Router)(unsafe.Pointer(in.Router)) - out.APIServerLoadBalancer = (*LoadBalancer)(unsafe.Pointer(in.APIServerLoadBalancer)) + if in.Router != nil { + in, out := &in.Router, &out.Router + *out = new(Router) + if err := Convert_v1alpha5_Router_To_v1alpha4_Router(*in, *out, s); err != nil { + return err + } + } else { + out.Router = nil + } + if in.APIServerLoadBalancer != nil { + in, out := &in.APIServerLoadBalancer, &out.APIServerLoadBalancer + *out = new(LoadBalancer) + if err := Convert_v1alpha5_LoadBalancer_To_v1alpha4_LoadBalancer(*in, *out, s); err != nil { + return err + } + } else { + out.APIServerLoadBalancer = nil + } return nil } @@ -1516,14 +1544,10 @@ func autoConvert_v1alpha5_Router_To_v1alpha4_Router(in *v1alpha5.Router, out *Ro out.Name = in.Name out.ID = in.ID out.Tags = *(*[]string)(unsafe.Pointer(&in.Tags)) + // WARNING: in.IPs requires manual conversion: does not exist in peer-type return nil } -// Convert_v1alpha5_Router_To_v1alpha4_Router is an autogenerated conversion function. -func Convert_v1alpha5_Router_To_v1alpha4_Router(in *v1alpha5.Router, out *Router, s conversion.Scope) error { - return autoConvert_v1alpha5_Router_To_v1alpha4_Router(in, out, s) -} - func autoConvert_v1alpha4_SecurityGroup_To_v1alpha5_SecurityGroup(in *SecurityGroup, out *v1alpha5.SecurityGroup, s conversion.Scope) error { out.Name = in.Name out.ID = in.ID diff --git a/api/v1alpha5/openstackcluster_webhook.go b/api/v1alpha5/openstackcluster_webhook.go index 8c2c0723b5..b72fbe05d2 100644 --- a/api/v1alpha5/openstackcluster_webhook.go +++ b/api/v1alpha5/openstackcluster_webhook.go @@ -117,6 +117,12 @@ func (r *OpenStackCluster) ValidateUpdate(oldRaw runtime.Object) error { r.Spec.Bastion.Enabled = true } + // Allow changes on AllowedCIDRs + if r.Spec.APIServerLoadBalancer.Enabled { + old.Spec.APIServerLoadBalancer.AllowedCIDRs = []string{} + r.Spec.APIServerLoadBalancer.AllowedCIDRs = []string{} + } + if !reflect.DeepEqual(old.Spec, r.Spec) { allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "cannot be modified")) } diff --git a/api/v1alpha5/openstackcluster_webhook_test.go b/api/v1alpha5/openstackcluster_webhook_test.go index 12c655edb3..74dfc0cebc 100644 --- a/api/v1alpha5/openstackcluster_webhook_test.go +++ b/api/v1alpha5/openstackcluster_webhook_test.go @@ -269,6 +269,35 @@ func TestOpenStackCluster_ValidateUpdate(t *testing.T) { }, wantErr: false, }, + { + name: "Changing CIDRs on the OpenStackCluster.Spec.APIServerLoadBalancer.AllowedCIDRs is allowed", + oldTemplate: &OpenStackCluster{ + Spec: OpenStackClusterSpec{ + CloudName: "foobar", + APIServerLoadBalancer: APIServerLoadBalancer{ + Enabled: true, + AllowedCIDRs: []string{ + "0.0.0.0/0", + "192.168.10.0/24", + }, + }, + }, + }, + newTemplate: &OpenStackCluster{ + Spec: OpenStackClusterSpec{ + CloudName: "foobar", + APIServerLoadBalancer: APIServerLoadBalancer{ + Enabled: true, + AllowedCIDRs: []string{ + "0.0.0.0/0", + "192.168.10.0/24", + "10.6.0.0/16", + }, + }, + }, + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/api/v1alpha5/types.go b/api/v1alpha5/types.go index 3988dd5581..c035a77860 100644 --- a/api/v1alpha5/types.go +++ b/api/v1alpha5/types.go @@ -220,6 +220,8 @@ type Router struct { ID string `json:"id"` //+optional Tags []string `json:"tags,omitempty"` + //+optional + IPs []string `json:"ips,omitempty"` } // LoadBalancer represents basic information about the associated OpenStack LoadBalancer. @@ -228,6 +230,8 @@ type LoadBalancer struct { ID string `json:"id"` IP string `json:"ip"` InternalIP string `json:"internalIP"` + //+optional + AllowedCIDRs []string `json:"allowedCIDRs,omitempty"` } // SecurityGroup represents the basic information of the associated @@ -301,8 +305,10 @@ type Bastion struct { } type APIServerLoadBalancer struct { - // Enabled defines whether a LoadBalancer should be created. + // Enabled defines whether a load balancer should be created. Enabled bool `json:"enabled,omitempty"` - // AdditionalPorts adds additional tcp ports to the Loadbalacner + // AdditionalPorts adds additional tcp ports to the load balancer. AdditionalPorts []int `json:"additionalPorts,omitempty"` + // AllowedCIDRs restrict access to all API-Server listeners to the given address CIDRs. + AllowedCIDRs []string `json:"allowedCidrs,omitempty"` } diff --git a/api/v1alpha5/zz_generated.deepcopy.go b/api/v1alpha5/zz_generated.deepcopy.go index ff943dc2ef..d784e164a4 100644 --- a/api/v1alpha5/zz_generated.deepcopy.go +++ b/api/v1alpha5/zz_generated.deepcopy.go @@ -36,6 +36,11 @@ func (in *APIServerLoadBalancer) DeepCopyInto(out *APIServerLoadBalancer) { *out = make([]int, len(*in)) copy(*out, *in) } + if in.AllowedCIDRs != nil { + in, out := &in.AllowedCIDRs, &out.AllowedCIDRs + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIServerLoadBalancer. @@ -175,6 +180,11 @@ func (in *Instance) DeepCopy() *Instance { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoadBalancer) DeepCopyInto(out *LoadBalancer) { *out = *in + if in.AllowedCIDRs != nil { + in, out := &in.AllowedCIDRs, &out.AllowedCIDRs + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancer. @@ -213,7 +223,7 @@ func (in *Network) DeepCopyInto(out *Network) { if in.APIServerLoadBalancer != nil { in, out := &in.APIServerLoadBalancer, &out.APIServerLoadBalancer *out = new(LoadBalancer) - **out = **in + (*in).DeepCopyInto(*out) } } @@ -889,6 +899,11 @@ func (in *Router) DeepCopyInto(out *Router) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.IPs != nil { + in, out := &in.IPs, &out.IPs + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Router. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml index 85c47240d3..3241fba5de 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml @@ -103,11 +103,11 @@ spec: credentials properties: name: - description: Name is unique within a namespace to reference + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -330,11 +330,11 @@ spec: (startup script in most cases) properties: name: - description: Name is unique within a namespace to reference + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -349,11 +349,11 @@ spec: description: The name of the secret containing the openstack credentials properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -2511,12 +2511,18 @@ spec: properties: additionalPorts: description: AdditionalPorts adds additional tcp ports to the - Loadbalacner + load balancer. items: type: integer type: array + allowedCidrs: + description: AllowedCIDRs restrict access to all API-Server listeners + to the given address CIDRs. + items: + type: string + type: array enabled: - description: Enabled defines whether a LoadBalancer should be + description: Enabled defines whether a load balancer should be created. type: boolean type: object @@ -3172,6 +3178,10 @@ spec: because this field is optional and therefore not set in all cases properties: + allowedCIDRs: + items: + type: string + type: array id: type: string internalIP: @@ -3373,6 +3383,10 @@ spec: properties: id: type: string + ips: + items: + type: string + type: array name: type: string tags: @@ -3555,6 +3569,10 @@ spec: description: Be careful when using APIServerLoadBalancer, because this field is optional and therefore not set in all cases properties: + allowedCIDRs: + items: + type: string + type: array id: type: string internalIP: @@ -3753,6 +3771,10 @@ spec: properties: id: type: string + ips: + items: + type: string + type: array name: type: string tags: @@ -3849,6 +3871,10 @@ spec: description: Be careful when using APIServerLoadBalancer, because this field is optional and therefore not set in all cases properties: + allowedCIDRs: + items: + type: string + type: array id: type: string internalIP: @@ -4047,6 +4073,10 @@ spec: properties: id: type: string + ips: + items: + type: string + type: array name: type: string tags: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml index 65226a1deb..cd0f280546 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml @@ -773,12 +773,18 @@ spec: properties: additionalPorts: description: AdditionalPorts adds additional tcp ports - to the Loadbalacner + to the load balancer. items: type: integer type: array + allowedCidrs: + description: AllowedCIDRs restrict access to all API-Server + listeners to the given address CIDRs. + items: + type: string + type: array enabled: - description: Enabled defines whether a LoadBalancer should + description: Enabled defines whether a load balancer should be created. type: boolean type: object diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachines.yaml index 202937302d..e5f9e59625 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachines.yaml @@ -68,11 +68,11 @@ spec: description: The name of the secret containing the openstack credentials properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -291,11 +291,11 @@ spec: script in most cases) properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachinetemplates.yaml index a2807d39fc..91293fe3ad 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackmachinetemplates.yaml @@ -59,11 +59,11 @@ spec: credentials properties: name: - description: Name is unique within a namespace to reference + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -286,11 +286,11 @@ spec: (startup script in most cases) properties: name: - description: Name is unique within a namespace to reference + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which + description: namespace defines the space within which the secret name must be unique. type: string type: object diff --git a/docs/book/src/clusteropenstack/configuration.md b/docs/book/src/clusteropenstack/configuration.md index 394bad98d6..320f8d1d8e 100644 --- a/docs/book/src/clusteropenstack/configuration.md +++ b/docs/book/src/clusteropenstack/configuration.md @@ -16,6 +16,7 @@ - [External network](#external-network) - [API server floating IP](#api-server-floating-ip) - [Disabling the API server floating IP](#disabling-the-api-server-floating-ip) + - [Restrict Access to the API server](#restrict-access-to-the-api-server) - [Network Filters](#network-filters) - [Multiple Networks](#multiple-networks) - [Subnet Filters](#subnet-filters) @@ -86,6 +87,7 @@ or [configure custom security groups](#security-groups) with rules allowing ingr ## OpenStack credential ### Generate credentials + The [env.rc](https://github.com/kubernetes-sigs/cluster-api-provider-openstack/blob/main/templates/env.rc) script sets the environment variables related to credentials. It's highly recommend to avoid using `admin` credential. ```bash @@ -177,6 +179,61 @@ during an upgrade. When the API server has a floating IP, this role is fulfilled floating IP even if there is no load balancer. When the API server does not have a floating IP, the load balancer virtual IP on the cluster network is used. +## Restrict Access to the API server + +> **NOTE** +> +> This requires "amphora" as load balancer provider at in version >= `v2.12` + +It is possible to restrict access to the Kubernetes API server on a network level. If required, you can specify +the allowed CIDRs by `spec.APIServerLoadBalancer.AllowedCIDRs` of `OpenStackCluster`. + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 +kind: OpenStackCluster +metadata: + name: + namespace: +spec: + allowAllInClusterTraffic: true + apiServerLoadBalancer: + allowedCidrs: + - 192.168.10/24 + - 10.10.0.0/16 +``` + +All known IPs of the target cluster will be discovered dynamically (e.g. you don't have to take care of target Cluster own Router IP, internal CIDRs or any Bastion Host IP). +**Note**: Please ensure, that at least the outgoing IP of your management Cluster is added to the list of allowed CIDRs. Otherwise CAPO can't reconcile the target Cluster correctly. + +All applied CIDRs (user defined + dynamically discovered) are written back into `status.network.apiServerLoadBalancer.allowedCIDRs` + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5 +kind: OpenStackCluster +metadata: + name: + namespace: +status: + network: + apiServerLoadBalancer: + allowedCIDRs: + - 10.6.0.0/24 # openStackCluster.Status.Network.Subnet.CIDR + - 10.6.0.90/32 # bastion Host internal IP + - 10.10.0.0/16 # user defined + - 192.168.10/24 # user defined + - 172.16.111.100/32 # bastion host floating IP + - 172.16.111.85/32 # router IP + internalIP: 10.6.0.144 + ip: 172.16.111.159 + name: k8s-clusterapi-cluster-- +``` + +If you locked out yourself or the CAPO management cluster, you can easily clear the `allowed_cidrs` field on OpenStack via + +```bash +openstack loadbalancer listener unset --allowed-cidrs +``` + ## Network Filters If you have a complex query that you want to use to lookup a network, then you can do this by using a network filter. More details about the filter can be found in [NetworkParam](https://github.com/kubernetes-sigs/cluster-api-provider-openstack/blob/main/api/v1beta1/types.go) @@ -295,15 +352,15 @@ plane and worker nodes respectively. By default, these groups have rules that allow the following traffic: - * Control plane nodes - * API server traffic from anywhere - * Etcd traffic from other control plane nodes - * Kubelet traffic from other cluster nodes - * Calico CNI traffic from other cluster nodes - * Worker nodes - * Node port traffic from anywhere - * Kubelet traffic from other cluster nodes - * Calico CNI traffic from other cluster nodes +- Control plane nodes + - API server traffic from anywhere + - Etcd traffic from other control plane nodes + - Kubelet traffic from other cluster nodes + - Calico CNI traffic from other cluster nodes +- Worker nodes + - Node port traffic from anywhere + - Kubelet traffic from other cluster nodes + - Calico CNI traffic from other cluster nodes To use a CNI other than Calico, the flag `OpenStackCluster.spec.allowAllInClusterTraffic` can be set to `true`. With this flag set, the rules for the managed security groups permit all traffic @@ -440,7 +497,6 @@ If `managedSecurityGroups: true`, security group rule opening 22/tcp is added to Once the workload cluster is up and running after being configured for an SSH bastion host, you can use the kubectl get openstackcluster command to look up the floating IP address of the bastion host (make sure the kubectl context is set to the management cluster). The output will look something like this: - ```yaml $ kubectl get openstackcluster NAME CLUSTER READY NETWORK SUBNET BASTION diff --git a/go.mod b/go.mod index 7cccdf5a6a..5a3a5b897d 100644 --- a/go.mod +++ b/go.mod @@ -6,23 +6,24 @@ require ( github.com/go-logr/logr v1.2.0 github.com/golang/mock v1.6.0 github.com/google/gofuzz v1.2.0 - github.com/gophercloud/gophercloud v0.16.0 - github.com/gophercloud/utils v0.0.0-20210323225332-7b186010c04f + github.com/gophercloud/gophercloud v0.24.0 + github.com/gophercloud/utils v0.0.0-20220307143606-8e7800759d16 + github.com/hashicorp/go-version v1.4.0 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.17.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.11.1 + github.com/prometheus/client_golang v1.12.1 github.com/spf13/pflag v1.0.5 - golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 + golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f golang.org/x/text v0.3.7 gopkg.in/ini.v1 v1.63.2 - k8s.io/api v0.23.0 + k8s.io/api v0.24.0 k8s.io/apiextensions-apiserver v0.23.0 - k8s.io/apimachinery v0.23.0 - k8s.io/client-go v0.23.0 - k8s.io/component-base v0.23.0 - k8s.io/klog/v2 v2.30.0 - k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b + k8s.io/apimachinery v0.24.0 + k8s.io/client-go v0.24.0 + k8s.io/component-base v0.24.0 + k8s.io/klog/v2 v2.60.1 + k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 sigs.k8s.io/cluster-api v1.1.3 sigs.k8s.io/cluster-api/test v1.1.3 sigs.k8s.io/controller-runtime v0.11.1 @@ -43,16 +44,17 @@ require ( github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect - github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/containerd/containerd v1.5.9 // indirect github.com/coredns/caddy v1.1.0 // indirect github.com/coredns/corefile-migration v1.0.14 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/docker v20.10.12+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect + github.com/emicklei/go-restful v2.9.5+incompatible // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect @@ -64,11 +66,11 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/cel-go v0.9.0 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.6 // indirect github.com/google/go-github/v33 v33.0.0 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/uuid v1.2.0 // indirect - github.com/googleapis/gnostic v0.5.5 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.12 // indirect @@ -85,43 +87,44 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.28.0 // indirect - github.com/prometheus/procfs v0.6.0 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.4.1 // indirect - github.com/spf13/cobra v1.2.1 // indirect + github.com/spf13/cobra v1.4.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.9.0 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/valyala/fastjson v1.6.3 // indirect - golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect + golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 // indirect - golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect - golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect + golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect + golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 // indirect + google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect google.golang.org/grpc v1.42.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - k8s.io/apiserver v0.23.0 // indirect + k8s.io/apiserver v0.24.0 // indirect k8s.io/cluster-bootstrap v0.23.0 // indirect - k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect - sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect + k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect + sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect sigs.k8s.io/kind v0.11.1 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect ) replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.1.3 diff --git a/go.sum b/go.sum index 82c9d785af..f3fd3a4e0c 100644 --- a/go.sum +++ b/go.sum @@ -121,6 +121,7 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= @@ -140,6 +141,7 @@ github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqO github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= @@ -152,8 +154,9 @@ github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6 github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= @@ -287,6 +290,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -306,8 +310,9 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v20.10.12+incompatible h1:CEeNmFM0QZIsJCZKMkZx0ZcahTiewkrgiwfYD+dfl1U= github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= @@ -327,6 +332,7 @@ github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:Htrtb github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -462,6 +468,8 @@ github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9 github.com/google/cel-go v0.9.0 h1:u1hg7lcZ/XWw2d3aV1jFS30ijQQ6q0/h1C2ZBeBD1gY= github.com/google/cel-go v0.9.0/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -513,13 +521,12 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gophercloud/gophercloud v0.15.1-0.20210202035223-633d73521055/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4= -github.com/gophercloud/gophercloud v0.16.0 h1:sWjPfypuzxRxjVbk3/MsU4H8jS0NNlyauZtIUl78BPU= -github.com/gophercloud/gophercloud v0.16.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4= -github.com/gophercloud/utils v0.0.0-20210323225332-7b186010c04f h1:+SO5iEqu9QjNWL9TfAmOE5u0Uizv1T3jpBuMJfMOVJ0= -github.com/gophercloud/utils v0.0.0-20210323225332-7b186010c04f/go.mod h1:wx8HMD8oQD0Ryhz6+6ykq75PJ79iPyEqYHfwZ4l7OsA= +github.com/gophercloud/gophercloud v0.20.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4= +github.com/gophercloud/gophercloud v0.24.0 h1:jDsIMGJ1KZpAjYfQgGI2coNQj5Q83oPzuiGJRFWgMzw= +github.com/gophercloud/gophercloud v0.24.0/go.mod h1:Q8fZtyi5zZxPS/j9aj3sSxtvj41AdQMDwyo1myduD5c= +github.com/gophercloud/utils v0.0.0-20220307143606-8e7800759d16 h1:slt/exMiitZNY+5OrKJ6ZvSogqN+SyzeYzAtyI6db9A= +github.com/gophercloud/utils v0.0.0-20220307143606-8e7800759d16/go.mod h1:qOGlfG6OIJ193/c3Xt/XjOfHataNZdQcVgiu93LxBUM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -557,6 +564,8 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.4.0 h1:aAQzgqIrRKRa7w75CKpbBxYsmUoPjzVm1W59ca1L0J4= +github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -685,8 +694,9 @@ github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2J github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -699,6 +709,7 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -786,8 +797,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= -github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -801,8 +812,9 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.28.0 h1:vGVfV9KrDTvWt5boZO0I19g2E3CsWfpPPKZM9dt3mEw= github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -813,8 +825,9 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -822,6 +835,7 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= @@ -865,8 +879,9 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -935,6 +950,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= @@ -1011,8 +1027,11 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc= +golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1048,6 +1067,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1105,8 +1125,10 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1236,12 +1258,17 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 h1:M69LAlWZCshgp0QSzyDcSsSIejIEeuaCVpmwcKwyLMk= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1259,8 +1286,9 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1325,6 +1353,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= +golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1428,8 +1457,9 @@ google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKr google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 h1:NHN4wOCScVzKhPenJ2dt+BTs3X/XkBVI/Rh4iDt55T8= google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 h1:Et6SkiuvnBn+SgrSYXs/BrUpGB4mbdwt4R3vaPIlicA= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1530,35 +1560,40 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/api v0.23.0 h1:WrL1gb73VSC8obi8cuYETJGXEoFNEh3LU0Pt+Sokgro= k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg= +k8s.io/api v0.24.0 h1:J0hann2hfxWr1hinZIDefw7Q96wmCBx6SSB8IY0MdDg= +k8s.io/api v0.24.0/go.mod h1:5Jl90IUrJHUJYEMANRURMiVvJ0g7Ax7r3R1bqO8zx8I= k8s.io/apiextensions-apiserver v0.23.0 h1:uii8BYmHYiT2ZTAJxmvc3X8UhNYMxl2A0z0Xq3Pm+WY= k8s.io/apiextensions-apiserver v0.23.0/go.mod h1:xIFAEEDlAZgpVBl/1VSjGDmLoXAWRG40+GsWhKhAxY4= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apimachinery v0.23.0 h1:mIfWRMjBuMdolAWJ3Fd+aPTMv3X9z+waiARMpvvb0HQ= k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc= +k8s.io/apimachinery v0.24.0 h1:ydFCyC/DjCvFCHK5OPMKBlxayQytB8pxy8YQInd5UyQ= +k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= -k8s.io/apiserver v0.23.0 h1:Ds/QveXWi9aJ8ISB0CJa4zBNc5njxAs5u3rmMIexqCY= k8s.io/apiserver v0.23.0/go.mod h1:Cec35u/9zAepDPPFyT+UMrgqOCjgJ5qtfVJDxjZYmt4= +k8s.io/apiserver v0.24.0 h1:GR7kGsjOMfilRvlG3Stxv/3uz/ryvJ/aZXc5pqdsNV0= +k8s.io/apiserver v0.24.0/go.mod h1:WFx2yiOMawnogNToVvUYT9nn1jaIkMKj41ZYCVycsBA= k8s.io/cli-runtime v0.23.0/go.mod h1:B5N3YH0KP1iKr6gEuJ/RRmGjO0mJQ/f/JrsmEiPQAlU= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/client-go v0.23.0 h1:vcsOqyPq7XV3QmQRCBH/t9BICJM9Q1M18qahjv+rebY= k8s.io/client-go v0.23.0/go.mod h1:hrDnpnK1mSr65lHHcUuIZIXDgEbzc7/683c6hyG4jTA= +k8s.io/client-go v0.24.0 h1:lbE4aB1gTHvYFSwm6eD3OF14NhFDKCejlnsGYlSJe5U= +k8s.io/client-go v0.24.0/go.mod h1:VFPQET+cAFpYxh6Bq6f4xyMY80G6jKKktU6G0m00VDw= k8s.io/cluster-bootstrap v0.23.0 h1:8pZuuAWPoygewSNB4IddX3HBwXcQkPDXL/ca7GtGf4o= k8s.io/cluster-bootstrap v0.23.0/go.mod h1:VltEnKWfrRTiKgOXp3ts3vh7yqNlH6KFKFflo9GtCBg= k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/component-base v0.23.0 h1:UAnyzjvVZ2ZR1lF35YwtNY6VMN94WtOnArcXBu34es8= k8s.io/component-base v0.23.0/go.mod h1:DHH5uiFvLC1edCpvcTDV++NKULdYYU6pR9Tt3HIKMKI= +k8s.io/component-base v0.24.0 h1:h5jieHZQoHrY/lHG+HyrSbJeyfuitheBvqvKwKHVC0g= +k8s.io/component-base v0.24.0/go.mod h1:Dgazgon0i7KYUsS8krG8muGiMVtUZxG037l1MKyXgrA= k8s.io/component-helpers v0.23.0/go.mod h1:liXMh6FZS4qamKtMJQ7uLHnFe3tlC86RX5mJEk/aerg= k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= @@ -1569,33 +1604,38 @@ k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= +k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 h1:Gii5eqf+GmIEwGNKQYQClCayuJCe2/4fZUvF7VG99sU= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= k8s.io/kubectl v0.23.0/go.mod h1:TfcGEs3u4dkmoC2eku1GYymdGaMtPMcaLLFrX/RB2kI= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/metrics v0.23.0/go.mod h1:NDiZTwppEtAuKJ1Rxt3S4dhyRzdp6yUcJf0vo023dPo= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25/go.mod h1:Mlj9PNLmG9bZ6BHFwFKDo5afkpWyUISkb9Me0GnK66I= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= sigs.k8s.io/cluster-api v1.1.3 h1:t682KcIPFeKGwe2SlxGvZa/HVmLA80XJ45KHBhzUETM= sigs.k8s.io/cluster-api v1.1.3/go.mod h1:XqFZ0s9+KKjI/K39/EzHyAb4Sljprqvnm/XKWPgPp3Y= sigs.k8s.io/cluster-api/test v1.1.3 h1:orrawaLueFkAllsy49MqDuUjmRRxy7hmTqfbxjOA+Dk= sigs.k8s.io/cluster-api/test v1.1.3/go.mod h1:WpD3cv0ZyNt5aulQmA7HvQbu7TQ/Eb8OPbwayGxUazs= sigs.k8s.io/controller-runtime v0.11.1 h1:7YIHT2QnHJArj/dk9aUkYhfqfK5cIxPOX5gPECfdZLU= sigs.k8s.io/controller-runtime v0.11.1/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA= -sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/kind v0.11.1 h1:pVzOkhUwMBrCB0Q/WllQDO3v14Y+o2V0tFgjTqIUjwA= sigs.k8s.io/kind v0.11.1/go.mod h1:fRpgVhtqAWrtLB9ED7zQahUimpUXuG/iHT88xYqEGIA= sigs.k8s.io/kustomize/api v0.10.1/go.mod h1:2FigT1QN6xKdcnGS2Ppp1uIWrtWN28Ms8A3OZUZhwr8= @@ -1605,8 +1645,9 @@ sigs.k8s.io/kustomize/kyaml v0.13.0/go.mod h1:FTJxEZ86ScK184NpGSAQcfEqee0nul8oLC sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.0 h1:kDvPBbnPk+qYmkHmSo8vKGp438IASWofnbbUKDE/bv0= sigs.k8s.io/structured-merge-diff/v4 v4.2.0/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= diff --git a/main.go b/main.go index bcfc0e53f6..65a488061e 100644 --- a/main.go +++ b/main.go @@ -65,6 +65,7 @@ var ( webhookPort int webhookCertDir string healthAddr string + lbProvider string ) func init() { @@ -123,6 +124,9 @@ func InitFlags(fs *pflag.FlagSet) { fs.StringVar(&healthAddr, "health-addr", ":9440", "The address the health endpoint binds to.") + + fs.StringVar(&lbProvider, "lb-provider", "amphora", + "The name of the load balancer provider (amphora or ovn) to use (defaults to amphora).") } func main() { diff --git a/pkg/cloud/services/loadbalancer/client.go b/pkg/cloud/services/loadbalancer/client.go index 1ce285ec02..c6f117fe78 100644 --- a/pkg/cloud/services/loadbalancer/client.go +++ b/pkg/cloud/services/loadbalancer/client.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/gophercloud/gophercloud" + "github.com/gophercloud/gophercloud/openstack/compute/apiversions" "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/listeners" "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers" "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/monitors" @@ -37,6 +38,7 @@ type LbClient interface { DeleteLoadBalancer(id string, opts loadbalancers.DeleteOptsBuilder) error CreateListener(opts listeners.CreateOptsBuilder) (*listeners.Listener, error) ListListeners(opts listeners.ListOptsBuilder) ([]listeners.Listener, error) + UpdateListener(id string, opts listeners.UpdateOpts) (*listeners.Listener, error) GetListener(id string) (*listeners.Listener, error) DeleteListener(id string) error CreatePool(opts pools.CreateOptsBuilder) (*pools.Pool, error) @@ -50,6 +52,7 @@ type LbClient interface { ListMonitors(opts monitors.ListOptsBuilder) ([]monitors.Monitor, error) DeleteMonitor(id string) error ListLoadBalancerProviders() ([]providers.Provider, error) + ListOctaviaVersions() ([]apiversions.APIVersion, error) } type lbClient struct { @@ -101,6 +104,15 @@ func (l lbClient) CreateListener(opts listeners.CreateOptsBuilder) (*listeners.L return listener, nil } +func (l lbClient) UpdateListener(id string, opts listeners.UpdateOpts) (*listeners.Listener, error) { + mc := metrics.NewMetricPrometheusContext("loadbalancer_listener", "update") + listener, err := listeners.Update(l.serviceClient, id, opts).Extract() + if mc.ObserveRequest(err) != nil { + return nil, err + } + return listener, nil +} + func (l lbClient) ListListeners(opts listeners.ListOptsBuilder) ([]listeners.Listener, error) { mc := metrics.NewMetricPrometheusContext("loadbalancer_listener", "list") allPages, err := listeners.List(l.serviceClient, opts).AllPages() @@ -229,3 +241,12 @@ func (l lbClient) ListLoadBalancerProviders() ([]providers.Provider, error) { } return providersList, nil } + +func (l lbClient) ListOctaviaVersions() ([]apiversions.APIVersion, error) { + mc := metrics.NewMetricPrometheusContext("version", "list") + allPages, err := apiversions.List(l.serviceClient).AllPages() + if mc.ObserveRequest(err) != nil { + return nil, err + } + return apiversions.ExtractAPIVersions(allPages) +} diff --git a/pkg/cloud/services/loadbalancer/loadbalancer.go b/pkg/cloud/services/loadbalancer/loadbalancer.go index 761dbaef5a..5e883543ab 100644 --- a/pkg/cloud/services/loadbalancer/loadbalancer.go +++ b/pkg/cloud/services/loadbalancer/loadbalancer.go @@ -19,6 +19,7 @@ package loadbalancer import ( "errors" "fmt" + "reflect" "time" "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/listeners" @@ -26,6 +27,7 @@ import ( "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/monitors" "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/pools" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/net" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util" @@ -33,11 +35,14 @@ import ( "sigs.k8s.io/cluster-api-provider-openstack/pkg/record" capoerrors "sigs.k8s.io/cluster-api-provider-openstack/pkg/utils/errors" "sigs.k8s.io/cluster-api-provider-openstack/pkg/utils/names" + openstackutil "sigs.k8s.io/cluster-api-provider-openstack/pkg/utils/openstack" + capostrings "sigs.k8s.io/cluster-api-provider-openstack/pkg/utils/strings" ) const ( - networkPrefix string = "k8s-clusterapi" - kubeapiLBSuffix string = "kubeapi" + networkPrefix string = "k8s-clusterapi" + kubeapiLBSuffix string = "kubeapi" + defaultLoadBalancerProvider string = "amphora" ) const loadBalancerProvisioningStatusActive = "ACTIVE" @@ -54,7 +59,22 @@ func (s *Service) ReconcileLoadBalancer(openStackCluster *infrav1.OpenStackClust fixedIPAddress = openStackCluster.Spec.ControlPlaneEndpoint.Host } - lb, err := s.getOrCreateLoadBalancer(openStackCluster, loadBalancerName, openStackCluster.Status.Network.Subnet.ID, clusterName, fixedIPAddress) + providers, err := s.loadbalancerClient.ListLoadBalancerProviders() + if err != nil { + return err + } + + // As mostly all LoadBalancer features are only supported on "amphora" we explicitly set the provider + // in the LoadBalancer create call to make sure to get the desired features - even if multiple providers exist. + var lbProvider string + for _, v := range providers { + if v.Name == defaultLoadBalancerProvider { + lbProvider = v.Name + break + } + } + + lb, err := s.getOrCreateLoadBalancer(openStackCluster, loadBalancerName, openStackCluster.Status.Network.Subnet.ID, clusterName, fixedIPAddress, lbProvider) if err != nil { return err } @@ -81,10 +101,24 @@ func (s *Service) ReconcileLoadBalancer(openStackCluster *infrav1.OpenStackClust lbFloatingIP = fp.FloatingIP } + allowedCIDRs := []string{} + // To reduce API calls towards OpenStack API, let's handle the CIDR support verification for all Ports only once. + allowedCIDRsSupported := false + octaviaVersions, err := s.loadbalancerClient.ListOctaviaVersions() + if err != nil { + return err + } + // The current version is always the last one in the list. + octaviaVersion := octaviaVersions[len(octaviaVersions)-1].ID + if openstackutil.IsOctaviaFeatureSupported(octaviaVersion, openstackutil.OctaviaFeatureVIPACL, lbProvider) { + allowedCIDRsSupported = true + } + portList := []int{apiServerPort} portList = append(portList, openStackCluster.Spec.APIServerLoadBalancer.AdditionalPorts...) for _, port := range portList { lbPortObjectsName := fmt.Sprintf("%s-%d", loadBalancerName, port) + listener, err := s.getOrCreateListener(openStackCluster, lbPortObjectsName, lb.ID, port) if err != nil { return err @@ -98,18 +132,26 @@ func (s *Service) ReconcileLoadBalancer(openStackCluster *infrav1.OpenStackClust if err := s.getOrCreateMonitor(openStackCluster, lbPortObjectsName, pool.ID, lb.ID); err != nil { return err } + + if allowedCIDRsSupported { + if err := s.getOrUpdateAllowedCIDRS(openStackCluster, listener); err != nil { + return err + } + allowedCIDRs = listener.AllowedCIDRs + } } openStackCluster.Status.Network.APIServerLoadBalancer = &infrav1.LoadBalancer{ - Name: lb.Name, - ID: lb.ID, - InternalIP: lb.VipAddress, - IP: lbFloatingIP, + Name: lb.Name, + ID: lb.ID, + InternalIP: lb.VipAddress, + IP: lbFloatingIP, + AllowedCIDRs: allowedCIDRs, } return nil } -func (s *Service) getOrCreateLoadBalancer(openStackCluster *infrav1.OpenStackCluster, loadBalancerName, subnetID, clusterName string, vipAddress string) (*loadbalancers.LoadBalancer, error) { +func (s *Service) getOrCreateLoadBalancer(openStackCluster *infrav1.OpenStackCluster, loadBalancerName, subnetID, clusterName, vipAddress, provider string) (*loadbalancers.LoadBalancer, error) { lb, err := s.checkIfLbExists(loadBalancerName) if err != nil { return nil, err @@ -126,6 +168,7 @@ func (s *Service) getOrCreateLoadBalancer(openStackCluster *infrav1.OpenStackClu VipSubnetID: subnetID, VipAddress: vipAddress, Description: names.GetDescription(clusterName), + Provider: provider, } lb, err = s.loadbalancerClient.CreateLoadBalancer(lbCreateOpts) if err != nil { @@ -175,6 +218,71 @@ func (s *Service) getOrCreateListener(openStackCluster *infrav1.OpenStackCluster return listener, nil } +func (s *Service) getOrUpdateAllowedCIDRS(openStackCluster *infrav1.OpenStackCluster, listener *listeners.Listener) error { + allowedCIDRs := []string{} + + if len(openStackCluster.Spec.APIServerLoadBalancer.AllowedCIDRs) > 0 { + allowedCIDRs = append(allowedCIDRs, openStackCluster.Spec.APIServerLoadBalancer.AllowedCIDRs...) + + if openStackCluster.Spec.Bastion.Enabled { + allowedCIDRs = append(allowedCIDRs, openStackCluster.Status.Bastion.FloatingIP, openStackCluster.Status.Bastion.IP) + } + + if openStackCluster.Status.Network.Subnet.CIDR != "" { + allowedCIDRs = append(allowedCIDRs, openStackCluster.Status.Network.Subnet.CIDR) + } + + if len(openStackCluster.Status.Network.Router.IPs) > 0 { + allowedCIDRs = append(allowedCIDRs, openStackCluster.Status.Network.Router.IPs...) + } + } + + // Validate CIDRs and convert any given IP into a CIDR. + allowedCIDRs = validateIPs(openStackCluster, allowedCIDRs) + + // Remove duplicates. + allowedCIDRs = capostrings.Unique(allowedCIDRs) + listener.AllowedCIDRs = capostrings.Unique(listener.AllowedCIDRs) + + if !reflect.DeepEqual(allowedCIDRs, listener.AllowedCIDRs) { + listenerUpdateOpts := listeners.UpdateOpts{ + AllowedCIDRs: &allowedCIDRs, + } + + listener, err := s.loadbalancerClient.UpdateListener(listener.ID, listenerUpdateOpts) + if err != nil { + record.Warnf(openStackCluster, "FailedUpdateListener", "Failed to update listener %s: %v", listener.Name, err) + return err + } + + if err := s.waitForListener(listener.ID, "ACTIVE"); err != nil { + record.Warnf(openStackCluster, "FailedUpdateListener", "Failed to update listener %s with id %s: wait for listener active: %v", listener.Name, listener.ID, err) + return err + } + + record.Eventf(openStackCluster, "SuccessfulUpdateListener", "Updated allowed_cidrs %s for listener %s with id %s", listener.AllowedCIDRs, listener.Name, listener.ID) + } + return nil +} + +// validateIPs validates given IPs/CIDRs and removes non valid network objects. +func validateIPs(openStackCluster *infrav1.OpenStackCluster, definedCIDRs []string) []string { + marshaledCIDRs := []string{} + + for _, v := range definedCIDRs { + switch { + case net.IsIPv4String(v): + marshaledCIDRs = append(marshaledCIDRs, v+"/32") + case net.IsIPv4CIDRString(v): + marshaledCIDRs = append(marshaledCIDRs, v) + default: + record.Warnf(openStackCluster, "FailedIPAddressValidation", "%s is not a valid IPv4 nor CIDR address and will not get applied to allowed_cidrs", v) + } + } + + return marshaledCIDRs +} + func (s *Service) getOrCreatePool(openStackCluster *infrav1.OpenStackCluster, poolName, listenerID, lbID string) (*pools.Pool, error) { pool, err := s.checkIfPoolExists(poolName) if err != nil { diff --git a/pkg/cloud/services/loadbalancer/loadbalancer_test.go b/pkg/cloud/services/loadbalancer/loadbalancer_test.go index 8c7de04304..6dfc07fa4b 100644 --- a/pkg/cloud/services/loadbalancer/loadbalancer_test.go +++ b/pkg/cloud/services/loadbalancer/loadbalancer_test.go @@ -21,10 +21,12 @@ import ( "github.com/go-logr/logr" "github.com/golang/mock/gomock" + "github.com/gophercloud/gophercloud/openstack/compute/apiversions" "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/listeners" "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers" "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/monitors" "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/pools" + "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/providers" . "github.com/onsi/gomega" infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha5" @@ -75,6 +77,13 @@ func Test_ReconcileLoadBalancer(t *testing.T) { // add network api call results here }, expectLoadBalancer: func(m *mock_loadbalancer.MockLbClientMockRecorder) { + // return loadbalancer providers + providers := []providers.Provider{ + {Name: "amphora", Description: "The Octavia Amphora driver."}, + {Name: "octavia", Description: "Deprecated alias of the Octavia Amphora driver."}, + } + m.ListLoadBalancerProviders().Return(providers, nil) + pendingLB := loadbalancers.LoadBalancer{ ID: "aaaaaaaa-bbbb-cccc-dddd-333333333333", Name: "k8s-clusterapi-cluster-AAAAA-kubeapi", @@ -90,6 +99,14 @@ func Test_ReconcileLoadBalancer(t *testing.T) { // wait for active loadbalancer by returning active loadbalancer on second call m.GetLoadBalancer("aaaaaaaa-bbbb-cccc-dddd-333333333333").Return(&pendingLB, nil).Return(&activeLB, nil) + // return octavia versions + versions := []apiversions.APIVersion{ + {ID: "2.24"}, + {ID: "2.23"}, + {ID: "2.22"}, + } + m.ListOctaviaVersions().Return(versions, nil) + listenerList := []listeners.Listener{ { ID: "aaaaaaaa-bbbb-cccc-dddd-444444444444", diff --git a/pkg/cloud/services/loadbalancer/mock_loadbalancer/doc.go b/pkg/cloud/services/loadbalancer/mock_loadbalancer/doc.go new file mode 100644 index 0000000000..23ae8dee72 --- /dev/null +++ b/pkg/cloud/services/loadbalancer/mock_loadbalancer/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mock_loadbalancer // nolint + +//go:generate mockgen -destination=loadbalancer_service_mock.go -package=mock_loadbalancer sigs.k8s.io/cluster-api-provider-openstack/pkg/cloud/services/loadbalancer LbClient +//go:generate /usr/bin/env bash -c "cat ../../../../../hack/boilerplate/boilerplate.generatego.txt loadbalancer_service_mock.go > _loadbalancer_service_mock.go && mv _loadbalancer_service_mock.go loadbalancer_service_mock.go" diff --git a/pkg/cloud/services/loadbalancer/mock_loadbalancer/loadbalancer_service_mock.go b/pkg/cloud/services/loadbalancer/mock_loadbalancer/loadbalancer_service_mock.go index a42977549e..b346c04d46 100644 --- a/pkg/cloud/services/loadbalancer/mock_loadbalancer/loadbalancer_service_mock.go +++ b/pkg/cloud/services/loadbalancer/mock_loadbalancer/loadbalancer_service_mock.go @@ -24,6 +24,7 @@ import ( reflect "reflect" gomock "github.com/golang/mock/gomock" + apiversions "github.com/gophercloud/gophercloud/openstack/compute/apiversions" listeners "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/listeners" loadbalancers "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers" monitors "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/monitors" @@ -304,6 +305,21 @@ func (mr *MockLbClientMockRecorder) ListMonitors(arg0 interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMonitors", reflect.TypeOf((*MockLbClient)(nil).ListMonitors), arg0) } +// ListOctaviaVersions mocks base method. +func (m *MockLbClient) ListOctaviaVersions() ([]apiversions.APIVersion, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListOctaviaVersions") + ret0, _ := ret[0].([]apiversions.APIVersion) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListOctaviaVersions indicates an expected call of ListOctaviaVersions. +func (mr *MockLbClientMockRecorder) ListOctaviaVersions() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListOctaviaVersions", reflect.TypeOf((*MockLbClient)(nil).ListOctaviaVersions)) +} + // ListPoolMember mocks base method. func (m *MockLbClient) ListPoolMember(arg0 string, arg1 pools.ListMembersOptsBuilder) ([]pools.Member, error) { m.ctrl.T.Helper() @@ -333,3 +349,18 @@ func (mr *MockLbClientMockRecorder) ListPools(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPools", reflect.TypeOf((*MockLbClient)(nil).ListPools), arg0) } + +// UpdateListener mocks base method. +func (m *MockLbClient) UpdateListener(arg0 string, arg1 listeners.UpdateOpts) (*listeners.Listener, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateListener", arg0, arg1) + ret0, _ := ret[0].(*listeners.Listener) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateListener indicates an expected call of UpdateListener. +func (mr *MockLbClientMockRecorder) UpdateListener(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateListener", reflect.TypeOf((*MockLbClient)(nil).UpdateListener), arg0, arg1) +} diff --git a/pkg/cloud/services/networking/router.go b/pkg/cloud/services/networking/router.go index f2298dcd01..1a19eb0104 100644 --- a/pkg/cloud/services/networking/router.go +++ b/pkg/cloud/services/networking/router.go @@ -70,10 +70,16 @@ func (s *Service) ReconcileRouter(openStackCluster *infrav1.OpenStackCluster, cl s.scope.Logger.V(6).Info(fmt.Sprintf("Reuse existing Router %s with id %s", routerName, router.ID)) } + routerIPs := []string{} + for _, ip := range router.GatewayInfo.ExternalFixedIPs { + routerIPs = append(routerIPs, ip.IPAddress) + } + openStackCluster.Status.Network.Router = &infrav1.Router{ Name: router.Name, ID: router.ID, Tags: router.Tags, + IPs: routerIPs, } if len(openStackCluster.Spec.ExternalRouterIPs) > 0 { diff --git a/pkg/utils/openstack/loadbalancer.go b/pkg/utils/openstack/loadbalancer.go new file mode 100644 index 0000000000..7d1820e109 --- /dev/null +++ b/pkg/utils/openstack/loadbalancer.go @@ -0,0 +1,81 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openstack + +import ( + version "github.com/hashicorp/go-version" + klog "k8s.io/klog/v2" +) + +const ( + OctaviaFeatureTags = 0 + OctaviaFeatureVIPACL = 1 + OctaviaFeatureFlavors = 2 + OctaviaFeatureTimeout = 3 + OctaviaFeatureAvailabilityZones = 4 + lbProviderOVN = "ovn" +) + +// IsOctaviaFeatureSupported returns true if the given feature is supported in the deployed Octavia version. +// copied from https://github.com/kubernetes/cloud-provider-openstack/blob/master/pkg/util/openstack/loadbalancer.go#L95-L148 +func IsOctaviaFeatureSupported(octaviaVer string, feature int, lbProvider string) bool { + currentVer, _ := version.NewVersion(octaviaVer) + + switch feature { + case OctaviaFeatureVIPACL: + if lbProvider == lbProviderOVN { + return false + } + verACL, _ := version.NewVersion("v2.12") + if currentVer.GreaterThanOrEqual(verACL) { + return true + } + case OctaviaFeatureTags: + verTags, _ := version.NewVersion("v2.5") + if currentVer.GreaterThanOrEqual(verTags) { + return true + } + case OctaviaFeatureFlavors: + if lbProvider == lbProviderOVN { + return false + } + verFlavors, _ := version.NewVersion("v2.6") + if currentVer.GreaterThanOrEqual(verFlavors) { + return true + } + case OctaviaFeatureTimeout: + if lbProvider == lbProviderOVN { + return false + } + verFlavors, _ := version.NewVersion("v2.1") + if currentVer.GreaterThanOrEqual(verFlavors) { + return true + } + case OctaviaFeatureAvailabilityZones: + if lbProvider == lbProviderOVN { + return false + } + verAvailabilityZones, _ := version.NewVersion("v2.14") + if currentVer.GreaterThanOrEqual(verAvailabilityZones) { + return true + } + default: + klog.Warningf("Feature %d not recognized", feature) + } + + return false +} diff --git a/pkg/utils/strings/strings.go b/pkg/utils/strings/strings.go new file mode 100644 index 0000000000..944e1f0359 --- /dev/null +++ b/pkg/utils/strings/strings.go @@ -0,0 +1,29 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package strings + +func Unique(s []string) []string { + inResult := make(map[string]bool) + var result []string + for _, str := range s { + if _, ok := inResult[str]; !ok { + inResult[str] = true + result = append(result, str) + } + } + return result +}