From e27ccf337ff3e1c5017591b265901df3af49391f Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 22 Jul 2025 12:50:22 -0700 Subject: [PATCH 01/20] validations: allow IPv6 configurations for unmanaged clusters (AWSCluster resource) --- api/v1beta1/network_types.go | 2 -- api/v1beta2/awscluster_webhook.go | 6 ----- api/v1beta2/awscluster_webhook_test.go | 12 +++++----- api/v1beta2/network_types.go | 5 +--- ...ster.x-k8s.io_awsmanagedcontrolplanes.yaml | 24 ++++++++----------- ...tructure.cluster.x-k8s.io_awsclusters.yaml | 19 +++++++-------- ....cluster.x-k8s.io_awsclustertemplates.yaml | 19 +++++++-------- 7 files changed, 33 insertions(+), 54 deletions(-) diff --git a/api/v1beta1/network_types.go b/api/v1beta1/network_types.go index f72940f45b..7ff98f56f3 100644 --- a/api/v1beta1/network_types.go +++ b/api/v1beta1/network_types.go @@ -249,7 +249,6 @@ type SubnetSpec struct { // IPv6CidrBlock is the IPv6 CIDR block to be used when the provider creates a managed VPC. // A subnet can have an IPv4 and an IPv6 address. - // IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. // +optional IPv6CidrBlock string `json:"ipv6CidrBlock,omitempty"` @@ -261,7 +260,6 @@ type SubnetSpec struct { IsPublic bool `json:"isPublic"` // IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with a VPC that has IPv6 enabled. - // IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. // +optional IsIPv6 bool `json:"isIpv6,omitempty"` diff --git a/api/v1beta2/awscluster_webhook.go b/api/v1beta2/awscluster_webhook.go index ec4fac40af..755978a878 100644 --- a/api/v1beta2/awscluster_webhook.go +++ b/api/v1beta2/awscluster_webhook.go @@ -301,13 +301,7 @@ func (r *AWSCluster) validateSSHKeyName() field.ErrorList { func (r *AWSCluster) validateNetwork() field.ErrorList { var allErrs field.ErrorList - if r.Spec.NetworkSpec.VPC.IsIPv6Enabled() { - allErrs = append(allErrs, field.Invalid(field.NewPath("ipv6"), r.Spec.NetworkSpec.VPC.IPv6, "IPv6 cannot be used with unmanaged clusters at this time.")) - } for _, subnet := range r.Spec.NetworkSpec.Subnets { - if subnet.IsIPv6 || subnet.IPv6CidrBlock != "" { - allErrs = append(allErrs, field.Invalid(field.NewPath("subnets"), r.Spec.NetworkSpec.Subnets, "IPv6 cannot be used with unmanaged clusters at this time.")) - } if subnet.ZoneType != nil && subnet.IsEdge() { if subnet.ParentZoneName == nil { allErrs = append(allErrs, field.Invalid(field.NewPath("subnets"), r.Spec.NetworkSpec.Subnets, "ParentZoneName must be set when ZoneType is 'local-zone'.")) diff --git a/api/v1beta2/awscluster_webhook_test.go b/api/v1beta2/awscluster_webhook_test.go index ad1b22d5fb..6fc8de1846 100644 --- a/api/v1beta2/awscluster_webhook_test.go +++ b/api/v1beta2/awscluster_webhook_test.go @@ -324,7 +324,7 @@ func TestAWSClusterValidateCreate(t *testing.T) { wantErr: false, }, { - name: "rejects ipv6", + name: "accepts ipv6", cluster: &AWSCluster{ Spec: AWSClusterSpec{ NetworkSpec: NetworkSpec{ @@ -337,10 +337,10 @@ func TestAWSClusterValidateCreate(t *testing.T) { }, }, }, - wantErr: true, + wantErr: false, }, { - name: "rejects ipv6 enabled subnet", + name: "accepts ipv6 enabled subnet", cluster: &AWSCluster{ Spec: AWSClusterSpec{ NetworkSpec: NetworkSpec{ @@ -356,10 +356,10 @@ func TestAWSClusterValidateCreate(t *testing.T) { }, }, }, - wantErr: true, + wantErr: false, }, { - name: "rejects ipv6 cidr block for subnets", + name: "accepts ipv6 cidr block for subnets", cluster: &AWSCluster{ Spec: AWSClusterSpec{ NetworkSpec: NetworkSpec{ @@ -372,7 +372,7 @@ func TestAWSClusterValidateCreate(t *testing.T) { }, }, }, - wantErr: true, + wantErr: false, }, { name: "rejects ingress rules with cidr block and source security group id", diff --git a/api/v1beta2/network_types.go b/api/v1beta2/network_types.go index 26e38bc934..d60778c538 100644 --- a/api/v1beta2/network_types.go +++ b/api/v1beta2/network_types.go @@ -432,8 +432,7 @@ type VPCSpec struct { // Mutually exclusive with CidrBlock. IPAMPool *IPAMPool `json:"ipamPool,omitempty"` - // IPv6 contains ipv6 specific settings for the network. Supported only in managed clusters. - // This field cannot be set on AWSCluster object. + // IPv6 contains ipv6 specific settings for the network. // +optional IPv6 *IPv6 `json:"ipv6,omitempty"` @@ -561,7 +560,6 @@ type SubnetSpec struct { // IPv6CidrBlock is the IPv6 CIDR block to be used when the provider creates a managed VPC. // A subnet can have an IPv4 and an IPv6 address. - // IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. // +optional IPv6CidrBlock string `json:"ipv6CidrBlock,omitempty"` @@ -573,7 +571,6 @@ type SubnetSpec struct { IsPublic bool `json:"isPublic"` // IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with a VPC that has IPv6 enabled. - // IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. // +optional IsIPv6 bool `json:"isIpv6,omitempty"` diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml index 858dac8a4c..ab37e69b22 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml @@ -599,12 +599,11 @@ spec: description: |- IPv6CidrBlock is the IPv6 CIDR block to be used when the provider creates a managed VPC. A subnet can have an IPv4 and an IPv6 address. - IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. type: string isIpv6: - description: |- - IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with a VPC that has IPv6 enabled. - IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. + description: IsIPv6 defines the subnet as an IPv6 subnet. + A subnet is IPv6 when it is associated with a VPC that + has IPv6 enabled. type: boolean isPublic: description: IsPublic defines the subnet as a public subnet. @@ -785,9 +784,8 @@ spec: type: integer type: object ipv6: - description: |- - IPv6 contains ipv6 specific settings for the network. Supported only in managed clusters. - This field cannot be set on AWSCluster object. + description: IPv6 contains ipv6 specific settings for the + network. properties: cidrBlock: description: |- @@ -2823,12 +2821,11 @@ spec: description: |- IPv6CidrBlock is the IPv6 CIDR block to be used when the provider creates a managed VPC. A subnet can have an IPv4 and an IPv6 address. - IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. type: string isIpv6: - description: |- - IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with a VPC that has IPv6 enabled. - IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. + description: IsIPv6 defines the subnet as an IPv6 subnet. + A subnet is IPv6 when it is associated with a VPC that + has IPv6 enabled. type: boolean isPublic: description: IsPublic defines the subnet as a public subnet. @@ -3009,9 +3006,8 @@ spec: type: integer type: object ipv6: - description: |- - IPv6 contains ipv6 specific settings for the network. Supported only in managed clusters. - This field cannot be set on AWSCluster object. + description: IPv6 contains ipv6 specific settings for the + network. properties: cidrBlock: description: |- diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml index 83416aa9ae..111c87c54f 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml @@ -284,12 +284,11 @@ spec: description: |- IPv6CidrBlock is the IPv6 CIDR block to be used when the provider creates a managed VPC. A subnet can have an IPv4 and an IPv6 address. - IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. type: string isIpv6: - description: |- - IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with a VPC that has IPv6 enabled. - IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. + description: IsIPv6 defines the subnet as an IPv6 subnet. + A subnet is IPv6 when it is associated with a VPC that + has IPv6 enabled. type: boolean isPublic: description: IsPublic defines the subnet as a public subnet. @@ -1546,12 +1545,11 @@ spec: description: |- IPv6CidrBlock is the IPv6 CIDR block to be used when the provider creates a managed VPC. A subnet can have an IPv4 and an IPv6 address. - IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. type: string isIpv6: - description: |- - IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with a VPC that has IPv6 enabled. - IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. + description: IsIPv6 defines the subnet as an IPv6 subnet. + A subnet is IPv6 when it is associated with a VPC that + has IPv6 enabled. type: boolean isPublic: description: IsPublic defines the subnet as a public subnet. @@ -1732,9 +1730,8 @@ spec: type: integer type: object ipv6: - description: |- - IPv6 contains ipv6 specific settings for the network. Supported only in managed clusters. - This field cannot be set on AWSCluster object. + description: IPv6 contains ipv6 specific settings for the + network. properties: cidrBlock: description: |- diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml index e4a0a6cf58..411a3a5bcf 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml @@ -303,12 +303,11 @@ spec: description: |- IPv6CidrBlock is the IPv6 CIDR block to be used when the provider creates a managed VPC. A subnet can have an IPv4 and an IPv6 address. - IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. type: string isIpv6: - description: |- - IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with a VPC that has IPv6 enabled. - IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. + description: IsIPv6 defines the subnet as an IPv6 + subnet. A subnet is IPv6 when it is associated + with a VPC that has IPv6 enabled. type: boolean isPublic: description: IsPublic defines the subnet as a public @@ -1133,12 +1132,11 @@ spec: description: |- IPv6CidrBlock is the IPv6 CIDR block to be used when the provider creates a managed VPC. A subnet can have an IPv4 and an IPv6 address. - IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. type: string isIpv6: - description: |- - IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with a VPC that has IPv6 enabled. - IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. + description: IsIPv6 defines the subnet as an IPv6 + subnet. A subnet is IPv6 when it is associated + with a VPC that has IPv6 enabled. type: boolean isPublic: description: IsPublic defines the subnet as a public @@ -1320,9 +1318,8 @@ spec: type: integer type: object ipv6: - description: |- - IPv6 contains ipv6 specific settings for the network. Supported only in managed clusters. - This field cannot be set on AWSCluster object. + description: IPv6 contains ipv6 specific settings + for the network. properties: cidrBlock: description: |- From 7b3846ef85496909f221c6bb114ab06ab9da733d Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 22 Jul 2025 14:03:52 -0700 Subject: [PATCH 02/20] ec2: enable primary IPv6 on ENI for EC2 instances AWS requires that when registering targets by instance ID for an IPv6 target group, the targets must have an assigned primary IPv6 address. Note: The default subnets managed by CAPA are already set up to assign IPv6 addresses to newly created ENIs. --- pkg/cloud/services/ec2/instances.go | 56 +++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/pkg/cloud/services/ec2/instances.go b/pkg/cloud/services/ec2/instances.go index 6e5813c74a..3842b4aa4c 100644 --- a/pkg/cloud/services/ec2/instances.go +++ b/pkg/cloud/services/ec2/instances.go @@ -585,14 +585,25 @@ func (s *Service) runInstance(role string, i *infrav1.Instance) (*infrav1.Instan input.NetworkInterfaces = netInterfaces } else { - input.NetworkInterfaces = []types.InstanceNetworkInterfaceSpecification{ - { - DeviceIndex: aws.Int32(0), - SubnetId: aws.String(i.SubnetID), - Groups: i.SecurityGroupIDs, - AssociatePublicIpAddress: i.PublicIPOnLaunch, - }, + netInterface := types.InstanceNetworkInterfaceSpecification{ + DeviceIndex: aws.Int32(0), + SubnetId: aws.String(i.SubnetID), + Groups: i.SecurityGroupIDs, + AssociatePublicIpAddress: i.PublicIPOnLaunch, + } + + // When registering targets by instance ID for an IPv6 target group, the targets must have an assigned primary IPv6 address. + // Use case: registering controlplane nodes to the API LBs. + enablePrimaryIpv6, err := s.shouldEnablePrimaryIpv6(i) + if err != nil { + return nil, fmt.Errorf("failed to determine whether to enable PrimaryIpv6 for instance: %w", err) + } + if enablePrimaryIpv6 { + netInterface.PrimaryIpv6 = aws.Bool(true) + netInterface.Ipv6AddressCount = aws.Int32(1) } + + input.NetworkInterfaces = []types.InstanceNetworkInterfaceSpecification{netInterface} } if i.NetworkInterfaceType != "" { @@ -1307,3 +1318,34 @@ func getInstanceCPUOptionsRequest(cpuOptions infrav1.CPUOptions) *types.CpuOptio return request } + +func (s *Service) shouldEnablePrimaryIpv6(i *infrav1.Instance) (bool, error) { + var enablePrimaryIpv6 bool + + // We should enable IPv6 capabilities only when the users explicitly configure so. + if !s.scope.VPC().IsIPv6Enabled() { + return false, nil + } + + sn := s.scope.Subnets().FindByID(i.SubnetID) + if sn != nil { + enablePrimaryIpv6 = sn.IsIPv6 + } else { + // The subnet is in a different VPC than the cluster VPC. Then, we query AWS API. + sns, err := s.getFilteredSubnets(types.Filter{Name: aws.String("subnet-id"), Values: []string{i.SubnetID}}) + if err != nil { + return false, fmt.Errorf("failed to find subnet info with id %q for instance: %w", i.SubnetID, err) + } + if len(sns) == 0 { + return false, fmt.Errorf("expected subnet %q for instance to exist, but found none", i.SubnetID) + } + for _, set := range sns[0].Ipv6CidrBlockAssociationSet { + if set.Ipv6CidrBlockState.State == types.SubnetCidrBlockStateCodeAssociated { + enablePrimaryIpv6 = true + break + } + } + } + + return enablePrimaryIpv6, nil +} From aa0221eb354eedc2627d868612f69a70147c78d0 Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 22 Jul 2025 14:46:12 -0700 Subject: [PATCH 03/20] ec2: support option HTTPProtocolIPv6 for EC2 IMDS The httpProtocolIPv6 field enables or disables the IPv6 endpoint of the instance metadata service. The SDK only applies this field if httpEndpoint is enabled. When running on single-stack IPv6, pods only have IPv6, thus requiring an IPv6 endpoint to query IMDS as IPv4 network is unreachable. --- .../awsmachinetemplate_webhook_test.go | 1 + api/v1beta2/types.go | 12 ++++++++++ ...ster.x-k8s.io_awsmanagedcontrolplanes.yaml | 22 +++++++++++++++++++ ...tructure.cluster.x-k8s.io_awsclusters.yaml | 11 ++++++++++ ...ture.cluster.x-k8s.io_awsmachinepools.yaml | 11 ++++++++++ ...tructure.cluster.x-k8s.io_awsmachines.yaml | 11 ++++++++++ ....cluster.x-k8s.io_awsmachinetemplates.yaml | 11 ++++++++++ ...uster.x-k8s.io_awsmanagedmachinepools.yaml | 11 ++++++++++ .../awsmachine_controller_unit_test.go | 1 + pkg/cloud/services/ec2/instances.go | 5 +++++ pkg/cloud/services/ec2/launchtemplate.go | 4 ++++ 11 files changed, 100 insertions(+) diff --git a/api/v1beta2/awsmachinetemplate_webhook_test.go b/api/v1beta2/awsmachinetemplate_webhook_test.go index ce355d1e4b..c0c6349bf1 100644 --- a/api/v1beta2/awsmachinetemplate_webhook_test.go +++ b/api/v1beta2/awsmachinetemplate_webhook_test.go @@ -127,6 +127,7 @@ func TestAWSMachineTemplateValidateUpdate(t *testing.T) { InstanceType: "test", InstanceMetadataOptions: &InstanceMetadataOptions{ HTTPEndpoint: InstanceMetadataEndpointStateEnabled, + HTTPProtocolIPv6: InstanceMetadataEndpointStateDisabled, HTTPPutResponseHopLimit: 1, HTTPTokens: HTTPTokensStateOptional, InstanceMetadataTags: InstanceMetadataEndpointStateDisabled, diff --git a/api/v1beta2/types.go b/api/v1beta2/types.go index c268165c10..0ad8e8c901 100644 --- a/api/v1beta2/types.go +++ b/api/v1beta2/types.go @@ -365,6 +365,15 @@ type InstanceMetadataOptions struct { // +kubebuilder:default=enabled HTTPEndpoint InstanceMetadataState `json:"httpEndpoint,omitempty"` + // Enables or disables the IPv6 endpoint for the instance metadata service. + // This applies only if you enabled the HTTP metadata endpoint. + // + // Default: disabled + // + // +kubebuilder:validation:Enum:=enabled;disabled + // +kubebuilder:default=disabled + HTTPProtocolIPv6 InstanceMetadataState `json:"httpProtocolIpv6,omitempty"` + // The desired HTTP PUT response hop limit for instance metadata requests. The // larger the number, the further instance metadata requests can travel. // @@ -411,6 +420,9 @@ func (obj *InstanceMetadataOptions) SetDefaults() { if obj.HTTPEndpoint == "" { obj.HTTPEndpoint = InstanceMetadataEndpointStateEnabled } + if obj.HTTPProtocolIPv6 == "" { + obj.HTTPProtocolIPv6 = InstanceMetadataEndpointStateDisabled + } if obj.HTTPPutResponseHopLimit == 0 { obj.HTTPPutResponseHopLimit = 1 } diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml index ab37e69b22..194aa5c61a 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml @@ -1307,6 +1307,17 @@ spec: - enabled - disabled type: string + httpProtocolIpv6: + default: disabled + description: |- + Enables or disables the IPv6 endpoint for the instance metadata service. + This applies only if you enabled the HTTP metadata endpoint. + + Default: disabled + enum: + - enabled + - disabled + type: string httpPutResponseHopLimit: default: 1 description: |- @@ -3571,6 +3582,17 @@ spec: - enabled - disabled type: string + httpProtocolIpv6: + default: disabled + description: |- + Enables or disables the IPv6 endpoint for the instance metadata service. + This applies only if you enabled the HTTP metadata endpoint. + + Default: disabled + enum: + - enabled + - disabled + type: string httpPutResponseHopLimit: default: 1 description: |- diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml index 111c87c54f..2270cddf1a 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml @@ -2284,6 +2284,17 @@ spec: - enabled - disabled type: string + httpProtocolIpv6: + default: disabled + description: |- + Enables or disables the IPv6 endpoint for the instance metadata service. + This applies only if you enabled the HTTP metadata endpoint. + + Default: disabled + enum: + - enabled + - disabled + type: string httpPutResponseHopLimit: default: 1 description: |- diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml index 7bface8e4d..fa1b42930a 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinepools.yaml @@ -707,6 +707,17 @@ spec: - enabled - disabled type: string + httpProtocolIpv6: + default: disabled + description: |- + Enables or disables the IPv6 endpoint for the instance metadata service. + This applies only if you enabled the HTTP metadata endpoint. + + Default: disabled + enum: + - enabled + - disabled + type: string httpPutResponseHopLimit: default: 1 description: |- diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml index d7aa2cfef6..568d5566e8 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml @@ -902,6 +902,17 @@ spec: - enabled - disabled type: string + httpProtocolIpv6: + default: disabled + description: |- + Enables or disables the IPv6 endpoint for the instance metadata service. + This applies only if you enabled the HTTP metadata endpoint. + + Default: disabled + enum: + - enabled + - disabled + type: string httpPutResponseHopLimit: default: 1 description: |- diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml index 5e3f55519d..eb469ba4db 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml @@ -821,6 +821,17 @@ spec: - enabled - disabled type: string + httpProtocolIpv6: + default: disabled + description: |- + Enables or disables the IPv6 endpoint for the instance metadata service. + This applies only if you enabled the HTTP metadata endpoint. + + Default: disabled + enum: + - enabled + - disabled + type: string httpPutResponseHopLimit: default: 1 description: |- diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml index 11fdfa422c..ecd1026b42 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml @@ -716,6 +716,17 @@ spec: - enabled - disabled type: string + httpProtocolIpv6: + default: disabled + description: |- + Enables or disables the IPv6 endpoint for the instance metadata service. + This applies only if you enabled the HTTP metadata endpoint. + + Default: disabled + enum: + - enabled + - disabled + type: string httpPutResponseHopLimit: default: 1 description: |- diff --git a/controllers/awsmachine_controller_unit_test.go b/controllers/awsmachine_controller_unit_test.go index e5e9827bdd..79e9dc4a04 100644 --- a/controllers/awsmachine_controller_unit_test.go +++ b/controllers/awsmachine_controller_unit_test.go @@ -2733,6 +2733,7 @@ func TestAWSMachineReconcilerReconcileDefaultsToLoadBalancerTypeClassic(t *testi }, MetadataOptions: &ec2types.InstanceMetadataOptionsResponse{ HttpEndpoint: ec2types.InstanceMetadataEndpointState(string(infrav1.InstanceMetadataEndpointStateEnabled)), + HttpProtocolIpv6: ec2types.InstanceMetadataProtocolState(string(infrav1.InstanceMetadataEndpointStateDisabled)), HttpPutResponseHopLimit: aws.Int32(1), HttpTokens: ec2types.HttpTokensState(string(infrav1.HTTPTokensStateOptional)), InstanceMetadataTags: ec2types.InstanceMetadataTagsState(string(infrav1.InstanceMetadataEndpointStateDisabled)), diff --git a/pkg/cloud/services/ec2/instances.go b/pkg/cloud/services/ec2/instances.go index 3842b4aa4c..1fc604f104 100644 --- a/pkg/cloud/services/ec2/instances.go +++ b/pkg/cloud/services/ec2/instances.go @@ -969,6 +969,7 @@ func (s *Service) SDKToInstance(v types.Instance) (*infrav1.Instance, error) { metadataOptions.HTTPEndpoint = infrav1.InstanceMetadataState(string(v.MetadataOptions.HttpEndpoint)) metadataOptions.HTTPTokens = infrav1.HTTPTokensState(string(v.MetadataOptions.HttpTokens)) metadataOptions.InstanceMetadataTags = infrav1.InstanceMetadataState(string(v.MetadataOptions.InstanceMetadataTags)) + metadataOptions.HTTPProtocolIPv6 = infrav1.InstanceMetadataState(v.MetadataOptions.HttpProtocolIpv6) if v.MetadataOptions.HttpPutResponseHopLimit != nil { metadataOptions.HTTPPutResponseHopLimit = int64(*v.MetadataOptions.HttpPutResponseHopLimit) } @@ -1124,6 +1125,7 @@ func (s *Service) ModifyInstanceMetadataOptions(instanceID string, options *infr HttpPutResponseHopLimit: utils.ToInt32Pointer(&options.HTTPPutResponseHopLimit), HttpTokens: types.HttpTokensState(string(options.HTTPTokens)), InstanceMetadataTags: types.InstanceMetadataTagsState(string(options.InstanceMetadataTags)), + HttpProtocolIpv6: types.InstanceMetadataProtocolState(string(options.HTTPProtocolIPv6)), InstanceId: aws.String(instanceID), } @@ -1277,6 +1279,9 @@ func getInstanceMetadataOptionsRequest(metadataOptions *infrav1.InstanceMetadata if metadataOptions.HTTPEndpoint != "" { request.HttpEndpoint = types.InstanceMetadataEndpointState(string(metadataOptions.HTTPEndpoint)) } + if metadataOptions.HTTPProtocolIPv6 != "" { + request.HttpProtocolIpv6 = types.InstanceMetadataProtocolState(string(metadataOptions.HTTPProtocolIPv6)) + } if metadataOptions.HTTPPutResponseHopLimit != 0 { request.HttpPutResponseHopLimit = utils.ToInt32Pointer(&metadataOptions.HTTPPutResponseHopLimit) } diff --git a/pkg/cloud/services/ec2/launchtemplate.go b/pkg/cloud/services/ec2/launchtemplate.go index c805a00e7a..a13671d309 100644 --- a/pkg/cloud/services/ec2/launchtemplate.go +++ b/pkg/cloud/services/ec2/launchtemplate.go @@ -906,11 +906,15 @@ func (s *Service) SDKToLaunchTemplate(d types.LaunchTemplateVersion) (*expinfrav HTTPPutResponseHopLimit: utils.ToInt64Value(v.MetadataOptions.HttpPutResponseHopLimit), HTTPTokens: infrav1.HTTPTokensState(string(v.MetadataOptions.HttpTokens)), HTTPEndpoint: infrav1.InstanceMetadataEndpointStateEnabled, + HTTPProtocolIPv6: infrav1.InstanceMetadataEndpointStateDisabled, InstanceMetadataTags: infrav1.InstanceMetadataEndpointStateDisabled, } if v.MetadataOptions.HttpEndpoint == types.LaunchTemplateInstanceMetadataEndpointStateDisabled { i.InstanceMetadataOptions.HTTPEndpoint = infrav1.InstanceMetadataEndpointStateDisabled } + if v.MetadataOptions.HttpProtocolIpv6 == types.LaunchTemplateInstanceMetadataProtocolIpv6Enabled { + i.InstanceMetadataOptions.HTTPProtocolIPv6 = infrav1.InstanceMetadataEndpointStateEnabled + } if v.MetadataOptions.InstanceMetadataTags == types.LaunchTemplateInstanceMetadataTagsStateEnabled { i.InstanceMetadataOptions.InstanceMetadataTags = infrav1.InstanceMetadataEndpointStateEnabled } From 17f3816fdd719ccedbf49584fcdb4f2626d1aca2 Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 22 Jul 2025 15:35:19 -0700 Subject: [PATCH 04/20] routing: ensure routes to eigw are up to date In the case where egress-only-internet-gateway is deleted, CAPA reconcilation loop will create a new one. Thus, CAPA needs to modify the routes to point to the new eigw ID. --- pkg/cloud/services/network/routetables.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/cloud/services/network/routetables.go b/pkg/cloud/services/network/routetables.go index 21dd039ff1..e038038eb0 100644 --- a/pkg/cloud/services/network/routetables.go +++ b/pkg/cloud/services/network/routetables.go @@ -145,7 +145,8 @@ func (s *Service) fixMismatchedRouting(specRoute *ec2.CreateRouteInput, currentR if (currentRoute.DestinationIpv6CidrBlock != nil && aws.ToString(currentRoute.DestinationIpv6CidrBlock) == aws.ToString(specRoute.DestinationIpv6CidrBlock)) && ((currentRoute.GatewayId != nil && aws.ToString(currentRoute.GatewayId) != aws.ToString(specRoute.GatewayId)) || - (currentRoute.NatGatewayId != nil && aws.ToString(currentRoute.NatGatewayId) != aws.ToString(specRoute.NatGatewayId))) { + (currentRoute.NatGatewayId != nil && aws.ToString(currentRoute.NatGatewayId) != aws.ToString(specRoute.NatGatewayId)) || + (currentRoute.EgressOnlyInternetGatewayId != nil && aws.ToString(currentRoute.EgressOnlyInternetGatewayId) != aws.ToString(specRoute.EgressOnlyInternetGatewayId))) { input = &ec2.ReplaceRouteInput{ RouteTableId: rt.RouteTableId, DestinationIpv6CidrBlock: specRoute.DestinationIpv6CidrBlock, From b910741f44e22a38a5266afd2ccc757605e9d6ee Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 22 Jul 2025 18:59:32 -0700 Subject: [PATCH 05/20] subnets: configure default subnets to use NAT64/DNS64 This allows IPv6-only workloads to reach IPv4-only services. AWS supports this via NAT64/DNS64. More details: https://docs.aws.amazon.com/vpc/latest/userguide/nat-gateway-nat64-dns64.html --- pkg/cloud/services/interfaces.go | 2 + pkg/cloud/services/network/routetables.go | 8 ++ .../services/network/routetables_test.go | 18 ++++ pkg/cloud/services/network/subnets.go | 22 ++++ pkg/cloud/services/network/subnets_test.go | 102 ++++++++++++------ 5 files changed, 118 insertions(+), 34 deletions(-) diff --git a/pkg/cloud/services/interfaces.go b/pkg/cloud/services/interfaces.go index 65b08a2ecd..37a7906813 100644 --- a/pkg/cloud/services/interfaces.go +++ b/pkg/cloud/services/interfaces.go @@ -35,6 +35,8 @@ const ( AnyIPv4CidrBlock = "0.0.0.0/0" // AnyIPv6CidrBlock is the CIDR block to match all IPv6 addresses. AnyIPv6CidrBlock = "::/0" + // NAT64CidrBlock is the well-known CIDR block defined in RFC6052 for NAT64. + NAT64CidrBlock = "64:ff9b::/96" ) // ASGInterface encapsulates the methods exposed to the machinepool diff --git a/pkg/cloud/services/network/routetables.go b/pkg/cloud/services/network/routetables.go index e038038eb0..ad0856f254 100644 --- a/pkg/cloud/services/network/routetables.go +++ b/pkg/cloud/services/network/routetables.go @@ -321,6 +321,13 @@ func (s *Service) getNatGatewayPrivateRoute(natGatewayID string) *ec2.CreateRout } } +func (s *Service) getNat64PrivateRoute(natGatewayID string) *ec2.CreateRouteInput { + return &ec2.CreateRouteInput{ + NatGatewayId: aws.String(natGatewayID), + DestinationIpv6CidrBlock: aws.String(services.NAT64CidrBlock), + } +} + func (s *Service) getEgressOnlyInternetGateway() *ec2.CreateRouteInput { return &ec2.CreateRouteInput{ DestinationIpv6CidrBlock: aws.String(services.AnyIPv6CidrBlock), @@ -415,6 +422,7 @@ func (s *Service) getRoutesToPrivateSubnet(sn *infrav1.SubnetSpec) (routes []*ec routes = append(routes, s.getNatGatewayPrivateRoute(natGatewayID)) if sn.IsIPv6 { + routes = append(routes, s.getNat64PrivateRoute(natGatewayID)) if !s.scope.VPC().IsIPv6Enabled() { // Safety net because EgressOnlyInternetGateway needs the ID from the ipv6 block. // if, for whatever reason by this point that is not available, we don't want to diff --git a/pkg/cloud/services/network/routetables_test.go b/pkg/cloud/services/network/routetables_test.go index eb131b8217..1c5b901037 100644 --- a/pkg/cloud/services/network/routetables_test.go +++ b/pkg/cloud/services/network/routetables_test.go @@ -161,6 +161,13 @@ func TestReconcileRouteTables(t *testing.T) { })). After(privateRouteTable) + m.CreateRoute(context.TODO(), gomock.Eq(&ec2.CreateRouteInput{ + DestinationIpv6CidrBlock: aws.String("64:ff9b::/96"), + NatGatewayId: aws.String("nat-01"), + RouteTableId: aws.String("rt-1"), + })). + After(privateRouteTable) + m.CreateRoute(context.TODO(), gomock.Eq(&ec2.CreateRouteInput{ DestinationIpv6CidrBlock: aws.String("::/0"), EgressOnlyInternetGatewayId: aws.String("eigw-01"), @@ -247,6 +254,13 @@ func TestReconcileRouteTables(t *testing.T) { })). After(privateRouteTable) + m.CreateRoute(context.TODO(), gomock.Eq(&ec2.CreateRouteInput{ + DestinationIpv6CidrBlock: aws.String("64:ff9b::/96"), + NatGatewayId: aws.String("nat-01"), + RouteTableId: aws.String("rt-1"), + })). + After(privateRouteTable) + m.CreateRoute(context.TODO(), gomock.Eq(&ec2.CreateRouteInput{ DestinationIpv6CidrBlock: aws.String("::/0"), EgressOnlyInternetGatewayId: aws.String("eigw-01"), @@ -1199,6 +1213,10 @@ func TestService_getRoutesForSubnet(t *testing.T) { DestinationCidrBlock: aws.String("0.0.0.0/0"), NatGatewayId: aws.String("nat-gw-fromZone-us-east-1a"), }, + { + DestinationIpv6CidrBlock: aws.String("64:ff9b::/96"), + NatGatewayId: aws.String("nat-gw-fromZone-us-east-1a"), + }, { DestinationIpv6CidrBlock: aws.String("::/0"), EgressOnlyInternetGatewayId: aws.String("vpc-eigw"), diff --git a/pkg/cloud/services/network/subnets.go b/pkg/cloud/services/network/subnets.go index f339a9a8c0..f859df7448 100644 --- a/pkg/cloud/services/network/subnets.go +++ b/pkg/cloud/services/network/subnets.go @@ -545,6 +545,28 @@ func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, err return nil, errors.Wrapf(err, "failed to set subnet %q attribute assign ipv6 address on creation", *out.Subnet.SubnetId) } record.Eventf(s.scope.InfraCluster(), "SuccessfulModifySubnetAttributes", "Modified managed Subnet %q attributes", *out.Subnet.SubnetId) + + // Enable DNS64 so that the Route 53 Resolver returns DNS records for IPv4-only services + // containing a synthesized IPv6 address prefixed 64:ff9b::/96. + // This is needed alongside NAT64 to allow IPv6-only workloads to reach IPv4-only services. + // We only need to enable on private subnets. + if !sn.IsPublic { + if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { + if _, err := s.EC2Client.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ + SubnetId: out.Subnet.SubnetId, + EnableDns64: &types.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + }); err != nil { + return false, err + } + return true, nil + }, awserrors.SubnetNotFound); err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedModifySubnetAttributes", "Failed modifying managed Subnet %q attributes: %v", *out.Subnet.SubnetId, err) + return nil, errors.Wrapf(err, "failed to set subnet %q attribute enable dns64", *out.Subnet.SubnetId) + } + record.Eventf(s.scope.InfraCluster(), "SuccessfulModifySubnetAttributes", "Modified managed Subnet %q attributes", *out.Subnet.SubnetId) + } } // AWS Wavelength Zone's public subnets does not support to map Carrier IP address on launch, and diff --git a/pkg/cloud/services/network/subnets_test.go b/pkg/cloud/services/network/subnets_test.go index f14c9b7deb..87c6d88a72 100644 --- a/pkg/cloud/services/network/subnets_test.go +++ b/pkg/cloud/services/network/subnets_test.go @@ -1781,14 +1781,6 @@ func TestReconcileSubnets(t *testing.T) { }).Return(&ec2.ModifySubnetAttributeOutput{}, nil). After(firstSubnet) - m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ - AssignIpv6AddressOnCreation: &types.AttributeBooleanValue{ - Value: aws.Bool(true), - }, - SubnetId: aws.String("subnet-2"), - }).Return(&ec2.ModifySubnetAttributeOutput{}, nil). - After(firstSubnet) - m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ MapPublicIpOnLaunch: &types.AttributeBooleanValue{ Value: aws.Bool(true), @@ -1865,6 +1857,22 @@ func TestReconcileSubnets(t *testing.T) { }, nil). After(secondSubnet) + m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &types.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-2"), + }).Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(secondSubnet) + + m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ + EnableDns64: &types.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-2"), + }).Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(secondSubnet) + m.DescribeAvailabilityZones(context.TODO(), gomock.Any()). Return(&ec2.DescribeAvailabilityZonesOutput{ AvailabilityZones: []types.AvailabilityZone{ @@ -3656,15 +3664,6 @@ func TestReconcileSubnets(t *testing.T) { Return(&ec2.ModifySubnetAttributeOutput{}, nil). After(firstSubnet) - m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ - AssignIpv6AddressOnCreation: &types.AttributeBooleanValue{ - Value: aws.Bool(true), - }, - SubnetId: aws.String("subnet-2"), - }). - Return(&ec2.ModifySubnetAttributeOutput{}, nil). - After(firstSubnet) - m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ MapPublicIpOnLaunch: &types.AttributeBooleanValue{ Value: aws.Bool(true), @@ -3741,6 +3740,22 @@ func TestReconcileSubnets(t *testing.T) { }, nil). After(secondSubnet) + m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &types.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-2"), + }).Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(secondSubnet) + + m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ + EnableDns64: &types.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-2"), + }).Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(secondSubnet) + m.DescribeAvailabilityZones(context.TODO(), gomock.Any()). Return(&ec2.DescribeAvailabilityZonesOutput{ AvailabilityZones: []types.AvailabilityZone{ @@ -3903,15 +3918,6 @@ func TestReconcileSubnets(t *testing.T) { Return(&ec2.ModifySubnetAttributeOutput{}, nil). After(zone1PublicSubnet) - m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ - AssignIpv6AddressOnCreation: &types.AttributeBooleanValue{ - Value: aws.Bool(true), - }, - SubnetId: aws.String("subnet-2"), - }). - Return(&ec2.ModifySubnetAttributeOutput{}, nil). - After(zone1PublicSubnet) - m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ MapPublicIpOnLaunch: &types.AttributeBooleanValue{ Value: aws.Bool(true), @@ -3988,6 +3994,24 @@ func TestReconcileSubnets(t *testing.T) { }, nil). After(zone1PrivateSubnet) + m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &types.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-2"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone1PrivateSubnet) + + m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ + EnableDns64: &types.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-2"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone1PrivateSubnet) + // zone 2 m.DescribeAvailabilityZones(context.TODO(), &ec2.DescribeAvailabilityZonesInput{ ZoneNames: []string{"us-east-1c"}, @@ -4077,14 +4101,6 @@ func TestReconcileSubnets(t *testing.T) { Return(&ec2.ModifySubnetAttributeOutput{}, nil). After(zone2PublicSubnet) - m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ - AssignIpv6AddressOnCreation: &types.AttributeBooleanValue{ - Value: aws.Bool(true), - }, - SubnetId: aws.String("subnet-2"), - }). - Return(&ec2.ModifySubnetAttributeOutput{}, nil). - After(zone2PublicSubnet) m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ MapPublicIpOnLaunch: &types.AttributeBooleanValue{ Value: aws.Bool(true), @@ -4160,6 +4176,24 @@ func TestReconcileSubnets(t *testing.T) { }, }, nil). After(zone2PrivateSubnet) + + m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &types.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-2"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone2PrivateSubnet) + + m.ModifySubnetAttribute(context.TODO(), &ec2.ModifySubnetAttributeInput{ + EnableDns64: &types.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-2"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone2PrivateSubnet) }, }, } From cbddc1a0df1814cda0a55282041c39f01b8d172a Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 22 Jul 2025 22:18:58 -0700 Subject: [PATCH 06/20] eigw: use cluster tag key to list managed egress-only internet gateway The API for DescribeEgressOnlyInternetGateways does not support attachment.vpc-id filter. Thus, the call will return all available eigw. Consequences: - CAPA incorrectly selects an unintended eigw for use. Leading to route creation failure since the eigw belongs to a different VPC. - CAPA incorrectly destroys all eigw of all VPCs. This is very catastrophic as it can break other workloads. This commit changes the filter to use cluster tag instead. Additional safeguard is also included to check if the eigw is truly attached the VPC. --- .../services/network/egress_only_gateways.go | 20 ++++++++++++++++--- .../network/egress_only_gateways_test.go | 4 ++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/pkg/cloud/services/network/egress_only_gateways.go b/pkg/cloud/services/network/egress_only_gateways.go index e710adecf7..03e263e0fa 100644 --- a/pkg/cloud/services/network/egress_only_gateways.go +++ b/pkg/cloud/services/network/egress_only_gateways.go @@ -136,9 +136,12 @@ func (s *Service) createEgressOnlyInternetGateway() (*types.EgressOnlyInternetGa } func (s *Service) describeEgressOnlyVpcInternetGateways() ([]types.EgressOnlyInternetGateway, error) { + // The API for DescribeEgressOnlyInternetGateways does not support filtering by VPC ID attachment. + // More details: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeEgressOnlyInternetGateways.html + // Since the eigw is managed by CAPA, we can filter by the kubernetes cluster tag. out, err := s.EC2Client.DescribeEgressOnlyInternetGateways(context.TODO(), &ec2.DescribeEgressOnlyInternetGatewaysInput{ Filters: []types.Filter{ - filter.EC2.VPCAttachment(s.scope.VPC().ID), + filter.EC2.Cluster(s.scope.Name()), }, }) if err != nil { @@ -146,11 +149,22 @@ func (s *Service) describeEgressOnlyVpcInternetGateways() ([]types.EgressOnlyInt return nil, errors.Wrapf(err, "failed to describe egress only internet gateways in vpc %q", s.scope.VPC().ID) } - if len(out.EgressOnlyInternetGateways) == 0 { + // For safeguarding, we collect only egress-only internet gateways + // that are attached to the VPC. + eigws := make([]types.EgressOnlyInternetGateway, 0) + for _, eigw := range out.EgressOnlyInternetGateways { + for _, attachment := range eigw.Attachments { + if aws.ToString(attachment.VpcId) == s.scope.VPC().ID { + eigws = append(eigws, eigw) + } + } + } + + if len(eigws) == 0 { return nil, awserrors.NewNotFound(fmt.Sprintf("no egress only internet gateways found in vpc %q", s.scope.VPC().ID)) } - return out.EgressOnlyInternetGateways, nil + return eigws, nil } func (s *Service) getEgressOnlyGatewayTagParams(id string) infrav1.BuildParams { diff --git a/pkg/cloud/services/network/egress_only_gateways_test.go b/pkg/cloud/services/network/egress_only_gateways_test.go index fbd859ab80..9168805381 100644 --- a/pkg/cloud/services/network/egress_only_gateways_test.go +++ b/pkg/cloud/services/network/egress_only_gateways_test.go @@ -199,8 +199,8 @@ func TestDeleteEgressOnlyInternetGateways(t *testing.T) { m.DescribeEgressOnlyInternetGateways(context.TODO(), gomock.Eq(&ec2.DescribeEgressOnlyInternetGatewaysInput{ Filters: []types.Filter{ { - Name: aws.String("attachment.vpc-id"), - Values: []string{"vpc-gateways"}, + Name: aws.String("tag-key"), + Values: []string{infrav1.ClusterTagKey("test-cluster")}, }, }, })).Return(&ec2.DescribeEgressOnlyInternetGatewaysOutput{}, nil) From b11e1fc90972adc4e627bdc24c20c9ecd8dc9213 Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 22 Jul 2025 19:53:53 -0700 Subject: [PATCH 07/20] securitygroup: ensure icmpv6 is supported CAPA handles icmpv6 as a protocol number 58. AWS accepts protocol number when creating rules. However, describing a rule from AWS API returns the protocol name, thus causing CAPA to not recognize it and fail. --- pkg/cloud/services/securitygroup/securitygroups.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/cloud/services/securitygroup/securitygroups.go b/pkg/cloud/services/securitygroup/securitygroups.go index 9de501f7c5..99c909779a 100644 --- a/pkg/cloud/services/securitygroup/securitygroups.go +++ b/pkg/cloud/services/securitygroup/securitygroups.go @@ -53,7 +53,7 @@ const ( IPProtocolICMP = "icmp" // IPProtocolICMPv6 is how EC2 represents the ICMPv6 protocol in ingress rules. - IPProtocolICMPv6 = "58" + IPProtocolICMPv6 = "icmpv6" ) // ReconcileSecurityGroups will reconcile security groups against the Service object. @@ -921,8 +921,14 @@ func ingressRuleFromSDKProtocol(v types.IpPermission) infrav1.IngressRule { IPProtocolUDP, IPProtocolICMP, IPProtocolICMPv6: + // The API returns IpProtocol values as protocol names. + // But icmpv6 is handled as its protocol number in CAPA. + protocol := *v.IpProtocol + if protocol == IPProtocolICMPv6 { + protocol = string(infrav1.SecurityGroupProtocolICMPv6) + } return infrav1.IngressRule{ - Protocol: infrav1.SecurityGroupProtocol(*v.IpProtocol), + Protocol: infrav1.SecurityGroupProtocol(protocol), FromPort: utils.ToInt64Value(v.FromPort), ToPort: utils.ToInt64Value(v.ToPort), } From b60ecb1f432bcb583808628d52a828fc68215fb0 Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Mon, 28 Jul 2025 13:10:03 -0700 Subject: [PATCH 08/20] securitygroup: allow setting allowed IPv6 CIDR for node NodePort services For IPv4, we have field NodePortIngressRuleCidrBlocks that specifies the allowed source IPv4 CIDR for node NodePort services on port 30000-32767. This extends that field to also accept IPv6 source CIDRs. --- api/v1beta2/network_types.go | 28 ++++- api/v1beta2/zz_generated.deepcopy.go | 23 +++- pkg/cloud/scope/cluster.go | 2 +- pkg/cloud/scope/managedcontrolplane.go | 2 +- pkg/cloud/scope/sg.go | 2 +- .../services/securitygroup/securitygroups.go | 36 +++--- .../securitygroup/securitygroups_test.go | 110 ++++++++++++++++-- 7 files changed, 169 insertions(+), 34 deletions(-) diff --git a/api/v1beta2/network_types.go b/api/v1beta2/network_types.go index d60778c538..61478166a7 100644 --- a/api/v1beta2/network_types.go +++ b/api/v1beta2/network_types.go @@ -23,6 +23,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "k8s.io/utils/net" "k8s.io/utils/ptr" ) @@ -367,7 +368,32 @@ type NetworkSpec struct { // NodePortIngressRuleCidrBlocks is an optional set of CIDR blocks to allow traffic to nodes' NodePort services. // If none are specified here, all IPs are allowed to connect. // +optional - NodePortIngressRuleCidrBlocks []string `json:"nodePortIngressRuleCidrBlocks,omitempty"` + NodePortIngressRuleCidrBlocks CidrBlocks `json:"nodePortIngressRuleCidrBlocks,omitempty"` +} + +// CidrBlocks defines a set of CIDR blocks. +type CidrBlocks []string + +// IPv4CidrBlocks returns only IPv4 CIDR blocks. +func (c CidrBlocks) IPv4CidrBlocks() CidrBlocks { + var cidrs CidrBlocks + for _, cidr := range c { + if net.IsIPv4CIDRString(cidr) { + cidrs = append(cidrs, cidr) + } + } + return cidrs +} + +// IPv6CidrBlocks returns only IPv6 CIDR blocks. +func (c CidrBlocks) IPv6CidrBlocks() CidrBlocks { + var cidrs CidrBlocks + for _, cidr := range c { + if net.IsIPv6CIDRString(cidr) { + cidrs = append(cidrs, cidr) + } + } + return cidrs } // IPv6 contains ipv6 specific settings for the network. diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 197cffba66..dd41f32792 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1beta2 import ( corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/cluster-api/api/v1beta1" ) @@ -1353,6 +1353,25 @@ func (in *CPUOptions) DeepCopy() *CPUOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in CidrBlocks) DeepCopyInto(out *CidrBlocks) { + { + in := &in + *out = make(CidrBlocks, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CidrBlocks. +func (in CidrBlocks) DeepCopy() CidrBlocks { + if in == nil { + return nil + } + out := new(CidrBlocks) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClassicELBAttributes) DeepCopyInto(out *ClassicELBAttributes) { *out = *in @@ -1890,7 +1909,7 @@ func (in *NetworkSpec) DeepCopyInto(out *NetworkSpec) { } if in.NodePortIngressRuleCidrBlocks != nil { in, out := &in.NodePortIngressRuleCidrBlocks, &out.NodePortIngressRuleCidrBlocks - *out = make([]string, len(*in)) + *out = make(CidrBlocks, len(*in)) copy(*out, *in) } } diff --git a/pkg/cloud/scope/cluster.go b/pkg/cloud/scope/cluster.go index 730b977578..f290453d9b 100644 --- a/pkg/cloud/scope/cluster.go +++ b/pkg/cloud/scope/cluster.go @@ -433,6 +433,6 @@ func (s *ClusterScope) UnstructuredControlPlane() (*unstructured.Unstructured, e } // NodePortIngressRuleCidrBlocks returns the CIDR blocks for the node NodePort ingress rules. -func (s *ClusterScope) NodePortIngressRuleCidrBlocks() []string { +func (s *ClusterScope) NodePortIngressRuleCidrBlocks() infrav1.CidrBlocks { return s.AWSCluster.Spec.NetworkSpec.DeepCopy().NodePortIngressRuleCidrBlocks } diff --git a/pkg/cloud/scope/managedcontrolplane.go b/pkg/cloud/scope/managedcontrolplane.go index be0bc76864..60012e51ec 100644 --- a/pkg/cloud/scope/managedcontrolplane.go +++ b/pkg/cloud/scope/managedcontrolplane.go @@ -493,7 +493,7 @@ func (s *ManagedControlPlaneScope) UnstructuredControlPlane() (*unstructured.Uns } // NodePortIngressRuleCidrBlocks returns the CIDR blocks for the node NodePort ingress rules. -func (s *ManagedControlPlaneScope) NodePortIngressRuleCidrBlocks() []string { +func (s *ManagedControlPlaneScope) NodePortIngressRuleCidrBlocks() infrav1.CidrBlocks { return nil } diff --git a/pkg/cloud/scope/sg.go b/pkg/cloud/scope/sg.go index 05409d835c..7673386acd 100644 --- a/pkg/cloud/scope/sg.go +++ b/pkg/cloud/scope/sg.go @@ -64,5 +64,5 @@ type SGScope interface { ControlPlaneLoadBalancers() []*infrav1.AWSLoadBalancerSpec // NodePortIngressRuleCidrBlocks returns the CIDR blocks for the node NodePort ingress rules. - NodePortIngressRuleCidrBlocks() []string + NodePortIngressRuleCidrBlocks() infrav1.CidrBlocks } diff --git a/pkg/cloud/services/securitygroup/securitygroups.go b/pkg/cloud/services/securitygroup/securitygroups.go index 99c909779a..eb98d81c14 100644 --- a/pkg/cloud/services/securitygroup/securitygroups.go +++ b/pkg/cloud/services/securitygroup/securitygroups.go @@ -647,17 +647,27 @@ func (s *Service) getSecurityGroupIngressRules(role infrav1.SecurityGroupRole) ( return append(cniRules, rules...), nil case infrav1.SecurityGroupNode: - cidrBlocks := []string{services.AnyIPv4CidrBlock} - if scopeCidrBlocks := s.scope.NodePortIngressRuleCidrBlocks(); len(scopeCidrBlocks) > 0 { - cidrBlocks = scopeCidrBlocks + ipv4CidrBlocks := []string{services.AnyIPv4CidrBlock} + if scopeCidrBlocks := s.scope.NodePortIngressRuleCidrBlocks().IPv4CidrBlocks(); len(scopeCidrBlocks) > 0 { + ipv4CidrBlocks = scopeCidrBlocks } + + var ipv6CidrBlocks []string + if s.scope.VPC().IsIPv6Enabled() { + ipv6CidrBlocks = []string{services.AnyIPv6CidrBlock} + if scopeCidrBlocks := s.scope.NodePortIngressRuleCidrBlocks().IPv6CidrBlocks(); len(scopeCidrBlocks) > 0 { + ipv6CidrBlocks = scopeCidrBlocks + } + } + rules := infrav1.IngressRules{ { - Description: "Node Port Services", - Protocol: infrav1.SecurityGroupProtocolTCP, - FromPort: 30000, - ToPort: 32767, - CidrBlocks: cidrBlocks, + Description: "Node Port Services", + Protocol: infrav1.SecurityGroupProtocolTCP, + FromPort: 30000, + ToPort: 32767, + CidrBlocks: ipv4CidrBlocks, + IPv6CidrBlocks: ipv6CidrBlocks, }, { Description: "Kubelet API", @@ -671,18 +681,10 @@ func (s *Service) getSecurityGroupIngressRules(role infrav1.SecurityGroupRole) ( }, }, } + if s.scope.Bastion().Enabled { rules = append(rules, s.defaultSSHIngressRule(s.scope.SecurityGroups()[infrav1.SecurityGroupBastion].ID)) } - if s.scope.VPC().IsIPv6Enabled() { - rules = append(rules, infrav1.IngressRule{ - Description: "Node Port Services IPv6", - Protocol: infrav1.SecurityGroupProtocolTCP, - FromPort: 30000, - ToPort: 32767, - IPv6CidrBlocks: []string{services.AnyIPv6CidrBlock}, - }) - } additionalIngressRules, err := s.processIngressRulesSGs(s.scope.AdditionalNodeIngressRules()) if err != nil { diff --git a/pkg/cloud/services/securitygroup/securitygroups_test.go b/pkg/cloud/services/securitygroup/securitygroups_test.go index 2fd1cc64db..6440905c41 100644 --- a/pkg/cloud/services/securitygroup/securitygroups_test.go +++ b/pkg/cloud/services/securitygroup/securitygroups_test.go @@ -2344,12 +2344,16 @@ func TestNodePortServicesIngressRules(t *testing.T) { testCases := []struct { name string - cidrBlocks []string + networkSpec infrav1.NetworkSpec expectedIngresRules infrav1.IngressRules }{ { - name: "default node ports services ingress rules, no node port cidr block provided", - cidrBlocks: nil, + name: "default node ports services ingress rules, no node port cidr block provided", + networkSpec: infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + CidrBlock: "10.0.0.0/16", + }, + }, expectedIngresRules: infrav1.IngressRules{ { Description: "Node Port Services", @@ -2368,8 +2372,39 @@ func TestNodePortServicesIngressRules(t *testing.T) { }, }, { - name: "node port cidr block provided, no default cidr block used for node port services ingress rule", - cidrBlocks: []string{"10.0.0.0/16"}, + name: "default node ports services ingress rules for IPv6, no node port cidr block provided", + networkSpec: infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + CidrBlock: "10.0.0.0/16", + IPv6: &infrav1.IPv6{}, + }, + }, + expectedIngresRules: infrav1.IngressRules{ + { + Description: "Node Port Services", + Protocol: infrav1.SecurityGroupProtocolTCP, + FromPort: 30000, + ToPort: 32767, + CidrBlocks: []string{services.AnyIPv4CidrBlock}, + IPv6CidrBlocks: []string{services.AnyIPv6CidrBlock}, + }, + { + Description: "Kubelet API", + Protocol: infrav1.SecurityGroupProtocolTCP, + FromPort: 10250, + ToPort: 10250, + SourceSecurityGroupIDs: []string{"Id1", "Id2"}, + }, + }, + }, + { + name: "node port cidr block provided, no default cidr block used for node port services ingress rule", + networkSpec: infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + CidrBlock: "10.0.0.0/16", + }, + NodePortIngressRuleCidrBlocks: []string{"10.0.0.0/16"}, + }, expectedIngresRules: infrav1.IngressRules{ { Description: "Node Port Services", @@ -2387,6 +2422,64 @@ func TestNodePortServicesIngressRules(t *testing.T) { }, }, }, + { + name: "node port cidr block provided for only IPv6, no default cidr block used for node port services ingress rule", + networkSpec: infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + CidrBlock: "10.0.0.0/16", + IPv6: &infrav1.IPv6{ + CidrBlock: "2001:1234:5678:9a40::/56", + }, + }, + NodePortIngressRuleCidrBlocks: []string{"2001:1234:5678:9a40::/56"}, + }, + expectedIngresRules: infrav1.IngressRules{ + { + Description: "Node Port Services", + Protocol: infrav1.SecurityGroupProtocolTCP, + FromPort: 30000, + ToPort: 32767, + CidrBlocks: []string{services.AnyIPv4CidrBlock}, + IPv6CidrBlocks: []string{"2001:1234:5678:9a40::/56"}, + }, + { + Description: "Kubelet API", + Protocol: infrav1.SecurityGroupProtocolTCP, + FromPort: 10250, + ToPort: 10250, + SourceSecurityGroupIDs: []string{"Id1", "Id2"}, + }, + }, + }, + { + name: "node port cidr block provided for both IPv4 and IPv6, no default cidr block used for node port services ingress rule", + networkSpec: infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + CidrBlock: "10.0.0.0/16", + IPv6: &infrav1.IPv6{ + CidrBlock: "2001:1234:5678:9a40::/56", + }, + }, + NodePortIngressRuleCidrBlocks: []string{"10.0.0.0/16", "2001:1234:5678:9a40::/56"}, + }, + expectedIngresRules: infrav1.IngressRules{ + { + Description: "Node Port Services", + Protocol: infrav1.SecurityGroupProtocolTCP, + FromPort: 30000, + ToPort: 32767, + CidrBlocks: []string{"10.0.0.0/16"}, + IPv6CidrBlocks: []string{"2001:1234:5678:9a40::/56"}, + }, + { + Description: "Kubelet API", + Protocol: infrav1.SecurityGroupProtocolTCP, + FromPort: 10250, + ToPort: 10250, + SourceSecurityGroupIDs: []string{"Id1", "Id2"}, + }, + }, + }, } for _, tc := range testCases { @@ -2399,12 +2492,7 @@ func TestNodePortServicesIngressRules(t *testing.T) { AWSCluster: &infrav1.AWSCluster{ Spec: infrav1.AWSClusterSpec{ ControlPlaneLoadBalancer: &infrav1.AWSLoadBalancerSpec{}, - NetworkSpec: infrav1.NetworkSpec{ - VPC: infrav1.VPCSpec{ - CidrBlock: "10.0.0.0/16", - }, - NodePortIngressRuleCidrBlocks: tc.cidrBlocks, - }, + NetworkSpec: tc.networkSpec, }, Status: infrav1.AWSClusterStatus{ Network: infrav1.NetworkStatus{ From ace87c9785205cbe1660c0987874a38110b8b7d0 Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Mon, 28 Jul 2025 13:19:50 -0700 Subject: [PATCH 09/20] securitygroup: allow configuring IPv6 source CIDRs for bastion SSH We need an option to configure IPv6 source CIDRs for SSH ingress rule of the bastion host. This extends the field allowedCIDRBlocks to also accepts IPv6 CIDR blocks. --- api/v1beta1/zz_generated.conversion.go | 2 +- api/v1beta2/awscluster_types.go | 3 ++- api/v1beta2/awscluster_webhook_test.go | 6 +++++- api/v1beta2/defaults.go | 2 +- api/v1beta2/zz_generated.deepcopy.go | 2 +- ...ster.x-k8s.io_awsmanagedcontrolplanes.yaml | 2 ++ ...tructure.cluster.x-k8s.io_awsclusters.yaml | 1 + ....cluster.x-k8s.io_awsclustertemplates.yaml | 1 + .../awsmanagedcontrolplane_webhook_test.go | 19 ++++++++++++++++--- .../services/securitygroup/securitygroups.go | 16 +++++++++++----- 10 files changed, 41 insertions(+), 13 deletions(-) diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index 9c7a33e9fb..1495f63979 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -1707,7 +1707,7 @@ func Convert_v1beta2_AllowedNamespaces_To_v1beta1_AllowedNamespaces(in *v1beta2. func autoConvert_v1beta1_Bastion_To_v1beta2_Bastion(in *Bastion, out *v1beta2.Bastion, s conversion.Scope) error { out.Enabled = in.Enabled out.DisableIngressRules = in.DisableIngressRules - out.AllowedCIDRBlocks = *(*[]string)(unsafe.Pointer(&in.AllowedCIDRBlocks)) + out.AllowedCIDRBlocks = *(*v1beta2.CidrBlocks)(unsafe.Pointer(&in.AllowedCIDRBlocks)) out.InstanceType = in.InstanceType out.AMI = in.AMI return nil diff --git a/api/v1beta2/awscluster_types.go b/api/v1beta2/awscluster_types.go index 213ad99c56..ace245bdca 100644 --- a/api/v1beta2/awscluster_types.go +++ b/api/v1beta2/awscluster_types.go @@ -152,8 +152,9 @@ type Bastion struct { // AllowedCIDRBlocks is a list of CIDR blocks allowed to access the bastion host. // They are set as ingress rules for the Bastion host's Security Group (defaults to 0.0.0.0/0). + // If the cluster has IPv6 enabled, defaults to ::/0 and 0.0.0.0/0. // +optional - AllowedCIDRBlocks []string `json:"allowedCIDRBlocks,omitempty"` + AllowedCIDRBlocks CidrBlocks `json:"allowedCIDRBlocks,omitempty"` // InstanceType will use the specified instance type for the bastion. If not specified, // Cluster API Provider AWS will use t3.micro for all regions except us-east-1, where t2.micro diff --git a/api/v1beta2/awscluster_webhook_test.go b/api/v1beta2/awscluster_webhook_test.go index 6fc8de1846..8cc89954a8 100644 --- a/api/v1beta2/awscluster_webhook_test.go +++ b/api/v1beta2/awscluster_webhook_test.go @@ -1353,6 +1353,7 @@ func TestAWSClusterValidateAllowedCIDRBlocks(t *testing.T) { AllowedCIDRBlocks: []string{ "192.168.0.0/16", "192.168.0.1/32", + "2001:1234:5678:9a40::/56", }, }, }, @@ -1379,6 +1380,7 @@ func TestAWSClusterValidateAllowedCIDRBlocks(t *testing.T) { AllowedCIDRBlocks: []string{ "192.168.0.0/16", "192.168.0.1/32", + "2001:1234:5678:9a40::/56", }, DisableIngressRules: true, }, @@ -1393,6 +1395,7 @@ func TestAWSClusterValidateAllowedCIDRBlocks(t *testing.T) { Bastion: Bastion{ AllowedCIDRBlocks: []string{ "100.200.300.400/99", + "2001:1234:5678:9a40::/129", }, }, }, @@ -1445,6 +1448,7 @@ func TestAWSClusterDefaultAllowedCIDRBlocks(t *testing.T) { Bastion: Bastion{ AllowedCIDRBlocks: []string{ "0.0.0.0/0", + "::/0", }, }, }, @@ -1455,7 +1459,7 @@ func TestAWSClusterDefaultAllowedCIDRBlocks(t *testing.T) { beforeCluster: &AWSCluster{ Spec: AWSClusterSpec{ Bastion: Bastion{ - AllowedCIDRBlocks: []string{"0.0.0.0/0"}, + AllowedCIDRBlocks: []string{"0.0.0.0/0", "::/0"}, DisableIngressRules: true, Enabled: true, }, diff --git a/api/v1beta2/defaults.go b/api/v1beta2/defaults.go index f10bb895c1..540e1e7474 100644 --- a/api/v1beta2/defaults.go +++ b/api/v1beta2/defaults.go @@ -26,7 +26,7 @@ import ( func SetDefaults_Bastion(obj *Bastion) { //nolint:golint,stylecheck // Default to allow open access to the bastion host if no CIDR Blocks have been set if len(obj.AllowedCIDRBlocks) == 0 && !obj.DisableIngressRules { - obj.AllowedCIDRBlocks = []string{"0.0.0.0/0"} + obj.AllowedCIDRBlocks = []string{"0.0.0.0/0", "::/0"} } } diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index dd41f32792..6b4ac58666 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -1237,7 +1237,7 @@ func (in *Bastion) DeepCopyInto(out *Bastion) { *out = *in if in.AllowedCIDRBlocks != nil { in, out := &in.AllowedCIDRBlocks, &out.AllowedCIDRBlocks - *out = make([]string, len(*in)) + *out = make(CidrBlocks, len(*in)) copy(*out, *in) } } diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml index 194aa5c61a..deddd8fd73 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml @@ -126,6 +126,7 @@ spec: description: |- AllowedCIDRBlocks is a list of CIDR blocks allowed to access the bastion host. They are set as ingress rules for the Bastion host's Security Group (defaults to 0.0.0.0/0). + If the cluster has IPv6 enabled, defaults to ::/0 and 0.0.0.0/0. items: type: string type: array @@ -2361,6 +2362,7 @@ spec: description: |- AllowedCIDRBlocks is a list of CIDR blocks allowed to access the bastion host. They are set as ingress rules for the Bastion host's Security Group (defaults to 0.0.0.0/0). + If the cluster has IPv6 enabled, defaults to ::/0 and 0.0.0.0/0. items: type: string type: array diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml index 2270cddf1a..a348ae662f 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml @@ -946,6 +946,7 @@ spec: description: |- AllowedCIDRBlocks is a list of CIDR blocks allowed to access the bastion host. They are set as ingress rules for the Bastion host's Security Group (defaults to 0.0.0.0/0). + If the cluster has IPv6 enabled, defaults to ::/0 and 0.0.0.0/0. items: type: string type: array diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml index 411a3a5bcf..77aca1a9e5 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml @@ -524,6 +524,7 @@ spec: description: |- AllowedCIDRBlocks is a list of CIDR blocks allowed to access the bastion host. They are set as ingress rules for the Bastion host's Security Group (defaults to 0.0.0.0/0). + If the cluster has IPv6 enabled, defaults to ::/0 and 0.0.0.0/0. items: type: string type: array diff --git a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go index 40de7b369b..9817f9cf58 100644 --- a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go +++ b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go @@ -39,7 +39,7 @@ var ( func TestDefaultingWebhook(t *testing.T) { defaultTestBastion := infrav1.Bastion{ - AllowedCIDRBlocks: []string{"0.0.0.0/0"}, + AllowedCIDRBlocks: []string{"0.0.0.0/0", "::/0"}, } AZUsageLimit := 3 defaultVPCSpec := infrav1.VPCSpec{ @@ -114,8 +114,21 @@ func TestDefaultingWebhook(t *testing.T) { resourceName: "cluster1", resourceNS: "default", expectHash: false, - spec: AWSManagedControlPlaneSpec{Bastion: infrav1.Bastion{AllowedCIDRBlocks: []string{"100.100.100.100/0"}}}, - expectSpec: AWSManagedControlPlaneSpec{EKSClusterName: "default_cluster1", IdentityRef: defaultIdentityRef, Bastion: infrav1.Bastion{AllowedCIDRBlocks: []string{"100.100.100.100/0"}}, NetworkSpec: defaultNetworkSpec, TokenMethod: &EKSTokenMethodIAMAuthenticator, BootstrapSelfManagedAddons: true}, + spec: AWSManagedControlPlaneSpec{ + Bastion: infrav1.Bastion{ + AllowedCIDRBlocks: []string{"100.100.100.100/0", "2001:1234:5678:9a40::/56"}, + }, + }, + expectSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + IdentityRef: defaultIdentityRef, + Bastion: infrav1.Bastion{ + AllowedCIDRBlocks: []string{"100.100.100.100/0", "2001:1234:5678:9a40::/56"}, + }, + NetworkSpec: defaultNetworkSpec, + TokenMethod: &EKSTokenMethodIAMAuthenticator, + BootstrapSelfManagedAddons: true, + }, }, { name: "with CNI on network", diff --git a/pkg/cloud/services/securitygroup/securitygroups.go b/pkg/cloud/services/securitygroup/securitygroups.go index eb98d81c14..5b23224b34 100644 --- a/pkg/cloud/services/securitygroup/securitygroups.go +++ b/pkg/cloud/services/securitygroup/securitygroups.go @@ -596,13 +596,19 @@ func (s *Service) getSecurityGroupIngressRules(role infrav1.SecurityGroupRole) ( } switch role { case infrav1.SecurityGroupBastion: + ipv4CidrBlocks := s.scope.Bastion().AllowedCIDRBlocks.IPv4CidrBlocks() + var ipv6CidrBlocks []string + if s.scope.VPC().IsIPv6Enabled() { + ipv6CidrBlocks = s.scope.Bastion().AllowedCIDRBlocks.IPv6CidrBlocks() + } return infrav1.IngressRules{ { - Description: "SSH", - Protocol: infrav1.SecurityGroupProtocolTCP, - FromPort: 22, - ToPort: 22, - CidrBlocks: s.scope.Bastion().AllowedCIDRBlocks, + Description: "SSH", + Protocol: infrav1.SecurityGroupProtocolTCP, + FromPort: 22, + ToPort: 22, + CidrBlocks: ipv4CidrBlocks, + IPv6CidrBlocks: ipv6CidrBlocks, }, }, nil case infrav1.SecurityGroupControlPlane: From cc1fed5be005982bf6d469103b3ea108eae513ca Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 29 Jul 2025 21:00:07 -0700 Subject: [PATCH 10/20] crd: add IPv6 of bastion host to cluster status When creating a bastion host for an IPv6 cluster, the instance has both public IPv4 and IPv6. Thus, we need to report them in the cluster status if any. This also adds an additional print column to display that bastion IPv6. --- api/v1beta1/awscluster_conversion.go | 1 + api/v1beta1/zz_generated.conversion.go | 1 + api/v1beta2/awscluster_types.go | 3 ++- api/v1beta2/types.go | 3 +++ api/v1beta2/zz_generated.deepcopy.go | 7 ++++++- ...olplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml | 6 ++++++ .../infrastructure.cluster.x-k8s.io_awsclusters.yaml | 9 ++++++++- pkg/cloud/services/ec2/instances.go | 1 + 8 files changed, 28 insertions(+), 3 deletions(-) diff --git a/api/v1beta1/awscluster_conversion.go b/api/v1beta1/awscluster_conversion.go index a201fd6935..9878db8ab1 100644 --- a/api/v1beta1/awscluster_conversion.go +++ b/api/v1beta1/awscluster_conversion.go @@ -67,6 +67,7 @@ func (src *AWSCluster) ConvertTo(dstRaw conversion.Hub) error { dst.Status.Bastion.HostID = restored.Status.Bastion.HostID dst.Status.Bastion.CapacityReservationPreference = restored.Status.Bastion.CapacityReservationPreference dst.Status.Bastion.CPUOptions = restored.Status.Bastion.CPUOptions + dst.Status.Bastion.IPv6Address = restored.Status.Bastion.IPv6Address } dst.Spec.Partition = restored.Spec.Partition diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index 1495f63979..baf626e19e 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -2042,6 +2042,7 @@ func autoConvert_v1beta2_Instance_To_v1beta1_Instance(in *v1beta2.Instance, out out.IAMProfile = in.IAMProfile out.Addresses = *(*[]apiv1beta1.MachineAddress)(unsafe.Pointer(&in.Addresses)) out.PrivateIP = (*string)(unsafe.Pointer(in.PrivateIP)) + // WARNING: in.IPv6Address requires manual conversion: does not exist in peer-type out.PublicIP = (*string)(unsafe.Pointer(in.PublicIP)) out.ENASupport = (*bool)(unsafe.Pointer(in.ENASupport)) out.EBSOptimized = (*bool)(unsafe.Pointer(in.EBSOptimized)) diff --git a/api/v1beta2/awscluster_types.go b/api/v1beta2/awscluster_types.go index ace245bdca..13b2c389e0 100644 --- a/api/v1beta2/awscluster_types.go +++ b/api/v1beta2/awscluster_types.go @@ -324,7 +324,8 @@ type S3Bucket struct { // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.ready",description="Cluster infrastructure is ready for EC2 instances" // +kubebuilder:printcolumn:name="VPC",type="string",JSONPath=".spec.network.vpc.id",description="AWS VPC the cluster is using" // +kubebuilder:printcolumn:name="Endpoint",type="string",JSONPath=".spec.controlPlaneEndpoint",description="API Endpoint",priority=1 -// +kubebuilder:printcolumn:name="Bastion IP",type="string",JSONPath=".status.bastion.publicIp",description="Bastion IP address for breakglass access" +// +kubebuilder:printcolumn:name="Bastion IP",type="string",JSONPath=".status.bastion.publicIp",description="Bastion IPv4 address for breakglass access" +// +kubebuilder:printcolumn:name="Bastion IPv6",type="string",JSONPath=".status.bastion.ipv6Address",description="Bastion IPv6 address for breakglass access" // +k8s:defaulter-gen=true // AWSCluster is the schema for Amazon EC2 based Kubernetes Cluster API. diff --git a/api/v1beta2/types.go b/api/v1beta2/types.go index 0ad8e8c901..b52bf9956d 100644 --- a/api/v1beta2/types.go +++ b/api/v1beta2/types.go @@ -197,6 +197,9 @@ type Instance struct { // The private IPv4 address assigned to the instance. PrivateIP *string `json:"privateIp,omitempty"` + // The IPv6 address assigned to the instance. + IPv6Address *string `json:"ipv6Address,omitempty"` + // The public IPv4 address assigned to the instance, if applicable. PublicIP *string `json:"publicIp,omitempty"` diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 6b4ac58666..2f2360d4f3 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1beta2 import ( corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/cluster-api/api/v1beta1" ) @@ -1676,6 +1676,11 @@ func (in *Instance) DeepCopyInto(out *Instance) { *out = new(string) **out = **in } + if in.IPv6Address != nil { + in, out := &in.IPv6Address, &out.IPv6Address + *out = new(string) + **out = **in + } if in.PublicIP != nil { in, out := &in.PublicIP, &out.PublicIP *out = new(string) diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml index deddd8fd73..df9f3aa6b1 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml @@ -1368,6 +1368,9 @@ spec: instanceState: description: The current state of the instance. type: string + ipv6Address: + description: The IPv6 address assigned to the instance. + type: string marketType: description: |- MarketType specifies the type of market for the EC2 instance. Valid values include: @@ -3644,6 +3647,9 @@ spec: instanceState: description: The current state of the instance. type: string + ipv6Address: + description: The IPv6 address assigned to the instance. + type: string marketType: description: |- MarketType specifies the type of market for the EC2 instance. Valid values include: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml index a348ae662f..e3f30d6f4b 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml @@ -901,10 +901,14 @@ spec: name: Endpoint priority: 1 type: string - - description: Bastion IP address for breakglass access + - description: Bastion IPv4 address for breakglass access jsonPath: .status.bastion.publicIp name: Bastion IP type: string + - description: Bastion IPv6 address for breakglass access + jsonPath: .status.bastion.ipv6Address + name: Bastion IPv6 + type: string name: v1beta2 schema: openAPIV3Schema: @@ -2345,6 +2349,9 @@ spec: instanceState: description: The current state of the instance. type: string + ipv6Address: + description: The IPv6 address assigned to the instance. + type: string marketType: description: |- MarketType specifies the type of market for the EC2 instance. Valid values include: diff --git a/pkg/cloud/services/ec2/instances.go b/pkg/cloud/services/ec2/instances.go index 1fc604f104..a9b772ec50 100644 --- a/pkg/cloud/services/ec2/instances.go +++ b/pkg/cloud/services/ec2/instances.go @@ -933,6 +933,7 @@ func (s *Service) SDKToInstance(v types.Instance) (*infrav1.Instance, error) { ImageID: aws.ToString(v.ImageId), SSHKeyName: v.KeyName, PrivateIP: v.PrivateIpAddress, + IPv6Address: v.Ipv6Address, PublicIP: v.PublicIpAddress, ENASupport: v.EnaSupport, EBSOptimized: v.EbsOptimized, From 543f7d3612f4f4516a2ef76b893959a5f4dfb54d Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 29 Jul 2025 13:57:28 -0700 Subject: [PATCH 11/20] template: manifest templates for IPv6-enabled cluster This is a minimal template set to install an IPv6-enabled cluster. Both the controlplane and worker nodes must use nitro-based instance type (with IPv6 support). --- templates/cluster-template-ipv6.yaml | 1087 ++++++++++++++++++++++++++ 1 file changed, 1087 insertions(+) create mode 100644 templates/cluster-template-ipv6.yaml diff --git a/templates/cluster-template-ipv6.yaml b/templates/cluster-template-ipv6.yaml new file mode 100644 index 0000000000..059092f96b --- /dev/null +++ b/templates/cluster-template-ipv6.yaml @@ -0,0 +1,1087 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + ccm: external + csi: external + name: "${CLUSTER_NAME}" +spec: + clusterNetwork: + pods: + cidrBlocks: + - fd01::/48 + services: + cidrBlocks: + - fd02::/112 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: "${CLUSTER_NAME}-control-plane" + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSCluster + name: "${CLUSTER_NAME}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSCluster +metadata: + name: "${CLUSTER_NAME}" +spec: + controlPlaneLoadBalancer: + loadBalancerType: nlb + healthCheckProtocol: HTTPS + network: + vpc: + ipv6: {} + region: "${AWS_REGION}" + sshKeyName: "${AWS_SSH_KEY_NAME}" +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + bind-address: "::" + cloud-provider: external + controllerManager: + extraArgs: + bind-address: "::" + cloud-provider: external + scheduler: + extraArgs: + bind-address: "::" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + # node-ip: pass "::" to make kubelet prefer the default IPv6 address + # rather than the default IPv4 address. + node-ip: "::" + name: "{{ ds.meta_data.local_hostname }}" + localAPIEndpoint: + advertiseAddress: "::" + bindPort: 6443 + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + # node-ip: pass "::" to make kubelet prefer the default IPv6 address + # rather than the default IPv4 address. + node-ip: "::" + name: "{{ ds.meta_data.local_hostname }}" + controlPlane: + localAPIEndpoint: + advertiseAddress: "::" + bindPort: 6443 + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate + name: "${CLUSTER_NAME}-control-plane" + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: "${KUBERNETES_VERSION}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + iamInstanceProfile: "control-plane.cluster-api-provider-aws.sigs.k8s.io" + instanceType: "${AWS_CONTROL_PLANE_MACHINE_TYPE}" + sshKeyName: "${AWS_SSH_KEY_NAME}" + # Resource-based naming (RBN) allows AAAA record DNS query. + # IP-based hostname only support A records. + privateDnsName: + enableResourceNameDnsAAAARecord: true + enableResourceNameDnsARecord: true + hostnameType: resource-name +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: "${CLUSTER_NAME}-md-0" + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate + name: "${CLUSTER_NAME}-md-0" + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io + instanceType: "${AWS_NODE_MACHINE_TYPE}" + sshKeyName: "${AWS_SSH_KEY_NAME}" + # Resource-based naming (RBN) allows AAAA record DNS query. + # IP-based hostname only support A records. + privateDnsName: + enableResourceNameDnsAAAARecord: true + enableResourceNameDnsARecord: true + hostnameType: resource-name +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + clusterConfiguration: + apiServer: + extraArgs: + bind-address: '::' + controllerManager: + extraArgs: + bind-address: '::' + scheduler: + extraArgs: + bind-address: '::' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + # node-ip: pass "::" to make kubelet prefer the default IPv6 address + # rather than the default IPv4 address. + node-ip: "::" + name: '{{ ds.meta_data.local_hostname }}' +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +# We need to provide cloud-config to the CCM via a ConfigMap to +# set the NodeIPFamilies to IPv6. This instructs the CCM to consider +# IPv6 in the node's network interface. +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3 + args: + - --v=2 + - --cloud-provider=aws + - --use-service-account-credentials=true + - --configure-cloud-routes=false + - --cloud-config=/etc/kubernetes/cloud-config.conf + volumeMounts: + - name: cloud-config + mountPath: /etc/kubernetes/cloud-config.conf + subPath: cloud-config.conf + resources: + requests: + cpu: 200m + hostNetwork: true + volumes: + - name: cloud-config + configMap: + name: cloud-config + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: cloud-config + namespace: kube-system + data: + cloud-config.conf: | + [Global] + NodeIPFamilies=ipv6 + NodeIPFamilies=ipv4 + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: |- + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + containers: + - args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=2 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.25.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=2 + - --feature-gates=Topology=true + - --extra-create-metadata + - --leader-election=true + - --default-fstype=ext4 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.2 + name: csi-provisioner + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=2 + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-attacher:v4.4.2 + name: csi-attacher + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.2 + name: csi-snapshotter + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=2 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.9.2 + imagePullPolicy: Always + name: csi-resizer + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=2 + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.25.0 + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=2 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.2 + name: node-driver-registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: ebs-csi-node-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon From 55bb29f994aa3b54f746d36fdf2f505418d2179a Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 29 Jul 2025 14:09:13 -0700 Subject: [PATCH 12/20] cni: customized calico manifests for single-stack IPv6 This is a set of customized calico CNI manifests to support single-stack IPv6 cluster. Note that VXLAN is used since IP-in-IP currently only supports IPv4. References: - https://docs.tigera.io/calico/latest/networking/ipam/ipv6#ipv6 - https://docs.tigera.io/calico/latest/getting-started/kubernetes/self-managed-onprem/config-options#switching-from-ip-in-ip-to-vxlan - https://docs.tigera.io/calico/latest/networking/configuring/vxlan-ipip --- test/e2e/data/cni/calico_ipv6.yaml | 10560 +++++++++++++++++++++++++++ 1 file changed, 10560 insertions(+) create mode 100644 test/e2e/data/cni/calico_ipv6.yaml diff --git a/test/e2e/data/cni/calico_ipv6.yaml b/test/e2e/data/cni/calico_ipv6.yaml new file mode 100644 index 0000000000..07c98e6b32 --- /dev/null +++ b/test/e2e/data/cni/calico_ipv6.yaml @@ -0,0 +1,10560 @@ +--- +# Source: calico/templates/calico-kube-controllers.yaml +# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers +--- +# Source: calico/templates/calico-kube-controllers.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-kube-controllers + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-cni-plugin + namespace: kube-system +--- +# Source: calico/templates/calico-config.yaml +# This ConfigMap is used to configure a self-hosted Calico installation. +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-config + namespace: kube-system +data: + # Typha is disabled. + typha_service_name: "none" + # Configure the backend to use. + # Required for VXLAN only. + calico_backend: "vxlan" + + # Configure the MTU to use for workload interfaces and tunnels. + # By default, MTU is auto-detected, and explicitly setting this field should not be required. + # You can override auto-detection by providing a non-zero value. + veth_mtu: "0" + + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. + cni_network_config: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "calico", + "log_level": "info", + "log_file_path": "/var/log/calico/cni/cni.log", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": { + "type": "calico-ipam", + "assign_ipv4": "false", + "assign_ipv6": "true" + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + } + ] + } +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: bgpconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPConfiguration + listKind: BGPConfigurationList + plural: bgpconfigurations + singular: bgpconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: BGPConfiguration contains the configuration for any BGP routing. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BGPConfigurationSpec contains the values of the BGP configuration. + properties: + asNumber: + description: + "ASNumber is the default AS number used by a node. [Default: + 64512]" + format: int32 + type: integer + bindMode: + description: |- + BindMode indicates whether to listen for BGP connections on all addresses (None) + or only on the node's canonical IP address Node.Spec.BGP.IPvXAddress (NodeIP). + Default behaviour is to listen for BGP connections on all addresses. + type: string + communities: + description: + Communities is a list of BGP community values and their + arbitrary names for tagging routes. + items: + description: + Community contains standard or large community value + and its name. + properties: + name: + description: Name given to community value. + type: string + value: + description: |- + Value must be of format `aa:nn` or `aa:nn:mm`. + For standard community use `aa:nn` format, where `aa` and `nn` are 16 bit number. + For large community use `aa:nn:mm` format, where `aa`, `nn` and `mm` are 32 bit number. + Where, `aa` is an AS Number, `nn` and `mm` are per-AS identifier. + pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ + type: string + type: object + type: array + ignoredInterfaces: + description: + IgnoredInterfaces indicates the network interfaces that + needs to be excluded when reading device routes. + items: + type: string + type: array + listenPort: + description: + ListenPort is the port where BGP protocol should listen. + Defaults to 179 + maximum: 65535 + minimum: 1 + type: integer + localWorkloadPeeringIPV4: + description: |- + The virtual IPv4 address of the node with which its local workload is expected to peer. + It is recommended to use a link-local address. + type: string + localWorkloadPeeringIPV6: + description: |- + The virtual IPv6 address of the node with which its local workload is expected to peer. + It is recommended to use a link-local address. + type: string + logSeverityScreen: + description: + "LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: INFO]" + type: string + nodeMeshMaxRestartTime: + description: |- + Time to allow for software restart for node-to-mesh peerings. When specified, this is configured + as the graceful restart timeout. When not specified, the BIRD default of 120s is used. + This field can only be set on the default BGPConfiguration instance and requires that NodeMesh is enabled + type: string + nodeMeshPassword: + description: |- + Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: + The key of the secret to select from. Must be + a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: + Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + nodeToNodeMeshEnabled: + description: + "NodeToNodeMeshEnabled sets whether full node to node + BGP mesh is enabled. [Default: true]" + type: boolean + prefixAdvertisements: + description: + PrefixAdvertisements contains per-prefix advertisement + configuration. + items: + description: + PrefixAdvertisement configures advertisement properties + for the specified CIDR. + properties: + cidr: + description: CIDR for which properties should be advertised. + type: string + communities: + description: |- + Communities can be list of either community names already defined in `Specs.Communities` or community value of format `aa:nn` or `aa:nn:mm`. + For standard community use `aa:nn` format, where `aa` and `nn` are 16 bit number. + For large community use `aa:nn:mm` format, where `aa`, `nn` and `mm` are 32 bit number. + Where,`aa` is an AS Number, `nn` and `mm` are per-AS identifier. + items: + type: string + type: array + type: object + type: array + serviceClusterIPs: + description: |- + ServiceClusterIPs are the CIDR blocks from which service cluster IPs are allocated. + If specified, Calico will advertise these blocks, as well as any cluster IPs within them. + items: + description: + ServiceClusterIPBlock represents a single allowed ClusterIP + CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceExternalIPs: + description: |- + ServiceExternalIPs are the CIDR blocks for Kubernetes Service External IPs. + Kubernetes Service ExternalIPs will only be advertised if they are within one of these blocks. + items: + description: + ServiceExternalIPBlock represents a single allowed + External IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceLoadBalancerIPs: + description: |- + ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes Service LoadBalancer IPs. + Kubernetes Service status.LoadBalancer.Ingress IPs will only be advertised if they are within one of these blocks. + items: + description: + ServiceLoadBalancerIPBlock represents a single allowed + LoadBalancer IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: bgpfilters.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPFilter + listKind: BGPFilterList + plural: bgpfilters + singular: bgpfilter + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: + BGPFilterSpec contains the IPv4 and IPv6 filter rules of + the BGP Filter. + properties: + exportV4: + description: + The ordered set of IPv4 BGPFilter rules acting on exporting + routes to a peer. + items: + description: + BGPFilterRuleV4 defines a BGP filter rule consisting + a single IPv4 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + interface: + type: string + matchOperator: + type: string + prefixLength: + properties: + max: + format: int32 + maximum: 32 + minimum: 0 + type: integer + min: + format: int32 + maximum: 32 + minimum: 0 + type: integer + type: object + source: + type: string + required: + - action + type: object + type: array + exportV6: + description: + The ordered set of IPv6 BGPFilter rules acting on exporting + routes to a peer. + items: + description: + BGPFilterRuleV6 defines a BGP filter rule consisting + a single IPv6 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + interface: + type: string + matchOperator: + type: string + prefixLength: + properties: + max: + format: int32 + maximum: 128 + minimum: 0 + type: integer + min: + format: int32 + maximum: 128 + minimum: 0 + type: integer + type: object + source: + type: string + required: + - action + type: object + type: array + importV4: + description: + The ordered set of IPv4 BGPFilter rules acting on importing + routes from a peer. + items: + description: + BGPFilterRuleV4 defines a BGP filter rule consisting + a single IPv4 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + interface: + type: string + matchOperator: + type: string + prefixLength: + properties: + max: + format: int32 + maximum: 32 + minimum: 0 + type: integer + min: + format: int32 + maximum: 32 + minimum: 0 + type: integer + type: object + source: + type: string + required: + - action + type: object + type: array + importV6: + description: + The ordered set of IPv6 BGPFilter rules acting on importing + routes from a peer. + items: + description: + BGPFilterRuleV6 defines a BGP filter rule consisting + a single IPv6 CIDR block and a filter action for this CIDR. + properties: + action: + type: string + cidr: + type: string + interface: + type: string + matchOperator: + type: string + prefixLength: + properties: + max: + format: int32 + maximum: 128 + minimum: 0 + type: integer + min: + format: int32 + maximum: 128 + minimum: 0 + type: integer + type: object + source: + type: string + required: + - action + type: object + type: array + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: bgppeers.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPPeer + listKind: BGPPeerList + plural: bgppeers + singular: bgppeer + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BGPPeerSpec contains the specification for a BGPPeer resource. + properties: + asNumber: + description: The AS Number of the peer. + format: int32 + type: integer + filters: + description: The ordered set of BGPFilters applied on this BGP peer. + items: + type: string + type: array + keepOriginalNextHop: + description: |- + Option to keep the original nexthop field when routes are sent to a BGP Peer. + Setting "true" configures the selected BGP Peers node to use the "next hop keep;" + instead of "next hop self;"(default) in the specific branch of the Node on "bird.cfg". + Note: that this field is deprecated. Users should use the NextHopMode field to control + the next hop attribute for a BGP peer. + type: boolean + localWorkloadSelector: + description: |- + Selector for the local workload that the node should peer with. When this is set, the peerSelector and peerIP fields must be empty, + and the ASNumber must not be empty. + type: string + maxRestartTime: + description: |- + Time to allow for software restart. When specified, this is configured as the graceful + restart timeout. When not specified, the BIRD default of 120s is used. + type: string + nextHopMode: + allOf: + - enum: + - Auto + - Self + - Keep + - enum: + - Auto + - Self + - Keep + description: |- + NextHopMode defines the method of calculating the next hop attribute for received routes. + This replaces and expands the deprecated KeepOriginalNextHop field. + Users should use this setting to control the next hop attribute for a BGP peer. + When this is set, the value of the KeepOriginalNextHop field is ignored. + if neither keepOriginalNextHop or nextHopMode is specified, BGP's default behaviour is used. + Set it to “Auto” to apply BGP’s default behaviour. + Set it to "Self" to configure "next hop self;" in "bird.cfg". + Set it to "Keep" to configure "next hop keep;" in "bird.cfg". + type: string + node: + description: |- + The node name identifying the Calico node instance that is targeted by this peer. + If this is not set, and no nodeSelector is specified, then this BGP peer selects all + nodes in the cluster. + type: string + nodeSelector: + description: |- + Selector for the nodes that should have this peering. When this is set, the Node + field must be empty. + type: string + numAllowedLocalASNumbers: + description: |- + Maximum number of local AS numbers that are allowed in the AS path for received routes. + This removes BGP loop prevention and should only be used if absolutely necessary. + format: int32 + type: integer + password: + description: + Optional BGP password for the peerings generated by this + BGPPeer resource. + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: + The key of the secret to select from. Must be + a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: + Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + peerIP: + description: |- + The IP address of the peer followed by an optional port number to peer with. + If port number is given, format should be `[]:port` or `:` for IPv4. + If optional port number is not set, and this peer IP and ASNumber belongs to a calico/node + with ListenPort set in BGPConfiguration, then we use that port to peer. + type: string + peerSelector: + description: |- + Selector for the remote nodes to peer with. When this is set, the PeerIP and + ASNumber fields must be empty. For each peering between the local node and + selected remote nodes, we configure an IPv4 peering if both ends have + NodeBGPSpec.IPv4Address specified, and an IPv6 peering if both ends have + NodeBGPSpec.IPv6Address specified. The remote AS number comes from the remote + node's NodeBGPSpec.ASNumber, or the global default if that is not set. + type: string + reachableBy: + description: |- + Add an exact, i.e. /32, static route toward peer IP in order to prevent route flapping. + ReachableBy contains the address of the gateway which peer can be reached by. + type: string + sourceAddress: + description: |- + Specifies whether and how to configure a source address for the peerings generated by + this BGPPeer resource. Default value "UseNodeIP" means to configure the node IP as the + source address. "None" means not to configure a source address. + type: string + ttlSecurity: + description: |- + TTLSecurity enables the generalized TTL security mechanism (GTSM) which protects against spoofed packets by + ignoring received packets with a smaller than expected TTL value. The provided value is the number of hops + (edges) between the peers. + type: integer + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: blockaffinities.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BlockAffinity + listKind: BlockAffinityList + plural: blockaffinities + singular: blockaffinity + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: + BlockAffinitySpec contains the specification for a BlockAffinity + resource. + properties: + cidr: + type: string + deleted: + description: |- + Deleted indicates that this block affinity is being deleted. + This field is a string for compatibility with older releases that + mistakenly treat this field as a string. + type: string + node: + type: string + state: + type: string + type: + type: string + required: + - cidr + - deleted + - node + - state + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: caliconodestatuses.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: CalicoNodeStatus + listKind: CalicoNodeStatusList + plural: caliconodestatuses + singular: caliconodestatus + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: + CalicoNodeStatusSpec contains the specification for a CalicoNodeStatus + resource. + properties: + classes: + description: |- + Classes declares the types of information to monitor for this calico/node, + and allows for selective status reporting about certain subsets of information. + items: + type: string + type: array + node: + description: + The node name identifies the Calico node instance for + node status. + type: string + updatePeriodSeconds: + description: |- + UpdatePeriodSeconds is the period at which CalicoNodeStatus should be updated. + Set to 0 to disable CalicoNodeStatus refresh. Maximum update period is one day. + format: int32 + type: integer + type: object + status: + description: |- + CalicoNodeStatusStatus defines the observed state of CalicoNodeStatus. + No validation needed for status since it is updated by Calico. + properties: + agent: + description: Agent holds agent status on the node. + properties: + birdV4: + description: BIRDV4 represents the latest observed status of bird4. + properties: + lastBootTime: + description: + LastBootTime holds the value of lastBootTime + from bird.ctl output. + type: string + lastReconfigurationTime: + description: + LastReconfigurationTime holds the value of lastReconfigTime + from bird.ctl output. + type: string + routerID: + description: Router ID used by bird. + type: string + state: + description: The state of the BGP Daemon. + type: string + version: + description: Version of the BGP daemon + type: string + type: object + birdV6: + description: BIRDV6 represents the latest observed status of bird6. + properties: + lastBootTime: + description: + LastBootTime holds the value of lastBootTime + from bird.ctl output. + type: string + lastReconfigurationTime: + description: + LastReconfigurationTime holds the value of lastReconfigTime + from bird.ctl output. + type: string + routerID: + description: Router ID used by bird. + type: string + state: + description: The state of the BGP Daemon. + type: string + version: + description: Version of the BGP daemon + type: string + type: object + type: object + bgp: + description: BGP holds node BGP status. + properties: + numberEstablishedV4: + description: The total number of IPv4 established bgp sessions. + type: integer + numberEstablishedV6: + description: The total number of IPv6 established bgp sessions. + type: integer + numberNotEstablishedV4: + description: The total number of IPv4 non-established bgp sessions. + type: integer + numberNotEstablishedV6: + description: The total number of IPv6 non-established bgp sessions. + type: integer + peersV4: + description: PeersV4 represents IPv4 BGP peers status on the node. + items: + description: + CalicoNodePeer contains the status of BGP peers + on the node. + properties: + peerIP: + description: + IP address of the peer whose condition we are + reporting. + type: string + since: + description: Since the state or reason last changed. + type: string + state: + description: State is the BGP session state. + type: string + type: + description: |- + Type indicates whether this peer is configured via the node-to-node mesh, + or via en explicit global or per-node BGPPeer object. + type: string + type: object + type: array + peersV6: + description: PeersV6 represents IPv6 BGP peers status on the node. + items: + description: + CalicoNodePeer contains the status of BGP peers + on the node. + properties: + peerIP: + description: + IP address of the peer whose condition we are + reporting. + type: string + since: + description: Since the state or reason last changed. + type: string + state: + description: State is the BGP session state. + type: string + type: + description: |- + Type indicates whether this peer is configured via the node-to-node mesh, + or via en explicit global or per-node BGPPeer object. + type: string + type: object + type: array + required: + - numberEstablishedV4 + - numberEstablishedV6 + - numberNotEstablishedV4 + - numberNotEstablishedV6 + type: object + lastUpdated: + description: |- + LastUpdated is a timestamp representing the server time when CalicoNodeStatus object + last updated. It is represented in RFC3339 form and is in UTC. + format: date-time + nullable: true + type: string + routes: + description: + Routes reports routes known to the Calico BGP daemon + on the node. + properties: + routesV4: + description: RoutesV4 represents IPv4 routes on the node. + items: + description: + CalicoNodeRoute contains the status of BGP routes + on the node. + properties: + destination: + description: Destination of the route. + type: string + gateway: + description: Gateway for the destination. + type: string + interface: + description: Interface for the destination + type: string + learnedFrom: + description: + LearnedFrom contains information regarding + where this route originated. + properties: + peerIP: + description: + If sourceType is NodeMesh or BGPPeer, IP + address of the router that sent us this route. + type: string + sourceType: + description: + Type of the source where a route is learned + from. + type: string + type: object + type: + description: + Type indicates if the route is being used for + forwarding or not. + type: string + type: object + type: array + routesV6: + description: RoutesV6 represents IPv6 routes on the node. + items: + description: + CalicoNodeRoute contains the status of BGP routes + on the node. + properties: + destination: + description: Destination of the route. + type: string + gateway: + description: Gateway for the destination. + type: string + interface: + description: Interface for the destination + type: string + learnedFrom: + description: + LearnedFrom contains information regarding + where this route originated. + properties: + peerIP: + description: + If sourceType is NodeMesh or BGPPeer, IP + address of the router that sent us this route. + type: string + sourceType: + description: + Type of the source where a route is learned + from. + type: string + type: object + type: + description: + Type indicates if the route is being used for + forwarding or not. + type: string + type: object + type: array + type: object + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: clusterinformations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: ClusterInformation + listKind: ClusterInformationList + plural: clusterinformations + singular: clusterinformation + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: ClusterInformation contains the cluster specific information. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: + ClusterInformationSpec contains the values of describing + the cluster. + properties: + calicoVersion: + description: + CalicoVersion is the version of Calico that the cluster + is running + type: string + clusterGUID: + description: ClusterGUID is the GUID of the cluster + type: string + clusterType: + description: ClusterType describes the type of the cluster + type: string + datastoreReady: + description: |- + DatastoreReady is used during significant datastore migrations to signal to components + such as Felix that it should wait before accessing the datastore. + type: boolean + variant: + description: Variant declares which variant of Calico should be active. + type: string + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: felixconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: FelixConfiguration + listKind: FelixConfigurationList + plural: felixconfigurations + singular: felixconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: Felix Configuration contains the configuration for Felix. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: FelixConfigurationSpec contains the values of the Felix configuration. + properties: + allowIPIPPacketsFromWorkloads: + description: |- + AllowIPIPPacketsFromWorkloads controls whether Felix will add a rule to drop IPIP encapsulated traffic + from workloads. [Default: false] + type: boolean + allowVXLANPacketsFromWorkloads: + description: |- + AllowVXLANPacketsFromWorkloads controls whether Felix will add a rule to drop VXLAN encapsulated traffic + from workloads. [Default: false] + type: boolean + awsSrcDstCheck: + description: |- + AWSSrcDstCheck controls whether Felix will try to change the "source/dest check" setting on the EC2 instance + on which it is running. A value of "Disable" will try to disable the source/dest check. Disabling the check + allows for sending workload traffic without encapsulation within the same AWS subnet. + [Default: DoNothing] + enum: + - DoNothing + - Enable + - Disable + type: string + bpfCTLBLogFilter: + description: |- + BPFCTLBLogFilter specifies, what is logged by connect time load balancer when BPFLogLevel is + debug. Currently has to be specified as 'all' when BPFLogFilters is set + to see CTLB logs. + [Default: unset - means logs are emitted when BPFLogLevel id debug and BPFLogFilters not set.] + type: string + bpfConnectTimeLoadBalancing: + description: |- + BPFConnectTimeLoadBalancing when in BPF mode, controls whether Felix installs the connect-time load + balancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services + and it improves the performance of pod-to-service connections.When set to TCP, connect time load balancing + is available only for services with TCP ports. [Default: TCP] + enum: + - TCP + - Enabled + - Disabled + type: string + bpfConnectTimeLoadBalancingEnabled: + description: |- + BPFConnectTimeLoadBalancingEnabled when in BPF mode, controls whether Felix installs the connection-time load + balancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services + and it improves the performance of pod-to-service connections. The only reason to disable it is for debugging + purposes. + + Deprecated: Use BPFConnectTimeLoadBalancing [Default: true] + type: boolean + bpfConntrackLogLevel: + description: |- + BPFConntrackLogLevel controls the log level of the BPF conntrack cleanup program, which runs periodically + to clean up expired BPF conntrack entries. + [Default: Off]. + enum: + - "Off" + - Debug + type: string + bpfConntrackMode: + description: |- + BPFConntrackCleanupMode controls how BPF conntrack entries are cleaned up. `Auto` will use a BPF program if supported, + falling back to userspace if not. `Userspace` will always use the userspace cleanup code. `BPFProgram` will + always use the BPF program (failing if not supported). + [Default: Auto] + enum: + - Auto + - Userspace + - BPFProgram + type: string + bpfConntrackTimeouts: + description: |- + BPFConntrackTimers overrides the default values for the specified conntrack timer if + set. Each value can be either a duration or `Auto` to pick the value from + a Linux conntrack timeout. + + Configurable timers are: CreationGracePeriod, TCPSynSent, + TCPEstablished, TCPFinsSeen, TCPResetSeen, UDPTimeout, GenericTimeout, + ICMPTimeout. + + Unset values are replaced by the default values with a warning log for + incorrect values. + properties: + creationGracePeriod: + description: |2- + CreationGracePeriod gives a generic grace period to new connection + before they are considered for cleanup [Default: 10s]. + pattern: ^(([0-9]*(\.[0-9]*)?(ms|s|h|m|us)+)+|Auto)$ + type: string + genericTimeout: + description: |- + GenericTimeout controls how long it takes before considering this + entry for cleanup after the connection became idle. If set to 'Auto', the + value from nf_conntrack_generic_timeout is used. If nil, Calico uses its + own default value. [Default: 10m]. + pattern: ^(([0-9]*(\.[0-9]*)?(ms|s|h|m|us)+)+|Auto)$ + type: string + icmpTimeout: + description: |- + ICMPTimeout controls how long it takes before considering this + entry for cleanup after the connection became idle. If set to 'Auto', the + value from nf_conntrack_icmp_timeout is used. If nil, Calico uses its + own default value. [Default: 5s]. + pattern: ^(([0-9]*(\.[0-9]*)?(ms|s|h|m|us)+)+|Auto)$ + type: string + tcpEstablished: + description: |- + TCPEstablished controls how long it takes before considering this entry for + cleanup after the connection became idle. If set to 'Auto', the + value from nf_conntrack_tcp_timeout_established is used. If nil, Calico uses + its own default value. [Default: 1h]. + pattern: ^(([0-9]*(\.[0-9]*)?(ms|s|h|m|us)+)+|Auto)$ + type: string + tcpFinsSeen: + description: |- + TCPFinsSeen controls how long it takes before considering this entry for + cleanup after the connection was closed gracefully. If set to 'Auto', the + value from nf_conntrack_tcp_timeout_time_wait is used. If nil, Calico uses + its own default value. [Default: Auto]. + pattern: ^(([0-9]*(\.[0-9]*)?(ms|s|h|m|us)+)+|Auto)$ + type: string + tcpResetSeen: + description: |- + TCPResetSeen controls how long it takes before considering this entry for + cleanup after the connection was aborted. If nil, Calico uses its own + default value. [Default: 40s]. + pattern: ^(([0-9]*(\.[0-9]*)?(ms|s|h|m|us)+)+|Auto)$ + type: string + tcpSynSent: + description: |- + TCPSynSent controls how long it takes before considering this entry for + cleanup after the last SYN without a response. If set to 'Auto', the + value from nf_conntrack_tcp_timeout_syn_sent is used. If nil, Calico uses + its own default value. [Default: 20s]. + pattern: ^(([0-9]*(\.[0-9]*)?(ms|s|h|m|us)+)+|Auto)$ + type: string + udpTimeout: + description: |- + UDPTimeout controls how long it takes before considering this entry for + cleanup after the connection became idle. If nil, Calico uses its own + default value. [Default: 60s]. + pattern: ^(([0-9]*(\.[0-9]*)?(ms|s|h|m|us)+)+|Auto)$ + type: string + type: object + bpfDSROptoutCIDRs: + description: |- + BPFDSROptoutCIDRs is a list of CIDRs which are excluded from DSR. That is, clients + in those CIDRs will access service node ports as if BPFExternalServiceMode was set to + Tunnel. + items: + type: string + type: array + bpfDataIfacePattern: + description: |- + BPFDataIfacePattern is a regular expression that controls which interfaces Felix should attach BPF programs to + in order to catch traffic to/from the network. This needs to match the interfaces that Calico workload traffic + flows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the + cluster. It should not match the workload interfaces (usually named cali...) or any other special device managed + by Calico itself (e.g., tunnels). + type: string + bpfDisableGROForIfaces: + description: |- + BPFDisableGROForIfaces is a regular expression that controls which interfaces Felix should disable the + Generic Receive Offload [GRO] option. It should not match the workload interfaces (usually named cali...). + type: string + bpfDisableUnprivileged: + description: |- + BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled sysctl to disable + unprivileged use of BPF. This ensures that unprivileged users cannot access Calico's BPF maps and + cannot insert their own BPF programs to interfere with Calico's. [Default: true] + type: boolean + bpfEnabled: + description: + "BPFEnabled, if enabled Felix will use the BPF dataplane. + [Default: false]" + type: boolean + bpfEnforceRPF: + description: |- + BPFEnforceRPF enforce strict RPF on all host interfaces with BPF programs regardless of + what is the per-interfaces or global setting. Possible values are Disabled, Strict + or Loose. [Default: Loose] + pattern: ^(?i)(Disabled|Strict|Loose)?$ + type: string + bpfExcludeCIDRsFromNAT: + description: |- + BPFExcludeCIDRsFromNAT is a list of CIDRs that are to be excluded from NAT + resolution so that host can handle them. A typical usecase is node local + DNS cache. + items: + type: string + type: array + bpfExportBufferSizeMB: + description: |- + BPFExportBufferSizeMB in BPF mode, controls the buffer size used for sending BPF events to felix. + [Default: 1] + type: integer + bpfExtToServiceConnmark: + description: |- + BPFExtToServiceConnmark in BPF mode, controls a 32bit mark that is set on connections from an + external client to a local service. This mark allows us to control how packets of that + connection are routed within the host and how is routing interpreted by RPF check. [Default: 0] + type: integer + bpfExternalServiceMode: + description: |- + BPFExternalServiceMode in BPF mode, controls how connections from outside the cluster to services (node ports + and cluster IPs) are forwarded to remote workloads. If set to "Tunnel" then both request and response traffic + is tunneled to the remote node. If set to "DSR", the request traffic is tunneled but the response traffic + is sent directly from the remote node. In "DSR" mode, the remote node appears to use the IP of the ingress + node; this requires a permissive L2 network. [Default: Tunnel] + pattern: ^(?i)(Tunnel|DSR)?$ + type: string + bpfForceTrackPacketsFromIfaces: + description: |- + BPFForceTrackPacketsFromIfaces in BPF mode, forces traffic from these interfaces + to skip Calico's iptables NOTRACK rule, allowing traffic from those interfaces to be + tracked by Linux conntrack. Should only be used for interfaces that are not used for + the Calico fabric. For example, a docker bridge device for non-Calico-networked + containers. [Default: docker+] + items: + type: string + type: array + bpfHostConntrackBypass: + description: |- + BPFHostConntrackBypass Controls whether to bypass Linux conntrack in BPF mode for + workloads and services. [Default: true - bypass Linux conntrack] + type: boolean + bpfHostNetworkedNATWithoutCTLB: + description: |- + BPFHostNetworkedNATWithoutCTLB when in BPF mode, controls whether Felix does a NAT without CTLB. This along with BPFConnectTimeLoadBalancing + determines the CTLB behavior. [Default: Enabled] + enum: + - Enabled + - Disabled + type: string + bpfKubeProxyEndpointSlicesEnabled: + description: |- + BPFKubeProxyEndpointSlicesEnabled is deprecated and has no effect. BPF + kube-proxy always accepts endpoint slices. This option will be removed in + the next release. + type: boolean + bpfKubeProxyIptablesCleanupEnabled: + description: |- + BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF mode, Felix will proactively clean up the upstream + Kubernetes kube-proxy's iptables chains. Should only be enabled if kube-proxy is not running. [Default: true] + type: boolean + bpfKubeProxyMinSyncPeriod: + description: |- + BPFKubeProxyMinSyncPeriod, in BPF mode, controls the minimum time between updates to the dataplane for Felix's + embedded kube-proxy. Lower values give reduced set-up latency. Higher values reduce Felix CPU usage by + batching up more work. [Default: 1s] + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + bpfL3IfacePattern: + description: |- + BPFL3IfacePattern is a regular expression that allows to list tunnel devices like wireguard or vxlan (i.e., L3 devices) + in addition to BPFDataIfacePattern. That is, tunnel interfaces not created by Calico, that Calico workload traffic flows + over as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster. + type: string + bpfLogFilters: + additionalProperties: + type: string + description: |- + BPFLogFilters is a map of key=values where the value is + a pcap filter expression and the key is an interface name with 'all' + denoting all interfaces, 'weps' all workload endpoints and 'heps' all host + endpoints. + + When specified as an env var, it accepts a comma-separated list of + key=values. + [Default: unset - means all debug logs are emitted] + type: object + bpfLogLevel: + description: |- + BPFLogLevel controls the log level of the BPF programs when in BPF dataplane mode. One of "Off", "Info", or + "Debug". The logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`. + [Default: Off]. + pattern: ^(?i)(Off|Info|Debug)?$ + type: string + bpfMapSizeConntrack: + description: |- + BPFMapSizeConntrack sets the size for the conntrack map. This map must be large enough to hold + an entry for each active connection. Warning: changing the size of the conntrack map can cause disruption. + type: integer + bpfMapSizeConntrackCleanupQueue: + description: |- + BPFMapSizeConntrackCleanupQueue sets the size for the map used to hold NAT conntrack entries that are queued + for cleanup. This should be big enough to hold all the NAT entries that expire within one cleanup interval. + minimum: 1 + type: integer + bpfMapSizeConntrackScaling: + description: |- + BPFMapSizeConntrackScaling controls whether and how we scale the conntrack map size depending + on its usage. 'Disabled' make the size stay at the default or whatever is set by + BPFMapSizeConntrack*. 'DoubleIfFull' doubles the size when the map is pretty much full even + after cleanups. [Default: DoubleIfFull] + pattern: ^(?i)(Disabled|DoubleIfFull)?$ + type: string + bpfMapSizeIPSets: + description: |- + BPFMapSizeIPSets sets the size for ipsets map. The IP sets map must be large enough to hold an entry + for each endpoint matched by every selector in the source/destination matches in network policy. Selectors + such as "all()" can result in large numbers of entries (one entry per endpoint in that case). + type: integer + bpfMapSizeIfState: + description: |- + BPFMapSizeIfState sets the size for ifstate map. The ifstate map must be large enough to hold an entry + for each device (host + workloads) on a host. + type: integer + bpfMapSizeNATAffinity: + description: |- + BPFMapSizeNATAffinity sets the size of the BPF map that stores the affinity of a connection (for services that + enable that feature. + type: integer + bpfMapSizeNATBackend: + description: |- + BPFMapSizeNATBackend sets the size for NAT back end map. + This is the total number of endpoints. This is mostly + more than the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: |- + BPFMapSizeNATFrontend sets the size for NAT front end map. + FrontendMap should be large enough to hold an entry for each nodeport, + external IP and each port in each service. + type: integer + bpfMapSizePerCpuConntrack: + description: |- + BPFMapSizePerCPUConntrack determines the size of conntrack map based on the number of CPUs. If set to a + non-zero value, overrides BPFMapSizeConntrack with `BPFMapSizePerCPUConntrack * (Number of CPUs)`. + This map must be large enough to hold an entry for each active connection. Warning: changing the size of the + conntrack map can cause disruption. + type: integer + bpfMapSizeRoute: + description: |- + BPFMapSizeRoute sets the size for the routes map. The routes map should be large enough + to hold one entry per workload and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: |- + BPFPSNATPorts sets the range from which we randomly pick a port if there is a source port + collision. This should be within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating systems. Linux uses + 32768–60999, while others mostly use the IANA defined range 49152–65535. It is not necessarily + a problem if this range overlaps with the operating systems. Both ends of the range are + inclusive. [Default: 20000:29999] + pattern: ^.* + x-kubernetes-int-or-string: true + bpfPolicyDebugEnabled: + description: |- + BPFPolicyDebugEnabled when true, Felix records detailed information + about the BPF policy programs, which can be examined with the calico-bpf command-line tool. + type: boolean + bpfProfiling: + description: |- + BPFProfiling controls profiling of BPF programs. At the monent, it can be + Disabled or Enabled. [Default: Disabled] + enum: + - Enabled + - Disabled + type: string + bpfRedirectToPeer: + description: |- + BPFRedirectToPeer controls which whether it is allowed to forward straight to the + peer side of the workload devices. It is allowed for any host L2 devices by default + (L2Only), but it breaks TCP dump on the host side of workload device as it bypasses + it on ingress. Value of Enabled also allows redirection from L3 host devices like + IPIP tunnel or Wireguard directly to the peer side of the workload's device. This + makes redirection faster, however, it breaks tools like tcpdump on the peer side. + Use Enabled with caution. [Default: L2Only] + enum: + - Enabled + - Disabled + - L2Only + type: string + chainInsertMode: + description: |- + ChainInsertMode controls whether Felix hooks the kernel's top-level iptables chains by inserting a rule + at the top of the chain or by appending a rule at the bottom. insert is the safe default since it prevents + Calico's rules from being bypassed. If you switch to append mode, be sure that the other rules in the chains + signal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed. + [Default: insert] + pattern: ^(?i)(Insert|Append)?$ + type: string + dataplaneDriver: + description: |- + DataplaneDriver filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver + is set to false. + type: string + dataplaneWatchdogTimeout: + description: |- + DataplaneWatchdogTimeout is the readiness/liveness timeout used for Felix's (internal) dataplane driver. + Deprecated: replaced by the generic HealthTimeoutOverrides. + type: string + debugDisableLogDropping: + description: |- + DebugDisableLogDropping disables the dropping of log messages when the log buffer is full. This can + significantly impact performance if log write-out is a bottleneck. [Default: false] + type: boolean + debugHost: + description: |- + DebugHost is the host IP or hostname to bind the debug port to. Only used + if DebugPort is set. [Default:localhost] + type: string + debugMemoryProfilePath: + description: + DebugMemoryProfilePath is the path to write the memory + profile to when triggered by signal. + type: string + debugPort: + description: |- + DebugPort if set, enables Felix's debug HTTP port, which allows memory and CPU profiles + to be retrieved. The debug port is not secure, it should not be exposed to the internet. + type: integer + debugSimulateCalcGraphHangAfter: + description: |- + DebugSimulateCalcGraphHangAfter is used to simulate a hang in the calculation graph after the specified duration. + This is useful in tests of the watchdog system only! + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + debugSimulateDataplaneApplyDelay: + description: |- + DebugSimulateDataplaneApplyDelay adds an artificial delay to every dataplane operation. This is useful for + simulating a heavily loaded system for test purposes only. + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + debugSimulateDataplaneHangAfter: + description: |- + DebugSimulateDataplaneHangAfter is used to simulate a hang in the dataplane after the specified duration. + This is useful in tests of the watchdog system only! + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + defaultEndpointToHostAction: + description: |- + DefaultEndpointToHostAction controls what happens to traffic that goes from a workload endpoint to the host + itself (after the endpoint's egress policy is applied). By default, Calico blocks traffic from workload + endpoints to the host itself with an iptables "DROP" action. If you want to allow some or all traffic from + endpoint to host, set this parameter to RETURN or ACCEPT. Use RETURN if you have your own rules in the iptables + "INPUT" chain; Calico will insert its rules at the top of that chain, then "RETURN" packets to the "INPUT" chain + once it has completed processing workload endpoint egress policy. Use ACCEPT to unconditionally accept packets + from workloads after processing workload endpoint egress policy. [Default: Drop] + pattern: ^(?i)(Drop|Accept|Return)?$ + type: string + deviceRouteProtocol: + description: |- + DeviceRouteProtocol controls the protocol to set on routes programmed by Felix. The protocol is an 8-bit label + used to identify the owner of the route. + type: integer + deviceRouteSourceAddress: + description: |- + DeviceRouteSourceAddress IPv4 address to set as the source hint for routes programmed by Felix. When not set + the source address for local traffic from host to workload will be determined by the kernel. + type: string + deviceRouteSourceAddressIPv6: + description: |- + DeviceRouteSourceAddressIPv6 IPv6 address to set as the source hint for routes programmed by Felix. When not set + the source address for local traffic from host to workload will be determined by the kernel. + type: string + disableConntrackInvalidCheck: + description: |- + DisableConntrackInvalidCheck disables the check for invalid connections in conntrack. While the conntrack + invalid check helps to detect malicious traffic, it can also cause issues with certain multi-NIC scenarios. + type: boolean + endpointReportingDelay: + description: |- + EndpointReportingDelay is the delay before Felix reports endpoint status to the datastore. This is only used + by the OpenStack integration. [Default: 1s] + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + endpointReportingEnabled: + description: |- + EndpointReportingEnabled controls whether Felix reports endpoint status to the datastore. This is only used + by the OpenStack integration. [Default: false] + type: boolean + endpointStatusPathPrefix: + description: |- + EndpointStatusPathPrefix is the path to the directory where endpoint status will be written. Endpoint status + file reporting is disabled if field is left empty. + + Chosen directory should match the directory used by the CNI plugin for PodStartupDelay. + [Default: /var/run/calico] + type: string + externalNodesList: + description: |- + ExternalNodesCIDRList is a list of CIDR's of external, non-Calico nodes from which VXLAN/IPIP overlay traffic + will be allowed. By default, external tunneled traffic is blocked to reduce attack surface. + items: + type: string + type: array + failsafeInboundHostPorts: + description: |- + FailsafeInboundHostPorts is a list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will + allow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally + cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, + it defaults to "tcp". If a CIDR is not specified, it will allow traffic from all addresses. To disable all inbound host ports, + use the value "[]". The default value allows ssh access, DHCP, BGP, etcd and the Kubernetes API. + [Default: tcp:22, udp:68, tcp:179, tcp:2379, tcp:2380, tcp:5473, tcp:6443, tcp:6666, tcp:6667 ] + items: + description: + ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + type: object + type: array + failsafeOutboundHostPorts: + description: |- + FailsafeOutboundHostPorts is a list of PortProto struct objects including UDP/TCP/SCTP ports and CIDRs that Felix + will allow outgoing traffic from host endpoints to irrespective of the security policy. This is useful to avoid accidentally + cutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from all addresses. To disable all outbound host ports, + use the value "[]". The default value opens etcd's standard ports to ensure that Felix does not get cut off from etcd + as well as allowing DHCP, DNS, BGP and the Kubernetes API. + [Default: udp:53, udp:67, tcp:179, tcp:2379, tcp:2380, tcp:5473, tcp:6443, tcp:6666, tcp:6667 ] + items: + description: + ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + type: object + type: array + featureDetectOverride: + description: |- + FeatureDetectOverride is used to override feature detection based on auto-detected platform + capabilities. Values are specified in a comma separated list with no spaces, example; + "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". A value of "true" or "false" will + force enable/disable feature, empty or omitted values fall back to auto-detection. + pattern: ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$ + type: string + featureGates: + description: |- + FeatureGates is used to enable or disable tech-preview Calico features. + Values are specified in a comma separated list with no spaces, example; + "BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false". This is + used to enable features that are not fully production ready. + pattern: ^([a-zA-Z0-9-_]+=([^=]+),)*([a-zA-Z0-9-_]+=([^=]+))?$ + type: string + floatingIPs: + description: |- + FloatingIPs configures whether or not Felix will program non-OpenStack floating IP addresses. (OpenStack-derived + floating IPs are always programmed, regardless of this setting.) + enum: + - Enabled + - Disabled + type: string + flowLogsCollectorDebugTrace: + description: |- + When FlowLogsCollectorDebugTrace is set to true, enables the logs in the collector to be + printed in their entirety. + type: boolean + flowLogsFlushInterval: + description: + FlowLogsFlushInterval configures the interval at which + Felix exports flow logs. + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + flowLogsGoldmaneServer: + description: + FlowLogGoldmaneServer is the flow server endpoint to + which flow data should be published. + type: string + flowLogsLocalReporter: + description: + "FlowLogsLocalReporter configures local unix socket for + reporting flow data from each node. [Default: Disabled]" + enum: + - Disabled + - Enabled + type: string + flowLogsPolicyEvaluationMode: + description: |- + Continuous - Felix evaluates active flows on a regular basis to determine the rule + traces in the flow logs. Any policy updates that impact a flow will be reflected in the + pending_policies field, offering a near-real-time view of policy changes across flows. + None - Felix stops evaluating pending traces. + [Default: Continuous] + enum: + - None + - Continuous + type: string + genericXDPEnabled: + description: |- + GenericXDPEnabled enables Generic XDP so network cards that don't support XDP offload or driver + modes can use XDP. This is not recommended since it doesn't provide better performance than + iptables. [Default: false] + type: boolean + goGCThreshold: + description: |- + GoGCThreshold Sets the Go runtime's garbage collection threshold. I.e. the percentage that the heap is + allowed to grow before garbage collection is triggered. In general, doubling the value halves the CPU time + spent doing GC, but it also doubles peak GC memory overhead. A special value of -1 can be used + to disable GC entirely; this should only be used in conjunction with the GoMemoryLimitMB setting. + + This setting is overridden by the GOGC environment variable. + + [Default: 40] + type: integer + goMaxProcs: + description: |- + GoMaxProcs sets the maximum number of CPUs that the Go runtime will use concurrently. A value of -1 means + "use the system default"; typically the number of real CPUs on the system. + + this setting is overridden by the GOMAXPROCS environment variable. + + [Default: -1] + type: integer + goMemoryLimitMB: + description: |- + GoMemoryLimitMB sets a (soft) memory limit for the Go runtime in MB. The Go runtime will try to keep its memory + usage under the limit by triggering GC as needed. To avoid thrashing, it will exceed the limit if GC starts to + take more than 50% of the process's CPU time. A value of -1 disables the memory limit. + + Note that the memory limit, if used, must be considerably less than any hard resource limit set at the container + or pod level. This is because felix is not the only process that must run in the container or pod. + + This setting is overridden by the GOMEMLIMIT environment variable. + + [Default: -1] + type: integer + healthEnabled: + description: |- + HealthEnabled if set to true, enables Felix's health port, which provides readiness and liveness endpoints. + [Default: false] + type: boolean + healthHost: + description: + "HealthHost is the host that the health server should + bind to. [Default: localhost]" + type: string + healthPort: + description: + "HealthPort is the TCP port that the health server should + bind to. [Default: 9099]" + type: integer + healthTimeoutOverrides: + description: |- + HealthTimeoutOverrides allows the internal watchdog timeouts of individual subcomponents to be + overridden. This is useful for working around "false positive" liveness timeouts that can occur + in particularly stressful workloads or if CPU is constrained. For a list of active + subcomponents, see Felix's logs. + items: + properties: + name: + type: string + timeout: + type: string + required: + - name + - timeout + type: object + type: array + interfaceExclude: + description: |- + InterfaceExclude A comma-separated list of interface names that should be excluded when Felix is resolving + host endpoints. The default value ensures that Felix ignores Kubernetes' internal `kube-ipvs0` device. If you + want to exclude multiple interface names using a single value, the list supports regular expressions. For + regular expressions you must wrap the value with `/`. For example having values `/^kube/,veth1` will exclude + all interfaces that begin with `kube` and also the interface `veth1`. [Default: kube-ipvs0] + type: string + interfacePrefix: + description: |- + InterfacePrefix is the interface name prefix that identifies workload endpoints and so distinguishes + them from host endpoint interfaces. Note: in environments other than bare metal, the orchestrators + configure this appropriately. For example our Kubernetes and Docker integrations set the 'cali' value, + and our OpenStack integration sets the 'tap' value. [Default: cali] + type: string + interfaceRefreshInterval: + description: |- + InterfaceRefreshInterval is the period at which Felix rescans local interfaces to verify their state. + The rescan can be disabled by setting the interval to 0. + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + ipForwarding: + description: |- + IPForwarding controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required + when using Calico for workload networking. This should be disabled only on hosts where Calico is used solely for + host protection. In BPF mode, due to a kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF + must be disabled. [Default: Enabled] + enum: + - Enabled + - Disabled + type: string + ipipEnabled: + description: |- + IPIPEnabled overrides whether Felix should configure an IPIP interface on the host. Optional as Felix + determines this based on the existing IP pools. [Default: nil (unset)] + type: boolean + ipipMTU: + description: |- + IPIPMTU controls the MTU to set on the IPIP tunnel device. Optional as Felix auto-detects the MTU based on the + MTU of the host's interfaces. [Default: 0 (auto-detect)] + type: integer + ipsetsRefreshInterval: + description: |- + IpsetsRefreshInterval controls the period at which Felix re-checks all IP sets to look for discrepancies. + Set to 0 to disable the periodic refresh. [Default: 90s] + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesBackend: + description: |- + IptablesBackend controls which backend of iptables will be used. The default is `Auto`. + + Warning: changing this on a running system can leave "orphaned" rules in the "other" backend. These + should be cleaned up to avoid confusing interactions. + pattern: ^(?i)(Auto|Legacy|NFT)?$ + type: string + iptablesFilterAllowAction: + description: |- + IptablesFilterAllowAction controls what happens to traffic that is accepted by a Felix policy chain in the + iptables filter table (which is used for "normal" policy). The default will immediately `Accept` the traffic. Use + `Return` to send the traffic back up to the system chains for further processing. + pattern: ^(?i)(Accept|Return)?$ + type: string + iptablesFilterDenyAction: + description: |- + IptablesFilterDenyAction controls what happens to traffic that is denied by network policy. By default Calico blocks traffic + with an iptables "DROP" action. If you want to use "REJECT" action instead you can configure it in here. + pattern: ^(?i)(Drop|Reject)?$ + type: string + iptablesLockFilePath: + description: |- + IptablesLockFilePath is the location of the iptables lock file. You may need to change this + if the lock file is not in its standard location (for example if you have mapped it into Felix's + container at a different path). [Default: /run/xtables.lock] + type: string + iptablesLockProbeInterval: + description: |- + IptablesLockProbeInterval when IptablesLockTimeout is enabled: the time that Felix will wait between + attempts to acquire the iptables lock if it is not available. Lower values make Felix more + responsive when the lock is contended, but use more CPU. [Default: 50ms] + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesLockTimeout: + description: |- + IptablesLockTimeout is the time that Felix itself will wait for the iptables lock (rather than delegating the + lock handling to the `iptables` command). + + Deprecated: `iptables-restore` v1.8+ always takes the lock, so enabling this feature results in deadlock. + [Default: 0s disabled] + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesMangleAllowAction: + description: |- + IptablesMangleAllowAction controls what happens to traffic that is accepted by a Felix policy chain in the + iptables mangle table (which is used for "pre-DNAT" policy). The default will immediately `Accept` the traffic. + Use `Return` to send the traffic back up to the system chains for further processing. + pattern: ^(?i)(Accept|Return)?$ + type: string + iptablesMarkMask: + description: |- + IptablesMarkMask is the mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal + number with at least 8 bits set, none of which clash with any other mark bits in use on the system. + [Default: 0xffff0000] + format: int32 + type: integer + iptablesNATOutgoingInterfaceFilter: + description: |- + This parameter can be used to limit the host interfaces on which Calico will apply SNAT to traffic leaving a + Calico IPAM pool with "NAT outgoing" enabled. This can be useful if you have a main data interface, where + traffic should be SNATted and a secondary device (such as the docker bridge) which is local to the host and + doesn't require SNAT. This parameter uses the iptables interface matching syntax, which allows + as a + wildcard. Most users will not need to set this. Example: if your data interfaces are eth0 and eth1 and you + want to exclude the docker bridge, you could set this to eth+ + type: string + iptablesPostWriteCheckInterval: + description: |- + IptablesPostWriteCheckInterval is the period after Felix has done a write + to the dataplane that it schedules an extra read back in order to check the write was not + clobbered by another process. This should only occur if another application on the system + doesn't respect the iptables lock. [Default: 1s] + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + iptablesRefreshInterval: + description: |- + IptablesRefreshInterval is the period at which Felix re-checks the IP sets + in the dataplane to ensure that no other process has accidentally broken Calico's rules. + Set to 0 to disable IP sets refresh. Note: the default for this value is lower than the + other refresh intervals as a workaround for a Linux kernel bug that was fixed in kernel + version 4.11. If you are using v4.11 or greater you may want to set this to, a higher value + to reduce Felix CPU usage. [Default: 10s] + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + ipv6Support: + description: + IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). + type: boolean + kubeNodePortRanges: + description: |- + KubeNodePortRanges holds list of port ranges used for service node ports. Only used if felix detects kube-proxy running in ipvs mode. + Felix uses these ranges to separate host and workload traffic. [Default: 30000:32767]. + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + logDebugFilenameRegex: + description: |- + LogDebugFilenameRegex controls which source code files have their Debug log output included in the logs. + Only logs from files with names that match the given regular expression are included. The filter only applies + to Debug level logs. + type: string + logFilePath: + description: + "LogFilePath is the full path to the Felix log. Set to + none to disable file logging. [Default: /var/log/calico/felix.log]" + type: string + logPrefix: + description: + "LogPrefix is the log prefix that Felix uses when rendering + LOG rules. [Default: calico-packet]" + type: string + logSeverityFile: + description: + "LogSeverityFile is the log severity above which logs + are sent to the log file. [Default: Info]" + pattern: ^(?i)(Trace|Debug|Info|Warning|Error|Fatal)?$ + type: string + logSeverityScreen: + description: + "LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]" + pattern: ^(?i)(Trace|Debug|Info|Warning|Error|Fatal)?$ + type: string + logSeveritySys: + description: |- + LogSeveritySys is the log severity above which logs are sent to the syslog. Set to None for no logging to syslog. + [Default: Info] + pattern: ^(?i)(Trace|Debug|Info|Warning|Error|Fatal)?$ + type: string + maxIpsetSize: + description: |- + MaxIpsetSize is the maximum number of IP addresses that can be stored in an IP set. Not applicable + if using the nftables backend. + type: integer + metadataAddr: + description: |- + MetadataAddr is the IP address or domain name of the server that can answer VM queries for + cloud-init metadata. In OpenStack, this corresponds to the machine running nova-api (or in + Ubuntu, nova-api-metadata). A value of none (case-insensitive) means that Felix should not + set up any NAT rule for the metadata path. [Default: 127.0.0.1] + type: string + metadataPort: + description: |- + MetadataPort is the port of the metadata server. This, combined with global.MetadataAddr (if + not 'None'), is used to set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. + In most cases this should not need to be changed [Default: 8775]. + type: integer + mtuIfacePattern: + description: |- + MTUIfacePattern is a regular expression that controls which interfaces Felix should scan in order + to calculate the host's MTU. + This should not match workload interfaces (usually named cali...). + type: string + natOutgoingAddress: + description: |- + NATOutgoingAddress specifies an address to use when performing source NAT for traffic in a natOutgoing pool that + is leaving the network. By default the address used is an address on the interface the traffic is leaving on + (i.e. it uses the iptables MASQUERADE target). + type: string + natPortRange: + anyOf: + - type: integer + - type: string + description: |- + NATPortRange specifies the range of ports that is used for port mapping when doing outgoing NAT. When unset the default behavior of the + network stack is used. + pattern: ^.* + x-kubernetes-int-or-string: true + netlinkTimeout: + description: |- + NetlinkTimeout is the timeout when talking to the kernel over the netlink protocol, used for programming + routes, rules, and other kernel objects. [Default: 10s] + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + nftablesFilterAllowAction: + description: |- + NftablesFilterAllowAction controls the nftables action that Felix uses to represent the "allow" policy verdict + in the filter table. The default is to `ACCEPT` the traffic, which is a terminal action. Alternatively, + `RETURN` can be used to return the traffic back to the top-level chain for further processing by your rules. + pattern: ^(?i)(Accept|Return)?$ + type: string + nftablesFilterDenyAction: + description: |- + NftablesFilterDenyAction controls what happens to traffic that is denied by network policy. By default, Calico + blocks traffic with a "drop" action. If you want to use a "reject" action instead you can configure it here. + pattern: ^(?i)(Drop|Reject)?$ + type: string + nftablesMangleAllowAction: + description: |- + NftablesMangleAllowAction controls the nftables action that Felix uses to represent the "allow" policy verdict + in the mangle table. The default is to `ACCEPT` the traffic, which is a terminal action. Alternatively, + `RETURN` can be used to return the traffic back to the top-level chain for further processing by your rules. + pattern: ^(?i)(Accept|Return)?$ + type: string + nftablesMarkMask: + description: |- + NftablesMarkMask is the mask that Felix selects its nftables Mark bits from. Should be a 32 bit hexadecimal + number with at least 8 bits set, none of which clash with any other mark bits in use on the system. + [Default: 0xffff0000] + format: int32 + type: integer + nftablesMode: + description: + "NFTablesMode configures nftables support in Felix. [Default: + Disabled]" + enum: + - Disabled + - Enabled + - Auto + type: string + nftablesRefreshInterval: + description: + "NftablesRefreshInterval controls the interval at which + Felix periodically refreshes the nftables rules. [Default: 90s]" + type: string + openstackRegion: + description: |- + OpenstackRegion is the name of the region that a particular Felix belongs to. In a multi-region + Calico/OpenStack deployment, this must be configured somehow for each Felix (here in the datamodel, + or in felix.cfg or the environment on each compute node), and must match the [calico] + openstack_region value configured in neutron.conf on each node. [Default: Empty] + type: string + policySyncPathPrefix: + description: |- + PolicySyncPathPrefix is used to by Felix to communicate policy changes to external services, + like Application layer policy. [Default: Empty] + type: string + prometheusGoMetricsEnabled: + description: |- + PrometheusGoMetricsEnabled disables Go runtime metrics collection, which the Prometheus client does by default, when + set to false. This reduces the number of metrics reported, reducing Prometheus load. [Default: true] + type: boolean + prometheusMetricsEnabled: + description: + "PrometheusMetricsEnabled enables the Prometheus metrics + server in Felix if set to true. [Default: false]" + type: boolean + prometheusMetricsHost: + description: + "PrometheusMetricsHost is the host that the Prometheus + metrics server should bind to. [Default: empty]" + type: string + prometheusMetricsPort: + description: + "PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. [Default: 9091]" + type: integer + prometheusProcessMetricsEnabled: + description: |- + PrometheusProcessMetricsEnabled disables process metrics collection, which the Prometheus client does by default, when + set to false. This reduces the number of metrics reported, reducing Prometheus load. [Default: true] + type: boolean + prometheusWireGuardMetricsEnabled: + description: |- + PrometheusWireGuardMetricsEnabled disables wireguard metrics collection, which the Prometheus client does by default, when + set to false. This reduces the number of metrics reported, reducing Prometheus load. [Default: true] + type: boolean + removeExternalRoutes: + description: |- + RemoveExternalRoutes Controls whether Felix will remove unexpected routes to workload interfaces. Felix will + always clean up expected routes that use the configured DeviceRouteProtocol. To add your own routes, you must + use a distinct protocol (in addition to setting this field to false). + type: boolean + reportingInterval: + description: |- + ReportingInterval is the interval at which Felix reports its status into the datastore or 0 to disable. + Must be non-zero in OpenStack deployments. [Default: 30s] + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + reportingTTL: + description: + "ReportingTTL is the time-to-live setting for process-wide + status reports. [Default: 90s]" + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + routeRefreshInterval: + description: |- + RouteRefreshInterval is the period at which Felix re-checks the routes + in the dataplane to ensure that no other process has accidentally broken Calico's rules. + Set to 0 to disable route refresh. [Default: 90s] + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + routeSource: + description: |- + RouteSource configures where Felix gets its routing information. + - WorkloadIPs: use workload endpoints to construct routes. + - CalicoIPAM: the default - use IPAM data to construct routes. + pattern: ^(?i)(WorkloadIPs|CalicoIPAM)?$ + type: string + routeSyncDisabled: + description: |- + RouteSyncDisabled will disable all operations performed on the route table. Set to true to + run in network-policy mode only. + type: boolean + routeTableRange: + description: |- + Deprecated in favor of RouteTableRanges. + Calico programs additional Linux route tables for various purposes. + RouteTableRange specifies the indices of the route tables that Calico should use. + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + routeTableRanges: + description: |- + Calico programs additional Linux route tables for various purposes. + RouteTableRanges specifies a set of table index ranges that Calico should use. + Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array + serviceLoopPrevention: + description: |- + When service IP advertisement is enabled, prevent routing loops to service IPs that are + not in use, by dropping or rejecting packets that do not get DNAT'd by kube-proxy. + Unless set to "Disabled", in which case such routing loops continue to be allowed. + [Default: Drop] + pattern: ^(?i)(Drop|Reject|Disabled)?$ + type: string + sidecarAccelerationEnabled: + description: + "SidecarAccelerationEnabled enables experimental sidecar + acceleration [Default: false]" + type: boolean + usageReportingEnabled: + description: |- + UsageReportingEnabled reports anonymous Calico version number and cluster size to projectcalico.org. Logs warnings returned by the usage + server. For example, if a significant security vulnerability has been discovered in the version of Calico being used. [Default: true] + type: boolean + usageReportingInitialDelay: + description: + "UsageReportingInitialDelay controls the minimum delay + before Felix makes a report. [Default: 300s]" + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + usageReportingInterval: + description: + "UsageReportingInterval controls the interval at which + Felix makes reports. [Default: 86400s]" + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + useInternalDataplaneDriver: + description: |- + UseInternalDataplaneDriver, if true, Felix will use its internal dataplane programming logic. If false, it + will launch an external dataplane driver and communicate with it over protobuf. + type: boolean + vxlanEnabled: + description: |- + VXLANEnabled overrides whether Felix should create the VXLAN tunnel device for IPv4 VXLAN networking. + Optional as Felix determines this based on the existing IP pools. [Default: nil (unset)] + type: boolean + vxlanMTU: + description: |- + VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the + MTU of the host's interfaces. [Default: 0 (auto-detect)] + type: integer + vxlanMTUV6: + description: |- + VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the + MTU of the host's interfaces. [Default: 0 (auto-detect)] + type: integer + vxlanPort: + description: + "VXLANPort is the UDP port number to use for VXLAN traffic. + [Default: 4789]" + type: integer + vxlanVNI: + description: |- + VXLANVNI is the VXLAN VNI to use for VXLAN traffic. You may need to change this if the default value is + in use on your system. [Default: 4096] + type: integer + windowsManageFirewallRules: + description: + "WindowsManageFirewallRules configures whether or not + Felix will program Windows Firewall rules (to allow inbound access + to its own metrics ports). [Default: Disabled]" + enum: + - Enabled + - Disabled + type: string + wireguardEnabled: + description: + "WireguardEnabled controls whether Wireguard is enabled + for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network). + [Default: false]" + type: boolean + wireguardEnabledV6: + description: + "WireguardEnabledV6 controls whether Wireguard is enabled + for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network). + [Default: false]" + type: boolean + wireguardHostEncryptionEnabled: + description: + "WireguardHostEncryptionEnabled controls whether Wireguard + host-to-host encryption is enabled. [Default: false]" + type: boolean + wireguardInterfaceName: + description: + "WireguardInterfaceName specifies the name to use for + the IPv4 Wireguard interface. [Default: wireguard.cali]" + type: string + wireguardInterfaceNameV6: + description: + "WireguardInterfaceNameV6 specifies the name to use for + the IPv6 Wireguard interface. [Default: wg-v6.cali]" + type: string + wireguardKeepAlive: + description: + "WireguardPersistentKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]" + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + wireguardListeningPort: + description: + "WireguardListeningPort controls the listening port used + by IPv4 Wireguard. [Default: 51820]" + type: integer + wireguardListeningPortV6: + description: + "WireguardListeningPortV6 controls the listening port + used by IPv6 Wireguard. [Default: 51821]" + type: integer + wireguardMTU: + description: + "WireguardMTU controls the MTU on the IPv4 Wireguard + interface. See Configuring MTU [Default: 1440]" + type: integer + wireguardMTUV6: + description: + "WireguardMTUV6 controls the MTU on the IPv6 Wireguard + interface. See Configuring MTU [Default: 1420]" + type: integer + wireguardRoutingRulePriority: + description: + "WireguardRoutingRulePriority controls the priority value + to use for the Wireguard routing rule. [Default: 99]" + type: integer + wireguardThreadingEnabled: + description: |- + WireguardThreadingEnabled controls whether Wireguard has Threaded NAPI enabled. [Default: false] + This increases the maximum number of packets a Wireguard interface can process. + Consider threaded NAPI only if you have high packets per second workloads that are causing dropping packets due to a saturated `softirq` CPU core. + There is a [known issue](https://lore.kernel.org/netdev/CALrw=nEoT2emQ0OAYCjM1d_6Xe_kNLSZ6dhjb5FxrLFYh4kozA@mail.gmail.com/T/) with this setting + that may cause NAPI to get stuck holding the global `rtnl_mutex` when a peer is removed. + Workaround: Make sure your Linux kernel [includes this patch](https://github.com/torvalds/linux/commit/56364c910691f6d10ba88c964c9041b9ab777bd6) to unwedge NAPI. + type: boolean + workloadSourceSpoofing: + description: |- + WorkloadSourceSpoofing controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source IP + address that is not theirs. This is disabled by default. When set to "Any", pods can request any prefix. + pattern: ^(?i)(Disabled|Any)?$ + type: string + xdpEnabled: + description: + "XDPEnabled enables XDP acceleration for suitable untracked + incoming deny rules. [Default: true]" + type: boolean + xdpRefreshInterval: + description: |- + XDPRefreshInterval is the period at which Felix re-checks all XDP state to ensure that no + other process has accidentally broken Calico's BPF maps or attached programs. Set to 0 to + disable XDP refresh. [Default: 90s] + pattern: ^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$ + type: string + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: globalnetworkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkPolicy + listKind: GlobalNetworkPolicyList + plural: globalnetworkpolicies + singular: globalnetworkpolicy + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + applyOnForward: + description: + ApplyOnForward indicates to apply the rules in this policy + on forward traffic. + type: boolean + doNotTrack: + description: |- + DoNotTrack indicates whether packets matched by the rules in this policy should go through + the data plane's connection tracking, such as Linux conntrack. If True, the rules in + this policy are applied before any data plane connection tracking, and packets allowed by + this policy are marked as not to be tracked. + type: boolean + egress: + description: |- + The ordered set of egress rules. Each rule contains a set of packet match criteria and + a corresponding action to apply. + items: + description: |- + A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy + and security Profiles reference rules - separated out as a list of rules for both + ingress and egress packet matching. + + Each positive match criteria has a negated version, prefixed with "Not". All the match + criteria within a rule must be satisfied for a packet to match. A single rule can contain + the positive and negative version of a match and both must be satisfied for the rule to match. + properties: + action: + type: string + destination: + description: + Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + http: + description: + HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: |- + Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed + HTTP Methods (e.g. GET, PUT, etc.) + Multiple methods are OR'd together. + items: + type: string + type: array + paths: + description: |- + Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed + HTTP Paths. + Multiple paths are OR'd together. + e.g: + - exact: /foo + - prefix: /bar + NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it. + items: + description: |- + HTTPPath specifies an HTTP path to match. It may be either of the form: + exact: : which matches the path exactly or + prefix: : which matches the path prefix + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: |- + ICMP is an optional field that restricts the rule to apply to a specific type and + code of ICMP traffic. This should only be specified if the Protocol field is set to + "ICMP" or "ICMPv6". + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + ipVersion: + description: |- + IPVersion is an optional field that restricts the rule to only match a specific IP + version. + type: integer + metadata: + description: + Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: + Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: + NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: |- + Protocol is an optional field that restricts the rule to only apply to traffic of + a specific IP protocol. Required if any of the EntityRules contain Ports + (because ports only apply to certain protocols). + + Must be one of these string values: "TCP", "UDP", "ICMP", "ICMPv6", "SCTP", "UDPLite" + or an integer in the range 1-255. + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: + Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: |- + The ordered set of ingress rules. Each rule contains a set of packet match criteria and + a corresponding action to apply. + items: + description: |- + A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy + and security Profiles reference rules - separated out as a list of rules for both + ingress and egress packet matching. + + Each positive match criteria has a negated version, prefixed with "Not". All the match + criteria within a rule must be satisfied for a packet to match. A single rule can contain + the positive and negative version of a match and both must be satisfied for the rule to match. + properties: + action: + type: string + destination: + description: + Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + http: + description: + HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: |- + Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed + HTTP Methods (e.g. GET, PUT, etc.) + Multiple methods are OR'd together. + items: + type: string + type: array + paths: + description: |- + Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed + HTTP Paths. + Multiple paths are OR'd together. + e.g: + - exact: /foo + - prefix: /bar + NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it. + items: + description: |- + HTTPPath specifies an HTTP path to match. It may be either of the form: + exact: : which matches the path exactly or + prefix: : which matches the path prefix + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: |- + ICMP is an optional field that restricts the rule to apply to a specific type and + code of ICMP traffic. This should only be specified if the Protocol field is set to + "ICMP" or "ICMPv6". + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + ipVersion: + description: |- + IPVersion is an optional field that restricts the rule to only match a specific IP + version. + type: integer + metadata: + description: + Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: + Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: + NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: |- + Protocol is an optional field that restricts the rule to only apply to traffic of + a specific IP protocol. Required if any of the EntityRules contain Ports + (because ports only apply to certain protocols). + + Must be one of these string values: "TCP", "UDP", "ICMP", "ICMPv6", "SCTP", "UDPLite" + or an integer in the range 1-255. + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: + Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + namespaceSelector: + description: + NamespaceSelector is an optional field for an expression + used to select a pod based on namespaces. + type: string + order: + description: |- + Order is an optional field that specifies the order in which the policy is applied. + Policies with higher "order" are applied after those with lower + order within the same tier. If the order is omitted, it may be considered to be "infinite" - i.e. the + policy will be applied last. Policies with identical order will be applied in + alphanumerical order based on the Policy "Name" within the tier. + type: number + performanceHints: + description: |- + PerformanceHints contains a list of hints to Calico's policy engine to + help process the policy more efficiently. Hints never change the + enforcement behaviour of the policy. + + Currently, the only available hint is "AssumeNeededOnEveryNode". When + that hint is set on a policy, Felix will act as if the policy matches + a local endpoint even if it does not. This is useful for "preloading" + any large static policies that are known to be used on every node. + If the policy is _not_ used on a particular node then the work + done to preload the policy (and to maintain it) is wasted. + items: + type: string + type: array + preDNAT: + description: + PreDNAT indicates to apply the rules in this policy before + any DNAT. + type: boolean + selector: + description: + "The selector is an expression used to pick out the endpoints + that the policy should\nbe applied to.\n\nSelector expressions follow + this syntax:\n\n\tlabel == \"string_literal\" -> comparison, e.g. + my_label == \"foo bar\"\n\tlabel != \"string_literal\" -> not + equal; also matches if label is not present\n\tlabel in { \"a\", + \"b\", \"c\", ... } -> true if the value of label X is one of + \"a\", \"b\", \"c\"\n\tlabel not in { \"a\", \"b\", \"c\", ... } + \ -> true if the value of label X is not one of \"a\", \"b\", \"c\"\n\thas(label_name) + \ -> True if that label is present\n\t! expr -> negation of expr\n\texpr + && expr -> Short-circuit and\n\texpr || expr -> Short-circuit + or\n\t( expr ) -> parens for grouping\n\tall() or the empty selector + -> matches all endpoints.\n\nLabel names are allowed to contain + alphanumerics, -, _ and /. String literals are more permissive\nbut + they do not support escape characters.\n\nExamples (with made-up + labels):\n\n\ttype == \"webserver\" && deployment == \"prod\"\n\ttype + in {\"frontend\", \"backend\"}\n\tdeployment != \"dev\"\n\t! has(label_name)" + type: string + serviceAccountSelector: + description: + ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + tier: + description: |- + The name of the tier that this policy belongs to. If this is omitted, the default + tier (name is "default") is assumed. The specified tier must exist in order to create + security policies within the tier, the "default" tier is created automatically if it + does not exist, this means for deployments requiring only a single Tier, the tier name + may be omitted on all policy management requests. + type: string + types: + description: |- + Types indicates whether this policy applies to ingress, or to egress, or to both. When + not explicitly specified (and so the value on creation is empty or nil), Calico defaults + Types according to what Ingress and Egress rules are present in the policy. The + default is: + + - [ PolicyTypeIngress ], if there are no Egress rules (including the case where there are + also no Ingress rules) + + - [ PolicyTypeEgress ], if there are Egress rules but no Ingress rules + + - [ PolicyTypeIngress, PolicyTypeEgress ], if there are both Ingress and Egress rules. + + When the policy is read back again, Types will always be one of these values, never empty + or nil. + items: + description: + PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: globalnetworksets.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkSet + listKind: GlobalNetworkSetList + plural: globalnetworksets + singular: globalnetworkset + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: |- + GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs that share labels to + allow rules to refer to them via selectors. The labels of GlobalNetworkSet are not namespaced. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: + GlobalNetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: hostendpoints.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: HostEndpoint + listKind: HostEndpointList + plural: hostendpoints + singular: hostendpoint + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: + HostEndpointSpec contains the specification for a HostEndpoint + resource. + properties: + expectedIPs: + description: + "The expected IP addresses (IPv4 and IPv6) of the endpoint.\nIf + \"InterfaceName\" is not present, Calico will look for an interface + matching any\nof the IPs in the list and apply policy to that.\nNote:\n\tWhen + using the selector match criteria in an ingress or egress security + Policy\n\tor Profile, Calico converts the selector into a set of + IP addresses. For host\n\tendpoints, the ExpectedIPs field is used + for that purpose. (If only the interface\n\tname is specified, Calico + does not learn the IPs of the interface for use in match\n\tcriteria.)" + items: + type: string + type: array + interfaceName: + description: |- + Either "*", or the name of a specific Linux interface to apply policy to; or empty. "*" + indicates that this HostEndpoint governs all traffic to, from or through the default + network namespace of the host named by the "Node" field; entering and leaving that + namespace via any interface, including those from/to non-host-networked local workloads. + + If InterfaceName is not "*", this HostEndpoint only governs traffic that enters or leaves + the host through the specific interface named by InterfaceName, or - when InterfaceName + is empty - through the specific interface that has one of the IPs in ExpectedIPs. + Therefore, when InterfaceName is empty, at least one expected IP must be specified. Only + external interfaces (such as "eth0") are supported here; it isn't possible for a + HostEndpoint to protect traffic through a specific local workload interface. + + Note: Only some kinds of policy are implemented for "*" HostEndpoints; initially just + pre-DNAT policy. Please check Calico documentation for the latest position. + type: string + node: + description: The node name identifying the Calico node instance. + type: string + ports: + description: + Ports contains the endpoint's named ports, which may + be referenced in security policy rules. + items: + properties: + name: + type: string + port: + type: integer + protocol: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + required: + - name + - port + - protocol + type: object + type: array + profiles: + description: |- + A list of identifiers of security Profile objects that apply to this endpoint. Each + profile is applied in the order that they appear in this list. Profile rules are applied + after the selector-based security policy. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: ipamblocks.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMBlock + listKind: IPAMBlockList + plural: ipamblocks + singular: ipamblock + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: + IPAMBlockSpec contains the specification for an IPAMBlock + resource. + properties: + affinity: + description: |- + Affinity of the block, if this block has one. If set, it will be of the form + "host:". If not set, this block is not affine to a host. + type: string + allocations: + description: |- + Array of allocations in-use within this block. nil entries mean the allocation is free. + For non-nil entries at index i, the index is the ordinal of the allocation within this block + and the value is the index of the associated attributes in the Attributes array. + items: + type: integer + # TODO: This nullable is manually added in. We should update controller-gen + # to handle []*int properly itself. + nullable: true + type: array + attributes: + description: |- + Attributes is an array of arbitrary metadata associated with allocations in the block. To find + attributes for a given allocation, use the value of the allocation's entry in the Allocations array + as the index of the element in this array. + items: + properties: + handle_id: + type: string + secondary: + additionalProperties: + type: string + type: object + type: object + type: array + cidr: + description: The block's CIDR. + type: string + deleted: + description: |- + Deleted is an internal boolean used to workaround a limitation in the Kubernetes API whereby + deletion will not return a conflict error if the block has been updated. It should not be set manually. + type: boolean + sequenceNumber: + default: 0 + description: |- + We store a sequence number that is updated each time the block is written. + Each allocation will also store the sequence number of the block at the time of its creation. + When releasing an IP, passing the sequence number associated with the allocation allows us + to protect against a race condition and ensure the IP hasn't been released and re-allocated + since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: |- + Map of allocated ordinal within the block to sequence number of the block at + the time of allocation. Kubernetes does not allow numerical keys for maps, so + the key is cast to a string. + type: object + strictAffinity: + description: + StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. + type: boolean + unallocated: + description: + Unallocated is an ordered list of allocations which are + free in the block. + items: + type: integer + type: array + required: + - allocations + - attributes + - cidr + - strictAffinity + - unallocated + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: ipamconfigs.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMConfig + listKind: IPAMConfigList + plural: ipamconfigs + singular: ipamconfig + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: + IPAMConfigSpec contains the specification for an IPAMConfig + resource. + properties: + autoAllocateBlocks: + type: boolean + maxBlocksPerHost: + description: |- + MaxBlocksPerHost, if non-zero, is the max number of blocks that can be + affine to each host. + maximum: 2147483647 + minimum: 0 + type: integer + strictAffinity: + type: boolean + required: + - autoAllocateBlocks + - strictAffinity + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: ipamhandles.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMHandle + listKind: IPAMHandleList + plural: ipamhandles + singular: ipamhandle + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: + IPAMHandleSpec contains the specification for an IPAMHandle + resource. + properties: + block: + additionalProperties: + type: integer + type: object + deleted: + type: boolean + handleID: + type: string + required: + - block + - handleID + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: ippools.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPPool + listKind: IPPoolList + plural: ippools + singular: ippool + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: IPPoolSpec contains the specification for an IPPool resource. + properties: + allowedUses: + description: |- + AllowedUse controls what the IP pool will be used for. If not specified or empty, defaults to + ["Tunnel", "Workload"] for back-compatibility + items: + type: string + type: array + assignmentMode: + description: + Determines the mode how IP addresses should be assigned + from this pool + enum: + - Automatic + - Manual + type: string + blockSize: + description: + The block size to use for IP address assignments from + this pool. Defaults to 26 for IPv4 and 122 for IPv6. + type: integer + cidr: + description: The pool CIDR. + type: string + disableBGPExport: + description: + "Disable exporting routes from this IP Pool's CIDR over + BGP. [Default: false]" + type: boolean + disabled: + description: + When disabled is true, Calico IPAM will not assign addresses + from this pool. + type: boolean + ipip: + description: |- + Deprecated: this field is only used for APIv1 backwards compatibility. + Setting this field is not allowed, this field is for internal use only. + properties: + enabled: + description: |- + When enabled is true, ipip tunneling will be used to deliver packets to + destinations within this pool. + type: boolean + mode: + description: |- + The IPIP mode. This can be one of "always" or "cross-subnet". A mode + of "always" will also use IPIP tunneling for routing to destination IP + addresses within this pool. A mode of "cross-subnet" will only use IPIP + tunneling when the destination node is on a different subnet to the + originating node. The default value (if not specified) is "always". + type: string + type: object + ipipMode: + description: |- + Contains configuration for IPIP tunneling for this pool. If not specified, + then this is defaulted to "Never" (i.e. IPIP tunneling is disabled). + type: string + nat-outgoing: + description: |- + Deprecated: this field is only used for APIv1 backwards compatibility. + Setting this field is not allowed, this field is for internal use only. + type: boolean + natOutgoing: + description: |- + When natOutgoing is true, packets sent from Calico networked containers in + this pool to destinations outside of this pool will be masqueraded. + type: boolean + nodeSelector: + description: + Allows IPPool to allocate for a specific node by label + selector. + type: string + vxlanMode: + description: |- + Contains configuration for VXLAN tunneling for this pool. If not specified, + then this is defaulted to "Never" (i.e. VXLAN tunneling is disabled). + type: string + required: + - cidr + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: ipreservations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPReservation + listKind: IPReservationList + plural: ipreservations + singular: ipreservation + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: + IPReservationSpec contains the specification for an IPReservation + resource. + properties: + reservedCIDRs: + description: + ReservedCIDRs is a list of CIDRs and/or IP addresses + that Calico IPAM will exclude from new allocations. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: kubecontrollersconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: KubeControllersConfiguration + listKind: KubeControllersConfigurationList + plural: kubecontrollersconfigurations + singular: kubecontrollersconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: + KubeControllersConfigurationSpec contains the values of the + Kubernetes controllers configuration. + properties: + controllers: + description: + Controllers enables and configures individual Kubernetes + controllers + properties: + loadBalancer: + description: + LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object + namespace: + description: + Namespace enables and configures the namespace controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: + "ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]" + type: string + type: object + node: + description: + Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: + HostEndpoint controls syncing nodes to host endpoints. + Disabled by default, set to nil to disable. + properties: + autoCreate: + description: + "AutoCreate enables automatic creation of + host endpoints for every node. [Default: Disabled]" + type: string + createDefaultHostEndpoint: + type: string + templates: + description: + Templates contains definition for creating + AutoHostEndpoints + items: + properties: + generateName: + description: + GenerateName is appended to the end + of the generated AutoHostEndpoint name + type: string + interfaceCIDRs: + description: + InterfaceCIDRs contains a list of CIRDs + used for matching nodeIPs to the AutoHostEndpoint + items: + type: string + type: array + labels: + additionalProperties: + type: string + description: + Labels adds the specified labels to + the generated AutoHostEndpoint, labels from node + with the same name will be overwritten by values + from the template label + type: object + nodeSelector: + description: + NodeSelector allows the AutoHostEndpoint + to be created only for specific nodes + type: string + type: object + type: array + type: object + leakGracePeriod: + description: |- + LeakGracePeriod is the period used by the controller to determine if an IP address has been leaked. + Set to 0 to disable IP garbage collection. [Default: 15m] + type: string + reconcilerPeriod: + description: + "ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]" + type: string + syncLabels: + description: + "SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]" + type: string + type: object + policy: + description: + Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: + "ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]" + type: string + type: object + serviceAccount: + description: + ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: + "ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]" + type: string + type: object + workloadEndpoint: + description: + WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: + "ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]" + type: string + type: object + type: object + debugProfilePort: + description: |- + DebugProfilePort configures the port to serve memory and cpu profiles on. If not specified, profiling + is disabled. + format: int32 + type: integer + etcdV3CompactionPeriod: + description: + "EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]" + type: string + healthChecks: + description: + "HealthChecks enables or disables support for health + checks [Default: Enabled]" + type: string + logSeverityScreen: + description: + "LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]" + type: string + prometheusMetricsPort: + description: + "PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: 9094]" + type: integer + required: + - controllers + type: object + status: + description: |- + KubeControllersConfigurationStatus represents the status of the configuration. It's useful for admins to + be able to see the actual config that was applied, which can be modified by environment variables on the + kube-controllers process. + properties: + environmentVars: + additionalProperties: + type: string + description: |- + EnvironmentVars contains the environment variables on the kube-controllers that influenced + the RunningConfig. + type: object + runningConfig: + description: |- + RunningConfig contains the effective config that is running in the kube-controllers pod, after + merging the API resource with any environment variables. + properties: + controllers: + description: + Controllers enables and configures individual Kubernetes + controllers + properties: + loadBalancer: + description: + LoadBalancer enables and configures the LoadBalancer + controller. Enabled by default, set to nil to disable. + properties: + assignIPs: + type: string + type: object + namespace: + description: + Namespace enables and configures the namespace + controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: + "ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]" + type: string + type: object + node: + description: + Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: + HostEndpoint controls syncing nodes to host + endpoints. Disabled by default, set to nil to disable. + properties: + autoCreate: + description: + "AutoCreate enables automatic creation + of host endpoints for every node. [Default: Disabled]" + type: string + createDefaultHostEndpoint: + type: string + templates: + description: + Templates contains definition for creating + AutoHostEndpoints + items: + properties: + generateName: + description: + GenerateName is appended to the + end of the generated AutoHostEndpoint name + type: string + interfaceCIDRs: + description: + InterfaceCIDRs contains a list + of CIRDs used for matching nodeIPs to the + AutoHostEndpoint + items: + type: string + type: array + labels: + additionalProperties: + type: string + description: + Labels adds the specified labels + to the generated AutoHostEndpoint, labels + from node with the same name will be overwritten + by values from the template label + type: object + nodeSelector: + description: + NodeSelector allows the AutoHostEndpoint + to be created only for specific nodes + type: string + type: object + type: array + type: object + leakGracePeriod: + description: |- + LeakGracePeriod is the period used by the controller to determine if an IP address has been leaked. + Set to 0 to disable IP garbage collection. [Default: 15m] + type: string + reconcilerPeriod: + description: + "ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]" + type: string + syncLabels: + description: + "SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]" + type: string + type: object + policy: + description: + Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: + "ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]" + type: string + type: object + serviceAccount: + description: + ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: + "ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]" + type: string + type: object + workloadEndpoint: + description: + WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: + "ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]" + type: string + type: object + type: object + debugProfilePort: + description: |- + DebugProfilePort configures the port to serve memory and cpu profiles on. If not specified, profiling + is disabled. + format: int32 + type: integer + etcdV3CompactionPeriod: + description: + "EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]" + type: string + healthChecks: + description: + "HealthChecks enables or disables support for health + checks [Default: Enabled]" + type: string + logSeverityScreen: + description: + "LogSeverityScreen is the log severity above which + logs are sent to the stdout. [Default: Info]" + type: string + prometheusMetricsPort: + description: + "PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: + 9094]" + type: integer + required: + - controllers + type: object + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: networkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: NetworkPolicy + listKind: NetworkPolicyList + plural: networkpolicies + singular: networkpolicy + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + egress: + description: |- + The ordered set of egress rules. Each rule contains a set of packet match criteria and + a corresponding action to apply. + items: + description: |- + A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy + and security Profiles reference rules - separated out as a list of rules for both + ingress and egress packet matching. + + Each positive match criteria has a negated version, prefixed with "Not". All the match + criteria within a rule must be satisfied for a packet to match. A single rule can contain + the positive and negative version of a match and both must be satisfied for the rule to match. + properties: + action: + type: string + destination: + description: + Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + http: + description: + HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: |- + Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed + HTTP Methods (e.g. GET, PUT, etc.) + Multiple methods are OR'd together. + items: + type: string + type: array + paths: + description: |- + Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed + HTTP Paths. + Multiple paths are OR'd together. + e.g: + - exact: /foo + - prefix: /bar + NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it. + items: + description: |- + HTTPPath specifies an HTTP path to match. It may be either of the form: + exact: : which matches the path exactly or + prefix: : which matches the path prefix + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: |- + ICMP is an optional field that restricts the rule to apply to a specific type and + code of ICMP traffic. This should only be specified if the Protocol field is set to + "ICMP" or "ICMPv6". + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + ipVersion: + description: |- + IPVersion is an optional field that restricts the rule to only match a specific IP + version. + type: integer + metadata: + description: + Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: + Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: + NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: |- + Protocol is an optional field that restricts the rule to only apply to traffic of + a specific IP protocol. Required if any of the EntityRules contain Ports + (because ports only apply to certain protocols). + + Must be one of these string values: "TCP", "UDP", "ICMP", "ICMPv6", "SCTP", "UDPLite" + or an integer in the range 1-255. + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: + Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: |- + The ordered set of ingress rules. Each rule contains a set of packet match criteria and + a corresponding action to apply. + items: + description: |- + A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy + and security Profiles reference rules - separated out as a list of rules for both + ingress and egress packet matching. + + Each positive match criteria has a negated version, prefixed with "Not". All the match + criteria within a rule must be satisfied for a packet to match. A single rule can contain + the positive and negative version of a match and both must be satisfied for the rule to match. + properties: + action: + type: string + destination: + description: + Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + http: + description: + HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: |- + Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed + HTTP Methods (e.g. GET, PUT, etc.) + Multiple methods are OR'd together. + items: + type: string + type: array + paths: + description: |- + Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed + HTTP Paths. + Multiple paths are OR'd together. + e.g: + - exact: /foo + - prefix: /bar + NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it. + items: + description: |- + HTTPPath specifies an HTTP path to match. It may be either of the form: + exact: : which matches the path exactly or + prefix: : which matches the path prefix + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: |- + ICMP is an optional field that restricts the rule to apply to a specific type and + code of ICMP traffic. This should only be specified if the Protocol field is set to + "ICMP" or "ICMPv6". + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + ipVersion: + description: |- + IPVersion is an optional field that restricts the rule to only match a specific IP + version. + type: integer + metadata: + description: + Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: + Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: + NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: |- + Protocol is an optional field that restricts the rule to only apply to traffic of + a specific IP protocol. Required if any of the EntityRules contain Ports + (because ports only apply to certain protocols). + + Must be one of these string values: "TCP", "UDP", "ICMP", "ICMPv6", "SCTP", "UDPLite" + or an integer in the range 1-255. + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: + Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + order: + description: |- + Order is an optional field that specifies the order in which the policy is applied. + Policies with higher "order" are applied after those with lower + order within the same tier. If the order is omitted, it may be considered to be "infinite" - i.e. the + policy will be applied last. Policies with identical order will be applied in + alphanumerical order based on the Policy "Name" within the tier. + type: number + performanceHints: + description: |- + PerformanceHints contains a list of hints to Calico's policy engine to + help process the policy more efficiently. Hints never change the + enforcement behaviour of the policy. + + Currently, the only available hint is "AssumeNeededOnEveryNode". When + that hint is set on a policy, Felix will act as if the policy matches + a local endpoint even if it does not. This is useful for "preloading" + any large static policies that are known to be used on every node. + If the policy is _not_ used on a particular node then the work + done to preload the policy (and to maintain it) is wasted. + items: + type: string + type: array + selector: + description: + "The selector is an expression used to pick out the endpoints + that the policy should\nbe applied to.\n\nSelector expressions follow + this syntax:\n\n\tlabel == \"string_literal\" -> comparison, e.g. + my_label == \"foo bar\"\n\tlabel != \"string_literal\" -> not + equal; also matches if label is not present\n\tlabel in { \"a\", + \"b\", \"c\", ... } -> true if the value of label X is one of + \"a\", \"b\", \"c\"\n\tlabel not in { \"a\", \"b\", \"c\", ... } + \ -> true if the value of label X is not one of \"a\", \"b\", \"c\"\n\thas(label_name) + \ -> True if that label is present\n\t! expr -> negation of expr\n\texpr + && expr -> Short-circuit and\n\texpr || expr -> Short-circuit + or\n\t( expr ) -> parens for grouping\n\tall() or the empty selector + -> matches all endpoints.\n\nLabel names are allowed to contain + alphanumerics, -, _ and /. String literals are more permissive\nbut + they do not support escape characters.\n\nExamples (with made-up + labels):\n\n\ttype == \"webserver\" && deployment == \"prod\"\n\ttype + in {\"frontend\", \"backend\"}\n\tdeployment != \"dev\"\n\t! has(label_name)" + type: string + serviceAccountSelector: + description: + ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + tier: + description: |- + The name of the tier that this policy belongs to. If this is omitted, the default + tier (name is "default") is assumed. The specified tier must exist in order to create + security policies within the tier, the "default" tier is created automatically if it + does not exist, this means for deployments requiring only a single Tier, the tier name + may be omitted on all policy management requests. + type: string + types: + description: |- + Types indicates whether this policy applies to ingress, or to egress, or to both. When + not explicitly specified (and so the value on creation is empty or nil), Calico defaults + Types according to what Ingress and Egress are present in the policy. The + default is: + + - [ PolicyTypeIngress ], if there are no Egress rules (including the case where there are + also no Ingress rules) + + - [ PolicyTypeEgress ], if there are Egress rules but no Ingress rules + + - [ PolicyTypeIngress, PolicyTypeEgress ], if there are both Ingress and Egress rules. + + When the policy is read back again, Types will always be one of these values, never empty + or nil. + items: + description: + PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: networksets.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: NetworkSet + listKind: NetworkSetList + plural: networksets + singular: networkset + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: + NetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: stagedglobalnetworkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: StagedGlobalNetworkPolicy + listKind: StagedGlobalNetworkPolicyList + plural: stagedglobalnetworkpolicies + singular: stagedglobalnetworkpolicy + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + applyOnForward: + description: + ApplyOnForward indicates to apply the rules in this policy + on forward traffic. + type: boolean + doNotTrack: + description: |- + DoNotTrack indicates whether packets matched by the rules in this policy should go through + the data plane's connection tracking, such as Linux conntrack. If True, the rules in + this policy are applied before any data plane connection tracking, and packets allowed by + this policy are marked as not to be tracked. + type: boolean + egress: + description: |- + The ordered set of egress rules. Each rule contains a set of packet match criteria and + a corresponding action to apply. + items: + description: |- + A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy + and security Profiles reference rules - separated out as a list of rules for both + ingress and egress packet matching. + + Each positive match criteria has a negated version, prefixed with "Not". All the match + criteria within a rule must be satisfied for a packet to match. A single rule can contain + the positive and negative version of a match and both must be satisfied for the rule to match. + properties: + action: + type: string + destination: + description: + Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + http: + description: + HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: |- + Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed + HTTP Methods (e.g. GET, PUT, etc.) + Multiple methods are OR'd together. + items: + type: string + type: array + paths: + description: |- + Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed + HTTP Paths. + Multiple paths are OR'd together. + e.g: + - exact: /foo + - prefix: /bar + NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it. + items: + description: |- + HTTPPath specifies an HTTP path to match. It may be either of the form: + exact: : which matches the path exactly or + prefix: : which matches the path prefix + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: |- + ICMP is an optional field that restricts the rule to apply to a specific type and + code of ICMP traffic. This should only be specified if the Protocol field is set to + "ICMP" or "ICMPv6". + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + ipVersion: + description: |- + IPVersion is an optional field that restricts the rule to only match a specific IP + version. + type: integer + metadata: + description: + Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: + Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: + NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: |- + Protocol is an optional field that restricts the rule to only apply to traffic of + a specific IP protocol. Required if any of the EntityRules contain Ports + (because ports only apply to certain protocols). + + Must be one of these string values: "TCP", "UDP", "ICMP", "ICMPv6", "SCTP", "UDPLite" + or an integer in the range 1-255. + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: + Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: |- + The ordered set of ingress rules. Each rule contains a set of packet match criteria and + a corresponding action to apply. + items: + description: |- + A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy + and security Profiles reference rules - separated out as a list of rules for both + ingress and egress packet matching. + + Each positive match criteria has a negated version, prefixed with "Not". All the match + criteria within a rule must be satisfied for a packet to match. A single rule can contain + the positive and negative version of a match and both must be satisfied for the rule to match. + properties: + action: + type: string + destination: + description: + Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + http: + description: + HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: |- + Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed + HTTP Methods (e.g. GET, PUT, etc.) + Multiple methods are OR'd together. + items: + type: string + type: array + paths: + description: |- + Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed + HTTP Paths. + Multiple paths are OR'd together. + e.g: + - exact: /foo + - prefix: /bar + NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it. + items: + description: |- + HTTPPath specifies an HTTP path to match. It may be either of the form: + exact: : which matches the path exactly or + prefix: : which matches the path prefix + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: |- + ICMP is an optional field that restricts the rule to apply to a specific type and + code of ICMP traffic. This should only be specified if the Protocol field is set to + "ICMP" or "ICMPv6". + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + ipVersion: + description: |- + IPVersion is an optional field that restricts the rule to only match a specific IP + version. + type: integer + metadata: + description: + Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: + Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: + NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: |- + Protocol is an optional field that restricts the rule to only apply to traffic of + a specific IP protocol. Required if any of the EntityRules contain Ports + (because ports only apply to certain protocols). + + Must be one of these string values: "TCP", "UDP", "ICMP", "ICMPv6", "SCTP", "UDPLite" + or an integer in the range 1-255. + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: + Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + namespaceSelector: + description: + NamespaceSelector is an optional field for an expression + used to select a pod based on namespaces. + type: string + order: + description: |- + Order is an optional field that specifies the order in which the policy is applied. + Policies with higher "order" are applied after those with lower + order within the same tier. If the order is omitted, it may be considered to be "infinite" - i.e. the + policy will be applied last. Policies with identical order will be applied in + alphanumerical order based on the Policy "Name" within the tier. + type: number + performanceHints: + description: |- + PerformanceHints contains a list of hints to Calico's policy engine to + help process the policy more efficiently. Hints never change the + enforcement behaviour of the policy. + + Currently, the only available hint is "AssumeNeededOnEveryNode". When + that hint is set on a policy, Felix will act as if the policy matches + a local endpoint even if it does not. This is useful for "preloading" + any large static policies that are known to be used on every node. + If the policy is _not_ used on a particular node then the work + done to preload the policy (and to maintain it) is wasted. + items: + type: string + type: array + preDNAT: + description: + PreDNAT indicates to apply the rules in this policy before + any DNAT. + type: boolean + selector: + description: + "The selector is an expression used to pick pick out + the endpoints that the policy should\nbe applied to.\n\nSelector + expressions follow this syntax:\n\n\tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\"\n\tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present\n\tlabel + in { \"a\", \"b\", \"c\", ... } -> true if the value of label + X is one of \"a\", \"b\", \"c\"\n\tlabel not in { \"a\", \"b\", + \"c\", ... } -> true if the value of label X is not one of \"a\", + \"b\", \"c\"\n\thas(label_name) -> True if that label is present\n\t! + expr -> negation of expr\n\texpr && expr -> Short-circuit and\n\texpr + || expr -> Short-circuit or\n\t( expr ) -> parens for grouping\n\tall() + or the empty selector -> matches all endpoints.\n\nLabel names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive\nbut they do not support escape characters.\n\nExamples + (with made-up labels):\n\n\ttype == \"webserver\" && deployment + == \"prod\"\n\ttype in {\"frontend\", \"backend\"}\n\tdeployment + != \"dev\"\n\t! has(label_name)" + type: string + serviceAccountSelector: + description: + ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + stagedAction: + description: + The staged action. If this is omitted, the default is + Set. + type: string + tier: + description: |- + The name of the tier that this policy belongs to. If this is omitted, the default + tier (name is "default") is assumed. The specified tier must exist in order to create + security policies within the tier, the "default" tier is created automatically if it + does not exist, this means for deployments requiring only a single Tier, the tier name + may be omitted on all policy management requests. + type: string + types: + description: |- + Types indicates whether this policy applies to ingress, or to egress, or to both. When + not explicitly specified (and so the value on creation is empty or nil), Calico defaults + Types according to what Ingress and Egress rules are present in the policy. The + default is: + + - [ PolicyTypeIngress ], if there are no Egress rules (including the case where there are + also no Ingress rules) + + - [ PolicyTypeEgress ], if there are Egress rules but no Ingress rules + + - [ PolicyTypeIngress, PolicyTypeEgress ], if there are both Ingress and Egress rules. + + When the policy is read back again, Types will always be one of these values, never empty + or nil. + items: + description: + PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: stagedkubernetesnetworkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: StagedKubernetesNetworkPolicy + listKind: StagedKubernetesNetworkPolicyList + plural: stagedkubernetesnetworkpolicies + singular: stagedkubernetesnetworkpolicy + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + egress: + description: |- + List of egress rules to be applied to the selected pods. Outgoing traffic is + allowed if there are no NetworkPolicies selecting the pod (and cluster policy + otherwise allows the traffic), OR if the traffic matches at least one egress rule + across all of the NetworkPolicy objects whose podSelector matches the pod. If + this field is empty then this NetworkPolicy limits all outgoing traffic (and serves + solely to ensure that the pods it selects are isolated by default). + This field is beta-level in 1.8 + items: + description: |- + NetworkPolicyEgressRule describes a particular set of traffic that is allowed out of pods + matched by a NetworkPolicySpec's podSelector. The traffic must match both ports and to. + This type is beta-level in 1.8 + properties: + ports: + description: |- + ports is a list of destination ports for outgoing traffic. + Each item in this list is combined using a logical OR. If this field is + empty or missing, this rule matches all ports (traffic not restricted by port). + If this field is present and contains at least one item, then this rule allows + traffic only if the traffic matches at least one port in the list. + items: + description: + NetworkPolicyPort describes a port to allow traffic + on + properties: + endPort: + description: |- + endPort indicates that the range of ports from port to endPort if set, inclusive, + should be allowed by the policy. This field cannot be defined if the port field + is not defined or if the port field is defined as a named (string) port. + The endPort must be equal or greater than port. + format: int32 + type: integer + port: + anyOf: + - type: integer + - type: string + description: |- + port represents the port on the given protocol. This can either be a numerical or named + port on a pod. If this field is not provided, this matches all port names and + numbers. + If present, only traffic on the specified protocol AND port will be matched. + x-kubernetes-int-or-string: true + protocol: + description: |- + protocol represents the protocol (TCP, UDP, or SCTP) which traffic must match. + If not specified, this field defaults to TCP. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + to: + description: |- + to is a list of destinations for outgoing traffic of pods selected for this rule. + Items in this list are combined using a logical OR operation. If this field is + empty or missing, this rule matches all destinations (traffic not restricted by + destination). If this field is present and contains at least one item, this rule + allows traffic only if the traffic matches at least one item in the to list. + items: + description: |- + NetworkPolicyPeer describes a peer to allow traffic to/from. Only certain combinations of + fields are allowed + properties: + ipBlock: + description: |- + ipBlock defines policy on a particular IPBlock. If this field is set then + neither of the other fields can be. + properties: + cidr: + description: |- + cidr is a string representing the IPBlock + Valid examples are "192.168.1.0/24" or "2001:db8::/64" + type: string + except: + description: |- + except is a slice of CIDRs that should not be included within an IPBlock + Valid examples are "192.168.1.0/24" or "2001:db8::/64" + Except values will be rejected if they are outside the cidr range + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - cidr + type: object + namespaceSelector: + description: |- + namespaceSelector selects namespaces using cluster-scoped labels. This field follows + standard label selector semantics; if present but empty, it selects all namespaces. + + If podSelector is also set, then the NetworkPolicyPeer as a whole selects + the pods matching podSelector in the namespaces selected by namespaceSelector. + Otherwise it selects all pods in the namespaces selected by namespaceSelector. + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + podSelector is a label selector which selects pods. This field follows standard label + selector semantics; if present but empty, it selects all pods. + + If namespaceSelector is also set, then the NetworkPolicyPeer as a whole selects + the pods matching podSelector in the Namespaces selected by NamespaceSelector. + Otherwise it selects the pods matching podSelector in the policy's own namespace. + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + ingress: + description: |- + List of ingress rules to be applied to the selected pods. Traffic is allowed to + a pod if there are no NetworkPolicies selecting the pod + (and cluster policy otherwise allows the traffic), OR if the traffic source is + the pod's local node, OR if the traffic matches at least one ingress rule + across all of the NetworkPolicy objects whose podSelector matches the pod. If + this field is empty then this NetworkPolicy does not allow any traffic (and serves + solely to ensure that the pods it selects are isolated by default) + items: + description: |- + NetworkPolicyIngressRule describes a particular set of traffic that is allowed to the pods + matched by a NetworkPolicySpec's podSelector. The traffic must match both ports and from. + properties: + from: + description: |- + from is a list of sources which should be able to access the pods selected for this rule. + Items in this list are combined using a logical OR operation. If this field is + empty or missing, this rule matches all sources (traffic not restricted by + source). If this field is present and contains at least one item, this rule + allows traffic only if the traffic matches at least one item in the from list. + items: + description: |- + NetworkPolicyPeer describes a peer to allow traffic to/from. Only certain combinations of + fields are allowed + properties: + ipBlock: + description: |- + ipBlock defines policy on a particular IPBlock. If this field is set then + neither of the other fields can be. + properties: + cidr: + description: |- + cidr is a string representing the IPBlock + Valid examples are "192.168.1.0/24" or "2001:db8::/64" + type: string + except: + description: |- + except is a slice of CIDRs that should not be included within an IPBlock + Valid examples are "192.168.1.0/24" or "2001:db8::/64" + Except values will be rejected if they are outside the cidr range + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - cidr + type: object + namespaceSelector: + description: |- + namespaceSelector selects namespaces using cluster-scoped labels. This field follows + standard label selector semantics; if present but empty, it selects all namespaces. + + If podSelector is also set, then the NetworkPolicyPeer as a whole selects + the pods matching podSelector in the namespaces selected by namespaceSelector. + Otherwise it selects all pods in the namespaces selected by namespaceSelector. + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + podSelector is a label selector which selects pods. This field follows standard label + selector semantics; if present but empty, it selects all pods. + + If namespaceSelector is also set, then the NetworkPolicyPeer as a whole selects + the pods matching podSelector in the Namespaces selected by NamespaceSelector. + Otherwise it selects the pods matching podSelector in the policy's own namespace. + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + ports: + description: |- + ports is a list of ports which should be made accessible on the pods selected for + this rule. Each item in this list is combined using a logical OR. If this field is + empty or missing, this rule matches all ports (traffic not restricted by port). + If this field is present and contains at least one item, then this rule allows + traffic only if the traffic matches at least one port in the list. + items: + description: + NetworkPolicyPort describes a port to allow traffic + on + properties: + endPort: + description: |- + endPort indicates that the range of ports from port to endPort if set, inclusive, + should be allowed by the policy. This field cannot be defined if the port field + is not defined or if the port field is defined as a named (string) port. + The endPort must be equal or greater than port. + format: int32 + type: integer + port: + anyOf: + - type: integer + - type: string + description: |- + port represents the port on the given protocol. This can either be a numerical or named + port on a pod. If this field is not provided, this matches all port names and + numbers. + If present, only traffic on the specified protocol AND port will be matched. + x-kubernetes-int-or-string: true + protocol: + description: |- + protocol represents the protocol (TCP, UDP, or SCTP) which traffic must match. + If not specified, this field defaults to TCP. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + podSelector: + description: |- + Selects the pods to which this NetworkPolicy object applies. The array of + ingress rules is applied to any pods selected by this field. Multiple network + policies can select the same set of pods. In this case, the ingress rules for + each are combined additively. This field is NOT optional and follows standard + label selector semantics. An empty podSelector matches all pods in this + namespace. + properties: + matchExpressions: + description: + matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + policyTypes: + description: |- + List of rule types that the NetworkPolicy relates to. + Valid options are Ingress, Egress, or Ingress,Egress. + If this field is not specified, it will default based on the existence of Ingress or Egress rules; + policies that contain an Egress section are assumed to affect Egress, and all policies + (whether or not they contain an Ingress section) are assumed to affect Ingress. + If you want to write an egress-only policy, you must explicitly specify policyTypes [ "Egress" ]. + Likewise, if you want to write a policy that specifies that no egress is allowed, + you must specify a policyTypes value that include "Egress" (since such a policy would not include + an Egress section and would otherwise default to just [ "Ingress" ]). + This field is beta-level in 1.8 + items: + description: |- + PolicyType string describes the NetworkPolicy type + This type is beta-level in 1.8 + type: string + type: array + stagedAction: + description: + The staged action. If this is omitted, the default is + Set. + type: string + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: stagednetworkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: StagedNetworkPolicy + listKind: StagedNetworkPolicyList + plural: stagednetworkpolicies + singular: stagednetworkpolicy + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + egress: + description: |- + The ordered set of egress rules. Each rule contains a set of packet match criteria and + a corresponding action to apply. + items: + description: |- + A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy + and security Profiles reference rules - separated out as a list of rules for both + ingress and egress packet matching. + + Each positive match criteria has a negated version, prefixed with "Not". All the match + criteria within a rule must be satisfied for a packet to match. A single rule can contain + the positive and negative version of a match and both must be satisfied for the rule to match. + properties: + action: + type: string + destination: + description: + Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + http: + description: + HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: |- + Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed + HTTP Methods (e.g. GET, PUT, etc.) + Multiple methods are OR'd together. + items: + type: string + type: array + paths: + description: |- + Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed + HTTP Paths. + Multiple paths are OR'd together. + e.g: + - exact: /foo + - prefix: /bar + NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it. + items: + description: |- + HTTPPath specifies an HTTP path to match. It may be either of the form: + exact: : which matches the path exactly or + prefix: : which matches the path prefix + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: |- + ICMP is an optional field that restricts the rule to apply to a specific type and + code of ICMP traffic. This should only be specified if the Protocol field is set to + "ICMP" or "ICMPv6". + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + ipVersion: + description: |- + IPVersion is an optional field that restricts the rule to only match a specific IP + version. + type: integer + metadata: + description: + Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: + Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: + NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: |- + Protocol is an optional field that restricts the rule to only apply to traffic of + a specific IP protocol. Required if any of the EntityRules contain Ports + (because ports only apply to certain protocols). + + Must be one of these string values: "TCP", "UDP", "ICMP", "ICMPv6", "SCTP", "UDPLite" + or an integer in the range 1-255. + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: + Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: |- + The ordered set of ingress rules. Each rule contains a set of packet match criteria and + a corresponding action to apply. + items: + description: |- + A Rule encapsulates a set of match criteria and an action. Both selector-based security Policy + and security Profiles reference rules - separated out as a list of rules for both + ingress and egress packet matching. + + Each positive match criteria has a negated version, prefixed with "Not". All the match + criteria within a rule must be satisfied for a packet to match. A single rule can contain + the positive and negative version of a match and both must be satisfied for the rule to match. + properties: + action: + type: string + destination: + description: + Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + http: + description: + HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: |- + Methods is an optional field that restricts the rule to apply only to HTTP requests that use one of the listed + HTTP Methods (e.g. GET, PUT, etc.) + Multiple methods are OR'd together. + items: + type: string + type: array + paths: + description: |- + Paths is an optional field that restricts the rule to apply to HTTP requests that use one of the listed + HTTP Paths. + Multiple paths are OR'd together. + e.g: + - exact: /foo + - prefix: /bar + NOTE: Each entry may ONLY specify either a `exact` or a `prefix` match. The validator will check for it. + items: + description: |- + HTTPPath specifies an HTTP path to match. It may be either of the form: + exact: : which matches the path exactly or + prefix: : which matches the path prefix + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: |- + ICMP is an optional field that restricts the rule to apply to a specific type and + code of ICMP traffic. This should only be specified if the Protocol field is set to + "ICMP" or "ICMPv6". + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + ipVersion: + description: |- + IPVersion is an optional field that restricts the rule to only match a specific IP + version. + type: integer + metadata: + description: + Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: + Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: |- + Match on a specific ICMP code. If specified, the Type value must also be specified. + This is a technical limitation imposed by the kernel's iptables firewall, which + Calico uses to enforce the rule. + type: integer + type: + description: |- + Match on a specific ICMP type. For example a value of 8 refers to ICMP Echo Request + (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: + NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: |- + Protocol is an optional field that restricts the rule to only apply to traffic of + a specific IP protocol. Required if any of the EntityRules contain Ports + (because ports only apply to certain protocols). + + Must be one of these string values: "TCP", "UDP", "ICMP", "ICMPv6", "SCTP", "UDPLite" + or an integer in the range 1-255. + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: + Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: |- + NamespaceSelector is an optional field that contains a selector expression. Only traffic + that originates from (or terminates at) endpoints within the selected namespaces will be + matched. When both NamespaceSelector and another selector are defined on the same rule, then only + workload endpoints that are matched by both selectors will be selected by the rule. + + For NetworkPolicy, an empty NamespaceSelector implies that the Selector is limited to selecting + only workload endpoints in the same namespace as the NetworkPolicy. + + For NetworkPolicy, `global()` NamespaceSelector implies that the Selector is limited to selecting + only GlobalNetworkSet or HostEndpoint. + + For GlobalNetworkPolicy, an empty NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces. + type: string + nets: + description: |- + Nets is an optional field that restricts the rule to only apply to traffic that + originates from (or terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: + NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: |- + NotPorts is the negated version of the Ports field. + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: |- + NotSelector is the negated version of the Selector field. See Selector field for + subtleties with negated selectors. + type: string + ports: + description: |- + Ports is an optional field that restricts the rule to only apply to traffic that has a + source (destination) port that matches one of these ranges/values. This value is a + list of integers or strings that represent ranges of ports. + + Since only some protocols have ports, if any ports are specified it requires the + Protocol match in the Rule to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: + "Selector is an optional field that contains + a selector expression (see Policy for\nsample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching\nthe selector will be matched.\n\nNote that: + in addition to the negated version of the Selector (see + NotSelector below), the\nselector expression syntax itself + supports negation. The two types of negation are subtly\ndifferent. + One negates the set of matched endpoints, the other negates + the whole match:\n\n\tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled\n\tendpoints + that do not have the label \"my_label\".\n\n\tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled\n\tendpoints that do have the label + \"my_label\".\n\nThe effect is that the latter will accept + packets from non-Calico sources whereas the\nformer is + limited to packets from Calico-controlled endpoints." + type: string + serviceAccounts: + description: |- + ServiceAccounts is an optional field that restricts the rule to only apply to traffic that originates from (or + terminates at) a pod running as a matching service account. + properties: + names: + description: |- + Names is an optional field that restricts the rule to only apply to traffic that originates from (or terminates + at) a pod running as a service account whose name is in the list. + items: + type: string + type: array + selector: + description: |- + Selector is an optional field that restricts the rule to only apply to traffic that originates from + (or terminates at) a pod running as a service account that matches the given label selector. + If both Names and Selector are specified then they are AND'ed. + type: string + type: object + services: + description: |- + Services is an optional field that contains options for matching Kubernetes Services. + If specified, only traffic that originates from or terminates at endpoints within the selected + service(s) will be matched, and only to/from each endpoint's port. + + Services cannot be specified on the same rule as Selector, NotSelector, NamespaceSelector, Nets, + NotNets or ServiceAccounts. + + Ports and NotPorts can only be specified with Services on ingress rules. + properties: + name: + description: + Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: |- + Namespace specifies the namespace of the given Service. If left empty, the rule + will match within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + order: + description: |- + Order is an optional field that specifies the order in which the policy is applied. + Policies with higher "order" are applied after those with lower + order within the same tier. If the order is omitted, it may be considered to be "infinite" - i.e. the + policy will be applied last. Policies with identical order will be applied in + alphanumerical order based on the Policy "Name" within the tier. + type: number + performanceHints: + description: |- + PerformanceHints contains a list of hints to Calico's policy engine to + help process the policy more efficiently. Hints never change the + enforcement behaviour of the policy. + + Currently, the only available hint is "AssumeNeededOnEveryNode". When + that hint is set on a policy, Felix will act as if the policy matches + a local endpoint even if it does not. This is useful for "preloading" + any large static policies that are known to be used on every node. + If the policy is _not_ used on a particular node then the work + done to preload the policy (and to maintain it) is wasted. + items: + type: string + type: array + selector: + description: + "The selector is an expression used to pick pick out + the endpoints that the policy should\nbe applied to.\n\nSelector + expressions follow this syntax:\n\n\tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\"\n\tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present\n\tlabel + in { \"a\", \"b\", \"c\", ... } -> true if the value of label + X is one of \"a\", \"b\", \"c\"\n\tlabel not in { \"a\", \"b\", + \"c\", ... } -> true if the value of label X is not one of \"a\", + \"b\", \"c\"\n\thas(label_name) -> True if that label is present\n\t! + expr -> negation of expr\n\texpr && expr -> Short-circuit and\n\texpr + || expr -> Short-circuit or\n\t( expr ) -> parens for grouping\n\tall() + or the empty selector -> matches all endpoints.\n\nLabel names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive\nbut they do not support escape characters.\n\nExamples + (with made-up labels):\n\n\ttype == \"webserver\" && deployment + == \"prod\"\n\ttype in {\"frontend\", \"backend\"}\n\tdeployment + != \"dev\"\n\t! has(label_name)" + type: string + serviceAccountSelector: + description: + ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + stagedAction: + description: + The staged action. If this is omitted, the default is + Set. + type: string + tier: + description: |- + The name of the tier that this policy belongs to. If this is omitted, the default + tier (name is "default") is assumed. The specified tier must exist in order to create + security policies within the tier, the "default" tier is created automatically if it + does not exist, this means for deployments requiring only a single Tier, the tier name + may be omitted on all policy management requests. + type: string + types: + description: |- + Types indicates whether this policy applies to ingress, or to egress, or to both. When + not explicitly specified (and so the value on creation is empty or nil), Calico defaults + Types according to what Ingress and Egress are present in the policy. The + default is: + + - [ PolicyTypeIngress ], if there are no Egress rules (including the case where there are + also no Ingress rules) + + - [ PolicyTypeEgress ], if there are Egress rules but no Ingress rules + + - [ PolicyTypeIngress, PolicyTypeEgress ], if there are both Ingress and Egress rules. + + When the policy is read back again, Types will always be one of these values, never empty + or nil. + items: + description: + PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: tiers.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: Tier + listKind: TierList + plural: tiers + singular: tier + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: + TierSpec contains the specification for a security policy + tier resource. + properties: + defaultAction: + description: |- + DefaultAction specifies the action applied to workloads selected by a policy in the tier, + but not rule matched the workload's traffic. + [Default: Deny] + enum: + - Pass + - Deny + type: string + order: + description: |- + Order is an optional field that specifies the order in which the tier is applied. + Tiers with higher "order" are applied after those with lower order. If the order + is omitted, it may be considered to be "infinite" - i.e. the tier will be applied + last. Tiers with identical order will be applied in alphanumerical order based + on the Tier "Name". + type: number + type: object + type: object + served: true + storage: true +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 + policy.networking.k8s.io/bundle-version: v0.1.1 + policy.networking.k8s.io/channel: experimental + creationTimestamp: null + name: adminnetworkpolicies.policy.networking.k8s.io +spec: + group: policy.networking.k8s.io + names: + kind: AdminNetworkPolicy + listKind: AdminNetworkPolicyList + plural: adminnetworkpolicies + shortNames: + - anp + singular: adminnetworkpolicy + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.priority + name: Priority + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + AdminNetworkPolicy is a cluster level resource that is part of the + AdminNetworkPolicy API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired behavior of AdminNetworkPolicy. + properties: + egress: + description: |- + Egress is the list of Egress rules to be applied to the selected pods. + A total of 100 rules will be allowed in each ANP instance. + The relative precedence of egress rules within a single ANP object (all of + which share the priority) will be determined by the order in which the rule + is written. Thus, a rule that appears at the top of the egress rules + would take the highest precedence. + ANPs with no egress rules do not affect egress traffic. + + + Support: Core + items: + description: |- + AdminNetworkPolicyEgressRule describes an action to take on a particular + set of traffic originating from pods selected by a AdminNetworkPolicy's + Subject field. + + properties: + action: + description: |- + Action specifies the effect this rule will have on matching traffic. + Currently the following actions are supported: + Allow: allows the selected traffic (even if it would otherwise have been denied by NetworkPolicy) + Deny: denies the selected traffic + Pass: instructs the selected traffic to skip any remaining ANP rules, and + then pass execution to any NetworkPolicies that select the pod. + If the pod is not selected by any NetworkPolicies then execution + is passed to any BaselineAdminNetworkPolicies that select the pod. + + + Support: Core + enum: + - Allow + - Deny + - Pass + type: string + name: + description: |- + Name is an identifier for this rule, that may be no more than 100 characters + in length. This field should be used by the implementation to help + improve observability, readability and error-reporting for any applied + AdminNetworkPolicies. + + + Support: Core + maxLength: 100 + type: string + ports: + description: |- + Ports allows for matching traffic based on port and protocols. + This field is a list of destination ports for the outgoing egress traffic. + If Ports is not set then the rule does not filter traffic via port. + + + Support: Core + items: + description: |- + AdminNetworkPolicyPort describes how to select network ports on pod(s). + Exactly one field must be set. + maxProperties: 1 + minProperties: 1 + properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string + portNumber: + description: |- + Port selects a port on a pod(s) based on number. + + + Support: Core + properties: + port: + description: |- + Number defines a network port value. + + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + + Support: Core + type: string + required: + - port + - protocol + type: object + portRange: + description: |- + PortRange selects a port range on a pod(s) based on provided start and end + values. + + + Support: Core + properties: + end: + description: |- + End defines a network port that is the end of a port range, the End value + must be greater than Start. + + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + + Support: Core + type: string + start: + description: |- + Start defines a network port that is the start of a port range, the Start + value must be less than End. + + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - end + - start + type: object + type: object + maxItems: 100 + type: array + to: + description: |- + To is the List of destinations whose traffic this rule applies to. + If any AdminNetworkPolicyEgressPeer matches the destination of outgoing + traffic then the specified action is applied. + This field must be defined and contain at least one item. + + + Support: Core + items: + description: |- + AdminNetworkPolicyEgressPeer defines a peer to allow traffic to. + Exactly one of the selector pointers must be set for a given peer. If a + consumer observes none of its fields are set, they must assume an unknown + option has been specified and fail closed. + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: |- + Namespaces defines a way to select all pods within a set of Namespaces. + Note that host-networked pods are not included in this type of peer. + + + Support: Core + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + + Networks can have upto 25 CIDRs specified. + + + Support: Extended + + + + items: + description: |- + CIDR is an IP address range in CIDR notation (for example, "10.0.0.0/8" or "fd00::/8"). + This string must be validated by implementations using net.ParseCIDR + TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: + CIDR must be either an IPv4 or IPv6 address. + IPv4 address embedded in IPv6 addresses are not + supported + rule: self.contains(':') != self.contains('.') + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster. This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + + Support: Extended + + + + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: |- + Pods defines a way to select a set of pods in + a set of namespaces. Note that host-networked pods + are not included in this type of peer. + + + Support: Core + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + required: + - action + - to + type: object + x-kubernetes-validations: + - message: + networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: + "!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))" + maxItems: 100 + type: array + ingress: + description: |- + Ingress is the list of Ingress rules to be applied to the selected pods. + A total of 100 rules will be allowed in each ANP instance. + The relative precedence of ingress rules within a single ANP object (all of + which share the priority) will be determined by the order in which the rule + is written. Thus, a rule that appears at the top of the ingress rules + would take the highest precedence. + ANPs with no ingress rules do not affect ingress traffic. + + + Support: Core + items: + description: |- + AdminNetworkPolicyIngressRule describes an action to take on a particular + set of traffic destined for pods selected by an AdminNetworkPolicy's + Subject field. + properties: + action: + description: |- + Action specifies the effect this rule will have on matching traffic. + Currently the following actions are supported: + Allow: allows the selected traffic (even if it would otherwise have been denied by NetworkPolicy) + Deny: denies the selected traffic + Pass: instructs the selected traffic to skip any remaining ANP rules, and + then pass execution to any NetworkPolicies that select the pod. + If the pod is not selected by any NetworkPolicies then execution + is passed to any BaselineAdminNetworkPolicies that select the pod. + + + Support: Core + enum: + - Allow + - Deny + - Pass + type: string + from: + description: |- + From is the list of sources whose traffic this rule applies to. + If any AdminNetworkPolicyIngressPeer matches the source of incoming + traffic then the specified action is applied. + This field must be defined and contain at least one item. + + + Support: Core + items: + description: |- + AdminNetworkPolicyIngressPeer defines an in-cluster peer to allow traffic from. + Exactly one of the selector pointers must be set for a given peer. If a + consumer observes none of its fields are set, they must assume an unknown + option has been specified and fail closed. + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: |- + Namespaces defines a way to select all pods within a set of Namespaces. + Note that host-networked pods are not included in this type of peer. + + + Support: Core + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: |- + Pods defines a way to select a set of pods in + a set of namespaces. Note that host-networked pods + are not included in this type of peer. + + + Support: Core + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + name: + description: |- + Name is an identifier for this rule, that may be no more than 100 characters + in length. This field should be used by the implementation to help + improve observability, readability and error-reporting for any applied + AdminNetworkPolicies. + + + Support: Core + maxLength: 100 + type: string + ports: + description: |- + Ports allows for matching traffic based on port and protocols. + This field is a list of ports which should be matched on + the pods selected for this policy i.e the subject of the policy. + So it matches on the destination port for the ingress traffic. + If Ports is not set then the rule does not filter traffic via port. + + + Support: Core + items: + description: |- + AdminNetworkPolicyPort describes how to select network ports on pod(s). + Exactly one field must be set. + maxProperties: 1 + minProperties: 1 + properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string + portNumber: + description: |- + Port selects a port on a pod(s) based on number. + + + Support: Core + properties: + port: + description: |- + Number defines a network port value. + + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + + Support: Core + type: string + required: + - port + - protocol + type: object + portRange: + description: |- + PortRange selects a port range on a pod(s) based on provided start and end + values. + + + Support: Core + properties: + end: + description: |- + End defines a network port that is the end of a port range, the End value + must be greater than Start. + + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + + Support: Core + type: string + start: + description: |- + Start defines a network port that is the start of a port range, the Start + value must be less than End. + + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - end + - start + type: object + type: object + maxItems: 100 + type: array + required: + - action + - from + type: object + maxItems: 100 + type: array + priority: + description: |- + Priority is a value from 0 to 1000. Rules with lower priority values have + higher precedence, and are checked before rules with higher priority values. + All AdminNetworkPolicy rules have higher precedence than NetworkPolicy or + BaselineAdminNetworkPolicy rules + The behavior is undefined if two ANP objects have same priority. + + + Support: Core + format: int32 + maximum: 1000 + minimum: 0 + type: integer + subject: + description: |- + Subject defines the pods to which this AdminNetworkPolicy applies. + Note that host-networked pods are not included in subject selection. + + + Support: Core + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: Namespaces is used to select pods via namespace selectors. + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: + Pods is used to select pods via namespace AND pod + selectors. + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + required: + - priority + - subject + type: object + status: + description: Status is the status to be reported by the implementation. + properties: + conditions: + items: + description: + "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + required: + - conditions + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 + policy.networking.k8s.io/bundle-version: v0.1.1 + policy.networking.k8s.io/channel: experimental + creationTimestamp: null + name: baselineadminnetworkpolicies.policy.networking.k8s.io +spec: + group: policy.networking.k8s.io + names: + kind: BaselineAdminNetworkPolicy + listKind: BaselineAdminNetworkPolicyList + plural: baselineadminnetworkpolicies + shortNames: + - banp + singular: baselineadminnetworkpolicy + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + BaselineAdminNetworkPolicy is a cluster level resource that is part of the + AdminNetworkPolicy API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired behavior of BaselineAdminNetworkPolicy. + properties: + egress: + description: |- + Egress is the list of Egress rules to be applied to the selected pods if + they are not matched by any AdminNetworkPolicy or NetworkPolicy rules. + A total of 100 Egress rules will be allowed in each BANP instance. + The relative precedence of egress rules within a single BANP object + will be determined by the order in which the rule is written. + Thus, a rule that appears at the top of the egress rules + would take the highest precedence. + BANPs with no egress rules do not affect egress traffic. + + + Support: Core + items: + description: |- + BaselineAdminNetworkPolicyEgressRule describes an action to take on a particular + set of traffic originating from pods selected by a BaselineAdminNetworkPolicy's + Subject field. + + properties: + action: + description: |- + Action specifies the effect this rule will have on matching traffic. + Currently the following actions are supported: + Allow: allows the selected traffic + Deny: denies the selected traffic + + + Support: Core + enum: + - Allow + - Deny + type: string + name: + description: |- + Name is an identifier for this rule, that may be no more than 100 characters + in length. This field should be used by the implementation to help + improve observability, readability and error-reporting for any applied + BaselineAdminNetworkPolicies. + + + Support: Core + maxLength: 100 + type: string + ports: + description: |- + Ports allows for matching traffic based on port and protocols. + This field is a list of destination ports for the outgoing egress traffic. + If Ports is not set then the rule does not filter traffic via port. + items: + description: |- + AdminNetworkPolicyPort describes how to select network ports on pod(s). + Exactly one field must be set. + maxProperties: 1 + minProperties: 1 + properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string + portNumber: + description: |- + Port selects a port on a pod(s) based on number. + + + Support: Core + properties: + port: + description: |- + Number defines a network port value. + + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + + Support: Core + type: string + required: + - port + - protocol + type: object + portRange: + description: |- + PortRange selects a port range on a pod(s) based on provided start and end + values. + + + Support: Core + properties: + end: + description: |- + End defines a network port that is the end of a port range, the End value + must be greater than Start. + + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + + Support: Core + type: string + start: + description: |- + Start defines a network port that is the start of a port range, the Start + value must be less than End. + + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - end + - start + type: object + type: object + maxItems: 100 + type: array + to: + description: |- + To is the list of destinations whose traffic this rule applies to. + If any AdminNetworkPolicyEgressPeer matches the destination of outgoing + traffic then the specified action is applied. + This field must be defined and contain at least one item. + + + Support: Core + items: + description: |- + AdminNetworkPolicyEgressPeer defines a peer to allow traffic to. + Exactly one of the selector pointers must be set for a given peer. If a + consumer observes none of its fields are set, they must assume an unknown + option has been specified and fail closed. + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: |- + Namespaces defines a way to select all pods within a set of Namespaces. + Note that host-networked pods are not included in this type of peer. + + + Support: Core + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + + Networks can have upto 25 CIDRs specified. + + + Support: Extended + + + + items: + description: |- + CIDR is an IP address range in CIDR notation (for example, "10.0.0.0/8" or "fd00::/8"). + This string must be validated by implementations using net.ParseCIDR + TODO: Introduce CEL CIDR validation regex isCIDR() in Kube 1.31 when it is available. + maxLength: 43 + type: string + x-kubernetes-validations: + - message: + CIDR must be either an IPv4 or IPv6 address. + IPv4 address embedded in IPv6 addresses are not + supported + rule: self.contains(':') != self.contains('.') + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster. This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + + Support: Extended + + + + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: |- + Pods defines a way to select a set of pods in + a set of namespaces. Note that host-networked pods + are not included in this type of peer. + + + Support: Core + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + required: + - action + - to + type: object + x-kubernetes-validations: + - message: + networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: + "!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))" + maxItems: 100 + type: array + ingress: + description: |- + Ingress is the list of Ingress rules to be applied to the selected pods + if they are not matched by any AdminNetworkPolicy or NetworkPolicy rules. + A total of 100 Ingress rules will be allowed in each BANP instance. + The relative precedence of ingress rules within a single BANP object + will be determined by the order in which the rule is written. + Thus, a rule that appears at the top of the ingress rules + would take the highest precedence. + BANPs with no ingress rules do not affect ingress traffic. + + + Support: Core + items: + description: |- + BaselineAdminNetworkPolicyIngressRule describes an action to take on a particular + set of traffic destined for pods selected by a BaselineAdminNetworkPolicy's + Subject field. + properties: + action: + description: |- + Action specifies the effect this rule will have on matching traffic. + Currently the following actions are supported: + Allow: allows the selected traffic + Deny: denies the selected traffic + + + Support: Core + enum: + - Allow + - Deny + type: string + from: + description: |- + From is the list of sources whose traffic this rule applies to. + If any AdminNetworkPolicyIngressPeer matches the source of incoming + traffic then the specified action is applied. + This field must be defined and contain at least one item. + + + Support: Core + items: + description: |- + AdminNetworkPolicyIngressPeer defines an in-cluster peer to allow traffic from. + Exactly one of the selector pointers must be set for a given peer. If a + consumer observes none of its fields are set, they must assume an unknown + option has been specified and fail closed. + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: |- + Namespaces defines a way to select all pods within a set of Namespaces. + Note that host-networked pods are not included in this type of peer. + + + Support: Core + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: |- + Pods defines a way to select a set of pods in + a set of namespaces. Note that host-networked pods + are not included in this type of peer. + + + Support: Core + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: + matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + name: + description: |- + Name is an identifier for this rule, that may be no more than 100 characters + in length. This field should be used by the implementation to help + improve observability, readability and error-reporting for any applied + BaselineAdminNetworkPolicies. + + + Support: Core + maxLength: 100 + type: string + ports: + description: |- + Ports allows for matching traffic based on port and protocols. + This field is a list of ports which should be matched on + the pods selected for this policy i.e the subject of the policy. + So it matches on the destination port for the ingress traffic. + If Ports is not set then the rule does not filter traffic via port. + + + Support: Core + items: + description: |- + AdminNetworkPolicyPort describes how to select network ports on pod(s). + Exactly one field must be set. + maxProperties: 1 + minProperties: 1 + properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + + Support: Extended + + + + type: string + portNumber: + description: |- + Port selects a port on a pod(s) based on number. + + + Support: Core + properties: + port: + description: |- + Number defines a network port value. + + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + + Support: Core + type: string + required: + - port + - protocol + type: object + portRange: + description: |- + PortRange selects a port range on a pod(s) based on provided start and end + values. + + + Support: Core + properties: + end: + description: |- + End defines a network port that is the end of a port range, the End value + must be greater than Start. + + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + + Support: Core + type: string + start: + description: |- + Start defines a network port that is the start of a port range, the Start + value must be less than End. + + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - end + - start + type: object + type: object + maxItems: 100 + type: array + required: + - action + - from + type: object + maxItems: 100 + type: array + subject: + description: |- + Subject defines the pods to which this BaselineAdminNetworkPolicy applies. + Note that host-networked pods are not included in subject selection. + + + Support: Core + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: Namespaces is used to select pods via namespace selectors. + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: + Pods is used to select pods via namespace AND pod + selectors. + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: + matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: + key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + required: + - subject + type: object + status: + description: Status is the status to be reported by the implementation. + properties: + conditions: + items: + description: + "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + required: + - conditions + type: object + required: + - metadata + - spec + type: object + x-kubernetes-validations: + - message: + Only one baseline admin network policy with metadata.name="default" + can be created in the cluster + rule: self.metadata.name == 'default' + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null +--- +# Source: calico/templates/calico-kube-controllers-rbac.yaml +# Include a clusterrole for the kube-controllers component, +# and bind it to the calico-kube-controllers serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +rules: + # Nodes are watched to monitor for deletions. + - apiGroups: [""] + resources: + - nodes + verbs: + - watch + - list + - get + # Pods are watched to check for existence as part of IPAM controller. + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch + # Services are monitored for service LoadBalancer IP allocation + - apiGroups: [""] + resources: + - services + - services/status + verbs: + - get + - list + - update + - watch + # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipreservations + verbs: + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + - ipamconfigs + - tiers + verbs: + - get + - list + - create + - update + - delete + - watch + # Pools are watched to maintain a mapping of blocks to IP pools. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - watch + # kube-controllers manages hostendpoints. + - apiGroups: ["crd.projectcalico.org"] + resources: + - hostendpoints + verbs: + - get + - list + - create + - update + - delete + - watch + # Needs access to update clusterinformations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - clusterinformations + verbs: + - get + - list + - create + - update + - watch + # KubeControllersConfiguration is where it gets its config + - apiGroups: ["crd.projectcalico.org"] + resources: + - kubecontrollersconfigurations + verbs: + # read its own config + - get + - list + # create a default if none exists + - create + # update status + - update + # watch for changes + - watch +--- +# Source: calico/templates/calico-node-rbac.yaml +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-node +rules: + # Used for creating service account tokens to be used by the CNI plugin + - apiGroups: [""] + resources: + - serviceaccounts/token + resourceNames: + - calico-cni-plugin + verbs: + - create + # The CNI plugin needs to get pods, nodes, and namespaces. + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + # EndpointSlices are used for Service-based network policy rule + # enforcement. + - apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: + - watch + - list + - apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get + # Pod CIDR auto-detection on kubeadm needs access to config maps. + - apiGroups: [""] + resources: + - configmaps + verbs: + - get + - apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update + # Watch for changes to Kubernetes NetworkPolicies. + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + # Watch for changes to Kubernetes (Baseline)AdminNetworkPolicies. + - apiGroups: ["policy.networking.k8s.io"] + resources: + - adminnetworkpolicies + - baselineadminnetworkpolicies + verbs: + - watch + - list + # Used by Calico for policy information. + - apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch + # The CNI plugin patches pods/status. + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + # Calico monitors various CRDs for config. + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - bgpfilters + - globalbgpconfigs + - bgpconfigurations + - ippools + - ipreservations + - ipamblocks + - globalnetworkpolicies + - stagedglobalnetworkpolicies + - networkpolicies + - stagednetworkpolicies + - stagedkubernetesnetworkpolicies + - globalnetworksets + - networksets + - clusterinformations + - hostendpoints + - blockaffinities + - caliconodestatuses + - tiers + verbs: + - get + - list + - watch + # Calico creates some tiers on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - tiers + verbs: + - create + # Calico must create and update some CRDs on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update + # Calico must update some CRDs. + - apiGroups: ["crd.projectcalico.org"] + resources: + - caliconodestatuses + verbs: + - update + # Calico stores some configuration information on the node. + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch + # These permissions are only required for upgrade from v2.6, and can + # be removed after upgrade or on fresh installations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update + # These permissions are required for Calico CNI to perform IPAM allocations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + # The CNI plugin and calico/node need to be able to create a default + # IPAMConfiguration + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipamconfigs + verbs: + - get + - create + # Block affinities must also be watchable by confd for route aggregation. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + verbs: + - watch + # The Calico IPAM migration needs to get daemonsets. These permissions can be + # removed if not upgrading from an installation using host-local IPAM. + - apiGroups: ["apps"] + resources: + - daemonsets + verbs: + - get +--- +# Source: calico/templates/calico-node-rbac.yaml +# CNI cluster role +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-cni-plugin +rules: + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + - clusterinformations + - ippools + - ipreservations + - ipamconfigs + verbs: + - get + - list + - create + - update + - delete +--- +# Source: calico/templates/tier-getter.yaml +# Implements the necessary permissions for the kube-controller-manager to interact with +# Tiers and Tiered Policies for GC. +# +# https://github.com/tigera/operator/blob/v1.37.0/pkg/render/apiserver.go#L1505-L1545 +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-tier-getter +rules: + - apiGroups: + - "projectcalico.org" + resources: + - "tiers" + verbs: + - "get" +--- +# Source: calico/templates/calico-kube-controllers-rbac.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers +subjects: + - kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system +--- +# Source: calico/templates/calico-node-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: + - kind: ServiceAccount + name: calico-node + namespace: kube-system +--- +# Source: calico/templates/calico-node-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-cni-plugin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-cni-plugin +subjects: + - kind: ServiceAccount + name: calico-cni-plugin + namespace: kube-system +--- +# Source: calico/templates/tier-getter.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-tier-getter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-tier-getter +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: User + name: system:kube-controller-manager +--- +# Source: calico/templates/calico-node.yaml +# This manifest installs the calico-node container, as well +# as the CNI plugins and network config on +# each master and worker node in a Kubernetes cluster. +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node +spec: + selector: + matchLabels: + k8s-app: calico-node + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + k8s-app: calico-node + spec: + nodeSelector: + kubernetes.io/os: linux + hostNetwork: true + tolerations: + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: calico-node + securityContext: + seccompProfile: + type: RuntimeDefault + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + priorityClassName: system-node-critical + initContainers: + # This container performs upgrade from host-local IPAM to calico-ipam. + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam + image: docker.io/calico/cni:v3.30.2 + imagePullPolicy: IfNotPresent + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + volumeMounts: + - mountPath: /var/lib/cni/networks + name: host-local-net-dir + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + securityContext: + privileged: true + # This container installs the CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: docker.io/calico/cni:v3.30.2 + imagePullPolicy: IfNotPresent + command: ["/opt/cni/bin/install"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # CNI MTU Config variable + - name: CNI_MTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + securityContext: + privileged: true + # This init container mounts the necessary filesystems needed by the BPF data plane + # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed + # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. + - name: "mount-bpffs" + image: docker.io/calico/node:v3.30.2 + imagePullPolicy: IfNotPresent + command: ["calico-node", "-init", "-best-effort"] + volumeMounts: + - mountPath: /sys/fs + name: sys-fs + # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host + # so that it outlives the init container. + mountPropagation: Bidirectional + - mountPath: /var/run/calico + name: var-run-calico + # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host + # so that it outlives the init container. + mountPropagation: Bidirectional + # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary, + # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly. + - mountPath: /nodeproc + name: nodeproc + readOnly: true + securityContext: + privileged: true + containers: + # Runs calico-node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: docker.io/calico/node:v3.30.2 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Choose the backend to use. + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s,bgp" + # --------------------------------------------- + # Enable IPv6 on Kubernetes. + # --------------------------------------------- + # Disable IPv4 detection + - name: IP + value: "none" + # Enable IPv6 detection + - name: IP6 + value: "autodetect" + # Since podCIDR is ULA IPv6 CIDR, NAT is required + # for internet access. + - name: CALICO_IPV6POOL_NAT_OUTGOING + value: "true" + # This is required when IPv4 detection is disabled. + - name: CALICO_ROUTER_ID + value: "hash" + - name: FELIX_IPV6SUPPORT + value: "true" + # Enable VXLAN on the IPv6 IP pool. + - name: CALICO_IPV6POOL_VXLAN + value: "Always" + # Disabl IPIP (not supporting IPv6) + - name: CALICO_IPV4POOL_IPIP + value: "Never" + # Disable VXLAN on IPv4 pool (not used) + - name: CALICO_IPV4POOL_VXLAN + value: "Never" + # --------------------------------------------- + # Enable IPv6 on Kubernetes. + # --------------------------------------------- + # Set MTU for tunnel device used if ipip is enabled + - name: FELIX_IPINIPMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Set MTU for the VXLAN tunnel device. + - name: FELIX_VXLANMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Set MTU for the Wireguard tunnel device. + - name: FELIX_WIREGUARDMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + # - name: CALICO_IPV4POOL_CIDR + # value: "192.168.0.0/16" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + - name: FELIX_HEALTHENABLED + value: "true" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + lifecycle: + preStop: + exec: + command: + - /bin/calico-node + - -shutdown + livenessProbe: + exec: + command: + - /bin/calico-node + - -felix-live + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + # - -bird-ready + periodSeconds: 10 + timeoutSeconds: 10 + volumeMounts: + # For maintaining CNI plugin API credentials. + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + readOnly: false + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false + - name: policysync + mountPath: /var/run/nodeagent + # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the + # parent directory. + - name: bpffs + mountPath: /sys/fs/bpf + - name: cni-log-dir + mountPath: /var/log/calico/cni + readOnly: true + volumes: + # Used by calico-node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + type: DirectoryOrCreate + - name: var-lib-calico + hostPath: + path: /var/lib/calico + type: DirectoryOrCreate + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: sys-fs + hostPath: + path: /sys/fs/ + type: DirectoryOrCreate + - name: bpffs + hostPath: + path: /sys/fs/bpf + type: Directory + # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs. + - name: nodeproc + hostPath: + path: /proc + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + # Used to access CNI logs. + - name: cni-log-dir + hostPath: + path: /var/log/calico/cni + # Mount in the directory for host-local IPAM allocations. This is + # used when upgrading from host-local to calico-ipam, and can be removed + # if not using the upgrade-ipam init container. + - name: host-local-net-dir + hostPath: + path: /var/lib/cni/networks + # Used to create per-pod Unix Domain Sockets + - name: policysync + hostPath: + type: DirectoryOrCreate + path: /var/run/nodeagent +--- +# Source: calico/templates/calico-kube-controllers.yaml +# See https://github.com/projectcalico/kube-controllers +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + # The controllers can only have a single active instance. + replicas: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + strategy: + type: Recreate + template: + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + spec: + nodeSelector: + kubernetes.io/os: linux + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + serviceAccountName: calico-kube-controllers + securityContext: + seccompProfile: + type: RuntimeDefault + priorityClassName: system-cluster-critical + containers: + - name: calico-kube-controllers + image: docker.io/calico/kube-controllers:v3.30.2 + imagePullPolicy: IfNotPresent + env: + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS + value: node,loadbalancer + - name: DATASTORE_TYPE + value: kubernetes + livenessProbe: + exec: + command: + - /usr/bin/check-status + - -l + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r + periodSeconds: 10 + securityContext: + runAsNonRoot: true From f6cdcc97d69fb6d4b8da0150d05f542a7cccfbae Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 29 Jul 2025 15:14:37 -0700 Subject: [PATCH 13/20] docs: add documentations for enabling IPv6 in non-eks clusters This combines existing docs for IPv6 EKS clusters with non-EKS ones, and also properly register the topic page into the documentation TOC. --- docs/book/src/SUMMARY_PREFIX.md | 1 + .../bring-your-own-aws-infrastructure.md | 6 + .../src/topics/eks/ipv6-enabled-cluster.md | 101 -------------- docs/book/src/topics/ipv6-enabled-cluster.md | 130 ++++++++++++++++++ 4 files changed, 137 insertions(+), 101 deletions(-) delete mode 100644 docs/book/src/topics/eks/ipv6-enabled-cluster.md create mode 100644 docs/book/src/topics/ipv6-enabled-cluster.md diff --git a/docs/book/src/SUMMARY_PREFIX.md b/docs/book/src/SUMMARY_PREFIX.md index de1756f422..2f24a78213 100644 --- a/docs/book/src/SUMMARY_PREFIX.md +++ b/docs/book/src/SUMMARY_PREFIX.md @@ -29,6 +29,7 @@ - [Upgrades](./topics/rosa/upgrades.md) - [External Auth Providers](./topics/rosa/external-auth.md) - [Support](./topics/rosa/support.md) + - [Enabling IPv6](./topics/ipv6-enabled-cluster.md) - [Bring Your Own AWS Infrastructure](./topics/bring-your-own-aws-infrastructure.md) - [Specifying the IAM Role to use for Management Components](./topics/specify-management-iam-role.md) - [Using external cloud provider with EBS CSI driver](./topics/external-cloud-provider-with-ebs-csi-driver.md) diff --git a/docs/book/src/topics/bring-your-own-aws-infrastructure.md b/docs/book/src/topics/bring-your-own-aws-infrastructure.md index 9dce04250a..2df203e573 100644 --- a/docs/book/src/topics/bring-your-own-aws-infrastructure.md +++ b/docs/book/src/topics/bring-your-own-aws-infrastructure.md @@ -29,6 +29,12 @@ In order to have Cluster API consume existing AWS infrastructure, you will need * Route table associations that provide connectivity to the Internet through a NAT gateway (for private subnets) or the Internet gateway (for public subnets) * VPC endpoints for `ec2`, `elasticloadbalancing`, `secretsmanager` an `autoscaling` (if using MachinePools) when the private Subnets do not have a NAT gateway +If you enable IPv6 for the workload cluster, you will need to ensure the following additional requirements: +- An IPv6 CIDR associated with the VPC (i.e. dualstack VPC). +- An egress-only internet gateway for IPv6 egress traffic from private subnets (only needed if the nodes require access to the Internet) + - In the route table associated with private subnets, a route that sends all internet-bound IPv6 traffic (`::/0`) to the egress-only internet gateway. +- (Optional) Enable DNS64 for private subnets to allow IPv6-only workloads to access IPv4-only services via NAT64. + You will need the ID of the VPC and subnet IDs that Cluster API should use. This information is available via the AWS Management Console or the AWS CLI. Note that there is no need to create an Elastic Load Balancer (ELB), security groups, or EC2 instances; Cluster API will take care of these items. diff --git a/docs/book/src/topics/eks/ipv6-enabled-cluster.md b/docs/book/src/topics/eks/ipv6-enabled-cluster.md deleted file mode 100644 index 7c10965102..0000000000 --- a/docs/book/src/topics/eks/ipv6-enabled-cluster.md +++ /dev/null @@ -1,101 +0,0 @@ -# IPv6 Enabled Cluster - -CAPA supports IPv6 enabled clusters. Dual stack clusters are not yet supported, but -dual VPC, meaning both ipv6 and ipv4 are defined, is supported and in fact, it's the -only mode of operation at the writing of this doc. - -Upcoming feature will be IPv6 _only_. - -## Managed Clusters - -### How to set up - -Two modes of operations are supported. Request AWS to generate and assign an address -or BYOIP which is Bring Your Own IP. There must already be a provisioned pool and a -set of IPv6 CIDRs for that. - -#### Automatically Generated IP - -To request AWS to assign a set of IPv6 addresses from an AWS defined address pool, -use the following setting: - -```yaml -kind: AWSManagedControlPlane -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-control-plane" -spec: - network: - vpc: - ipv6: {} -``` - -#### BYOIP ( Bring Your Own IP ) - -To define your own IPv6 address pool and CIDR set the following values: - -```yaml -spec: - network: - vpc: - ipv6: - poolId: pool-id - cidrBlock: "2009:1234:ff00::/56" -``` - -If you have a VPC that is IPv6 enabled and you would like to use it, please define it in the config: - -```yaml -spec: - network: - vpc: - ipv6: {} -``` - -This has to be done explicitly because otherwise, it would break in the following two scenarios: -- During an upgrade from 1.5 to >=2.0 where the VPC is ipv6 enabled, but CAPA was only recently made aware -- During a migration on the VPC, switching it from only IPv4 to Dual Stack ( it would see that ipv6 is enabled and - enforce it while doing that would not have been the intention of the user ) - - -### Requirements - -The use of a Nitro enabled instance is required. To see a list of nitro instances in your region -run the following command: - -```bash -aws ec2 describe-instance-types --filters Name=hypervisor,Values=nitro --region us-west-2 | grep "InstanceType" -``` - -This will list all available Nitro hypervisor based instances in your region. - -All addons **must** be enabled. A working cluster configuration looks like this: - -```yaml -kind: AWSManagedControlPlane -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-control-plane" -spec: - network: - vpc: - ipv6: {} - region: "${AWS_REGION}" - sshKeyName: "${AWS_SSH_KEY_NAME}" - version: "${KUBERNETES_VERSION}" - addons: - - name: "vpc-cni" - version: "v1.11.0-eksbuild.1" - conflictResolution: "overwrite" # this is important, otherwise environment property update will not work - - name: "coredns" - version: "v1.8.7-eksbuild.1" - - name: "kube-proxy" - version: "v1.22.6-eksbuild.1" -``` - -You can't define custom POD CIDRs on EKS with IPv6. EKS automatically assigns an address range from a unique local -address range of `fc00::/7`. - -## Unmanaged Clusters - -Unmanaged clusters are not supported at this time. diff --git a/docs/book/src/topics/ipv6-enabled-cluster.md b/docs/book/src/topics/ipv6-enabled-cluster.md new file mode 100644 index 0000000000..2623cd6d73 --- /dev/null +++ b/docs/book/src/topics/ipv6-enabled-cluster.md @@ -0,0 +1,130 @@ +# Enabling IPv6 + +## Overview + +CAPA enables you to create an IPv6 Kubernetes clusters on Amazon Web Service (AWS). + +Only single-stack IPv6 clusters are supported. However, CAPA utilizes a dual stack infrastructure (e.g. dual stack VPC) to support IPv6. In fact, it is the only mode of operation at the time of writing. + +> **IMPORTANT NOTE**: Dual stack clusters are not yet supported. + +## Prerequisites + +The instance types for control plane and worker machines must be [Nitro-based](https://docs.aws.amazon.com/ec2/latest/instancetypes/ec2-nitro-instances.html) in order to support IPv6. To see a list of Nitro instance types in your region, run the following command: + +```bash +aws ec2 describe-instance-types \ + --filters Name=hypervisor,Values=nitro \ + --query="InstanceTypes[*].InstanceType" +``` + +## Creating IPv6 EKS-managed Clusters + +To quickly deploy an IPv6 EKS cluster, use the [IPv6 EKS cluster template](https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-aws/refs/heads/main/templates/cluster-template-eks-ipv6.yaml). + + + +**Notes**: All addons **must** be enabled. A working cluster configuration looks like this: + +```yaml +kind: AWSManagedControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + network: + vpc: + ipv6: {} + region: "${AWS_REGION}" + sshKeyName: "${AWS_SSH_KEY_NAME}" + version: "${KUBERNETES_VERSION}" + addons: + - name: "vpc-cni" + version: "v1.11.0-eksbuild.1" + # this is important, otherwise environment property update will not work + conflictResolution: "overwrite" + - name: "coredns" + version: "v1.8.7-eksbuild.1" + - name: "kube-proxy" + version: "v1.22.6-eksbuild.1" +``` + +## Creating IPv6 Self-managed Clusters + +To quickly deploy an IPv6 self-managed cluster, use the [IPv6 cluster template](https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-aws/refs/heads/main/templates/cluster-template-ipv6.yaml). + +When creating a self-managed cluster, you can define the Pod and Service CIDR. For example, you can define ULA IPv6 range `fd01::/48` for pod networking and `fd02::/112` for service networking. + + + +### CNI IPv6 support + +By default, no CNI plugin is installed when provisioning a self-managed cluster. You need to install your own CNI solution that supports IPv6, for example, Calico with VXLAN. + +You can find the guides to enable [IPv6](https://docs.tigera.io/calico/latest/networking/ipam/ipv6#ipv6) and [VXLAN](https://docs.tigera.io/calico/latest/networking/configuring/vxlan-ipip) support for Calico on their official documentation. Or you can use a customized Calico manifests [here](https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-aws/refs/heads/main/test/e2e/data/cni/calico_ipv6.yaml) for IPv6. + +## IPv6 CIDR Allocations + +### AWS-assigned IPv6 VPC CIDR + +To request AWS to automatically assign an IPv6 CIDR from an AWS defined address pool, use the following setting: + +```yaml +spec: + network: + vpc: + ipv6: {} +``` + +### BYOIPv6 VPC CIDR + +To define your own IPv6 address pool and CIDR set the following values: + +```yaml +spec: + network: + vpc: + ipv6: + poolId: pool-id + cidrBlock: "2009:1234:ff00::/56" +``` + +There must already be a provisioned pool and a set of IPv6 CIDRs for that. + +### BYO IPv6 VPC + +If you have a VPC that is IPv6 enabled (i.e. dual stack VPC) and you would like to use it, please define it in the `AWSCluster` specs: + +```yaml +spec: + network: + vpc: + id: vpc-1234567890abcdefg + cidrBlock: 10.0.0.0/16 + ipv6: + cidrBlock: "2001:1234:ff00::/56" + egressOnlyInternetGatewayId: eigw-1234567890abcdefg +``` + +This has to be done explicitly because otherwise, it would break in the following two scenarios: +- During an upgrade from 1.5 to >=2.0 where the VPC is ipv6 enabled, but CAPA was only recently made aware. +- During a migration on the VPC, switching it from only IPv4 to Dual Stack (it would see that ipv6 is enabled and + enforce it while doing that would not have been the intention of the user). From c11fd5114e04240865c5eb98f1624514e7be0770 Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 5 Aug 2025 14:50:14 -0700 Subject: [PATCH 14/20] validations: validate vpc and subnet CIDR Validation for specified VPC and subnet CIDRs is added for early feedback from the webhook. There are already existing checks for bastion and nodePort CIDRs. --- api/v1beta2/awscluster_webhook.go | 36 +++++- api/v1beta2/awscluster_webhook_test.go | 146 ++++++++++++++++++++++++- 2 files changed, 178 insertions(+), 4 deletions(-) diff --git a/api/v1beta2/awscluster_webhook.go b/api/v1beta2/awscluster_webhook.go index 755978a878..9d93260db5 100644 --- a/api/v1beta2/awscluster_webhook.go +++ b/api/v1beta2/awscluster_webhook.go @@ -301,10 +301,35 @@ func (r *AWSCluster) validateSSHKeyName() field.ErrorList { func (r *AWSCluster) validateNetwork() field.ErrorList { var allErrs field.ErrorList - for _, subnet := range r.Spec.NetworkSpec.Subnets { + + vpcSpec := r.Spec.NetworkSpec.VPC + vpcField := field.NewPath("spec", "network", "vpc") + if vpcSpec.CidrBlock != "" { + if _, _, err := net.ParseCIDR(vpcSpec.CidrBlock); err != nil { + allErrs = append(allErrs, field.Invalid(vpcField.Child("cidrBlock"), vpcSpec.CidrBlock, "VPC CIDR block is invalid")) + } + } + if vpcSpec.IPv6 != nil && vpcSpec.IPv6.CidrBlock != "" { + if _, _, err := net.ParseCIDR(vpcSpec.IPv6.CidrBlock); err != nil { + allErrs = append(allErrs, field.Invalid(vpcField.Child("ipv6", "cidrBlock"), vpcSpec.IPv6.CidrBlock, "VPC IPv6 CIDR block is invalid")) + } + } + + subnetField := field.NewPath("spec", "network", "subnets") + for i, subnet := range r.Spec.NetworkSpec.Subnets { if subnet.ZoneType != nil && subnet.IsEdge() { if subnet.ParentZoneName == nil { - allErrs = append(allErrs, field.Invalid(field.NewPath("subnets"), r.Spec.NetworkSpec.Subnets, "ParentZoneName must be set when ZoneType is 'local-zone'.")) + allErrs = append(allErrs, field.Invalid(subnetField.Index(i).Child("parentZoneName"), subnet.ParentZoneName, "ParentZoneName must be set when ZoneType is 'local-zone'.")) + } + } + if subnet.CidrBlock != "" { + if _, _, err := net.ParseCIDR(subnet.CidrBlock); err != nil { + allErrs = append(allErrs, field.Invalid(subnetField.Index(i).Child("cidrBlock"), subnet.CidrBlock, "subnet CIDR block is invalid")) + } + } + if subnet.IPv6CidrBlock != "" { + if _, _, err := net.ParseCIDR(subnet.IPv6CidrBlock); err != nil { + allErrs = append(allErrs, field.Invalid(subnetField.Index(i).Child("ipv6CidrBlock"), subnet.IPv6CidrBlock, "subnet IPv6 CIDR block is invalid")) } } } @@ -344,10 +369,15 @@ func (r *AWSCluster) validateNetwork() field.ErrorList { secondaryCidrBlocks := r.Spec.NetworkSpec.VPC.SecondaryCidrBlocks secondaryCidrBlocksField := field.NewPath("spec", "network", "vpc", "secondaryCidrBlocks") - for _, cidrBlock := range secondaryCidrBlocks { + for i, cidrBlock := range secondaryCidrBlocks { if r.Spec.NetworkSpec.VPC.CidrBlock != "" && r.Spec.NetworkSpec.VPC.CidrBlock == cidrBlock.IPv4CidrBlock { allErrs = append(allErrs, field.Invalid(secondaryCidrBlocksField, secondaryCidrBlocks, fmt.Sprintf("AWSCluster.spec.network.vpc.secondaryCidrBlocks must not contain the primary AWSCluster.spec.network.vpc.cidrBlock %v", r.Spec.NetworkSpec.VPC.CidrBlock))) } + if cidrBlock.IPv4CidrBlock != "" { + if _, _, err := net.ParseCIDR(cidrBlock.IPv4CidrBlock); err != nil { + allErrs = append(allErrs, field.Invalid(secondaryCidrBlocksField.Index(i).Child("ipv4CidrBlock"), cidrBlock.IPv4CidrBlock, "secondary VPC CIDR block is invalid")) + } + } } return allErrs diff --git a/api/v1beta2/awscluster_webhook_test.go b/api/v1beta2/awscluster_webhook_test.go index 8cc89954a8..16549b8c18 100644 --- a/api/v1beta2/awscluster_webhook_test.go +++ b/api/v1beta2/awscluster_webhook_test.go @@ -324,7 +324,87 @@ func TestAWSClusterValidateCreate(t *testing.T) { wantErr: false, }, { - name: "accepts ipv6", + name: "accepts vpc cidr", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + NetworkSpec: NetworkSpec{ + VPC: VPCSpec{ + CidrBlock: "10.0.0.0/16", + }, + }, + }, + }, + wantErr: false, + }, + { + name: "rejects invalid vpc cidr", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + NetworkSpec: NetworkSpec{ + VPC: VPCSpec{ + CidrBlock: "10.0.0.0", + }, + }, + }, + }, + wantErr: true, + }, + { + name: "accepts vpc secondary cidr", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + NetworkSpec: NetworkSpec{ + VPC: VPCSpec{ + CidrBlock: "10.0.0.0/16", + SecondaryCidrBlocks: []VpcCidrBlock{ + { + IPv4CidrBlock: "10.0.1.0/24", + }, + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "rejects invalid vpc secondary cidr", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + NetworkSpec: NetworkSpec{ + VPC: VPCSpec{ + CidrBlock: "10.0.0.0/16", + SecondaryCidrBlocks: []VpcCidrBlock{ + { + IPv4CidrBlock: "10.0.1.0", + }, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "rejects vpc secondary cidr duplicate with vpc primary cidr", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + NetworkSpec: NetworkSpec{ + VPC: VPCSpec{ + CidrBlock: "10.0.0.0/16", + SecondaryCidrBlocks: []VpcCidrBlock{ + { + IPv4CidrBlock: "10.0.0.0/16", + }, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "accepts vpc ipv6 cidr", cluster: &AWSCluster{ Spec: AWSClusterSpec{ NetworkSpec: NetworkSpec{ @@ -339,6 +419,22 @@ func TestAWSClusterValidateCreate(t *testing.T) { }, wantErr: false, }, + { + name: "reject invalid vpc ipv6 cidr", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + NetworkSpec: NetworkSpec{ + VPC: VPCSpec{ + IPv6: &IPv6{ + CidrBlock: "2001:2345:5678::", + PoolID: "pool-id", + }, + }, + }, + }, + }, + wantErr: true, + }, { name: "accepts ipv6 enabled subnet", cluster: &AWSCluster{ @@ -358,6 +454,38 @@ func TestAWSClusterValidateCreate(t *testing.T) { }, wantErr: false, }, + { + name: "accepts cidr block for subnets", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + NetworkSpec: NetworkSpec{ + Subnets: []SubnetSpec{ + { + ID: "sub-1", + CidrBlock: "10.0.10.0/24", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "rejects invalid cidr block for subnets", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + NetworkSpec: NetworkSpec{ + Subnets: []SubnetSpec{ + { + ID: "sub-1", + CidrBlock: "10.0.10.0", + }, + }, + }, + }, + }, + wantErr: true, + }, { name: "accepts ipv6 cidr block for subnets", cluster: &AWSCluster{ @@ -374,6 +502,22 @@ func TestAWSClusterValidateCreate(t *testing.T) { }, wantErr: false, }, + { + name: "rejects invalid ipv6 cidr block for subnets", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + NetworkSpec: NetworkSpec{ + Subnets: []SubnetSpec{ + { + ID: "sub-1", + IPv6CidrBlock: "2022:1234:5678:9101::", + }, + }, + }, + }, + }, + wantErr: true, + }, { name: "rejects ingress rules with cidr block and source security group id", cluster: &AWSCluster{ From b0363f4aac39f1a12f18c697bd0a68853800395f Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 5 Aug 2025 17:08:03 -0700 Subject: [PATCH 15/20] docs: update doc for enabling ipv6 The following is added: - [BYO VPC] Mention the required route when enabling DNS64. - [BYO VPC] Mention that CAPA only utilizes the IPv6 aspect of the dual stack VPC. --- docs/book/src/topics/bring-your-own-aws-infrastructure.md | 1 + docs/book/src/topics/ipv6-enabled-cluster.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/book/src/topics/bring-your-own-aws-infrastructure.md b/docs/book/src/topics/bring-your-own-aws-infrastructure.md index 2df203e573..db2e55737e 100644 --- a/docs/book/src/topics/bring-your-own-aws-infrastructure.md +++ b/docs/book/src/topics/bring-your-own-aws-infrastructure.md @@ -34,6 +34,7 @@ If you enable IPv6 for the workload cluster, you will need to ensure the followi - An egress-only internet gateway for IPv6 egress traffic from private subnets (only needed if the nodes require access to the Internet) - In the route table associated with private subnets, a route that sends all internet-bound IPv6 traffic (`::/0`) to the egress-only internet gateway. - (Optional) Enable DNS64 for private subnets to allow IPv6-only workloads to access IPv4-only services via NAT64. + - In the route table associated with private subnets, a route that sends traffic for destination `64:ff9b::/96` to the NAT gateways. More details [here](https://docs.aws.amazon.com/vpc/latest/userguide/nat-gateway-nat64-dns64.html). You will need the ID of the VPC and subnet IDs that Cluster API should use. This information is available via the AWS Management Console or the AWS CLI. diff --git a/docs/book/src/topics/ipv6-enabled-cluster.md b/docs/book/src/topics/ipv6-enabled-cluster.md index 2623cd6d73..2d1390a9f7 100644 --- a/docs/book/src/topics/ipv6-enabled-cluster.md +++ b/docs/book/src/topics/ipv6-enabled-cluster.md @@ -111,7 +111,7 @@ There must already be a provisioned pool and a set of IPv6 CIDRs for that. ### BYO IPv6 VPC -If you have a VPC that is IPv6 enabled (i.e. dual stack VPC) and you would like to use it, please define it in the `AWSCluster` specs: +If you have a dual stack VPC (i.e. CAPA will only use IPv6 for the cluster in this configuration) and you would like to use it, please define it in the `AWSCluster` specs: ```yaml spec: From 32551c3540a7679daa43791deab9d925b4100fc9 Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Fri, 8 Aug 2025 13:18:27 -0700 Subject: [PATCH 16/20] cni: document the requirement for calico ipv6 support --- docs/book/src/topics/ipv6-enabled-cluster.md | 14 ++++++++++++++ templates/cluster-template-ipv6.yaml | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/docs/book/src/topics/ipv6-enabled-cluster.md b/docs/book/src/topics/ipv6-enabled-cluster.md index 2d1390a9f7..901d6b469f 100644 --- a/docs/book/src/topics/ipv6-enabled-cluster.md +++ b/docs/book/src/topics/ipv6-enabled-cluster.md @@ -81,6 +81,20 @@ By default, no CNI plugin is installed when provisioning a self-managed cluster. You can find the guides to enable [IPv6](https://docs.tigera.io/calico/latest/networking/ipam/ipv6#ipv6) and [VXLAN](https://docs.tigera.io/calico/latest/networking/configuring/vxlan-ipip) support for Calico on their official documentation. Or you can use a customized Calico manifests [here](https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-aws/refs/heads/main/test/e2e/data/cni/calico_ipv6.yaml) for IPv6. +**Note**: If you are using Calico as the CNI provider, ensure the CNI ingress rule allows VXLAN. You can set the rule in the `AWSCluster` resource, for example: +```yaml +spec: + network: + cni: + cniIngressRules: + # If using Calico as CNI provider, this rule is required. + # Note: Calico currently supports IPv6 with VXLAN. + - description: "IPv6 VXLAN (calico)" + protocol: udp + fromPort: 4789 + toPort: 4789 +``` + ## IPv6 CIDR Allocations ### AWS-assigned IPv6 VPC CIDR diff --git a/templates/cluster-template-ipv6.yaml b/templates/cluster-template-ipv6.yaml index 059092f96b..4c1edf0904 100644 --- a/templates/cluster-template-ipv6.yaml +++ b/templates/cluster-template-ipv6.yaml @@ -31,6 +31,14 @@ spec: loadBalancerType: nlb healthCheckProtocol: HTTPS network: + cni: + cniIngressRules: + # If using Calico as CNI provider, this rule is required. + # Note: Calico currently supports IPv6 with VXLAN. + - description: "IPv6 VXLAN (calico)" + protocol: udp + fromPort: 4789 + toPort: 4789 vpc: ipv6: {} region: "${AWS_REGION}" From 01b3c1290706cdd17389adfce3acdb5e62585f7a Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Thu, 18 Sep 2025 20:48:18 -0700 Subject: [PATCH 17/20] subnets: wait till IPv6 CIDR is associated with subnets There is a brief period where the IPv6 CIDR is not yet associated with the subnets. Thus, when CAPA creates the default dualstack subnets, it should wait until the IPv6 CIDR is associated before proceeding. If not, CAPA will misinterprete the subnet as non-IPv6 and proceed its reconcilation. The consequence is that CAPA will skip creating a route to eigw. Route to eigw for destination "::/0" to eigw is required for EC2 instance time sync on start-up. --- pkg/cloud/services/network/subnets.go | 50 ++++++++++++++++++++------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/pkg/cloud/services/network/subnets.go b/pkg/cloud/services/network/subnets.go index f859df7448..94125af558 100644 --- a/pkg/cloud/services/network/subnets.go +++ b/pkg/cloud/services/network/subnets.go @@ -507,9 +507,8 @@ func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, err ), }, } - if s.scope.VPC().IsIPv6Enabled() { + if sn.IsIPv6 { input.Ipv6CidrBlock = aws.String(sn.IPv6CidrBlock) - sn.IsIPv6 = true } out, err := s.EC2Client.CreateSubnet(context.TODO(), input) if err != nil { @@ -521,7 +520,32 @@ func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, err s.scope.Info("Created subnet", "id", *out.Subnet.SubnetId, "public", sn.IsPublic, "az", sn.AvailabilityZone, "cidr", sn.CidrBlock, "ipv6", sn.IsIPv6, "ipv6-cidr", sn.IPv6CidrBlock) wReq := &ec2.DescribeSubnetsInput{SubnetIds: []string{aws.ToString(out.Subnet.SubnetId)}} - if err := ec2.NewSubnetAvailableWaiter(s.EC2Client).Wait(context.TODO(), wReq, time.Minute*5); err != nil { + if err := ec2.NewSubnetAvailableWaiter(s.EC2Client).Wait(context.TODO(), wReq, time.Minute*5, func(sawo *ec2.SubnetAvailableWaiterOptions) { + // There is a brief period where the IPv6 CIDR is not yet associated with the subnets. + // We need to additionally wait till the CIDR is associated. + if sn.IsIPv6 { + // Default handler will check for subnet state "available". + subnetStateCheck := sawo.Retryable + sawo.Retryable = func(ctx context.Context, dsi *ec2.DescribeSubnetsInput, dso *ec2.DescribeSubnetsOutput, err error) (bool, error) { + available, err := subnetStateCheck(ctx, dsi, dso, err) + if err != nil { + return false, err + } + + cidrAssociated := true + for _, subnet := range dso.Subnets { + for _, set := range subnet.Ipv6CidrBlockAssociationSet { + if set.Ipv6CidrBlockState.State != types.SubnetCidrBlockStateCodeAssociated { + cidrAssociated = false + break + } + } + } + + return available && cidrAssociated, nil + } + } + }); err != nil { return nil, errors.Wrapf(err, "failed to wait for subnet %q", *out.Subnet.SubnetId) } @@ -613,23 +637,23 @@ func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, err ID: sn.ID, ResourceID: *out.Subnet.SubnetId, AvailabilityZone: *out.Subnet.AvailabilityZone, - CidrBlock: *out.Subnet.CidrBlock, // TODO: this will panic in case of IPv6 only subnets... - IsPublic: sn.IsPublic, - Tags: sn.Tags, + // In case of IPv6-only subnets, cidrBlock (IPv4) is empty. + CidrBlock: aws.ToString(out.Subnet.CidrBlock), + IsPublic: sn.IsPublic, + Tags: sn.Tags, } for _, set := range out.Subnet.Ipv6CidrBlockAssociationSet { - if set.Ipv6CidrBlockState.State == types.SubnetCidrBlockStateCodeAssociated { - subnet.IPv6CidrBlock = aws.ToString(set.Ipv6CidrBlock) - subnet.IsIPv6 = true - } + // The IPv6 CIDR is already ensured to be associated so we don't need to check for its association state. + subnet.IPv6CidrBlock = aws.ToString(set.Ipv6CidrBlock) + subnet.IsIPv6 = true } s.scope.Debug("Created new subnet in VPC with cidr and availability zone ", - "subnet-id", *out.Subnet.SubnetId, + "subnet-id", subnet.ResourceID, "vpc-id", *out.Subnet.VpcId, - "cidr-block", *out.Subnet.CidrBlock, + "cidr-block", subnet.CidrBlock, "ipv6-cidr-block", subnet.IPv6CidrBlock, - "availability-zone", *out.Subnet.AvailabilityZone) + "availability-zone", subnet.AvailabilityZone) return subnet, nil } From 6b3cded47725dfc585c8ea3855944d6706322b28 Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Mon, 29 Sep 2025 16:26:19 -0700 Subject: [PATCH 18/20] sg: allow both ipv4 and ipv6 cidrs to API LB if vpc ipv6 block is defined When AWSCluster.spec.network.vpc.ipv6 is non-nil, most handlers in CAPA treats it as "adding" IPv6 capabilities on top of IPv4 infrastructure. Except security group ingress rules for API LB. This commit aligns the API LB SG handler with the rest of the code base. These rules can be overriden in the AWSCluster LB spec to allow only IPv6 CIDRs if needed. --- .../services/securitygroup/securitygroups.go | 48 +++++++++---------- .../securitygroup/securitygroups_test.go | 28 +++++++++-- 2 files changed, 47 insertions(+), 29 deletions(-) diff --git a/pkg/cloud/services/securitygroup/securitygroups.go b/pkg/cloud/services/securitygroup/securitygroups.go index 5b23224b34..8bebb68051 100644 --- a/pkg/cloud/services/securitygroup/securitygroups.go +++ b/pkg/cloud/services/securitygroup/securitygroups.go @@ -994,19 +994,7 @@ func (s *Service) getControlPlaneLBIngressRules() infrav1.IngressRules { } func (s *Service) getIngressRuleToAllowAnyIPInTheAPIServer() infrav1.IngressRules { - if s.scope.VPC().IsIPv6Enabled() { - return infrav1.IngressRules{ - { - Description: "Kubernetes API IPv6", - Protocol: infrav1.SecurityGroupProtocolTCP, - FromPort: int64(s.scope.APIServerPort()), - ToPort: int64(s.scope.APIServerPort()), - IPv6CidrBlocks: []string{services.AnyIPv6CidrBlock}, - }, - } - } - - return infrav1.IngressRules{ + rules := infrav1.IngressRules{ { Description: "Kubernetes API", Protocol: infrav1.SecurityGroupProtocolTCP, @@ -1015,22 +1003,20 @@ func (s *Service) getIngressRuleToAllowAnyIPInTheAPIServer() infrav1.IngressRule CidrBlocks: []string{services.AnyIPv4CidrBlock}, }, } + if s.scope.VPC().IsIPv6Enabled() { + rules = append(rules, infrav1.IngressRule{ + Description: "Kubernetes API IPv6", + Protocol: infrav1.SecurityGroupProtocolTCP, + FromPort: int64(s.scope.APIServerPort()), + ToPort: int64(s.scope.APIServerPort()), + IPv6CidrBlocks: []string{services.AnyIPv6CidrBlock}, + }) + } + return rules } func (s *Service) getIngressRuleToAllowVPCCidrInTheAPIServer() infrav1.IngressRules { - if s.scope.VPC().IsIPv6Enabled() { - return infrav1.IngressRules{ - { - Description: "Kubernetes API IPv6", - Protocol: infrav1.SecurityGroupProtocolTCP, - FromPort: int64(s.scope.APIServerPort()), - ToPort: int64(s.scope.APIServerPort()), - IPv6CidrBlocks: []string{s.scope.VPC().IPv6.CidrBlock}, - }, - } - } - - return infrav1.IngressRules{ + rules := infrav1.IngressRules{ { Description: "Kubernetes API", Protocol: infrav1.SecurityGroupProtocolTCP, @@ -1039,6 +1025,16 @@ func (s *Service) getIngressRuleToAllowVPCCidrInTheAPIServer() infrav1.IngressRu CidrBlocks: []string{s.scope.VPC().CidrBlock}, }, } + if s.scope.VPC().IsIPv6Enabled() { + rules = append(rules, infrav1.IngressRule{ + Description: "Kubernetes API IPv6", + Protocol: infrav1.SecurityGroupProtocolTCP, + FromPort: int64(s.scope.APIServerPort()), + ToPort: int64(s.scope.APIServerPort()), + IPv6CidrBlocks: []string{s.scope.VPC().IPv6.CidrBlock}, + }) + } + return rules } func (s *Service) processIngressRulesSGs(ingressRules []infrav1.IngressRule) (infrav1.IngressRules, error) { diff --git a/pkg/cloud/services/securitygroup/securitygroups_test.go b/pkg/cloud/services/securitygroup/securitygroups_test.go index 6440905c41..3583cd1fe1 100644 --- a/pkg/cloud/services/securitygroup/securitygroups_test.go +++ b/pkg/cloud/services/securitygroup/securitygroups_test.go @@ -1607,7 +1607,7 @@ func TestControlPlaneLoadBalancerIngressRules(t *testing.T) { }, }, { - name: "when no ingress rules are passed and nat gateway IPs are not available, the default for IPv6 is set", + name: "when no ingress rules are passed and nat gateway IPs are not available with vpc ipv6 block is defined, the default for IPv4 and IPv6 are set", awsCluster: &infrav1.AWSCluster{ Spec: infrav1.AWSClusterSpec{ ControlPlaneLoadBalancer: &infrav1.AWSLoadBalancerSpec{}, @@ -1621,6 +1621,13 @@ func TestControlPlaneLoadBalancerIngressRules(t *testing.T) { Status: infrav1.AWSClusterStatus{}, }, expectedIngresRules: infrav1.IngressRules{ + infrav1.IngressRule{ + Description: "Kubernetes API", + Protocol: infrav1.SecurityGroupProtocolTCP, + FromPort: 6443, + ToPort: 6443, + CidrBlocks: []string{services.AnyIPv4CidrBlock}, + }, infrav1.IngressRule{ Description: "Kubernetes API IPv6", Protocol: infrav1.SecurityGroupProtocolTCP, @@ -1748,20 +1755,35 @@ func TestControlPlaneLoadBalancerIngressRules(t *testing.T) { }, NetworkSpec: infrav1.NetworkSpec{ VPC: infrav1.VPCSpec{ + CidrBlock: "10.0.0.0/16", IPv6: &infrav1.IPv6{ - CidrBlock: "10.0.0.0/16", + CidrBlock: "2001:1234:5678:9a40::/56", }, }, }, }, }, expectedIngresRules: infrav1.IngressRules{ + infrav1.IngressRule{ + Description: "Kubernetes API", + Protocol: infrav1.SecurityGroupProtocolTCP, + FromPort: 6443, + ToPort: 6443, + CidrBlocks: []string{"10.0.0.0/16"}, + }, infrav1.IngressRule{ Description: "Kubernetes API IPv6", Protocol: infrav1.SecurityGroupProtocolTCP, FromPort: 6443, ToPort: 6443, - IPv6CidrBlocks: []string{"10.0.0.0/16"}, + IPv6CidrBlocks: []string{"2001:1234:5678:9a40::/56"}, + }, + infrav1.IngressRule{ + Description: "Kubernetes API", + Protocol: infrav1.SecurityGroupProtocolTCP, + FromPort: 6443, + ToPort: 6443, + CidrBlocks: []string{services.AnyIPv4CidrBlock}, }, infrav1.IngressRule{ Description: "Kubernetes API IPv6", From 4171752af6cacb6a7d58e2ff61d5d2ba840e855a Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 29 Jul 2025 14:47:48 -0700 Subject: [PATCH 19/20] crd: clarify isIpv6 field on subnet spec The field isIpv6 is set to true if and only if the subnet has an associated IPv6 CIDR. This means the VPC is also associated with an IPv6 CIDR. --- api/v1beta1/network_types.go | 2 +- api/v1beta2/network_types.go | 2 +- ...ster.x-k8s.io_awsmanagedcontrolplanes.yaml | 6 +- ...8s.io_awsmanagedcontrolplanetemplates.yaml | 13 +- ...tructure.cluster.x-k8s.io_awsclusters.yaml | 6 +- ....cluster.x-k8s.io_awsclustertemplates.yaml | 4 +- docs/book/src/crd/index.md | 1578 ++++++++++++++--- 7 files changed, 1363 insertions(+), 248 deletions(-) diff --git a/api/v1beta1/network_types.go b/api/v1beta1/network_types.go index 7ff98f56f3..2f8f9795cb 100644 --- a/api/v1beta1/network_types.go +++ b/api/v1beta1/network_types.go @@ -259,7 +259,7 @@ type SubnetSpec struct { // +optional IsPublic bool `json:"isPublic"` - // IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with a VPC that has IPv6 enabled. + // IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with an IPv6 CIDR. // +optional IsIPv6 bool `json:"isIpv6,omitempty"` diff --git a/api/v1beta2/network_types.go b/api/v1beta2/network_types.go index 61478166a7..849ef34e49 100644 --- a/api/v1beta2/network_types.go +++ b/api/v1beta2/network_types.go @@ -596,7 +596,7 @@ type SubnetSpec struct { // +optional IsPublic bool `json:"isPublic"` - // IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with a VPC that has IPv6 enabled. + // IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with an IPv6 CIDR. // +optional IsIPv6 bool `json:"isIpv6,omitempty"` diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml index df9f3aa6b1..4b6ff573f3 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml @@ -603,8 +603,7 @@ spec: type: string isIpv6: description: IsIPv6 defines the subnet as an IPv6 subnet. - A subnet is IPv6 when it is associated with a VPC that - has IPv6 enabled. + A subnet is IPv6 when it is associated with an IPv6 CIDR. type: boolean isPublic: description: IsPublic defines the subnet as a public subnet. @@ -2840,8 +2839,7 @@ spec: type: string isIpv6: description: IsIPv6 defines the subnet as an IPv6 subnet. - A subnet is IPv6 when it is associated with a VPC that - has IPv6 enabled. + A subnet is IPv6 when it is associated with an IPv6 CIDR. type: boolean isPublic: description: IsPublic defines the subnet as a public subnet. diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml index ad5c56c54b..9de5f127d8 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml @@ -137,6 +137,7 @@ spec: description: |- AllowedCIDRBlocks is a list of CIDR blocks allowed to access the bastion host. They are set as ingress rules for the Bastion host's Security Group (defaults to 0.0.0.0/0). + If the cluster has IPv6 enabled, defaults to ::/0 and 0.0.0.0/0. items: type: string type: array @@ -620,12 +621,11 @@ spec: description: |- IPv6CidrBlock is the IPv6 CIDR block to be used when the provider creates a managed VPC. A subnet can have an IPv4 and an IPv6 address. - IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. type: string isIpv6: - description: |- - IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with a VPC that has IPv6 enabled. - IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object. + description: IsIPv6 defines the subnet as an IPv6 + subnet. A subnet is IPv6 when it is associated + with an IPv6 CIDR. type: boolean isPublic: description: IsPublic defines the subnet as a public @@ -807,9 +807,8 @@ spec: type: integer type: object ipv6: - description: |- - IPv6 contains ipv6 specific settings for the network. Supported only in managed clusters. - This field cannot be set on AWSCluster object. + description: IPv6 contains ipv6 specific settings + for the network. properties: cidrBlock: description: |- diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml index e3f30d6f4b..f69bd4926a 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml @@ -287,8 +287,7 @@ spec: type: string isIpv6: description: IsIPv6 defines the subnet as an IPv6 subnet. - A subnet is IPv6 when it is associated with a VPC that - has IPv6 enabled. + A subnet is IPv6 when it is associated with an IPv6 CIDR. type: boolean isPublic: description: IsPublic defines the subnet as a public subnet. @@ -1553,8 +1552,7 @@ spec: type: string isIpv6: description: IsIPv6 defines the subnet as an IPv6 subnet. - A subnet is IPv6 when it is associated with a VPC that - has IPv6 enabled. + A subnet is IPv6 when it is associated with an IPv6 CIDR. type: boolean isPublic: description: IsPublic defines the subnet as a public subnet. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml index 77aca1a9e5..c3d219903f 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml @@ -307,7 +307,7 @@ spec: isIpv6: description: IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated - with a VPC that has IPv6 enabled. + with an IPv6 CIDR. type: boolean isPublic: description: IsPublic defines the subnet as a public @@ -1137,7 +1137,7 @@ spec: isIpv6: description: IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated - with a VPC that has IPv6 enabled. + with an IPv6 CIDR. type: boolean isPublic: description: IsPublic defines the subnet as a public diff --git a/docs/book/src/crd/index.md b/docs/book/src/crd/index.md index 363550ffb5..f19020dd1c 100644 --- a/docs/book/src/crd/index.md +++ b/docs/book/src/crd/index.md @@ -5374,6 +5374,19 @@ string

ServiceAccountRoleArn is the ARN of an IAM role to bind to the addons service account

+ + +preserveOnDelete
+ +bool + + + +(Optional) +

PreserveOnDelete indicates that the addon resources should be +preserved in the cluster on delete.

+ +

AddonIssue @@ -6265,6 +6278,7 @@ AWSIdentityReference +(Optional)

IdentityRef is a reference to an identity to be used when reconciling the managed control plane. If no identity is specified, the default identity for this controller will be used.

@@ -6692,7 +6706,7 @@ AWSManagedControlPlaneStatus

AWSManagedControlPlaneSpec

-(Appears on:AWSManagedControlPlane) +(Appears on:AWSManagedControlPlane, AWSManagedControlPlaneTemplateResource)

AWSManagedControlPlaneSpec defines the desired state of an Amazon EKS Cluster.

@@ -6729,6 +6743,7 @@ AWSIdentityReference +(Optional)

IdentityRef is a reference to an identity to be used when reconciling the managed control plane. If no identity is specified, the default identity for this controller will be used.

@@ -7317,10 +7332,10 @@ in the cluster.

-

Addon +

AWSManagedControlPlaneTemplate

-

Addon represents a EKS addon.

+

AWSManagedControlPlaneTemplate is the Schema for the AWSManagedControlPlaneTemplates API.

@@ -7332,73 +7347,55 @@ in the cluster.

- - - - - +
+
+
-name
- -string - -		 -

Name is the name of the addon

-
-version
+metadata
-string + +Kubernetes meta/v1.ObjectMeta + 		-

Version is the version of the addon to use

+Refer to the Kubernetes API documentation for the fields of the +metadata field.
-configuration
+spec
-string + +AWSManagedControlPlaneTemplateSpec + 		-(Optional) -

Configuration of the EKS addon

-
- - -
-conflictResolution
+template
- -AddonResolution + +AWSManagedControlPlaneTemplateResource 		-

ConflictResolution is used to declare what should happen if there -are parameter conflicts. Defaults to none
-serviceAccountRoleARN
- -string - -		 -(Optional) -

ServiceAccountRoleArn is the ARN of an IAM role to bind to the addons service account

+
-

AddonIssue +

AWSManagedControlPlaneTemplateResource

-(Appears on:AddonState) +(Appears on:AWSManagedControlPlaneTemplateSpec)

-

AddonIssue represents an issue with an addon.

+

AWSManagedControlPlaneTemplateResource describes the data needed to create an AWSManagedCluster from a template.

@@ -7410,229 +7407,813 @@ string - +
+
+
-code
+spec
-string + +AWSManagedControlPlaneSpec + 		-

Code is the issue code

-
- -
-message
+eksClusterName
string 		-

Message is the textual description of the issue

+(Optional) +

EKSClusterName allows you to specify the name of the EKS cluster in +AWS. If you don’t specify a name then a default name will be created +based on the namespace and name of the managed control plane.
-resourceIds
+identityRef
-[]string + +AWSIdentityReference + 		-

ResourceIDs is a list of resource ids for the issue

+(Optional) +

IdentityRef is a reference to an identity to be used when reconciling the managed control plane. +If no identity is specified, the default identity for this controller will be used.
-

AddonResolution -(string alias)

-

-(Appears on:Addon) -

-

-

AddonResolution defines the method for resolving parameter conflicts.

-

-

AddonState -

-

-(Appears on:AWSManagedControlPlaneStatus) -

-

-

AddonState represents the state of an addon.

-

- - - - - - - - - -
FieldDescription
-name
+network
-string + +NetworkSpec + 		-

Name is the name of the addon

+

NetworkSpec encapsulates all things related to AWS network.
-version
+secondaryCidrBlock
string 		-

Version is the version of the addon to use

+(Optional) +

SecondaryCidrBlock is the additional CIDR range to use for pod IPs. +Must be within the 100.64.0.0/10 or 198.19.0.0/16 range.
-arn
+region
string 		-

ARN is the AWS ARN of the addon

+

The AWS Region the cluster lives in.
-serviceAccountRoleARN
+partition
string 		-

ServiceAccountRoleArn is the ARN of the IAM role used for the service account

+(Optional) +

Partition is the AWS security partition being used. Defaults to “aws”
-createdAt
+sshKeyName
- -Kubernetes meta/v1.Time - +string 		-

CreatedAt is the date and time the addon was created at

+(Optional) +

SSHKeyName is the name of the ssh key to attach to the bastion host. Valid values are empty string (do not use SSH keys), a valid SSH key name, or omitted (use the default SSH key name)
-modifiedAt
+version
- -Kubernetes meta/v1.Time - +string 		-

ModifiedAt is the date and time the addon was last modified

+(Optional) +

Version defines the desired Kubernetes version. If no version number +is supplied then the latest version of Kubernetes that EKS supports +will be used.
-status
+roleName
string 		-

Status is the status of the addon

+(Optional) +

RoleName specifies the name of IAM role that gives EKS +permission to make API calls. If the role is pre-existing +we will treat it as unmanaged and not delete it on +deletion. If the EKSEnableIAM feature flag is true +and no name is supplied then a role is created.
-issues
+roleAdditionalPolicies
- -[]AddonIssue - +[]string 		-

Issues is a list of issue associated with the addon

+(Optional) +

RoleAdditionalPolicies allows you to attach additional polices to +the control plane role. You must enable the EKSAllowAddRoles +feature flag to incorporate these into the created role.
-

AddonStatus -(string alias)

-

-

AddonStatus defines the status for an addon.

-

-

ControlPlaneLoggingSpec -

-

-(Appears on:AWSManagedControlPlaneSpec) -

-

-

ControlPlaneLoggingSpec defines what EKS control plane logs that should be enabled.

-

- - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
-apiServer
+rolePath
-bool +string 		-

APIServer indicates if the Kubernetes API Server log (kube-apiserver) shoulkd be enabled

+(Optional) +

RolePath sets the path to the role. For more information about paths, see IAM Identifiers +(https://docs.aws.amazon.com/IAM/latest/UserGuide/Using_Identifiers.html) +in the IAM User Guide.

+

This parameter is optional. If it is not included, it defaults to a slash +(/).
-audit
+rolePermissionsBoundary
-bool +string 		-

Audit indicates if the Kubernetes API audit log should be enabled

+(Optional) +

RolePermissionsBoundary sets the ARN of the managed policy that is used +to set the permissions boundary for the role.

+

A permissions boundary policy defines the maximum permissions that identity-based +policies can grant to an entity, but does not grant permissions. Permissions +boundaries do not define the maximum permissions that a resource-based policy +can grant to an entity. To learn more, see Permissions boundaries for IAM +entities (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_boundaries.html) +in the IAM User Guide.

+

For more information about policy types, see Policy types (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#access_policy-types) +in the IAM User Guide.
-authenticator
+logging
-bool + +ControlPlaneLoggingSpec + 		-

Authenticator indicates if the iam authenticator log should be enabled

+(Optional) +

Logging specifies which EKS Cluster logs should be enabled. Entries for +each of the enabled logs will be sent to CloudWatch
-controllerManager
+encryptionConfig
-bool + +EncryptionConfig + 		-

ControllerManager indicates if the controller manager (kube-controller-manager) log should be enabled

+(Optional) +

EncryptionConfig specifies the encryption configuration for the cluster
-scheduler
+additionalTags
-bool + +Tags + + +		 +(Optional) +

AdditionalTags is an optional set of tags to add to AWS resources managed by the AWS provider, in addition to the +ones added by default.

+
+iamAuthenticatorConfig
+ + +IAMAuthenticatorConfig + + +		 +(Optional) +

IAMAuthenticatorConfig allows the specification of any additional user or role mappings +for use when generating the aws-iam-authenticator configuration. If this is nil the +default configuration is still generated for the cluster.

+
+endpointAccess
+ + +EndpointAccess + + +		 +(Optional) +

Endpoints specifies access to this cluster’s control plane endpoints

+
+controlPlaneEndpoint
+ + +Cluster API api/v1beta1.APIEndpoint + + +		 +(Optional) +

ControlPlaneEndpoint represents the endpoint used to communicate with the control plane.

+
+imageLookupFormat
+ +string + +		 +(Optional) +

ImageLookupFormat is the AMI naming format to look up machine images when +a machine does not specify an AMI. When set, this will be used for all +cluster machines unless a machine specifies a different ImageLookupOrg. +Supports substitutions for {{.BaseOS}} and {{.K8sVersion}} with the base +OS and kubernetes version, respectively. The BaseOS will be the value in +ImageLookupBaseOS or ubuntu (the default), and the kubernetes version as +defined by the packages produced by kubernetes/release without v as a +prefix: 1.13.0, 1.12.5-mybuild.1, or 1.17.3. For example, the default +image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* will end up +searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* for a +Machine that is targeting kubernetes v1.18.0 and the ubuntu base OS. See +also: https://golang.org/pkg/text/template/

+
+imageLookupOrg
+ +string + +		 +(Optional) +

ImageLookupOrg is the AWS Organization ID to look up machine images when a +machine does not specify an AMI. When set, this will be used for all +cluster machines unless a machine specifies a different ImageLookupOrg.

+
+imageLookupBaseOS
+ +string + +		 +

ImageLookupBaseOS is the name of the base operating system used to look +up machine images when a machine does not specify an AMI. When set, this +will be used for all cluster machines unless a machine specifies a +different ImageLookupBaseOS.

+
+bastion
+ + +Bastion + + +		 +(Optional) +

Bastion contains options to configure the bastion host.

+
+tokenMethod
+ + +EKSTokenMethod + + +		 +

TokenMethod is used to specify the method for obtaining a client token for communicating with EKS +iam-authenticator - obtains a client token using iam-authentictor +aws-cli - obtains a client token using the AWS CLI +Defaults to iam-authenticator

+
+associateOIDCProvider
+ +bool + +		 +

AssociateOIDCProvider can be enabled to automatically create an identity +provider for the controller for use with IAM roles for service accounts

+
+addons
+ + +[]sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2.Addon + + +		 +(Optional) +

Addons defines the EKS addons to enable with the EKS cluster.

+
+oidcIdentityProviderConfig
+ + +OIDCIdentityProviderConfig + + +		 +(Optional) +

IdentityProviderconfig is used to specify the oidc provider config +to be attached with this eks cluster

+
+vpcCni
+ + +VpcCni + + +		 +(Optional) +

VpcCni is used to set configuration options for the VPC CNI plugin

+
+bootstrapSelfManagedAddons
+ +bool + +		 +

BootstrapSelfManagedAddons is used to set configuration options for +bare EKS cluster without EKS default networking addons +If you set this value to false when creating a cluster, the default networking add-ons will not be installed

+
+restrictPrivateSubnets
+ +bool + +		 +

RestrictPrivateSubnets indicates that the EKS control plane should only use private subnets.

+
+kubeProxy
+ + +KubeProxy + + +		 +

KubeProxy defines managed attributes of the kube-proxy daemonset

+
+ + + + +

AWSManagedControlPlaneTemplateSpec +

+

+(Appears on:AWSManagedControlPlaneTemplate) +

+

+

AWSManagedControlPlaneTemplateSpec defines the desired state of AWSManagedControlPlaneTemplate.

+

+ + + + + + + + + + + + + +
FieldDescription
+template
+ + +AWSManagedControlPlaneTemplateResource + + +		 +
+

Addon +

+

+

Addon represents a EKS addon.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+name
+ +string + +		 +

Name is the name of the addon

+
+version
+ +string + +		 +

Version is the version of the addon to use

+
+configuration
+ +string + +		 +(Optional) +

Configuration of the EKS addon

+
+conflictResolution
+ + +AddonResolution + + +		 +

ConflictResolution is used to declare what should happen if there +are parameter conflicts. Defaults to overwrite

+
+serviceAccountRoleARN
+ +string + +		 +(Optional) +

ServiceAccountRoleArn is the ARN of an IAM role to bind to the addons service account

+
+preserveOnDelete
+ +bool + +		 +(Optional) +

PreserveOnDelete indicates that the addon resources should be +preserved in the cluster on delete.

+
+

AddonIssue +

+

+(Appears on:AddonState) +

+

+

AddonIssue represents an issue with an addon.

+

+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+code
+ +string + +		 +

Code is the issue code

+
+message
+ +string + +		 +

Message is the textual description of the issue

+
+resourceIds
+ +[]string + +		 +

ResourceIDs is a list of resource ids for the issue

+
+

AddonResolution +(string alias)

+

+(Appears on:Addon) +

+

+

AddonResolution defines the method for resolving parameter conflicts.

+

+

AddonState +

+

+(Appears on:AWSManagedControlPlaneStatus) +

+

+

AddonState represents the state of an addon.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+name
+ +string + +		 +

Name is the name of the addon

+
+version
+ +string + +		 +

Version is the version of the addon to use

+
+arn
+ +string + +		 +

ARN is the AWS ARN of the addon

+
+serviceAccountRoleARN
+ +string + +		 +

ServiceAccountRoleArn is the ARN of the IAM role used for the service account

+
+createdAt
+ + +Kubernetes meta/v1.Time + + +		 +

CreatedAt is the date and time the addon was created at

+
+modifiedAt
+ + +Kubernetes meta/v1.Time + + +		 +

ModifiedAt is the date and time the addon was last modified

+
+status
+ +string + +		 +

Status is the status of the addon

+
+issues
+ + +[]AddonIssue + + +		 +

Issues is a list of issue associated with the addon

+
+

AddonStatus +(string alias)

+

+

AddonStatus defines the status for an addon.

+

+

ControlPlaneLoggingSpec +

+

+(Appears on:AWSManagedControlPlaneSpec) +

+

+

ControlPlaneLoggingSpec defines what EKS control plane logs that should be enabled.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -15078,8 +15665,7 @@ string @@ -15114,8 +15700,7 @@ bool @@ -19361,6 +19946,14 @@ AWSClusterTemplateResource
FieldDescription
+apiServer
+ +bool + +		 +

APIServer indicates if the Kubernetes API Server log (kube-apiserver) shoulkd be enabled

+
+audit
+ +bool + +		 +

Audit indicates if the Kubernetes API audit log should be enabled

+
+authenticator
+ +bool + +		 +

Authenticator indicates if the iam authenticator log should be enabled

+
+controllerManager
+ +bool + +		 +

ControllerManager indicates if the controller manager (kube-controller-manager) log should be enabled

+
+scheduler
+ +bool 		@@ -8682,6 +9263,12 @@ string

"candidate"

Candidate channel group is for testing candidate builds.

"eus"

Eus channel group is for eus channel releases.

+

"fast"

Fast channel group is for fast channel releases.

+

"nightly"

Nightly channel group is for testing nigtly builds.

 (Optional)

IPv6CidrBlock is the IPv6 CIDR block to be used when the provider creates a managed VPC. -A subnet can have an IPv4 and an IPv6 address. -IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object.

+A subnet can have an IPv4 and an IPv6 address.
(Optional) -

IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with a VPC that has IPv6 enabled. -IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object.

+

IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with an IPv6 CIDR.
+

AWSConfidentialComputePolicy +(string alias)

+

+(Appears on:CPUOptions) +

+

+

AWSConfidentialComputePolicy represents the confidential compute configuration for the instance.

+

AWSIdentityKind (string alias)

@@ -19731,13 +20324,28 @@ image lookup the AMI is not set.

-instanceType
+instanceType
+ +string + + + +

InstanceType is the type of instance to create. Example: m4.xlarge

+ + + + +cpuOptions,omitempty,omitzero
-string + +CPUOptions + -

InstanceType is the type of instance to create. Example: m4.xlarge

+(Optional) +

CPUOptions defines CPU-related settings for the instance, including the confidential computing policy. +When omitted, this means no opinion and the AWS platform is left to choose a reasonable default.

@@ -20051,6 +20659,50 @@ If this value is selected, CapacityReservationID must be specified to identify t If marketType is not specified and spotMarketOptions is provided, the marketType defaults to “Spot”.

+ + +hostID
+ +string + + + +(Optional) +

HostID specifies the Dedicated Host on which the instance must be started.

+ + + + +hostAffinity
+ +string + + + +(Optional) +

HostAffinity specifies the dedicated host affinity setting for the instance. +When hostAffinity is set to host, an instance started onto a specific host always restarts on the same host if stopped. +When hostAffinity is set to default, and you stop and restart the instance, it can be restarted on any available host. +When HostAffinity is defined, HostID is required.

+ + + + +capacityReservationPreference
+ + +CapacityReservationPreference + + + + +(Optional) +

CapacityReservationPreference specifies the preference for use of Capacity Reservations by the instance. Valid values include: +“Open”: The instance may make use of open Capacity Reservations that match its AZ and InstanceType +“None”: The instance may not make use of any Capacity Reservations. This is to conserve open reservations for desired workloads +“CapacityReservationsOnly”: The instance will only run if matched or targeted to a Capacity Reservation. Note that this is incompatible with a MarketType of Spot

+ + @@ -20196,6 +20848,21 @@ string +cpuOptions,omitempty,omitzero
+ + +CPUOptions + + + + +(Optional) +

CPUOptions defines CPU-related settings for the instance, including the confidential computing policy. +When omitted, this means no opinion and the AWS platform is left to choose a reasonable default.

+ + + + additionalTags
@@ -20505,6 +21172,50 @@ If this value is selected, CapacityReservationID must be specified to identify t If marketType is not specified and spotMarketOptions is provided, the marketType defaults to “Spot”.

+ + +hostID
+ +string + + + +(Optional) +

HostID specifies the Dedicated Host on which the instance must be started.

+ + + + +hostAffinity
+ +string + + + +(Optional) +

HostAffinity specifies the dedicated host affinity setting for the instance. +When hostAffinity is set to host, an instance started onto a specific host always restarts on the same host if stopped. +When hostAffinity is set to default, and you stop and restart the instance, it can be restarted on any available host. +When HostAffinity is defined, HostID is required.

+ + + + +capacityReservationPreference
+ + +CapacityReservationPreference + + + + +(Optional) +

CapacityReservationPreference specifies the preference for use of Capacity Reservations by the instance. Valid values include: +“Open”: The instance may make use of open Capacity Reservations that match its AZ and InstanceType +“None”: The instance may not make use of any Capacity Reservations. This is to conserve open reservations for desired workloads +“CapacityReservationsOnly”: The instance will only run if matched or targeted to a Capacity Reservation. Note that this is incompatible with a MarketType of Spot

+ +

AWSMachineStatus @@ -20864,6 +21575,21 @@ string +cpuOptions,omitempty,omitzero
+ + +CPUOptions + + + + +(Optional) +

CPUOptions defines CPU-related settings for the instance, including the confidential computing policy. +When omitted, this means no opinion and the AWS platform is left to choose a reasonable default.

+ + + + additionalTags
@@ -21173,6 +21899,50 @@ If this value is selected, CapacityReservationID must be specified to identify t If marketType is not specified and spotMarketOptions is provided, the marketType defaults to “Spot”.

+ + +hostID
+ +string + + + +(Optional) +

HostID specifies the Dedicated Host on which the instance must be started.

+ + + + +hostAffinity
+ +string + + + +(Optional) +

HostAffinity specifies the dedicated host affinity setting for the instance. +When hostAffinity is set to host, an instance started onto a specific host always restarts on the same host if stopped. +When hostAffinity is set to default, and you stop and restart the instance, it can be restarted on any available host. +When HostAffinity is defined, HostID is required.

+ + + + +capacityReservationPreference
+ + +CapacityReservationPreference + + + + +(Optional) +

CapacityReservationPreference specifies the preference for use of Capacity Reservations by the instance. Valid values include: +“Open”: The instance may make use of open Capacity Reservations that match its AZ and InstanceType +“None”: The instance may not make use of any Capacity Reservations. This is to conserve open reservations for desired workloads +“CapacityReservationsOnly”: The instance will only run if matched or targeted to a Capacity Reservation. Note that this is incompatible with a MarketType of Spot

+ + @@ -21214,7 +21984,150 @@ AWSMachineTemplateResource (Appears on:AWSMachineTemplate)

-

AWSMachineTemplateStatus defines a status for an AWSMachineTemplate.

+

AWSMachineTemplateStatus defines a status for an AWSMachineTemplate.

+

+ + + + + + + + + + + + + +
FieldDescription
+capacity
+ + +Kubernetes core/v1.ResourceList + + +		 +(Optional) +

Capacity defines the resource capacity for this machine. +This value is used for autoscaling from zero operations as defined in: +https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20210310-opt-in-autoscaling-from-zero.md

+
+

AWSMachineTemplateWebhook +

+

+

AWSMachineTemplateWebhook implements a custom validation webhook for AWSMachineTemplate. +Note: we use a custom validator to access the request context for SSA of AWSMachineTemplate.

+

+

AWSManagedCluster +

+

+

AWSManagedCluster is the Schema for the awsmanagedclusters API

+

+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+metadata
+ + +Kubernetes meta/v1.ObjectMeta + + +		 +Refer to the Kubernetes API documentation for the fields of the +metadata field. +
+spec
+ + +AWSManagedClusterSpec + + +		 +
+
+ + + + + +
+controlPlaneEndpoint
+ + +Cluster API api/v1beta1.APIEndpoint + + +		 +(Optional) +

ControlPlaneEndpoint represents the endpoint used to communicate with the control plane.

+
+
+status
+ + +AWSManagedClusterStatus + + +		 +
+

AWSManagedClusterSpec +

+

+(Appears on:AWSManagedCluster, AWSManagedClusterTemplateResource) +

+

+

AWSManagedClusterSpec defines the desired state of AWSManagedCluster

+

+ + + + + + + + + + + + + +
FieldDescription
+controlPlaneEndpoint
+ + +Cluster API api/v1beta1.APIEndpoint + + +		 +(Optional) +

ControlPlaneEndpoint represents the endpoint used to communicate with the control plane.

+
+

AWSManagedClusterStatus +

+

+(Appears on:AWSManagedCluster) +

+

+

AWSManagedClusterStatus defines the observed state of AWSManagedCluster

@@ -21226,32 +22139,50 @@ AWSMachineTemplateResource + + + + + + + +
-capacity
+ready
- -Kubernetes core/v1.ResourceList +bool + +		 +(Optional) +

Ready is when the AWSManagedControlPlane has a API server URL.

+
+failureDomains
+ + +Cluster API api/v1beta1.FailureDomains 		(Optional) -

Capacity defines the resource capacity for this machine. -This value is used for autoscaling from zero operations as defined in: -https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20210310-opt-in-autoscaling-from-zero.md

+

FailureDomains specifies a list fo available availability zones that can be used

+
+conditions
+ + +Cluster API api/v1beta1.Conditions + + +		 +(Optional) +

Conditions defines current service state of the AWSManagedCluster.
-

AWSMachineTemplateWebhook +

AWSManagedClusterTemplate

-

AWSMachineTemplateWebhook implements a custom validation webhook for AWSMachineTemplate. -Note: we use a custom validator to access the request context for SSA of AWSMachineTemplate.

-

-

AWSManagedCluster -

-

-

AWSManagedCluster is the Schema for the awsmanagedclusters API

+

AWSManagedClusterTemplate is the Schema for the AWSManagedClusterTemplates API.

@@ -21279,8 +22210,8 @@ Refer to the Kubernetes API documentation for the fields of the @@ -21290,42 +22221,28 @@ AWSManagedClusterSpec
spec
- -AWSManagedClusterSpec + +AWSManagedClusterTemplateSpec
-controlPlaneEndpoint
+template
- -Cluster API api/v1beta1.APIEndpoint + +AWSManagedClusterTemplateResource 		-(Optional) -

ControlPlaneEndpoint represents the endpoint used to communicate with the control plane.
- - -status
- - -AWSManagedClusterStatus - - - - - - -

AWSManagedClusterSpec +

AWSManagedClusterTemplateResource

-(Appears on:AWSManagedCluster) +(Appears on:AWSManagedClusterTemplateSpec)

-

AWSManagedClusterSpec defines the desired state of AWSManagedCluster

+

AWSManagedClusterTemplateResource describes the data needed to create an AWSManagedCluster from a template.

@@ -21337,6 +22254,19 @@ AWSManagedClusterStatus + +
+spec
+ + +AWSManagedClusterSpec + + +		 +
+
+ + + +
controlPlaneEndpoint
@@ -21349,15 +22279,18 @@ Cluster API api/v1beta1.APIEndpoint

ControlPlaneEndpoint represents the endpoint used to communicate with the control plane.
+
-

AWSManagedClusterStatus +

AWSManagedClusterTemplateSpec

-(Appears on:AWSManagedCluster) +(Appears on:AWSManagedClusterTemplate)

-

AWSManagedClusterStatus defines the observed state of AWSManagedCluster

+

AWSManagedClusterTemplateSpec defines the desired state of AWSManagedClusterTemplate.

@@ -21369,42 +22302,14 @@ Cluster API api/v1beta1.APIEndpoint - - - - - - - - @@ -21692,13 +22597,16 @@ Requires AllowedCIDRBlocks to be empty.

@@ -21914,6 +22822,66 @@ The source for the rule will be set to control plane and worker security group I
-ready
- -bool - -		 -(Optional) -

Ready is when the AWSManagedControlPlane has a API server URL.

-
-failureDomains
- - -Cluster API api/v1beta1.FailureDomains - - -		 -(Optional) -

FailureDomains specifies a list fo available availability zones that can be used

-
-conditions
+template
- -Cluster API api/v1beta1.Conditions + +AWSManagedClusterTemplateResource 		-(Optional) -

Conditions defines current service state of the AWSManagedCluster.
allowedCIDRBlocks
-[]string + +CidrBlocks + 		(Optional)

AllowedCIDRBlocks is a list of CIDR blocks allowed to access the bastion host. -They are set as ingress rules for the Bastion host’s Security Group (defaults to 0.0.0.0/0).

+They are set as ingress rules for the Bastion host’s Security Group (defaults to 0.0.0.0/0). +If the cluster has IPv6 enabled, defaults to ::/0 and 0.0.0.0/0.
+

CPUOptions +

+

+(Appears on:AWSMachineSpec, Instance) +

+

+

CPUOptions defines CPU-related settings for the instance, including the confidential computing policy.

+

+ + + + + + + + + + + + + +
FieldDescription
+confidentialCompute
+ + +AWSConfidentialComputePolicy + + +		 +(Optional) +

ConfidentialCompute specifies whether confidential computing should be enabled for the instance, +and, if so, which confidential computing technology to use. +Valid values are: Disabled, AMDEncryptedVirtualizationNestedPaging +When set to Disabled, confidential computing will be disabled for the instance. +When set to AMDEncryptedVirtualizationNestedPaging, AMD SEV-SNP will be used as the confidential computing technology for the instance. +In this case, ensure the following conditions are met: +1) The selected instance type supports AMD SEV-SNP. +2) The selected AWS region supports AMD SEV-SNP. +3) The selected AMI supports AMD SEV-SNP. +More details can be checked at https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/sev-snp.html +When omitted, this means no opinion and the AWS platform is left to choose a reasonable default, +which is subject to change without notice. The current default is Disabled.

+
+

CapacityReservationPreference +(string alias)

+

+(Appears on:AWSMachineSpec, Instance, AWSLaunchTemplate) +

+

+

CapacityReservationPreference describes the preferred use of capacity reservations +of an instance

+

+

CidrBlocks +([]string alias)

+

+(Appears on:Bastion, NetworkSpec) +

+

+

CidrBlocks defines a set of CIDR blocks.

+

ClassicELBAttributes

@@ -22447,7 +23415,9 @@ string (Optional) -

Version defines which version of Ignition will be used to generate bootstrap data.

+

Version defines which version of Ignition will be used to generate bootstrap data. +Defaults to 2.3 if storageType is set to ClusterObjectStore. +It will be ignored if storageType is set to UnencryptedUserData, as the userdata defines its own version.

@@ -22911,6 +23881,17 @@ string +ipv6Address
+ +string + + + +

The IPv6 address assigned to the instance.

+ + + + publicIp
string @@ -23150,6 +24131,65 @@ If this value is selected, CapacityReservationID must be specified to identify t If marketType is not specified and spotMarketOptions is provided, the marketType defaults to “Spot”.

+ + +hostAffinity
+ +string + + + +(Optional) +

HostAffinity specifies the dedicated host affinity setting for the instance. +When hostAffinity is set to host, an instance started onto a specific host always restarts on the same host if stopped. +When hostAffinity is set to default, and you stop and restart the instance, it can be restarted on any available host. +When HostAffinity is defined, HostID is required.

+ + + + +hostID
+ +string + + + +(Optional) +

HostID specifies the dedicated host on which the instance should be started.

+ + + + +capacityReservationPreference
+ + +CapacityReservationPreference + + + + +(Optional) +

CapacityReservationPreference specifies the preference for use of Capacity Reservations by the instance. Valid values include: +“Open”: The instance may make use of open Capacity Reservations that match its AZ and InstanceType +“None”: The instance may not make use of any Capacity Reservations. This is to conserve open reservations for desired workloads +“CapacityReservationsOnly”: The instance will only run if matched or targeted to a Capacity Reservation. Note that this is incompatible with a MarketType of Spot

+ + + + +cpuOptions,omitempty,omitzero
+ + +CPUOptions + + + + +(Optional) +

CPUOptions defines CPU-related settings for the instance, including the confidential computing policy. +When omitted, this means no opinion and the AWS platform is left to choose a reasonable default.

+ +

InstanceMetadataOptions @@ -23185,6 +24225,21 @@ InstanceMetadataState +httpProtocolIpv6
+ + +InstanceMetadataState + + + + +

Enables or disables the IPv6 endpoint for the instance metadata service. +This applies only if you enabled the HTTP metadata endpoint.

+

Default: disabled

+ + + + httpPutResponseHopLimit
int64 @@ -23625,7 +24680,9 @@ This is optional - if not provided new security groups will be created for the c nodePortIngressRuleCidrBlocks
-[]string + +CidrBlocks + @@ -24081,8 +25138,7 @@ string (Optional)

IPv6CidrBlock is the IPv6 CIDR block to be used when the provider creates a managed VPC. -A subnet can have an IPv4 and an IPv6 address. -IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object.

+A subnet can have an IPv4 and an IPv6 address.

@@ -24117,8 +25173,7 @@ bool (Optional) -

IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with a VPC that has IPv6 enabled. -IPv6 is only supported in managed clusters, this field cannot be set on AWSCluster object.

+

IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with an IPv6 CIDR.

@@ -24649,8 +25704,7 @@ IPv6 (Optional) -

IPv6 contains ipv6 specific settings for the network. Supported only in managed clusters. -This field cannot be set on AWSCluster object.

+

IPv6 contains ipv6 specific settings for the network.

@@ -25375,6 +26429,23 @@ If this value is selected, CapacityReservationID must be specified to identify t If marketType is not specified and spotMarketOptions is provided, the marketType defaults to “Spot”.

+ + +capacityReservationPreference
+ + +CapacityReservationPreference + + + + +(Optional) +

CapacityReservationPreference specifies the preference for use of Capacity Reservations by the instance. Valid values include: +“Open”: The instance may make use of open Capacity Reservations that match its AZ and InstanceType +“None”: The instance may not make use of any Capacity Reservations. This is to conserve open reservations for desired workloads +“CapacityReservationsOnly”: The instance will only run if matched or targeted to a Capacity Reservation

+ +

AWSLifecycleHook @@ -26225,6 +27296,11 @@ ASGStatus +

AWSMachinePoolWebhook +

+

+

AWSMachinePoolWebhook implements a custom validation webhook for AWSMachinePool.

+

AWSManagedMachinePool

@@ -27735,7 +28811,9 @@ int64 (Appears on:AWSManagedMachinePoolSpec)

-

ManagedMachineAMIType specifies which AWS AMI to use for a managed MachinePool.

+

ManagedMachineAMIType specifies which AWS AMI to use for a managed MachinePool. +Source of truth can be found using the link below: +https://docs.aws.amazon.com/eks/latest/APIReference/API_CreateNodegroup.html#AmazonEKS-CreateNodegroup-request-amiType

@@ -27747,9 +28825,18 @@ int64 + + + + + + @@ -27759,6 +28846,39 @@ int64 + + + + + + + + + + + + + + + + + + + + + +

"AL2023_ARM_64_STANDARD"

Al2023Arm64 is the AL2023 Arm AMI type.

"AL2023_ARM_64_NVIDIA"

Al2023Arm64Nvidia is the AL2023 Arm Nvidia AMI type.

+

"AL2023_x86_64_STANDARD"

Al2023x86_64 is the AL2023 x86-64 AMI type.

"AL2023_x86_64_NEURON"

Al2023x86_64Neuron is the AL2023 x86-64 Neuron AMI type.

+

"AL2023_x86_64_NVIDIA"

Al2023x86_64Nvidia is the AL2023 x86-64 Nvidia AMI type.

+

"AL2_ARM_64"

Al2Arm64 is the Arm AMI type.

"AL2_x86_64_GPU"

Al2x86_64GPU is the x86-64 GPU AMI type.

"BOTTLEROCKET_ARM_64"

BottleRocketArm64 is the Arm AMI type.

+

"BOTTLEROCKET_ARM_64_FIPS"

BottleRocketArm64Fips is the BottleRocket Arm Fips AMI type.

+

"BOTTLEROCKET_ARM_64_NVIDIA"

BottleRocketArm64Nvidia is the BottleRocket Arm Nvidia AMI type.

+

"BOTTLEROCKET_x86_64"

BottleRocketx86_64 is the BottleRocket x86-64 AMI type.

+

"BOTTLEROCKET_x86_64_FIPS"

BottleRocketx86_64Fips is the BottleRocket x86-64 Fips AMI type.

+

"BOTTLEROCKET_x86_64_NVIDIA"

BottleRocketx86_64Nvidia is the BottleRocket x86-64 Nvidia AMI type.

+

"CUSTOM"

Custom is the custom AMI type.

+

"WINDOWS_CORE_2019_x86_64"

WindowsCore2019x86_64 is the Windows Core 2019 x86-64 AMI type.

+

"WINDOWS_CORE_2022_x86_64"

WindowsCore2022x86_64 is the Windows Core 2022 x86-64 AMI type.

+

"WINDOWS_FULL_2019_x86_64"

WindowsFull2019x86_64 is the Windows Full 2019 x86-64 AMI type.

+

"WINDOWS_FULL_2022_x86_64"

WindowsFull2022x86_64 is the Windows Full 2022 x86-64 AMI type.

+

ManagedMachinePoolCapacityType From 3abd5a128eabe69fffe5a75bc12948889d68c8d1 Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Wed, 1 Oct 2025 22:55:19 -0700 Subject: [PATCH 20/20] api: add spec field to configure target group ipType The field targetGroupIPType is added to the loadbalancer spec to allow configuring ip address type of target group for API load balancers. This field is not applicable to Classic Load Balancers (CLB). This commit also defines a new network status field to determine the ip type of API load balancers. --- api/v1beta1/awscluster_conversion.go | 2 + api/v1beta1/zz_generated.conversion.go | 1 + api/v1beta2/awscluster_types.go | 17 + api/v1beta2/awscluster_webhook.go | 52 + api/v1beta2/awscluster_webhook_test.go | 152 +++ api/v1beta2/network_types.go | 53 + api/v1beta2/zz_generated.deepcopy.go | 10 + ...ster.x-k8s.io_awsmanagedcontrolplanes.yaml | 48 + ...tructure.cluster.x-k8s.io_awsclusters.yaml | 66 ++ ....cluster.x-k8s.io_awsclustertemplates.yaml | 42 + docs/book/src/crd/index.md | 960 ++++++++++++++++-- pkg/cloud/services/elb/loadbalancer.go | 66 +- pkg/cloud/services/elb/loadbalancer_test.go | 17 + 13 files changed, 1395 insertions(+), 91 deletions(-) diff --git a/api/v1beta1/awscluster_conversion.go b/api/v1beta1/awscluster_conversion.go index 9878db8ab1..f8542e042b 100644 --- a/api/v1beta1/awscluster_conversion.go +++ b/api/v1beta1/awscluster_conversion.go @@ -156,6 +156,7 @@ func (src *AWSCluster) ConvertTo(dstRaw conversion.Hub) error { func restoreControlPlaneLoadBalancerStatus(restored, dst *infrav1.LoadBalancer) { dst.ARN = restored.ARN dst.LoadBalancerType = restored.LoadBalancerType + dst.LoadBalancerIPAddressType = restored.LoadBalancerIPAddressType dst.ELBAttributes = restored.ELBAttributes dst.ELBListeners = restored.ELBListeners dst.Name = restored.Name @@ -193,6 +194,7 @@ func restoreControlPlaneLoadBalancer(restored, dst *infrav1.AWSLoadBalancerSpec) dst.Scheme = restored.Scheme dst.CrossZoneLoadBalancing = restored.CrossZoneLoadBalancing dst.Subnets = restored.Subnets + dst.TargetGroupIPType = restored.TargetGroupIPType } // ConvertFrom converts the v1beta1 AWSCluster receiver to a v1beta1 AWSCluster. diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index baf626e19e..617aa5346b 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -1251,6 +1251,7 @@ func autoConvert_v1beta2_AWSLoadBalancerSpec_To_v1beta1_AWSLoadBalancerSpec(in * // WARNING: in.LoadBalancerType requires manual conversion: does not exist in peer-type // WARNING: in.DisableHostsRewrite requires manual conversion: does not exist in peer-type // WARNING: in.PreserveClientIP requires manual conversion: does not exist in peer-type + // WARNING: in.TargetGroupIPType requires manual conversion: does not exist in peer-type return nil } diff --git a/api/v1beta2/awscluster_types.go b/api/v1beta2/awscluster_types.go index 13b2c389e0..dff88ab765 100644 --- a/api/v1beta2/awscluster_types.go +++ b/api/v1beta2/awscluster_types.go @@ -253,6 +253,15 @@ type AWSLoadBalancerSpec struct { // PreserveClientIP lets the user control if preservation of client ips must be retained or not. // If this is enabled 6443 will be opened to 0.0.0.0/0. PreserveClientIP bool `json:"preserveClientIP,omitempty"` + + // TargetGroupIPType sets the IP address type for the target group. + // Valid values are ipv4 and ipv6. If not specified, defaults to ipv4 unless + // the VPC has IPv6 enabled, in which case it defaults to ipv6. + // This applies to the API server target group. + // This field cannot be set if LoadBalancerType is classic or disabled. + // +kubebuilder:validation:Enum=ipv4;ipv6 + // +optional + TargetGroupIPType *TargetGroupIPType `json:"targetGroupIPType,omitempty"` } // AdditionalListenerSpec defines the desired state of an @@ -272,6 +281,14 @@ type AdditionalListenerSpec struct { // HealthCheck sets the optional custom health check configuration to the API target group. // +optional HealthCheck *TargetGroupHealthCheckAdditionalSpec `json:"healthCheck,omitempty"` + + // TargetGroupIPType sets the IP address type for the target group. + // Valid values are ipv4 and ipv6. If not specified, defaults to ipv4 unless + // the VPC has IPv6 enabled, in which case it defaults to ipv6. + // This field cannot be set if LoadBalancerType is classic or disabled. + // +kubebuilder:validation:Enum=ipv4;ipv6 + // +optional + TargetGroupIPType *TargetGroupIPType `json:"targetGroupIPType,omitempty"` } // AWSClusterStatus defines the observed state of AWSCluster. diff --git a/api/v1beta2/awscluster_webhook.go b/api/v1beta2/awscluster_webhook.go index 9d93260db5..252ec0c118 100644 --- a/api/v1beta2/awscluster_webhook.go +++ b/api/v1beta2/awscluster_webhook.go @@ -238,6 +238,14 @@ func (r *AWSCluster) validateControlPlaneLoadBalancerUpdate(oldlb, newlb *AWSLoa ) } } + + // TargetGroupIPType is immutable after creation. + if !cmp.Equal(oldlb.TargetGroupIPType, newlb.TargetGroupIPType) { + allErrs = append(allErrs, + field.Forbidden(field.NewPath("spec", "controlPlaneLoadBalancer", "targetGroupIPType"), + "field is immutable and cannot be changed after target group creation"), + ) + } } return allErrs @@ -467,6 +475,33 @@ func (r *AWSCluster) validateControlPlaneLBs() (admission.Warnings, field.ErrorL if r.Spec.ControlPlaneLoadBalancer.DisableHostsRewrite { allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "disableHostsRewrite"), r.Spec.ControlPlaneLoadBalancer.DisableHostsRewrite, "cannot disable hosts rewrite if the LoadBalancer reconciliation is disabled")) } + + if r.Spec.ControlPlaneLoadBalancer.TargetGroupIPType != nil { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "targetGroupIPType"), r.Spec.ControlPlaneLoadBalancer.TargetGroupIPType, "cannot set target group IP type if the LoadBalancer reconciliation is disabled")) + } + } + + if r.Spec.ControlPlaneLoadBalancer != nil { + basePath := field.NewPath("spec", "controlPlaneLoadBalancer") + if r.Spec.ControlPlaneLoadBalancer.TargetGroupIPType != nil { + allErrs = append(allErrs, r.validateTargetGroupIPType(basePath.Child("targetGroupIPType"), r.Spec.ControlPlaneLoadBalancer.TargetGroupIPType, r.Spec.ControlPlaneLoadBalancer)...) + } + for i, listener := range r.Spec.ControlPlaneLoadBalancer.AdditionalListeners { + if listener.TargetGroupIPType != nil { + allErrs = append(allErrs, r.validateTargetGroupIPType(basePath.Child("additionalListeners").Index(i).Child("targetGroupIPType"), listener.TargetGroupIPType, r.Spec.ControlPlaneLoadBalancer)...) + } + } + } + if r.Spec.SecondaryControlPlaneLoadBalancer != nil { + basePath := field.NewPath("spec", "secondaryControlPlaneLoadBalancer") + if r.Spec.SecondaryControlPlaneLoadBalancer.TargetGroupIPType != nil { + allErrs = append(allErrs, r.validateTargetGroupIPType(basePath.Child("targetGroupIPType"), r.Spec.SecondaryControlPlaneLoadBalancer.TargetGroupIPType, r.Spec.SecondaryControlPlaneLoadBalancer)...) + } + for i, listener := range r.Spec.SecondaryControlPlaneLoadBalancer.AdditionalListeners { + if listener.TargetGroupIPType != nil { + allErrs = append(allErrs, r.validateTargetGroupIPType(basePath.Child("additionalListeners").Index(i).Child("targetGroupIPType"), listener.TargetGroupIPType, r.Spec.SecondaryControlPlaneLoadBalancer)...) + } + } } return allWarnings, allErrs @@ -488,3 +523,20 @@ func (r *AWSCluster) validateIngressRules(path *field.Path, rules []IngressRule) } return allErrs } + +// validateTargetGroupIPType validates that the target group IP type is compatible +// with the load balancer type and VPC configuration. +func (r *AWSCluster) validateTargetGroupIPType(path *field.Path, targetGroupIPType *TargetGroupIPType, lbSpec *AWSLoadBalancerSpec) field.ErrorList { + var allErrs field.ErrorList + + if targetGroupIPType != nil { + if lbSpec.LoadBalancerType == LoadBalancerTypeClassic { + allErrs = append(allErrs, field.Invalid(path, targetGroupIPType, "targetGroupIPType cannot be used with classic load balancer types")) + } + if TargetGroupIPTypeIPv6.Equals(targetGroupIPType) && !r.Spec.NetworkSpec.VPC.IsIPv6Enabled() { + allErrs = append(allErrs, field.Invalid(path, targetGroupIPType, "targetGroupIPType IPv6 requires IPv6 to be enabled on the VPC. Set spec.network.vpc.ipv6 to enable IPv6")) + } + } + + return allErrs +} diff --git a/api/v1beta2/awscluster_webhook_test.go b/api/v1beta2/awscluster_webhook_test.go index 16549b8c18..43fccb162a 100644 --- a/api/v1beta2/awscluster_webhook_test.go +++ b/api/v1beta2/awscluster_webhook_test.go @@ -890,6 +890,111 @@ func TestAWSClusterValidateCreate(t *testing.T) { }, wantErr: true, }, + { + name: "rejects targetGroupIPType when LoadBalancer is disabled", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{ + TargetGroupIPType: &TargetGroupIPTypeIPv4, + LoadBalancerType: LoadBalancerTypeDisabled, + }, + }, + }, + wantErr: true, + }, + { + name: "rejects targetGroupIPType with Classic Load Balancer", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{ + LoadBalancerType: LoadBalancerTypeClassic, + TargetGroupIPType: &TargetGroupIPTypeIPv4, + }, + }, + }, + wantErr: true, + }, + { + name: "accepts targetGroupIPType IPv4 with Network Load Balancer", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{ + LoadBalancerType: LoadBalancerTypeNLB, + TargetGroupIPType: &TargetGroupIPTypeIPv4, + }, + }, + }, + wantErr: false, + }, + { + name: "rejects targetGroupIPType IPv6 with VPC IPv6 disabled", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{ + LoadBalancerType: LoadBalancerTypeNLB, + TargetGroupIPType: &TargetGroupIPTypeIPv6, + }, + NetworkSpec: NetworkSpec{}, + }, + }, + wantErr: true, + }, + { + name: "accepts targetGroupIPType IPv6 with NLB and VPC IPv6 enabled", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{ + LoadBalancerType: LoadBalancerTypeNLB, + TargetGroupIPType: &TargetGroupIPTypeIPv6, + }, + NetworkSpec: NetworkSpec{ + VPC: VPCSpec{ + IPv6: &IPv6{ + CidrBlock: "2001:db8::/56", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "rejects additionalListener targetGroupIPType with Classic Load Balancer", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{ + LoadBalancerType: LoadBalancerTypeClassic, + AdditionalListeners: []AdditionalListenerSpec{ + { + Port: 22623, + Protocol: ELBProtocolTCP, + TargetGroupIPType: &TargetGroupIPTypeIPv4, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "rejects additionalListener targetGroupIPType IPv6 with VPC IPv6 disabled", + cluster: &AWSCluster{ + Spec: AWSClusterSpec{ + ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{ + LoadBalancerType: LoadBalancerTypeNLB, + AdditionalListeners: []AdditionalListenerSpec{ + { + Port: 8443, + Protocol: ELBProtocolTCP, + TargetGroupIPType: &TargetGroupIPTypeIPv6, + }, + }, + }, + NetworkSpec: NetworkSpec{}, + }, + }, + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1348,6 +1453,53 @@ func TestAWSClusterValidateUpdate(t *testing.T) { }, wantErr: true, }, + { + name: "should failed if controlPlaneLoadBalancer targetGroupIPType is changed", + oldCluster: &AWSCluster{ + Spec: AWSClusterSpec{ + ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{ + LoadBalancerType: LoadBalancerTypeNLB, + TargetGroupIPType: &TargetGroupIPTypeIPv4, + }, + }, + }, + newCluster: &AWSCluster{ + Spec: AWSClusterSpec{ + ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{ + LoadBalancerType: LoadBalancerTypeNLB, + TargetGroupIPType: &TargetGroupIPTypeIPv6, + }, + NetworkSpec: NetworkSpec{ + VPC: VPCSpec{ + IPv6: &IPv6{ + CidrBlock: "2001:db8::/56", + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "should pass controlPlaneLoadBalancer targetGroupIPType is the same on update", + oldCluster: &AWSCluster{ + Spec: AWSClusterSpec{ + ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{ + LoadBalancerType: LoadBalancerTypeNLB, + TargetGroupIPType: &TargetGroupIPTypeIPv4, + }, + }, + }, + newCluster: &AWSCluster{ + Spec: AWSClusterSpec{ + ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{ + LoadBalancerType: LoadBalancerTypeNLB, + TargetGroupIPType: &TargetGroupIPTypeIPv4, + }, + }, + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/api/v1beta2/network_types.go b/api/v1beta2/network_types.go index 849ef34e49..6268235934 100644 --- a/api/v1beta2/network_types.go +++ b/api/v1beta2/network_types.go @@ -218,6 +218,53 @@ var ( TargetGroupAttributeUnhealthyDrainingIntervalSeconds = "target_health_state.unhealthy.draining_interval_seconds" ) +// TargetGroupIPType defines the IP address type for target groups. +type TargetGroupIPType string + +var ( + // TargetGroupIPTypeIPv4 defines the IPv4 address type for target groups. + TargetGroupIPTypeIPv4 = TargetGroupIPType("ipv4") + + // TargetGroupIPTypeIPv6 defines the IPv6 address type for target groups. + TargetGroupIPTypeIPv6 = TargetGroupIPType("ipv6") +) + +func (t TargetGroupIPType) String() string { + return string(t) +} + +// Equals returns true if two TargetGroupIPType are equal. +func (t TargetGroupIPType) Equals(other *TargetGroupIPType) bool { + if other == nil { + return false + } + + return t == *other +} + +// LoadBalancerIPAddressType defines the IP address type for load balancers. +type LoadBalancerIPAddressType string + +// Enum values for LoadBalancerIPAddressType +const ( + LoadBalancerIPAddressTypeIPv4 = LoadBalancerIPAddressType("ipv4") + LoadBalancerIPAddressTypeDualstack = LoadBalancerIPAddressType("dualstack") + LoadBalancerIPAddressTypeDualstackWithoutPublicIPv4 = LoadBalancerIPAddressType("dualstack-without-public-ipv4") +) + +func (t LoadBalancerIPAddressType) String() string { + return string(t) +} + +// Equals returns true if two LoadBalancerIPAddressType are equal. +func (t LoadBalancerIPAddressType) Equals(other *LoadBalancerIPAddressType) bool { + if other == nil { + return false + } + + return t == *other +} + // LoadBalancerAttribute defines a set of attributes for a V2 load balancer. type LoadBalancerAttribute string @@ -243,6 +290,8 @@ type TargetGroupSpec struct { VpcID string `json:"vpcId"` // HealthCheck is the elb health check associated with the load balancer. HealthCheck *TargetGroupHealthCheck `json:"targetGroupHealthCheck,omitempty"` + // IPType is the IP address type for the target group. + IPType TargetGroupIPType `json:"ipType,omitempty"` } // Listener defines an AWS network load balancer listener. @@ -298,6 +347,10 @@ type LoadBalancer struct { // LoadBalancerType sets the type for a load balancer. The default type is classic. // +kubebuilder:validation:Enum:=classic;elb;alb;nlb LoadBalancerType LoadBalancerType `json:"loadBalancerType,omitempty"` + + // LoadBalancerIPAddressType specifies the IP address type for the load balancer. + // +kubebuilder:validation:Enum:=ipv4;dualstack;dualstack-without-public-ipv4 + LoadBalancerIPAddressType LoadBalancerIPAddressType `json:"loadBalancerIPAddressType,omitempty"` } // IsUnmanaged returns true if the Classic ELB is unmanaged. diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 2f2360d4f3..b0e8f5a2a0 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -597,6 +597,11 @@ func (in *AWSLoadBalancerSpec) DeepCopyInto(out *AWSLoadBalancerSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.TargetGroupIPType != nil { + in, out := &in.TargetGroupIPType, &out.TargetGroupIPType + *out = new(TargetGroupIPType) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSLoadBalancerSpec. @@ -1199,6 +1204,11 @@ func (in *AdditionalListenerSpec) DeepCopyInto(out *AdditionalListenerSpec) { *out = new(TargetGroupHealthCheckAdditionalSpec) (*in).DeepCopyInto(*out) } + if in.TargetGroupIPType != nil { + in, out := &in.TargetGroupIPType, &out.TargetGroupIPType + *out = new(TargetGroupIPType) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdditionalListenerSpec. diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml index 4b6ff573f3..b0b22cec0b 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml @@ -1729,6 +1729,10 @@ spec: TargetGroupSpec specifies target group settings for a given listener. This is created first, and the ARN is then passed to the listener. properties: + ipType: + description: IPType is the IP address type for the + target group. + type: string name: description: Name of the TargetGroup. Must be unique over the same group of listeners. @@ -1848,6 +1852,14 @@ spec: - protocol type: object type: array + loadBalancerIPAddressType: + description: LoadBalancerIPAddressType specifies the IP address + type for the load balancer. + enum: + - ipv4 + - dualstack + - dualstack-without-public-ipv4 + type: string loadBalancerType: description: LoadBalancerType sets the type for a load balancer. The default type is classic. @@ -1949,6 +1961,10 @@ spec: TargetGroupSpec specifies target group settings for a given listener. This is created first, and the ARN is then passed to the listener. properties: + ipType: + description: IPType is the IP address type for the + target group. + type: string name: description: Name of the TargetGroup. Must be unique over the same group of listeners. @@ -2068,6 +2084,14 @@ spec: - protocol type: object type: array + loadBalancerIPAddressType: + description: LoadBalancerIPAddressType specifies the IP address + type for the load balancer. + enum: + - ipv4 + - dualstack + - dualstack-without-public-ipv4 + type: string loadBalancerType: description: LoadBalancerType sets the type for a load balancer. The default type is classic. @@ -4007,6 +4031,10 @@ spec: TargetGroupSpec specifies target group settings for a given listener. This is created first, and the ARN is then passed to the listener. properties: + ipType: + description: IPType is the IP address type for the + target group. + type: string name: description: Name of the TargetGroup. Must be unique over the same group of listeners. @@ -4126,6 +4154,14 @@ spec: - protocol type: object type: array + loadBalancerIPAddressType: + description: LoadBalancerIPAddressType specifies the IP address + type for the load balancer. + enum: + - ipv4 + - dualstack + - dualstack-without-public-ipv4 + type: string loadBalancerType: description: LoadBalancerType sets the type for a load balancer. The default type is classic. @@ -4227,6 +4263,10 @@ spec: TargetGroupSpec specifies target group settings for a given listener. This is created first, and the ARN is then passed to the listener. properties: + ipType: + description: IPType is the IP address type for the + target group. + type: string name: description: Name of the TargetGroup. Must be unique over the same group of listeners. @@ -4346,6 +4386,14 @@ spec: - protocol type: object type: array + loadBalancerIPAddressType: + description: LoadBalancerIPAddressType specifies the IP address + type for the load balancer. + enum: + - ipv4 + - dualstack + - dualstack-without-public-ipv4 + type: string loadBalancerType: description: LoadBalancerType sets the type for a load balancer. The default type is classic. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml index f69bd4926a..ada56b0f51 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml @@ -1074,6 +1074,16 @@ spec: enum: - TCP type: string + targetGroupIPType: + description: |- + TargetGroupIPType sets the IP address type for the target group. + Valid values are ipv4 and ipv6. If not specified, defaults to ipv4 unless + the VPC has IPv6 enabled, in which case it defaults to ipv6. + This field cannot be set if LoadBalancerType is classic or disabled. + enum: + - ipv4 + - ipv6 + type: string required: - port type: object @@ -1270,6 +1280,17 @@ spec: items: type: string type: array + targetGroupIPType: + description: |- + TargetGroupIPType sets the IP address type for the target group. + Valid values are ipv4 and ipv6. If not specified, defaults to ipv4 unless + the VPC has IPv6 enabled, in which case it defaults to ipv6. + This applies to the API server target group. + This field cannot be set if LoadBalancerType is classic or disabled. + enum: + - ipv4 + - ipv6 + type: string type: object identityRef: description: |- @@ -1955,6 +1976,16 @@ spec: enum: - TCP type: string + targetGroupIPType: + description: |- + TargetGroupIPType sets the IP address type for the target group. + Valid values are ipv4 and ipv6. If not specified, defaults to ipv4 unless + the VPC has IPv6 enabled, in which case it defaults to ipv6. + This field cannot be set if LoadBalancerType is classic or disabled. + enum: + - ipv4 + - ipv6 + type: string required: - port type: object @@ -2151,6 +2182,17 @@ spec: items: type: string type: array + targetGroupIPType: + description: |- + TargetGroupIPType sets the IP address type for the target group. + Valid values are ipv4 and ipv6. If not specified, defaults to ipv4 unless + the VPC has IPv6 enabled, in which case it defaults to ipv6. + This applies to the API server target group. + This field cannot be set if LoadBalancerType is classic or disabled. + enum: + - ipv4 + - ipv6 + type: string type: object sshKeyName: description: SSHKeyName is the name of the ssh key to attach to the @@ -2678,6 +2720,10 @@ spec: TargetGroupSpec specifies target group settings for a given listener. This is created first, and the ARN is then passed to the listener. properties: + ipType: + description: IPType is the IP address type for the + target group. + type: string name: description: Name of the TargetGroup. Must be unique over the same group of listeners. @@ -2797,6 +2843,14 @@ spec: - protocol type: object type: array + loadBalancerIPAddressType: + description: LoadBalancerIPAddressType specifies the IP address + type for the load balancer. + enum: + - ipv4 + - dualstack + - dualstack-without-public-ipv4 + type: string loadBalancerType: description: LoadBalancerType sets the type for a load balancer. The default type is classic. @@ -2898,6 +2952,10 @@ spec: TargetGroupSpec specifies target group settings for a given listener. This is created first, and the ARN is then passed to the listener. properties: + ipType: + description: IPType is the IP address type for the + target group. + type: string name: description: Name of the TargetGroup. Must be unique over the same group of listeners. @@ -3017,6 +3075,14 @@ spec: - protocol type: object type: array + loadBalancerIPAddressType: + description: LoadBalancerIPAddressType specifies the IP address + type for the load balancer. + enum: + - ipv4 + - dualstack + - dualstack-without-public-ipv4 + type: string loadBalancerType: description: LoadBalancerType sets the type for a load balancer. The default type is classic. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml index c3d219903f..068e899fda 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml @@ -652,6 +652,16 @@ spec: enum: - TCP type: string + targetGroupIPType: + description: |- + TargetGroupIPType sets the IP address type for the target group. + Valid values are ipv4 and ipv6. If not specified, defaults to ipv4 unless + the VPC has IPv6 enabled, in which case it defaults to ipv6. + This field cannot be set if LoadBalancerType is classic or disabled. + enum: + - ipv4 + - ipv6 + type: string required: - port type: object @@ -850,6 +860,17 @@ spec: items: type: string type: array + targetGroupIPType: + description: |- + TargetGroupIPType sets the IP address type for the target group. + Valid values are ipv4 and ipv6. If not specified, defaults to ipv4 unless + the VPC has IPv6 enabled, in which case it defaults to ipv6. + This applies to the API server target group. + This field cannot be set if LoadBalancerType is classic or disabled. + enum: + - ipv4 + - ipv6 + type: string type: object identityRef: description: |- @@ -1544,6 +1565,16 @@ spec: enum: - TCP type: string + targetGroupIPType: + description: |- + TargetGroupIPType sets the IP address type for the target group. + Valid values are ipv4 and ipv6. If not specified, defaults to ipv4 unless + the VPC has IPv6 enabled, in which case it defaults to ipv6. + This field cannot be set if LoadBalancerType is classic or disabled. + enum: + - ipv4 + - ipv6 + type: string required: - port type: object @@ -1742,6 +1773,17 @@ spec: items: type: string type: array + targetGroupIPType: + description: |- + TargetGroupIPType sets the IP address type for the target group. + Valid values are ipv4 and ipv6. If not specified, defaults to ipv4 unless + the VPC has IPv6 enabled, in which case it defaults to ipv6. + This applies to the API server target group. + This field cannot be set if LoadBalancerType is classic or disabled. + enum: + - ipv4 + - ipv6 + type: string type: object sshKeyName: description: SSHKeyName is the name of the ssh key to attach diff --git a/docs/book/src/crd/index.md b/docs/book/src/crd/index.md index f19020dd1c..90b8bb19d1 100644 --- a/docs/book/src/crd/index.md +++ b/docs/book/src/crd/index.md @@ -6637,6 +6637,20 @@ to be attached with this eks cluster

+accessConfig
+ + +AccessConfig + + + + +(Optional) +

AccessConfig specifies the access configuration information for the cluster

+ + + + vpcCni
@@ -7102,6 +7116,20 @@ to be attached with this eks cluster

+accessConfig
+ + +AccessConfig + + + + +(Optional) +

AccessConfig specifies the access configuration information for the cluster

+ + + + vpcCni
@@ -7801,6 +7829,20 @@ to be attached with this eks cluster

+accessConfig
+ + +AccessConfig + + + + +(Optional) +

AccessConfig specifies the access configuration information for the cluster

+ + + + vpcCni
@@ -7885,6 +7927,51 @@ AWSManagedControlPlaneTemplateResource +

AccessConfig +

+

+(Appears on:AWSManagedControlPlaneSpec) +

+

+

AccessConfig represents the access configuration information for the cluster

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+authenticationMode
+ + +EKSAuthenticationMode + + +		 +

AuthenticationMode specifies the desired authentication mode for the cluster +Defaults to config_map

+
+bootstrapClusterCreatorAdminPermissions
+ +bool + +		 +

BootstrapClusterCreatorAdminPermissions grants cluster admin permissions +to the IAM identity creating the cluster. Only applied during creation, +ignored when updating existing clusters. Defaults to true.

+

Addon

@@ -8222,6 +8309,14 @@ bool +

EKSAuthenticationMode +(string alias)

+

+(Appears on:AccessConfig) +

+

+

EKSAuthenticationMode defines the authentication mode for the cluster

+

EKSTokenMethod (string alias)

@@ -8809,7 +8904,7 @@ Amazon VPC CNI addon.

AWSRolesRef

-(Appears on:RosaControlPlaneSpec) +(Appears on:RosaControlPlaneSpec, ROSARoleConfigStatus)

AWSRolesRef contains references to various AWS IAM roles required for operators to make calls against the AWS API.

@@ -9245,6 +9340,44 @@ string +

AutoScaling +

+

+(Appears on:DefaultMachinePoolSpec, RosaMachinePoolSpec) +

+

+

AutoScaling specifies scaling options.

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+minReplicas
+ +int + +		 +
+maxReplicas
+ +int + +		 +

ChannelGroupType (string alias)

@@ -9309,8 +9442,8 @@ string autoscaling
- -RosaMachinePoolAutoScaling + +AutoScaling @@ -9798,6 +9931,21 @@ AlwaysAcknowledge: If acknowledgment is required, apply it and proceed with the +rosaRoleConfigRef
+ + +Kubernetes core/v1.LocalObjectReference + + + + +(Optional) +

RosaRoleConfigRef is a reference to a RosaRoleConfig resource that contains account roles, operator roles and OIDC configuration. +RosaRoleConfigRef and role fields such as installerRoleARN, supportRoleARN, workerRoleARN, rolesRef and oidcID are mutually exclusive.

+ + + + rolesRef
@@ -9806,7 +9954,9 @@ AWSRolesRef -

AWS IAM roles used to perform credential requests by the openshift operators.

+(Optional) +

AWS IAM roles used to perform credential requests by the openshift operators. +Required if RosaRoleConfigRef is not specified.

@@ -9817,7 +9967,9 @@ string -

The ID of the internal OpenID Connect Provider.

+(Optional) +

The ID of the internal OpenID Connect Provider. +Required if RosaRoleConfigRef is not specified.

@@ -9855,7 +10007,9 @@ string -

InstallerRoleARN is an AWS IAM role that OpenShift Cluster Manager will assume to create the cluster..

+(Optional) +

InstallerRoleARN is an AWS IAM role that OpenShift Cluster Manager will assume to create the cluster. +Required if RosaRoleConfigRef is not specified.

@@ -9866,8 +10020,10 @@ string +(Optional)

SupportRoleARN is an AWS IAM role used by Red Hat SREs to enable -access to the cluster account in order to provide support.

+access to the cluster account in order to provide support. +Required if RosaRoleConfigRef is not specified.

@@ -9878,7 +10034,9 @@ string -

WorkerRoleARN is an AWS IAM role that will be attached to worker instances.

+(Optional) +

WorkerRoleARN is an AWS IAM role that will be attached to worker instances. +Required if RosaRoleConfigRef is not specified.

@@ -10366,6 +10524,21 @@ AlwaysAcknowledge: If acknowledgment is required, apply it and proceed with the +rosaRoleConfigRef
+ + +Kubernetes core/v1.LocalObjectReference + + + + +(Optional) +

RosaRoleConfigRef is a reference to a RosaRoleConfig resource that contains account roles, operator roles and OIDC configuration. +RosaRoleConfigRef and role fields such as installerRoleARN, supportRoleARN, workerRoleARN, rolesRef and oidcID are mutually exclusive.

+ + + + rolesRef
@@ -10374,7 +10547,9 @@ AWSRolesRef -

AWS IAM roles used to perform credential requests by the openshift operators.

+(Optional) +

AWS IAM roles used to perform credential requests by the openshift operators. +Required if RosaRoleConfigRef is not specified.

@@ -10385,7 +10560,9 @@ string -

The ID of the internal OpenID Connect Provider.

+(Optional) +

The ID of the internal OpenID Connect Provider. +Required if RosaRoleConfigRef is not specified.

@@ -10423,7 +10600,9 @@ string -

InstallerRoleARN is an AWS IAM role that OpenShift Cluster Manager will assume to create the cluster..

+(Optional) +

InstallerRoleARN is an AWS IAM role that OpenShift Cluster Manager will assume to create the cluster. +Required if RosaRoleConfigRef is not specified.

@@ -10434,8 +10613,10 @@ string +(Optional)

SupportRoleARN is an AWS IAM role used by Red Hat SREs to enable -access to the cluster account in order to provide support.

+access to the cluster account in order to provide support. +Required if RosaRoleConfigRef is not specified.

@@ -10446,7 +10627,9 @@ string -

WorkerRoleARN is an AWS IAM role that will be attached to worker instances.

+(Optional) +

WorkerRoleARN is an AWS IAM role that will be attached to worker instances. +Required if RosaRoleConfigRef is not specified.

@@ -19965,7 +20148,7 @@ AWSClusterTemplateResource

AWSIdentityReference

-(Appears on:AWSClusterRoleIdentitySpec, AWSClusterSpec, AWSManagedControlPlaneSpec, AWSManagedControlPlaneSpec, RosaControlPlaneSpec) +(Appears on:AWSClusterRoleIdentitySpec, AWSClusterSpec, AWSManagedControlPlaneSpec, AWSManagedControlPlaneSpec, RosaControlPlaneSpec, ROSARoleConfigSpec)

AWSIdentityReference specifies a identity.

@@ -20186,6 +20369,24 @@ bool If this is enabled 6443 will be opened to 0.0.0.0/0.

+ + +targetGroupIPType
+ + +TargetGroupIPType + + + + +(Optional) +

TargetGroupIPType sets the IP address type for the target group. +Valid values are ipv4 and ipv6. If not specified, defaults to ipv4 unless +the VPC has IPv6 enabled, in which case it defaults to ipv6. +This applies to the API server target group. +This field cannot be set if LoadBalancerType is classic or disabled.

+ +

AWSMachine @@ -22501,6 +22702,23 @@ TargetGroupHealthCheckAdditionalSpec

HealthCheck sets the optional custom health check configuration to the API target group.

+ + +targetGroupIPType
+ + +TargetGroupIPType + + + + +(Optional) +

TargetGroupIPType sets the IP address type for the target group. +Valid values are ipv4 and ipv6. If not specified, defaults to ipv4 unless +the VPC has IPv6 enabled, in which case it defaults to ipv6. +This field cannot be set if LoadBalancerType is classic or disabled.

+ +

AllowedNamespaces @@ -24546,6 +24764,19 @@ LoadBalancerType

LoadBalancerType sets the type for a load balancer. The default type is classic.

+ + +loadBalancerIPAddressType
+ + +LoadBalancerIPAddressType + + + + +

LoadBalancerIPAddressType specifies the IP address type for the load balancer.

+ +

LoadBalancerAttribute @@ -24553,6 +24784,14 @@ LoadBalancerType

LoadBalancerAttribute defines a set of attributes for a V2 load balancer.

+

LoadBalancerIPAddressType +(string alias)

+

+(Appears on:LoadBalancer) +

+

+

LoadBalancerIPAddressType defines the IP address type for load balancers.

+

LoadBalancerType (string alias)

@@ -25547,6 +25786,14 @@ a target unhealthy.

+

TargetGroupIPType +(string alias)

+

+(Appears on:AWSLoadBalancerSpec, AdditionalListenerSpec, TargetGroupSpec) +

+

+

TargetGroupIPType defines the IP address type for target groups.

+

TargetGroupSpec

@@ -25621,6 +25868,19 @@ TargetGroupHealthCheck

HealthCheck is the elb health check associated with the load balancer.

+ + +ipType
+ + +TargetGroupIPType + + + + +

IPType is the IP address type for the target group.

+ +

VPCSpec @@ -28154,10 +28414,13 @@ Cluster API api/v1beta1.Conditions -

AutoScalingGroup +

AccountRoleConfig

-

AutoScalingGroup describes an AWS autoscaling group.

+(Appears on:ROSARoleConfigSpec) +

+

+

AccountRoleConfig defines account IAM roles before creating your ROSA cluster.

@@ -28169,93 +28432,221 @@ Cluster API api/v1beta1.Conditions + +
-id
+prefix
string 		-

The tags associated with the instance.

+

User-defined prefix for all generated AWS account role
-tags
+permissionsBoundaryARN
- -Tags - +string 		+(Optional) +

The ARN of the policy that is used to set the permissions boundary for the account roles.
-name
+path
string 		+(Optional) +

The arn path for the account/operator roles as well as their policies.
-desiredCapacity
+version
-int32 +string 		+

Version of OpenShift that will be used to the roles tag in formate of x.y.z example; “4.19.0” +Setting the role OpenShift version tag does not affect the associated ROSAControlplane version.
-maxSize
+sharedVPCConfig
-int32 + +SharedVPCConfig + 		+(Optional) +

SharedVPCConfig is used to set up shared VPC.
+

AccountRolesRef +

+

+(Appears on:ROSARoleConfigStatus) +

+

+

AccountRolesRef defscribes ARNs used as Account roles.

+

+ + + + + + + + - - +
FieldDescription
-minSize
+installerRoleARN
-int32 +string 		+

InstallerRoleARN is an AWS IAM role that OpenShift Cluster Manager will assume to create the cluster..
-placementGroup
+supportRoleARN
string 		+

SupportRoleARN is an AWS IAM role used by Red Hat SREs to enable +access to the cluster account in order to provide support.
-subnets
+workerRoleARN
-[]string +string 		+

WorkerRoleARN is an AWS IAM role that will be attached to worker instances.
-defaultCoolDown
- - -Kubernetes meta/v1.Duration +
+

AutoScalingGroup +

+

+

AutoScalingGroup describes an AWS autoscaling group.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -29036,6 +29427,29 @@ InstancesDistribution
FieldDescription
+id
+ +string + +		 +

The tags associated with the instance.

+
+tags
+ + +Tags + + +		 +
+name
+ +string + +		 +
+desiredCapacity
+ +int32 + +		 +
+maxSize
+ +int32 + +		 +
+minSize
+ +int32 + +		 +
+placementGroup
+ +string + +		 +
+subnets
+ +[]string + +		 +
+defaultCoolDown
+ + +Kubernetes meta/v1.Duration
+

OidcProviderType +(string alias)

+

+(Appears on:ROSARoleConfigSpec) +

+

+

OidcProviderType set to Managed or UnManaged

+

+ + + + + + + + + + + + +
ValueDescription

"Managed"

Managed OIDC Provider type

+

"Unmanaged"

Unmanaged OIDC Provider type

+

OnDemandAllocationStrategy (string alias)

@@ -29044,6 +29458,74 @@ InstancesDistribution

OnDemandAllocationStrategy indicates how to allocate instance types to fulfill On-Demand capacity.

+

OperatorRoleConfig +

+

+(Appears on:ROSARoleConfigSpec) +

+

+

OperatorRoleConfig defines cluster-specific operator IAM roles based on your cluster configuration.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+prefix
+ +string + +		 +

User-defined prefix for generated AWS operator roles.

+
+permissionsBoundaryARN
+ +string + +		 +(Optional) +

The ARN of the policy that is used to set the permissions boundary for the operator roles.

+
+sharedVPCConfig
+ + +SharedVPCConfig + + +		 +(Optional) +

SharedVPCConfig is used to set up shared VPC.

+
+oidcID
+ +string + +		 +(Optional) +

OIDCID is the ID of the OIDC config that will be used to create the operator roles. +Cannot be set when OidcProviderType set to Managed

+

Overrides

@@ -29499,8 +29981,8 @@ string autoscaling
- -RosaMachinePoolAutoScaling + +AutoScaling @@ -29592,6 +30074,19 @@ RosaUpdateConfig

UpdateConfig specifies update configurations.

+ + +capacityReservationID
+ +string + + + +(Optional) +

CapacityReservationID specifies the ID of an AWS On-Demand Capacity Reservation and Capacity Blocks for ML. +The CapacityReservationID must be pre-created in advance, before creating a NodePool.

+ + @@ -29609,6 +30104,296 @@ RosaMachinePoolStatus +

ROSARoleConfig +

+

+

ROSARoleConfig is the Schema for the rosaroleconfigs API

+

+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+metadata
+ + +Kubernetes meta/v1.ObjectMeta + + +		 +Refer to the Kubernetes API documentation for the fields of the +metadata field. +
+spec
+ + +ROSARoleConfigSpec + + +		 +
+
+ + + + + + + + + + + + + + + + + + + + + +
+accountRoleConfig
+ + +AccountRoleConfig + + +		 +

AccountRoleConfig defines account-wide IAM roles before creating your ROSA cluster.

+
+operatorRoleConfig
+ + +OperatorRoleConfig + + +		 +

OperatorRoleConfig defines cluster-specific operator IAM roles based on your cluster configuration.

+
+identityRef
+ + +AWSIdentityReference + + +		 +(Optional) +

IdentityRef is a reference to an identity to be used when reconciling the ROSA Role Config. +If no identity is specified, the default identity for this controller will be used.

+
+credentialsSecretRef
+ + +Kubernetes core/v1.LocalObjectReference + + +		 +(Optional) +

CredentialsSecretRef references a secret with necessary credentials to connect to the OCM API.

+
+oidcProviderType
+ + +OidcProviderType + + +		 +

OIDC provider type values are Managed or UnManaged. When set to Unmanged OperatorRoleConfig OIDCID field must be provided.

+
+
+status
+ + +ROSARoleConfigStatus + + +		 +
+

ROSARoleConfigSpec +

+

+(Appears on:ROSARoleConfig) +

+

+

ROSARoleConfigSpec defines the desired state of ROSARoleConfig

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+accountRoleConfig
+ + +AccountRoleConfig + + +		 +

AccountRoleConfig defines account-wide IAM roles before creating your ROSA cluster.

+
+operatorRoleConfig
+ + +OperatorRoleConfig + + +		 +

OperatorRoleConfig defines cluster-specific operator IAM roles based on your cluster configuration.

+
+identityRef
+ + +AWSIdentityReference + + +		 +(Optional) +

IdentityRef is a reference to an identity to be used when reconciling the ROSA Role Config. +If no identity is specified, the default identity for this controller will be used.

+
+credentialsSecretRef
+ + +Kubernetes core/v1.LocalObjectReference + + +		 +(Optional) +

CredentialsSecretRef references a secret with necessary credentials to connect to the OCM API.

+
+oidcProviderType
+ + +OidcProviderType + + +		 +

OIDC provider type values are Managed or UnManaged. When set to Unmanged OperatorRoleConfig OIDCID field must be provided.

+
+

ROSARoleConfigStatus +

+

+(Appears on:ROSARoleConfig) +

+

+

ROSARoleConfigStatus defines the observed state of ROSARoleConfig

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+oidcID
+ +string + +		 +

ID of created OIDC config

+
+oidcProviderARN
+ +string + +		 +

Create OIDC provider for operators to authenticate against in an STS cluster.

+
+accountRolesRef
+ + +AccountRolesRef + + +		 +

Created Account roles that can be used to

+
+operatorRolesRef
+ + +AWSRolesRef + + +		 +

AWS IAM roles used to perform credential requests by the openshift operators.

+
+conditions
+ + +Cluster API api/v1beta1.Conditions + + +		 +

Conditions specifies the ROSARoleConfig conditions

+

RefreshPreferences

@@ -29758,44 +30543,6 @@ running at any time during the update is at most 130% of desired nodes.

-

RosaMachinePoolAutoScaling -

-

-(Appears on:DefaultMachinePoolSpec, RosaMachinePoolSpec) -

-

-

RosaMachinePoolAutoScaling specifies scaling options.

-

- - - - - - - - - - - - - - - - - -
FieldDescription
-minReplicas
- -int - -		 -
-maxReplicas
- -int - -		 -

RosaMachinePoolSpec

@@ -29929,8 +30676,8 @@ string autoscaling
- -RosaMachinePoolAutoScaling + +AutoScaling @@ -30022,6 +30769,19 @@ RosaUpdateConfig

UpdateConfig specifies update configurations.

+ + +capacityReservationID
+ +string + + + +(Optional) +

CapacityReservationID specifies the ID of an AWS On-Demand Capacity Reservation and Capacity Blocks for ML. +The CapacityReservationID must be pre-created in advance, before creating a NodePool.

+ +

RosaMachinePoolStatus @@ -30207,6 +30967,46 @@ RollingUpdate +

SharedVPCConfig +

+

+(Appears on:AccountRoleConfig, OperatorRoleConfig) +

+

+

SharedVPCConfig is used to set up shared VPC.

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+routeRoleARN
+ +string + +		 +

Role ARN associated with the private hosted zone used for Hosted Control Plane cluster shared VPC, this role contains policies to be used with Route 53

+
+vpcEndpointRoleArn
+ +string + +		 +

Role ARN associated with the shared VPC used for Hosted Control Plane clusters, this role contains policies to be used with the VPC endpoint

+

SpotAllocationStrategy (string alias)

diff --git a/pkg/cloud/services/elb/loadbalancer.go b/pkg/cloud/services/elb/loadbalancer.go index 874ea2d815..f36a8b6467 100644 --- a/pkg/cloud/services/elb/loadbalancer.go +++ b/pkg/cloud/services/elb/loadbalancer.go @@ -306,6 +306,32 @@ func (s *Service) getAdditionalTargetGroupHealthCheck(ln infrav1.AdditionalListe return healthCheck } +// getAPITargetGroupIPType determines the IP address type for the API server target group. +// It defaults to IPv4, uses IPv6 if the VPC has IPv6 enabled, and can be overridden by the load balancer spec. +func (s *Service) getAPITargetGroupIPType(lbSpec *infrav1.AWSLoadBalancerSpec) infrav1.TargetGroupIPType { + ipType := infrav1.TargetGroupIPTypeIPv4 + if s.scope.VPC().IsIPv6Enabled() { + ipType = infrav1.TargetGroupIPTypeIPv6 + } + if lbSpec != nil && lbSpec.TargetGroupIPType != nil { + ipType = *lbSpec.TargetGroupIPType + } + return ipType +} + +// getAdditionalTargetGroupIPType determines the IP address type for an additional listener's target group. +// It defaults to IPv4, uses IPv6 if the VPC has IPv6 enabled, and can be overridden by the listener spec. +func (s *Service) getAdditionalTargetGroupIPType(ln infrav1.AdditionalListenerSpec) infrav1.TargetGroupIPType { + ipType := infrav1.TargetGroupIPTypeIPv4 + if s.scope.VPC().IsIPv6Enabled() { + ipType = infrav1.TargetGroupIPTypeIPv6 + } + if ln.TargetGroupIPType != nil { + ipType = *ln.TargetGroupIPType + } + return ipType +} + func (s *Service) getAPIServerLBSpec(ctx context.Context, elbName string, lbSpec *infrav1.AWSLoadBalancerSpec) (*infrav1.LoadBalancer, error) { var securityGroupIDs []string if lbSpec != nil { @@ -335,6 +361,7 @@ func (s *Service) getAPIServerLBSpec(ctx context.Context, elbName string, lbSpec Protocol: infrav1.ELBProtocolTCP, VpcID: s.scope.VPC().ID, HealthCheck: apiHealthCheck, + IPType: s.getAPITargetGroupIPType(lbSpec), }, }, }, @@ -360,6 +387,7 @@ func (s *Service) getAPIServerLBSpec(ctx context.Context, elbName string, lbSpec Protocol: listener.Protocol, VpcID: s.scope.VPC().ID, HealthCheck: lnHealthCheck, + IPType: s.getAdditionalTargetGroupIPType(listener), }, }) } @@ -1780,9 +1808,11 @@ func (s *Service) createTargetGroup(ctx context.Context, ln infrav1.Listener, ta HealthyThresholdCount: aws.Int32(infrav1.DefaultAPIServerHealthThresholdCount), UnhealthyThresholdCount: aws.Int32(infrav1.DefaultAPIServerUnhealthThresholdCount), } - if s.scope.VPC().IsIPv6Enabled() { - targetGroupInput.IpAddressType = elbv2types.TargetGroupIpAddressTypeEnumIpv6 + + if ln.TargetGroup.IPType != "" { + targetGroupInput.IpAddressType = elbv2types.TargetGroupIpAddressTypeEnum(ln.TargetGroup.IPType) } + if ln.TargetGroup.HealthCheck != nil { targetGroupInput.HealthCheckEnabled = aws.Bool(true) @@ -1841,6 +1871,8 @@ func fromSDKTypeToClassicELB(v *elbtypes.LoadBalancerDescription, attrs *elbtype DNSName: aws.ToString(v.DNSName), Tags: converters.ELBTagsToMap(tags), LoadBalancerType: infrav1.LoadBalancerTypeClassic, + // Classic Load Balancers only support IPv4. + LoadBalancerIPAddressType: infrav1.LoadBalancerIPAddressTypeIPv4, } if attrs.ConnectionSettings != nil && attrs.ConnectionSettings.IdleTimeout != nil { @@ -1860,14 +1892,24 @@ func fromSDKTypeToLB(v elbv2types.LoadBalancer, attrs []elbv2types.LoadBalancerA availabilityZones[i] = aws.ToString(az.ZoneName) } res := &infrav1.LoadBalancer{ - ARN: aws.ToString(v.LoadBalancerArn), - Name: aws.ToString(v.LoadBalancerName), - Scheme: infrav1.ELBScheme(v.Scheme), - SubnetIDs: subnetIDs, - SecurityGroupIDs: v.SecurityGroups, - AvailabilityZones: availabilityZones, - DNSName: aws.ToString(v.DNSName), - Tags: converters.V2TagsToMap(tags), + ARN: aws.ToString(v.LoadBalancerArn), + Name: aws.ToString(v.LoadBalancerName), + Scheme: infrav1.ELBScheme(v.Scheme), + SubnetIDs: subnetIDs, + SecurityGroupIDs: v.SecurityGroups, + AvailabilityZones: availabilityZones, + DNSName: aws.ToString(v.DNSName), + Tags: converters.V2TagsToMap(tags), + LoadBalancerIPAddressType: infrav1.LoadBalancerIPAddressType(v.IpAddressType), + } + + switch v.Type { + case elbv2types.LoadBalancerTypeEnumApplication: + res.LoadBalancerType = infrav1.LoadBalancerTypeALB + case elbv2types.LoadBalancerTypeEnumNetwork: + res.LoadBalancerType = infrav1.LoadBalancerTypeNLB + case elbv2types.LoadBalancerTypeEnumGateway: + res.LoadBalancerType = infrav1.LoadBalancerTypeELB } infraAttrs := make(map[string]*string, len(attrs)) @@ -1927,7 +1969,9 @@ func isSDKTargetGroupEqualToTargetGroup(elbTG *elbv2types.TargetGroup, spec *inf // Not created by CAPA return false } - return int64(ptr.Deref(elbTG.Port, 0)) == spec.Port && strings.EqualFold(string(elbTG.Protocol), spec.Protocol.String()) + return int64(ptr.Deref(elbTG.Port, 0)) == spec.Port && + strings.EqualFold(string(elbTG.Protocol), spec.Protocol.String()) && + strings.EqualFold(string(elbTG.IpAddressType), string(spec.IPType)) } // SchemeToSDKScheme converts infrav1.ELBScheme to elbv2types.LoadBalancerSchemeEnum. diff --git a/pkg/cloud/services/elb/loadbalancer_test.go b/pkg/cloud/services/elb/loadbalancer_test.go index d59c15c91b..ba654c54ea 100644 --- a/pkg/cloud/services/elb/loadbalancer_test.go +++ b/pkg/cloud/services/elb/loadbalancer_test.go @@ -1619,6 +1619,23 @@ func TestReconcileTargetGroupsAndListeners(t *testing.T) { { name: "created with ipv6 vpc", spec: func(spec infrav1.LoadBalancer) infrav1.LoadBalancer { + spec.ELBListeners = []infrav1.Listener{ + { + Protocol: "TCP", + Port: infrav1.DefaultAPIServerPort, + TargetGroup: infrav1.TargetGroupSpec{ + Name: "name", + Port: infrav1.DefaultAPIServerPort, + Protocol: "TCP", + VpcID: vpcID, + HealthCheck: &infrav1.TargetGroupHealthCheck{ + Protocol: aws.String("tcp"), + Port: aws.String(infrav1.DefaultAPIServerPortString), + }, + IPType: infrav1.TargetGroupIPTypeIPv6, + }, + }, + } return spec }, awsCluster: func(acl infrav1.AWSCluster) infrav1.AWSCluster {