From 83c456666a6f19e159427d9bb8ee0b170225cad0 Mon Sep 17 00:00:00 2001 From: Saylor Berman Date: Fri, 7 Feb 2025 14:06:46 -0700 Subject: [PATCH] CP/DP Split: Support basic NGINX OSS provisioning This commit updates the control plane to deploy an NGINX data plane when a valid Gateway resource is created. When the Gateway is deleted or becomes invalid, the data plane is removed. The NginxProxy resource has been updated with numerous configuration options related to the k8s deployment and service configs, which the control plane will apply to the NGINX resources when set. The control plane fully owns the NGINX deployment resources, so users who want to change any configuration must do so using the NginxProxy resource. This does not yet support NGINX Plus or NGINX debug mode. Those will be added in followup pull requests. This also adds some basic daemonset fields, but does not yet support deploying a daemosnet. That will also be added soon. --- .github/workflows/helm.yml | 2 +- .yamllint.yaml | 4 +- Makefile | 4 +- apis/v1alpha2/nginxproxy_types.go | 320 +- apis/v1alpha2/zz_generated.deepcopy.go | 250 ++ charts/nginx-gateway-fabric/README.md | 64 +- charts/nginx-gateway-fabric/README.md.gotmpl | 7 +- .../templates/_helpers.tpl | 2 +- .../templates/clusterrole.yaml | 23 +- .../templates/configmap.yaml | 34 - .../templates/deployment.yaml | 54 +- .../templates/gatewayclass.yaml | 2 - .../templates/nginxproxy.yaml | 23 +- .../templates/serviceaccount.yaml | 12 +- .../templates/tmp-nginx-agent-conf.yaml | 43 - .../templates/tmp-nginx-deployment.yaml | 204 - .../templates/tmp-nginx-service.yaml | 36 - .../nginx-gateway-fabric/values.schema.json | 422 +- charts/nginx-gateway-fabric/values.yaml | 255 +- cmd/gateway/commands.go | 37 +- cmd/gateway/commands_test.go | 51 +- .../bases/gateway.nginx.org_nginxproxies.yaml | 3462 +++++++++++++++++ config/tests/static-deployment.yaml | 10 +- deploy/aws-nlb/deploy.yaml | 269 +- deploy/azure/deploy.yaml | 268 +- deploy/crds.yaml | 3462 +++++++++++++++++ deploy/default/deploy.yaml | 263 +- deploy/experimental-nginx-plus/deploy.yaml | 287 +- deploy/experimental/deploy.yaml | 264 +- deploy/nginx-plus/deploy.yaml | 286 +- deploy/nodeport/deploy.yaml | 263 +- deploy/openshift/deploy.yaml | 265 +- .../snippets-filters-nginx-plus/deploy.yaml | 286 +- deploy/snippets-filters/deploy.yaml | 263 +- docs/developer/quickstart.md | 4 +- examples/helm/aws-nlb/values.yaml | 11 +- examples/helm/azure/values.yaml | 8 +- .../helm/experimental-nginx-plus/values.yaml | 2 - examples/helm/nginx-plus/values.yaml | 2 - examples/helm/nodeport/values.yaml | 5 +- .../snippets-filters-nginx-plus/values.yaml | 2 - go.mod | 2 +- internal/framework/controller/labels.go | 12 + .../controller/predicate/annotation.go | 41 + .../controller/predicate/annotation_test.go | 178 + .../framework/controller/predicate/label.go | 18 + .../framework/controller/predicate/service.go | 53 - .../controller/predicate/service_test.go | 220 +- internal/framework/controller/resource.go | 9 + internal/mode/static/config/config.go | 13 +- internal/mode/static/handler.go | 194 +- internal/mode/static/handler_test.go | 171 +- internal/mode/static/manager.go | 47 +- internal/mode/static/nginx/agent/agent.go | 2 + internal/mode/static/nginx/agent/command.go | 4 +- .../mode/static/nginx/agent/command_test.go | 6 + .../mode/static/nginx/agent/deployment.go | 12 +- .../static/nginx/agent/deployment_test.go | 3 + .../static/nginx/agent/grpc/connections.go | 12 +- .../nginx/agent/grpc/connections_test.go | 24 +- .../grpcfakes/fake_connections_tracker.go | 79 +- internal/mode/static/provisioner/doc.go | 4 + internal/mode/static/provisioner/eventloop.go | 126 + internal/mode/static/provisioner/handler.go | 162 + internal/mode/static/provisioner/objects.go | 524 +++ .../mode/static/provisioner/provisioner.go | 354 ++ .../provisionerfakes/fake_provisioner.go | 117 + internal/mode/static/provisioner/setter.go | 45 + internal/mode/static/provisioner/store.go | 196 + internal/mode/static/provisioner/templates.go | 73 + .../mode/static/state/change_processor.go | 3 - .../static/state/change_processor_test.go | 9 + internal/mode/static/state/graph/gateway.go | 11 +- .../mode/static/state/graph/gateway_test.go | 5 +- internal/mode/static/state/graph/graph.go | 16 +- .../mode/static/state/graph/graph_test.go | 10 +- .../mode/static/state/graph/nginxproxy.go | 13 + .../static/state/graph/nginxproxy_test.go | 77 + internal/mode/static/status/queue.go | 19 +- internal/mode/static/status/queue_test.go | 4 + scripts/generate-manifests.sh | 2 +- tests/Makefile | 4 +- tests/framework/ngf.go | 2 +- 83 files changed, 10824 insertions(+), 3583 deletions(-) delete mode 100644 charts/nginx-gateway-fabric/templates/configmap.yaml delete mode 100644 charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml delete mode 100644 charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml delete mode 100644 charts/nginx-gateway-fabric/templates/tmp-nginx-service.yaml create mode 100644 internal/framework/controller/labels.go create mode 100644 internal/framework/controller/predicate/label.go create mode 100644 internal/framework/controller/resource.go create mode 100644 internal/mode/static/provisioner/doc.go create mode 100644 internal/mode/static/provisioner/eventloop.go create mode 100644 internal/mode/static/provisioner/handler.go create mode 100644 internal/mode/static/provisioner/objects.go create mode 100644 internal/mode/static/provisioner/provisioner.go create mode 100644 internal/mode/static/provisioner/provisionerfakes/fake_provisioner.go create mode 100644 internal/mode/static/provisioner/setter.go create mode 100644 internal/mode/static/provisioner/store.go create mode 100644 internal/mode/static/provisioner/templates.go diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 0c402aa8c8..3fbd654f32 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -176,4 +176,4 @@ jobs: --set=nginx.plus=${{ inputs.image == 'plus' }} \ --set=nginx.image.tag=nightly \ --set=nginxGateway.productTelemetry.enable=false \ - ${{ inputs.image == 'plus' && '--set=serviceAccount.imagePullSecret=nginx-plus-registry-secret --set=nginx.image.repository=private-registry.nginx.com/nginx-gateway-fabric/nginx-plus' || '' }}" + ${{ inputs.image == 'plus' && '--set=nginx.imagePullSecret=nginx-plus-registry-secret --set=nginx.image.repository=private-registry.nginx.com/nginx-gateway-fabric/nginx-plus' || '' }}" diff --git a/.yamllint.yaml b/.yamllint.yaml index 83713689aa..e52cae4940 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -14,7 +14,9 @@ rules: require-starting-space: true ignore-shebangs: true min-spaces-from-content: 1 - comments-indentation: enable + comments-indentation: + ignore: | + charts/nginx-gateway-fabric/values.yaml document-end: disable document-start: disable empty-lines: enable diff --git a/Makefile b/Makefile index 375e48950f..0fb227f514 100644 --- a/Makefile +++ b/Makefile @@ -226,13 +226,13 @@ install-ngf-local-build-with-plus: check-for-plus-usage-endpoint build-images-wi .PHONY: helm-install-local helm-install-local: install-gateway-crds ## Helm install NGF on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build. - helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PREFIX) --create-namespace --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway $(HELM_PARAMETERS) + helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PREFIX) --create-namespace --wait --set nginxGateway.image.pullPolicy=Never --set nginx.service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway $(HELM_PARAMETERS) .PHONY: helm-install-local-with-plus helm-install-local-with-plus: check-for-plus-usage-endpoint install-gateway-crds ## Helm install NGF with NGINX Plus on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build-with-plus. kubectl create namespace nginx-gateway || true kubectl -n nginx-gateway create secret generic nplus-license --from-file $(PLUS_LICENSE_FILE) || true - helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --wait --set nginxGateway.image.pullPolicy=Never --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true --set nginx.usage.endpoint=$(PLUS_USAGE_ENDPOINT) $(HELM_PARAMETERS) + helm install nginx-gateway $(CHART_DIR) --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --wait --set nginxGateway.image.pullPolicy=Never --set nginx.service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway --set nginx.plus=true --set nginx.usage.endpoint=$(PLUS_USAGE_ENDPOINT) $(HELM_PARAMETERS) .PHONY: check-for-plus-usage-endpoint check-for-plus-usage-endpoint: ## Checks that the PLUS_USAGE_ENDPOINT is set in the environment. This env var is required when deploying or testing with N+. diff --git a/apis/v1alpha2/nginxproxy_types.go b/apis/v1alpha2/nginxproxy_types.go index 7c10bd9f3c..8f8a2671e8 100644 --- a/apis/v1alpha2/nginxproxy_types.go +++ b/apis/v1alpha2/nginxproxy_types.go @@ -1,6 +1,7 @@ package v1alpha2 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1" @@ -47,6 +48,11 @@ type NginxProxySpec struct { // // +optional Telemetry *Telemetry `json:"telemetry,omitempty"` + // Metrics defines the configuration for Prometheus scraping metrics. Changing this value results in a + // re-roll of the NGINX deployment. + // + // +optional + Metrics *Metrics `json:"metrics,omitempty"` // RewriteClientIP defines configuration for rewriting the client IP to the original client's IP. // +kubebuilder:validation:XValidation:message="if mode is set, trustedAddresses is a required field",rule="!(has(self.mode) && (!has(self.trustedAddresses) || size(self.trustedAddresses) == 0))" // @@ -66,14 +72,10 @@ type NginxProxySpec struct { // // +optional DisableHTTP2 *bool `json:"disableHTTP2,omitempty"` -} - -// NginxPlus specifies NGINX Plus additional settings. These will only be applied if NGINX Plus is being used. -type NginxPlus struct { - // AllowedAddresses specifies IPAddresses or CIDR blocks to the allow list for accessing the NGINX Plus API. + // Kubernetes contains the configuration for the NGINX Deployment and Service Kubernetes objects. // // +optional - AllowedAddresses []NginxPlusAllowAddress `json:"allowedAddresses,omitempty"` + Kubernetes *KubernetesSpec `json:"kubernetes,omitempty"` } // Telemetry specifies the OpenTelemetry configuration. @@ -82,6 +84,7 @@ type Telemetry struct { // // +optional DisabledFeatures []DisableTelemetryFeature `json:"disabledFeatures,omitempty"` + // Exporter specifies OpenTelemetry export parameters. // // +optional @@ -105,6 +108,16 @@ type Telemetry struct { SpanAttributes []v1alpha1.SpanAttribute `json:"spanAttributes,omitempty"` } +// DisableTelemetryFeature is a telemetry feature that can be disabled. +// +// +kubebuilder:validation:Enum=DisableTracing +type DisableTelemetryFeature string + +const ( + // DisableTracing disables the OpenTelemetry tracing feature. + DisableTracing DisableTelemetryFeature = "DisableTracing" +) + // TelemetryExporter specifies OpenTelemetry export parameters. type TelemetryExporter struct { // Interval is the maximum interval between two exports. @@ -136,6 +149,21 @@ type TelemetryExporter struct { Endpoint *string `json:"endpoint,omitempty"` } +// Metrics defines the configuration for Prometheus scraping metrics. +type Metrics struct { + // Port where the Prometheus metrics are exposed. + // + // +optional + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + Port *int32 `json:"port,omitempty"` + + // Disable serving Prometheus metrics on the listen port. + // + // +optional + Disable *bool `json:"disable,omitempty"` +} + // RewriteClientIP specifies the configuration for rewriting the client's IP address. type RewriteClientIP struct { // Mode defines how NGINX will rewrite the client's IP address. @@ -229,27 +257,6 @@ const ( RewriteClientIPHostnameAddressType RewriteClientIPAddressType = "Hostname" ) -// NginxPlusAllowAddress specifies the address type and value for an NginxPlus allow address. -type NginxPlusAllowAddress struct { - // Type specifies the type of address. - Type NginxPlusAllowAddressType `json:"type"` - - // Value specifies the address value. - Value string `json:"value"` -} - -// NginxPlusAllowAddressType specifies the type of address. -// +kubebuilder:validation:Enum=CIDR;IPAddress -type NginxPlusAllowAddressType string - -const ( - // NginxPlusAllowCIDRAddressType specifies that the address is a CIDR block. - NginxPlusAllowCIDRAddressType NginxPlusAllowAddressType = "CIDR" - - // NginxPlusAllowIPAddressType specifies that the address is an IP address. - NginxPlusAllowIPAddressType NginxPlusAllowAddressType = "IPAddress" -) - // NginxLogging defines logging related settings for NGINX. type NginxLogging struct { // ErrorLevel defines the error log level. Possible log levels listed in order of increasing severity are @@ -260,6 +267,13 @@ type NginxLogging struct { // +optional // +kubebuilder:default=info ErrorLevel *NginxErrorLogLevel `json:"errorLevel,omitempty"` + + // AgentLevel defines the log level of the NGINX agent process. Changing this value results in a + // re-roll of the NGINX deployment. + // + // +optional + // +kubebuilder:default=info + AgentLevel *AgentLogLevel `json:"agentLevel,omitempty"` } // NginxErrorLogLevel type defines the log level of error logs for NGINX. @@ -293,12 +307,254 @@ const ( NginxLogLevelEmerg NginxErrorLogLevel = "emerg" ) -// DisableTelemetryFeature is a telemetry feature that can be disabled. +// AgentLevel defines the log level of the NGINX agent process. // -// +kubebuilder:validation:Enum=DisableTracing -type DisableTelemetryFeature string +// +kubebuilder:validation:Enum=debug;info;error;panic;fatal +type AgentLogLevel string const ( - // DisableTracing disables the OpenTelemetry tracing feature. - DisableTracing DisableTelemetryFeature = "DisableTracing" + // AgentLogLevelDebug is the debug level NGINX agent logs. + AgentLogLevelDebug AgentLogLevel = "debug" + + // AgentLogLevelInfo is the info level NGINX agent logs. + AgentLogLevelInfo AgentLogLevel = "info" + + // AgentLogLevelError is the error level NGINX agent logs. + AgentLogLevelError AgentLogLevel = "error" + + // AgentLogLevelPanic is the panic level NGINX agent logs. + AgentLogLevelPanic AgentLogLevel = "panic" + + // AgentLogLevelFatal is the fatal level NGINX agent logs. + AgentLogLevelFatal AgentLogLevel = "fatal" +) + +// NginxPlus specifies NGINX Plus additional settings. These will only be applied if NGINX Plus is being used. +type NginxPlus struct { + // AllowedAddresses specifies IPAddresses or CIDR blocks to the allow list for accessing the NGINX Plus API. + // + // +optional + AllowedAddresses []NginxPlusAllowAddress `json:"allowedAddresses,omitempty"` +} + +// NginxPlusAllowAddress specifies the address type and value for an NginxPlus allow address. +type NginxPlusAllowAddress struct { + // Type specifies the type of address. + Type NginxPlusAllowAddressType `json:"type"` + + // Value specifies the address value. + Value string `json:"value"` +} + +// NginxPlusAllowAddressType specifies the type of address. +// +kubebuilder:validation:Enum=CIDR;IPAddress +type NginxPlusAllowAddressType string + +const ( + // NginxPlusAllowCIDRAddressType specifies that the address is a CIDR block. + NginxPlusAllowCIDRAddressType NginxPlusAllowAddressType = "CIDR" + + // NginxPlusAllowIPAddressType specifies that the address is an IP address. + NginxPlusAllowIPAddressType NginxPlusAllowAddressType = "IPAddress" +) + +// KubernetesSpec contains the configuration for the NGINX Deployment and Service Kubernetes objects. +type KubernetesSpec struct { + // Deployment is the configuration for the NGINX Deployment. + // This is the default deployment option. + // + // +optional + Deployment *DeploymentSpec `json:"deployment,omitempty"` + + // Service is the configuration for the NGINX Service. + // + // +optional + Service *ServiceSpec `json:"service,omitempty"` +} + +// Deployment is the configuration for the NGINX Deployment. +type DeploymentSpec struct { + // Number of desired Pods. + // + // +optional + Replicas *int32 `json:"replicas,omitempty"` + + // Pod defines Pod-specific fields. + // + // +optional + Pod PodSpec `json:"pod,omitempty"` + + // Container defines container fields for the NGINX container. + // + // +optional + Container ContainerSpec `json:"container,omitempty"` +} + +// PodSpec defines Pod-specific fields. +type PodSpec struct { + // TerminationGracePeriodSeconds is the optional duration in seconds the pod needs to terminate gracefully. + // Value must be non-negative integer. The value zero indicates stop immediately via + // the kill signal (no opportunity to shut down). + // If this value is nil, the default grace period will be used instead. + // The grace period is the duration in seconds after the processes running in the pod are sent + // a termination signal and the time when the processes are forcibly halted with a kill signal. + // Set this value longer than the expected cleanup time for your process. + // Defaults to 30 seconds. + // + // +optional + TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"` + + // Affinity is the pod's scheduling constraints. + // + // +optional + Affinity *corev1.Affinity `json:"affinity,omitempty"` + + // NodeSelector is a selector which must be true for the pod to fit on a node. + // Selector which must match a node's labels for the pod to be scheduled on that node. + // + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // Tolerations allow the scheduler to schedule Pods with matching taints. + // + // +optional + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + + // Volumes represents named volumes in a pod that may be accessed by any container in the pod. + // + // +optional + Volumes []corev1.Volume `json:"volumes,omitempty"` + + // TopologySpreadConstraints describes how a group of Pods ought to spread across topology + // domains. Scheduler will schedule Pods in a way which abides by the constraints. + // All topologySpreadConstraints are ANDed. + // + // +optional + TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` +} + +// ContainerSpec defines container fields for the NGINX container. +type ContainerSpec struct { + // Image is the NGINX image to use. + // + // +optional + Image *Image `json:"image,omitempty"` + + // Resources describes the compute resource requirements. + // + // +optional + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + + // Lifecycle describes actions that the management system should take in response to container lifecycle + // events. For the PostStart and PreStop lifecycle handlers, management of the container blocks + // until the action is complete, unless the container process fails, in which case the handler is aborted. + // + // +optional + Lifecycle *corev1.Lifecycle `json:"lifecycle,omitempty"` + + // VolumeMounts describe the mounting of Volumes within a container. + // + // +optional + VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"` +} + +// Image is the NGINX image to use. +type Image struct { + // Repository is the image path. + // Default is ghcr.io/nginx/nginx-gateway-fabric/nginx. + // + // +optional + Repository *string `json:"repository,omitempty"` + // Tag is the image tag to use. Default matches the tag of the control plane. + // + // +optional + Tag *string `json:"tag,omitempty"` + // PullPolicy describes a policy for if/when to pull a container image. + // + // +optional + // +kubebuilder:default:=IfNotPresent + PullPolicy *PullPolicy `json:"pullPolicy,omitempty"` +} + +// PullPolicy describes a policy for if/when to pull a container image. +// +kubebuilder:validation:Enum=Always;Never;IfNotPresent +type PullPolicy corev1.PullPolicy + +const ( + // PullAlways means that kubelet always attempts to pull the latest image. Container will fail if the pull fails. + PullAlways PullPolicy = PullPolicy(corev1.PullAlways) + // PullNever means that kubelet never pulls an image, but only uses a local image. Container will fail if the + // image isn't present. + PullNever PullPolicy = PullPolicy(corev1.PullNever) + // PullIfNotPresent means that kubelet pulls if the image isn't present on disk. Container will fail if the image + // isn't present and the pull fails. + PullIfNotPresent PullPolicy = PullPolicy(corev1.PullIfNotPresent) +) + +// ServiceSpec is the configuration for the NGINX Service. +type ServiceSpec struct { + // ServiceType describes ingress method for the Service. + // + // +optional + // +kubebuilder:default:=LoadBalancer + ServiceType *ServiceType `json:"type,omitempty"` + + // ExternalTrafficPolicy describes how nodes distribute service traffic they + // receive on one of the Service's "externally-facing" addresses (NodePorts, ExternalIPs, + // and LoadBalancer IPs. + // + // +optional + // +kubebuilder:default:=Local + ExternalTrafficPolicy *ExternalTrafficPolicy `json:"externalTrafficPolicy,omitempty"` + + // LoadBalancerIP is a static IP address for the load balancer. Requires service type to be LoadBalancer. + // + // +optional + LoadBalancerIP *string `json:"loadBalancerIP,omitempty"` + + // Annotations contain any Service-specific annotations. + // + // +optional + Annotations map[string]string `json:"annotations,omitempty"` + + // LoadBalancerSourceRanges are the IP ranges (CIDR) that are allowed to access the load balancer. + // Requires service type to be LoadBalancer. + // + // +optional + LoadBalancerSourceRanges []string `json:"loadBalancerSourceRanges,omitempty"` +} + +// ServiceType describes ingress method for the Service. +// +kubebuilder:validation:Enum=ClusterIP;LoadBalancer;NodePort +type ServiceType corev1.ServiceType + +const ( + // ServiceTypeClusterIP means a Service will only be accessible inside the + // cluster, via the cluster IP. + ServiceTypeClusterIP ServiceType = ServiceType(corev1.ServiceTypeClusterIP) + + // ServiceTypeNodePort means a Service will be exposed on one port of + // every node, in addition to 'ClusterIP' type. + ServiceTypeNodePort ServiceType = ServiceType(corev1.ServiceTypeNodePort) + + // ServiceTypeLoadBalancer means a Service will be exposed via an + // external load balancer (if the cloud provider supports it), in addition + // to 'NodePort' type. + ServiceTypeLoadBalancer ServiceType = ServiceType(corev1.ServiceTypeLoadBalancer) +) + +// ExternalTrafficPolicy describes how nodes distribute service traffic they +// receive on one of the Service's "externally-facing" addresses (NodePorts, ExternalIPs, +// and LoadBalancer IPs. +// +kubebuilder:validation:Enum=Cluster;Local +type ExternalTrafficPolicy corev1.ServiceExternalTrafficPolicy + +const ( + // ExternalTrafficPolicyCluster routes traffic to all endpoints. + ExternalTrafficPolicyCluster ExternalTrafficPolicy = ExternalTrafficPolicy(corev1.ServiceExternalTrafficPolicyCluster) + + // ExternalTrafficPolicyLocal preserves the source IP of the traffic by + // routing only to endpoints on the same node as the traffic was received on + // (dropping the traffic if there are no local endpoints). + ExternalTrafficPolicyLocal ExternalTrafficPolicy = ExternalTrafficPolicy(corev1.ServiceExternalTrafficPolicyLocal) ) diff --git a/apis/v1alpha2/zz_generated.deepcopy.go b/apis/v1alpha2/zz_generated.deepcopy.go index 6e0856a220..c6420fc2f2 100644 --- a/apis/v1alpha2/zz_generated.deepcopy.go +++ b/apis/v1alpha2/zz_generated.deepcopy.go @@ -6,10 +6,150 @@ package v1alpha2 import ( "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" apisv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContainerSpec) DeepCopyInto(out *ContainerSpec) { + *out = *in + if in.Image != nil { + in, out := &in.Image, &out.Image + *out = new(Image) + (*in).DeepCopyInto(*out) + } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + if in.Lifecycle != nil { + in, out := &in.Lifecycle, &out.Lifecycle + *out = new(v1.Lifecycle) + (*in).DeepCopyInto(*out) + } + if in.VolumeMounts != nil { + in, out := &in.VolumeMounts, &out.VolumeMounts + *out = make([]v1.VolumeMount, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerSpec. +func (in *ContainerSpec) DeepCopy() *ContainerSpec { + if in == nil { + return nil + } + out := new(ContainerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + in.Pod.DeepCopyInto(&out.Pod) + in.Container.DeepCopyInto(&out.Container) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentSpec. +func (in *DeploymentSpec) DeepCopy() *DeploymentSpec { + if in == nil { + return nil + } + out := new(DeploymentSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Image) DeepCopyInto(out *Image) { + *out = *in + if in.Repository != nil { + in, out := &in.Repository, &out.Repository + *out = new(string) + **out = **in + } + if in.Tag != nil { + in, out := &in.Tag, &out.Tag + *out = new(string) + **out = **in + } + if in.PullPolicy != nil { + in, out := &in.PullPolicy, &out.PullPolicy + *out = new(PullPolicy) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Image. +func (in *Image) DeepCopy() *Image { + if in == nil { + return nil + } + out := new(Image) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesSpec) DeepCopyInto(out *KubernetesSpec) { + *out = *in + if in.Deployment != nil { + in, out := &in.Deployment, &out.Deployment + *out = new(DeploymentSpec) + (*in).DeepCopyInto(*out) + } + if in.Service != nil { + in, out := &in.Service, &out.Service + *out = new(ServiceSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesSpec. +func (in *KubernetesSpec) DeepCopy() *KubernetesSpec { + if in == nil { + return nil + } + out := new(KubernetesSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Metrics) DeepCopyInto(out *Metrics) { + *out = *in + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(int32) + **out = **in + } + if in.Disable != nil { + in, out := &in.Disable, &out.Disable + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Metrics. +func (in *Metrics) DeepCopy() *Metrics { + if in == nil { + return nil + } + out := new(Metrics) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NginxLogging) DeepCopyInto(out *NginxLogging) { *out = *in @@ -18,6 +158,11 @@ func (in *NginxLogging) DeepCopyInto(out *NginxLogging) { *out = new(NginxErrorLogLevel) **out = **in } + if in.AgentLevel != nil { + in, out := &in.AgentLevel, &out.AgentLevel + *out = new(AgentLogLevel) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NginxLogging. @@ -136,6 +281,11 @@ func (in *NginxProxySpec) DeepCopyInto(out *NginxProxySpec) { *out = new(Telemetry) (*in).DeepCopyInto(*out) } + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = new(Metrics) + (*in).DeepCopyInto(*out) + } if in.RewriteClientIP != nil { in, out := &in.RewriteClientIP, &out.RewriteClientIP *out = new(RewriteClientIP) @@ -156,6 +306,11 @@ func (in *NginxProxySpec) DeepCopyInto(out *NginxProxySpec) { *out = new(bool) **out = **in } + if in.Kubernetes != nil { + in, out := &in.Kubernetes, &out.Kubernetes + *out = new(KubernetesSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NginxProxySpec. @@ -254,6 +409,59 @@ func (in *ObservabilityPolicySpec) DeepCopy() *ObservabilityPolicySpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodSpec) DeepCopyInto(out *PodSpec) { + *out = *in + if in.TerminationGracePeriodSeconds != nil { + in, out := &in.TerminationGracePeriodSeconds, &out.TerminationGracePeriodSeconds + *out = new(int64) + **out = **in + } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(v1.Affinity) + (*in).DeepCopyInto(*out) + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]v1.Volume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.TopologySpreadConstraints != nil { + in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints + *out = make([]v1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodSpec. +func (in *PodSpec) DeepCopy() *PodSpec { + if in == nil { + return nil + } + out := new(PodSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RewriteClientIP) DeepCopyInto(out *RewriteClientIP) { *out = *in @@ -299,6 +507,48 @@ func (in *RewriteClientIPAddress) DeepCopy() *RewriteClientIPAddress { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { + *out = *in + if in.ServiceType != nil { + in, out := &in.ServiceType, &out.ServiceType + *out = new(ServiceType) + **out = **in + } + if in.ExternalTrafficPolicy != nil { + in, out := &in.ExternalTrafficPolicy, &out.ExternalTrafficPolicy + *out = new(ExternalTrafficPolicy) + **out = **in + } + if in.LoadBalancerIP != nil { + in, out := &in.LoadBalancerIP, &out.LoadBalancerIP + *out = new(string) + **out = **in + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.LoadBalancerSourceRanges != nil { + in, out := &in.LoadBalancerSourceRanges, &out.LoadBalancerSourceRanges + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceSpec. +func (in *ServiceSpec) DeepCopy() *ServiceSpec { + if in == nil { + return nil + } + out := new(ServiceSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Telemetry) DeepCopyInto(out *Telemetry) { *out = *in diff --git a/charts/nginx-gateway-fabric/README.md b/charts/nginx-gateway-fabric/README.md index 6eaa78d31f..142c5eee65 100644 --- a/charts/nginx-gateway-fabric/README.md +++ b/charts/nginx-gateway-fabric/README.md @@ -112,13 +112,13 @@ By default, the NGINX Gateway Fabric helm chart deploys a LoadBalancer Service. To use a NodePort Service instead: ```shell -helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set service.type=NodePort +helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set nginx.service.type=NodePort ``` To disable the creation of a Service: ```shell -helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set service.create=false +helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set nginx.service.create=false ``` ## Upgrading the Chart @@ -253,65 +253,69 @@ kubectl kustomize https://github.com/nginx/nginx-gateway-fabric/config/crd/gatew The following table lists the configurable parameters of the NGINX Gateway Fabric chart and their default values. +> More granular configuration options may not show up in this table. +> Viewing the `values.yaml` file directly can show all available options. + | Key | Description | Type | Default | |-----|-------------|------|---------| -| `affinity` | The affinity of the NGINX Gateway Fabric pod. | object | `{}` | -| `extraVolumes` | extraVolumes for the NGINX Gateway Fabric pod. Use in conjunction with nginxGateway.extraVolumeMounts and nginx.extraVolumeMounts to mount additional volumes to the containers. | list | `[]` | -| `metrics.enable` | Enable exposing metrics in the Prometheus format. | bool | `true` | -| `metrics.port` | Set the port where the Prometheus metrics are exposed. | int | `9113` | -| `metrics.secure` | Enable serving metrics via https. By default metrics are served via http. Please note that this endpoint will be secured with a self-signed certificate. | bool | `false` | +| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"config":{},"container":{},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` | | `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. | object | `{}` | +| `nginx.container` | The container configuration for the NGINX container. | object | `{}` | | `nginx.debug` | Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. | bool | `false` | -| `nginx.extraVolumeMounts` | extraVolumeMounts are the additional volume mounts for the nginx container. | list | `[]` | -| `nginx.image.pullPolicy` | | string | `"Always"` | | `nginx.image.repository` | The NGINX image to use. | string | `"ghcr.io/nginx/nginx-gateway-fabric/nginx"` | -| `nginx.image.tag` | | string | `"edge"` | -| `nginx.lifecycle` | The lifecycle of the nginx container. | object | `{}` | -| `nginx.plus` | Is NGINX Plus image being used | bool | `false` | +| `nginx.imagePullSecret` | The name of the secret containing docker registry credentials. Secret must exist in the same namespace as the helm release. The control plane will copy this secret into any namespace where NGINX is deployed. | string | `""` | +| `nginx.imagePullSecrets` | A list of secret names containing docker registry credentials. Secrets must exist in the same namespace as the helm release. The control plane will copy these secrets into any namespace where NGINX is deployed. | list | `[]` | +| `nginx.kind` | The kind of NGINX deployment. | string | `"deployment"` | +| `nginx.plus` | Is NGINX Plus image being used. | bool | `false` | +| `nginx.pod` | The pod configuration for the NGINX data plane pod. | object | `{}` | +| `nginx.replicas` | The number of replicas of the NGINX Deployment. | int | `1` | +| `nginx.service` | The service configuration for the NGINX data plane. | object | `{"externalTrafficPolicy":"Local","type":"LoadBalancer"}` | +| `nginx.service.externalTrafficPolicy` | The externalTrafficPolicy of the service. The value Local preserves the client source IP. | string | `"Local"` | +| `nginx.service.type` | The type of service to create for the NGINX data plane. | string | `"LoadBalancer"` | | `nginx.usage.caSecretName` | The name of the Secret containing the NGINX Instance Manager CA certificate. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` | | `nginx.usage.clientSSLSecretName` | The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` | | `nginx.usage.endpoint` | The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com | string | `""` | | `nginx.usage.resolver` | The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. | string | `""` | | `nginx.usage.secretName` | The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"nplus-license"` | | `nginx.usage.skipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` | +| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` | +| `nginxGateway.affinity` | The affinity of the NGINX Gateway Fabric control plane pod. | object | `{}` | | `nginxGateway.config.logging.level` | Log level. | string | `"info"` | | `nginxGateway.configAnnotations` | Set of custom annotations for NginxGateway objects. | object | `{}` | | `nginxGateway.extraVolumeMounts` | extraVolumeMounts are the additional volume mounts for the nginx-gateway container. | list | `[]` | +| `nginxGateway.extraVolumes` | extraVolumes for the NGINX Gateway Fabric control plane pod. Use in conjunction with nginxGateway.extraVolumeMounts mount additional volumes to the container. | list | `[]` | | `nginxGateway.gatewayClassAnnotations` | Set of custom annotations for GatewayClass objects. | object | `{}` | | `nginxGateway.gatewayClassName` | The name of the GatewayClass that will be created as part of this release. Every NGINX Gateway Fabric must have a unique corresponding GatewayClass resource. NGINX Gateway Fabric only processes resources that belong to its class - i.e. have the "gatewayClassName" field resource equal to the class. | string | `"nginx"` | | `nginxGateway.gatewayControllerName` | The name of the Gateway controller. The controller name must be of the form: DOMAIN/PATH. The controller's domain is gateway.nginx.org. | string | `"gateway.nginx.org/nginx-gateway-controller"` | | `nginxGateway.gwAPIExperimentalFeatures.enable` | Enable the experimental features of Gateway API which are supported by NGINX Gateway Fabric. Requires the Gateway APIs installed from the experimental channel. | bool | `false` | -| `nginxGateway.image.pullPolicy` | | string | `"Always"` | +| `nginxGateway.image` | The image configuration for the NGINX Gateway Fabric control plane. | object | `{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"}` | | `nginxGateway.image.repository` | The NGINX Gateway Fabric image to use | string | `"ghcr.io/nginx/nginx-gateway-fabric"` | -| `nginxGateway.image.tag` | | string | `"edge"` | | `nginxGateway.kind` | The kind of the NGINX Gateway Fabric installation - currently, only deployment is supported. | string | `"deployment"` | | `nginxGateway.leaderElection.enable` | Enable leader election. Leader election is used to avoid multiple replicas of the NGINX Gateway Fabric reporting the status of the Gateway API resources. If not enabled, all replicas of NGINX Gateway Fabric will update the statuses of the Gateway API resources. | bool | `true` | | `nginxGateway.leaderElection.lockName` | The name of the leader election lock. A Lease object with this name will be created in the same Namespace as the controller. | string | Autogenerated if not set or set to "". | | `nginxGateway.lifecycle` | The lifecycle of the nginx-gateway container. | object | `{}` | +| `nginxGateway.metrics.enable` | Enable exposing metrics in the Prometheus format. | bool | `true` | +| `nginxGateway.metrics.port` | Set the port where the Prometheus metrics are exposed. | int | `9113` | +| `nginxGateway.metrics.secure` | Enable serving metrics via https. By default metrics are served via http. Please note that this endpoint will be secured with a self-signed certificate. | bool | `false` | +| `nginxGateway.nodeSelector` | The nodeSelector of the NGINX Gateway Fabric control plane pod. | object | `{}` | | `nginxGateway.podAnnotations` | Set of custom annotations for the NGINX Gateway Fabric pods. | object | `{}` | | `nginxGateway.productTelemetry.enable` | Enable the collection of product telemetry. | bool | `true` | | `nginxGateway.readinessProbe.enable` | Enable the /readyz endpoint on the control plane. | bool | `true` | | `nginxGateway.readinessProbe.initialDelaySeconds` | The number of seconds after the Pod has started before the readiness probes are initiated. | int | `3` | | `nginxGateway.readinessProbe.port` | Port in which the readiness endpoint is exposed. | int | `8081` | -| `nginxGateway.replicaCount` | The number of replicas of the NGINX Gateway Fabric Deployment. | int | `1` | +| `nginxGateway.replicas` | The number of replicas of the NGINX Gateway Fabric Deployment. | int | `1` | | `nginxGateway.resources` | The resource requests and/or limits of the nginx-gateway container. | object | `{}` | +| `nginxGateway.service` | The service configuration for the NGINX Gateway Fabric control plane. | object | `{"annotations":{}}` | | `nginxGateway.service.annotations` | The annotations of the NGINX Gateway Fabric control plane service. | object | `{}` | +| `nginxGateway.serviceAccount` | The serviceaccount configuration for the NGINX Gateway Fabric control plane. | object | `{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""}` | +| `nginxGateway.serviceAccount.annotations` | Set of custom annotations for the NGINX Gateway Fabric control plane service account. | object | `{}` | +| `nginxGateway.serviceAccount.imagePullSecret` | The name of the secret containing docker registry credentials for the control plane. Secret must exist in the same namespace as the helm release. | string | `""` | +| `nginxGateway.serviceAccount.imagePullSecrets` | A list of secret names containing docker registry credentials for the control plane. Secrets must exist in the same namespace as the helm release. | list | `[]` | +| `nginxGateway.serviceAccount.name` | The name of the service account of the NGINX Gateway Fabric control plane pods. Used for RBAC. | string | Autogenerated if not set or set to "" | | `nginxGateway.snippetsFilters.enable` | Enable SnippetsFilters feature. SnippetsFilters allow inserting NGINX configuration into the generated NGINX config for HTTPRoute and GRPCRoute resources. | bool | `false` | -| `nodeSelector` | The nodeSelector of the NGINX Gateway Fabric pod. | object | `{}` | -| `service.annotations` | The annotations of the NGINX Gateway Fabric service. | object | `{}` | -| `service.create` | Creates a service to expose the NGINX Gateway Fabric pods. | bool | `true` | -| `service.externalTrafficPolicy` | The externalTrafficPolicy of the service. The value Local preserves the client source IP. | string | `"Local"` | -| `service.loadBalancerIP` | The static IP address for the load balancer. Requires service.type set to LoadBalancer. | string | `""` | -| `service.loadBalancerSourceRanges` | The IP ranges (CIDR) that are allowed to access the load balancer. Requires service.type set to LoadBalancer. | list | `[]` | -| `service.ports` | A list of ports to expose through the NGINX Gateway Fabric service. Update it to match the listener ports from your Gateway resource. Follows the conventional Kubernetes yaml syntax for service ports. | list | `[{"name":"http","port":80,"protocol":"TCP","targetPort":80},{"name":"https","port":443,"protocol":"TCP","targetPort":443}]` | -| `service.type` | The type of service to create for the NGINX Gateway Fabric. | string | `"LoadBalancer"` | -| `serviceAccount.annotations` | Set of custom annotations for the NGINX Gateway Fabric service account. | object | `{}` | -| `serviceAccount.imagePullSecret` | The name of the secret containing docker registry credentials. Secret must exist in the same namespace as the helm release. | string | `""` | -| `serviceAccount.imagePullSecrets` | A list of secret names containing docker registry credentials. Secrets must exist in the same namespace as the helm release. | list | `[]` | -| `serviceAccount.name` | The name of the service account of the NGINX Gateway Fabric pods. Used for RBAC. | string | Autogenerated if not set or set to "" | -| `terminationGracePeriodSeconds` | The termination grace period of the NGINX Gateway Fabric pod. | int | `30` | -| `tolerations` | Tolerations for the NGINX Gateway Fabric pod. | list | `[]` | -| `topologySpreadConstraints` | The topology spread constraints for the NGINX Gateway Fabric pod. | list | `[]` | +| `nginxGateway.terminationGracePeriodSeconds` | The termination grace period of the NGINX Gateway Fabric control plane pod. | int | `30` | +| `nginxGateway.tolerations` | Tolerations for the NGINX Gateway Fabric control plane pod. | list | `[]` | +| `nginxGateway.topologySpreadConstraints` | The topology spread constraints for the NGINX Gateway Fabric control plane pod. | list | `[]` | ---------------------------------------------- Autogenerated from chart metadata using [helm-docs](https://github.com/norwoodj/helm-docs) diff --git a/charts/nginx-gateway-fabric/README.md.gotmpl b/charts/nginx-gateway-fabric/README.md.gotmpl index f89de6bd00..6306d2a647 100644 --- a/charts/nginx-gateway-fabric/README.md.gotmpl +++ b/charts/nginx-gateway-fabric/README.md.gotmpl @@ -110,13 +110,13 @@ By default, the NGINX Gateway Fabric helm chart deploys a LoadBalancer Service. To use a NodePort Service instead: ```shell -helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set service.type=NodePort +helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set nginx.service.type=NodePort ``` To disable the creation of a Service: ```shell -helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set service.create=false +helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway --set nginx.service.create=false ``` ## Upgrading the Chart @@ -251,6 +251,9 @@ kubectl kustomize https://github.com/nginx/nginx-gateway-fabric/config/crd/gatew The following table lists the configurable parameters of the NGINX Gateway Fabric chart and their default values. +> More granular configuration options may not show up in this table. +> Viewing the `values.yaml` file directly can show all available options. + {{ template "chart.valuesTable" . }} ---------------------------------------------- diff --git a/charts/nginx-gateway-fabric/templates/_helpers.tpl b/charts/nginx-gateway-fabric/templates/_helpers.tpl index 65b6c5e6f8..2a137d5fbd 100644 --- a/charts/nginx-gateway-fabric/templates/_helpers.tpl +++ b/charts/nginx-gateway-fabric/templates/_helpers.tpl @@ -78,7 +78,7 @@ app.kubernetes.io/instance: {{ .Release.Name }} Create the name of the ServiceAccount to use */}} {{- define "nginx-gateway.serviceAccountName" -}} -{{- default (include "nginx-gateway.fullname" .) .Values.serviceAccount.name }} +{{- default (include "nginx-gateway.fullname" .) .Values.nginxGateway.serviceAccount.name }} {{- end }} {{/* diff --git a/charts/nginx-gateway-fabric/templates/clusterrole.yaml b/charts/nginx-gateway-fabric/templates/clusterrole.yaml index 830fe1b391..2ae9c5a2c0 100644 --- a/charts/nginx-gateway-fabric/templates/clusterrole.yaml +++ b/charts/nginx-gateway-fabric/templates/clusterrole.yaml @@ -7,14 +7,25 @@ metadata: rules: - apiGroups: - "" + - apps resources: - - namespaces - - services - secrets - - pods -{{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }} - configmaps -{{- end }} + - serviceaccounts + - services + - deployments + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - "" + resources: + - namespaces + - pods verbs: - get - list @@ -135,4 +146,6 @@ rules: - {{ include "nginx-gateway.scc-name" . }} verbs: - use + - create + - watch {{- end }} diff --git a/charts/nginx-gateway-fabric/templates/configmap.yaml b/charts/nginx-gateway-fabric/templates/configmap.yaml deleted file mode 100644 index 8b99c60650..0000000000 --- a/charts/nginx-gateway-fabric/templates/configmap.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: nginx-includes-bootstrap - namespace: {{ .Release.Namespace }} - labels: - {{- include "nginx-gateway.labels" . | nindent 4 }} -data: - main.conf: | - {{- if and .Values.nginx.config .Values.nginx.config.logging .Values.nginx.config.logging.errorLevel }} - error_log stderr {{ .Values.nginx.config.logging.errorLevel }}; - {{ else }} - error_log stderr info; - {{- end }} - {{- if .Values.nginx.plus }} - mgmt.conf: | - mgmt { - {{- if .Values.nginx.usage.endpoint }} - usage_report endpoint={{ .Values.nginx.usage.endpoint }}; - {{- end }} - {{- if .Values.nginx.usage.skipVerify }} - ssl_verify off; - {{- end }} - {{- if .Values.nginx.usage.caSecretName }} - ssl_trusted_certificate /etc/nginx/certs-bootstrap/ca.crt; - {{- end }} - {{- if .Values.nginx.usage.clientSSLSecretName }} - ssl_certificate /etc/nginx/certs-bootstrap/tls.crt; - ssl_certificate_key /etc/nginx/certs-bootstrap/tls.key; - {{- end }} - enforce_initial_report off; - deployment_context /etc/nginx/main-includes/deployment_ctx.json; - } - {{- end }} diff --git a/charts/nginx-gateway-fabric/templates/deployment.yaml b/charts/nginx-gateway-fabric/templates/deployment.yaml index 7a254a3bf6..0c9a6c06b7 100644 --- a/charts/nginx-gateway-fabric/templates/deployment.yaml +++ b/charts/nginx-gateway-fabric/templates/deployment.yaml @@ -7,7 +7,7 @@ metadata: labels: {{- include "nginx-gateway.labels" . | nindent 4 }} spec: - replicas: {{ .Values.nginxGateway.replicaCount }} + replicas: {{ .Values.nginxGateway.replicas }} selector: matchLabels: {{- include "nginx-gateway.selectorLabels" . | nindent 6 }} @@ -15,24 +15,20 @@ spec: metadata: labels: {{- include "nginx-gateway.selectorLabels" . | nindent 8 }} - {{- if or .Values.nginxGateway.podAnnotations .Values.metrics.enable }} + {{- if or .Values.nginxGateway.podAnnotations .Values.nginxGateway.metrics.enable }} annotations: {{- if .Values.nginxGateway.podAnnotations }} {{- toYaml .Values.nginxGateway.podAnnotations | nindent 8 }} {{- end }} - {{- if .Values.metrics.enable }} + {{- if .Values.nginxGateway.metrics.enable }} prometheus.io/scrape: "true" - prometheus.io/port: "{{ .Values.metrics.port }}" - {{- if .Values.metrics.secure }} + prometheus.io/port: "{{ .Values.nginxGateway.metrics.port }}" + {{- if .Values.nginxGateway.metrics.secure }} prometheus.io/scheme: "https" {{- end }} {{- end }} {{- end }} spec: - {{- if .Values.topologySpreadConstraints }} - topologySpreadConstraints: - {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} - {{- end }} containers: - args: - static-mode @@ -61,9 +57,9 @@ spec: - --usage-report-client-ssl-secret={{ .Values.nginx.usage.clientSSLSecretName }} {{- end }} {{- end }} - {{- if .Values.metrics.enable }} - - --metrics-port={{ .Values.metrics.port }} - {{- if .Values.metrics.secure }} + {{- if .Values.nginxGateway.metrics.enable }} + - --metrics-port={{ .Values.nginxGateway.metrics.port }} + {{- if .Values.nginxGateway.metrics.secure }} - --metrics-secure-serving {{- end }} {{- else }} @@ -89,10 +85,6 @@ spec: - --snippets-filters {{- end }} env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - name: POD_NAMESPACE valueFrom: fieldRef: @@ -105,6 +97,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid + - name: INSTANCE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/instance'] + - name: IMAGE_NAME + value: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }} image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }} imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }} name: nginx-gateway @@ -119,9 +117,9 @@ spec: ports: - name: agent-grpc containerPort: 8443 - {{- if .Values.metrics.enable }} + {{- if .Values.nginxGateway.metrics.enable }} - name: metrics - containerPort: {{ .Values.metrics.port }} + containerPort: {{ .Values.nginxGateway.metrics.port }} {{- end }} {{- if .Values.nginxGateway.readinessProbe.enable }} - name: health @@ -146,24 +144,28 @@ spec: {{- with .Values.nginxGateway.extraVolumeMounts -}} {{ toYaml . | nindent 8 }} {{- end }} - terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} - {{- if .Values.affinity }} + {{- if .Values.nginxGateway.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml .Values.nginxGateway.topologySpreadConstraints | nindent 8 }} + {{- end }} + terminationGracePeriodSeconds: {{ .Values.nginxGateway.terminationGracePeriodSeconds }} + {{- if .Values.nginxGateway.affinity }} affinity: - {{- toYaml .Values.affinity | nindent 8 }} + {{- toYaml .Values.nginxGateway.affinity | nindent 8 }} {{- end }} serviceAccountName: {{ include "nginx-gateway.serviceAccountName" . }} securityContext: fsGroup: 1001 runAsNonRoot: true - {{- if .Values.tolerations }} + {{- if .Values.nginxGateway.tolerations }} tolerations: - {{- toYaml .Values.tolerations | nindent 6 }} + {{- toYaml .Values.nginxGateway.tolerations | nindent 6 }} {{- end }} - {{- if .Values.nodeSelector }} + {{- if .Values.nginxGateway.nodeSelector }} nodeSelector: - {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- toYaml .Values.nginxGateway.nodeSelector | nindent 8 }} {{- end }} - {{- with .Values.extraVolumes -}} + {{- with .Values.nginxGateway.extraVolumes -}} {{ toYaml . | nindent 6 }} {{- end }} {{- end }} diff --git a/charts/nginx-gateway-fabric/templates/gatewayclass.yaml b/charts/nginx-gateway-fabric/templates/gatewayclass.yaml index aecd54e8ad..b6905cd33c 100644 --- a/charts/nginx-gateway-fabric/templates/gatewayclass.yaml +++ b/charts/nginx-gateway-fabric/templates/gatewayclass.yaml @@ -12,10 +12,8 @@ metadata: {{- end }} spec: controllerName: {{ .Values.nginxGateway.gatewayControllerName }} - {{- if .Values.nginx.config }} parametersRef: group: gateway.nginx.org kind: NginxProxy name: {{ include "nginx-gateway.proxy-config-name" . }} namespace: {{ .Release.Namespace }} - {{- end }} diff --git a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml index bc4105ee37..1dd6f44155 100644 --- a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml +++ b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml @@ -1,4 +1,3 @@ -{{- if .Values.nginx.config }} apiVersion: gateway.nginx.org/v1alpha2 kind: NginxProxy metadata: @@ -7,5 +6,25 @@ metadata: labels: {{- include "nginx-gateway.labels" . | nindent 4 }} spec: + {{- if .Values.nginx.config }} {{- toYaml .Values.nginx.config | nindent 2 }} -{{- end }} + {{- end }} + kubernetes: + {{- if eq .Values.nginx.kind "deployment" }} + deployment: + replicas: {{ .Values.nginx.replicas }} + {{- if .Values.nginx.pod }} + pod: + {{- toYaml .Values.nginx.pod | nindent 8 }} + {{- end }} + container: + {{- if .Values.nginx.container }} + {{- toYaml .Values.nginx.container | nindent 8 }} + {{- end }} + image: + {{- toYaml .Values.nginx.image | nindent 10 }} + {{- end }} + {{- if .Values.nginx.service }} + service: + {{- toYaml .Values.nginx.service | nindent 6 }} + {{- end }} diff --git a/charts/nginx-gateway-fabric/templates/serviceaccount.yaml b/charts/nginx-gateway-fabric/templates/serviceaccount.yaml index 069a2066b9..fa3439759d 100644 --- a/charts/nginx-gateway-fabric/templates/serviceaccount.yaml +++ b/charts/nginx-gateway-fabric/templates/serviceaccount.yaml @@ -6,14 +6,14 @@ metadata: labels: {{- include "nginx-gateway.labels" . | nindent 4 }} annotations: - {{- toYaml .Values.serviceAccount.annotations | nindent 4 }} -{{- if or .Values.serviceAccount.imagePullSecret .Values.serviceAccount.imagePullSecrets }} + {{- toYaml .Values.nginxGateway.serviceAccount.annotations | nindent 4 }} +{{- if or .Values.nginxGateway.serviceAccount.imagePullSecret .Values.nginxGateway.serviceAccount.imagePullSecrets }} imagePullSecrets: - {{- if .Values.serviceAccount.imagePullSecret }} - - name: {{ .Values.serviceAccount.imagePullSecret }} + {{- if .Values.nginxGateway.serviceAccount.imagePullSecret }} + - name: {{ .Values.nginxGateway.serviceAccount.imagePullSecret }} {{- end }} - {{- if .Values.serviceAccount.imagePullSecrets }} - {{- range .Values.serviceAccount.imagePullSecrets }} + {{- if .Values.nginxGateway.serviceAccount.imagePullSecrets }} + {{- range .Values.nginxGateway.serviceAccount.imagePullSecrets }} - name: {{ . }} {{- end }} {{- end }} diff --git a/charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml b/charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml deleted file mode 100644 index 6e85efffeb..0000000000 --- a/charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: nginx-agent-config - namespace: {{ .Release.Namespace }} -data: - nginx-agent.conf: |- - command: - server: - host: {{ include "nginx-gateway.fullname" . }}.{{ .Release.Namespace }}.svc - port: 443 - allowed_directories: - - /etc/nginx - - /usr/share/nginx - - /var/run/nginx - features: - - connection - - configuration - - certificates - - metrics - {{- if .Values.nginx.plus }} - - api-action - {{- end }} - log: - level: debug - collector: - receivers: - host_metrics: - collection_interval: 1m0s - initial_delay: 1s - scrapers: - cpu: {} - memory: {} - disk: {} - network: {} - filesystem: {} - processors: - batch: {} - exporters: - prometheus_exporter: - server: - host: "0.0.0.0" - port: 9113 diff --git a/charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml b/charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml deleted file mode 100644 index bb04bf46eb..0000000000 --- a/charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml +++ /dev/null @@ -1,204 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tmp-nginx-deployment - namespace: {{ .Release.Namespace }} -spec: - selector: - matchLabels: - app.kubernetes.io/name: tmp-nginx-deployment - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - labels: - app.kubernetes.io/name: tmp-nginx-deployment - app.kubernetes.io/instance: {{ .Release.Name }} - annotations: - {{- if .Values.metrics.enable }} - prometheus.io/scrape: "true" - prometheus.io/port: "{{ .Values.metrics.port }}" - {{- if .Values.metrics.secure }} - prometheus.io/scheme: "https" - {{- end }} - {{- end }} - spec: - initContainers: - - name: init - image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }} - imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }} - command: - - /usr/bin/gateway - - initialize - - --source - - /agent/nginx-agent.conf - - --destination - - /etc/nginx-agent - - --source - - /includes/main.conf - - --destination - - /etc/nginx/main-includes - {{- if .Values.nginx.plus }} - - --source - - /includes/mgmt.conf - - --nginx-plus - - --destination - - /etc/nginx/main-includes - {{- end }} - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - securityContext: - seccompProfile: - type: RuntimeDefault - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsUser: 101 - runAsGroup: 1001 - volumeMounts: - - name: nginx-agent-config - mountPath: /agent - - name: nginx-agent - mountPath: /etc/nginx-agent - - name: nginx-includes-bootstrap - mountPath: /includes - - name: nginx-main-includes - mountPath: /etc/nginx/main-includes - containers: - - image: {{ .Values.nginx.image.repository }}:{{ .Values.nginx.image.tag | default .Chart.AppVersion }} - imagePullPolicy: {{ .Values.nginx.image.pullPolicy }} - name: nginx - {{- if .Values.nginx.lifecycle }} - lifecycle: - {{- toYaml .Values.nginx.lifecycle | nindent 10 }} - {{- end }} - ports: - - containerPort: 80 - name: http - - containerPort: 443 - name: https - - name: metrics - containerPort: {{ .Values.metrics.port }} - securityContext: - seccompProfile: - type: RuntimeDefault - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - runAsUser: 101 - runAsGroup: 1001 - volumeMounts: - - name: nginx-agent - mountPath: /etc/nginx-agent - - name: nginx-agent-log - mountPath: /var/log/nginx-agent - - name: nginx-conf - mountPath: /etc/nginx/conf.d - - name: nginx-stream-conf - mountPath: /etc/nginx/stream-conf.d - - name: nginx-main-includes - mountPath: /etc/nginx/main-includes - - name: nginx-secrets - mountPath: /etc/nginx/secrets - - name: nginx-run - mountPath: /var/run/nginx - - name: nginx-cache - mountPath: /var/cache/nginx - - name: nginx-includes - mountPath: /etc/nginx/includes - {{- if .Values.nginx.plus }} - - name: nginx-lib - mountPath: /var/lib/nginx/state - {{- if .Values.nginx.usage.secretName }} - - name: nginx-plus-license - mountPath: /etc/nginx/license.jwt - subPath: license.jwt - {{- end }} - {{- if or .Values.nginx.usage.caSecretName .Values.nginx.usage.clientSSLSecretName }} - - name: nginx-plus-usage-certs - mountPath: /etc/nginx/certs-bootstrap/ - {{- end }} - {{- end }} - {{- with .Values.nginx.extraVolumeMounts -}} - {{ toYaml . | nindent 8 }} - {{- end }} - {{- if .Values.nginx.debug }} - command: - - "/bin/sh" - args: - - "-c" - - "rm -rf /var/run/nginx/*.sock && nginx-debug -g 'daemon off;'" - {{- end }} - terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} - {{- if .Values.affinity }} - affinity: - {{- toYaml .Values.affinity | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "nginx-gateway.serviceAccountName" . }} - securityContext: - fsGroup: 1001 - runAsNonRoot: true - {{- if .Values.tolerations }} - tolerations: - {{- toYaml .Values.tolerations | nindent 6 }} - {{- end }} - {{- if .Values.nodeSelector }} - nodeSelector: - {{- toYaml .Values.nodeSelector | nindent 8 }} - {{- end }} - volumes: - - name: nginx-agent - emptyDir: {} - - name: nginx-agent-config - configMap: - name: nginx-agent-config - - name: nginx-agent-log - emptyDir: {} - - name: nginx-conf - emptyDir: {} - - name: nginx-stream-conf - emptyDir: {} - - name: nginx-main-includes - emptyDir: {} - - name: nginx-secrets - emptyDir: {} - - name: nginx-run - emptyDir: {} - - name: nginx-cache - emptyDir: {} - - name: nginx-includes - emptyDir: {} - - name: nginx-includes-bootstrap - configMap: - name: nginx-includes-bootstrap - {{- if .Values.nginx.plus }} - - name: nginx-lib - emptyDir: {} - {{- if .Values.nginx.usage.secretName }} - - name: nginx-plus-license - secret: - secretName: {{ .Values.nginx.usage.secretName }} - {{- end }} - {{- if or .Values.nginx.usage.caSecretName .Values.nginx.usage.clientSSLSecretName }} - - name: nginx-plus-usage-certs - projected: - sources: - {{- if .Values.nginx.usage.caSecretName }} - - secret: - name: {{ .Values.nginx.usage.caSecretName }} - {{- end }} - {{- if .Values.nginx.usage.clientSSLSecretName }} - - secret: - name: {{ .Values.nginx.usage.clientSSLSecretName }} - {{- end }} - {{- end }} - {{- end }} - {{- with .Values.extraVolumes -}} - {{ toYaml . | nindent 6 }} - {{- end }} diff --git a/charts/nginx-gateway-fabric/templates/tmp-nginx-service.yaml b/charts/nginx-gateway-fabric/templates/tmp-nginx-service.yaml deleted file mode 100644 index 6b82fd1e78..0000000000 --- a/charts/nginx-gateway-fabric/templates/tmp-nginx-service.yaml +++ /dev/null @@ -1,36 +0,0 @@ -{{- if .Values.service.create }} -apiVersion: v1 -kind: Service -metadata: - name: tmp-nginx-deployment - namespace: {{ .Release.Namespace }} - labels: - {{- include "nginx-gateway.labels" . | nindent 4 }} -{{- if .Values.service.annotations }} - annotations: -{{ toYaml .Values.service.annotations | indent 4 }} -{{- end }} -spec: -{{- if or (eq .Values.service.type "LoadBalancer") (eq .Values.service.type "NodePort") }} - {{- if .Values.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} - {{- end }} -{{- end }} - type: {{ .Values.service.type }} -{{- if eq .Values.service.type "LoadBalancer" }} - {{- if .Values.service.loadBalancerIP }} - loadBalancerIP: {{ .Values.service.loadBalancerIP }} - {{- end }} - {{- if .Values.service.loadBalancerSourceRanges }} - loadBalancerSourceRanges: - {{ toYaml .Values.service.loadBalancerSourceRanges | nindent 2 }} - {{- end }} -{{- end}} - selector: - app.kubernetes.io/name: tmp-nginx-deployment - app.kubernetes.io/instance: {{ .Release.Name }} - ports: # Update the following ports to match your Gateway Listener ports -{{- if .Values.service.ports }} -{{ toYaml .Values.service.ports | indent 2 }} -{{ end }} -{{- end }} diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index 4155650fa2..fac20c8777 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -1,58 +1,14 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { - "affinity": { - "description": "The affinity of the NGINX Gateway Fabric pod.", - "required": [], - "title": "affinity", - "type": "object" - }, - "extraVolumes": { - "description": "extraVolumes for the NGINX Gateway Fabric pod. Use in conjunction with\nnginxGateway.extraVolumeMounts and nginx.extraVolumeMounts to mount additional volumes to the containers.", - "items": { - "required": [] - }, - "required": [], - "title": "extraVolumes", - "type": "array" - }, "global": { "description": "Global values are values that can be accessed from any chart or subchart by exactly the same name.", "required": [], "title": "global", "type": "object" }, - "metrics": { - "properties": { - "enable": { - "default": true, - "description": "Enable exposing metrics in the Prometheus format.", - "required": [], - "title": "enable", - "type": "boolean" - }, - "port": { - "default": 9113, - "description": "Set the port where the Prometheus metrics are exposed.", - "maximum": 65535, - "minimum": 1, - "required": [], - "title": "port", - "type": "integer" - }, - "secure": { - "default": false, - "description": "Enable serving metrics via https. By default metrics are served via http.\nPlease note that this endpoint will be secured with a self-signed certificate.", - "required": [], - "title": "secure", - "type": "boolean" - } - }, - "required": [], - "title": "metrics", - "type": "object" - }, "nginx": { + "description": "The nginx section contains the configuration for all NGINX data plane deployments\ninstalled by the NGINX Gateway Fabric control plane.", "properties": { "config": { "description": "The configuration for the data plane that is contained in the NginxProxy resource.", @@ -75,6 +31,17 @@ "logging": { "description": "Logging defines logging related settings for NGINX.", "properties": { + "agentLevel": { + "enum": [ + "debug", + "info", + "error", + "panic", + "fatal" + ], + "required": [], + "type": "string" + }, "errorLevel": { "enum": [ "debug", @@ -93,6 +60,23 @@ "required": [], "type": "object" }, + "metrics": { + "description": "Metrics defines the configuration for Prometheus scraping metrics.", + "properties": { + "disable": { + "required": [], + "type": "boolean" + }, + "port": { + "maximum": 65535, + "minimum": 1, + "required": [], + "type": "integer" + } + }, + "required": [], + "type": "object" + }, "nginxPlus": { "description": "NginxPlus specifies NGINX Plus additional settings.", "properties": { @@ -239,6 +223,12 @@ "title": "config", "type": "object" }, + "container": { + "description": "The container configuration for the NGINX container.", + "required": [], + "title": "container", + "type": "object" + }, "debug": { "default": false, "description": "Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource.", @@ -246,15 +236,6 @@ "title": "debug", "type": "boolean" }, - "extraVolumeMounts": { - "description": "extraVolumeMounts are the additional volume mounts for the nginx container.", - "items": { - "required": [] - }, - "required": [], - "title": "extraVolumeMounts", - "type": "array" - }, "image": { "properties": { "pullPolicy": { @@ -285,19 +266,80 @@ "title": "image", "type": "object" }, - "lifecycle": { - "description": "The lifecycle of the nginx container.", + "imagePullSecret": { + "default": "", + "description": "The name of the secret containing docker registry credentials.\nSecret must exist in the same namespace as the helm release. The control\nplane will copy this secret into any namespace where NGINX is deployed.", "required": [], - "title": "lifecycle", - "type": "object" + "title": "imagePullSecret", + "type": "string" + }, + "imagePullSecrets": { + "description": "A list of secret names containing docker registry credentials.\nSecrets must exist in the same namespace as the helm release. The control\nplane will copy these secrets into any namespace where NGINX is deployed.", + "items": { + "required": [] + }, + "required": [], + "title": "imagePullSecrets", + "type": "array" + }, + "kind": { + "default": "deployment", + "description": "The kind of NGINX deployment.", + "enum": [ + "deployment" + ], + "required": [], + "title": "kind" }, "plus": { "default": false, - "description": "Is NGINX Plus image being used", + "description": "Is NGINX Plus image being used.", "required": [], "title": "plus", "type": "boolean" }, + "pod": { + "description": "The pod configuration for the NGINX data plane pod.", + "required": [], + "title": "pod", + "type": "object" + }, + "replicas": { + "default": 1, + "description": "The number of replicas of the NGINX Deployment.", + "required": [], + "title": "replicas", + "type": "integer" + }, + "service": { + "description": "The service configuration for the NGINX data plane.", + "properties": { + "externalTrafficPolicy": { + "default": "Local", + "description": "The externalTrafficPolicy of the service. The value Local preserves the client source IP.", + "enum": [ + "Cluster", + "Local" + ], + "required": [], + "title": "externalTrafficPolicy" + }, + "type": { + "default": "LoadBalancer", + "description": "The type of service to create for the NGINX data plane.", + "enum": [ + "ClusterIP", + "NodePort", + "LoadBalancer" + ], + "required": [], + "title": "type" + } + }, + "required": [], + "title": "service", + "type": "object" + }, "usage": { "description": "Configuration for NGINX Plus usage reporting.", "properties": { @@ -354,7 +396,14 @@ "type": "object" }, "nginxGateway": { + "description": "The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment.", "properties": { + "affinity": { + "description": "The affinity of the NGINX Gateway Fabric control plane pod.", + "required": [], + "title": "affinity", + "type": "object" + }, "config": { "description": "The dynamic configuration for the control plane that is contained in the NginxGateway resource.", "properties": { @@ -396,6 +445,15 @@ "title": "extraVolumeMounts", "type": "array" }, + "extraVolumes": { + "description": "extraVolumes for the NGINX Gateway Fabric control plane pod. Use in conjunction with\nnginxGateway.extraVolumeMounts mount additional volumes to the container.", + "items": { + "required": [] + }, + "required": [], + "title": "extraVolumes", + "type": "array" + }, "gatewayClassAnnotations": { "description": "Set of custom annotations for GatewayClass objects.", "required": [], @@ -431,6 +489,7 @@ "type": "object" }, "image": { + "description": "The image configuration for the NGINX Gateway Fabric control plane.", "properties": { "pullPolicy": { "default": "Always", @@ -495,6 +554,42 @@ "title": "lifecycle", "type": "object" }, + "metrics": { + "properties": { + "enable": { + "default": true, + "description": "Enable exposing metrics in the Prometheus format.", + "required": [], + "title": "enable", + "type": "boolean" + }, + "port": { + "default": 9113, + "description": "Set the port where the Prometheus metrics are exposed.", + "maximum": 65535, + "minimum": 1, + "required": [], + "title": "port", + "type": "integer" + }, + "secure": { + "default": false, + "description": "Enable serving metrics via https. By default metrics are served via http.\nPlease note that this endpoint will be secured with a self-signed certificate.", + "required": [], + "title": "secure", + "type": "boolean" + } + }, + "required": [], + "title": "metrics", + "type": "object" + }, + "nodeSelector": { + "description": "The nodeSelector of the NGINX Gateway Fabric control plane pod.", + "required": [], + "title": "nodeSelector", + "type": "object" + }, "podAnnotations": { "description": "Set of custom annotations for the NGINX Gateway Fabric pods.", "required": [], @@ -546,11 +641,11 @@ "title": "readinessProbe", "type": "object" }, - "replicaCount": { + "replicas": { "default": 1, "description": "The number of replicas of the NGINX Gateway Fabric Deployment.", "required": [], - "title": "replicaCount", + "title": "replicas", "type": "integer" }, "resources": { @@ -560,6 +655,7 @@ "type": "object" }, "service": { + "description": "The service configuration for the NGINX Gateway Fabric control plane.", "properties": { "annotations": { "description": "The annotations of the NGINX Gateway Fabric control plane service.", @@ -572,6 +668,43 @@ "title": "service", "type": "object" }, + "serviceAccount": { + "description": "The serviceaccount configuration for the NGINX Gateway Fabric control plane.", + "properties": { + "annotations": { + "description": "Set of custom annotations for the NGINX Gateway Fabric control plane service account.", + "required": [], + "title": "annotations", + "type": "object" + }, + "imagePullSecret": { + "default": "", + "description": "The name of the secret containing docker registry credentials for the control plane.\nSecret must exist in the same namespace as the helm release.", + "required": [], + "title": "imagePullSecret", + "type": "string" + }, + "imagePullSecrets": { + "description": "A list of secret names containing docker registry credentials for the control plane.\nSecrets must exist in the same namespace as the helm release.", + "items": { + "required": [] + }, + "required": [], + "title": "imagePullSecrets", + "type": "array" + }, + "name": { + "default": "", + "description": "The name of the service account of the NGINX Gateway Fabric control plane pods. Used for RBAC.", + "required": [], + "title": "name", + "type": "string" + } + }, + "required": [], + "title": "serviceAccount", + "type": "object" + }, "snippetsFilters": { "properties": { "enable": { @@ -585,174 +718,39 @@ "required": [], "title": "snippetsFilters", "type": "object" - } - }, - "required": [ - "gatewayClassName", - "gatewayControllerName" - ], - "title": "nginxGateway", - "type": "object" - }, - "nodeSelector": { - "description": "The nodeSelector of the NGINX Gateway Fabric pod.", - "required": [], - "title": "nodeSelector", - "type": "object" - }, - "service": { - "properties": { - "annotations": { - "description": "The annotations of the NGINX Gateway Fabric service.", - "required": [], - "title": "annotations", - "type": "object" - }, - "create": { - "default": true, - "description": "Creates a service to expose the NGINX Gateway Fabric pods.", - "required": [], - "title": "create", - "type": "boolean" }, - "externalTrafficPolicy": { - "default": "Local", - "description": "The externalTrafficPolicy of the service. The value Local preserves the client source IP.", - "enum": [ - "Cluster", - "Local" - ], + "terminationGracePeriodSeconds": { + "default": 30, + "description": "The termination grace period of the NGINX Gateway Fabric control plane pod.", "required": [], - "title": "externalTrafficPolicy" - }, - "loadBalancerIP": { - "default": "", - "description": "The static IP address for the load balancer. Requires service.type set to LoadBalancer.", - "required": [], - "title": "loadBalancerIP", - "type": "string" + "title": "terminationGracePeriodSeconds", + "type": "integer" }, - "loadBalancerSourceRanges": { - "description": "The IP ranges (CIDR) that are allowed to access the load balancer. Requires service.type set to LoadBalancer.", + "tolerations": { + "description": "Tolerations for the NGINX Gateway Fabric control plane pod.", "items": { "required": [] }, "required": [], - "title": "loadBalancerSourceRanges", + "title": "tolerations", "type": "array" }, - "ports": { - "description": "A list of ports to expose through the NGINX Gateway Fabric service. Update it to match the listener ports from\nyour Gateway resource. Follows the conventional Kubernetes yaml syntax for service ports.", - "items": { - "properties": { - "name": { - "required": [], - "type": "string" - }, - "port": { - "maximum": 65535, - "minimum": 1, - "required": [], - "type": "integer" - }, - "protocol": { - "enum": [ - "TCP", - "UDP" - ], - "required": [], - "type": "string" - }, - "targetPort": { - "maximum": 65535, - "minimum": 1, - "required": [], - "type": "integer" - } - }, - "required": [], - "type": "object" - }, - "required": [], - "title": "ports", - "type": "array" - }, - "type": { - "default": "LoadBalancer", - "description": "The type of service to create for the NGINX Gateway Fabric.", - "enum": [ - "ClusterIP", - "NodePort", - "LoadBalancer" - ], - "required": [], - "title": "type" - } - }, - "required": [], - "title": "service", - "type": "object" - }, - "serviceAccount": { - "properties": { - "annotations": { - "description": "Set of custom annotations for the NGINX Gateway Fabric service account.", - "required": [], - "title": "annotations", - "type": "object" - }, - "imagePullSecret": { - "default": "", - "description": "The name of the secret containing docker registry credentials.\nSecret must exist in the same namespace as the helm release.", - "required": [], - "title": "imagePullSecret", - "type": "string" - }, - "imagePullSecrets": { - "description": "A list of secret names containing docker registry credentials.\nSecrets must exist in the same namespace as the helm release.", + "topologySpreadConstraints": { + "description": "The topology spread constraints for the NGINX Gateway Fabric control plane pod.", "items": { "required": [] }, "required": [], - "title": "imagePullSecrets", + "title": "topologySpreadConstraints", "type": "array" - }, - "name": { - "default": "", - "description": "The name of the service account of the NGINX Gateway Fabric pods. Used for RBAC.", - "required": [], - "title": "name", - "type": "string" } }, - "required": [], - "title": "serviceAccount", + "required": [ + "gatewayClassName", + "gatewayControllerName" + ], + "title": "nginxGateway", "type": "object" - }, - "terminationGracePeriodSeconds": { - "default": 30, - "description": "The termination grace period of the NGINX Gateway Fabric pod.", - "required": [], - "title": "terminationGracePeriodSeconds", - "type": "integer" - }, - "tolerations": { - "description": "Tolerations for the NGINX Gateway Fabric pod.", - "items": { - "required": [] - }, - "required": [], - "title": "tolerations", - "type": "array" - }, - "topologySpreadConstraints": { - "description": "The topology spread constraints for the NGINX Gateway Fabric pod.", - "items": { - "required": [] - }, - "required": [], - "title": "topologySpreadConstraints", - "type": "array" } }, "required": [], diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index 8f1488b8d0..5289392b38 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -1,5 +1,6 @@ # yaml-language-server: $schema=values.schema.json +# -- The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. nginxGateway: # FIXME(lucacome): https://github.com/nginx/nginx-gateway-fabric/issues/2490 @@ -47,12 +48,30 @@ nginxGateway: # -- Set of custom annotations for NginxGateway objects. configAnnotations: {} + # -- The service configuration for the NGINX Gateway Fabric control plane. service: # -- The annotations of the NGINX Gateway Fabric control plane service. annotations: {} + # -- The serviceaccount configuration for the NGINX Gateway Fabric control plane. + serviceAccount: + # -- Set of custom annotations for the NGINX Gateway Fabric control plane service account. + annotations: {} + + # -- The name of the service account of the NGINX Gateway Fabric control plane pods. Used for RBAC. + # @default -- Autogenerated if not set or set to "" + name: "" + + # -- The name of the secret containing docker registry credentials for the control plane. + # Secret must exist in the same namespace as the helm release. + imagePullSecret: "" + + # -- A list of secret names containing docker registry credentials for the control plane. + # Secrets must exist in the same namespace as the helm release. + imagePullSecrets: [] + # -- The number of replicas of the NGINX Gateway Fabric Deployment. - replicaCount: 1 + replicas: 1 # The configuration for leader election. leaderElection: @@ -83,6 +102,7 @@ nginxGateway: # -- The number of seconds after the Pod has started before the readiness probes are initiated. initialDelaySeconds: 3 + # -- The image configuration for the NGINX Gateway Fabric control plane. image: # -- The NGINX Gateway Fabric image to use repository: ghcr.io/nginx/nginx-gateway-fabric @@ -105,9 +125,44 @@ nginxGateway: # -- The resource requests and/or limits of the nginx-gateway container. resources: {} + # -- extraVolumes for the NGINX Gateway Fabric control plane pod. Use in conjunction with + # nginxGateway.extraVolumeMounts mount additional volumes to the container. + extraVolumes: [] + # -- extraVolumeMounts are the additional volume mounts for the nginx-gateway container. extraVolumeMounts: [] + # -- The termination grace period of the NGINX Gateway Fabric control plane pod. + terminationGracePeriodSeconds: 30 + + # -- Tolerations for the NGINX Gateway Fabric control plane pod. + tolerations: [] + + # -- The nodeSelector of the NGINX Gateway Fabric control plane pod. + nodeSelector: {} + + # -- The affinity of the NGINX Gateway Fabric control plane pod. + affinity: {} + + # -- The topology spread constraints for the NGINX Gateway Fabric control plane pod. + topologySpreadConstraints: [] + + metrics: + # -- Enable exposing metrics in the Prometheus format. + enable: true + + # @schema + # type: integer + # minimum: 1 + # maximum: 65535 + # @schema + # -- Set the port where the Prometheus metrics are exposed. + port: 9113 + + # -- Enable serving metrics via https. By default metrics are served via http. + # Please note that this endpoint will be secured with a self-signed certificate. + secure: false + gwAPIExperimentalFeatures: # -- Enable the experimental features of Gateway API which are supported by NGINX Gateway Fabric. Requires the Gateway # APIs installed from the experimental channel. @@ -118,7 +173,19 @@ nginxGateway: # config for HTTPRoute and GRPCRoute resources. enable: false +# -- The nginx section contains the configuration for all NGINX data plane deployments +# installed by the NGINX Gateway Fabric control plane. nginx: + # @schema + # enum: + # - deployment + # @schema + # -- The kind of NGINX deployment. + kind: deployment + + # -- The number of replicas of the NGINX Deployment. + replicas: 1 + image: # -- The NGINX image to use. repository: ghcr.io/nginx/nginx-gateway-fabric/nginx @@ -131,9 +198,19 @@ nginx: # @schema pullPolicy: Always - # -- Is NGINX Plus image being used + # -- Is NGINX Plus image being used. plus: false + # -- The name of the secret containing docker registry credentials. + # Secret must exist in the same namespace as the helm release. The control + # plane will copy this secret into any namespace where NGINX is deployed. + imagePullSecret: "" + + # -- A list of secret names containing docker registry credentials. + # Secrets must exist in the same namespace as the helm release. The control + # plane will copy these secrets into any namespace where NGINX is deployed. + imagePullSecrets: [] + # Configuration for NGINX Plus usage reporting. usage: # -- The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace @@ -235,6 +312,16 @@ nginx: # type: string # enum: # - DisableTracing + # metrics: + # type: object + # description: Metrics defines the configuration for Prometheus scraping metrics. + # properties: + # disable: + # type: boolean + # port: + # type: integer + # minimum: 1 + # maximum: 65535 # logging: # type: object # description: Logging defines logging related settings for NGINX. @@ -250,6 +337,14 @@ nginx: # - crit # - alert # - emerg + # agentLevel: + # type: string + # enum: + # - debug + # - info + # - error + # - panic + # - fatal # nginxPlus: # type: object # description: NginxPlus specifies NGINX Plus additional settings. @@ -269,125 +364,65 @@ nginx: # -- The configuration for the data plane that is contained in the NginxProxy resource. config: {} - # -- Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. - debug: false - - # -- The lifecycle of the nginx container. - lifecycle: {} + # -- The pod configuration for the NGINX data plane pod. + pod: {} + # -- The termination grace period of the NGINX data plane pod. + # terminationGracePeriodSeconds: 30 - # -- extraVolumeMounts are the additional volume mounts for the nginx container. - extraVolumeMounts: [] + # -- Tolerations for the NGINX Gateway Fabric control plane pod. + # tolerations: [] -# -- The termination grace period of the NGINX Gateway Fabric pod. -terminationGracePeriodSeconds: 30 + # -- The nodeSelector of the NGINX Gateway Fabric control plane pod. + # nodeSelector: {} -# -- Tolerations for the NGINX Gateway Fabric pod. -tolerations: [] + # -- The affinity of the NGINX Gateway Fabric control plane pod. + # affinity: {} -# -- The nodeSelector of the NGINX Gateway Fabric pod. -nodeSelector: {} + # -- The topology spread constraints for the NGINX Gateway Fabric control plane pod. + # topologySpreadConstraints: [] -# -- The affinity of the NGINX Gateway Fabric pod. -affinity: {} + # -- extraVolumes for the NGINX Gateway Fabric control plane pod. Use in conjunction with + # nginx.container.extraVolumeMounts mount additional volumes to the container. + # extraVolumes: [] -# -- The topology spread constraints for the NGINX Gateway Fabric pod. -topologySpreadConstraints: [] + # -- The container configuration for the NGINX container. + container: {} + # -- The resource requirements of the NGINX container. + # resources: {} -serviceAccount: - # -- Set of custom annotations for the NGINX Gateway Fabric service account. - annotations: {} + # -- The lifecycle of the NGINX container. + # lifecycle: {} - # -- The name of the service account of the NGINX Gateway Fabric pods. Used for RBAC. - # @default -- Autogenerated if not set or set to "" - name: "" - - # -- The name of the secret containing docker registry credentials. - # Secret must exist in the same namespace as the helm release. - imagePullSecret: "" - - # -- A list of secret names containing docker registry credentials. - # Secrets must exist in the same namespace as the helm release. - imagePullSecrets: [] - -service: - # -- Creates a service to expose the NGINX Gateway Fabric pods. - create: true - - # @schema - # enum: - # - ClusterIP - # - NodePort - # - LoadBalancer - # @schema - # -- The type of service to create for the NGINX Gateway Fabric. - type: LoadBalancer + # -- extraVolumeMounts are the additional volume mounts for the NGINX container. + # extraVolumeMounts: [] - # @schema - # enum: - # - Cluster - # - Local - # @schema - # -- The externalTrafficPolicy of the service. The value Local preserves the client source IP. - externalTrafficPolicy: Local - - # -- The annotations of the NGINX Gateway Fabric service. - annotations: {} - - # -- The static IP address for the load balancer. Requires service.type set to LoadBalancer. - loadBalancerIP: "" + # -- The service configuration for the NGINX data plane. + service: + # @schema + # enum: + # - ClusterIP + # - NodePort + # - LoadBalancer + # @schema + # -- The type of service to create for the NGINX data plane. + type: LoadBalancer - # -- The IP ranges (CIDR) that are allowed to access the load balancer. Requires service.type set to LoadBalancer. - loadBalancerSourceRanges: [] + # @schema + # enum: + # - Cluster + # - Local + # @schema + # -- The externalTrafficPolicy of the service. The value Local preserves the client source IP. + externalTrafficPolicy: Local - # @schema - # type: array - # items: - # type: object - # properties: - # port: - # type: integer - # minimum: 1 - # maximum: 65535 - # targetPort: - # type: integer - # minimum: 1 - # maximum: 65535 - # protocol: - # type: string - # enum: - # - TCP - # - UDP - # name: - # type: string - # @schema - # -- A list of ports to expose through the NGINX Gateway Fabric service. Update it to match the listener ports from - # your Gateway resource. Follows the conventional Kubernetes yaml syntax for service ports. - ports: - - port: 80 - targetPort: 80 - protocol: TCP - name: http - - port: 443 - targetPort: 443 - protocol: TCP - name: https - -metrics: - # -- Enable exposing metrics in the Prometheus format. - enable: true + # -- The annotations of the NGINX data plane service. + # annotations: {} - # @schema - # type: integer - # minimum: 1 - # maximum: 65535 - # @schema - # -- Set the port where the Prometheus metrics are exposed. - port: 9113 + # -- The static IP address for the load balancer. Requires nginx.service.type set to LoadBalancer. + # loadBalancerIP: "" - # -- Enable serving metrics via https. By default metrics are served via http. - # Please note that this endpoint will be secured with a self-signed certificate. - secure: false + # -- The IP ranges (CIDR) that are allowed to access the load balancer. Requires nginx.service.type set to LoadBalancer. + # loadBalancerSourceRanges: [] -# -- extraVolumes for the NGINX Gateway Fabric pod. Use in conjunction with -# nginxGateway.extraVolumeMounts and nginx.extraVolumeMounts to mount additional volumes to the containers. -extraVolumes: [] + # -- Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. + debug: false diff --git a/cmd/gateway/commands.go b/cmd/gateway/commands.go index da92044068..0a572c922e 100644 --- a/cmd/gateway/commands.go +++ b/cmd/gateway/commands.go @@ -6,6 +6,7 @@ import ( "os" "runtime/debug" "strconv" + "strings" "time" "github.com/spf13/cobra" @@ -206,7 +207,7 @@ func createStaticModeCommand() *cobra.Command { flagKeys, flagValues := parseFlags(cmd.Flags()) - podConfig, err := createGatewayPodConfig(serviceName.value) + podConfig, err := createGatewayPodConfig(version, serviceName.value) if err != nil { return fmt.Errorf("error creating gateway pod config: %w", err) } @@ -242,7 +243,6 @@ func createStaticModeCommand() *cobra.Command { EndpointInsecure: telemetryEndpointInsecure, }, Plus: plus, - Version: version, ExperimentalFeatures: gwExperimentalFeatures, ImageSource: imageSource, Flags: config.Flags{ @@ -649,33 +649,46 @@ func getBuildInfo() (commitHash string, commitTime string, dirtyBuild string) { return } -func createGatewayPodConfig(svcName string) (config.GatewayPodConfig, error) { - podIP, err := getValueFromEnv("POD_IP") +func createGatewayPodConfig(version, svcName string) (config.GatewayPodConfig, error) { + podUID, err := getValueFromEnv("POD_UID") if err != nil { return config.GatewayPodConfig{}, err } - podUID, err := getValueFromEnv("POD_UID") + ns, err := getValueFromEnv("POD_NAMESPACE") if err != nil { return config.GatewayPodConfig{}, err } - ns, err := getValueFromEnv("POD_NAMESPACE") + name, err := getValueFromEnv("POD_NAME") if err != nil { return config.GatewayPodConfig{}, err } - name, err := getValueFromEnv("POD_NAME") + instance, err := getValueFromEnv("INSTANCE_NAME") if err != nil { return config.GatewayPodConfig{}, err } + image, err := getValueFromEnv("IMAGE_NAME") + if err != nil { + return config.GatewayPodConfig{}, err + } + + // use image tag version if set, otherwise fall back to binary version + ngfVersion := version + if imageParts := strings.Split(image, ":"); len(imageParts) == 2 { + ngfVersion = imageParts[1] + } + c := config.GatewayPodConfig{ - PodIP: podIP, - ServiceName: svcName, - Namespace: ns, - Name: name, - UID: podUID, + ServiceName: svcName, + Namespace: ns, + Name: name, + UID: podUID, + InstanceName: instance, + Version: ngfVersion, + Image: image, } return c, nil diff --git a/cmd/gateway/commands_test.go b/cmd/gateway/commands_test.go index 2c1ac5d266..b58fa3331b 100644 --- a/cmd/gateway/commands_test.go +++ b/cmd/gateway/commands_test.go @@ -669,43 +669,62 @@ func TestCreateGatewayPodConfig(t *testing.T) { // Order matters here // We start with all env vars set - g.Expect(os.Setenv("POD_IP", "10.0.0.0")).To(Succeed()) g.Expect(os.Setenv("POD_UID", "1234")).To(Succeed()) g.Expect(os.Setenv("POD_NAMESPACE", "default")).To(Succeed()) g.Expect(os.Setenv("POD_NAME", "my-pod")).To(Succeed()) + g.Expect(os.Setenv("INSTANCE_NAME", "my-pod-xyz")).To(Succeed()) + g.Expect(os.Setenv("IMAGE_NAME", "my-pod-image:tag")).To(Succeed()) + + version := "0.0.0" expCfg := config.GatewayPodConfig{ - PodIP: "10.0.0.0", - ServiceName: "svc", - Namespace: "default", - Name: "my-pod", - UID: "1234", + ServiceName: "svc", + Namespace: "default", + Name: "my-pod", + UID: "1234", + InstanceName: "my-pod-xyz", + Version: "tag", + Image: "my-pod-image:tag", } - cfg, err := createGatewayPodConfig("svc") + cfg, err := createGatewayPodConfig(version, "svc") g.Expect(err).To(Not(HaveOccurred())) g.Expect(cfg).To(Equal(expCfg)) + // unset image tag and use provided version + g.Expect(os.Setenv("IMAGE_NAME", "my-pod-image")).To(Succeed()) + expCfg.Version = version + expCfg.Image = "my-pod-image" + cfg, err = createGatewayPodConfig(version, "svc") + g.Expect(err).To(Not(HaveOccurred())) + g.Expect(cfg).To(Equal(expCfg)) + + // unset image name + g.Expect(os.Unsetenv("IMAGE_NAME")).To(Succeed()) + cfg, err = createGatewayPodConfig(version, "svc") + g.Expect(err).To(MatchError(errors.New("environment variable IMAGE_NAME not set"))) + g.Expect(cfg).To(Equal(config.GatewayPodConfig{})) + + // unset instance name + g.Expect(os.Unsetenv("INSTANCE_NAME")).To(Succeed()) + cfg, err = createGatewayPodConfig(version, "svc") + g.Expect(err).To(MatchError(errors.New("environment variable INSTANCE_NAME not set"))) + g.Expect(cfg).To(Equal(config.GatewayPodConfig{})) + // unset name g.Expect(os.Unsetenv("POD_NAME")).To(Succeed()) - cfg, err = createGatewayPodConfig("svc") + cfg, err = createGatewayPodConfig(version, "svc") g.Expect(err).To(MatchError(errors.New("environment variable POD_NAME not set"))) g.Expect(cfg).To(Equal(config.GatewayPodConfig{})) // unset namespace g.Expect(os.Unsetenv("POD_NAMESPACE")).To(Succeed()) - cfg, err = createGatewayPodConfig("svc") + cfg, err = createGatewayPodConfig(version, "svc") g.Expect(err).To(MatchError(errors.New("environment variable POD_NAMESPACE not set"))) g.Expect(cfg).To(Equal(config.GatewayPodConfig{})) // unset pod UID g.Expect(os.Unsetenv("POD_UID")).To(Succeed()) - cfg, err = createGatewayPodConfig("svc") + cfg, err = createGatewayPodConfig(version, "svc") g.Expect(err).To(MatchError(errors.New("environment variable POD_UID not set"))) g.Expect(cfg).To(Equal(config.GatewayPodConfig{})) - - // unset IP - g.Expect(os.Unsetenv("POD_IP")).To(Succeed()) - cfg, err = createGatewayPodConfig("svc") - g.Expect(err).To(MatchError(errors.New("environment variable POD_IP not set"))) - g.Expect(cfg).To(Equal(config.GatewayPodConfig{})) } diff --git a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml index 834ace5c6c..a86fcf723e 100644 --- a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml +++ b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml @@ -66,9 +66,3455 @@ spec: - ipv4 - ipv6 type: string + kubernetes: + description: Kubernetes contains the configuration for the NGINX Deployment + and Service Kubernetes objects. + properties: + deployment: + description: |- + Deployment is the configuration for the NGINX Deployment. + This is the default deployment option. + properties: + container: + description: Container defines container fields for the NGINX + container. + properties: + image: + description: Image is the NGINX image to use. + properties: + pullPolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when + to pull a container image. + enum: + - Always + - Never + - IfNotPresent + type: string + repository: + description: |- + Repository is the image path. + Default is ghcr.io/nginx/nginx-gateway-fabric/nginx. + type: string + tag: + description: Tag is the image tag to use. Default + matches the tag of the control plane. + type: string + type: object + lifecycle: + description: |- + Lifecycle describes actions that the management system should take in response to container lifecycle + events. For the PostStart and PreStop lifecycle handlers, management of the container blocks + until the action is complete, unless the container process fails, in which case the handler is aborted. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that + the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds + to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that + the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds + to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + resources: + description: Resources describes the compute resource + requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + volumeMounts: + description: VolumeMounts describe the mounting of Volumes + within a container. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + type: object + pod: + description: Pod defines Pod-specific fields. + properties: + affinity: + description: Affinity is the pod's scheduling constraints. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules + for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, in + the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same node, + zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + type: object + terminationGracePeriodSeconds: + description: |- + TerminationGracePeriodSeconds is the optional duration in seconds the pod needs to terminate gracefully. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: Tolerations allow the scheduler to schedule + Pods with matching taints. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of Pods ought to spread across topology + domains. Scheduler will schedule Pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how + to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + volumes: + description: Volumes represents named volumes in a pod + that may be accessed by any container in the pod. + items: + description: Volume represents a named volume in a pod + that may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree + awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: |- + azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. + Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type + are redirected to the disk.csi.azure.com CSI driver. + properties: + cachingMode: + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data + disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk + in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed availability + set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: |- + azureFile represents an Azure File Service mount on the host and bind mount to the pod. + Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type + are redirected to the file.csi.azure.com CSI driver. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret + that contains Azure Storage Account Name and + Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: |- + cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. + Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the + mounted root, rather than the full Ceph tree, + default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + Deprecated: Cinder is deprecated. All operations for the in-tree cinder type + are redirected to the cinder.csi.openstack.org CSI driver. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that + should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents + ephemeral storage that is handled by certain external + CSI drivers. + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API + about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API + volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. + Must not be absolute or contain the + ''..'' path. Must be utf-8 encoded. + The first item of the relative path + must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of + resource being referenced + type: string + name: + description: Name is the name of + resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of + resource being referenced + type: string + name: + description: Name is the name of + resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query + over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding + reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine and + then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun + number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target + worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. + properties: + driver: + description: driver is the name of the driver + to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field + holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: |- + flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. + Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the + dataset. This is unique identifier of a Flocker + dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree + gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for + the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun + number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for + iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: |- + photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. + Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: |- + portworxVolume represents a portworx volume attached and mounted on kubelets host machine. + Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type + are redirected to the pxd.portworx.com CSI driver when the CSIMigrationPortworx feature-gate + is on. + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a + Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources + secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the + volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about + the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about + the downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name, namespace and + uid are supported.' + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path + is the relative path name + of the file to be created. + Must not be absolute or contain + the ''..'' path. Must be utf-8 + encoded. The first item of + the relative path must not + start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about + the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify + whether the Secret or its key must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information + about the serviceAccountToken data to + project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: |- + quobyte represents a Quobyte mount on the host that shares a pod's lifetime. + Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: |- + scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. + Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of + the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of + the ScaleIO Protection Domain for the configured + storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage + Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage + system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: |- + storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. + Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: |- + vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. + Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type + are redirected to the csi.vsphere.vmware.com CSI driver. + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile ID + associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + replicas: + description: Number of desired Pods. + format: int32 + type: integer + type: object + service: + description: Service is the configuration for the NGINX Service. + properties: + annotations: + additionalProperties: + type: string + description: Annotations contain any Service-specific annotations. + type: object + externalTrafficPolicy: + default: Local + description: |- + ExternalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, ExternalIPs, + and LoadBalancer IPs. + enum: + - Cluster + - Local + type: string + loadBalancerIP: + description: LoadBalancerIP is a static IP address for the + load balancer. Requires service type to be LoadBalancer. + type: string + loadBalancerSourceRanges: + description: |- + LoadBalancerSourceRanges are the IP ranges (CIDR) that are allowed to access the load balancer. + Requires service type to be LoadBalancer. + items: + type: string + type: array + type: + default: LoadBalancer + description: ServiceType describes ingress method for the + Service. + enum: + - ClusterIP + - LoadBalancer + - NodePort + type: string + type: object + type: object logging: description: Logging defines logging related settings for NGINX. properties: + agentLevel: + default: info + description: |- + AgentLevel defines the log level of the NGINX agent process. Changing this value results in a + re-roll of the NGINX deployment. + enum: + - debug + - info + - error + - panic + - fatal + type: string errorLevel: default: info description: |- @@ -87,6 +3533,22 @@ spec: - emerg type: string type: object + metrics: + description: |- + Metrics defines the configuration for Prometheus scraping metrics. Changing this value results in a + re-roll of the NGINX deployment. + properties: + disable: + description: Disable serving Prometheus metrics on the listen + port. + type: boolean + port: + description: Port where the Prometheus metrics are exposed. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object nginxPlus: description: NginxPlus specifies NGINX Plus additional settings. properties: diff --git a/config/tests/static-deployment.yaml b/config/tests/static-deployment.yaml index 2a53887183..7c1b2df7e9 100644 --- a/config/tests/static-deployment.yaml +++ b/config/tests/static-deployment.yaml @@ -33,10 +33,6 @@ spec: - --leader-election-lock-name=nginx-gateway-leader-election - --product-telemetry-disable env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - name: POD_NAMESPACE valueFrom: fieldRef: @@ -49,6 +45,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid + - name: INSTANCE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/instance'] + - name: IMAGE_NAME + value: ghcr.io/nginx/nginx-gateway-fabric:edge image: ghcr.io/nginx/nginx-gateway-fabric:edge imagePullPolicy: Always name: nginx-gateway diff --git a/deploy/aws-nlb/deploy.yaml b/deploy/aws-nlb/deploy.yaml index c983b691fd..d49067708b 100644 --- a/deploy/aws-nlb/deploy.yaml +++ b/deploy/aws-nlb/deploy.yaml @@ -24,10 +24,24 @@ metadata: rules: - apiGroups: - "" + - apps resources: - - namespaces - - services - secrets + - configmaps + - serviceaccounts + - services + - deployments + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - "" + resources: + - namespaces - pods verbs: - get @@ -141,60 +155,6 @@ subjects: namespace: nginx-gateway --- apiVersion: v1 -data: - nginx-agent.conf: |- - command: - server: - host: nginx-gateway.nginx-gateway.svc - port: 443 - allowed_directories: - - /etc/nginx - - /usr/share/nginx - - /var/run/nginx - features: - - connection - - configuration - - certificates - - metrics - log: - level: debug - collector: - receivers: - host_metrics: - collection_interval: 1m0s - initial_delay: 1s - scrapers: - cpu: {} - memory: {} - disk: {} - network: {} - filesystem: {} - processors: - batch: {} - exporters: - prometheus_exporter: - server: - host: "0.0.0.0" - port: 9113 -kind: ConfigMap -metadata: - name: nginx-agent-config - namespace: nginx-gateway ---- -apiVersion: v1 -data: - main.conf: | - error_log stderr info; -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: nginx-includes-bootstrap - namespace: nginx-gateway ---- -apiVersion: v1 kind: Service metadata: labels: @@ -214,34 +174,6 @@ spec: app.kubernetes.io/name: nginx-gateway type: ClusterIP --- -apiVersion: v1 -kind: Service -metadata: - annotations: - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip - service.beta.kubernetes.io/aws-load-balancer-type: external - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - externalTrafficPolicy: Local - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 80 - - name: https - port: 443 - protocol: TCP - targetPort: 443 - selector: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - type: LoadBalancer ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -277,10 +209,6 @@ spec: - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - name: POD_NAMESPACE valueFrom: fieldRef: @@ -293,6 +221,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid + - name: INSTANCE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/instance'] + - name: IMAGE_NAME + value: ghcr.io/nginx/nginx-gateway-fabric:edge image: ghcr.io/nginx/nginx-gateway-fabric:edge imagePullPolicy: Always name: nginx-gateway @@ -325,135 +259,6 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 --- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - selector: - matchLabels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - template: - metadata: - annotations: - prometheus.io/port: "9113" - prometheus.io/scrape: "true" - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - spec: - containers: - - image: ghcr.io/nginx/nginx-gateway-fabric/nginx:edge - imagePullPolicy: Always - name: nginx - ports: - - containerPort: 80 - name: http - - containerPort: 443 - name: https - - containerPort: 9113 - name: metrics - securityContext: - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /var/log/nginx-agent - name: nginx-agent-log - - mountPath: /etc/nginx/conf.d - name: nginx-conf - - mountPath: /etc/nginx/stream-conf.d - name: nginx-stream-conf - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - - mountPath: /etc/nginx/secrets - name: nginx-secrets - - mountPath: /var/run/nginx - name: nginx-run - - mountPath: /var/cache/nginx - name: nginx-cache - - mountPath: /etc/nginx/includes - name: nginx-includes - initContainers: - - command: - - /usr/bin/gateway - - initialize - - --source - - /agent/nginx-agent.conf - - --destination - - /etc/nginx-agent - - --source - - /includes/main.conf - - --destination - - /etc/nginx/main-includes - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/nginx/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: init - securityContext: - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /agent - name: nginx-agent-config - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /includes - name: nginx-includes-bootstrap - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - securityContext: - fsGroup: 1001 - runAsNonRoot: true - serviceAccountName: nginx-gateway - terminationGracePeriodSeconds: 30 - volumes: - - emptyDir: {} - name: nginx-agent - - configMap: - name: nginx-agent-config - name: nginx-agent-config - - emptyDir: {} - name: nginx-agent-log - - emptyDir: {} - name: nginx-conf - - emptyDir: {} - name: nginx-stream-conf - - emptyDir: {} - name: nginx-main-includes - - emptyDir: {} - name: nginx-secrets - - emptyDir: {} - name: nginx-run - - emptyDir: {} - name: nginx-cache - - emptyDir: {} - name: nginx-includes - - configMap: - name: nginx-includes-bootstrap - name: nginx-includes-bootstrap ---- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: @@ -464,6 +269,11 @@ metadata: name: nginx spec: controllerName: gateway.nginx.org/nginx-gateway-controller + parametersRef: + group: gateway.nginx.org + kind: NginxProxy + name: nginx-gateway-proxy-config + namespace: nginx-gateway --- apiVersion: gateway.nginx.org/v1alpha1 kind: NginxGateway @@ -477,3 +287,28 @@ metadata: spec: logging: level: info +--- +apiVersion: gateway.nginx.org/v1alpha2 +kind: NginxProxy +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-proxy-config + namespace: nginx-gateway +spec: + kubernetes: + deployment: + container: + image: + pullPolicy: Always + repository: ghcr.io/nginx/nginx-gateway-fabric/nginx + tag: edge + replicas: 1 + service: + annotations: + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-type: external + externalTrafficPolicy: Local + type: LoadBalancer diff --git a/deploy/azure/deploy.yaml b/deploy/azure/deploy.yaml index 0e0330d243..e7d0ef976d 100644 --- a/deploy/azure/deploy.yaml +++ b/deploy/azure/deploy.yaml @@ -24,10 +24,24 @@ metadata: rules: - apiGroups: - "" + - apps resources: - - namespaces - - services - secrets + - configmaps + - serviceaccounts + - services + - deployments + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - "" + resources: + - namespaces - pods verbs: - get @@ -141,60 +155,6 @@ subjects: namespace: nginx-gateway --- apiVersion: v1 -data: - nginx-agent.conf: |- - command: - server: - host: nginx-gateway.nginx-gateway.svc - port: 443 - allowed_directories: - - /etc/nginx - - /usr/share/nginx - - /var/run/nginx - features: - - connection - - configuration - - certificates - - metrics - log: - level: debug - collector: - receivers: - host_metrics: - collection_interval: 1m0s - initial_delay: 1s - scrapers: - cpu: {} - memory: {} - disk: {} - network: {} - filesystem: {} - processors: - batch: {} - exporters: - prometheus_exporter: - server: - host: "0.0.0.0" - port: 9113 -kind: ConfigMap -metadata: - name: nginx-agent-config - namespace: nginx-gateway ---- -apiVersion: v1 -data: - main.conf: | - error_log stderr info; -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: nginx-includes-bootstrap - namespace: nginx-gateway ---- -apiVersion: v1 kind: Service metadata: labels: @@ -214,31 +174,6 @@ spec: app.kubernetes.io/name: nginx-gateway type: ClusterIP --- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - externalTrafficPolicy: Local - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 80 - - name: https - port: 443 - protocol: TCP - targetPort: 443 - selector: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - type: LoadBalancer ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -274,10 +209,6 @@ spec: - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - name: POD_NAMESPACE valueFrom: fieldRef: @@ -290,6 +221,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid + - name: INSTANCE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/instance'] + - name: IMAGE_NAME + value: ghcr.io/nginx/nginx-gateway-fabric:edge image: ghcr.io/nginx/nginx-gateway-fabric:edge imagePullPolicy: Always name: nginx-gateway @@ -324,137 +261,6 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 --- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - selector: - matchLabels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - template: - metadata: - annotations: - prometheus.io/port: "9113" - prometheus.io/scrape: "true" - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - spec: - containers: - - image: ghcr.io/nginx/nginx-gateway-fabric/nginx:edge - imagePullPolicy: Always - name: nginx - ports: - - containerPort: 80 - name: http - - containerPort: 443 - name: https - - containerPort: 9113 - name: metrics - securityContext: - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /var/log/nginx-agent - name: nginx-agent-log - - mountPath: /etc/nginx/conf.d - name: nginx-conf - - mountPath: /etc/nginx/stream-conf.d - name: nginx-stream-conf - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - - mountPath: /etc/nginx/secrets - name: nginx-secrets - - mountPath: /var/run/nginx - name: nginx-run - - mountPath: /var/cache/nginx - name: nginx-cache - - mountPath: /etc/nginx/includes - name: nginx-includes - initContainers: - - command: - - /usr/bin/gateway - - initialize - - --source - - /agent/nginx-agent.conf - - --destination - - /etc/nginx-agent - - --source - - /includes/main.conf - - --destination - - /etc/nginx/main-includes - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/nginx/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: init - securityContext: - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /agent - name: nginx-agent-config - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /includes - name: nginx-includes-bootstrap - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - nodeSelector: - kubernetes.io/os: linux - securityContext: - fsGroup: 1001 - runAsNonRoot: true - serviceAccountName: nginx-gateway - terminationGracePeriodSeconds: 30 - volumes: - - emptyDir: {} - name: nginx-agent - - configMap: - name: nginx-agent-config - name: nginx-agent-config - - emptyDir: {} - name: nginx-agent-log - - emptyDir: {} - name: nginx-conf - - emptyDir: {} - name: nginx-stream-conf - - emptyDir: {} - name: nginx-main-includes - - emptyDir: {} - name: nginx-secrets - - emptyDir: {} - name: nginx-run - - emptyDir: {} - name: nginx-cache - - emptyDir: {} - name: nginx-includes - - configMap: - name: nginx-includes-bootstrap - name: nginx-includes-bootstrap ---- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: @@ -465,6 +271,11 @@ metadata: name: nginx spec: controllerName: gateway.nginx.org/nginx-gateway-controller + parametersRef: + group: gateway.nginx.org + kind: NginxProxy + name: nginx-gateway-proxy-config + namespace: nginx-gateway --- apiVersion: gateway.nginx.org/v1alpha1 kind: NginxGateway @@ -478,3 +289,28 @@ metadata: spec: logging: level: info +--- +apiVersion: gateway.nginx.org/v1alpha2 +kind: NginxProxy +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-proxy-config + namespace: nginx-gateway +spec: + kubernetes: + deployment: + container: + image: + pullPolicy: Always + repository: ghcr.io/nginx/nginx-gateway-fabric/nginx + tag: edge + pod: + nodeSelector: + kubernetes.io/os: linux + replicas: 1 + service: + externalTrafficPolicy: Local + type: LoadBalancer diff --git a/deploy/crds.yaml b/deploy/crds.yaml index a7be218353..cc4ad633b3 100644 --- a/deploy/crds.yaml +++ b/deploy/crds.yaml @@ -651,9 +651,3455 @@ spec: - ipv4 - ipv6 type: string + kubernetes: + description: Kubernetes contains the configuration for the NGINX Deployment + and Service Kubernetes objects. + properties: + deployment: + description: |- + Deployment is the configuration for the NGINX Deployment. + This is the default deployment option. + properties: + container: + description: Container defines container fields for the NGINX + container. + properties: + image: + description: Image is the NGINX image to use. + properties: + pullPolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when + to pull a container image. + enum: + - Always + - Never + - IfNotPresent + type: string + repository: + description: |- + Repository is the image path. + Default is ghcr.io/nginx/nginx-gateway-fabric/nginx. + type: string + tag: + description: Tag is the image tag to use. Default + matches the tag of the control plane. + type: string + type: object + lifecycle: + description: |- + Lifecycle describes actions that the management system should take in response to container lifecycle + events. For the PostStart and PreStop lifecycle handlers, management of the container blocks + until the action is complete, unless the container process fails, in which case the handler is aborted. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that + the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds + to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies a command to execute + in the container. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies an HTTP GET request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents a duration that + the container should sleep. + properties: + seconds: + description: Seconds is the number of seconds + to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for backward compatibility. There is no validation of this field and + lifecycle hooks will fail at runtime when it is specified. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + resources: + description: Resources describes the compute resource + requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + volumeMounts: + description: VolumeMounts describe the mounting of Volumes + within a container. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + type: object + pod: + description: Pod defines Pod-specific fields. + properties: + affinity: + description: Affinity is the pod's scheduling constraints. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules + for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, in + the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same node, + zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + type: object + terminationGracePeriodSeconds: + description: |- + TerminationGracePeriodSeconds is the optional duration in seconds the pod needs to terminate gracefully. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: Tolerations allow the scheduler to schedule + Pods with matching taints. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of Pods ought to spread across topology + domains. Scheduler will schedule Pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how + to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + volumes: + description: Volumes represents named volumes in a pod + that may be accessed by any container in the pod. + items: + description: Volume represents a named volume in a pod + that may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree + awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: |- + azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. + Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type + are redirected to the disk.csi.azure.com CSI driver. + properties: + cachingMode: + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data + disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk + in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed availability + set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: |- + azureFile represents an Azure File Service mount on the host and bind mount to the pod. + Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type + are redirected to the file.csi.azure.com CSI driver. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret + that contains Azure Storage Account Name and + Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: |- + cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. + Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported. + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the + mounted root, rather than the full Ceph tree, + default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + Deprecated: Cinder is deprecated. All operations for the in-tree cinder type + are redirected to the cinder.csi.openstack.org CSI driver. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that + should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents + ephemeral storage that is handled by certain external + CSI drivers. + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API + about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API + volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. + Must not be absolute or contain the + ''..'' path. Must be utf-8 encoded. + The first item of the relative path + must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of + resource being referenced + type: string + name: + description: Name is the name of + resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of + resource being referenced + type: string + name: + description: Name is the name of + resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query + over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding + reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine and + then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun + number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target + worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead. + properties: + driver: + description: driver is the name of the driver + to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field + holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: |- + flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. + Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported. + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the + dataset. This is unique identifier of a Flocker + dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree + gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for + the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun + number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for + iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: |- + photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. + Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: |- + portworxVolume represents a portworx volume attached and mounted on kubelets host machine. + Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type + are redirected to the pxd.portworx.com CSI driver when the CSIMigrationPortworx feature-gate + is on. + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a + Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources + secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the + volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about + the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about + the downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name, namespace and + uid are supported.' + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path + is the relative path name + of the file to be created. + Must not be absolute or contain + the ''..'' path. Must be utf-8 + encoded. The first item of + the relative path must not + start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about + the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify + whether the Secret or its key must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information + about the serviceAccountToken data to + project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: |- + quobyte represents a Quobyte mount on the host that shares a pod's lifetime. + Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported. + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: |- + scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. + Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of + the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of + the ScaleIO Protection Domain for the configured + storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage + Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage + system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: |- + storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. + Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: |- + vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. + Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type + are redirected to the csi.vsphere.vmware.com CSI driver. + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile ID + associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + replicas: + description: Number of desired Pods. + format: int32 + type: integer + type: object + service: + description: Service is the configuration for the NGINX Service. + properties: + annotations: + additionalProperties: + type: string + description: Annotations contain any Service-specific annotations. + type: object + externalTrafficPolicy: + default: Local + description: |- + ExternalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, ExternalIPs, + and LoadBalancer IPs. + enum: + - Cluster + - Local + type: string + loadBalancerIP: + description: LoadBalancerIP is a static IP address for the + load balancer. Requires service type to be LoadBalancer. + type: string + loadBalancerSourceRanges: + description: |- + LoadBalancerSourceRanges are the IP ranges (CIDR) that are allowed to access the load balancer. + Requires service type to be LoadBalancer. + items: + type: string + type: array + type: + default: LoadBalancer + description: ServiceType describes ingress method for the + Service. + enum: + - ClusterIP + - LoadBalancer + - NodePort + type: string + type: object + type: object logging: description: Logging defines logging related settings for NGINX. properties: + agentLevel: + default: info + description: |- + AgentLevel defines the log level of the NGINX agent process. Changing this value results in a + re-roll of the NGINX deployment. + enum: + - debug + - info + - error + - panic + - fatal + type: string errorLevel: default: info description: |- @@ -672,6 +4118,22 @@ spec: - emerg type: string type: object + metrics: + description: |- + Metrics defines the configuration for Prometheus scraping metrics. Changing this value results in a + re-roll of the NGINX deployment. + properties: + disable: + description: Disable serving Prometheus metrics on the listen + port. + type: boolean + port: + description: Port where the Prometheus metrics are exposed. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object nginxPlus: description: NginxPlus specifies NGINX Plus additional settings. properties: diff --git a/deploy/default/deploy.yaml b/deploy/default/deploy.yaml index 8e51e699fc..42cc36de4a 100644 --- a/deploy/default/deploy.yaml +++ b/deploy/default/deploy.yaml @@ -24,10 +24,24 @@ metadata: rules: - apiGroups: - "" + - apps resources: - - namespaces - - services - secrets + - configmaps + - serviceaccounts + - services + - deployments + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - "" + resources: + - namespaces - pods verbs: - get @@ -141,60 +155,6 @@ subjects: namespace: nginx-gateway --- apiVersion: v1 -data: - nginx-agent.conf: |- - command: - server: - host: nginx-gateway.nginx-gateway.svc - port: 443 - allowed_directories: - - /etc/nginx - - /usr/share/nginx - - /var/run/nginx - features: - - connection - - configuration - - certificates - - metrics - log: - level: debug - collector: - receivers: - host_metrics: - collection_interval: 1m0s - initial_delay: 1s - scrapers: - cpu: {} - memory: {} - disk: {} - network: {} - filesystem: {} - processors: - batch: {} - exporters: - prometheus_exporter: - server: - host: "0.0.0.0" - port: 9113 -kind: ConfigMap -metadata: - name: nginx-agent-config - namespace: nginx-gateway ---- -apiVersion: v1 -data: - main.conf: | - error_log stderr info; -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: nginx-includes-bootstrap - namespace: nginx-gateway ---- -apiVersion: v1 kind: Service metadata: labels: @@ -214,31 +174,6 @@ spec: app.kubernetes.io/name: nginx-gateway type: ClusterIP --- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - externalTrafficPolicy: Local - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 80 - - name: https - port: 443 - protocol: TCP - targetPort: 443 - selector: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - type: LoadBalancer ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -274,10 +209,6 @@ spec: - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - name: POD_NAMESPACE valueFrom: fieldRef: @@ -290,6 +221,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid + - name: INSTANCE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/instance'] + - name: IMAGE_NAME + value: ghcr.io/nginx/nginx-gateway-fabric:edge image: ghcr.io/nginx/nginx-gateway-fabric:edge imagePullPolicy: Always name: nginx-gateway @@ -322,135 +259,6 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 --- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - selector: - matchLabels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - template: - metadata: - annotations: - prometheus.io/port: "9113" - prometheus.io/scrape: "true" - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - spec: - containers: - - image: ghcr.io/nginx/nginx-gateway-fabric/nginx:edge - imagePullPolicy: Always - name: nginx - ports: - - containerPort: 80 - name: http - - containerPort: 443 - name: https - - containerPort: 9113 - name: metrics - securityContext: - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /var/log/nginx-agent - name: nginx-agent-log - - mountPath: /etc/nginx/conf.d - name: nginx-conf - - mountPath: /etc/nginx/stream-conf.d - name: nginx-stream-conf - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - - mountPath: /etc/nginx/secrets - name: nginx-secrets - - mountPath: /var/run/nginx - name: nginx-run - - mountPath: /var/cache/nginx - name: nginx-cache - - mountPath: /etc/nginx/includes - name: nginx-includes - initContainers: - - command: - - /usr/bin/gateway - - initialize - - --source - - /agent/nginx-agent.conf - - --destination - - /etc/nginx-agent - - --source - - /includes/main.conf - - --destination - - /etc/nginx/main-includes - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/nginx/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: init - securityContext: - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /agent - name: nginx-agent-config - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /includes - name: nginx-includes-bootstrap - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - securityContext: - fsGroup: 1001 - runAsNonRoot: true - serviceAccountName: nginx-gateway - terminationGracePeriodSeconds: 30 - volumes: - - emptyDir: {} - name: nginx-agent - - configMap: - name: nginx-agent-config - name: nginx-agent-config - - emptyDir: {} - name: nginx-agent-log - - emptyDir: {} - name: nginx-conf - - emptyDir: {} - name: nginx-stream-conf - - emptyDir: {} - name: nginx-main-includes - - emptyDir: {} - name: nginx-secrets - - emptyDir: {} - name: nginx-run - - emptyDir: {} - name: nginx-cache - - emptyDir: {} - name: nginx-includes - - configMap: - name: nginx-includes-bootstrap - name: nginx-includes-bootstrap ---- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: @@ -461,6 +269,11 @@ metadata: name: nginx spec: controllerName: gateway.nginx.org/nginx-gateway-controller + parametersRef: + group: gateway.nginx.org + kind: NginxProxy + name: nginx-gateway-proxy-config + namespace: nginx-gateway --- apiVersion: gateway.nginx.org/v1alpha1 kind: NginxGateway @@ -474,3 +287,25 @@ metadata: spec: logging: level: info +--- +apiVersion: gateway.nginx.org/v1alpha2 +kind: NginxProxy +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-proxy-config + namespace: nginx-gateway +spec: + kubernetes: + deployment: + container: + image: + pullPolicy: Always + repository: ghcr.io/nginx/nginx-gateway-fabric/nginx + tag: edge + replicas: 1 + service: + externalTrafficPolicy: Local + type: LoadBalancer diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml index 009dd2aaad..88f5e771cd 100644 --- a/deploy/experimental-nginx-plus/deploy.yaml +++ b/deploy/experimental-nginx-plus/deploy.yaml @@ -4,8 +4,6 @@ metadata: name: nginx-gateway --- apiVersion: v1 -imagePullSecrets: -- name: nginx-plus-registry-secret kind: ServiceAccount metadata: labels: @@ -26,12 +24,25 @@ metadata: rules: - apiGroups: - "" + - apps resources: - - namespaces - - services - secrets - - pods - configmaps + - serviceaccounts + - services + - deployments + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - "" + resources: + - namespaces + - pods verbs: - get - list @@ -148,66 +159,6 @@ subjects: namespace: nginx-gateway --- apiVersion: v1 -data: - nginx-agent.conf: |- - command: - server: - host: nginx-gateway.nginx-gateway.svc - port: 443 - allowed_directories: - - /etc/nginx - - /usr/share/nginx - - /var/run/nginx - features: - - connection - - configuration - - certificates - - metrics - - api-action - log: - level: debug - collector: - receivers: - host_metrics: - collection_interval: 1m0s - initial_delay: 1s - scrapers: - cpu: {} - memory: {} - disk: {} - network: {} - filesystem: {} - processors: - batch: {} - exporters: - prometheus_exporter: - server: - host: "0.0.0.0" - port: 9113 -kind: ConfigMap -metadata: - name: nginx-agent-config - namespace: nginx-gateway ---- -apiVersion: v1 -data: - main.conf: | - error_log stderr info; - mgmt.conf: | - mgmt { - enforce_initial_report off; - deployment_context /etc/nginx/main-includes/deployment_ctx.json; - } -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: nginx-includes-bootstrap - namespace: nginx-gateway ---- -apiVersion: v1 kind: Service metadata: labels: @@ -227,31 +178,6 @@ spec: app.kubernetes.io/name: nginx-gateway type: ClusterIP --- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - externalTrafficPolicy: Local - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 80 - - name: https - port: 443 - protocol: TCP - targetPort: 443 - selector: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - type: LoadBalancer ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -290,10 +216,6 @@ spec: - --leader-election-lock-name=nginx-gateway-leader-election - --gateway-api-experimental-features env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - name: POD_NAMESPACE valueFrom: fieldRef: @@ -306,6 +228,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid + - name: INSTANCE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/instance'] + - name: IMAGE_NAME + value: ghcr.io/nginx/nginx-gateway-fabric:edge image: ghcr.io/nginx/nginx-gateway-fabric:edge imagePullPolicy: Always name: nginx-gateway @@ -338,150 +266,6 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 --- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - selector: - matchLabels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - template: - metadata: - annotations: - prometheus.io/port: "9113" - prometheus.io/scrape: "true" - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - spec: - containers: - - image: private-registry.nginx.com/nginx-gateway-fabric/nginx-plus:edge - imagePullPolicy: Always - name: nginx - ports: - - containerPort: 80 - name: http - - containerPort: 443 - name: https - - containerPort: 9113 - name: metrics - securityContext: - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /var/log/nginx-agent - name: nginx-agent-log - - mountPath: /etc/nginx/conf.d - name: nginx-conf - - mountPath: /etc/nginx/stream-conf.d - name: nginx-stream-conf - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - - mountPath: /etc/nginx/secrets - name: nginx-secrets - - mountPath: /var/run/nginx - name: nginx-run - - mountPath: /var/cache/nginx - name: nginx-cache - - mountPath: /etc/nginx/includes - name: nginx-includes - - mountPath: /var/lib/nginx/state - name: nginx-lib - - mountPath: /etc/nginx/license.jwt - name: nginx-plus-license - subPath: license.jwt - initContainers: - - command: - - /usr/bin/gateway - - initialize - - --source - - /agent/nginx-agent.conf - - --destination - - /etc/nginx-agent - - --source - - /includes/main.conf - - --destination - - /etc/nginx/main-includes - - --source - - /includes/mgmt.conf - - --nginx-plus - - --destination - - /etc/nginx/main-includes - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/nginx/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: init - securityContext: - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /agent - name: nginx-agent-config - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /includes - name: nginx-includes-bootstrap - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - securityContext: - fsGroup: 1001 - runAsNonRoot: true - serviceAccountName: nginx-gateway - terminationGracePeriodSeconds: 30 - volumes: - - emptyDir: {} - name: nginx-agent - - configMap: - name: nginx-agent-config - name: nginx-agent-config - - emptyDir: {} - name: nginx-agent-log - - emptyDir: {} - name: nginx-conf - - emptyDir: {} - name: nginx-stream-conf - - emptyDir: {} - name: nginx-main-includes - - emptyDir: {} - name: nginx-secrets - - emptyDir: {} - name: nginx-run - - emptyDir: {} - name: nginx-cache - - emptyDir: {} - name: nginx-includes - - configMap: - name: nginx-includes-bootstrap - name: nginx-includes-bootstrap - - emptyDir: {} - name: nginx-lib - - name: nginx-plus-license - secret: - secretName: nplus-license ---- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: @@ -492,6 +276,11 @@ metadata: name: nginx spec: controllerName: gateway.nginx.org/nginx-gateway-controller + parametersRef: + group: gateway.nginx.org + kind: NginxProxy + name: nginx-gateway-proxy-config + namespace: nginx-gateway --- apiVersion: gateway.nginx.org/v1alpha1 kind: NginxGateway @@ -505,3 +294,25 @@ metadata: spec: logging: level: info +--- +apiVersion: gateway.nginx.org/v1alpha2 +kind: NginxProxy +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-proxy-config + namespace: nginx-gateway +spec: + kubernetes: + deployment: + container: + image: + pullPolicy: Always + repository: private-registry.nginx.com/nginx-gateway-fabric/nginx-plus + tag: edge + replicas: 1 + service: + externalTrafficPolicy: Local + type: LoadBalancer diff --git a/deploy/experimental/deploy.yaml b/deploy/experimental/deploy.yaml index c847f0a4cd..15311817cc 100644 --- a/deploy/experimental/deploy.yaml +++ b/deploy/experimental/deploy.yaml @@ -24,12 +24,25 @@ metadata: rules: - apiGroups: - "" + - apps resources: - - namespaces - - services - secrets - - pods - configmaps + - serviceaccounts + - services + - deployments + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - "" + resources: + - namespaces + - pods verbs: - get - list @@ -146,60 +159,6 @@ subjects: namespace: nginx-gateway --- apiVersion: v1 -data: - nginx-agent.conf: |- - command: - server: - host: nginx-gateway.nginx-gateway.svc - port: 443 - allowed_directories: - - /etc/nginx - - /usr/share/nginx - - /var/run/nginx - features: - - connection - - configuration - - certificates - - metrics - log: - level: debug - collector: - receivers: - host_metrics: - collection_interval: 1m0s - initial_delay: 1s - scrapers: - cpu: {} - memory: {} - disk: {} - network: {} - filesystem: {} - processors: - batch: {} - exporters: - prometheus_exporter: - server: - host: "0.0.0.0" - port: 9113 -kind: ConfigMap -metadata: - name: nginx-agent-config - namespace: nginx-gateway ---- -apiVersion: v1 -data: - main.conf: | - error_log stderr info; -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: nginx-includes-bootstrap - namespace: nginx-gateway ---- -apiVersion: v1 kind: Service metadata: labels: @@ -219,31 +178,6 @@ spec: app.kubernetes.io/name: nginx-gateway type: ClusterIP --- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - externalTrafficPolicy: Local - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 80 - - name: https - port: 443 - protocol: TCP - targetPort: 443 - selector: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - type: LoadBalancer ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -280,10 +214,6 @@ spec: - --leader-election-lock-name=nginx-gateway-leader-election - --gateway-api-experimental-features env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - name: POD_NAMESPACE valueFrom: fieldRef: @@ -296,6 +226,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid + - name: INSTANCE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/instance'] + - name: IMAGE_NAME + value: ghcr.io/nginx/nginx-gateway-fabric:edge image: ghcr.io/nginx/nginx-gateway-fabric:edge imagePullPolicy: Always name: nginx-gateway @@ -328,135 +264,6 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 --- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - selector: - matchLabels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - template: - metadata: - annotations: - prometheus.io/port: "9113" - prometheus.io/scrape: "true" - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - spec: - containers: - - image: ghcr.io/nginx/nginx-gateway-fabric/nginx:edge - imagePullPolicy: Always - name: nginx - ports: - - containerPort: 80 - name: http - - containerPort: 443 - name: https - - containerPort: 9113 - name: metrics - securityContext: - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /var/log/nginx-agent - name: nginx-agent-log - - mountPath: /etc/nginx/conf.d - name: nginx-conf - - mountPath: /etc/nginx/stream-conf.d - name: nginx-stream-conf - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - - mountPath: /etc/nginx/secrets - name: nginx-secrets - - mountPath: /var/run/nginx - name: nginx-run - - mountPath: /var/cache/nginx - name: nginx-cache - - mountPath: /etc/nginx/includes - name: nginx-includes - initContainers: - - command: - - /usr/bin/gateway - - initialize - - --source - - /agent/nginx-agent.conf - - --destination - - /etc/nginx-agent - - --source - - /includes/main.conf - - --destination - - /etc/nginx/main-includes - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/nginx/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: init - securityContext: - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /agent - name: nginx-agent-config - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /includes - name: nginx-includes-bootstrap - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - securityContext: - fsGroup: 1001 - runAsNonRoot: true - serviceAccountName: nginx-gateway - terminationGracePeriodSeconds: 30 - volumes: - - emptyDir: {} - name: nginx-agent - - configMap: - name: nginx-agent-config - name: nginx-agent-config - - emptyDir: {} - name: nginx-agent-log - - emptyDir: {} - name: nginx-conf - - emptyDir: {} - name: nginx-stream-conf - - emptyDir: {} - name: nginx-main-includes - - emptyDir: {} - name: nginx-secrets - - emptyDir: {} - name: nginx-run - - emptyDir: {} - name: nginx-cache - - emptyDir: {} - name: nginx-includes - - configMap: - name: nginx-includes-bootstrap - name: nginx-includes-bootstrap ---- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: @@ -467,6 +274,11 @@ metadata: name: nginx spec: controllerName: gateway.nginx.org/nginx-gateway-controller + parametersRef: + group: gateway.nginx.org + kind: NginxProxy + name: nginx-gateway-proxy-config + namespace: nginx-gateway --- apiVersion: gateway.nginx.org/v1alpha1 kind: NginxGateway @@ -480,3 +292,25 @@ metadata: spec: logging: level: info +--- +apiVersion: gateway.nginx.org/v1alpha2 +kind: NginxProxy +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-proxy-config + namespace: nginx-gateway +spec: + kubernetes: + deployment: + container: + image: + pullPolicy: Always + repository: ghcr.io/nginx/nginx-gateway-fabric/nginx + tag: edge + replicas: 1 + service: + externalTrafficPolicy: Local + type: LoadBalancer diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml index 282a7b8878..ca6be2dd91 100644 --- a/deploy/nginx-plus/deploy.yaml +++ b/deploy/nginx-plus/deploy.yaml @@ -4,8 +4,6 @@ metadata: name: nginx-gateway --- apiVersion: v1 -imagePullSecrets: -- name: nginx-plus-registry-secret kind: ServiceAccount metadata: labels: @@ -26,10 +24,24 @@ metadata: rules: - apiGroups: - "" + - apps resources: - - namespaces - - services - secrets + - configmaps + - serviceaccounts + - services + - deployments + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - "" + resources: + - namespaces - pods verbs: - get @@ -143,66 +155,6 @@ subjects: namespace: nginx-gateway --- apiVersion: v1 -data: - nginx-agent.conf: |- - command: - server: - host: nginx-gateway.nginx-gateway.svc - port: 443 - allowed_directories: - - /etc/nginx - - /usr/share/nginx - - /var/run/nginx - features: - - connection - - configuration - - certificates - - metrics - - api-action - log: - level: debug - collector: - receivers: - host_metrics: - collection_interval: 1m0s - initial_delay: 1s - scrapers: - cpu: {} - memory: {} - disk: {} - network: {} - filesystem: {} - processors: - batch: {} - exporters: - prometheus_exporter: - server: - host: "0.0.0.0" - port: 9113 -kind: ConfigMap -metadata: - name: nginx-agent-config - namespace: nginx-gateway ---- -apiVersion: v1 -data: - main.conf: | - error_log stderr info; - mgmt.conf: | - mgmt { - enforce_initial_report off; - deployment_context /etc/nginx/main-includes/deployment_ctx.json; - } -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: nginx-includes-bootstrap - namespace: nginx-gateway ---- -apiVersion: v1 kind: Service metadata: labels: @@ -222,31 +174,6 @@ spec: app.kubernetes.io/name: nginx-gateway type: ClusterIP --- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - externalTrafficPolicy: Local - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 80 - - name: https - port: 443 - protocol: TCP - targetPort: 443 - selector: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - type: LoadBalancer ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -284,10 +211,6 @@ spec: - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - name: POD_NAMESPACE valueFrom: fieldRef: @@ -300,6 +223,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid + - name: INSTANCE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/instance'] + - name: IMAGE_NAME + value: ghcr.io/nginx/nginx-gateway-fabric:edge image: ghcr.io/nginx/nginx-gateway-fabric:edge imagePullPolicy: Always name: nginx-gateway @@ -332,150 +261,6 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 --- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - selector: - matchLabels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - template: - metadata: - annotations: - prometheus.io/port: "9113" - prometheus.io/scrape: "true" - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - spec: - containers: - - image: private-registry.nginx.com/nginx-gateway-fabric/nginx-plus:edge - imagePullPolicy: Always - name: nginx - ports: - - containerPort: 80 - name: http - - containerPort: 443 - name: https - - containerPort: 9113 - name: metrics - securityContext: - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /var/log/nginx-agent - name: nginx-agent-log - - mountPath: /etc/nginx/conf.d - name: nginx-conf - - mountPath: /etc/nginx/stream-conf.d - name: nginx-stream-conf - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - - mountPath: /etc/nginx/secrets - name: nginx-secrets - - mountPath: /var/run/nginx - name: nginx-run - - mountPath: /var/cache/nginx - name: nginx-cache - - mountPath: /etc/nginx/includes - name: nginx-includes - - mountPath: /var/lib/nginx/state - name: nginx-lib - - mountPath: /etc/nginx/license.jwt - name: nginx-plus-license - subPath: license.jwt - initContainers: - - command: - - /usr/bin/gateway - - initialize - - --source - - /agent/nginx-agent.conf - - --destination - - /etc/nginx-agent - - --source - - /includes/main.conf - - --destination - - /etc/nginx/main-includes - - --source - - /includes/mgmt.conf - - --nginx-plus - - --destination - - /etc/nginx/main-includes - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/nginx/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: init - securityContext: - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /agent - name: nginx-agent-config - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /includes - name: nginx-includes-bootstrap - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - securityContext: - fsGroup: 1001 - runAsNonRoot: true - serviceAccountName: nginx-gateway - terminationGracePeriodSeconds: 30 - volumes: - - emptyDir: {} - name: nginx-agent - - configMap: - name: nginx-agent-config - name: nginx-agent-config - - emptyDir: {} - name: nginx-agent-log - - emptyDir: {} - name: nginx-conf - - emptyDir: {} - name: nginx-stream-conf - - emptyDir: {} - name: nginx-main-includes - - emptyDir: {} - name: nginx-secrets - - emptyDir: {} - name: nginx-run - - emptyDir: {} - name: nginx-cache - - emptyDir: {} - name: nginx-includes - - configMap: - name: nginx-includes-bootstrap - name: nginx-includes-bootstrap - - emptyDir: {} - name: nginx-lib - - name: nginx-plus-license - secret: - secretName: nplus-license ---- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: @@ -486,6 +271,11 @@ metadata: name: nginx spec: controllerName: gateway.nginx.org/nginx-gateway-controller + parametersRef: + group: gateway.nginx.org + kind: NginxProxy + name: nginx-gateway-proxy-config + namespace: nginx-gateway --- apiVersion: gateway.nginx.org/v1alpha1 kind: NginxGateway @@ -499,3 +289,25 @@ metadata: spec: logging: level: info +--- +apiVersion: gateway.nginx.org/v1alpha2 +kind: NginxProxy +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-proxy-config + namespace: nginx-gateway +spec: + kubernetes: + deployment: + container: + image: + pullPolicy: Always + repository: private-registry.nginx.com/nginx-gateway-fabric/nginx-plus + tag: edge + replicas: 1 + service: + externalTrafficPolicy: Local + type: LoadBalancer diff --git a/deploy/nodeport/deploy.yaml b/deploy/nodeport/deploy.yaml index ec4d874a80..fa29623d9a 100644 --- a/deploy/nodeport/deploy.yaml +++ b/deploy/nodeport/deploy.yaml @@ -24,10 +24,24 @@ metadata: rules: - apiGroups: - "" + - apps resources: - - namespaces - - services - secrets + - configmaps + - serviceaccounts + - services + - deployments + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - "" + resources: + - namespaces - pods verbs: - get @@ -141,60 +155,6 @@ subjects: namespace: nginx-gateway --- apiVersion: v1 -data: - nginx-agent.conf: |- - command: - server: - host: nginx-gateway.nginx-gateway.svc - port: 443 - allowed_directories: - - /etc/nginx - - /usr/share/nginx - - /var/run/nginx - features: - - connection - - configuration - - certificates - - metrics - log: - level: debug - collector: - receivers: - host_metrics: - collection_interval: 1m0s - initial_delay: 1s - scrapers: - cpu: {} - memory: {} - disk: {} - network: {} - filesystem: {} - processors: - batch: {} - exporters: - prometheus_exporter: - server: - host: "0.0.0.0" - port: 9113 -kind: ConfigMap -metadata: - name: nginx-agent-config - namespace: nginx-gateway ---- -apiVersion: v1 -data: - main.conf: | - error_log stderr info; -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: nginx-includes-bootstrap - namespace: nginx-gateway ---- -apiVersion: v1 kind: Service metadata: labels: @@ -214,31 +174,6 @@ spec: app.kubernetes.io/name: nginx-gateway type: ClusterIP --- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - externalTrafficPolicy: Local - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 80 - - name: https - port: 443 - protocol: TCP - targetPort: 443 - selector: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - type: NodePort ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -274,10 +209,6 @@ spec: - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - name: POD_NAMESPACE valueFrom: fieldRef: @@ -290,6 +221,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid + - name: INSTANCE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/instance'] + - name: IMAGE_NAME + value: ghcr.io/nginx/nginx-gateway-fabric:edge image: ghcr.io/nginx/nginx-gateway-fabric:edge imagePullPolicy: Always name: nginx-gateway @@ -322,135 +259,6 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 --- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - selector: - matchLabels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - template: - metadata: - annotations: - prometheus.io/port: "9113" - prometheus.io/scrape: "true" - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - spec: - containers: - - image: ghcr.io/nginx/nginx-gateway-fabric/nginx:edge - imagePullPolicy: Always - name: nginx - ports: - - containerPort: 80 - name: http - - containerPort: 443 - name: https - - containerPort: 9113 - name: metrics - securityContext: - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /var/log/nginx-agent - name: nginx-agent-log - - mountPath: /etc/nginx/conf.d - name: nginx-conf - - mountPath: /etc/nginx/stream-conf.d - name: nginx-stream-conf - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - - mountPath: /etc/nginx/secrets - name: nginx-secrets - - mountPath: /var/run/nginx - name: nginx-run - - mountPath: /var/cache/nginx - name: nginx-cache - - mountPath: /etc/nginx/includes - name: nginx-includes - initContainers: - - command: - - /usr/bin/gateway - - initialize - - --source - - /agent/nginx-agent.conf - - --destination - - /etc/nginx-agent - - --source - - /includes/main.conf - - --destination - - /etc/nginx/main-includes - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/nginx/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: init - securityContext: - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /agent - name: nginx-agent-config - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /includes - name: nginx-includes-bootstrap - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - securityContext: - fsGroup: 1001 - runAsNonRoot: true - serviceAccountName: nginx-gateway - terminationGracePeriodSeconds: 30 - volumes: - - emptyDir: {} - name: nginx-agent - - configMap: - name: nginx-agent-config - name: nginx-agent-config - - emptyDir: {} - name: nginx-agent-log - - emptyDir: {} - name: nginx-conf - - emptyDir: {} - name: nginx-stream-conf - - emptyDir: {} - name: nginx-main-includes - - emptyDir: {} - name: nginx-secrets - - emptyDir: {} - name: nginx-run - - emptyDir: {} - name: nginx-cache - - emptyDir: {} - name: nginx-includes - - configMap: - name: nginx-includes-bootstrap - name: nginx-includes-bootstrap ---- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: @@ -461,6 +269,11 @@ metadata: name: nginx spec: controllerName: gateway.nginx.org/nginx-gateway-controller + parametersRef: + group: gateway.nginx.org + kind: NginxProxy + name: nginx-gateway-proxy-config + namespace: nginx-gateway --- apiVersion: gateway.nginx.org/v1alpha1 kind: NginxGateway @@ -474,3 +287,25 @@ metadata: spec: logging: level: info +--- +apiVersion: gateway.nginx.org/v1alpha2 +kind: NginxProxy +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-proxy-config + namespace: nginx-gateway +spec: + kubernetes: + deployment: + container: + image: + pullPolicy: Always + repository: ghcr.io/nginx/nginx-gateway-fabric/nginx + tag: edge + replicas: 1 + service: + externalTrafficPolicy: Local + type: NodePort diff --git a/deploy/openshift/deploy.yaml b/deploy/openshift/deploy.yaml index b22981cd1b..e2701bf885 100644 --- a/deploy/openshift/deploy.yaml +++ b/deploy/openshift/deploy.yaml @@ -24,10 +24,24 @@ metadata: rules: - apiGroups: - "" + - apps resources: - - namespaces - - services - secrets + - configmaps + - serviceaccounts + - services + - deployments + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - "" + resources: + - namespaces - pods verbs: - get @@ -130,6 +144,8 @@ rules: - securitycontextconstraints verbs: - use + - create + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -149,60 +165,6 @@ subjects: namespace: nginx-gateway --- apiVersion: v1 -data: - nginx-agent.conf: |- - command: - server: - host: nginx-gateway.nginx-gateway.svc - port: 443 - allowed_directories: - - /etc/nginx - - /usr/share/nginx - - /var/run/nginx - features: - - connection - - configuration - - certificates - - metrics - log: - level: debug - collector: - receivers: - host_metrics: - collection_interval: 1m0s - initial_delay: 1s - scrapers: - cpu: {} - memory: {} - disk: {} - network: {} - filesystem: {} - processors: - batch: {} - exporters: - prometheus_exporter: - server: - host: "0.0.0.0" - port: 9113 -kind: ConfigMap -metadata: - name: nginx-agent-config - namespace: nginx-gateway ---- -apiVersion: v1 -data: - main.conf: | - error_log stderr info; -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: nginx-includes-bootstrap - namespace: nginx-gateway ---- -apiVersion: v1 kind: Service metadata: labels: @@ -222,31 +184,6 @@ spec: app.kubernetes.io/name: nginx-gateway type: ClusterIP --- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - externalTrafficPolicy: Local - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 80 - - name: https - port: 443 - protocol: TCP - targetPort: 443 - selector: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - type: LoadBalancer ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -282,10 +219,6 @@ spec: - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - name: POD_NAMESPACE valueFrom: fieldRef: @@ -298,6 +231,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid + - name: INSTANCE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/instance'] + - name: IMAGE_NAME + value: ghcr.io/nginx/nginx-gateway-fabric:edge image: ghcr.io/nginx/nginx-gateway-fabric:edge imagePullPolicy: Always name: nginx-gateway @@ -330,135 +269,6 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 --- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - selector: - matchLabels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - template: - metadata: - annotations: - prometheus.io/port: "9113" - prometheus.io/scrape: "true" - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - spec: - containers: - - image: ghcr.io/nginx/nginx-gateway-fabric/nginx:edge - imagePullPolicy: Always - name: nginx - ports: - - containerPort: 80 - name: http - - containerPort: 443 - name: https - - containerPort: 9113 - name: metrics - securityContext: - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /var/log/nginx-agent - name: nginx-agent-log - - mountPath: /etc/nginx/conf.d - name: nginx-conf - - mountPath: /etc/nginx/stream-conf.d - name: nginx-stream-conf - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - - mountPath: /etc/nginx/secrets - name: nginx-secrets - - mountPath: /var/run/nginx - name: nginx-run - - mountPath: /var/cache/nginx - name: nginx-cache - - mountPath: /etc/nginx/includes - name: nginx-includes - initContainers: - - command: - - /usr/bin/gateway - - initialize - - --source - - /agent/nginx-agent.conf - - --destination - - /etc/nginx-agent - - --source - - /includes/main.conf - - --destination - - /etc/nginx/main-includes - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/nginx/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: init - securityContext: - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /agent - name: nginx-agent-config - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /includes - name: nginx-includes-bootstrap - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - securityContext: - fsGroup: 1001 - runAsNonRoot: true - serviceAccountName: nginx-gateway - terminationGracePeriodSeconds: 30 - volumes: - - emptyDir: {} - name: nginx-agent - - configMap: - name: nginx-agent-config - name: nginx-agent-config - - emptyDir: {} - name: nginx-agent-log - - emptyDir: {} - name: nginx-conf - - emptyDir: {} - name: nginx-stream-conf - - emptyDir: {} - name: nginx-main-includes - - emptyDir: {} - name: nginx-secrets - - emptyDir: {} - name: nginx-run - - emptyDir: {} - name: nginx-cache - - emptyDir: {} - name: nginx-includes - - configMap: - name: nginx-includes-bootstrap - name: nginx-includes-bootstrap ---- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: @@ -469,6 +279,11 @@ metadata: name: nginx spec: controllerName: gateway.nginx.org/nginx-gateway-controller + parametersRef: + group: gateway.nginx.org + kind: NginxProxy + name: nginx-gateway-proxy-config + namespace: nginx-gateway --- apiVersion: gateway.nginx.org/v1alpha1 kind: NginxGateway @@ -483,6 +298,28 @@ spec: logging: level: info --- +apiVersion: gateway.nginx.org/v1alpha2 +kind: NginxProxy +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-proxy-config + namespace: nginx-gateway +spec: + kubernetes: + deployment: + container: + image: + pullPolicy: Always + repository: ghcr.io/nginx/nginx-gateway-fabric/nginx + tag: edge + replicas: 1 + service: + externalTrafficPolicy: Local + type: LoadBalancer +--- allowHostDirVolumePlugin: false allowHostIPC: false allowHostNetwork: false diff --git a/deploy/snippets-filters-nginx-plus/deploy.yaml b/deploy/snippets-filters-nginx-plus/deploy.yaml index ffed3588fd..f452442bef 100644 --- a/deploy/snippets-filters-nginx-plus/deploy.yaml +++ b/deploy/snippets-filters-nginx-plus/deploy.yaml @@ -4,8 +4,6 @@ metadata: name: nginx-gateway --- apiVersion: v1 -imagePullSecrets: -- name: nginx-plus-registry-secret kind: ServiceAccount metadata: labels: @@ -26,10 +24,24 @@ metadata: rules: - apiGroups: - "" + - apps resources: - - namespaces - - services - secrets + - configmaps + - serviceaccounts + - services + - deployments + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - "" + resources: + - namespaces - pods verbs: - get @@ -145,66 +157,6 @@ subjects: namespace: nginx-gateway --- apiVersion: v1 -data: - nginx-agent.conf: |- - command: - server: - host: nginx-gateway.nginx-gateway.svc - port: 443 - allowed_directories: - - /etc/nginx - - /usr/share/nginx - - /var/run/nginx - features: - - connection - - configuration - - certificates - - metrics - - api-action - log: - level: debug - collector: - receivers: - host_metrics: - collection_interval: 1m0s - initial_delay: 1s - scrapers: - cpu: {} - memory: {} - disk: {} - network: {} - filesystem: {} - processors: - batch: {} - exporters: - prometheus_exporter: - server: - host: "0.0.0.0" - port: 9113 -kind: ConfigMap -metadata: - name: nginx-agent-config - namespace: nginx-gateway ---- -apiVersion: v1 -data: - main.conf: | - error_log stderr info; - mgmt.conf: | - mgmt { - enforce_initial_report off; - deployment_context /etc/nginx/main-includes/deployment_ctx.json; - } -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: nginx-includes-bootstrap - namespace: nginx-gateway ---- -apiVersion: v1 kind: Service metadata: labels: @@ -224,31 +176,6 @@ spec: app.kubernetes.io/name: nginx-gateway type: ClusterIP --- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - externalTrafficPolicy: Local - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 80 - - name: https - port: 443 - protocol: TCP - targetPort: 443 - selector: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - type: LoadBalancer ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -287,10 +214,6 @@ spec: - --leader-election-lock-name=nginx-gateway-leader-election - --snippets-filters env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - name: POD_NAMESPACE valueFrom: fieldRef: @@ -303,6 +226,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid + - name: INSTANCE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/instance'] + - name: IMAGE_NAME + value: ghcr.io/nginx/nginx-gateway-fabric:edge image: ghcr.io/nginx/nginx-gateway-fabric:edge imagePullPolicy: Always name: nginx-gateway @@ -335,150 +264,6 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 --- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - selector: - matchLabels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - template: - metadata: - annotations: - prometheus.io/port: "9113" - prometheus.io/scrape: "true" - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - spec: - containers: - - image: private-registry.nginx.com/nginx-gateway-fabric/nginx-plus:edge - imagePullPolicy: Always - name: nginx - ports: - - containerPort: 80 - name: http - - containerPort: 443 - name: https - - containerPort: 9113 - name: metrics - securityContext: - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /var/log/nginx-agent - name: nginx-agent-log - - mountPath: /etc/nginx/conf.d - name: nginx-conf - - mountPath: /etc/nginx/stream-conf.d - name: nginx-stream-conf - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - - mountPath: /etc/nginx/secrets - name: nginx-secrets - - mountPath: /var/run/nginx - name: nginx-run - - mountPath: /var/cache/nginx - name: nginx-cache - - mountPath: /etc/nginx/includes - name: nginx-includes - - mountPath: /var/lib/nginx/state - name: nginx-lib - - mountPath: /etc/nginx/license.jwt - name: nginx-plus-license - subPath: license.jwt - initContainers: - - command: - - /usr/bin/gateway - - initialize - - --source - - /agent/nginx-agent.conf - - --destination - - /etc/nginx-agent - - --source - - /includes/main.conf - - --destination - - /etc/nginx/main-includes - - --source - - /includes/mgmt.conf - - --nginx-plus - - --destination - - /etc/nginx/main-includes - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/nginx/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: init - securityContext: - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /agent - name: nginx-agent-config - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /includes - name: nginx-includes-bootstrap - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - securityContext: - fsGroup: 1001 - runAsNonRoot: true - serviceAccountName: nginx-gateway - terminationGracePeriodSeconds: 30 - volumes: - - emptyDir: {} - name: nginx-agent - - configMap: - name: nginx-agent-config - name: nginx-agent-config - - emptyDir: {} - name: nginx-agent-log - - emptyDir: {} - name: nginx-conf - - emptyDir: {} - name: nginx-stream-conf - - emptyDir: {} - name: nginx-main-includes - - emptyDir: {} - name: nginx-secrets - - emptyDir: {} - name: nginx-run - - emptyDir: {} - name: nginx-cache - - emptyDir: {} - name: nginx-includes - - configMap: - name: nginx-includes-bootstrap - name: nginx-includes-bootstrap - - emptyDir: {} - name: nginx-lib - - name: nginx-plus-license - secret: - secretName: nplus-license ---- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: @@ -489,6 +274,11 @@ metadata: name: nginx spec: controllerName: gateway.nginx.org/nginx-gateway-controller + parametersRef: + group: gateway.nginx.org + kind: NginxProxy + name: nginx-gateway-proxy-config + namespace: nginx-gateway --- apiVersion: gateway.nginx.org/v1alpha1 kind: NginxGateway @@ -502,3 +292,25 @@ metadata: spec: logging: level: info +--- +apiVersion: gateway.nginx.org/v1alpha2 +kind: NginxProxy +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-proxy-config + namespace: nginx-gateway +spec: + kubernetes: + deployment: + container: + image: + pullPolicy: Always + repository: private-registry.nginx.com/nginx-gateway-fabric/nginx-plus + tag: edge + replicas: 1 + service: + externalTrafficPolicy: Local + type: LoadBalancer diff --git a/deploy/snippets-filters/deploy.yaml b/deploy/snippets-filters/deploy.yaml index 6fa5e75077..dfe78332b5 100644 --- a/deploy/snippets-filters/deploy.yaml +++ b/deploy/snippets-filters/deploy.yaml @@ -24,10 +24,24 @@ metadata: rules: - apiGroups: - "" + - apps resources: - - namespaces - - services - secrets + - configmaps + - serviceaccounts + - services + - deployments + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - "" + resources: + - namespaces - pods verbs: - get @@ -143,60 +157,6 @@ subjects: namespace: nginx-gateway --- apiVersion: v1 -data: - nginx-agent.conf: |- - command: - server: - host: nginx-gateway.nginx-gateway.svc - port: 443 - allowed_directories: - - /etc/nginx - - /usr/share/nginx - - /var/run/nginx - features: - - connection - - configuration - - certificates - - metrics - log: - level: debug - collector: - receivers: - host_metrics: - collection_interval: 1m0s - initial_delay: 1s - scrapers: - cpu: {} - memory: {} - disk: {} - network: {} - filesystem: {} - processors: - batch: {} - exporters: - prometheus_exporter: - server: - host: "0.0.0.0" - port: 9113 -kind: ConfigMap -metadata: - name: nginx-agent-config - namespace: nginx-gateway ---- -apiVersion: v1 -data: - main.conf: | - error_log stderr info; -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: nginx-includes-bootstrap - namespace: nginx-gateway ---- -apiVersion: v1 kind: Service metadata: labels: @@ -216,31 +176,6 @@ spec: app.kubernetes.io/name: nginx-gateway type: ClusterIP --- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: nginx-gateway - app.kubernetes.io/version: edge - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - externalTrafficPolicy: Local - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 80 - - name: https - port: 443 - protocol: TCP - targetPort: 443 - selector: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - type: LoadBalancer ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -277,10 +212,6 @@ spec: - --leader-election-lock-name=nginx-gateway-leader-election - --snippets-filters env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - name: POD_NAMESPACE valueFrom: fieldRef: @@ -293,6 +224,12 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid + - name: INSTANCE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/instance'] + - name: IMAGE_NAME + value: ghcr.io/nginx/nginx-gateway-fabric:edge image: ghcr.io/nginx/nginx-gateway-fabric:edge imagePullPolicy: Always name: nginx-gateway @@ -325,135 +262,6 @@ spec: serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 --- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tmp-nginx-deployment - namespace: nginx-gateway -spec: - selector: - matchLabels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - template: - metadata: - annotations: - prometheus.io/port: "9113" - prometheus.io/scrape: "true" - labels: - app.kubernetes.io/instance: nginx-gateway - app.kubernetes.io/name: tmp-nginx-deployment - spec: - containers: - - image: ghcr.io/nginx/nginx-gateway-fabric/nginx:edge - imagePullPolicy: Always - name: nginx - ports: - - containerPort: 80 - name: http - - containerPort: 443 - name: https - - containerPort: 9113 - name: metrics - securityContext: - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /var/log/nginx-agent - name: nginx-agent-log - - mountPath: /etc/nginx/conf.d - name: nginx-conf - - mountPath: /etc/nginx/stream-conf.d - name: nginx-stream-conf - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - - mountPath: /etc/nginx/secrets - name: nginx-secrets - - mountPath: /var/run/nginx - name: nginx-run - - mountPath: /var/cache/nginx - name: nginx-cache - - mountPath: /etc/nginx/includes - name: nginx-includes - initContainers: - - command: - - /usr/bin/gateway - - initialize - - --source - - /agent/nginx-agent.conf - - --destination - - /etc/nginx-agent - - --source - - /includes/main.conf - - --destination - - /etc/nginx/main-includes - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - image: ghcr.io/nginx/nginx-gateway-fabric:edge - imagePullPolicy: Always - name: init - securityContext: - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 1001 - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /agent - name: nginx-agent-config - - mountPath: /etc/nginx-agent - name: nginx-agent - - mountPath: /includes - name: nginx-includes-bootstrap - - mountPath: /etc/nginx/main-includes - name: nginx-main-includes - securityContext: - fsGroup: 1001 - runAsNonRoot: true - serviceAccountName: nginx-gateway - terminationGracePeriodSeconds: 30 - volumes: - - emptyDir: {} - name: nginx-agent - - configMap: - name: nginx-agent-config - name: nginx-agent-config - - emptyDir: {} - name: nginx-agent-log - - emptyDir: {} - name: nginx-conf - - emptyDir: {} - name: nginx-stream-conf - - emptyDir: {} - name: nginx-main-includes - - emptyDir: {} - name: nginx-secrets - - emptyDir: {} - name: nginx-run - - emptyDir: {} - name: nginx-cache - - emptyDir: {} - name: nginx-includes - - configMap: - name: nginx-includes-bootstrap - name: nginx-includes-bootstrap ---- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: @@ -464,6 +272,11 @@ metadata: name: nginx spec: controllerName: gateway.nginx.org/nginx-gateway-controller + parametersRef: + group: gateway.nginx.org + kind: NginxProxy + name: nginx-gateway-proxy-config + namespace: nginx-gateway --- apiVersion: gateway.nginx.org/v1alpha1 kind: NginxGateway @@ -477,3 +290,25 @@ metadata: spec: logging: level: info +--- +apiVersion: gateway.nginx.org/v1alpha2 +kind: NginxProxy +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-proxy-config + namespace: nginx-gateway +spec: + kubernetes: + deployment: + container: + image: + pullPolicy: Always + repository: ghcr.io/nginx/nginx-gateway-fabric/nginx + tag: edge + replicas: 1 + service: + externalTrafficPolicy: Local + type: LoadBalancer diff --git a/docs/developer/quickstart.md b/docs/developer/quickstart.md index 058d3d5b9d..697acf5036 100644 --- a/docs/developer/quickstart.md +++ b/docs/developer/quickstart.md @@ -183,13 +183,13 @@ This will build the docker images `nginx-gateway-fabric:` and `nginx- - To install with Helm (where your release name is `my-release`): ```shell - helm install my-release ./charts/nginx-gateway-fabric --create-namespace --wait --set service.type=NodePort --set nginxGateway.image.repository=nginx-gateway-fabric --set nginxGateway.image.tag=$(whoami) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=nginx-gateway-fabric/nginx --set nginx.image.tag=$(whoami) --set nginx.image.pullPolicy=Never -n nginx-gateway + helm install my-release ./charts/nginx-gateway-fabric --create-namespace --wait --set nginx.service.type=NodePort --set nginxGateway.image.repository=nginx-gateway-fabric --set nginxGateway.image.tag=$(whoami) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=nginx-gateway-fabric/nginx --set nginx.image.tag=$(whoami) --set nginx.image.pullPolicy=Never -n nginx-gateway ``` - To install NGINX Plus with Helm (where your release name is `my-release`): ```shell - helm install my-release ./charts/nginx-gateway-fabric --create-namespace --wait --set service.type=NodePort --set nginxGateway.image.repository=nginx-gateway-fabric --set nginxGateway.image.tag=$(whoami) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=nginx-gateway-fabric/nginx-plus --set nginx.image.tag=$(whoami) --set nginx.image.pullPolicy=Never --set nginx.plus=true -n nginx-gateway + helm install my-release ./charts/nginx-gateway-fabric --create-namespace --wait --set nginx.service.type=NodePort --set nginxGateway.image.repository=nginx-gateway-fabric --set nginxGateway.image.tag=$(whoami) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=nginx-gateway-fabric/nginx-plus --set nginx.image.tag=$(whoami) --set nginx.image.pullPolicy=Never --set nginx.plus=true -n nginx-gateway ``` > For more information on Helm configuration options see the Helm [README](../../charts/nginx-gateway-fabric/README.md). diff --git a/examples/helm/aws-nlb/values.yaml b/examples/helm/aws-nlb/values.yaml index b1ffc87974..3034ca995f 100644 --- a/examples/helm/aws-nlb/values.yaml +++ b/examples/helm/aws-nlb/values.yaml @@ -1,7 +1,8 @@ nginxGateway: name: nginx-gateway -service: - type: LoadBalancer - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "external" - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" +nginx: + service: + type: LoadBalancer + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "external" + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" diff --git a/examples/helm/azure/values.yaml b/examples/helm/azure/values.yaml index 3dbfc24256..ee6669108c 100644 --- a/examples/helm/azure/values.yaml +++ b/examples/helm/azure/values.yaml @@ -1,4 +1,8 @@ nginxGateway: name: nginx-gateway -nodeSelector: - kubernetes.io/os: linux + nodeSelector: + kubernetes.io/os: linux +nginx: + pod: + nodeSelector: + kubernetes.io/os: linux diff --git a/examples/helm/experimental-nginx-plus/values.yaml b/examples/helm/experimental-nginx-plus/values.yaml index 08469ce364..e1d854fd3a 100644 --- a/examples/helm/experimental-nginx-plus/values.yaml +++ b/examples/helm/experimental-nginx-plus/values.yaml @@ -7,6 +7,4 @@ nginx: plus: true image: repository: private-registry.nginx.com/nginx-gateway-fabric/nginx-plus - -serviceAccount: imagePullSecret: nginx-plus-registry-secret diff --git a/examples/helm/nginx-plus/values.yaml b/examples/helm/nginx-plus/values.yaml index b8b842d16a..0b85bfc51b 100644 --- a/examples/helm/nginx-plus/values.yaml +++ b/examples/helm/nginx-plus/values.yaml @@ -5,6 +5,4 @@ nginx: plus: true image: repository: private-registry.nginx.com/nginx-gateway-fabric/nginx-plus - -serviceAccount: imagePullSecret: nginx-plus-registry-secret diff --git a/examples/helm/nodeport/values.yaml b/examples/helm/nodeport/values.yaml index 17da6a8849..93318a7b96 100644 --- a/examples/helm/nodeport/values.yaml +++ b/examples/helm/nodeport/values.yaml @@ -1,4 +1,5 @@ nginxGateway: name: nginx-gateway -service: - type: NodePort +nginx: + service: + type: NodePort diff --git a/examples/helm/snippets-filters-nginx-plus/values.yaml b/examples/helm/snippets-filters-nginx-plus/values.yaml index 9cacfdb168..89cc0b59b4 100644 --- a/examples/helm/snippets-filters-nginx-plus/values.yaml +++ b/examples/helm/snippets-filters-nginx-plus/values.yaml @@ -7,6 +7,4 @@ nginx: plus: true image: repository: private-registry.nginx.com/nginx-gateway-fabric/nginx-plus - -serviceAccount: imagePullSecret: nginx-plus-registry-secret diff --git a/go.mod b/go.mod index e4b98bcd75..9d9db005f9 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( go.opentelemetry.io/otel v1.34.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 go.uber.org/zap v1.27.0 + golang.org/x/text v0.21.0 google.golang.org/grpc v1.69.4 google.golang.org/protobuf v1.36.3 k8s.io/api v0.32.1 @@ -79,7 +80,6 @@ require ( golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/term v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.7.0 // indirect golang.org/x/tools v0.29.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect diff --git a/internal/framework/controller/labels.go b/internal/framework/controller/labels.go new file mode 100644 index 0000000000..79b6b55113 --- /dev/null +++ b/internal/framework/controller/labels.go @@ -0,0 +1,12 @@ +package controller + +// The following labels are added to each nginx resource created by the control plane. +const ( + GatewayLabel = "gateway.networking.k8s.io/gateway-name" + AppNameLabel = "app.kubernetes.io/name" + AppInstanceLabel = "app.kubernetes.io/instance" + AppManagedByLabel = "app.kubernetes.io/managed-by" +) + +// RestartedAnnotation is added to a Deployment or DaemonSet's PodSpec to trigger a rolling restart. +const RestartedAnnotation = "kubectl.kubernetes.io/restartedAt" diff --git a/internal/framework/controller/predicate/annotation.go b/internal/framework/controller/predicate/annotation.go index fdf1fd696f..46b48660de 100644 --- a/internal/framework/controller/predicate/annotation.go +++ b/internal/framework/controller/predicate/annotation.go @@ -1,8 +1,11 @@ package predicate import ( + appsv1 "k8s.io/api/apps/v1" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" ) // AnnotationPredicate implements a predicate function based on the Annotation. @@ -37,3 +40,41 @@ func (cp AnnotationPredicate) Update(e event.UpdateEvent) bool { return oldAnnotationVal != newAnnotationVal } + +// RestartDeploymentAnnotationPredicate skips update events if they are due to a rolling restart. +// This type of event is triggered by adding an annotation to the deployment's PodSpec. +// This is used by the provisioner to ensure it allows for rolling restarts of the nginx deployment +// without reverting the annotation and deleting the new pod(s). Otherwise, if a user changes +// the nginx deployment, we want to see that event so we can revert it back to the configuration +// that we expect it to have. +type RestartDeploymentAnnotationPredicate struct { + predicate.Funcs +} + +// Update filters UpdateEvents based on if the annotation is present or changed. +func (cp RestartDeploymentAnnotationPredicate) Update(e event.UpdateEvent) bool { + if e.ObjectOld == nil || e.ObjectNew == nil { + // this case should not happen + return false + } + + depOld, ok := e.ObjectOld.(*appsv1.Deployment) + if !ok { + return false + } + + depNew, ok := e.ObjectNew.(*appsv1.Deployment) + if !ok { + return false + } + + oldVal, oldExists := depOld.Spec.Template.Annotations[controller.RestartedAnnotation] + + if newVal, ok := depNew.Spec.Template.Annotations[controller.RestartedAnnotation]; ok { + if !oldExists || newVal != oldVal { + return false + } + } + + return true +} diff --git a/internal/framework/controller/predicate/annotation_test.go b/internal/framework/controller/predicate/annotation_test.go index 47cd762839..4ecc448b4d 100644 --- a/internal/framework/controller/predicate/annotation_test.go +++ b/internal/framework/controller/predicate/annotation_test.go @@ -4,9 +4,13 @@ import ( "testing" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/event" + + "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" ) func TestAnnotationPredicate_Create(t *testing.T) { @@ -222,3 +226,177 @@ func TestAnnotationPredicate_Update(t *testing.T) { }) } } + +func TestRestartDeploymentAnnotationPredicate_Update(t *testing.T) { + t.Parallel() + + tests := []struct { + event event.UpdateEvent + name string + expUpdate bool + }{ + { + name: "annotation added", + event: event.UpdateEvent{ + ObjectOld: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + }, + }, + }, + ObjectNew: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + controller.RestartedAnnotation: "true", + }, + }, + }, + }, + }, + }, + expUpdate: false, + }, + { + name: "annotation changed", + event: event.UpdateEvent{ + ObjectOld: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + controller.RestartedAnnotation: "false", + }, + }, + }, + }, + }, + ObjectNew: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + controller.RestartedAnnotation: "true", + }, + }, + }, + }, + }, + }, + expUpdate: false, + }, + { + name: "annotation removed", + event: event.UpdateEvent{ + ObjectOld: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + controller.RestartedAnnotation: "true", + }, + }, + }, + }, + }, + ObjectNew: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + }, + }, + }, + }, + expUpdate: true, + }, + { + name: "annotation unchanged", + event: event.UpdateEvent{ + ObjectOld: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + controller.RestartedAnnotation: "true", + }, + }, + }, + }, + }, + ObjectNew: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + controller.RestartedAnnotation: "true", + }, + }, + }, + }, + }, + }, + expUpdate: true, + }, + { + name: "old object is nil", + event: event.UpdateEvent{ + ObjectOld: nil, + ObjectNew: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + controller.RestartedAnnotation: "true", + }, + }, + }, + }, + }, + }, + expUpdate: false, + }, + { + name: "new object is nil", + event: event.UpdateEvent{ + ObjectOld: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + controller.RestartedAnnotation: "true", + }, + }, + }, + }, + }, + ObjectNew: nil, + }, + expUpdate: false, + }, + { + name: "both objects are nil", + event: event.UpdateEvent{ + ObjectOld: nil, + ObjectNew: nil, + }, + expUpdate: false, + }, + } + + p := RestartDeploymentAnnotationPredicate{} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + update := p.Update(test.event) + g.Expect(update).To(Equal(test.expUpdate)) + }) + } +} diff --git a/internal/framework/controller/predicate/label.go b/internal/framework/controller/predicate/label.go new file mode 100644 index 0000000000..06d1d157a6 --- /dev/null +++ b/internal/framework/controller/predicate/label.go @@ -0,0 +1,18 @@ +package predicate + +import ( + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8spredicate "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// NginxLabelPredicate returns a predicate that only matches resources with the nginx labels. +func NginxLabelPredicate(selector metav1.LabelSelector) k8spredicate.Predicate { + labelPredicate, err := k8spredicate.LabelSelectorPredicate(selector) + if err != nil { + panic(fmt.Sprintf("error creating label selector: %v", err)) + } + + return labelPredicate +} diff --git a/internal/framework/controller/predicate/service.go b/internal/framework/controller/predicate/service.go index 04eea8f5d2..21e59e6ee0 100644 --- a/internal/framework/controller/predicate/service.go +++ b/internal/framework/controller/predicate/service.go @@ -2,9 +2,7 @@ package predicate import ( apiv1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" ) @@ -65,54 +63,3 @@ func (ServicePortsChangedPredicate) Update(e event.UpdateEvent) bool { return len(newPortSet) > 0 } - -// GatewayServicePredicate implements predicate functions for this Pod's Service. -type GatewayServicePredicate struct { - predicate.Funcs - NSName types.NamespacedName -} - -// Update implements the default UpdateEvent filter for the Gateway Service. -func (gsp GatewayServicePredicate) Update(e event.UpdateEvent) bool { - if e.ObjectOld == nil { - return false - } - if e.ObjectNew == nil { - return false - } - - oldSvc, ok := e.ObjectOld.(*apiv1.Service) - if !ok { - return false - } - - newSvc, ok := e.ObjectNew.(*apiv1.Service) - if !ok { - return false - } - - if client.ObjectKeyFromObject(newSvc) != gsp.NSName { - return false - } - - if oldSvc.Spec.Type != newSvc.Spec.Type { - return true - } - - if newSvc.Spec.Type == apiv1.ServiceTypeLoadBalancer { - oldIngress := oldSvc.Status.LoadBalancer.Ingress - newIngress := newSvc.Status.LoadBalancer.Ingress - - if len(oldIngress) != len(newIngress) { - return true - } - - for i, ingress := range oldIngress { - if ingress.IP != newIngress[i].IP || ingress.Hostname != newIngress[i].Hostname { - return true - } - } - } - - return false -} diff --git a/internal/framework/controller/predicate/service_test.go b/internal/framework/controller/predicate/service_test.go index 98176774ec..fcb4aa694f 100644 --- a/internal/framework/controller/predicate/service_test.go +++ b/internal/framework/controller/predicate/service_test.go @@ -5,8 +5,6 @@ import ( . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" @@ -245,223 +243,7 @@ func TestServicePortsChangedPredicate(t *testing.T) { t.Parallel() g := NewWithT(t) - p := GatewayServicePredicate{} - - g.Expect(p.Delete(event.DeleteEvent{Object: &v1.Service{}})).To(BeTrue()) - g.Expect(p.Create(event.CreateEvent{Object: &v1.Service{}})).To(BeTrue()) - g.Expect(p.Generic(event.GenericEvent{Object: &v1.Service{}})).To(BeTrue()) -} - -func TestGatewayServicePredicate_Update(t *testing.T) { - t.Parallel() - testcases := []struct { - objectOld client.Object - objectNew client.Object - msg string - expUpdate bool - }{ - { - msg: "nil objectOld", - objectOld: nil, - objectNew: &v1.Service{}, - expUpdate: false, - }, - { - msg: "nil objectNew", - objectOld: &v1.Service{}, - objectNew: nil, - expUpdate: false, - }, - { - msg: "non-Service objectOld", - objectOld: &v1.Namespace{}, - objectNew: &v1.Service{}, - expUpdate: false, - }, - { - msg: "non-Service objectNew", - objectOld: &v1.Service{}, - objectNew: &v1.Namespace{}, - expUpdate: false, - }, - { - msg: "Service not watched", - objectOld: &v1.Service{}, - objectNew: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "nginx-gateway", - Name: "not-watched", - }, - }, - expUpdate: false, - }, - { - msg: "something irrelevant changed", - objectOld: &v1.Service{ - Spec: v1.ServiceSpec{ - ClusterIP: "1.2.3.4", - }, - }, - objectNew: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "nginx-gateway", - Name: "nginx", - }, - Spec: v1.ServiceSpec{ - ClusterIP: "5.6.7.8", - }, - }, - expUpdate: false, - }, - { - msg: "type changed", - objectOld: &v1.Service{ - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeLoadBalancer, - }, - }, - objectNew: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "nginx-gateway", - Name: "nginx", - }, - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeNodePort, - }, - }, - expUpdate: true, - }, - { - msg: "ingress changed length", - objectOld: &v1.Service{ - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeLoadBalancer, - }, - Status: v1.ServiceStatus{ - LoadBalancer: v1.LoadBalancerStatus{ - Ingress: []v1.LoadBalancerIngress{ - { - IP: "1.2.3.4", - }, - }, - }, - }, - }, - objectNew: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "nginx-gateway", - Name: "nginx", - }, - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeNodePort, - }, Status: v1.ServiceStatus{ - LoadBalancer: v1.LoadBalancerStatus{ - Ingress: []v1.LoadBalancerIngress{ - { - IP: "1.2.3.4", - }, - { - IP: "5.6.7.8", - }, - }, - }, - }, - }, - expUpdate: true, - }, - { - msg: "IP address changed", - objectOld: &v1.Service{ - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeLoadBalancer, - }, - Status: v1.ServiceStatus{ - LoadBalancer: v1.LoadBalancerStatus{ - Ingress: []v1.LoadBalancerIngress{ - { - IP: "1.2.3.4", - }, - }, - }, - }, - }, - objectNew: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "nginx-gateway", - Name: "nginx", - }, - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeNodePort, - }, Status: v1.ServiceStatus{ - LoadBalancer: v1.LoadBalancerStatus{ - Ingress: []v1.LoadBalancerIngress{ - { - IP: "5.6.7.8", - }, - }, - }, - }, - }, - expUpdate: true, - }, - { - msg: "Hostname changed", - objectOld: &v1.Service{ - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeLoadBalancer, - }, - Status: v1.ServiceStatus{ - LoadBalancer: v1.LoadBalancerStatus{ - Ingress: []v1.LoadBalancerIngress{ - { - Hostname: "one", - }, - }, - }, - }, - }, - objectNew: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "nginx-gateway", - Name: "nginx", - }, - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeNodePort, - }, Status: v1.ServiceStatus{ - LoadBalancer: v1.LoadBalancerStatus{ - Ingress: []v1.LoadBalancerIngress{ - { - Hostname: "two", - }, - }, - }, - }, - }, - expUpdate: true, - }, - } - - p := GatewayServicePredicate{NSName: types.NamespacedName{Namespace: "nginx-gateway", Name: "nginx"}} - - for _, tc := range testcases { - t.Run(tc.msg, func(t *testing.T) { - t.Parallel() - g := NewWithT(t) - update := p.Update(event.UpdateEvent{ - ObjectOld: tc.objectOld, - ObjectNew: tc.objectNew, - }) - - g.Expect(update).To(Equal(tc.expUpdate)) - }) - } -} - -func TestGatewayServicePredicate(t *testing.T) { - t.Parallel() - g := NewWithT(t) - - p := GatewayServicePredicate{} + p := ServicePortsChangedPredicate{} g.Expect(p.Delete(event.DeleteEvent{Object: &v1.Service{}})).To(BeTrue()) g.Expect(p.Create(event.CreateEvent{Object: &v1.Service{}})).To(BeTrue()) diff --git a/internal/framework/controller/resource.go b/internal/framework/controller/resource.go new file mode 100644 index 0000000000..c238b64924 --- /dev/null +++ b/internal/framework/controller/resource.go @@ -0,0 +1,9 @@ +package controller + +import "fmt" + +// CreateNginxResourceName creates the base resource name for all nginx resources +// created by the control plane. +func CreateNginxResourceName(gatewayName, gatewayClassName string) string { + return fmt.Sprintf("%s-%s", gatewayName, gatewayClassName) +} diff --git a/internal/mode/static/config/config.go b/internal/mode/static/config/config.go index 82b4238836..19837a780a 100644 --- a/internal/mode/static/config/config.go +++ b/internal/mode/static/config/config.go @@ -8,13 +8,13 @@ import ( "k8s.io/apimachinery/pkg/types" ) +const DefaultNginxMetricsPort = int32(9113) + type Config struct { // AtomicLevel is an atomically changeable, dynamic logging level. AtomicLevel zap.AtomicLevel // UsageReportConfig specifies the NGINX Plus usage reporting configuration. UsageReportConfig UsageReportConfig - // Version is the running NGF version. - Version string // ImageSource is the source of the NGINX Gateway image. ImageSource string // Flags contains the NGF command-line flag names and values. @@ -52,8 +52,6 @@ type Config struct { // GatewayPodConfig contains information about this Pod. type GatewayPodConfig struct { - // PodIP is the IP address of this Pod. - PodIP string // ServiceName is the name of the Service that fronts this Pod. ServiceName string // Namespace is the namespace of this Pod. @@ -62,6 +60,13 @@ type GatewayPodConfig struct { Name string // UID is the UID of the Pod. UID string + // InstanceName is the name used in the instance label. + // Generally this will be the name of the Helm release. + InstanceName string + // Version is the running NGF version. + Version string + // Image is the image path of the Pod. + Image string } // MetricsConfig specifies the metrics config. diff --git a/internal/mode/static/handler.go b/internal/mode/static/handler.go index 84a658ae4e..a77db52c3c 100644 --- a/internal/mode/static/handler.go +++ b/internal/mode/static/handler.go @@ -16,6 +16,7 @@ import ( gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" ngfAPI "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1" + "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" "github.com/nginx/nginx-gateway-fabric/internal/framework/events" "github.com/nginx/nginx-gateway-fabric/internal/framework/helpers" frameworkStatus "github.com/nginx/nginx-gateway-fabric/internal/framework/status" @@ -23,6 +24,7 @@ import ( "github.com/nginx/nginx-gateway-fabric/internal/mode/static/licensing" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/nginx/agent" ngxConfig "github.com/nginx/nginx-gateway-fabric/internal/mode/static/nginx/config" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/provisioner" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/dataplane" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/graph" @@ -39,6 +41,8 @@ type eventHandlerConfig struct { ctx context.Context // nginxUpdater updates nginx configuration using the NGINX agent. nginxUpdater agent.NginxUpdater + // nginxProvisioner handles provisioning and deprovisioning nginx resources. + nginxProvisioner provisioner.Provisioner // metricsCollector collects metrics for this controller. metricsCollector handlerMetricsCollector // statusUpdater updates statuses on Kubernetes resources. @@ -73,6 +77,8 @@ type eventHandlerConfig struct { controlConfigNSName types.NamespacedName // gatewayCtlrName is the name of the NGF controller. gatewayCtlrName string + // gatewayClassName is the name of the GatewayClass. + gatewayClassName string // updateGatewayClassStatus enables updating the status of the GatewayClass resource. updateGatewayClassStatus bool // plus is whether or not we are running NGINX Plus. @@ -129,18 +135,6 @@ func newEventHandlerImpl(cfg eventHandlerConfig) *eventHandlerImpl { upsert: handler.nginxGatewayCRDUpsert, delete: handler.nginxGatewayCRDDelete, }, - // NGF-fronting Service - objectFilterKey( - &v1.Service{}, - types.NamespacedName{ - Name: handler.cfg.gatewayPodConfig.ServiceName, - Namespace: handler.cfg.gatewayPodConfig.Namespace, - }, - ): { - upsert: handler.nginxGatewayServiceUpsert, - delete: handler.nginxGatewayServiceDelete, - captureChangeInGraph: true, - }, } go handler.waitForStatusUpdates(cfg.ctx) @@ -194,16 +188,37 @@ func (h *eventHandlerImpl) sendNginxConfig( gr *graph.Graph, changeType state.ChangeType, ) { - deploymentName := types.NamespacedName{ - Name: "tmp-nginx-deployment", - Namespace: h.cfg.gatewayPodConfig.Namespace, + if gr == nil { + logger.Info("Handling events didn't result into NGINX configuration changes") + return + } + + if gr.Gateway == nil { + // still need to update GatewayClass status + obj := &status.QueueObject{ + UpdateType: status.UpdateAll, + } + h.cfg.statusQueue.Enqueue(obj) + return + } + + go func() { + if err := h.cfg.nginxProvisioner.RegisterGateway(ctx, gr.Gateway, gr.DeploymentName.Name); err != nil { + logger.Error(err, "error from provisioner") + } + }() + + if !gr.Gateway.Valid { + obj := &status.QueueObject{ + Deployment: gr.DeploymentName, + UpdateType: status.UpdateAll, + } + h.cfg.statusQueue.Enqueue(obj) + return } - // TODO(sberman): if nginx Deployment is scaled down, we should remove the pod from the ConnectionsTracker - // and Deployment. - // If fully deleted, then delete the deployment from the Store and close the stopCh. stopCh := make(chan struct{}) - deployment := h.cfg.nginxDeployments.GetOrStore(ctx, deploymentName, stopCh) + deployment := h.cfg.nginxDeployments.GetOrStore(ctx, gr.DeploymentName, stopCh) if deployment == nil { panic("expected deployment, got nil") } @@ -216,8 +231,9 @@ func (h *eventHandlerImpl) sendNginxConfig( if configApplied || err != nil { obj := &status.QueueObject{ + UpdateType: status.UpdateAll, Error: err, - Deployment: deploymentName, + Deployment: gr.DeploymentName, } h.cfg.statusQueue.Enqueue(obj) } @@ -232,9 +248,6 @@ func (h *eventHandlerImpl) processStateAndBuildConfig( ) bool { var configApplied bool switch changeType { - case state.NoChange: - logger.Info("Handling events didn't result into NGINX configuration changes") - return false case state.EndpointsOnlyChange: h.version++ cfg := dataplane.BuildConfiguration(ctx, gr, h.cfg.serviceResolver, h.version, h.cfg.plus) @@ -279,28 +292,74 @@ func (h *eventHandlerImpl) waitForStatusUpdates(ctx context.Context) { return } + // TODO(sberman): once we support multiple Gateways, we'll have to get + // the correct Graph for the Deployment contained in the update message + gr := h.cfg.processor.GetLatestGraph() + if gr == nil { + continue + } + var nginxReloadRes graph.NginxReloadResult switch { case item.Error != nil: h.cfg.logger.Error(item.Error, "Failed to update NGINX configuration") nginxReloadRes.Error = item.Error - default: + case gr.Gateway != nil: h.cfg.logger.Info("NGINX configuration was successfully updated") } - - // TODO(sberman): once we support multiple Gateways, we'll have to get - // the correct Graph for the Deployment contained in the update message - gr := h.cfg.processor.GetLatestGraph() gr.LatestReloadResult = nginxReloadRes - h.updateStatuses(ctx, gr) + switch item.UpdateType { + case status.UpdateAll: + h.updateStatuses(ctx, gr) + case status.UpdateGateway: + gwAddresses, err := getGatewayAddresses( + ctx, + h.cfg.k8sClient, + item.GatewayService, + gr.Gateway, + h.cfg.gatewayClassName, + ) + if err != nil { + msg := "error getting Gateway Service IP address" + h.cfg.logger.Error(err, msg) + h.cfg.eventRecorder.Eventf( + item.GatewayService, + v1.EventTypeWarning, + "GetServiceIPFailed", + msg+": %s", + err.Error(), + ) + continue + } + + transitionTime := metav1.Now() + gatewayStatuses := status.PrepareGatewayRequests( + gr.Gateway, + gr.IgnoredGateways, + transitionTime, + gwAddresses, + gr.LatestReloadResult, + ) + h.cfg.statusUpdater.UpdateGroup(ctx, groupGateways, gatewayStatuses...) + default: + panic(fmt.Sprintf("unknown event type %T", item.UpdateType)) + } } } func (h *eventHandlerImpl) updateStatuses(ctx context.Context, gr *graph.Graph) { - gwAddresses, err := getGatewayAddresses(ctx, h.cfg.k8sClient, nil, h.cfg.gatewayPodConfig) + gwAddresses, err := getGatewayAddresses(ctx, h.cfg.k8sClient, nil, gr.Gateway, h.cfg.gatewayClassName) if err != nil { - h.cfg.logger.Error(err, "Setting GatewayStatusAddress to Pod IP Address") + msg := "error getting Gateway Service IP address" + h.cfg.logger.Error(err, msg) + h.cfg.eventRecorder.Eventf( + &v1.Service{}, + v1.EventTypeWarning, + "GetServiceIPFailed", + msg+": %s", + err.Error(), + ) } transitionTime := metav1.Now() @@ -440,27 +499,27 @@ func getGatewayAddresses( ctx context.Context, k8sClient client.Client, svc *v1.Service, - podConfig ngfConfig.GatewayPodConfig, + gateway *graph.Gateway, + gatewayClassName string, ) ([]gatewayv1.GatewayStatusAddress, error) { - podAddress := []gatewayv1.GatewayStatusAddress{ - { - Type: helpers.GetPointer(gatewayv1.IPAddressType), - Value: podConfig.PodIP, - }, + if gateway == nil { + return nil, nil } var gwSvc v1.Service if svc == nil { - key := types.NamespacedName{Name: podConfig.ServiceName, Namespace: podConfig.Namespace} + svcName := controller.CreateNginxResourceName(gateway.Source.GetName(), gatewayClassName) + key := types.NamespacedName{Name: svcName, Namespace: gateway.Source.GetNamespace()} if err := k8sClient.Get(ctx, key, &gwSvc); err != nil { - return podAddress, fmt.Errorf("error finding Service for Gateway: %w", err) + return nil, fmt.Errorf("error finding Service for Gateway: %w", err) } } else { gwSvc = *svc } var addresses, hostnames []string - if gwSvc.Spec.Type == v1.ServiceTypeLoadBalancer { + switch gwSvc.Spec.Type { + case v1.ServiceTypeLoadBalancer: for _, ingress := range gwSvc.Status.LoadBalancer.Ingress { if ingress.IP != "" { addresses = append(addresses, ingress.IP) @@ -468,6 +527,8 @@ func getGatewayAddresses( hostnames = append(hostnames, ingress.Hostname) } } + default: + addresses = append(addresses, gwSvc.Spec.ClusterIP) } gwAddresses := make([]gatewayv1.GatewayStatusAddress, 0, len(addresses)+len(hostnames)) @@ -546,56 +607,3 @@ func (h *eventHandlerImpl) nginxGatewayCRDDelete( ) { h.updateControlPlaneAndSetStatus(ctx, logger, nil) } - -func (h *eventHandlerImpl) nginxGatewayServiceUpsert(ctx context.Context, logger logr.Logger, obj client.Object) { - svc, ok := obj.(*v1.Service) - if !ok { - panic(fmt.Errorf("obj type mismatch: got %T, expected %T", svc, &v1.Service{})) - } - - gwAddresses, err := getGatewayAddresses(ctx, h.cfg.k8sClient, svc, h.cfg.gatewayPodConfig) - if err != nil { - logger.Error(err, "Setting GatewayStatusAddress to Pod IP Address") - } - - gr := h.cfg.processor.GetLatestGraph() - if gr == nil { - return - } - - transitionTime := metav1.Now() - gatewayStatuses := status.PrepareGatewayRequests( - gr.Gateway, - gr.IgnoredGateways, - transitionTime, - gwAddresses, - gr.LatestReloadResult, - ) - h.cfg.statusUpdater.UpdateGroup(ctx, groupGateways, gatewayStatuses...) -} - -func (h *eventHandlerImpl) nginxGatewayServiceDelete( - ctx context.Context, - logger logr.Logger, - _ types.NamespacedName, -) { - gwAddresses, err := getGatewayAddresses(ctx, h.cfg.k8sClient, nil, h.cfg.gatewayPodConfig) - if err != nil { - logger.Error(err, "Setting GatewayStatusAddress to Pod IP Address") - } - - gr := h.cfg.processor.GetLatestGraph() - if gr == nil { - return - } - - transitionTime := metav1.Now() - gatewayStatuses := status.PrepareGatewayRequests( - gr.Gateway, - gr.IgnoredGateways, - transitionTime, - gwAddresses, - gr.LatestReloadResult, - ) - h.cfg.statusUpdater.UpdateGroup(ctx, groupGateways, gatewayStatuses...) -} diff --git a/internal/mode/static/handler_test.go b/internal/mode/static/handler_test.go index dbc4ea9ed6..b479b8b34e 100644 --- a/internal/mode/static/handler_test.go +++ b/internal/mode/static/handler_test.go @@ -29,6 +29,7 @@ import ( "github.com/nginx/nginx-gateway-fabric/internal/mode/static/nginx/agent/agentfakes" agentgrpcfakes "github.com/nginx/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/grpcfakes" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/nginx/config/configfakes" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/provisioner/provisionerfakes" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/dataplane" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/graph" @@ -38,10 +39,12 @@ import ( var _ = Describe("eventHandler", func() { var ( + baseGraph *graph.Graph handler *eventHandlerImpl fakeProcessor *statefakes.FakeChangeProcessor fakeGenerator *configfakes.FakeGenerator fakeNginxUpdater *agentfakes.FakeNginxUpdater + fakeProvisioner *provisionerfakes.FakeProvisioner fakeStatusUpdater *statusfakes.FakeGroupUpdater fakeEventRecorder *record.FakeRecorder fakeK8sClient client.WithWatch @@ -84,18 +87,29 @@ var _ = Describe("eventHandler", func() { _, name, reqs = fakeStatusUpdater.UpdateGroupArgsForCall(1) Expect(name).To(Equal(groupGateways)) - Expect(reqs).To(BeEmpty()) + Expect(reqs).To(HaveLen(1)) + + Expect(fakeProvisioner.RegisterGatewayCallCount()).Should(Equal(1)) } BeforeEach(func() { ctx, cancel = context.WithCancel(context.Background()) //nolint:fatcontext // ignore for test + baseGraph = &graph.Graph{ + Gateway: &graph.Gateway{ + Valid: true, + Source: &gatewayv1.Gateway{}, + }, + } + fakeProcessor = &statefakes.FakeChangeProcessor{} fakeProcessor.ProcessReturns(state.NoChange, &graph.Graph{}) - fakeProcessor.GetLatestGraphReturns(&graph.Graph{}) + fakeProcessor.GetLatestGraphReturns(baseGraph) fakeGenerator = &configfakes.FakeGenerator{} fakeNginxUpdater = &agentfakes.FakeNginxUpdater{} fakeNginxUpdater.UpdateConfigReturns(true) + fakeProvisioner = &provisionerfakes.FakeProvisioner{} + fakeProvisioner.RegisterGatewayReturns(nil) fakeStatusUpdater = &statusfakes.FakeGroupUpdater{} fakeEventRecorder = record.NewFakeRecorder(1) zapLogLevelSetter = newZapLogLevelSetter(zap.NewAtomicLevel()) @@ -112,6 +126,7 @@ var _ = Describe("eventHandler", func() { generator: fakeGenerator, logLevelSetter: zapLogLevelSetter, nginxUpdater: fakeNginxUpdater, + nginxProvisioner: fakeProvisioner, statusUpdater: fakeStatusUpdater, eventRecorder: fakeEventRecorder, deployCtxCollector: &licensingfakes.FakeCollector{}, @@ -157,8 +172,7 @@ var _ = Describe("eventHandler", func() { } BeforeEach(func() { - fakeProcessor.ProcessReturns(state.ClusterStateChange /* changed */, &graph.Graph{}) - + fakeProcessor.ProcessReturns(state.ClusterStateChange, baseGraph) fakeGenerator.GenerateReturns(fakeCfgFiles) }) @@ -195,11 +209,24 @@ var _ = Describe("eventHandler", func() { expectReconfig(dcfg, fakeCfgFiles) Expect(helpers.Diff(handler.GetLatestConfiguration(), &dcfg)).To(BeEmpty()) }) + + It("should not build anything if Gateway isn't set", func() { + fakeProcessor.ProcessReturns(state.ClusterStateChange, &graph.Graph{}) + + e := &events.UpsertEvent{Resource: &gatewayv1.HTTPRoute{}} + batch := []interface{}{e} + + handler.HandleEventBatch(context.Background(), logr.Discard(), batch) + + checkUpsertEventExpectations(e) + Expect(fakeProvisioner.RegisterGatewayCallCount()).Should(Equal(0)) + Expect(fakeGenerator.GenerateCallCount()).Should(Equal(0)) + }) }) When("a batch has multiple events", func() { It("should process events", func() { - upsertEvent := &events.UpsertEvent{Resource: &gatewayv1.HTTPRoute{}} + upsertEvent := &events.UpsertEvent{Resource: &gatewayv1.Gateway{}} deleteEvent := &events.DeleteEvent{ Type: &gatewayv1.HTTPRoute{}, NamespacedName: types.NamespacedName{Namespace: "test", Name: "route"}, @@ -357,77 +384,6 @@ var _ = Describe("eventHandler", func() { }) }) - When("receiving Service updates", func() { - const notNginxGatewayServiceName = "not-nginx-gateway" - - BeforeEach(func() { - fakeProcessor.GetLatestGraphReturns(&graph.Graph{}) - - Expect(fakeK8sClient.Create(context.Background(), createService(notNginxGatewayServiceName))).To(Succeed()) - }) - - It("should not call UpdateAddresses if the Service is not for the Gateway Pod", func() { - e := &events.UpsertEvent{Resource: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "not-nginx-gateway", - }, - }} - batch := []interface{}{e} - - handler.HandleEventBatch(context.Background(), logr.Discard(), batch) - - Expect(fakeStatusUpdater.UpdateGroupCallCount()).To(BeZero()) - - de := &events.DeleteEvent{Type: &v1.Service{}} - batch = []interface{}{de} - Expect(fakeK8sClient.Delete(context.Background(), createService(notNginxGatewayServiceName))).To(Succeed()) - - handler.HandleEventBatch(context.Background(), logr.Discard(), batch) - - Expect(handler.GetLatestConfiguration()).To(BeNil()) - - Expect(fakeStatusUpdater.UpdateGroupCallCount()).To(BeZero()) - }) - - It("should update the addresses when the Gateway Service is upserted", func() { - e := &events.UpsertEvent{Resource: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nginx-gateway", - Namespace: "nginx-gateway", - }, - }} - batch := []interface{}{e} - - handler.HandleEventBatch(context.Background(), logr.Discard(), batch) - - Expect(handler.GetLatestConfiguration()).To(BeNil()) - Expect(fakeStatusUpdater.UpdateGroupCallCount()).To(Equal(1)) - _, name, reqs := fakeStatusUpdater.UpdateGroupArgsForCall(0) - Expect(name).To(Equal(groupGateways)) - Expect(reqs).To(BeEmpty()) - }) - - It("should update the addresses when the Gateway Service is deleted", func() { - e := &events.DeleteEvent{ - Type: &v1.Service{}, - NamespacedName: types.NamespacedName{ - Name: "nginx-gateway", - Namespace: "nginx-gateway", - }, - } - batch := []interface{}{e} - Expect(fakeK8sClient.Delete(context.Background(), createService(nginxGatewayServiceName))).To(Succeed()) - - handler.HandleEventBatch(context.Background(), logr.Discard(), batch) - - Expect(handler.GetLatestConfiguration()).To(BeNil()) - Expect(fakeStatusUpdater.UpdateGroupCallCount()).To(Equal(1)) - _, name, reqs := fakeStatusUpdater.UpdateGroupArgsForCall(0) - Expect(name).To(Equal(groupGateways)) - Expect(reqs).To(BeEmpty()) - }) - }) - When("receiving an EndpointsOnlyChange update", func() { e := &events.UpsertEvent{Resource: &discoveryV1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ @@ -438,7 +394,7 @@ var _ = Describe("eventHandler", func() { batch := []interface{}{e} BeforeEach(func() { - fakeProcessor.ProcessReturns(state.EndpointsOnlyChange, &graph.Graph{}) + fakeProcessor.ProcessReturns(state.EndpointsOnlyChange, &graph.Graph{Gateway: &graph.Gateway{Valid: true}}) }) When("running NGINX Plus", func() { @@ -472,6 +428,7 @@ var _ = Describe("eventHandler", func() { It("should update status when receiving a queue event", func() { obj := &status.QueueObject{ + UpdateType: status.UpdateAll, Deployment: types.NamespacedName{}, Error: errors.New("status error"), } @@ -486,6 +443,20 @@ var _ = Describe("eventHandler", func() { Expect(gr.LatestReloadResult.Error.Error()).To(Equal("status error")) }) + It("should update Gateway status when receiving a queue event", func() { + obj := &status.QueueObject{ + UpdateType: status.UpdateGateway, + Deployment: types.NamespacedName{}, + GatewayService: &v1.Service{}, + } + queue.Enqueue(obj) + + Eventually( + func() int { + return fakeStatusUpdater.UpdateGroupCallCount() + }).Should(Equal(1)) + }) + It("should update nginx conf only when leader", func() { ctx := context.Background() handler.cfg.graphBuiltHealthChecker.leader = false @@ -494,7 +465,7 @@ var _ = Describe("eventHandler", func() { batch := []interface{}{e} readyChannel := handler.cfg.graphBuiltHealthChecker.getReadyCh() - fakeProcessor.ProcessReturns(state.ClusterStateChange, &graph.Graph{}) + fakeProcessor.ProcessReturns(state.ClusterStateChange, &graph.Graph{Gateway: &graph.Gateway{Valid: true}}) handler.HandleEventBatch(context.Background(), logr.Discard(), batch) @@ -535,23 +506,25 @@ var _ = Describe("eventHandler", func() { var _ = Describe("getGatewayAddresses", func() { It("gets gateway addresses from a Service", func() { fakeClient := fake.NewFakeClient() - podConfig := config.GatewayPodConfig{ - PodIP: "1.2.3.4", - ServiceName: "my-service", - Namespace: "nginx-gateway", - } // no Service exists yet, should get error and Pod Address - addrs, err := getGatewayAddresses(context.Background(), fakeClient, nil, podConfig) + gateway := &graph.Gateway{ + Source: &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + Namespace: "test-ns", + }, + }, + } + addrs, err := getGatewayAddresses(context.Background(), fakeClient, nil, gateway, "nginx") Expect(err).To(HaveOccurred()) - Expect(addrs).To(HaveLen(1)) - Expect(addrs[0].Value).To(Equal("1.2.3.4")) + Expect(addrs).To(BeNil()) // Create LoadBalancer Service svc := v1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "my-service", - Namespace: "nginx-gateway", + Name: "gateway-nginx", + Namespace: "test-ns", }, Spec: v1.ServiceSpec{ Type: v1.ServiceTypeLoadBalancer, @@ -572,11 +545,31 @@ var _ = Describe("getGatewayAddresses", func() { Expect(fakeClient.Create(context.Background(), &svc)).To(Succeed()) - addrs, err = getGatewayAddresses(context.Background(), fakeClient, &svc, podConfig) + addrs, err = getGatewayAddresses(context.Background(), fakeClient, &svc, gateway, "nginx") Expect(err).ToNot(HaveOccurred()) Expect(addrs).To(HaveLen(2)) Expect(addrs[0].Value).To(Equal("34.35.36.37")) Expect(addrs[1].Value).To(Equal("myhost")) + + Expect(fakeClient.Delete(context.Background(), &svc)).To(Succeed()) + // Create ClusterIP Service + svc = v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway-nginx", + Namespace: "test-ns", + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + ClusterIP: "12.13.14.15", + }, + } + + Expect(fakeClient.Create(context.Background(), &svc)).To(Succeed()) + + addrs, err = getGatewayAddresses(context.Background(), fakeClient, &svc, gateway, "nginx") + Expect(err).ToNot(HaveOccurred()) + Expect(addrs).To(HaveLen(1)) + Expect(addrs[0].Value).To(Equal("12.13.14.15")) }) }) diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index 31574a9f64..119c49a8c3 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -58,6 +58,7 @@ import ( "github.com/nginx/nginx-gateway-fabric/internal/mode/static/nginx/config/policies/observability" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/nginx/config/policies/upstreamsettings" ngxvalidation "github.com/nginx/nginx-gateway-fabric/internal/mode/static/nginx/config/validation" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/provisioner" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/graph" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/resolver" @@ -122,12 +123,6 @@ func StartManager(cfg config.Config) error { return err } - // protectedPorts is the map of ports that may not be configured by a listener, and the name of what it is used for - protectedPorts := map[int32]string{ - int32(cfg.MetricsConfig.Port): "MetricsPort", //nolint:gosec // port will not overflow int32 - int32(cfg.HealthConfig.Port): "HealthPort", //nolint:gosec // port will not overflow int32 - } - mustExtractGVK := kinds.NewMustExtractGKV(scheme) genericValidator := ngxvalidation.GenericValidator{} @@ -149,7 +144,6 @@ func StartManager(cfg config.Config) error { }, EventRecorder: recorder, MustExtractGVK: mustExtractGVK, - ProtectedPorts: protectedPorts, PlusSecrets: plusSecrets, }) @@ -205,9 +199,31 @@ func StartManager(cfg config.Config) error { return fmt.Errorf("cannot register grpc server: %w", err) } + nginxProvisioner, provLoop, err := provisioner.NewNginxProvisioner( + ctx, + mgr, + provisioner.Config{ + DeploymentStore: nginxUpdater.NginxDeployments, + StatusQueue: statusQueue, + Logger: cfg.Logger.WithName("provisioner"), + EventRecorder: recorder, + GatewayPodConfig: cfg.GatewayPodConfig, + GCName: cfg.GatewayClassName, + Plus: cfg.Plus, + }, + ) + if err != nil { + return fmt.Errorf("error building provisioner: %w", err) + } + + if err := mgr.Add(&runnables.LeaderOrNonLeader{Runnable: provLoop}); err != nil { + return fmt.Errorf("cannot register provisioner event loop: %w", err) + } + eventHandler := newEventHandlerImpl(eventHandlerConfig{ ctx: ctx, nginxUpdater: nginxUpdater, + nginxProvisioner: nginxProvisioner, metricsCollector: handlerCollector, statusUpdater: groupStatusUpdater, processor: processor, @@ -227,6 +243,7 @@ func StartManager(cfg config.Config) error { gatewayPodConfig: cfg.GatewayPodConfig, controlConfigNSName: controlConfigNSName, gatewayCtlrName: cfg.GatewayCtlrName, + gatewayClassName: cfg.GatewayClassName, updateGatewayClassStatus: cfg.UpdateGatewayClassStatus, plus: cfg.Plus, statusQueue: statusQueue, @@ -249,6 +266,7 @@ func StartManager(cfg config.Config) error { if err = mgr.Add(runnables.NewCallFunctionsAfterBecameLeader([]func(context.Context){ groupStatusUpdater.Enable, + nginxProvisioner.Enable, healthChecker.setAsLeader, eventHandler.eventHandlerEnable, })); err != nil { @@ -260,7 +278,7 @@ func StartManager(cfg config.Config) error { K8sClientReader: mgr.GetAPIReader(), GraphGetter: processor, ConfigurationGetter: eventHandler, - Version: cfg.Version, + Version: cfg.GatewayPodConfig.Version, PodNSName: types.NamespacedName{ Namespace: cfg.GatewayPodConfig.Namespace, Name: cfg.GatewayPodConfig.Name, @@ -417,19 +435,6 @@ func registerControllers( controller.WithK8sPredicate(predicate.ServicePortsChangedPredicate{}), }, }, - { - objectType: &apiv1.Service{}, - name: "ngf-service", // unique controller names are needed and we have multiple Service ctlrs - options: func() []controller.Option { - svcNSName := types.NamespacedName{ - Namespace: cfg.GatewayPodConfig.Namespace, - Name: cfg.GatewayPodConfig.ServiceName, - } - return []controller.Option{ - controller.WithK8sPredicate(predicate.GatewayServicePredicate{NSName: svcNSName}), - } - }(), - }, { objectType: &apiv1.Secret{}, options: []controller.Option{ diff --git a/internal/mode/static/nginx/agent/agent.go b/internal/mode/static/nginx/agent/agent.go index 58fad509db..3d2ff78b4a 100644 --- a/internal/mode/static/nginx/agent/agent.go +++ b/internal/mode/static/nginx/agent/agent.go @@ -115,6 +115,8 @@ func (n *NginxUpdaterImpl) UpdateUpstreamServers( // TODO(sberman): optimize this by only sending updates that are necessary. // Call GetUpstreams first (will need Subscribers to send responses back), and // then determine which upstreams actually need to be updated. + // OR we can possibly just use the most recent NGINXPlusActions to see what the last state + // of upstreams were, and only update the diff. var errs []error var applied bool diff --git a/internal/mode/static/nginx/agent/command.go b/internal/mode/static/nginx/agent/command.go index 236a34f57d..31d96143a7 100644 --- a/internal/mode/static/nginx/agent/command.go +++ b/internal/mode/static/nginx/agent/command.go @@ -128,6 +128,7 @@ func (cs *commandService) Subscribe(in pb.CommandService_SubscribeServer) error if !ok { return agentgrpc.ErrStatusInvalidConnection } + defer cs.connTracker.RemoveConnection(gi.IPAddress) // wait for the agent to report itself and nginx conn, deployment, err := cs.waitForConnection(ctx, gi) @@ -135,6 +136,7 @@ func (cs *commandService) Subscribe(in pb.CommandService_SubscribeServer) error cs.logger.Error(err, "error waiting for connection") return err } + defer deployment.RemovePodStatus(conn.PodName) cs.logger.Info(fmt.Sprintf("Successfully connected to nginx agent %s", conn.PodName)) @@ -367,6 +369,7 @@ func (cs *commandService) logAndSendErrorStatus(deployment *Deployment, conn *ag queueObj := &status.QueueObject{ Deployment: conn.Parent, Error: deployment.GetConfigurationStatus(), + UpdateType: status.UpdateAll, } cs.statusQueue.Enqueue(queueObj) } @@ -504,7 +507,6 @@ func getNginxInstanceID(instances []*pb.Instance) string { } // UpdateDataPlaneHealth includes full health information about the data plane as reported by the agent. -// TODO(sberman): Is health monitoring the data planes something useful for us to do? func (cs *commandService) UpdateDataPlaneHealth( _ context.Context, _ *pb.UpdateDataPlaneHealthRequest, diff --git a/internal/mode/static/nginx/agent/command_test.go b/internal/mode/static/nginx/agent/command_test.go index 340c4deda9..714ffaafe5 100644 --- a/internal/mode/static/nginx/agent/command_test.go +++ b/internal/mode/static/nginx/agent/command_test.go @@ -399,11 +399,17 @@ func TestSubscribe(t *testing.T) { ensureAPIRequestWasSent(g, mockServer, loopAction) verifyResponse(g, mockServer, responseCh) + g.Eventually(func() map[string]error { + return deployment.podStatuses + }).Should(HaveKey("nginx-pod")) + cancel() g.Eventually(func() error { return <-errCh }).Should(MatchError(ContainSubstring("context canceled"))) + + g.Expect(deployment.podStatuses).ToNot(HaveKey("nginx-pod")) } func TestSubscribe_Errors(t *testing.T) { diff --git a/internal/mode/static/nginx/agent/deployment.go b/internal/mode/static/nginx/agent/deployment.go index c0bd2bca1d..bafdc6ad9e 100644 --- a/internal/mode/static/nginx/agent/deployment.go +++ b/internal/mode/static/nginx/agent/deployment.go @@ -104,6 +104,14 @@ func (d *Deployment) GetLatestUpstreamError() error { return d.latestUpstreamError } +// RemovePodStatus deletes a pod from the pod status map. +func (d *Deployment) RemovePodStatus(podName string) { + d.Lock.Lock() + defer d.Lock.Unlock() + + delete(d.podStatuses, podName) +} + /* The following functions for the Deployment object are UNLOCKED, meaning that they are unsafe. Callers of these functions MUST ensure the lock is set before calling. @@ -255,9 +263,7 @@ func (d *DeploymentStore) StoreWithBroadcaster( return deployment } -// Remove cleans up any connections that are tracked for this deployment, and then removes -// the deployment from the store. +// Remove the deployment from the store. func (d *DeploymentStore) Remove(nsName types.NamespacedName) { - d.connTracker.UntrackConnectionsForParent(nsName) d.deployments.Delete(nsName) } diff --git a/internal/mode/static/nginx/agent/deployment_test.go b/internal/mode/static/nginx/agent/deployment_test.go index e4881b9934..3c6dc4c859 100644 --- a/internal/mode/static/nginx/agent/deployment_test.go +++ b/internal/mode/static/nginx/agent/deployment_test.go @@ -91,6 +91,9 @@ func TestSetPodErrorStatus(t *testing.T) { g.Expect(deployment.GetConfigurationStatus()).To(MatchError(ContainSubstring("test error"))) g.Expect(deployment.GetConfigurationStatus()).To(MatchError(ContainSubstring("test error 2"))) + + deployment.RemovePodStatus("test-pod") + g.Expect(deployment.podStatuses).ToNot(HaveKey("test-pod")) } func TestSetLatestConfigError(t *testing.T) { diff --git a/internal/mode/static/nginx/agent/grpc/connections.go b/internal/mode/static/nginx/agent/grpc/connections.go index 8f1adc2c75..6b30ce4b59 100644 --- a/internal/mode/static/nginx/agent/grpc/connections.go +++ b/internal/mode/static/nginx/agent/grpc/connections.go @@ -16,7 +16,7 @@ type ConnectionsTracker interface { Track(key string, conn Connection) GetConnection(key string) Connection SetInstanceID(key, id string) - UntrackConnectionsForParent(parent types.NamespacedName) + RemoveConnection(key string) } // Connection contains the data about a single nginx agent connection. @@ -77,14 +77,10 @@ func (c *AgentConnectionsTracker) SetInstanceID(key, id string) { } } -// UntrackConnectionsForParent removes all Connections that reference the specified parent. -func (c *AgentConnectionsTracker) UntrackConnectionsForParent(parent types.NamespacedName) { +// RemoveConnection removes a connection from the tracking map. +func (c *AgentConnectionsTracker) RemoveConnection(key string) { c.lock.Lock() defer c.lock.Unlock() - for key, conn := range c.connections { - if conn.Parent == parent { - delete(c.connections, key) - } - } + delete(c.connections, key) } diff --git a/internal/mode/static/nginx/agent/grpc/connections_test.go b/internal/mode/static/nginx/agent/grpc/connections_test.go index be0ca18a8b..c9d7b3cdc3 100644 --- a/internal/mode/static/nginx/agent/grpc/connections_test.go +++ b/internal/mode/static/nginx/agent/grpc/connections_test.go @@ -75,25 +75,21 @@ func TestSetInstanceID(t *testing.T) { g.Expect(trackedConn.InstanceID).To(Equal("instance1")) } -func TestUntrackConnectionsForParent(t *testing.T) { +func TestRemoveConnection(t *testing.T) { t.Parallel() g := NewWithT(t) tracker := agentgrpc.NewConnectionsTracker() + conn := agentgrpc.Connection{ + PodName: "pod1", + InstanceID: "instance1", + Parent: types.NamespacedName{Namespace: "default", Name: "parent1"}, + } + tracker.Track("key1", conn) - parent1 := types.NamespacedName{Namespace: "default", Name: "parent1"} - conn1 := agentgrpc.Connection{PodName: "pod1", InstanceID: "instance1", Parent: parent1} - conn2 := agentgrpc.Connection{PodName: "pod2", InstanceID: "instance2", Parent: parent1} - - parent2 := types.NamespacedName{Namespace: "default", Name: "parent2"} - conn3 := agentgrpc.Connection{PodName: "pod3", InstanceID: "instance3", Parent: parent2} - - tracker.Track("key1", conn1) - tracker.Track("key2", conn2) - tracker.Track("key3", conn3) + trackedConn := tracker.GetConnection("key1") + g.Expect(trackedConn).To(Equal(conn)) - tracker.UntrackConnectionsForParent(parent1) + tracker.RemoveConnection("key1") g.Expect(tracker.GetConnection("key1")).To(Equal(agentgrpc.Connection{})) - g.Expect(tracker.GetConnection("key2")).To(Equal(agentgrpc.Connection{})) - g.Expect(tracker.GetConnection("key3")).To(Equal(conn3)) } diff --git a/internal/mode/static/nginx/agent/grpc/grpcfakes/fake_connections_tracker.go b/internal/mode/static/nginx/agent/grpc/grpcfakes/fake_connections_tracker.go index a82da0a5a2..8ae97043cd 100644 --- a/internal/mode/static/nginx/agent/grpc/grpcfakes/fake_connections_tracker.go +++ b/internal/mode/static/nginx/agent/grpc/grpcfakes/fake_connections_tracker.go @@ -5,7 +5,6 @@ import ( "sync" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc" - "k8s.io/apimachinery/pkg/types" ) type FakeConnectionsTracker struct { @@ -20,6 +19,11 @@ type FakeConnectionsTracker struct { getConnectionReturnsOnCall map[int]struct { result1 grpc.Connection } + RemoveConnectionStub func(string) + removeConnectionMutex sync.RWMutex + removeConnectionArgsForCall []struct { + arg1 string + } SetInstanceIDStub func(string, string) setInstanceIDMutex sync.RWMutex setInstanceIDArgsForCall []struct { @@ -32,11 +36,6 @@ type FakeConnectionsTracker struct { arg1 string arg2 grpc.Connection } - UntrackConnectionsForParentStub func(types.NamespacedName) - untrackConnectionsForParentMutex sync.RWMutex - untrackConnectionsForParentArgsForCall []struct { - arg1 types.NamespacedName - } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } @@ -102,6 +101,38 @@ func (fake *FakeConnectionsTracker) GetConnectionReturnsOnCall(i int, result1 gr }{result1} } +func (fake *FakeConnectionsTracker) RemoveConnection(arg1 string) { + fake.removeConnectionMutex.Lock() + fake.removeConnectionArgsForCall = append(fake.removeConnectionArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.RemoveConnectionStub + fake.recordInvocation("RemoveConnection", []interface{}{arg1}) + fake.removeConnectionMutex.Unlock() + if stub != nil { + fake.RemoveConnectionStub(arg1) + } +} + +func (fake *FakeConnectionsTracker) RemoveConnectionCallCount() int { + fake.removeConnectionMutex.RLock() + defer fake.removeConnectionMutex.RUnlock() + return len(fake.removeConnectionArgsForCall) +} + +func (fake *FakeConnectionsTracker) RemoveConnectionCalls(stub func(string)) { + fake.removeConnectionMutex.Lock() + defer fake.removeConnectionMutex.Unlock() + fake.RemoveConnectionStub = stub +} + +func (fake *FakeConnectionsTracker) RemoveConnectionArgsForCall(i int) string { + fake.removeConnectionMutex.RLock() + defer fake.removeConnectionMutex.RUnlock() + argsForCall := fake.removeConnectionArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeConnectionsTracker) SetInstanceID(arg1 string, arg2 string) { fake.setInstanceIDMutex.Lock() fake.setInstanceIDArgsForCall = append(fake.setInstanceIDArgsForCall, struct { @@ -168,49 +199,17 @@ func (fake *FakeConnectionsTracker) TrackArgsForCall(i int) (string, grpc.Connec return argsForCall.arg1, argsForCall.arg2 } -func (fake *FakeConnectionsTracker) UntrackConnectionsForParent(arg1 types.NamespacedName) { - fake.untrackConnectionsForParentMutex.Lock() - fake.untrackConnectionsForParentArgsForCall = append(fake.untrackConnectionsForParentArgsForCall, struct { - arg1 types.NamespacedName - }{arg1}) - stub := fake.UntrackConnectionsForParentStub - fake.recordInvocation("UntrackConnectionsForParent", []interface{}{arg1}) - fake.untrackConnectionsForParentMutex.Unlock() - if stub != nil { - fake.UntrackConnectionsForParentStub(arg1) - } -} - -func (fake *FakeConnectionsTracker) UntrackConnectionsForParentCallCount() int { - fake.untrackConnectionsForParentMutex.RLock() - defer fake.untrackConnectionsForParentMutex.RUnlock() - return len(fake.untrackConnectionsForParentArgsForCall) -} - -func (fake *FakeConnectionsTracker) UntrackConnectionsForParentCalls(stub func(types.NamespacedName)) { - fake.untrackConnectionsForParentMutex.Lock() - defer fake.untrackConnectionsForParentMutex.Unlock() - fake.UntrackConnectionsForParentStub = stub -} - -func (fake *FakeConnectionsTracker) UntrackConnectionsForParentArgsForCall(i int) types.NamespacedName { - fake.untrackConnectionsForParentMutex.RLock() - defer fake.untrackConnectionsForParentMutex.RUnlock() - argsForCall := fake.untrackConnectionsForParentArgsForCall[i] - return argsForCall.arg1 -} - func (fake *FakeConnectionsTracker) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() fake.getConnectionMutex.RLock() defer fake.getConnectionMutex.RUnlock() + fake.removeConnectionMutex.RLock() + defer fake.removeConnectionMutex.RUnlock() fake.setInstanceIDMutex.RLock() defer fake.setInstanceIDMutex.RUnlock() fake.trackMutex.RLock() defer fake.trackMutex.RUnlock() - fake.untrackConnectionsForParentMutex.RLock() - defer fake.untrackConnectionsForParentMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/internal/mode/static/provisioner/doc.go b/internal/mode/static/provisioner/doc.go new file mode 100644 index 0000000000..14cffc569b --- /dev/null +++ b/internal/mode/static/provisioner/doc.go @@ -0,0 +1,4 @@ +/* +Package provisioner contains the functions for deploying an instance of nginx. +*/ +package provisioner diff --git a/internal/mode/static/provisioner/eventloop.go b/internal/mode/static/provisioner/eventloop.go new file mode 100644 index 0000000000..c4ccc2b2e1 --- /dev/null +++ b/internal/mode/static/provisioner/eventloop.go @@ -0,0 +1,126 @@ +package provisioner + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/manager" + k8spredicate "sigs.k8s.io/controller-runtime/pkg/predicate" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" + "github.com/nginx/nginx-gateway-fabric/internal/framework/controller/predicate" + "github.com/nginx/nginx-gateway-fabric/internal/framework/events" + ngftypes "github.com/nginx/nginx-gateway-fabric/internal/framework/types" +) + +func newEventLoop( + ctx context.Context, + mgr manager.Manager, + handler *eventHandler, + logger logr.Logger, + selector metav1.LabelSelector, +) (*events.EventLoop, error) { + nginxResourceLabelPredicate := predicate.NginxLabelPredicate(selector) + + controllerRegCfgs := []struct { + objectType ngftypes.ObjectType + options []controller.Option + }{ + { + objectType: &gatewayv1.Gateway{}, + }, + { + objectType: &appsv1.Deployment{}, + options: []controller.Option{ + controller.WithK8sPredicate( + k8spredicate.And( + k8spredicate.GenerationChangedPredicate{}, + nginxResourceLabelPredicate, + predicate.RestartDeploymentAnnotationPredicate{}, + ), + ), + }, + }, + { + objectType: &corev1.Service{}, + options: []controller.Option{ + controller.WithK8sPredicate( + k8spredicate.And( + nginxResourceLabelPredicate, + ), + ), + }, + }, + { + objectType: &corev1.ServiceAccount{}, + options: []controller.Option{ + controller.WithK8sPredicate( + k8spredicate.And( + k8spredicate.GenerationChangedPredicate{}, + nginxResourceLabelPredicate, + ), + ), + }, + }, + { + objectType: &corev1.ConfigMap{}, + options: []controller.Option{ + controller.WithK8sPredicate( + k8spredicate.And( + k8spredicate.GenerationChangedPredicate{}, + nginxResourceLabelPredicate, + ), + ), + }, + }, + } + + eventCh := make(chan interface{}) + for _, regCfg := range controllerRegCfgs { + gvk, err := apiutil.GVKForObject(regCfg.objectType, mgr.GetScheme()) + if err != nil { + panic(fmt.Sprintf("could not extract GVK for object: %T", regCfg.objectType)) + } + + if err := controller.Register( + ctx, + regCfg.objectType, + fmt.Sprintf("provisioner-%s", gvk.Kind), + mgr, + eventCh, + regCfg.options..., + ); err != nil { + return nil, fmt.Errorf("cannot register controller for %T: %w", regCfg.objectType, err) + } + } + + firstBatchPreparer := events.NewFirstEventBatchPreparerImpl( + mgr.GetCache(), + []client.Object{}, + []client.ObjectList{ + // GatewayList MUST be first in this list to ensure that we see it before attempting + // to provision or deprovision any nginx resources. + &gatewayv1.GatewayList{}, + &appsv1.DeploymentList{}, + &corev1.ServiceList{}, + &corev1.ServiceAccountList{}, + &corev1.ConfigMapList{}, + }, + ) + + eventLoop := events.NewEventLoop( + eventCh, + logger.WithName("eventLoop"), + handler, + firstBatchPreparer, + ) + + return eventLoop, nil +} diff --git a/internal/mode/static/provisioner/handler.go b/internal/mode/static/provisioner/handler.go new file mode 100644 index 0000000000..405b670c18 --- /dev/null +++ b/internal/mode/static/provisioner/handler.go @@ -0,0 +1,162 @@ +package provisioner + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" + "github.com/nginx/nginx-gateway-fabric/internal/framework/events" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/status" +) + +// eventHandler ensures each Gateway for the specific GatewayClass has a corresponding Deployment +// of NGF configured to use that specific Gateway. +// +// eventHandler implements events.Handler interface. +type eventHandler struct { + store *store + provisioner *NginxProvisioner + labelSelector labels.Selector + // gcName is the GatewayClass name for this control plane. + gcName string +} + +func newEventHandler( + store *store, + provisioner *NginxProvisioner, + selector metav1.LabelSelector, + gcName string, +) (*eventHandler, error) { + labelSelector, err := metav1.LabelSelectorAsSelector(&selector) + if err != nil { + return nil, fmt.Errorf("error initializing label selector: %w", err) + } + + return &eventHandler{ + store: store, + provisioner: provisioner, + labelSelector: labelSelector, + gcName: gcName, + }, nil +} + +func (h *eventHandler) HandleEventBatch(ctx context.Context, logger logr.Logger, batch events.EventBatch) { + for _, event := range batch { + switch e := event.(type) { + case *events.UpsertEvent: + switch obj := e.Resource.(type) { + case *gatewayv1.Gateway: + h.store.updateGateway(obj) + case *appsv1.Deployment, *corev1.ServiceAccount, *corev1.ConfigMap: + objLabels := labels.Set(obj.GetLabels()) + if h.labelSelector.Matches(objLabels) { + gatewayName := objLabels.Get(controller.GatewayLabel) + gatewayNSName := types.NamespacedName{Namespace: obj.GetNamespace(), Name: gatewayName} + + if err := h.updateOrDeleteResources(ctx, obj, gatewayNSName); err != nil { + logger.Error(err, "error handling resource update") + } + } + case *corev1.Service: + objLabels := labels.Set(obj.GetLabels()) + if h.labelSelector.Matches(objLabels) { + gatewayName := objLabels.Get(controller.GatewayLabel) + gatewayNSName := types.NamespacedName{Namespace: obj.GetNamespace(), Name: gatewayName} + + if err := h.updateOrDeleteResources(ctx, obj, gatewayNSName); err != nil { + logger.Error(err, "error handling resource update") + } + + statusUpdate := &status.QueueObject{ + Deployment: client.ObjectKeyFromObject(obj), + UpdateType: status.UpdateGateway, + GatewayService: obj, + } + h.provisioner.cfg.StatusQueue.Enqueue(statusUpdate) + } + default: + panic(fmt.Errorf("unknown resource type %T", e.Resource)) + } + case *events.DeleteEvent: + switch e.Type.(type) { + case *gatewayv1.Gateway: + if err := h.provisioner.deprovisionNginx(ctx, e.NamespacedName); err != nil { + logger.Error(err, "error deprovisioning nginx resources") + } + h.store.deleteGateway(e.NamespacedName) + case *appsv1.Deployment, *corev1.Service, *corev1.ServiceAccount, *corev1.ConfigMap: + if err := h.reprovisionResources(ctx, e); err != nil { + logger.Error(err, "error re-provisioning nginx resources") + } + default: + panic(fmt.Errorf("unknown resource type %T", e.Type)) + } + default: + panic(fmt.Errorf("unknown event type %T", e)) + } + } +} + +// updateOrDeleteResources ensures that nginx resources are either: +// - deleted if the Gateway no longer exists (this is for when the controller first starts up) +// - are updated to the proper state in case a user makes a change directly to the resource. +func (h *eventHandler) updateOrDeleteResources( + ctx context.Context, + obj client.Object, + gatewayNSName types.NamespacedName, +) error { + if gw := h.store.getGateway(gatewayNSName); gw == nil { + if !h.provisioner.isLeader() { + h.provisioner.setResourceToDelete(gatewayNSName) + + return nil + } + + if err := h.provisioner.deprovisionNginx(ctx, gatewayNSName); err != nil { + return fmt.Errorf("error deprovisioning nginx resources: %w", err) + } + return nil + } + + h.store.registerResourceInGatewayConfig(gatewayNSName, obj) + + resourceName := controller.CreateNginxResourceName(gatewayNSName.Name, h.gcName) + resources := h.store.getNginxResourcesForGateway(gatewayNSName) + if resources.Gateway != nil { + if err := h.provisioner.provisionNginx( + ctx, + resourceName, + resources.Gateway.Source, + resources.Gateway.EffectiveNginxProxy, + ); err != nil { + return fmt.Errorf("error updating nginx resource: %w", err) + } + } + + return nil +} + +// reprovisionResources redeploys nginx resources that have been deleted but should not have been. +func (h *eventHandler) reprovisionResources(ctx context.Context, event *events.DeleteEvent) error { + if gateway := h.store.gatewayExistsForResource(event.Type, event.NamespacedName); gateway != nil && gateway.Valid { + resourceName := controller.CreateNginxResourceName(gateway.Source.GetName(), h.gcName) + if err := h.provisioner.reprovisionNginx( + ctx, + resourceName, + gateway.Source, + gateway.EffectiveNginxProxy, + ); err != nil { + return err + } + } + return nil +} diff --git a/internal/mode/static/provisioner/objects.go b/internal/mode/static/provisioner/objects.go new file mode 100644 index 0000000000..19a24cb832 --- /dev/null +++ b/internal/mode/static/provisioner/objects.go @@ -0,0 +1,524 @@ +package provisioner + +import ( + "fmt" + "maps" + "strconv" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + ngfAPIv1alpha2 "github.com/nginx/nginx-gateway-fabric/apis/v1alpha2" + "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" + "github.com/nginx/nginx-gateway-fabric/internal/framework/helpers" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/config" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/graph" +) + +const ( + defaultNginxErrorLogLevel = "info" + nginxIncludesConfigMapNameSuffix = "includes-bootstrap" + nginxAgentConfigMapNameSuffix = "agent-config" + + defaultServiceType = corev1.ServiceTypeLoadBalancer + defaultServicePolicy = corev1.ServiceExternalTrafficPolicyLocal + + defaultNginxImagePath = "ghcr.io/nginx/nginx-gateway-fabric/nginx" + defaultNginxPlusImagePath = "private-registry.nginx.com/nginx-gateway-fabric/nginx-plus" + defaultImagePullPolicy = corev1.PullIfNotPresent +) + +var emptyDirVolumeSource = corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}} + +func (p *NginxProvisioner) buildNginxResourceObjects( + resourceName string, + gateway *gatewayv1.Gateway, + nProxyCfg *graph.EffectiveNginxProxy, +) []client.Object { + // TODO(sberman): handle nginx plus config + + ngxIncludesConfigMapName := controller.CreateNginxResourceName(resourceName, nginxIncludesConfigMapNameSuffix) + ngxAgentConfigMapName := controller.CreateNginxResourceName(resourceName, nginxAgentConfigMapNameSuffix) + + selectorLabels := make(map[string]string) + maps.Copy(selectorLabels, p.baseLabelSelector.MatchLabels) + selectorLabels[controller.GatewayLabel] = gateway.GetName() + selectorLabels[controller.AppNameLabel] = resourceName + + labels := make(map[string]string) + annotations := make(map[string]string) + + maps.Copy(labels, selectorLabels) + + if gateway.Spec.Infrastructure != nil { + for key, value := range gateway.Spec.Infrastructure.Labels { + labels[string(key)] = string(value) + } + + for key, value := range gateway.Spec.Infrastructure.Annotations { + annotations[string(key)] = string(value) + } + } + + objectMeta := metav1.ObjectMeta{ + Name: resourceName, + Namespace: gateway.GetNamespace(), + Labels: labels, + Annotations: annotations, + } + + configmaps := p.buildNginxConfigMaps( + objectMeta, + nProxyCfg, + ngxIncludesConfigMapName, + ngxAgentConfigMapName, + ) + + serviceAccount := &corev1.ServiceAccount{ + ObjectMeta: objectMeta, + } + + ports := make(map[int32]struct{}) + for _, listener := range gateway.Spec.Listeners { + ports[int32(listener.Port)] = struct{}{} + } + + service := buildNginxService(objectMeta, nProxyCfg, ports, selectorLabels) + deployment := p.buildNginxDeployment( + objectMeta, + nProxyCfg, + ngxIncludesConfigMapName, + ngxAgentConfigMapName, + ports, + selectorLabels, + ) + + // order to install resources: + // scc (if openshift) + // secrets + // configmaps + // serviceaccount + // service + // deployment/daemonset + + objects := make([]client.Object, 0, len(configmaps)+3) + objects = append(objects, configmaps...) + objects = append(objects, serviceAccount, service, deployment) + + return objects +} + +func (p *NginxProvisioner) buildNginxConfigMaps( + objectMeta metav1.ObjectMeta, + nProxyCfg *graph.EffectiveNginxProxy, + ngxIncludesConfigMapName string, + ngxAgentConfigMapName string, +) []client.Object { + var logging *ngfAPIv1alpha2.NginxLogging + if nProxyCfg != nil && nProxyCfg.Logging != nil { + logging = nProxyCfg.Logging + } + + logLevel := defaultNginxErrorLogLevel + if logging != nil && logging.ErrorLevel != nil { + logLevel = string(*nProxyCfg.Logging.ErrorLevel) + } + + mainFields := map[string]interface{}{ + "ErrorLevel": logLevel, + } + + bootstrapCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: ngxIncludesConfigMapName, + Namespace: objectMeta.Namespace, + Labels: objectMeta.Labels, + Annotations: objectMeta.Annotations, + }, + Data: map[string]string{ + "main.conf": string(helpers.MustExecuteTemplate(mainTemplate, mainFields)), + }, + } + + metricsPort := config.DefaultNginxMetricsPort + port, enableMetrics := graph.MetricsEnabledForNginxProxy(nProxyCfg) + if port != nil { + metricsPort = *port + } + + agentFields := map[string]interface{}{ + "Plus": p.cfg.Plus, + "ServiceName": p.cfg.GatewayPodConfig.ServiceName, + "Namespace": p.cfg.GatewayPodConfig.Namespace, + "EnableMetrics": enableMetrics, + "MetricsPort": metricsPort, + } + + if logging != nil && logging.AgentLevel != nil { + agentFields["LogLevel"] = *logging.AgentLevel + } + + agentCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: ngxAgentConfigMapName, + Namespace: objectMeta.Namespace, + Labels: objectMeta.Labels, + Annotations: objectMeta.Annotations, + }, + Data: map[string]string{ + "nginx-agent.conf": string(helpers.MustExecuteTemplate(agentTemplate, agentFields)), + }, + } + + return []client.Object{bootstrapCM, agentCM} +} + +func buildNginxService( + objectMeta metav1.ObjectMeta, + nProxyCfg *graph.EffectiveNginxProxy, + ports map[int32]struct{}, + selectorLabels map[string]string, +) *corev1.Service { + var serviceCfg ngfAPIv1alpha2.ServiceSpec + if nProxyCfg != nil && nProxyCfg.Kubernetes != nil && nProxyCfg.Kubernetes.Service != nil { + serviceCfg = *nProxyCfg.Kubernetes.Service + } + + serviceType := defaultServiceType + if serviceCfg.ServiceType != nil { + serviceType = corev1.ServiceType(*serviceCfg.ServiceType) + } + + servicePolicy := defaultServicePolicy + if serviceCfg.ExternalTrafficPolicy != nil { + servicePolicy = corev1.ServiceExternalTrafficPolicy(*serviceCfg.ExternalTrafficPolicy) + } + + servicePorts := make([]corev1.ServicePort, 0, len(ports)) + for port := range ports { + servicePort := corev1.ServicePort{ + Name: fmt.Sprintf("port-%d", port), + Port: port, + TargetPort: intstr.FromInt32(port), + } + servicePorts = append(servicePorts, servicePort) + } + + svc := &corev1.Service{ + ObjectMeta: objectMeta, + Spec: corev1.ServiceSpec{ + Type: serviceType, + Ports: servicePorts, + ExternalTrafficPolicy: servicePolicy, + Selector: selectorLabels, + }, + } + + if serviceCfg.LoadBalancerIP != nil { + svc.Spec.LoadBalancerIP = *serviceCfg.LoadBalancerIP + } + if serviceCfg.LoadBalancerSourceRanges != nil { + svc.Spec.LoadBalancerSourceRanges = serviceCfg.LoadBalancerSourceRanges + } + + return svc +} + +func (p *NginxProvisioner) buildNginxDeployment( + objectMeta metav1.ObjectMeta, + nProxyCfg *graph.EffectiveNginxProxy, + ngxIncludesConfigMapName string, + ngxAgentConfigMapName string, + ports map[int32]struct{}, + selectorLabels map[string]string, +) client.Object { + podTemplateSpec := p.buildNginxPodTemplateSpec( + objectMeta, + nProxyCfg, + ngxIncludesConfigMapName, + ngxAgentConfigMapName, + ports, + ) + + var object client.Object + // TODO(sberman): daemonset support + deployment := &appsv1.Deployment{ + ObjectMeta: objectMeta, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: selectorLabels, + }, + Template: podTemplateSpec, + }, + } + + var deploymentCfg ngfAPIv1alpha2.DeploymentSpec + if nProxyCfg != nil && nProxyCfg.Kubernetes != nil && nProxyCfg.Kubernetes.Deployment != nil { + deploymentCfg = *nProxyCfg.Kubernetes.Deployment + } + + if deploymentCfg.Replicas != nil { + deployment.Spec.Replicas = deploymentCfg.Replicas + } + + object = deployment + + return object +} + +func (p *NginxProvisioner) buildNginxPodTemplateSpec( + objectMeta metav1.ObjectMeta, + nProxyCfg *graph.EffectiveNginxProxy, + ngxIncludesConfigMapName string, + ngxAgentConfigMapName string, + ports map[int32]struct{}, +) corev1.PodTemplateSpec { + // TODO(sberman): handle nginx plus; debug + + containerPorts := make([]corev1.ContainerPort, 0, len(ports)) + for port := range ports { + containerPort := corev1.ContainerPort{ + Name: fmt.Sprintf("port-%d", port), + ContainerPort: port, + } + containerPorts = append(containerPorts, containerPort) + } + + podAnnotations := make(map[string]string) + maps.Copy(podAnnotations, objectMeta.Annotations) + + metricsPort := config.DefaultNginxMetricsPort + if port, enabled := graph.MetricsEnabledForNginxProxy(nProxyCfg); enabled { + if port != nil { + metricsPort = *port + } + + containerPorts = append(containerPorts, corev1.ContainerPort{ + Name: "metrics", + ContainerPort: metricsPort, + }) + + podAnnotations["prometheus.io/scrape"] = "true" + podAnnotations["prometheus.io/port"] = strconv.Itoa(int(metricsPort)) + } + + image, pullPolicy := p.buildImage(nProxyCfg) + + spec := corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: objectMeta.Labels, + Annotations: podAnnotations, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: image, + ImagePullPolicy: pullPolicy, + Ports: containerPorts, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_BIND_SERVICE"}, + Drop: []corev1.Capability{"ALL"}, + }, + ReadOnlyRootFilesystem: helpers.GetPointer[bool](true), + RunAsGroup: helpers.GetPointer[int64](1001), + RunAsUser: helpers.GetPointer[int64](101), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + {MountPath: "/etc/nginx-agent", Name: "nginx-agent"}, + {MountPath: "/var/log/nginx-agent", Name: "nginx-agent-log"}, + {MountPath: "/etc/nginx/conf.d", Name: "nginx-conf"}, + {MountPath: "/etc/nginx/stream-conf.d", Name: "nginx-stream-conf"}, + {MountPath: "/etc/nginx/main-includes", Name: "nginx-main-includes"}, + {MountPath: "/etc/nginx/secrets", Name: "nginx-secrets"}, + {MountPath: "/var/run/nginx", Name: "nginx-run"}, + {MountPath: "/var/cache/nginx", Name: "nginx-cache"}, + {MountPath: "/etc/nginx/includes", Name: "nginx-includes"}, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: "init", + Image: p.cfg.GatewayPodConfig.Image, + ImagePullPolicy: pullPolicy, + Command: []string{ + "/usr/bin/gateway", + "initialize", + "--source", "/agent/nginx-agent.conf", + "--destination", "/etc/nginx-agent", + "--source", "/includes/main.conf", + "--destination", "/etc/nginx/main-includes", + }, + Env: []corev1.EnvVar{ + { + Name: "POD_UID", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.uid", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + {MountPath: "/agent", Name: "nginx-agent-config"}, + {MountPath: "/etc/nginx-agent", Name: "nginx-agent"}, + {MountPath: "/includes", Name: "nginx-includes-bootstrap"}, + {MountPath: "/etc/nginx/main-includes", Name: "nginx-main-includes"}, + }, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + ReadOnlyRootFilesystem: helpers.GetPointer[bool](true), + RunAsGroup: helpers.GetPointer[int64](1001), + RunAsUser: helpers.GetPointer[int64](101), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + }, + }, + ServiceAccountName: objectMeta.Name, + Volumes: []corev1.Volume{ + {Name: "nginx-agent", VolumeSource: emptyDirVolumeSource}, + { + Name: "nginx-agent-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: ngxAgentConfigMapName, + }, + }, + }, + }, + {Name: "nginx-agent-log", VolumeSource: emptyDirVolumeSource}, + {Name: "nginx-conf", VolumeSource: emptyDirVolumeSource}, + {Name: "nginx-stream-conf", VolumeSource: emptyDirVolumeSource}, + {Name: "nginx-main-includes", VolumeSource: emptyDirVolumeSource}, + {Name: "nginx-secrets", VolumeSource: emptyDirVolumeSource}, + {Name: "nginx-run", VolumeSource: emptyDirVolumeSource}, + {Name: "nginx-cache", VolumeSource: emptyDirVolumeSource}, + {Name: "nginx-includes", VolumeSource: emptyDirVolumeSource}, + { + Name: "nginx-includes-bootstrap", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: ngxIncludesConfigMapName, + }, + }, + }, + }, + }, + }, + } + + if nProxyCfg != nil && nProxyCfg.Kubernetes != nil { + var podSpec *ngfAPIv1alpha2.PodSpec + var containerSpec *ngfAPIv1alpha2.ContainerSpec + if nProxyCfg.Kubernetes.Deployment != nil { + podSpec = &nProxyCfg.Kubernetes.Deployment.Pod + containerSpec = &nProxyCfg.Kubernetes.Deployment.Container + } + + if podSpec != nil { + spec.Spec.TerminationGracePeriodSeconds = podSpec.TerminationGracePeriodSeconds + spec.Spec.Affinity = podSpec.Affinity + spec.Spec.NodeSelector = podSpec.NodeSelector + spec.Spec.Tolerations = podSpec.Tolerations + spec.Spec.Volumes = append(spec.Spec.Volumes, podSpec.Volumes...) + spec.Spec.TopologySpreadConstraints = podSpec.TopologySpreadConstraints + } + + if containerSpec != nil { + container := spec.Spec.Containers[0] + if containerSpec.Resources != nil { + container.Resources = *containerSpec.Resources + } + container.Lifecycle = containerSpec.Lifecycle + container.VolumeMounts = append(container.VolumeMounts, containerSpec.VolumeMounts...) + spec.Spec.Containers[0] = container + } + } + + return spec +} + +func (p *NginxProvisioner) buildImage(nProxyCfg *graph.EffectiveNginxProxy) (string, corev1.PullPolicy) { + image := defaultNginxImagePath + tag := p.cfg.GatewayPodConfig.Version + pullPolicy := defaultImagePullPolicy + + getImageAndPullPolicy := func(container ngfAPIv1alpha2.ContainerSpec) (string, string, corev1.PullPolicy) { + if container.Image != nil { + if container.Image.Repository != nil { + image = *container.Image.Repository + } + if container.Image.Tag != nil { + tag = *container.Image.Tag + } + if container.Image.PullPolicy != nil { + pullPolicy = corev1.PullPolicy(*container.Image.PullPolicy) + } + } + + return image, tag, pullPolicy + } + + if nProxyCfg != nil && nProxyCfg.Kubernetes != nil { + if nProxyCfg.Kubernetes.Deployment != nil { + image, tag, pullPolicy = getImageAndPullPolicy(nProxyCfg.Kubernetes.Deployment.Container) + } + } + + return fmt.Sprintf("%s:%s", image, tag), pullPolicy +} + +func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName types.NamespacedName) []client.Object { + objectMeta := metav1.ObjectMeta{ + Name: deploymentNSName.Name, + Namespace: deploymentNSName.Namespace, + } + + deployment := &appsv1.Deployment{ + ObjectMeta: objectMeta, + } + service := &corev1.Service{ + ObjectMeta: objectMeta, + } + serviceAccount := &corev1.ServiceAccount{ + ObjectMeta: objectMeta, + } + bootstrapCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: controller.CreateNginxResourceName(deploymentNSName.Name, nginxIncludesConfigMapNameSuffix), + Namespace: deploymentNSName.Namespace, + }, + } + agentCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: controller.CreateNginxResourceName(deploymentNSName.Name, nginxAgentConfigMapNameSuffix), + Namespace: deploymentNSName.Namespace, + }, + } + + // order to delete: + // deployment/daemonset + // service + // serviceaccount + // configmaps + // secrets + // scc (if openshift) + + return []client.Object{deployment, service, serviceAccount, bootstrapCM, agentCM} +} diff --git a/internal/mode/static/provisioner/provisioner.go b/internal/mode/static/provisioner/provisioner.go new file mode 100644 index 0000000000..a505cf90ab --- /dev/null +++ b/internal/mode/static/provisioner/provisioner.go @@ -0,0 +1,354 @@ +package provisioner + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/go-logr/logr" + "golang.org/x/text/cases" + "golang.org/x/text/language" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/manager" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" + "github.com/nginx/nginx-gateway-fabric/internal/framework/events" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/config" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/nginx/agent" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/graph" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/status" +) + +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate + +//counterfeiter:generate . Provisioner + +// Provisioner is an interface for triggering NGINX resources to be created/updated/deleted. +type Provisioner interface { + RegisterGateway(ctx context.Context, gateway *graph.Gateway, resourceName string) error +} + +// Config is the configuration for the Provisioner. +type Config struct { + DeploymentStore *agent.DeploymentStore + StatusQueue *status.Queue + Logger logr.Logger + GatewayPodConfig config.GatewayPodConfig + EventRecorder record.EventRecorder + GCName string + Plus bool +} + +// NginxProvisioner handles provisioning nginx kubernetes resources. +type NginxProvisioner struct { + store *store + k8sClient client.Client + // resourcesToDeleteOnStartup contains a list of Gateway names that no longer exist + // but have nginx resources tied to them that need to be deleted. + resourcesToDeleteOnStartup []types.NamespacedName + baseLabelSelector metav1.LabelSelector + cfg Config + leader bool + + lock sync.RWMutex +} + +// NewNginxProvisioner returns a new instance of a Provisioner that will deploy nginx resources. +func NewNginxProvisioner( + ctx context.Context, + mgr manager.Manager, + cfg Config, +) (*NginxProvisioner, *events.EventLoop, error) { + store := newStore() + + selector := metav1.LabelSelector{ + MatchLabels: map[string]string{ + controller.AppInstanceLabel: cfg.GatewayPodConfig.InstanceName, + controller.AppManagedByLabel: controller.CreateNginxResourceName( + cfg.GatewayPodConfig.InstanceName, + cfg.GCName, + ), + }, + } + + provisioner := &NginxProvisioner{ + k8sClient: mgr.GetClient(), + store: store, + baseLabelSelector: selector, + resourcesToDeleteOnStartup: []types.NamespacedName{}, + cfg: cfg, + } + + handler, err := newEventHandler(store, provisioner, selector, cfg.GCName) + if err != nil { + return nil, nil, fmt.Errorf("error initializing eventHandler: %w", err) + } + + eventLoop, err := newEventLoop(ctx, mgr, handler, cfg.Logger, selector) + if err != nil { + return nil, nil, err + } + + return provisioner, eventLoop, nil +} + +// Enable is called when the Pod becomes leader and allows the provisioner to manage resources. +func (p *NginxProvisioner) Enable(ctx context.Context) { + p.lock.Lock() + p.leader = true + p.lock.Unlock() + + p.lock.RLock() + for _, gatewayNSName := range p.resourcesToDeleteOnStartup { + if err := p.deprovisionNginx(ctx, gatewayNSName); err != nil { + p.cfg.Logger.Error(err, "error deprovisioning nginx resources on startup") + } + } + p.lock.RUnlock() + + p.lock.Lock() + p.resourcesToDeleteOnStartup = []types.NamespacedName{} + p.lock.Unlock() +} + +// isLeader returns whether or not this provisioner is the leader. +func (p *NginxProvisioner) isLeader() bool { + p.lock.RLock() + defer p.lock.RUnlock() + + return p.leader +} + +// setResourceToDelete is called when there are resources to delete, but this pod is not leader. +// Once it becomes leader, it will delete those resources. +func (p *NginxProvisioner) setResourceToDelete(gatewayNSName types.NamespacedName) { + p.lock.Lock() + defer p.lock.Unlock() + + p.resourcesToDeleteOnStartup = append(p.resourcesToDeleteOnStartup, gatewayNSName) +} + +//nolint:gocyclo // will refactor at some point +func (p *NginxProvisioner) provisionNginx( + ctx context.Context, + resourceName string, + gateway *gatewayv1.Gateway, + nProxyCfg *graph.EffectiveNginxProxy, +) error { + if !p.isLeader() { + return nil + } + + objects := p.buildNginxResourceObjects(resourceName, gateway, nProxyCfg) + + p.cfg.Logger.Info( + "Creating/Updating nginx resources", + "namespace", gateway.GetNamespace(), + "name", resourceName, + ) + + var agentConfigMapUpdated, deploymentCreated bool + var deploymentObj *appsv1.Deployment + for _, obj := range objects { + createCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + + var res controllerutil.OperationResult + if err := wait.PollUntilContextCancel( + createCtx, + 500*time.Millisecond, + true, /* poll immediately */ + func(ctx context.Context) (bool, error) { + var upsertErr error + res, upsertErr = controllerutil.CreateOrUpdate(ctx, p.k8sClient, obj, objectSpecSetter(obj)) + if upsertErr != nil { + if !apierrors.IsAlreadyExists(upsertErr) && !apierrors.IsConflict(upsertErr) { + return false, upsertErr + } + if apierrors.IsConflict(upsertErr) { + return false, nil + } + } + return true, nil + }, + ); err != nil { + p.cfg.EventRecorder.Eventf( + obj, + corev1.EventTypeWarning, + "CreateOrUpdateFailed", + "Failed to create or update nginx resource: %s", + err.Error(), + ) + cancel() + return err + } + cancel() + + if res != controllerutil.OperationResultCreated && res != controllerutil.OperationResultUpdated { + continue + } + + switch o := obj.(type) { + case *appsv1.Deployment: + deploymentObj = o + if res == controllerutil.OperationResultCreated { + deploymentCreated = true + } + case *corev1.ConfigMap: + if res == controllerutil.OperationResultUpdated && + strings.Contains(obj.GetName(), nginxAgentConfigMapNameSuffix) { + agentConfigMapUpdated = true + } + } + + result := cases.Title(language.English, cases.Compact).String(string(res)) + p.cfg.Logger.V(1).Info( + fmt.Sprintf("%s nginx %s", result, obj.GetObjectKind().GroupVersionKind().Kind), + "namespace", gateway.GetNamespace(), + "name", resourceName, + ) + } + + // if agent configmap was updated, then we'll need to restart the deployment + if agentConfigMapUpdated && !deploymentCreated { + updateCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + p.cfg.Logger.V(1).Info( + "Restarting nginx deployment after agent configmap update", + "name", deploymentObj.GetName(), + "namespace", deploymentObj.GetNamespace(), + ) + + if deploymentObj.Spec.Template.Annotations == nil { + deploymentObj.Annotations = make(map[string]string) + } + deploymentObj.Spec.Template.Annotations[controller.RestartedAnnotation] = time.Now().Format(time.RFC3339) + + if err := p.k8sClient.Update(updateCtx, deploymentObj); err != nil && !apierrors.IsConflict(err) { + p.cfg.EventRecorder.Eventf( + deploymentObj, + corev1.EventTypeWarning, + "RestartFailed", + "Failed to restart nginx deployment after agent config update: %s", + err.Error(), + ) + return err + } + } + + return nil +} + +func (p *NginxProvisioner) reprovisionNginx( + ctx context.Context, + resourceName string, + gateway *gatewayv1.Gateway, + nProxyCfg *graph.EffectiveNginxProxy, +) error { + if !p.isLeader() { + return nil + } + objects := p.buildNginxResourceObjects(resourceName, gateway, nProxyCfg) + + p.cfg.Logger.Info( + "Re-creating nginx resources", + "namespace", gateway.GetNamespace(), + "name", resourceName, + ) + + createCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + for _, obj := range objects { + if err := p.k8sClient.Create(createCtx, obj); err != nil && !apierrors.IsAlreadyExists(err) { + p.cfg.EventRecorder.Eventf( + obj, + corev1.EventTypeWarning, + "CreateFailed", + "Failed to create nginx resource: %s", + err.Error(), + ) + return err + } + } + + return nil +} + +func (p *NginxProvisioner) deprovisionNginx(ctx context.Context, gatewayNSName types.NamespacedName) error { + if !p.isLeader() { + return nil + } + + p.cfg.Logger.Info( + "Removing nginx resources for Gateway", + "name", gatewayNSName.Name, + "namespace", gatewayNSName.Namespace, + ) + + deploymentNSName := types.NamespacedName{ + Name: controller.CreateNginxResourceName(gatewayNSName.Name, p.cfg.GCName), + Namespace: gatewayNSName.Namespace, + } + + objects := p.buildNginxResourceObjectsForDeletion(deploymentNSName) + + createCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + for _, obj := range objects { + if err := p.k8sClient.Delete(createCtx, obj); err != nil && !apierrors.IsNotFound(err) { + p.cfg.EventRecorder.Eventf( + obj, + corev1.EventTypeWarning, + "DeleteFailed", + "Failed to delete nginx resource: %s", + err.Error(), + ) + return err + } + } + + p.store.deleteResourcesForGateway(gatewayNSName) + p.cfg.DeploymentStore.Remove(deploymentNSName) + + return nil +} + +// RegisterGateway is called by the main event handler when a Gateway API resource event occurs +// and the graph is built. The provisioner updates the Gateway config in the store and then: +// - If it's a valid Gateway, create or update nginx resources associated with the Gateway, if necessary. +// - If it's an invalid Gateway, delete the associated nginx resources. +func (p *NginxProvisioner) RegisterGateway( + ctx context.Context, + gateway *graph.Gateway, + resourceName string, +) error { + gatewayNSName := client.ObjectKeyFromObject(gateway.Source) + if updated := p.store.registerResourceInGatewayConfig(gatewayNSName, gateway); !updated { + return nil + } + + if gateway.Valid { + if err := p.provisionNginx(ctx, resourceName, gateway.Source, gateway.EffectiveNginxProxy); err != nil { + return fmt.Errorf("error provisioning nginx resources: %w", err) + } + } else { + if err := p.deprovisionNginx(ctx, gatewayNSName); err != nil { + return fmt.Errorf("error deprovisioning nginx resources: %w", err) + } + } + + return nil +} diff --git a/internal/mode/static/provisioner/provisionerfakes/fake_provisioner.go b/internal/mode/static/provisioner/provisionerfakes/fake_provisioner.go new file mode 100644 index 0000000000..b4359a1ceb --- /dev/null +++ b/internal/mode/static/provisioner/provisionerfakes/fake_provisioner.go @@ -0,0 +1,117 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package provisionerfakes + +import ( + "context" + "sync" + + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/provisioner" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/graph" +) + +type FakeProvisioner struct { + RegisterGatewayStub func(context.Context, *graph.Gateway, string) error + registerGatewayMutex sync.RWMutex + registerGatewayArgsForCall []struct { + arg1 context.Context + arg2 *graph.Gateway + arg3 string + } + registerGatewayReturns struct { + result1 error + } + registerGatewayReturnsOnCall map[int]struct { + result1 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeProvisioner) RegisterGateway(arg1 context.Context, arg2 *graph.Gateway, arg3 string) error { + fake.registerGatewayMutex.Lock() + ret, specificReturn := fake.registerGatewayReturnsOnCall[len(fake.registerGatewayArgsForCall)] + fake.registerGatewayArgsForCall = append(fake.registerGatewayArgsForCall, struct { + arg1 context.Context + arg2 *graph.Gateway + arg3 string + }{arg1, arg2, arg3}) + stub := fake.RegisterGatewayStub + fakeReturns := fake.registerGatewayReturns + fake.recordInvocation("RegisterGateway", []interface{}{arg1, arg2, arg3}) + fake.registerGatewayMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeProvisioner) RegisterGatewayCallCount() int { + fake.registerGatewayMutex.RLock() + defer fake.registerGatewayMutex.RUnlock() + return len(fake.registerGatewayArgsForCall) +} + +func (fake *FakeProvisioner) RegisterGatewayCalls(stub func(context.Context, *graph.Gateway, string) error) { + fake.registerGatewayMutex.Lock() + defer fake.registerGatewayMutex.Unlock() + fake.RegisterGatewayStub = stub +} + +func (fake *FakeProvisioner) RegisterGatewayArgsForCall(i int) (context.Context, *graph.Gateway, string) { + fake.registerGatewayMutex.RLock() + defer fake.registerGatewayMutex.RUnlock() + argsForCall := fake.registerGatewayArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeProvisioner) RegisterGatewayReturns(result1 error) { + fake.registerGatewayMutex.Lock() + defer fake.registerGatewayMutex.Unlock() + fake.RegisterGatewayStub = nil + fake.registerGatewayReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeProvisioner) RegisterGatewayReturnsOnCall(i int, result1 error) { + fake.registerGatewayMutex.Lock() + defer fake.registerGatewayMutex.Unlock() + fake.RegisterGatewayStub = nil + if fake.registerGatewayReturnsOnCall == nil { + fake.registerGatewayReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.registerGatewayReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeProvisioner) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.registerGatewayMutex.RLock() + defer fake.registerGatewayMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeProvisioner) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ provisioner.Provisioner = new(FakeProvisioner) diff --git a/internal/mode/static/provisioner/setter.go b/internal/mode/static/provisioner/setter.go new file mode 100644 index 0000000000..4195fd6d2a --- /dev/null +++ b/internal/mode/static/provisioner/setter.go @@ -0,0 +1,45 @@ +package provisioner + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// objectSpecSetter sets the spec of the provided object. This is used when creating or updating the object. +func objectSpecSetter(object client.Object) controllerutil.MutateFn { + switch obj := object.(type) { + case *appsv1.Deployment: + return deploymentSpecSetter(obj, obj.Spec) + case *corev1.Service: + return serviceSpecSetter(obj, obj.Spec) + case *corev1.ServiceAccount: + return func() error { return nil } + case *corev1.ConfigMap: + return configMapSpecSetter(obj, obj.Data) + } + + return nil +} + +func deploymentSpecSetter(deployment *appsv1.Deployment, spec appsv1.DeploymentSpec) controllerutil.MutateFn { + return func() error { + deployment.Spec = spec + return nil + } +} + +func serviceSpecSetter(service *corev1.Service, spec corev1.ServiceSpec) controllerutil.MutateFn { + return func() error { + service.Spec = spec + return nil + } +} + +func configMapSpecSetter(configMap *corev1.ConfigMap, data map[string]string) controllerutil.MutateFn { + return func() error { + configMap.Data = data + return nil + } +} diff --git a/internal/mode/static/provisioner/store.go b/internal/mode/static/provisioner/store.go new file mode 100644 index 0000000000..bf78ee21c0 --- /dev/null +++ b/internal/mode/static/provisioner/store.go @@ -0,0 +1,196 @@ +package provisioner + +import ( + "reflect" + "strings" + "sync" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/graph" +) + +// NginxResources are all of the NGINX resources deployed in relation to a Gateway. +type NginxResources struct { + Gateway *graph.Gateway + Deployment *appsv1.Deployment + Service *corev1.Service + ServiceAccount *corev1.ServiceAccount + BootstrapConfigMap *corev1.ConfigMap + AgentConfigMap *corev1.ConfigMap +} + +// store stores the cluster state needed by the provisioner and allows to update it from the events. +type store struct { + // gateways is a map of all Gateway resources in the cluster. Used on startup to determine + // which nginx resources aren't tied to any Gateways and need to be cleaned up. + gateways map[types.NamespacedName]*gatewayv1.Gateway + // nginxResources is a map of Gateway NamespacedNames and their associated nginx resources. + nginxResources map[types.NamespacedName]*NginxResources + + lock sync.RWMutex +} + +func newStore() *store { + return &store{ + gateways: make(map[types.NamespacedName]*gatewayv1.Gateway), + nginxResources: make(map[types.NamespacedName]*NginxResources), + } +} + +func (s *store) updateGateway(obj *gatewayv1.Gateway) { + s.lock.Lock() + defer s.lock.Unlock() + + s.gateways[client.ObjectKeyFromObject(obj)] = obj +} + +func (s *store) deleteGateway(nsName types.NamespacedName) { + s.lock.Lock() + defer s.lock.Unlock() + + delete(s.gateways, nsName) +} + +func (s *store) getGateway(nsName types.NamespacedName) *gatewayv1.Gateway { + s.lock.RLock() + defer s.lock.RUnlock() + + return s.gateways[nsName] +} + +// registerResourceInGatewayConfig adds or updates the provided resource in the tracking map. +// If the object being updated is the Gateway, check if anything that we care about changed. This ensures that +// we don't attempt to update nginx resources when the main event handler triggers this call with an unrelated event +// (like a Route update) that shouldn't result in nginx resource changes. +func (s *store) registerResourceInGatewayConfig(gatewayNSName types.NamespacedName, object interface{}) bool { + s.lock.Lock() + defer s.lock.Unlock() + + switch obj := object.(type) { + case *graph.Gateway: + if cfg, ok := s.nginxResources[gatewayNSName]; !ok { + s.nginxResources[gatewayNSName] = &NginxResources{ + Gateway: obj, + } + } else { + changed := gatewayChanged(cfg.Gateway, obj) + cfg.Gateway = obj + return changed + } + case *appsv1.Deployment: + if cfg, ok := s.nginxResources[gatewayNSName]; !ok { + s.nginxResources[gatewayNSName] = &NginxResources{ + Deployment: obj, + } + } else { + cfg.Deployment = obj + } + case *corev1.Service: + if cfg, ok := s.nginxResources[gatewayNSName]; !ok { + s.nginxResources[gatewayNSName] = &NginxResources{ + Service: obj, + } + } else { + cfg.Service = obj + } + case *corev1.ServiceAccount: + if cfg, ok := s.nginxResources[gatewayNSName]; !ok { + s.nginxResources[gatewayNSName] = &NginxResources{ + ServiceAccount: obj, + } + } else { + cfg.ServiceAccount = obj + } + case *corev1.ConfigMap: + if cfg, ok := s.nginxResources[gatewayNSName]; !ok { + if strings.HasSuffix(obj.GetName(), nginxIncludesConfigMapNameSuffix) { + s.nginxResources[gatewayNSName] = &NginxResources{ + BootstrapConfigMap: obj, + } + } else if strings.HasSuffix(obj.GetName(), nginxAgentConfigMapNameSuffix) { + s.nginxResources[gatewayNSName] = &NginxResources{ + AgentConfigMap: obj, + } + } + } else { + if strings.HasSuffix(obj.GetName(), nginxIncludesConfigMapNameSuffix) { + cfg.BootstrapConfigMap = obj + } else if strings.HasSuffix(obj.GetName(), nginxAgentConfigMapNameSuffix) { + cfg.AgentConfigMap = obj + } + } + } + + return true +} + +func gatewayChanged(original, updated *graph.Gateway) bool { + if original == nil { + return true + } + + if original.Valid != updated.Valid { + return true + } + + if !reflect.DeepEqual(original.Source, updated.Source) { + return true + } + + return !reflect.DeepEqual(original.EffectiveNginxProxy, updated.EffectiveNginxProxy) +} + +func (s *store) getNginxResourcesForGateway(nsName types.NamespacedName) *NginxResources { + s.lock.RLock() + defer s.lock.RUnlock() + + return s.nginxResources[nsName] +} + +func (s *store) deleteResourcesForGateway(nsName types.NamespacedName) { + s.lock.Lock() + defer s.lock.Unlock() + + delete(s.nginxResources, nsName) +} + +//nolint:gocyclo // will refactor at some point +func (s *store) gatewayExistsForResource(object client.Object, nsName types.NamespacedName) *graph.Gateway { + s.lock.RLock() + defer s.lock.RUnlock() + + resourceMatches := func(obj client.Object) bool { + return obj.GetName() == nsName.Name && obj.GetNamespace() == nsName.Namespace + } + + for _, resources := range s.nginxResources { + switch object.(type) { + case *appsv1.Deployment: + if resources.Deployment != nil && resourceMatches(resources.Deployment) { + return resources.Gateway + } + case *corev1.Service: + if resources.Service != nil && resourceMatches(resources.Service) { + return resources.Gateway + } + case *corev1.ServiceAccount: + if resources.ServiceAccount != nil && resourceMatches(resources.ServiceAccount) { + return resources.Gateway + } + case *corev1.ConfigMap: + if resources.BootstrapConfigMap != nil && resourceMatches(resources.BootstrapConfigMap) { + return resources.Gateway + } + if resources.AgentConfigMap != nil && resourceMatches(resources.AgentConfigMap) { + return resources.Gateway + } + } + } + + return nil +} diff --git a/internal/mode/static/provisioner/templates.go b/internal/mode/static/provisioner/templates.go new file mode 100644 index 0000000000..0b4d1ca308 --- /dev/null +++ b/internal/mode/static/provisioner/templates.go @@ -0,0 +1,73 @@ +package provisioner + +import gotemplate "text/template" + +var ( + mainTemplate = gotemplate.Must(gotemplate.New("main").Parse(mainTemplateText)) + // mgmtTemplate = gotemplate.Must(gotemplate.New("mgmt").Parse(mgmtTemplateText)). + agentTemplate = gotemplate.Must(gotemplate.New("agent").Parse(agentTemplateText)) +) + +const mainTemplateText = ` +error_log stderr {{ .ErrorLevel }};` + +// const mgmtTemplateText = `mgmt { +// {{- if .Values.nginx.usage.endpoint }} +// usage_report endpoint={{ .Values.nginx.usage.endpoint }}; +// {{- end }} +// {{- if .Values.nginx.usage.skipVerify }} +// ssl_verify off; +// {{- end }} +// {{- if .Values.nginx.usage.caSecretName }} +// ssl_trusted_certificate /etc/nginx/certs-bootstrap/ca.crt; +// {{- end }} +// {{- if .Values.nginx.usage.clientSSLSecretName }} +// ssl_certificate /etc/nginx/certs-bootstrap/tls.crt; +// ssl_certificate_key /etc/nginx/certs-bootstrap/tls.key; +// {{- end }} +// enforce_initial_report off; +// deployment_context /etc/nginx/main-includes/deployment_ctx.json; +// }` + +const agentTemplateText = `command: + server: + host: {{ .ServiceName }}.{{ .Namespace }}.svc + port: 443 +allowed_directories: +- /etc/nginx +- /usr/share/nginx +- /var/run/nginx +features: +- connection +- configuration +- certificates +{{- if .EnableMetrics }} +- metrics +{{- end }} +{{- if eq true .Plus }} +- api-action +{{- end }} +{{- if .LogLevel }} +log: + level: {{ .LogLevel }} +{{- end }} +{{- if .EnableMetrics }} +collector: + receivers: + host_metrics: + collection_interval: 1m0s + initial_delay: 1s + scrapers: + cpu: {} + memory: {} + disk: {} + network: {} + filesystem: {} + processors: + batch: {} + exporters: + prometheus_exporter: + server: + host: "0.0.0.0" + port: {{ .MetricsPort }} +{{- end }}` diff --git a/internal/mode/static/state/change_processor.go b/internal/mode/static/state/change_processor.go index 43ac0e73d7..79cb6109ca 100644 --- a/internal/mode/static/state/change_processor.go +++ b/internal/mode/static/state/change_processor.go @@ -69,8 +69,6 @@ type ChangeProcessorConfig struct { EventRecorder record.EventRecorder // MustExtractGVK is a function that extracts schema.GroupVersionKind from a client.Object. MustExtractGVK kinds.MustExtractGVK - // ProtectedPorts are the ports that may not be configured by a listener with a descriptive name of the ports. - ProtectedPorts graph.ProtectedPorts // PlusSecrets is a list of secret files used for NGINX Plus reporting (JWT, client SSL, CA). PlusSecrets map[types.NamespacedName][]graph.PlusSecretFile // Logger is the logger for this Change Processor. @@ -285,7 +283,6 @@ func (c *ChangeProcessorImpl) Process() (ChangeType, *graph.Graph) { c.cfg.GatewayClassName, c.cfg.PlusSecrets, c.cfg.Validators, - c.cfg.ProtectedPorts, ) return changeType, c.latestGraph diff --git a/internal/mode/static/state/change_processor_test.go b/internal/mode/static/state/change_processor_test.go index b0a1d053a3..26140c88f4 100644 --- a/internal/mode/static/state/change_processor_test.go +++ b/internal/mode/static/state/change_processor_test.go @@ -945,6 +945,10 @@ var _ = Describe("ChangeProcessor", func() { refTLSSvc: {}, refGRPCSvc: {}, }, + DeploymentName: types.NamespacedName{ + Namespace: "test", + Name: "gateway-1-test-class", + }, } }) When("no upsert has occurred", func() { @@ -1501,6 +1505,7 @@ var _ = Describe("ChangeProcessor", func() { ) // gateway 2 takes over; + expGraph.DeploymentName.Name = "gateway-2-test-class" // route 1 has been replaced by route 2 listener80 := getListenerByName(expGraph.Gateway, httpListenerName) listener443 := getListenerByName(expGraph.Gateway, httpsListenerName) @@ -1553,6 +1558,7 @@ var _ = Describe("ChangeProcessor", func() { ) // gateway 2 still in charge; + expGraph.DeploymentName.Name = "gateway-2-test-class" // no HTTP routes remain // GRPCRoute 2 still exists // TLSRoute 2 still exists @@ -1604,6 +1610,7 @@ var _ = Describe("ChangeProcessor", func() { ) // gateway 2 still in charge; + expGraph.DeploymentName.Name = "gateway-2-test-class" // no routes remain listener80 := getListenerByName(expGraph.Gateway, httpListenerName) listener443 := getListenerByName(expGraph.Gateway, httpsListenerName) @@ -1648,6 +1655,7 @@ var _ = Describe("ChangeProcessor", func() { ) // gateway 2 still in charge; + expGraph.DeploymentName.Name = "gateway-2-test-class" // no HTTP or TLS routes remain listener80 := getListenerByName(expGraph.Gateway, httpListenerName) listener443 := getListenerByName(expGraph.Gateway, httpsListenerName) @@ -1692,6 +1700,7 @@ var _ = Describe("ChangeProcessor", func() { Source: gw2, Conditions: staticConds.NewGatewayInvalid("GatewayClass doesn't exist"), } + expGraph.DeploymentName.Name = "gateway-2-test-class" expGraph.Routes = map[graph.RouteKey]*graph.L7Route{} expGraph.L4Routes = map[graph.L4RouteKey]*graph.L4Route{} expGraph.ReferencedSecrets = nil diff --git a/internal/mode/static/state/graph/gateway.go b/internal/mode/static/state/graph/gateway.go index b6cfd49ebe..d41364288d 100644 --- a/internal/mode/static/state/graph/gateway.go +++ b/internal/mode/static/state/graph/gateway.go @@ -10,6 +10,7 @@ import ( "github.com/nginx/nginx-gateway-fabric/internal/framework/conditions" "github.com/nginx/nginx-gateway-fabric/internal/framework/kinds" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/config" ngfsort "github.com/nginx/nginx-gateway-fabric/internal/mode/static/sort" staticConds "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/conditions" ) @@ -104,7 +105,6 @@ func buildGateway( secretResolver *secretResolver, gc *GatewayClass, refGrantResolver *referenceGrantResolver, - protectedPorts ProtectedPorts, nps map[types.NamespacedName]*NginxProxy, ) *Gateway { if gw == nil { @@ -136,6 +136,15 @@ func buildGateway( } } + protectedPorts := make(ProtectedPorts) + if port, enabled := MetricsEnabledForNginxProxy(effectiveNginxProxy); enabled { + metricsPort := config.DefaultNginxMetricsPort + if port != nil { + metricsPort = *port + } + protectedPorts[metricsPort] = "MetricsPort" + } + return &Gateway{ Source: gw, Listeners: buildListeners(gw, secretResolver, refGrantResolver, protectedPorts), diff --git a/internal/mode/static/state/graph/gateway_test.go b/internal/mode/static/state/graph/gateway_test.go index 7d21f77b7c..0e50d14380 100644 --- a/internal/mode/static/state/graph/gateway_test.go +++ b/internal/mode/static/state/graph/gateway_test.go @@ -153,9 +153,6 @@ func TestBuildGateway(t *testing.T) { labelSet := map[string]string{ "key": "value", } - protectedPorts := ProtectedPorts{ - 9113: "MetricsPort", - } listenerAllowedRoutes := v1.Listener{ Name: "listener-with-allowed-routes", Hostname: helpers.GetPointer[v1.Hostname]("foo.example.com"), @@ -1304,7 +1301,7 @@ func TestBuildGateway(t *testing.T) { t.Run(test.name, func(t *testing.T) { g := NewWithT(t) resolver := newReferenceGrantResolver(test.refGrants) - result := buildGateway(test.gateway, secretResolver, test.gatewayClass, resolver, protectedPorts, nginxProxies) + result := buildGateway(test.gateway, secretResolver, test.gatewayClass, resolver, nginxProxies) g.Expect(helpers.Diff(test.expected, result)).To(BeEmpty()) }) } diff --git a/internal/mode/static/state/graph/graph.go b/internal/mode/static/state/graph/graph.go index e43ed94958..9178bf10a1 100644 --- a/internal/mode/static/state/graph/graph.go +++ b/internal/mode/static/state/graph/graph.go @@ -16,6 +16,7 @@ import ( ngfAPIv1alpha1 "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1" ngfAPIv1alpha2 "github.com/nginx/nginx-gateway-fabric/apis/v1alpha2" + "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" "github.com/nginx/nginx-gateway-fabric/internal/framework/controller/index" "github.com/nginx/nginx-gateway-fabric/internal/framework/kinds" ngftypes "github.com/nginx/nginx-gateway-fabric/internal/framework/types" @@ -84,8 +85,10 @@ type Graph struct { SnippetsFilters map[types.NamespacedName]*SnippetsFilter // PlusSecrets holds the secrets related to NGINX Plus licensing. PlusSecrets map[types.NamespacedName][]PlusSecretFile - + // LatestReloadResult is the latest result of applying config to nginx for this Gateway. LatestReloadResult NginxReloadResult + // DeploymentName is the name of the nginx Deployment for this Gateway. + DeploymentName types.NamespacedName } // NginxReloadResult describes the result of an NGINX reload. @@ -208,7 +211,6 @@ func BuildGraph( gcName string, plusSecrets map[types.NamespacedName][]PlusSecretFile, validators validation.Validators, - protectedPorts ProtectedPorts, ) *Graph { processedGwClasses, gcExists := processGatewayClasses(state.GatewayClasses, gcName, controllerName) if gcExists && processedGwClasses.Winner == nil { @@ -240,7 +242,6 @@ func BuildGraph( secretResolver, gc, refGrantResolver, - protectedPorts, processedNginxProxies, ) @@ -306,9 +307,18 @@ func BuildGraph( setPlusSecretContent(state.Secrets, plusSecrets) + var deploymentName types.NamespacedName + if gw != nil { + deploymentName = types.NamespacedName{ + Namespace: gw.Source.Namespace, + Name: controller.CreateNginxResourceName(gw.Source.Name, gcName), + } + } + g := &Graph{ GatewayClass: gc, Gateway: gw, + DeploymentName: deploymentName, Routes: routes, L4Routes: l4routes, IgnoredGatewayClasses: processedGwClasses.Ignored, diff --git a/internal/mode/static/state/graph/graph_test.go b/internal/mode/static/state/graph/graph_test.go index 0e52442b07..f59d091a18 100644 --- a/internal/mode/static/state/graph/graph_test.go +++ b/internal/mode/static/state/graph/graph_test.go @@ -36,11 +36,6 @@ func TestBuildGraph(t *testing.T) { controllerName = "my.controller" ) - protectedPorts := ProtectedPorts{ - 9113: "MetricsPort", - 8081: "HealthPort", - } - cm := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "configmap", @@ -1002,6 +997,10 @@ func TestBuildGraph(t *testing.T) { }, }, }, + DeploymentName: types.NamespacedName{ + Namespace: "test", + Name: "gateway-1-my-class", + }, } } @@ -1071,7 +1070,6 @@ func TestBuildGraph(t *testing.T) { GenericValidator: &validationfakes.FakeGenericValidator{}, PolicyValidator: fakePolicyValidator, }, - protectedPorts, ) g.Expect(helpers.Diff(test.expected, result)).To(BeEmpty()) diff --git a/internal/mode/static/state/graph/nginxproxy.go b/internal/mode/static/state/graph/nginxproxy.go index 3b013b8f39..e9993ab73f 100644 --- a/internal/mode/static/state/graph/nginxproxy.go +++ b/internal/mode/static/state/graph/nginxproxy.go @@ -110,6 +110,19 @@ func telemetryEnabledForNginxProxy(np *EffectiveNginxProxy) bool { return true } +// MetricsEnabledForNginxProxy returns whether metrics is enabled, and the associated port if specified. +// By default, metrics are enabled. +func MetricsEnabledForNginxProxy(np *EffectiveNginxProxy) (*int32, bool) { + if np != nil && np.Metrics != nil { + if np.Metrics.Disable != nil && *np.Metrics.Disable { + return nil, false + } + return np.Metrics.Port, true + } + + return nil, true +} + func processNginxProxies( nps map[types.NamespacedName]*ngfAPIv1alpha2.NginxProxy, validator validation.GenericValidator, diff --git a/internal/mode/static/state/graph/nginxproxy_test.go b/internal/mode/static/state/graph/nginxproxy_test.go index 80c7ef6401..2289bb7dab 100644 --- a/internal/mode/static/state/graph/nginxproxy_test.go +++ b/internal/mode/static/state/graph/nginxproxy_test.go @@ -373,6 +373,83 @@ func TestTelemetryEnabledForNginxProxy(t *testing.T) { } } +func TestMetricsEnabledForNginxProxy(t *testing.T) { + t.Parallel() + + tests := []struct { + ep *EffectiveNginxProxy + port *int32 + name string + enabled bool + }{ + { + name: "NginxProxy is nil", + port: nil, + enabled: true, + }, + { + name: "metrics struct is nil", + ep: &EffectiveNginxProxy{ + Metrics: nil, + }, + port: nil, + enabled: true, + }, + { + name: "metrics disable is nil", + ep: &EffectiveNginxProxy{ + Metrics: &ngfAPIv1alpha2.Metrics{ + Disable: nil, + }, + }, + port: nil, + enabled: true, + }, + { + name: "metrics is disabled", + ep: &EffectiveNginxProxy{ + Metrics: &ngfAPIv1alpha2.Metrics{ + Disable: helpers.GetPointer(true), + }, + }, + port: nil, + enabled: false, + }, + { + name: "metrics is enabled with no port specified", + ep: &EffectiveNginxProxy{ + Metrics: &ngfAPIv1alpha2.Metrics{ + Disable: helpers.GetPointer(false), + }, + }, + port: nil, + enabled: true, + }, + { + name: "metrics is enabled with port specified", + ep: &EffectiveNginxProxy{ + Metrics: &ngfAPIv1alpha2.Metrics{ + Disable: helpers.GetPointer(false), + Port: helpers.GetPointer[int32](8080), + }, + }, + port: helpers.GetPointer[int32](8080), + enabled: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + port, enabled := MetricsEnabledForNginxProxy(test.ep) + g.Expect(port).To(Equal(test.port)) + g.Expect(enabled).To(Equal(test.enabled)) + }) + } +} + func TestProcessNginxProxies(t *testing.T) { t.Parallel() diff --git a/internal/mode/static/status/queue.go b/internal/mode/static/status/queue.go index 5f31bbec6d..991718648b 100644 --- a/internal/mode/static/status/queue.go +++ b/internal/mode/static/status/queue.go @@ -4,13 +4,28 @@ import ( "context" "sync" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" ) +// UpdateType is the type of status update to perform. +type UpdateType int + +const ( + // UpdateAll means to update statuses of all Gateway API resources. + UpdateAll = iota + // UpdateGateway means to just update the status of the Gateway resource. + UpdateGateway +) + // QueueObject is the object to be passed to the queue for status updates. type QueueObject struct { - Error error - Deployment types.NamespacedName + // GatewayService is the Gateway Service that was updated. When set, UpdateType should be UpdateGateway. + // Set by the provisioner + GatewayService *corev1.Service + Error error + Deployment types.NamespacedName + UpdateType UpdateType } // Queue represents a queue with unlimited size. diff --git a/internal/mode/static/status/queue_test.go b/internal/mode/static/status/queue_test.go index 0bed3cee62..8ed8bbb5ab 100644 --- a/internal/mode/static/status/queue_test.go +++ b/internal/mode/static/status/queue_test.go @@ -26,6 +26,7 @@ func TestEnqueue(t *testing.T) { item := &QueueObject{ Error: nil, Deployment: types.NamespacedName{Namespace: "default", Name: "test-object"}, + UpdateType: UpdateAll, } q.Enqueue(item) @@ -41,6 +42,7 @@ func TestDequeue(t *testing.T) { item := &QueueObject{ Error: nil, Deployment: types.NamespacedName{Namespace: "default", Name: "test-object"}, + UpdateType: UpdateAll, } q.Enqueue(item) @@ -73,10 +75,12 @@ func TestDequeueWithMultipleItems(t *testing.T) { item1 := &QueueObject{ Error: nil, Deployment: types.NamespacedName{Namespace: "default", Name: "test-object-1"}, + UpdateType: UpdateAll, } item2 := &QueueObject{ Error: nil, Deployment: types.NamespacedName{Namespace: "default", Name: "test-object-2"}, + UpdateType: UpdateAll, } q.Enqueue(item1) q.Enqueue(item2) diff --git a/scripts/generate-manifests.sh b/scripts/generate-manifests.sh index 731b359272..f52743e382 100755 --- a/scripts/generate-manifests.sh +++ b/scripts/generate-manifests.sh @@ -33,7 +33,7 @@ generate_manifests openshift # FIXME(lucacome): Implement a better way to generate the static deployment file # https://github.com/nginx/nginx-gateway-fabric/issues/2326 -helm template nginx-gateway charts/nginx-gateway-fabric --set nameOverride=nginx-gateway --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml >config/tests/static-deployment.yaml +helm template nginx-gateway charts/nginx-gateway-fabric --set nameOverride=nginx-gateway --set nginxGateway.metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml >config/tests/static-deployment.yaml sed -i.bak '/app.kubernetes.io\/managed-by: Helm/d' config/tests/static-deployment.yaml sed -i.bak '/helm.sh/d' config/tests/static-deployment.yaml rm -f config/tests/static-deployment.yaml.bak diff --git a/tests/Makefile b/tests/Makefile index c95f337d19..a7e8e5717f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -14,7 +14,7 @@ NGF_VERSION ?= edge## NGF version to be tested PULL_POLICY = Never## Pull policy for the images NGINX_CONF_DIR = internal/mode/static/nginx/conf PROVISIONER_MANIFEST = conformance/provisioner/provisioner.yaml -SUPPORTED_EXTENDED_FEATURES = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,HTTPRouteResponseHeaderModification,HTTPRoutePathRedirect,GatewayHTTPListenerIsolation +SUPPORTED_EXTENDED_FEATURES = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,HTTPRouteResponseHeaderModification,HTTPRoutePathRedirect,GatewayHTTPListenerIsolation,GatewayInfrastructurePropagation STANDARD_CONFORMANCE_PROFILES = GATEWAY-HTTP,GATEWAY-GRPC EXPERIMENTAL_CONFORMANCE_PROFILES = GATEWAY-TLS CONFORMANCE_PROFILES = $(STANDARD_CONFORMANCE_PROFILES) # by default we use the standard conformance profiles. If experimental is enabled we override this and add the experimental profiles. @@ -169,7 +169,7 @@ delete-gke-cluster: ## Delete the GKE cluster add-local-ip-to-cluster: ## Add local IP to the GKE cluster master-authorized-networks ./scripts/add-local-ip-auth-networks.sh -HELM_PARAMETERS += --set nameOverride=nginx-gateway --set nginxGateway.kind=skip --set service.create=false --skip-schema-validation +HELM_PARAMETERS += --set nameOverride=nginx-gateway --set nginxGateway.kind=skip --set nginx.service.type=ClusterIP --skip-schema-validation .PHONY: deploy-updated-provisioner deploy-updated-provisioner: ## Update provisioner manifest and deploy to the configured kind cluster diff --git a/tests/framework/ngf.go b/tests/framework/ngf.go index 45a1eeecc9..48cd1b9ed7 100644 --- a/tests/framework/ngf.go +++ b/tests/framework/ngf.go @@ -230,7 +230,7 @@ func setImageArgs(cfg InstallationConfig) []string { } if cfg.ServiceType != "" { - args = append(args, formatValueSet("service.type", cfg.ServiceType)...) + args = append(args, formatValueSet("nginx.service.type", cfg.ServiceType)...) if cfg.ServiceType == "LoadBalancer" && cfg.IsGKEInternalLB { args = append( args,