Implement ClientSettingsPolicy (#1940)

Problems:
- As a Cluster Operator, I want to set defaults for client settings 
that will work for most applications so that most Application Developers 
will not have to tweak these settings.
- As an Application Developer, I want to be able to configure 
client settings for my application based on its behavior or requirements.
- As an Application Developer, I want to override the defaults 
for client settings set by the Cluster Operator because the defaults 
do not satisfy my application's requirements or behavior.

Solution: Implement ClientSettingsPolicy API.
- Cluster operators can create a ClientSettingsPolicy for a Gateway 
to set defaults for client settings that apply to all routes attached to that Gateway.
- App devs can create ClientSettingsPolicies for their routes and specify 
client settings that override the defaults set by the cluster operator.

Author: kate-osborn

apis/v1alpha1/clientsettingspolicy_types.go

@@ -48,8 +48,11 @@ type ClientSettingsPolicySpec struct {
 
 	// TargetRef identifies an API object to apply the policy to.
 	// Object must be in the same namespace as the policy.
+	// Support: Gateway, HTTPRoute, GRPCRoute.
 	//
-	// Support: Gateway, HTTPRoute
+	// +kubebuilder:validation:XValidation:message="TargetRef Kind must be one of: Gateway, HTTPRoute, or GRPCRoute",rule="(self.kind=='Gateway' || self.kind=='HTTPRoute' || self.kind=='GRPCRoute')"
+	// +kubebuilder:validation:XValidation:message="TargetRef Group must be gateway.networking.k8s.io.",rule="(self.group=='gateway.networking.k8s.io')"
+	//nolint:lll
 	TargetRef gatewayv1alpha2.LocalPolicyTargetReference `json:"targetRef"`
 }
 
@@ -95,7 +98,11 @@ type ClientKeepAlive struct {
 
 	// Timeout defines the keep-alive timeouts for clients.
 	//
+	// +kubebuilder:validation:XValidation:message="header can only be specified if server is specified",rule="!(has(self.header) && !has(self.server))"
+	//
+	//
 	// +optional
+	//nolint:lll
 	Timeout *ClientKeepAliveTimeout `json:"timeout,omitempty"`
 }
 

apis/v1alpha1/policy_methods.go

@@ -0,0 +1,21 @@
+package v1alpha1
+
+import (
+	"sigs.k8s.io/gateway-api/apis/v1alpha2"
+)
+
+// FIXME(kate-osborn): https://github.com/nginxinc/nginx-gateway-fabric/issues/1939.
+// Figure out a way to generate these methods for all our policies.
+// These methods implement the policies.Policy interface which extends client.Object to add the following methods.
+
+func (p *ClientSettingsPolicy) GetTargetRef() v1alpha2.LocalPolicyTargetReference {
+	return p.Spec.TargetRef
+}
+
+func (p *ClientSettingsPolicy) GetPolicyStatus() v1alpha2.PolicyStatus {
+	return p.Status
+}
+
+func (p *ClientSettingsPolicy) SetPolicyStatus(status v1alpha2.PolicyStatus) {
+	p.Status = status
+}

charts/nginx-gateway-fabric/templates/deployment.yaml

@@ -124,6 +124,8 @@ spec:
           mountPath: /etc/nginx/secrets
         - name: nginx-run
           mountPath: /var/run/nginx
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
         {{- with .Values.nginxGateway.extraVolumeMounts -}}
         {{ toYaml . | nindent 8 }}
         {{- end }}
@@ -161,6 +163,8 @@ spec:
           mountPath: /var/cache/nginx
         - name: nginx-lib
           mountPath: /var/lib/nginx
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
         {{- with .Values.nginx.extraVolumeMounts -}}
         {{ toYaml . | nindent 8 }}
         {{- end }}
@@ -195,6 +199,8 @@ spec:
         emptyDir: {}
       - name: nginx-lib
         emptyDir: {}
+      - name: nginx-includes
+        emptyDir: {}
       {{- with .Values.extraVolumes -}}
       {{ toYaml . | nindent 6 }}
       {{- end }}

charts/nginx-gateway-fabric/templates/rbac.yaml

@@ -121,13 +121,15 @@ rules:
   - gateway.nginx.org
   resources:
   - nginxproxies
+  - clientsettingspolicies
   verbs:
   - list
   - watch
 - apiGroups:
   - gateway.nginx.org
   resources:
   - nginxgateways/status
+  - clientsettingspolicies/status
   verbs:
   - update
 {{- if .Values.nginxGateway.leaderElection.enable }}

config/crd/bases/gateway.nginx.org_clientsettingspolicies.yaml

@@ -108,14 +108,15 @@ spec:
                         pattern: ^\d{1,4}(ms|s)?$
                         type: string
                     type: object
+                    x-kubernetes-validations:
+                    - message: header can only be specified if server is specified
+                      rule: '!(has(self.header) && !has(self.server))'
                 type: object
               targetRef:
                 description: |-
                   TargetRef identifies an API object to apply the policy to.
                   Object must be in the same namespace as the policy.
-
-
-                  Support: Gateway, HTTPRoute
+                  Support: Gateway, HTTPRoute, GRPCRoute.
                 properties:
                   group:
                     description: Group is the group of the target resource.
@@ -138,6 +139,12 @@ spec:
                 - kind
                 - name
                 type: object
+                x-kubernetes-validations:
+                - message: 'TargetRef Kind must be one of: Gateway, HTTPRoute, or
+                    GRPCRoute'
+                  rule: (self.kind=='Gateway' || self.kind=='HTTPRoute' || self.kind=='GRPCRoute')
+                - message: TargetRef Group must be gateway.networking.k8s.io.
+                  rule: (self.group=='gateway.networking.k8s.io')
             required:
             - targetRef
             type: object

conformance/provisioner/static-deployment.yaml

@@ -76,6 +76,8 @@ spec:
           mountPath: /etc/nginx/secrets
         - name: nginx-run
           mountPath: /var/run/nginx
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
       - image: ghcr.io/nginxinc/nginx-gateway-fabric/nginx:edge
         imagePullPolicy: Always
         name: nginx
@@ -106,6 +108,8 @@ spec:
           mountPath: /var/cache/nginx
         - name: nginx-lib
           mountPath: /var/lib/nginx
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
       terminationGracePeriodSeconds: 30
       serviceAccountName: nginx-gateway
       shareProcessNamespace: true
@@ -125,3 +129,5 @@ spec:
         emptyDir: {}
       - name: nginx-lib
         emptyDir: {}
+      - name: nginx-includes
+        emptyDir: {}

deploy/crds.yaml

@@ -107,14 +107,15 @@ spec:
                         pattern: ^\d{1,4}(ms|s)?$
                         type: string
                     type: object
+                    x-kubernetes-validations:
+                    - message: header can only be specified if server is specified
+                      rule: '!(has(self.header) && !has(self.server))'
                 type: object
               targetRef:
                 description: |-
                   TargetRef identifies an API object to apply the policy to.
                   Object must be in the same namespace as the policy.
-
-
-                  Support: Gateway, HTTPRoute
+                  Support: Gateway, HTTPRoute, GRPCRoute.
                 properties:
                   group:
                     description: Group is the group of the target resource.
@@ -137,6 +138,12 @@ spec:
                 - kind
                 - name
                 type: object
+                x-kubernetes-validations:
+                - message: 'TargetRef Kind must be one of: Gateway, HTTPRoute, or
+                    GRPCRoute'
+                  rule: (self.kind=='Gateway' || self.kind=='HTTPRoute' || self.kind=='GRPCRoute')
+                - message: TargetRef Group must be gateway.networking.k8s.io.
+                  rule: (self.group=='gateway.networking.k8s.io')
             required:
             - targetRef
             type: object

deploy/manifests/nginx-gateway-experimental.yaml

@@ -103,13 +103,15 @@ rules:
   - gateway.nginx.org
   resources:
   - nginxproxies
+  - clientsettingspolicies
   verbs:
   - list
   - watch
 - apiGroups:
   - gateway.nginx.org
   resources:
   - nginxgateways/status
+  - clientsettingspolicies/status
   verbs:
   - update
 - apiGroups:
@@ -228,6 +230,8 @@ spec:
           mountPath: /etc/nginx/secrets
         - name: nginx-run
           mountPath: /var/run/nginx
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
       - image: ghcr.io/nginxinc/nginx-gateway-fabric/nginx:edge
         imagePullPolicy: Always
         name: nginx
@@ -258,6 +262,8 @@ spec:
           mountPath: /var/cache/nginx
         - name: nginx-lib
           mountPath: /var/lib/nginx
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
       terminationGracePeriodSeconds: 30
       serviceAccountName: nginx-gateway
       shareProcessNamespace: true
@@ -277,6 +283,8 @@ spec:
         emptyDir: {}
       - name: nginx-lib
         emptyDir: {}
+      - name: nginx-includes
+        emptyDir: {}
 ---
 # Source: nginx-gateway-fabric/templates/gatewayclass.yaml
 apiVersion: gateway.networking.k8s.io/v1

deploy/manifests/nginx-gateway.yaml

@@ -100,13 +100,15 @@ rules:
   - gateway.nginx.org
   resources:
   - nginxproxies
+  - clientsettingspolicies
   verbs:
   - list
   - watch
 - apiGroups:
   - gateway.nginx.org
   resources:
   - nginxgateways/status
+  - clientsettingspolicies/status
   verbs:
   - update
 - apiGroups:
@@ -224,6 +226,8 @@ spec:
           mountPath: /etc/nginx/secrets
         - name: nginx-run
           mountPath: /var/run/nginx
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
       - image: ghcr.io/nginxinc/nginx-gateway-fabric/nginx:edge
         imagePullPolicy: Always
         name: nginx
@@ -254,6 +258,8 @@ spec:
           mountPath: /var/cache/nginx
         - name: nginx-lib
           mountPath: /var/lib/nginx
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
       terminationGracePeriodSeconds: 30
       serviceAccountName: nginx-gateway
       shareProcessNamespace: true
@@ -273,6 +279,8 @@ spec:
         emptyDir: {}
       - name: nginx-lib
         emptyDir: {}
+      - name: nginx-includes
+        emptyDir: {}
 ---
 # Source: nginx-gateway-fabric/templates/gatewayclass.yaml
 apiVersion: gateway.networking.k8s.io/v1

deploy/manifests/nginx-plus-gateway-experimental.yaml

@@ -109,13 +109,15 @@ rules:
   - gateway.nginx.org
   resources:
   - nginxproxies
+  - clientsettingspolicies
   verbs:
   - list
   - watch
 - apiGroups:
   - gateway.nginx.org
   resources:
   - nginxgateways/status
+  - clientsettingspolicies/status
   verbs:
   - update
 - apiGroups:
@@ -235,6 +237,8 @@ spec:
           mountPath: /etc/nginx/secrets
         - name: nginx-run
           mountPath: /var/run/nginx
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
       - image: nginx-gateway-fabric/nginx-plus:edge
         imagePullPolicy: Always
         name: nginx
@@ -265,6 +269,8 @@ spec:
           mountPath: /var/cache/nginx
         - name: nginx-lib
           mountPath: /var/lib/nginx
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
       terminationGracePeriodSeconds: 30
       serviceAccountName: nginx-gateway
       shareProcessNamespace: true
@@ -284,6 +290,8 @@ spec:
         emptyDir: {}
       - name: nginx-lib
         emptyDir: {}
+      - name: nginx-includes
+        emptyDir: {}
 ---
 # Source: nginx-gateway-fabric/templates/gatewayclass.yaml
 apiVersion: gateway.networking.k8s.io/v1