CP/DP Split: Remove NGINX manager and deployment (#2936)

Removing the nginx runtime manager and deployment container since nginx will live in its own pod managed by agent. Temporarily saving the nginx deployment and service for future use.

Updated the control plane liveness probe to return true once it's processed all resources, instead of after it's written config to nginx (since nginx may not be started yet in the future architecture).

Author: sjberman

build/Dockerfile

@@ -11,36 +11,20 @@ RUN make build
 
 FROM golang:1.23 AS ca-certs-provider
 
-FROM alpine:3.20 AS capabilizer
-RUN apk add --no-cache libcap
-
-FROM capabilizer AS local-capabilizer
-COPY ./build/out/gateway /usr/bin/
-RUN setcap 'cap_kill=+ep' /usr/bin/gateway
-
-FROM capabilizer AS container-capabilizer
-COPY --from=builder /go/src/github.com/nginxinc/nginx-gateway-fabric/build/out/gateway /usr/bin/
-RUN setcap 'cap_kill=+ep' /usr/bin/gateway
-
-FROM capabilizer AS goreleaser-capabilizer
-ARG TARGETARCH
-COPY dist/gateway_linux_$TARGETARCH*/gateway /usr/bin/
-RUN setcap 'cap_kill=+ep' /usr/bin/gateway
-
 FROM scratch AS common
-# CA certs are needed for telemetry report and NGINX Plus usage report features, so that
-# NGF can verify the server's certificate.
+# CA certs are needed for telemetry report so that NGF can verify the server's certificate.
 COPY --from=ca-certs-provider --link /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 USER 102:1001
 ARG BUILD_AGENT
 ENV BUILD_AGENT=${BUILD_AGENT}
 ENTRYPOINT [ "/usr/bin/gateway" ]
 
 FROM common AS container
-COPY --from=container-capabilizer /usr/bin/gateway /usr/bin/
+COPY --from=builder /go/src/github.com/nginxinc/nginx-gateway-fabric/build/out/gateway /usr/bin/
 
 FROM common AS local
-COPY --from=local-capabilizer /usr/bin/gateway /usr/bin/
+COPY ./build/out/gateway /usr/bin/
 
 FROM common AS goreleaser
-COPY --from=goreleaser-capabilizer /usr/bin/gateway /usr/bin/
+ARG TARGETARCH
+COPY dist/gateway_linux_$TARGETARCH*/gateway /usr/bin/

charts/nginx-gateway-fabric/README.md

@@ -268,6 +268,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `nginx.image.tag` |  | string | `"edge"` |
 | `nginx.lifecycle` | The lifecycle of the nginx container. | object | `{}` |
 | `nginx.plus` | Is NGINX Plus image being used | bool | `false` |
+| `nginx.securityContext.allowPrivilegeEscalation` | Some environments may need this set to true in order for the control plane to successfully reload NGINX. | bool | `false` |
 | `nginx.usage.caSecretName` | The name of the Secret containing the NGINX Instance Manager CA certificate. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
 | `nginx.usage.clientSSLSecretName` | The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
 | `nginx.usage.endpoint` | The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com | string | `""` |
@@ -295,7 +296,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `nginxGateway.readinessProbe.port` | Port in which the readiness endpoint is exposed. | int | `8081` |
 | `nginxGateway.replicaCount` | The number of replicas of the NGINX Gateway Fabric Deployment. | int | `1` |
 | `nginxGateway.resources` | The resource requests and/or limits of the nginx-gateway container. | object | `{}` |
-| `nginxGateway.securityContext.allowPrivilegeEscalation` | Some environments may need this set to true in order for the control plane to successfully reload NGINX. | bool | `false` |
+| `nginxGateway.service.annotations` | The annotations of the NGINX Gateway Fabric control plane service. | object | `{}` |
 | `nginxGateway.snippetsFilters.enable` | Enable SnippetsFilters feature. SnippetsFilters allow inserting NGINX configuration into the generated NGINX config for HTTPRoute and GRPCRoute resources. | bool | `false` |
 | `nodeSelector` | The nodeSelector of the NGINX Gateway Fabric pod. | object | `{}` |
 | `service.annotations` | The annotations of the NGINX Gateway Fabric service. | object | `{}` |

charts/nginx-gateway-fabric/templates/deployment.yaml

@@ -33,43 +33,6 @@ spec:
       topologySpreadConstraints:
         {{- toYaml .Values.topologySpreadConstraints | nindent 8 }}
       {{- end }}
-      initContainers:
-      - name: init
-        image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }}
-        imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }}
-        command:
-        - /usr/bin/gateway
-        - initialize
-        - --source
-        - /includes/main.conf
-        {{- if .Values.nginx.plus }}
-        - --source
-        - /includes/mgmt.conf
-        - --nginx-plus
-        {{- end }}
-        - --destination
-        - /etc/nginx/main-includes
-        env:
-          - name: POD_UID
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.uid
-        securityContext:
-          seccompProfile:
-            type: RuntimeDefault
-          capabilities:
-            add:
-            - KILL # Set because the binary has CAP_KILL for the main controller process. Not used by init.
-            drop:
-            - ALL
-          readOnlyRootFilesystem: true
-          runAsUser: 102
-          runAsGroup: 1001
-        volumeMounts:
-        - name: nginx-includes-bootstrap
-          mountPath: /includes
-        - name: nginx-main-includes
-          mountPath: /etc/nginx/main-includes
       containers:
       - args:
         - static-mode
@@ -171,99 +134,21 @@ spec:
         securityContext:
           seccompProfile:
             type: RuntimeDefault
-          allowPrivilegeEscalation: {{ .Values.nginxGateway.securityContext.allowPrivilegeEscalation }}
           capabilities:
-            add:
-            - KILL
             drop:
             - ALL
           readOnlyRootFilesystem: true
           runAsUser: 102
           runAsGroup: 1001
-        volumeMounts:
-        - name: nginx-conf
-          mountPath: /etc/nginx/conf.d
-        - name: nginx-stream-conf
-          mountPath: /etc/nginx/stream-conf.d
-        - name: nginx-main-includes
-          mountPath: /etc/nginx/main-includes
-        - name: nginx-secrets
-          mountPath: /etc/nginx/secrets
-        - name: nginx-run
-          mountPath: /var/run/nginx
-        - name: nginx-includes
-          mountPath: /etc/nginx/includes
         {{- with .Values.nginxGateway.extraVolumeMounts -}}
         {{ toYaml . | nindent 8 }}
         {{- end }}
-      - image: {{ .Values.nginx.image.repository }}:{{ .Values.nginx.image.tag | default .Chart.AppVersion }}
-        imagePullPolicy: {{ .Values.nginx.image.pullPolicy }}
-        name: nginx
-        {{- if .Values.nginx.lifecycle }}
-        lifecycle:
-        {{- toYaml .Values.nginx.lifecycle | nindent 10 }}
-        {{- end }}
-        ports:
-        - containerPort: 80
-          name: http
-        - containerPort: 443
-          name: https
-        securityContext:
-          seccompProfile:
-            type: RuntimeDefault
-          capabilities:
-            add:
-            - NET_BIND_SERVICE
-            drop:
-            - ALL
-          readOnlyRootFilesystem: true
-          runAsUser: 101
-          runAsGroup: 1001
-        volumeMounts:
-        - name: nginx-conf
-          mountPath: /etc/nginx/conf.d
-        - name: nginx-stream-conf
-          mountPath: /etc/nginx/stream-conf.d
-        - name: nginx-main-includes
-          mountPath: /etc/nginx/main-includes
-        - name: nginx-secrets
-          mountPath: /etc/nginx/secrets
-        - name: nginx-run
-          mountPath: /var/run/nginx
-        - name: nginx-cache
-          mountPath: /var/cache/nginx
-        - name: nginx-includes
-          mountPath: /etc/nginx/includes
-        {{- if .Values.nginx.plus }}
-        - name: nginx-lib
-          mountPath: /var/lib/nginx/state
-          {{- if .Values.nginx.usage.secretName }}
-        - name: nginx-plus-license
-          mountPath: /etc/nginx/license.jwt
-          subPath: license.jwt
-          {{- end }}
-          {{- if or .Values.nginx.usage.caSecretName .Values.nginx.usage.clientSSLSecretName }}
-        - name: nginx-plus-usage-certs
-          mountPath: /etc/nginx/certs-bootstrap/
-          {{- end }}
-        {{- end }}
-        {{- with .Values.nginx.extraVolumeMounts -}}
-        {{ toYaml . | nindent 8 }}
-        {{- end }}
-        {{- if .Values.nginx.debug }}
-        command:
-          - "/bin/sh"
-        args:
-          - "-c"
-          - "rm -rf /var/run/nginx/*.sock && nginx-debug -g 'daemon off;'"
-        {{- end }}
       terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
       {{- if .Values.affinity }}
       affinity:
       {{- toYaml .Values.affinity | nindent 8 }}
       {{- end }}
       serviceAccountName: {{ include "nginx-gateway.serviceAccountName" . }}
-      shareProcessNamespace: true
       securityContext:
         fsGroup: 1001
         runAsNonRoot: true
@@ -275,46 +160,6 @@ spec:
       nodeSelector:
       {{- toYaml .Values.nodeSelector | nindent 8 }}
       {{- end }}
-      volumes:
-      - name: nginx-conf
-        emptyDir: {}
-      - name: nginx-stream-conf
-        emptyDir: {}
-      - name: nginx-main-includes
-        emptyDir: {}
-      - name: nginx-secrets
-        emptyDir: {}
-      - name: nginx-run
-        emptyDir: {}
-      - name: nginx-cache
-        emptyDir: {}
-      - name: nginx-includes
-        emptyDir: {}
-      - name: nginx-includes-bootstrap
-        configMap:
-          name: nginx-includes-bootstrap
-      {{- if .Values.nginx.plus }}
-      - name: nginx-lib
-        emptyDir: {}
-        {{- if .Values.nginx.usage.secretName }}
-      - name: nginx-plus-license
-        secret:
-          secretName: {{ .Values.nginx.usage.secretName }}
-        {{- end }}
-        {{- if or .Values.nginx.usage.caSecretName .Values.nginx.usage.clientSSLSecretName }}
-      - name: nginx-plus-usage-certs
-        projected:
-          sources:
-          {{- if .Values.nginx.usage.caSecretName }}
-          - secret:
-              name: {{ .Values.nginx.usage.caSecretName }}
-          {{- end }}
-          {{- if .Values.nginx.usage.clientSSLSecretName }}
-          - secret:
-              name: {{ .Values.nginx.usage.clientSSLSecretName }}
-          {{- end }}
-        {{- end }}
-      {{- end }}
       {{- with .Values.extraVolumes -}}
       {{ toYaml . | nindent 6 }}
       {{- end }}

charts/nginx-gateway-fabric/templates/scc.yaml

@@ -3,7 +3,7 @@ kind: SecurityContextConstraints
 apiVersion: security.openshift.io/v1
 metadata:
   name: {{ include "nginx-gateway.scc-name" . }}
-allowPrivilegeEscalation: {{ .Values.nginxGateway.securityContext.allowPrivilegeEscalation }}
+allowPrivilegeEscalation: {{ .Values.nginx.securityContext.allowPrivilegeEscalation }}
 allowHostDirVolumePlugin: false
 allowHostIPC: false
 allowHostNetwork: false

charts/nginx-gateway-fabric/templates/service.yaml

@@ -1,35 +1,20 @@
-{{- if .Values.service.create }}
 apiVersion: v1
 kind: Service
 metadata:
   name: {{ include "nginx-gateway.fullname" . }}
   namespace: {{ .Release.Namespace }}
   labels:
     {{- include "nginx-gateway.labels" . | nindent 4 }}
-{{- if .Values.service.annotations }}
+{{- if .Values.nginxGateway.service.annotations }}
   annotations:
-{{ toYaml .Values.service.annotations | indent 4 }}
+{{ toYaml .Values.nginxGateway.service.annotations | indent 4 }}
 {{- end }}
 spec:
-{{- if or (eq .Values.service.type "LoadBalancer") (eq .Values.service.type "NodePort") }}
-  {{- if .Values.service.externalTrafficPolicy }}
-  externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }}
-  {{- end }}
-{{- end }}
-  type: {{ .Values.service.type }}
-{{- if eq .Values.service.type "LoadBalancer" }}
-  {{- if .Values.service.loadBalancerIP }}
-  loadBalancerIP: {{ .Values.service.loadBalancerIP }}
-  {{- end }}
-  {{- if .Values.service.loadBalancerSourceRanges }}
-  loadBalancerSourceRanges:
-  {{ toYaml .Values.service.loadBalancerSourceRanges | nindent 2 }}
-  {{- end }}
-{{- end}}
+  type: ClusterIP
   selector:
     {{- include "nginx-gateway.selectorLabels" . | nindent 4 }}
-  ports: # Update the following ports to match your Gateway Listener ports
-{{- if .Values.service.ports }}
-{{ toYaml .Values.service.ports | indent 2 }}
-{{ end }}
-{{- end }}
+  ports:
+  - name: grpc
+    port: 443
+    protocol: TCP
+    targetPort: 443