Skip to content

Commit c0c2d9c

Browse files
sjbermansjberman
committed
CP/DP Split: Add agent/nginx container and deployment (#2958)
Updating the nginx docker containers to build and include agent. Once agent is officially released, we can use the published binary instead of building. Added a temporary nginx deployment to the helm chart to deploy a standalone nginx pod. Added the basic gRPC server and agent API implementation to allow for the agent pod to connect to the control plane without errors.
1 parent b18a342 commit c0c2d9c

File tree

31 files changed

+2327
-259
lines changed

31 files changed

+2327
-259
lines changed

‎.yamllint.yaml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,7 @@
22
ignore:
33
- charts/nginx-gateway-fabric/templates
44
- config/crd/bases/
5-
- deploy/crds.yaml
6-
- deploy/*nginx-plus
5+
- deploy
76
- site/static
87

98
rules:

‎build/Dockerfile.nginx‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,41 @@
11
# syntax=docker/dockerfile:1.13
2+
# TODO(sberman): the commented out lines are for when we use the published agent release
3+
# FROM scratch AS nginx-files
4+
5+
# # the following links can be replaced with local files if needed, i.e. ADD --chown=101:1001 <local_file> <container_file>
6+
# ADD --link --chown=101:1001 https://cs.nginx.com/static/keys/nginx_signing.rsa.pub nginx_signing.rsa.pub
7+
8+
FROM golang:alpine AS builder
9+
10+
WORKDIR /tmp
11+
12+
RUN apk add --no-cache git make \
13+
&& git clone https://github.com/nginx/agent.git \
14+
&& cd agent \
15+
&& git checkout v3 \
16+
&& make build
17+
218
FROM nginx:1.27.4-alpine-otel
319

420
ARG NJS_DIR
521
ARG NGINX_CONF_DIR
622
ARG BUILD_AGENT
723

8-
RUN apk add --no-cache libcap \
24+
# RUN --mount=type=bind,from=nginx-files,src=nginx_signing.rsa.pub,target=/etc/apk/keys/nginx_signing.rsa.pub \
25+
# printf "%s\n" "http://packages.nginx.org/nginx-agent/alpine/v$(egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
26+
# && apk add --no-cache nginx-agent
27+
28+
RUN apk add --no-cache libcap bash \
929
&& mkdir -p /usr/lib/nginx/modules \
10-
&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
11-
&& setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \
30+
&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
31+
&& setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \
1232
&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug \
1333
&& setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug \
1434
&& apk del libcap
1535

36+
COPY --from=builder /tmp/agent/build/nginx-agent /usr/bin/nginx-agent
37+
38+
COPY build/entrypoint.sh /agent/entrypoint.sh
1639
COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js
1740
COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf
1841
COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
@@ -24,4 +47,4 @@ LABEL org.nginx.ngf.image.build.agent="${BUILD_AGENT}"
2447

2548
USER 101:1001
2649

27-
CMD ["sh", "-c", "rm -rf /var/run/nginx/*.sock && nginx -g 'daemon off;'"]
50+
ENTRYPOINT ["/agent/entrypoint.sh"]

‎build/Dockerfile.nginxplus‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,15 @@ FROM scratch AS nginx-files
44
# the following links can be replaced with local files if needed, i.e. ADD --chown=101:1001 <local_file> <container_file>
55
ADD --link --chown=101:1001 https://cs.nginx.com/static/keys/nginx_signing.rsa.pub nginx_signing.rsa.pub
66

7+
FROM golang:alpine AS builder
8+
9+
WORKDIR /tmp
10+
11+
RUN apk add --no-cache git make \
12+
&& git clone https://github.com/nginx/agent.git \
13+
&& cd agent \
14+
&& git checkout v3 \
15+
&& make build
716

817
FROM alpine:3.21
918

@@ -18,7 +27,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \
1827
addgroup -g 1001 -S nginx \
1928
&& adduser -S -D -H -u 101 -h /var/cache/nginx -s /sbin/nologin -G nginx -g nginx nginx \
2029
&& printf "%s\n" "https://pkgs.nginx.com/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
21-
&& apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-otel libcap \
30+
&& apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-otel libcap bash \
2231
&& mkdir -p /usr/lib/nginx/modules \
2332
&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
2433
&& setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \
@@ -29,6 +38,9 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \
2938
&& ln -sf /dev/stdout /var/log/nginx/access.log \
3039
&& ln -sf /dev/stderr /var/log/nginx/error.log
3140

41+
COPY --from=builder /tmp/agent/build/nginx-agent /usr/bin/nginx-agent
42+
43+
COPY build/entrypoint.sh /agent/entrypoint.sh
3244
COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js
3345
COPY ${NGINX_CONF_DIR}/nginx-plus.conf /etc/nginx/nginx.conf
3446
COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
@@ -40,4 +52,4 @@ USER 101:1001
4052

4153
LABEL org.nginx.ngf.image.build.agent="${BUILD_AGENT}"
4254

43-
CMD ["sh", "-c", "rm -rf /var/run/nginx/*.sock && nginx -g 'daemon off;'"]
55+
ENTRYPOINT ["/agent/entrypoint.sh"]

‎build/entrypoint.sh‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/bin/bash
2+
3+
set -euxo pipefail
4+
5+
handle_term() {
6+
echo "received TERM signal"
7+
echo "stopping nginx-agent ..."
8+
kill -TERM "${agent_pid}" 2>/dev/null
9+
echo "stopping nginx ..."
10+
kill -TERM "${nginx_pid}" 2>/dev/null
11+
}
12+
13+
trap 'handle_term' TERM
14+
15+
rm -rf /var/run/nginx/*.sock
16+
17+
# Launch nginx
18+
echo "starting nginx ..."
19+
/usr/sbin/nginx -g "daemon off;" &
20+
21+
nginx_pid=$!
22+
23+
SECONDS=0
24+
25+
while ! ps -ef | grep "nginx: master process" | grep -v grep; do
26+
if ((SECONDS > 5)); then
27+
echo "couldn't find nginx master process"
28+
exit 1
29+
fi
30+
done
31+
32+
# start nginx-agent, pass args
33+
echo "starting nginx-agent ..."
34+
nginx-agent "$@" &
35+
36+
agent_pid=$!
37+
38+
if [ $? != 0 ]; then
39+
echo "couldn't start the agent, please check the log file"
40+
exit 1
41+
fi
42+
43+
wait_term() {
44+
wait ${agent_pid}
45+
trap - TERM
46+
kill -QUIT "${nginx_pid}" 2>/dev/null
47+
echo "waiting for nginx to stop..."
48+
wait ${nginx_pid}
49+
}
50+
51+
wait_term
52+
53+
echo "nginx-agent process has stopped, exiting."

‎charts/nginx-gateway-fabric/templates/deployment.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ spec:
117117
{{- toYaml .Values.nginxGateway.resources | nindent 10 }}
118118
{{- end }}
119119
ports:
120+
- name: agent-grpc
121+
containerPort: 8443
120122
{{- if .Values.metrics.enable }}
121123
- name: metrics
122124
containerPort: {{ .Values.metrics.port }}

‎charts/nginx-gateway-fabric/templates/service.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ spec:
1414
selector:
1515
{{- include "nginx-gateway.selectorLabels" . | nindent 4 }}
1616
ports:
17-
- name: grpc
17+
- name: agent-grpc
1818
port: 443
1919
protocol: TCP
20-
targetPort: 443
20+
targetPort: 8443

‎charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: nginx-agent-config
5+
namespace: {{ .Release.Namespace }}
6+
data:
7+
nginx-agent.conf: |-
8+
command:
9+
server:
10+
host: {{ include "nginx-gateway.fullname" . }}.{{ .Release.Namespace }}.svc
11+
port: 443
12+
allowed_directories:
13+
- /etc/nginx
14+
- /usr/share/nginx
15+
- /var/run/nginx
16+
features:
17+
- connection
18+
log:
19+
level: debug

‎charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: tmp-nginx-deployment
5+
namespace: {{ .Release.Namespace }}
6+
spec:
7+
selector:
8+
matchLabels:
9+
app.kubernetes.io/name: tmp-nginx-deployment
10+
app.kubernetes.io/instance: {{ .Release.Name }}
11+
template:
12+
metadata:
13+
labels:
14+
app.kubernetes.io/name: tmp-nginx-deployment
15+
app.kubernetes.io/instance: {{ .Release.Name }}
16+
spec:
17+
initContainers:
18+
- name: sleep # wait for a bit for control plane to be ready
19+
image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }}
20+
imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }}
21+
command:
22+
- /usr/bin/gateway
23+
- sleep
24+
- --duration=15s
25+
- name: init
26+
image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }}
27+
imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }}
28+
command:
29+
- /usr/bin/gateway
30+
- initialize
31+
- --source
32+
- /includes/main.conf
33+
{{- if .Values.nginx.plus }}
34+
- --source
35+
- /includes/mgmt.conf
36+
- --nginx-plus
37+
{{- end }}
38+
- --destination
39+
- /etc/nginx/main-includes
40+
env:
41+
- name: POD_UID
42+
valueFrom:
43+
fieldRef:
44+
fieldPath: metadata.uid
45+
securityContext:
46+
seccompProfile:
47+
type: RuntimeDefault
48+
capabilities:
49+
drop:
50+
- ALL
51+
readOnlyRootFilesystem: true
52+
runAsUser: 102
53+
runAsGroup: 1001
54+
volumeMounts:
55+
- name: nginx-includes-bootstrap
56+
mountPath: /includes
57+
- name: nginx-main-includes
58+
mountPath: /etc/nginx/main-includes
59+
containers:
60+
- image: {{ .Values.nginx.image.repository }}:{{ .Values.nginx.image.tag | default .Chart.AppVersion }}
61+
imagePullPolicy: {{ .Values.nginx.image.pullPolicy }}
62+
name: nginx
63+
{{- if .Values.nginx.lifecycle }}
64+
lifecycle:
65+
{{- toYaml .Values.nginx.lifecycle | nindent 10 }}
66+
{{- end }}
67+
ports:
68+
- containerPort: 80
69+
name: http
70+
- containerPort: 443
71+
name: https
72+
securityContext:
73+
seccompProfile:
74+
type: RuntimeDefault
75+
allowPrivilegeEscalation: {{ .Values.nginx.securityContext.allowPrivilegeEscalation }}
76+
capabilities:
77+
add:
78+
- NET_BIND_SERVICE
79+
drop:
80+
- ALL
81+
readOnlyRootFilesystem: true
82+
runAsUser: 101
83+
runAsGroup: 1001
84+
volumeMounts:
85+
- name: nginx-agent
86+
mountPath: /etc/nginx-agent
87+
- name: nginx-conf
88+
mountPath: /etc/nginx/conf.d
89+
- name: nginx-stream-conf
90+
mountPath: /etc/nginx/stream-conf.d
91+
- name: nginx-main-includes
92+
mountPath: /etc/nginx/main-includes
93+
- name: nginx-secrets
94+
mountPath: /etc/nginx/secrets
95+
- name: nginx-run
96+
mountPath: /var/run/nginx
97+
- name: nginx-cache
98+
mountPath: /var/cache/nginx
99+
- name: nginx-includes
100+
mountPath: /etc/nginx/includes
101+
{{- if .Values.nginx.plus }}
102+
- name: nginx-lib
103+
mountPath: /var/lib/nginx/state
104+
{{- if .Values.nginx.usage.secretName }}
105+
- name: nginx-plus-license
106+
mountPath: /etc/nginx/license.jwt
107+
subPath: license.jwt
108+
{{- end }}
109+
{{- if or .Values.nginx.usage.caSecretName .Values.nginx.usage.clientSSLSecretName }}
110+
- name: nginx-plus-usage-certs
111+
mountPath: /etc/nginx/certs-bootstrap/
112+
{{- end }}
113+
{{- end }}
114+
{{- with .Values.nginx.extraVolumeMounts -}}
115+
{{ toYaml . | nindent 8 }}
116+
{{- end }}
117+
{{- if .Values.nginx.debug }}
118+
command:
119+
- "/bin/sh"
120+
args:
121+
- "-c"
122+
- "rm -rf /var/run/nginx/*.sock && nginx-debug -g 'daemon off;'"
123+
{{- end }}
124+
terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
125+
{{- if .Values.affinity }}
126+
affinity:
127+
{{- toYaml .Values.affinity | nindent 8 }}
128+
{{- end }}
129+
serviceAccountName: {{ include "nginx-gateway.serviceAccountName" . }}
130+
securityContext:
131+
fsGroup: 1001
132+
runAsNonRoot: true
133+
{{- if .Values.tolerations }}
134+
tolerations:
135+
{{- toYaml .Values.tolerations | nindent 6 }}
136+
{{- end }}
137+
{{- if .Values.nodeSelector }}
138+
nodeSelector:
139+
{{- toYaml .Values.nodeSelector | nindent 8 }}
140+
{{- end }}
141+
volumes:
142+
- name: nginx-agent
143+
configMap:
144+
name: nginx-agent-config
145+
- name: nginx-conf
146+
emptyDir: {}
147+
- name: nginx-stream-conf
148+
emptyDir: {}
149+
- name: nginx-main-includes
150+
emptyDir: {}
151+
- name: nginx-secrets
152+
emptyDir: {}
153+
- name: nginx-run
154+
emptyDir: {}
155+
- name: nginx-cache
156+
emptyDir: {}
157+
- name: nginx-includes
158+
emptyDir: {}
159+
- name: nginx-includes-bootstrap
160+
configMap:
161+
name: nginx-includes-bootstrap
162+
{{- if .Values.nginx.plus }}
163+
- name: nginx-lib
164+
emptyDir: {}
165+
{{- if .Values.nginx.usage.secretName }}
166+
- name: nginx-plus-license
167+
secret:
168+
secretName: {{ .Values.nginx.usage.secretName }}
169+
{{- end }}
170+
{{- if or .Values.nginx.usage.caSecretName .Values.nginx.usage.clientSSLSecretName }}
171+
- name: nginx-plus-usage-certs
172+
projected:
173+
sources:
174+
{{- if .Values.nginx.usage.caSecretName }}
175+
- secret:
176+
name: {{ .Values.nginx.usage.caSecretName }}
177+
{{- end }}
178+
{{- if .Values.nginx.usage.clientSSLSecretName }}
179+
- secret:
180+
name: {{ .Values.nginx.usage.clientSSLSecretName }}
181+
{{- end }}
182+
{{- end }}
183+
{{- end }}
184+
{{- with .Values.extraVolumes -}}
185+
{{ toYaml . | nindent 6 }}
186+
{{- end }}

0 commit comments

Comments
 (0)