From 7033bb1c6b451ef39da47d4faac5537cb2fbf5ff Mon Sep 17 00:00:00 2001 From: Saylor Berman Date: Fri, 14 Mar 2025 14:21:07 -0600 Subject: [PATCH] CP/DP split: Secure connection Problem: We want to ensure that the connection between the control plane and data plane is authenticated and secure. Solution: 1. Configure agent to send the kubernetes service token in the request. The control plane validates this token using the TokenReview API to ensure the agent is authenticated. 2. Configure TLS certificates for both the control and data planes. By default, a Job will run when installing NGF that creates self-signed certificates in the nginx-gateway namespace. The server Secret is mounted to the control plane, and the control plane copies the client Secret when deploying nginx resources. This Secret is mounted to the agent. The control plane will reset the agent connection if it detects that its own certs have changed. For production environments, we'll recommend a user configures TLS using cert-manager instead, for better security and certificate rotation. --- charts/nginx-gateway-fabric/README.md | 6 + .../templates/certs-job.yaml | 112 +++++++ .../templates/clusterrole.yaml | 6 + .../templates/deployment.yaml | 8 + .../nginx-gateway-fabric/templates/scc.yaml | 2 + .../nginx-gateway-fabric/values.schema.json | 42 +++ charts/nginx-gateway-fabric/values.yaml | 21 ++ cmd/gateway/certs.go | 230 ++++++++++++++ cmd/gateway/certs_test.go | 139 +++++++++ cmd/gateway/commands.go | 160 ++++++++-- cmd/gateway/commands_test.go | 132 +++++++- cmd/gateway/main.go | 1 + deploy/aws-nlb/deploy.yaml | 107 +++++++ deploy/azure/deploy.yaml | 107 +++++++ deploy/default/deploy.yaml | 107 +++++++ deploy/experimental-nginx-plus/deploy.yaml | 107 +++++++ deploy/experimental/deploy.yaml | 107 +++++++ deploy/nginx-plus/deploy.yaml | 107 +++++++ deploy/nodeport/deploy.yaml | 107 +++++++ deploy/openshift/deploy.yaml | 109 +++++++ .../snippets-filters-nginx-plus/deploy.yaml | 107 +++++++ deploy/snippets-filters/deploy.yaml | 107 +++++++ go.mod | 2 +- internal/framework/controller/index/pod.go | 19 ++ .../framework/controller/index/pod_test.go | 53 ++++ internal/framework/controller/register.go | 4 +- internal/mode/static/config/config.go | 2 + internal/mode/static/manager.go | 27 ++ internal/mode/static/nginx/agent/agent.go | 2 + .../mode/static/nginx/agent/agent_test.go | 4 +- internal/mode/static/nginx/agent/command.go | 7 +- .../mode/static/nginx/agent/command_test.go | 79 +++++ .../static/nginx/agent/grpc/connections.go | 3 - .../nginx/agent/grpc/context/context.go | 1 + .../nginx/agent/grpc/filewatcher/doc.go | 4 + .../agent/grpc/filewatcher/filewatcher.go | 106 +++++++ .../grpc/filewatcher/filewatcher_test.go | 69 +++++ internal/mode/static/nginx/agent/grpc/grpc.go | 74 ++++- .../agent/grpc/interceptor/interceptor.go | 143 ++++++++- .../grpc/interceptor/interceptor_test.go | 292 ++++++++++++++++++ internal/mode/static/provisioner/eventloop.go | 4 +- internal/mode/static/provisioner/handler.go | 9 +- .../mode/static/provisioner/handler_test.go | 19 +- internal/mode/static/provisioner/objects.go | 74 ++++- .../mode/static/provisioner/objects_test.go | 139 ++++++--- .../mode/static/provisioner/provisioner.go | 16 +- .../static/provisioner/provisioner_test.go | 49 ++- internal/mode/static/provisioner/store.go | 18 +- .../mode/static/provisioner/store_test.go | 45 ++- internal/mode/static/provisioner/templates.go | 7 + 50 files changed, 3082 insertions(+), 120 deletions(-) create mode 100644 charts/nginx-gateway-fabric/templates/certs-job.yaml create mode 100644 cmd/gateway/certs.go create mode 100644 cmd/gateway/certs_test.go create mode 100644 internal/framework/controller/index/pod.go create mode 100644 internal/framework/controller/index/pod_test.go create mode 100644 internal/mode/static/nginx/agent/grpc/filewatcher/doc.go create mode 100644 internal/mode/static/nginx/agent/grpc/filewatcher/filewatcher.go create mode 100644 internal/mode/static/nginx/agent/grpc/filewatcher/filewatcher_test.go create mode 100644 internal/mode/static/nginx/agent/grpc/interceptor/interceptor_test.go diff --git a/charts/nginx-gateway-fabric/README.md b/charts/nginx-gateway-fabric/README.md index 84f837c2bd..b71ad17677 100644 --- a/charts/nginx-gateway-fabric/README.md +++ b/charts/nginx-gateway-fabric/README.md @@ -258,6 +258,12 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | Key | Description | Type | Default | |-----|-------------|------|---------| +| `certGenerator` | The certGenerator section contains the configuration for the cert-generator Job. | object | `{"agentTLSSecretName":"agent-tls","annotations":{},"overwrite":false,"serverTLSSecretName":"server-tls"}` | +| `certGenerator.agentTLSSecretName` | The name of the base Secret containing TLS CA, certificate, and key for the NGINX Agent to securely communicate with the NGINX Gateway Fabric control plane. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"agent-tls"` | +| `certGenerator.annotations` | The annotations of the cert-generator Job. | object | `{}` | +| `certGenerator.overwrite` | Overwrite existing TLS Secrets on startup. | bool | `false` | +| `certGenerator.serverTLSSecretName` | The name of the Secret containing TLS CA, certificate, and key for the NGINX Gateway Fabric control plane to securely communicate with the NGINX Agent. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"server-tls"` | +| `clusterDomain` | The DNS cluster domain of your Kubernetes cluster. | string | `"cluster.local"` | | `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"config":{},"container":{},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` | | `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. | object | `{}` | | `nginx.container` | The container configuration for the NGINX container. | object | `{}` | diff --git a/charts/nginx-gateway-fabric/templates/certs-job.yaml b/charts/nginx-gateway-fabric/templates/certs-job.yaml new file mode 100644 index 0000000000..a2b529ae1b --- /dev/null +++ b/charts/nginx-gateway-fabric/templates/certs-job.yaml @@ -0,0 +1,112 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "nginx-gateway.fullname" . }}-cert-generator + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-gateway.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-install +{{- if or .Values.nginxGateway.serviceAccount.imagePullSecret .Values.nginxGateway.serviceAccount.imagePullSecrets }} +imagePullSecrets: + {{- if .Values.nginxGateway.serviceAccount.imagePullSecret }} + - name: {{ .Values.nginxGateway.serviceAccount.imagePullSecret }} + {{- end }} + {{- if .Values.nginxGateway.serviceAccount.imagePullSecrets }} + {{- range .Values.nginxGateway.serviceAccount.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "nginx-gateway.fullname" . }}-cert-generator + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-gateway.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-install +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "nginx-gateway.fullname" . }}-cert-generator + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-gateway.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-install +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "nginx-gateway.fullname" . }}-cert-generator +subjects: +- kind: ServiceAccount + name: {{ include "nginx-gateway.fullname" . }}-cert-generator + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "nginx-gateway.fullname" . }}-cert-generator + namespace: {{ .Release.Namespace }} + labels: + {{- include "nginx-gateway.labels" . | nindent 4 }} + annotations: + {{- with .Values.certGenerator.annotations -}} + {{ toYaml . | nindent 4 }} + {{- end }} + "helm.sh/hook": pre-install, pre-upgrade +spec: + template: + metadata: + annotations: + {{- with .Values.certGenerator.annotations -}} + {{ toYaml . | nindent 8 }} + {{- end }} + spec: + containers: + - args: + - generate-certs + - --service={{ include "nginx-gateway.fullname" . }} + - --cluster-domain={{ .Values.clusterDomain }} + - --server-tls-secret={{ .Values.certGenerator.serverTLSSecretName }} + - --agent-tls-secret={{ .Values.certGenerator.agentTLSSecretName }} + {{- if .Values.certGenerator.overwrite }} + - --overwrite + {{- end }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }} + imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }} + name: cert-generator + securityContext: + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 101 + runAsGroup: 1001 + restartPolicy: Never + serviceAccountName: {{ include "nginx-gateway.fullname" . }}-cert-generator + securityContext: + fsGroup: 1001 + runAsNonRoot: true + ttlSecondsAfterFinished: 0 diff --git a/charts/nginx-gateway-fabric/templates/clusterrole.yaml b/charts/nginx-gateway-fabric/templates/clusterrole.yaml index 2ae9c5a2c0..479c22adbc 100644 --- a/charts/nginx-gateway-fabric/templates/clusterrole.yaml +++ b/charts/nginx-gateway-fabric/templates/clusterrole.yaml @@ -59,6 +59,12 @@ rules: verbs: - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create - apiGroups: - gateway.networking.k8s.io resources: diff --git a/charts/nginx-gateway-fabric/templates/deployment.yaml b/charts/nginx-gateway-fabric/templates/deployment.yaml index 574cc274a3..49898c57b2 100644 --- a/charts/nginx-gateway-fabric/templates/deployment.yaml +++ b/charts/nginx-gateway-fabric/templates/deployment.yaml @@ -36,6 +36,7 @@ spec: - --gatewayclass={{ .Values.nginxGateway.gatewayClassName }} - --config={{ include "nginx-gateway.config-name" . }} - --service={{ include "nginx-gateway.fullname" . }} + - --agent-tls-secret={{ .Values.certGenerator.agentTLSSecretName }} {{- if .Values.nginx.imagePullSecret }} - --nginx-docker-secret={{ .Values.nginx.imagePullSecret }} {{- end }} @@ -149,6 +150,9 @@ spec: readOnlyRootFilesystem: true runAsUser: 101 runAsGroup: 1001 + volumeMounts: + - name: nginx-agent-tls + mountPath: /var/run/secrets/ngf {{- with .Values.nginxGateway.extraVolumeMounts -}} {{ toYaml . | nindent 8 }} {{- end }} @@ -173,6 +177,10 @@ spec: nodeSelector: {{- toYaml .Values.nginxGateway.nodeSelector | nindent 8 }} {{- end }} + volumes: + - name: nginx-agent-tls + secret: + secretName: {{ .Values.certGenerator.serverTLSSecretName }} {{- with .Values.nginxGateway.extraVolumes -}} {{ toYaml . | nindent 6 }} {{- end }} diff --git a/charts/nginx-gateway-fabric/templates/scc.yaml b/charts/nginx-gateway-fabric/templates/scc.yaml index 6ab7dc92c1..1564e84e32 100644 --- a/charts/nginx-gateway-fabric/templates/scc.yaml +++ b/charts/nginx-gateway-fabric/templates/scc.yaml @@ -34,4 +34,6 @@ users: - {{ printf "system:serviceaccount:%s:%s" .Release.Namespace (include "nginx-gateway.serviceAccountName" .) }} requiredDropCapabilities: - ALL +volumes: +- secret {{- end }} diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index fac20c8777..04faa2ca45 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -1,6 +1,48 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { + "certGenerator": { + "description": "The certGenerator section contains the configuration for the cert-generator Job.", + "properties": { + "agentTLSSecretName": { + "default": "agent-tls", + "description": "The name of the base Secret containing TLS CA, certificate, and key for the NGINX Agent to securely\ncommunicate with the NGINX Gateway Fabric control plane. Must exist in the same namespace that the\nNGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).", + "required": [], + "title": "agentTLSSecretName", + "type": "string" + }, + "annotations": { + "description": "The annotations of the cert-generator Job.", + "required": [], + "title": "annotations", + "type": "object" + }, + "overwrite": { + "default": false, + "description": "Overwrite existing TLS Secrets on startup.", + "required": [], + "title": "overwrite", + "type": "boolean" + }, + "serverTLSSecretName": { + "default": "server-tls", + "description": "The name of the Secret containing TLS CA, certificate, and key for the NGINX Gateway Fabric control plane\nto securely communicate with the NGINX Agent. Must exist in the same namespace that the NGINX Gateway Fabric\ncontrol plane is running in (default namespace: nginx-gateway).", + "required": [], + "title": "serverTLSSecretName", + "type": "string" + } + }, + "required": [], + "title": "certGenerator", + "type": "object" + }, + "clusterDomain": { + "default": "cluster.local", + "description": "The DNS cluster domain of your Kubernetes cluster.", + "required": [], + "title": "clusterDomain", + "type": "string" + }, "global": { "description": "Global values are values that can be accessed from any chart or subchart by exactly the same name.", "required": [], diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index 5289392b38..4ad0230669 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -1,5 +1,8 @@ # yaml-language-server: $schema=values.schema.json +# -- The DNS cluster domain of your Kubernetes cluster. +clusterDomain: cluster.local + # -- The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. nginxGateway: # FIXME(lucacome): https://github.com/nginx/nginx-gateway-fabric/issues/2490 @@ -426,3 +429,21 @@ nginx: # -- Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. debug: false + +# -- The certGenerator section contains the configuration for the cert-generator Job. +certGenerator: + # -- The annotations of the cert-generator Job. + annotations: {} + + # -- The name of the Secret containing TLS CA, certificate, and key for the NGINX Gateway Fabric control plane + # to securely communicate with the NGINX Agent. Must exist in the same namespace that the NGINX Gateway Fabric + # control plane is running in (default namespace: nginx-gateway). + serverTLSSecretName: server-tls + + # -- The name of the base Secret containing TLS CA, certificate, and key for the NGINX Agent to securely + # communicate with the NGINX Gateway Fabric control plane. Must exist in the same namespace that the + # NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). + agentTLSSecretName: agent-tls + + # -- Overwrite existing TLS Secrets on startup. + overwrite: false diff --git a/cmd/gateway/certs.go b/cmd/gateway/certs.go new file mode 100644 index 0000000000..6f7a22d97e --- /dev/null +++ b/cmd/gateway/certs.go @@ -0,0 +1,230 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/sha1" //nolint:gosec // using sha1 in this case is fine + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "reflect" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + ctlrZap "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +const ( + expiry = 365 * 3 * 24 * time.Hour // 3 years + defaultDomain = "cluster.local" +) + +var subject = pkix.Name{ + CommonName: "nginx-gateway", + Country: []string{"US"}, + Locality: []string{"SEA"}, + Organization: []string{"F5"}, + OrganizationalUnit: []string{"NGINX"}, +} + +type certificateConfig struct { + caCertificate []byte + serverCertificate []byte + serverKey []byte + clientCertificate []byte + clientKey []byte +} + +// generateCertificates creates a CA, server, and client certificates and keys. +func generateCertificates(service, namespace, clientDNSDomain string) (*certificateConfig, error) { + caCertPEM, caKeyPEM, err := generateCA() + if err != nil { + return nil, fmt.Errorf("error generating CA: %w", err) + } + + caKeyPair, err := tls.X509KeyPair(caCertPEM, caKeyPEM) + if err != nil { + return nil, err + } + + serverCert, serverKey, err := generateCert(caKeyPair, serverDNSNames(service, namespace)) + if err != nil { + return nil, fmt.Errorf("error generating server cert: %w", err) + } + + clientCert, clientKey, err := generateCert(caKeyPair, clientDNSNames(clientDNSDomain)) + if err != nil { + return nil, fmt.Errorf("error generating client cert: %w", err) + } + + return &certificateConfig{ + caCertificate: caCertPEM, + serverCertificate: serverCert, + serverKey: serverKey, + clientCertificate: clientCert, + clientKey: clientKey, + }, nil +} + +func generateCA() ([]byte, []byte, error) { + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, err + } + + ca := &x509.Certificate{ + Subject: subject, + NotBefore: time.Now(), + NotAfter: time.Now().Add(expiry), + SubjectKeyId: subjectKeyID(caKey.N), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + IsCA: true, + BasicConstraintsValid: true, + } + + caCertBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey) + if err != nil { + return nil, nil, err + } + + caCertPEM := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: caCertBytes, + }) + + caKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(caKey), + }) + + return caCertPEM, caKeyPEM, nil +} + +func generateCert(caKeyPair tls.Certificate, dnsNames []string) ([]byte, []byte, error) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, err + } + + cert := &x509.Certificate{ + Subject: subject, + NotBefore: time.Now(), + NotAfter: time.Now().Add(expiry), + SubjectKeyId: subjectKeyID(key.N), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + DNSNames: dnsNames, + } + + caCert, err := x509.ParseCertificate(caKeyPair.Certificate[0]) + if err != nil { + return nil, nil, err + } + + certBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, &key.PublicKey, caKeyPair.PrivateKey) + if err != nil { + return nil, nil, err + } + + certPEM := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + + keyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(key), + }) + + return certPEM, keyPEM, nil +} + +// subjectKeyID generates the SubjectKeyID using the modulus of the private key. +func subjectKeyID(n *big.Int) []byte { + h := sha1.New() //nolint:gosec // using sha1 in this case is fine + h.Write(n.Bytes()) + return h.Sum(nil) +} + +func serverDNSNames(service, namespace string) []string { + return []string{ + fmt.Sprintf("%s.%s.svc", service, namespace), + } +} + +func clientDNSNames(dnsDomain string) []string { + return []string{ + fmt.Sprintf("*.%s", dnsDomain), + } +} + +func createSecrets( + ctx context.Context, + k8sClient client.Client, + certConfig *certificateConfig, + serverSecretName, + clientSecretName, + namespace string, + overwrite bool, +) error { + serverSecret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: serverSecretName, + Namespace: namespace, + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + "ca.crt": certConfig.caCertificate, + "tls.crt": certConfig.serverCertificate, + "tls.key": certConfig.serverKey, + }, + } + + clientSecret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: clientSecretName, + Namespace: namespace, + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + "ca.crt": certConfig.caCertificate, + "tls.crt": certConfig.clientCertificate, + "tls.key": certConfig.clientKey, + }, + } + + logger := ctlrZap.New().WithName("cert-generator") + for _, secret := range []corev1.Secret{serverSecret, clientSecret} { + key := client.ObjectKeyFromObject(&secret) + currentSecret := &corev1.Secret{} + + if err := k8sClient.Get(ctx, key, currentSecret); err != nil { + if apierrors.IsNotFound(err) { + if err := k8sClient.Create(ctx, &secret); err != nil { + return fmt.Errorf("error creating secret %v: %w", key, err) + } + } else { + return fmt.Errorf("error getting secret %v: %w", key, err) + } + } else { + if !overwrite { + logger.Info("Skipping updating Secret. Must be updated manually or by another source.", "name", key) + continue + } + + if !reflect.DeepEqual(secret.Data, currentSecret.Data) { + if err := k8sClient.Update(ctx, &secret); err != nil { + return fmt.Errorf("error updating secret %v: %w", key, err) + } + } + } + } + + return nil +} diff --git a/cmd/gateway/certs_test.go b/cmd/gateway/certs_test.go new file mode 100644 index 0000000000..4a9bfbe164 --- /dev/null +++ b/cmd/gateway/certs_test.go @@ -0,0 +1,139 @@ +package main + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "testing" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestGenerateCertificates(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + certConfig, err := generateCertificates("nginx", "default", "cluster.local") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(certConfig).ToNot(BeNil()) + g.Expect(certConfig.caCertificate).ToNot(BeNil()) + g.Expect(certConfig.serverCertificate).ToNot(BeNil()) + g.Expect(certConfig.serverKey).ToNot(BeNil()) + g.Expect(certConfig.clientCertificate).ToNot(BeNil()) + g.Expect(certConfig.clientKey).ToNot(BeNil()) + + block, _ := pem.Decode(certConfig.caCertificate) + g.Expect(block).ToNot(BeNil()) + caCert, err := x509.ParseCertificate(block.Bytes) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(caCert.IsCA).To(BeTrue()) + + pool := x509.NewCertPool() + g.Expect(pool.AppendCertsFromPEM(certConfig.caCertificate)).To(BeTrue()) + + block, _ = pem.Decode(certConfig.serverCertificate) + g.Expect(block).ToNot(BeNil()) + serverCert, err := x509.ParseCertificate(block.Bytes) + g.Expect(err).ToNot(HaveOccurred()) + + _, err = serverCert.Verify(x509.VerifyOptions{ + DNSName: "nginx.default.svc", + Roots: pool, + }) + g.Expect(err).ToNot(HaveOccurred()) + + block, _ = pem.Decode(certConfig.clientCertificate) + g.Expect(block).ToNot(BeNil()) + clientCert, err := x509.ParseCertificate(block.Bytes) + g.Expect(err).ToNot(HaveOccurred()) + + _, err = clientCert.Verify(x509.VerifyOptions{ + DNSName: "*.cluster.local", + Roots: pool, + }) + g.Expect(err).ToNot(HaveOccurred()) +} + +func TestCreateSecrets(t *testing.T) { + t.Parallel() + + fakeClient := fake.NewFakeClient() + + tests := []struct { + name string + overwrite bool + }{ + { + name: "doesn't overwrite on updates", + overwrite: false, + }, + { + name: "overwrites on updates", + overwrite: true, + }, + } + + verifySecrets := func(g *WithT, name string, overwrite bool) { + certConfig, err := generateCertificates("nginx", "default", "cluster.local") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(certConfig).ToNot(BeNil()) + + serverSecretName := fmt.Sprintf("%s-server-secret", name) + clientSecretName := fmt.Sprintf("%s-client-secret", name) + err = createSecrets(t.Context(), fakeClient, certConfig, serverSecretName, clientSecretName, "default", overwrite) + g.Expect(err).ToNot(HaveOccurred()) + + serverSecret := &corev1.Secret{} + err = fakeClient.Get(t.Context(), client.ObjectKey{Name: serverSecretName, Namespace: "default"}, serverSecret) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(serverSecret.Data["ca.crt"]).To(Equal(certConfig.caCertificate)) + g.Expect(serverSecret.Data["tls.crt"]).To(Equal(certConfig.serverCertificate)) + g.Expect(serverSecret.Data["tls.key"]).To(Equal(certConfig.serverKey)) + + clientSecret := &corev1.Secret{} + err = fakeClient.Get(t.Context(), client.ObjectKey{Name: clientSecretName, Namespace: "default"}, clientSecret) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(clientSecret.Data["ca.crt"]).To(Equal(certConfig.caCertificate)) + g.Expect(clientSecret.Data["tls.crt"]).To(Equal(certConfig.clientCertificate)) + g.Expect(clientSecret.Data["tls.key"]).To(Equal(certConfig.clientKey)) + + // If overwrite is false, then no updates should occur. If true, then updates should occur. + newCertConfig, err := generateCertificates("nginx", "default", "new-DNS-name") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(newCertConfig).ToNot(BeNil()) + g.Expect(newCertConfig).ToNot(Equal(certConfig)) + + err = createSecrets(t.Context(), fakeClient, newCertConfig, serverSecretName, clientSecretName, "default", overwrite) + g.Expect(err).ToNot(HaveOccurred()) + + expCertConfig := certConfig + if overwrite { + expCertConfig = newCertConfig + } + + err = fakeClient.Get(t.Context(), client.ObjectKey{Name: serverSecretName, Namespace: "default"}, serverSecret) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(serverSecret.Data["tls.crt"]).To(Equal(expCertConfig.serverCertificate)) + + err = fakeClient.Get(t.Context(), client.ObjectKey{Name: clientSecretName, Namespace: "default"}, clientSecret) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(clientSecret.Data["tls.crt"]).To(Equal(expCertConfig.clientCertificate)) + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + name := "no-overwrite" + if test.overwrite { + name = "overwrite" + } + + verifySecrets(g, name, test.overwrite) + }) + } +} diff --git a/cmd/gateway/commands.go b/cmd/gateway/commands.go index efa990e29d..4d875cb6ac 100644 --- a/cmd/gateway/commands.go +++ b/cmd/gateway/commands.go @@ -17,6 +17,7 @@ import ( "k8s.io/klog/v2" ctlr "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + k8sConfig "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/log" ctlrZap "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -37,6 +38,9 @@ const ( gatewayCtlrNameUsageFmt = `The name of the Gateway controller. ` + `The controller name must be of the form: DOMAIN/PATH. The controller's domain is '%s'` plusFlag = "nginx-plus" + + serverTLSSecret = "server-tls" + agentTLSSecret = "agent-tls" ) func createRootCommand() *cobra.Command { @@ -58,6 +62,7 @@ func createControllerCommand() *cobra.Command { gatewayFlag = "gateway" configFlag = "config" serviceFlag = "service" + agentTLSSecretFlag = "agent-tls-secret" updateGCStatusFlag = "update-gatewayclass-status" metricsDisableFlag = "metrics-disable" metricsSecureFlag = "metrics-secure-serving" @@ -96,6 +101,10 @@ func createControllerCommand() *cobra.Command { serviceName = stringValidatingValue{ validator: validateResourceName, } + agentTLSSecretName = stringValidatingValue{ + validator: validateResourceName, + value: agentTLSSecret, + } disableMetrics bool metricsSecure bool metricsListenPort = intValidatingValue{ @@ -254,6 +263,7 @@ func createControllerCommand() *cobra.Command { }, SnippetsFilters: snippetsFilters, NginxDockerSecretNames: nginxDockerSecrets.values, + AgentTLSSecretName: agentTLSSecretName.value, } if err := static.StartManager(conf); err != nil { @@ -303,6 +313,14 @@ func createControllerCommand() *cobra.Command { ` Lives in the same Namespace as the controller.`, ) + cmd.Flags().Var( + &agentTLSSecretName, + agentTLSSecretFlag, + `The name of the base Secret containing TLS CA, certificate, and key for the NGINX Agent to securely `+ + `communicate with the NGINX Gateway Fabric control plane. Must exist in the same namespace that the `+ + `NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).`, + ) + cmd.Flags().BoolVar( &updateGCStatus, updateGCStatusFlag, @@ -442,32 +460,105 @@ func createControllerCommand() *cobra.Command { return cmd } -// FIXME(pleshakov): Remove this command once NGF min supported Kubernetes version supports sleep action in -// preStop hook. -// See https://github.com/kubernetes/enhancements/tree/4ec371d92dcd4f56a2ab18c8ba20bb85d8d20efe/keps/sig-node/3960-pod-lifecycle-sleep-action -// -//nolint:lll -func createSleepCommand() *cobra.Command { +func createGenerateCertsCommand() *cobra.Command { // flag names - const durationFlag = "duration" + const ( + serverTLSSecretFlag = "server-tls-secret" //nolint:gosec // not credentials + agentTLSSecretFlag = "agent-tls-secret" + serviceFlag = "service" + clusterDomainFlag = "cluster-domain" + overwriteFlag = "overwrite" + ) + // flag values - var duration time.Duration + var ( + serverTLSSecretName = stringValidatingValue{ + validator: validateResourceName, + value: serverTLSSecret, + } + agentTLSSecretName = stringValidatingValue{ + validator: validateResourceName, + value: agentTLSSecret, + } + serviceName = stringValidatingValue{ + validator: validateResourceName, + } + clusterDomain = stringValidatingValue{ + validator: validateQualifiedName, + value: defaultDomain, + } + overwrite bool + ) cmd := &cobra.Command{ - Use: "sleep", - Short: "Sleep for specified duration and exit", - Run: func(_ *cobra.Command, _ []string) { - // It is expected that this command is run from lifecycle hook. - // Because logs from hooks are not visible in the container logs, we don't log here at all. - time.Sleep(duration) + Use: "generate-certs", + Short: "Generate self-signed certificates for securing control plane to data plane communication", + RunE: func(cmd *cobra.Command, _ []string) error { + namespace, err := getValueFromEnv("POD_NAMESPACE") + if err != nil { + return fmt.Errorf("POD_NAMESPACE must be specified in the ENV") + } + + certConfig, err := generateCertificates(serviceName.value, namespace, clusterDomain.value) + if err != nil { + return fmt.Errorf("error generating certificates: %w", err) + } + + k8sClient, err := client.New(k8sConfig.GetConfigOrDie(), client.Options{}) + if err != nil { + return fmt.Errorf("error creating k8s client: %w", err) + } + + if err := createSecrets( + cmd.Context(), + k8sClient, + certConfig, + serverTLSSecretName.value, + agentTLSSecretName.value, + namespace, + overwrite, + ); err != nil { + return fmt.Errorf("error creating secrets: %w", err) + } + + return nil }, } - cmd.Flags().DurationVar( - &duration, - durationFlag, - 30*time.Second, - "Set the duration of sleep. Must be parsable by https://pkg.go.dev/time#ParseDuration", + cmd.Flags().Var( + &serverTLSSecretName, + serverTLSSecretFlag, + `The name of the Secret containing TLS CA, certificate, and key for the NGINX Gateway Fabric control plane `+ + `to securely communicate with the NGINX Agent. Must exist in the same namespace that the `+ + `NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).`, + ) + + cmd.Flags().Var( + &agentTLSSecretName, + agentTLSSecretFlag, + `The name of the base Secret containing TLS CA, certificate, and key for the NGINX Agent to securely `+ + `communicate with the NGINX Gateway Fabric control plane. Must exist in the same namespace that the `+ + `NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway).`, + ) + + cmd.Flags().Var( + &serviceName, + serviceFlag, + `The name of the Service that fronts the NGINX Gateway Fabric Pod.`+ + ` Lives in the same Namespace as the controller.`, + ) + + cmd.Flags().Var( + &clusterDomain, + clusterDomainFlag, + `The DNS domain of your Kubernetes cluster.`, + ) + + cmd.Flags().BoolVar( + &overwrite, + overwriteFlag, + false, + "Overwrite existing certificates.", ) return cmd @@ -564,6 +655,37 @@ func createInitializeCommand() *cobra.Command { return cmd } +// FIXME(pleshakov): Remove this command once NGF min supported Kubernetes version supports sleep action in +// preStop hook. +// See https://github.com/kubernetes/enhancements/tree/4ec371d92dcd4f56a2ab18c8ba20bb85d8d20efe/keps/sig-node/3960-pod-lifecycle-sleep-action +// +//nolint:lll +func createSleepCommand() *cobra.Command { + // flag names + const durationFlag = "duration" + // flag values + var duration time.Duration + + cmd := &cobra.Command{ + Use: "sleep", + Short: "Sleep for specified duration and exit", + Run: func(_ *cobra.Command, _ []string) { + // It is expected that this command is run from lifecycle hook. + // Because logs from hooks are not visible in the container logs, we don't log here at all. + time.Sleep(duration) + }, + } + + cmd.Flags().DurationVar( + &duration, + durationFlag, + 30*time.Second, + "Set the duration of sleep. Must be parsable by https://pkg.go.dev/time#ParseDuration", + ) + + return cmd +} + func parseFlags(flags *pflag.FlagSet) ([]string, []string) { var flagKeys, flagValues []string diff --git a/cmd/gateway/commands_test.go b/cmd/gateway/commands_test.go index 61459455f6..8662c4ef0d 100644 --- a/cmd/gateway/commands_test.go +++ b/cmd/gateway/commands_test.go @@ -129,7 +129,7 @@ func TestCommonFlagsValidation(t *testing.T) { } } -func TestStaticModeCmdFlagValidation(t *testing.T) { +func TestControllerCmdFlagValidation(t *testing.T) { t.Parallel() tests := []flagTestCase{ { @@ -140,6 +140,7 @@ func TestStaticModeCmdFlagValidation(t *testing.T) { "--gateway=nginx-gateway/nginx", "--config=nginx-gateway-config", "--service=nginx-gateway", + "--agent-tls-secret=agent-tls", "--update-gatewayclass-status=true", "--metrics-port=9114", "--metrics-disable", @@ -217,6 +218,22 @@ func TestStaticModeCmdFlagValidation(t *testing.T) { wantErr: true, expectedErrPrefix: `invalid argument "!@#$" for "--service" flag: invalid format`, }, + { + name: "agent-tls-secret is set to empty string", + args: []string{ + "--agent-tls-secret=", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "" for "--agent-tls-secret" flag: must be set`, + }, + { + name: "agent-tls-secret is set to invalid string", + args: []string{ + "--agent-tls-secret=!@#$", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "!@#$" for "--agent-tls-secret" flag: invalid format`, + }, { name: "update-gatewayclass-status is set to empty string", args: []string{ @@ -441,13 +458,18 @@ func TestStaticModeCmdFlagValidation(t *testing.T) { } } -func TestSleepCmdFlagValidation(t *testing.T) { +func TestGenerateCertsCmdFlagValidation(t *testing.T) { t.Parallel() + tests := []flagTestCase{ { name: "valid flags", args: []string{ - "--duration=1s", + "--server-tls-secret=server-secret", + "--agent-tls-secret=agent-secret", + "--service=my-service", + "--cluster-domain=cluster.local", + "--overwrite", }, wantErr: false, }, @@ -457,27 +479,75 @@ func TestSleepCmdFlagValidation(t *testing.T) { wantErr: false, }, { - name: "duration is set to empty string", + name: "server-tls-secret is set to empty string", args: []string{ - "--duration=", + "--server-tls-secret=", }, wantErr: true, - expectedErrPrefix: `invalid argument "" for "--duration" flag: time: invalid duration ""`, + expectedErrPrefix: `invalid argument "" for "--server-tls-secret" flag: must be set`, }, { - name: "duration is invalid", + name: "server-tls-secret is invalid", args: []string{ - "--duration=invalid", + "--server-tls-secret=!@#$", }, wantErr: true, - expectedErrPrefix: `invalid argument "invalid" for "--duration" flag: time: invalid duration "invalid"`, + expectedErrPrefix: `invalid argument "!@#$" for "--server-tls-secret" flag: invalid format`, + }, + { + name: "agent-tls-secret is set to empty string", + args: []string{ + "--agent-tls-secret=", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "" for "--agent-tls-secret" flag: must be set`, + }, + { + name: "agent-tls-secret is invalid", + args: []string{ + "--agent-tls-secret=!@#$", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "!@#$" for "--agent-tls-secret" flag: invalid format`, + }, + { + name: "service is set to empty string", + args: []string{ + "--service=", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "" for "--service" flag: must be set`, + }, + { + name: "service is invalid", + args: []string{ + "--service=!@#$", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "!@#$" for "--service" flag: invalid format`, + }, + { + name: "cluster-domain is set to empty string", + args: []string{ + "--cluster-domain=", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "" for "--cluster-domain" flag: must be set`, + }, + { + name: "cluster-domain is invalid", + args: []string{ + "--cluster-domain=!@#$", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "!@#$" for "--cluster-domain" flag: invalid format`, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { t.Parallel() - cmd := createSleepCommand() + cmd := createGenerateCertsCommand() testFlag(t, cmd, test) }) } @@ -529,6 +599,48 @@ func TestInitializeCmdFlagValidation(t *testing.T) { } } +func TestSleepCmdFlagValidation(t *testing.T) { + t.Parallel() + tests := []flagTestCase{ + { + name: "valid flags", + args: []string{ + "--duration=1s", + }, + wantErr: false, + }, + { + name: "omitted flags", + args: nil, + wantErr: false, + }, + { + name: "duration is set to empty string", + args: []string{ + "--duration=", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "" for "--duration" flag: time: invalid duration ""`, + }, + { + name: "duration is invalid", + args: []string{ + "--duration=invalid", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "invalid" for "--duration" flag: time: invalid duration "invalid"`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + cmd := createSleepCommand() + testFlag(t, cmd, test) + }) + } +} + func TestParseFlags(t *testing.T) { t.Parallel() g := NewWithT(t) diff --git a/cmd/gateway/main.go b/cmd/gateway/main.go index 203385b732..515fcc3f16 100644 --- a/cmd/gateway/main.go +++ b/cmd/gateway/main.go @@ -22,6 +22,7 @@ func main() { rootCmd.AddCommand( createControllerCommand(), + createGenerateCertsCommand(), createInitializeCommand(), createSleepCommand(), ) diff --git a/deploy/aws-nlb/deploy.yaml b/deploy/aws-nlb/deploy.yaml index c6309d395e..3f82b66bce 100644 --- a/deploy/aws-nlb/deploy.yaml +++ b/deploy/aws-nlb/deploy.yaml @@ -13,6 +13,35 @@ metadata: name: nginx-gateway namespace: nginx-gateway --- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - get +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -74,6 +103,12 @@ rules: verbs: - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create - apiGroups: - gateway.networking.k8s.io resources: @@ -138,6 +173,24 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-gateway-cert-generator +subjects: +- kind: ServiceAccount + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: @@ -205,6 +258,7 @@ spec: - --gatewayclass=nginx - --config=nginx-gateway-config - --service=nginx-gateway + - --agent-tls-secret=agent-tls - --metrics-port=9113 - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election @@ -253,11 +307,64 @@ spec: runAsUser: 101 seccompProfile: type: RuntimeDefault + volumeMounts: + - mountPath: /var/run/secrets/ngf + name: nginx-agent-tls securityContext: fsGroup: 1001 runAsNonRoot: true serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 + volumes: + - name: nginx-agent-tls + secret: + secretName: server-tls +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +spec: + template: + metadata: + annotations: null + spec: + containers: + - args: + - generate-certs + - --service=nginx-gateway + - --cluster-domain=cluster.local + - --server-tls-secret=server-tls + - --agent-tls-secret=agent-tls + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/nginx/nginx-gateway-fabric:edge + imagePullPolicy: Always + name: cert-generator + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + restartPolicy: Never + securityContext: + fsGroup: 1001 + runAsNonRoot: true + serviceAccountName: nginx-gateway-cert-generator + ttlSecondsAfterFinished: 0 --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/azure/deploy.yaml b/deploy/azure/deploy.yaml index 88ce668193..6fd37f8a86 100644 --- a/deploy/azure/deploy.yaml +++ b/deploy/azure/deploy.yaml @@ -13,6 +13,35 @@ metadata: name: nginx-gateway namespace: nginx-gateway --- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - get +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -74,6 +103,12 @@ rules: verbs: - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create - apiGroups: - gateway.networking.k8s.io resources: @@ -138,6 +173,24 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-gateway-cert-generator +subjects: +- kind: ServiceAccount + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: @@ -205,6 +258,7 @@ spec: - --gatewayclass=nginx - --config=nginx-gateway-config - --service=nginx-gateway + - --agent-tls-secret=agent-tls - --metrics-port=9113 - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election @@ -253,6 +307,9 @@ spec: runAsUser: 101 seccompProfile: type: RuntimeDefault + volumeMounts: + - mountPath: /var/run/secrets/ngf + name: nginx-agent-tls nodeSelector: kubernetes.io/os: linux securityContext: @@ -260,6 +317,56 @@ spec: runAsNonRoot: true serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 + volumes: + - name: nginx-agent-tls + secret: + secretName: server-tls +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +spec: + template: + metadata: + annotations: null + spec: + containers: + - args: + - generate-certs + - --service=nginx-gateway + - --cluster-domain=cluster.local + - --server-tls-secret=server-tls + - --agent-tls-secret=agent-tls + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/nginx/nginx-gateway-fabric:edge + imagePullPolicy: Always + name: cert-generator + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + restartPolicy: Never + securityContext: + fsGroup: 1001 + runAsNonRoot: true + serviceAccountName: nginx-gateway-cert-generator + ttlSecondsAfterFinished: 0 --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/default/deploy.yaml b/deploy/default/deploy.yaml index b73922cdae..28f9bbec55 100644 --- a/deploy/default/deploy.yaml +++ b/deploy/default/deploy.yaml @@ -13,6 +13,35 @@ metadata: name: nginx-gateway namespace: nginx-gateway --- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - get +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -74,6 +103,12 @@ rules: verbs: - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create - apiGroups: - gateway.networking.k8s.io resources: @@ -138,6 +173,24 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-gateway-cert-generator +subjects: +- kind: ServiceAccount + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: @@ -205,6 +258,7 @@ spec: - --gatewayclass=nginx - --config=nginx-gateway-config - --service=nginx-gateway + - --agent-tls-secret=agent-tls - --metrics-port=9113 - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election @@ -253,11 +307,64 @@ spec: runAsUser: 101 seccompProfile: type: RuntimeDefault + volumeMounts: + - mountPath: /var/run/secrets/ngf + name: nginx-agent-tls securityContext: fsGroup: 1001 runAsNonRoot: true serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 + volumes: + - name: nginx-agent-tls + secret: + secretName: server-tls +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +spec: + template: + metadata: + annotations: null + spec: + containers: + - args: + - generate-certs + - --service=nginx-gateway + - --cluster-domain=cluster.local + - --server-tls-secret=server-tls + - --agent-tls-secret=agent-tls + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/nginx/nginx-gateway-fabric:edge + imagePullPolicy: Always + name: cert-generator + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + restartPolicy: Never + securityContext: + fsGroup: 1001 + runAsNonRoot: true + serviceAccountName: nginx-gateway-cert-generator + ttlSecondsAfterFinished: 0 --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml index 23d4223234..68bdf72e43 100644 --- a/deploy/experimental-nginx-plus/deploy.yaml +++ b/deploy/experimental-nginx-plus/deploy.yaml @@ -13,6 +13,35 @@ metadata: name: nginx-gateway namespace: nginx-gateway --- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - get +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -74,6 +103,12 @@ rules: verbs: - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create - apiGroups: - gateway.networking.k8s.io resources: @@ -142,6 +177,24 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-gateway-cert-generator +subjects: +- kind: ServiceAccount + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: @@ -209,6 +262,7 @@ spec: - --gatewayclass=nginx - --config=nginx-gateway-config - --service=nginx-gateway + - --agent-tls-secret=agent-tls - --nginx-docker-secret=nginx-plus-registry-secret - --nginx-plus - --usage-report-secret=nplus-license @@ -261,11 +315,64 @@ spec: runAsUser: 101 seccompProfile: type: RuntimeDefault + volumeMounts: + - mountPath: /var/run/secrets/ngf + name: nginx-agent-tls securityContext: fsGroup: 1001 runAsNonRoot: true serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 + volumes: + - name: nginx-agent-tls + secret: + secretName: server-tls +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +spec: + template: + metadata: + annotations: null + spec: + containers: + - args: + - generate-certs + - --service=nginx-gateway + - --cluster-domain=cluster.local + - --server-tls-secret=server-tls + - --agent-tls-secret=agent-tls + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/nginx/nginx-gateway-fabric:edge + imagePullPolicy: Always + name: cert-generator + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + restartPolicy: Never + securityContext: + fsGroup: 1001 + runAsNonRoot: true + serviceAccountName: nginx-gateway-cert-generator + ttlSecondsAfterFinished: 0 --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/experimental/deploy.yaml b/deploy/experimental/deploy.yaml index 7ae7821f26..be7273edd4 100644 --- a/deploy/experimental/deploy.yaml +++ b/deploy/experimental/deploy.yaml @@ -13,6 +13,35 @@ metadata: name: nginx-gateway namespace: nginx-gateway --- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - get +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -74,6 +103,12 @@ rules: verbs: - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create - apiGroups: - gateway.networking.k8s.io resources: @@ -142,6 +177,24 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-gateway-cert-generator +subjects: +- kind: ServiceAccount + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: @@ -209,6 +262,7 @@ spec: - --gatewayclass=nginx - --config=nginx-gateway-config - --service=nginx-gateway + - --agent-tls-secret=agent-tls - --metrics-port=9113 - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election @@ -258,11 +312,64 @@ spec: runAsUser: 101 seccompProfile: type: RuntimeDefault + volumeMounts: + - mountPath: /var/run/secrets/ngf + name: nginx-agent-tls securityContext: fsGroup: 1001 runAsNonRoot: true serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 + volumes: + - name: nginx-agent-tls + secret: + secretName: server-tls +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +spec: + template: + metadata: + annotations: null + spec: + containers: + - args: + - generate-certs + - --service=nginx-gateway + - --cluster-domain=cluster.local + - --server-tls-secret=server-tls + - --agent-tls-secret=agent-tls + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/nginx/nginx-gateway-fabric:edge + imagePullPolicy: Always + name: cert-generator + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + restartPolicy: Never + securityContext: + fsGroup: 1001 + runAsNonRoot: true + serviceAccountName: nginx-gateway-cert-generator + ttlSecondsAfterFinished: 0 --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml index b6b6b1ca58..7bdb4fe3c9 100644 --- a/deploy/nginx-plus/deploy.yaml +++ b/deploy/nginx-plus/deploy.yaml @@ -13,6 +13,35 @@ metadata: name: nginx-gateway namespace: nginx-gateway --- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - get +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -74,6 +103,12 @@ rules: verbs: - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create - apiGroups: - gateway.networking.k8s.io resources: @@ -138,6 +173,24 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-gateway-cert-generator +subjects: +- kind: ServiceAccount + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: @@ -205,6 +258,7 @@ spec: - --gatewayclass=nginx - --config=nginx-gateway-config - --service=nginx-gateway + - --agent-tls-secret=agent-tls - --nginx-docker-secret=nginx-plus-registry-secret - --nginx-plus - --usage-report-secret=nplus-license @@ -256,11 +310,64 @@ spec: runAsUser: 101 seccompProfile: type: RuntimeDefault + volumeMounts: + - mountPath: /var/run/secrets/ngf + name: nginx-agent-tls securityContext: fsGroup: 1001 runAsNonRoot: true serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 + volumes: + - name: nginx-agent-tls + secret: + secretName: server-tls +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +spec: + template: + metadata: + annotations: null + spec: + containers: + - args: + - generate-certs + - --service=nginx-gateway + - --cluster-domain=cluster.local + - --server-tls-secret=server-tls + - --agent-tls-secret=agent-tls + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/nginx/nginx-gateway-fabric:edge + imagePullPolicy: Always + name: cert-generator + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + restartPolicy: Never + securityContext: + fsGroup: 1001 + runAsNonRoot: true + serviceAccountName: nginx-gateway-cert-generator + ttlSecondsAfterFinished: 0 --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/nodeport/deploy.yaml b/deploy/nodeport/deploy.yaml index 206401aac1..909270a96b 100644 --- a/deploy/nodeport/deploy.yaml +++ b/deploy/nodeport/deploy.yaml @@ -13,6 +13,35 @@ metadata: name: nginx-gateway namespace: nginx-gateway --- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - get +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -74,6 +103,12 @@ rules: verbs: - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create - apiGroups: - gateway.networking.k8s.io resources: @@ -138,6 +173,24 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-gateway-cert-generator +subjects: +- kind: ServiceAccount + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: @@ -205,6 +258,7 @@ spec: - --gatewayclass=nginx - --config=nginx-gateway-config - --service=nginx-gateway + - --agent-tls-secret=agent-tls - --metrics-port=9113 - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election @@ -253,11 +307,64 @@ spec: runAsUser: 101 seccompProfile: type: RuntimeDefault + volumeMounts: + - mountPath: /var/run/secrets/ngf + name: nginx-agent-tls securityContext: fsGroup: 1001 runAsNonRoot: true serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 + volumes: + - name: nginx-agent-tls + secret: + secretName: server-tls +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +spec: + template: + metadata: + annotations: null + spec: + containers: + - args: + - generate-certs + - --service=nginx-gateway + - --cluster-domain=cluster.local + - --server-tls-secret=server-tls + - --agent-tls-secret=agent-tls + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/nginx/nginx-gateway-fabric:edge + imagePullPolicy: Always + name: cert-generator + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + restartPolicy: Never + securityContext: + fsGroup: 1001 + runAsNonRoot: true + serviceAccountName: nginx-gateway-cert-generator + ttlSecondsAfterFinished: 0 --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/openshift/deploy.yaml b/deploy/openshift/deploy.yaml index 0a62309fcc..d61f5e49ac 100644 --- a/deploy/openshift/deploy.yaml +++ b/deploy/openshift/deploy.yaml @@ -13,6 +13,35 @@ metadata: name: nginx-gateway namespace: nginx-gateway --- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - get +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -74,6 +103,12 @@ rules: verbs: - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create - apiGroups: - gateway.networking.k8s.io resources: @@ -148,6 +183,24 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-gateway-cert-generator +subjects: +- kind: ServiceAccount + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: @@ -215,6 +268,7 @@ spec: - --gatewayclass=nginx - --config=nginx-gateway-config - --service=nginx-gateway + - --agent-tls-secret=agent-tls - --metrics-port=9113 - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election @@ -263,11 +317,64 @@ spec: runAsUser: 101 seccompProfile: type: RuntimeDefault + volumeMounts: + - mountPath: /var/run/secrets/ngf + name: nginx-agent-tls securityContext: fsGroup: 1001 runAsNonRoot: true serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 + volumes: + - name: nginx-agent-tls + secret: + secretName: server-tls +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +spec: + template: + metadata: + annotations: null + spec: + containers: + - args: + - generate-certs + - --service=nginx-gateway + - --cluster-domain=cluster.local + - --server-tls-secret=server-tls + - --agent-tls-secret=agent-tls + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/nginx/nginx-gateway-fabric:edge + imagePullPolicy: Always + name: cert-generator + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + restartPolicy: Never + securityContext: + fsGroup: 1001 + runAsNonRoot: true + serviceAccountName: nginx-gateway-cert-generator + ttlSecondsAfterFinished: 0 --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass @@ -354,3 +461,5 @@ supplementalGroups: type: MustRunAs users: - system:serviceaccount:nginx-gateway:nginx-gateway +volumes: +- secret diff --git a/deploy/snippets-filters-nginx-plus/deploy.yaml b/deploy/snippets-filters-nginx-plus/deploy.yaml index a25c7e1aa3..e6b27f01fb 100644 --- a/deploy/snippets-filters-nginx-plus/deploy.yaml +++ b/deploy/snippets-filters-nginx-plus/deploy.yaml @@ -13,6 +13,35 @@ metadata: name: nginx-gateway namespace: nginx-gateway --- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - get +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -74,6 +103,12 @@ rules: verbs: - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create - apiGroups: - gateway.networking.k8s.io resources: @@ -140,6 +175,24 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-gateway-cert-generator +subjects: +- kind: ServiceAccount + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: @@ -207,6 +260,7 @@ spec: - --gatewayclass=nginx - --config=nginx-gateway-config - --service=nginx-gateway + - --agent-tls-secret=agent-tls - --nginx-docker-secret=nginx-plus-registry-secret - --nginx-plus - --usage-report-secret=nplus-license @@ -259,11 +313,64 @@ spec: runAsUser: 101 seccompProfile: type: RuntimeDefault + volumeMounts: + - mountPath: /var/run/secrets/ngf + name: nginx-agent-tls securityContext: fsGroup: 1001 runAsNonRoot: true serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 + volumes: + - name: nginx-agent-tls + secret: + secretName: server-tls +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +spec: + template: + metadata: + annotations: null + spec: + containers: + - args: + - generate-certs + - --service=nginx-gateway + - --cluster-domain=cluster.local + - --server-tls-secret=server-tls + - --agent-tls-secret=agent-tls + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/nginx/nginx-gateway-fabric:edge + imagePullPolicy: Always + name: cert-generator + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + restartPolicy: Never + securityContext: + fsGroup: 1001 + runAsNonRoot: true + serviceAccountName: nginx-gateway-cert-generator + ttlSecondsAfterFinished: 0 --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/deploy/snippets-filters/deploy.yaml b/deploy/snippets-filters/deploy.yaml index 288058bac2..714376f7f7 100644 --- a/deploy/snippets-filters/deploy.yaml +++ b/deploy/snippets-filters/deploy.yaml @@ -13,6 +13,35 @@ metadata: name: nginx-gateway namespace: nginx-gateway --- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - update + - get +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -74,6 +103,12 @@ rules: verbs: - list - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create - apiGroups: - gateway.networking.k8s.io resources: @@ -140,6 +175,24 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-gateway-cert-generator +subjects: +- kind: ServiceAccount + name: nginx-gateway-cert-generator + namespace: nginx-gateway +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: @@ -207,6 +260,7 @@ spec: - --gatewayclass=nginx - --config=nginx-gateway-config - --service=nginx-gateway + - --agent-tls-secret=agent-tls - --metrics-port=9113 - --health-port=8081 - --leader-election-lock-name=nginx-gateway-leader-election @@ -256,11 +310,64 @@ spec: runAsUser: 101 seccompProfile: type: RuntimeDefault + volumeMounts: + - mountPath: /var/run/secrets/ngf + name: nginx-agent-tls securityContext: fsGroup: 1001 runAsNonRoot: true serviceAccountName: nginx-gateway terminationGracePeriodSeconds: 30 + volumes: + - name: nginx-agent-tls + secret: + secretName: server-tls +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/instance: nginx-gateway + app.kubernetes.io/name: nginx-gateway + app.kubernetes.io/version: edge + name: nginx-gateway-cert-generator + namespace: nginx-gateway +spec: + template: + metadata: + annotations: null + spec: + containers: + - args: + - generate-certs + - --service=nginx-gateway + - --cluster-domain=cluster.local + - --server-tls-secret=server-tls + - --agent-tls-secret=agent-tls + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/nginx/nginx-gateway-fabric:edge + imagePullPolicy: Always + name: cert-generator + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + restartPolicy: Never + securityContext: + fsGroup: 1001 + runAsNonRoot: true + serviceAccountName: nginx-gateway-cert-generator + ttlSecondsAfterFinished: 0 --- apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/go.mod b/go.mod index 497badb90b..beb7611cb7 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/nginx/nginx-gateway-fabric go 1.24.0 require ( + github.com/fsnotify/fsnotify v1.8.0 github.com/go-kit/log v0.2.1 github.com/go-logr/logr v1.4.2 github.com/google/go-cmp v0.7.0 @@ -38,7 +39,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect - github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/internal/framework/controller/index/pod.go b/internal/framework/controller/index/pod.go new file mode 100644 index 0000000000..2cd5cf6818 --- /dev/null +++ b/internal/framework/controller/index/pod.go @@ -0,0 +1,19 @@ +package index + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// PodIPIndexFunc is a client.IndexerFunc that parses a Pod object and returns the PodIP. +// Used by the gRPC token validator for validating a connection from NGINX agent. +func PodIPIndexFunc(obj client.Object) []string { + pod, ok := obj.(*corev1.Pod) + if !ok { + panic(fmt.Sprintf("expected an Pod; got %T", obj)) + } + + return []string{pod.Status.PodIP} +} diff --git a/internal/framework/controller/index/pod_test.go b/internal/framework/controller/index/pod_test.go new file mode 100644 index 0000000000..e89c0492da --- /dev/null +++ b/internal/framework/controller/index/pod_test.go @@ -0,0 +1,53 @@ +package index + +import ( + "testing" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestPodIPIndexFunc(t *testing.T) { + t.Parallel() + testcases := []struct { + msg string + obj client.Object + expOutput []string + }{ + { + msg: "normal case", + obj: &corev1.Pod{ + Status: corev1.PodStatus{ + PodIP: "1.2.3.4", + }, + }, + expOutput: []string{"1.2.3.4"}, + }, + { + msg: "empty status", + obj: &corev1.Pod{}, + expOutput: []string{""}, + }, + } + + for _, tc := range testcases { + t.Run(tc.msg, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + output := PodIPIndexFunc(tc.obj) + g.Expect(output).To(Equal(tc.expOutput)) + }) + } +} + +func TestPodIPIndexFuncPanics(t *testing.T) { + t.Parallel() + defer func() { + g := NewWithT(t) + g.Expect(recover()).ToNot(BeNil()) + }() + + PodIPIndexFunc(&corev1.Namespace{}) +} diff --git a/internal/framework/controller/register.go b/internal/framework/controller/register.go index c76db1f577..557438da98 100644 --- a/internal/framework/controller/register.go +++ b/internal/framework/controller/register.go @@ -96,7 +96,7 @@ func Register( } for field, indexerFunc := range cfg.fieldIndices { - if err := addIndex( + if err := AddIndex( ctx, mgr.GetFieldIndexer(), objectType, @@ -136,7 +136,7 @@ func Register( return nil } -func addIndex( +func AddIndex( ctx context.Context, indexer client.FieldIndexer, objectType ngftypes.ObjectType, diff --git a/internal/mode/static/config/config.go b/internal/mode/static/config/config.go index d8556e19f2..d37c6b55f7 100644 --- a/internal/mode/static/config/config.go +++ b/internal/mode/static/config/config.go @@ -32,6 +32,8 @@ type Config struct { ConfigName string // GatewayClassName is the name of the GatewayClass resource that the Gateway will use. GatewayClassName string + // AgentTLSSecretName is the name of the TLS Secret used by NGINX Agent to communicate with the control plane. + AgentTLSSecretName string // NginxDockerSecretNames are the names of any Docker registry Secrets for the NGINX container. NginxDockerSecretNames []string // LeaderElection contains the configuration for leader election. diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index 3459a7134f..bc68ba5510 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -11,6 +11,7 @@ import ( "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "google.golang.org/grpc" appsv1 "k8s.io/api/apps/v1" + authv1 "k8s.io/api/authentication/v1" apiv1 "k8s.io/api/core/v1" discoveryV1 "k8s.io/api/discovery/v1" apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -91,6 +92,7 @@ func init() { utilruntime.Must(ngfAPIv1alpha2.AddToScheme(scheme)) utilruntime.Must(apiext.AddToScheme(scheme)) utilruntime.Must(appsv1.AddToScheme(scheme)) + utilruntime.Must(authv1.AddToScheme(scheme)) } func StartManager(cfg config.Config) error { @@ -172,13 +174,21 @@ func StartManager(cfg config.Config) error { }) statusQueue := status.NewQueue() + resetConnChan := make(chan struct{}) nginxUpdater := agent.NewNginxUpdater( cfg.Logger.WithName("nginxUpdater"), mgr.GetAPIReader(), statusQueue, + resetConnChan, cfg.Plus, ) + tokenAudience := fmt.Sprintf( + "%s.%s.svc", + cfg.GatewayPodConfig.ServiceName, + cfg.GatewayPodConfig.Namespace, + ) + grpcServer := agentgrpc.NewServer( cfg.Logger.WithName("agentGRPCServer"), grpcServerPort, @@ -186,6 +196,9 @@ func StartManager(cfg config.Config) error { nginxUpdater.CommandService.Register, nginxUpdater.FileService.Register, }, + mgr.GetClient(), + tokenAudience, + resetConnChan, ) if err = mgr.Add(&runnables.LeaderOrNonLeader{Runnable: grpcServer}); err != nil { @@ -202,6 +215,7 @@ func StartManager(cfg config.Config) error { EventRecorder: recorder, GatewayPodConfig: &cfg.GatewayPodConfig, GCName: cfg.GatewayClassName, + AgentTLSSecretName: cfg.AgentTLSSecretName, Plus: cfg.Plus, NginxDockerSecretNames: cfg.NginxDockerSecretNames, PlusUsageConfig: &cfg.UsageReportConfig, @@ -362,6 +376,19 @@ func createManager(cfg config.Config, healthChecker *graphBuiltHealthChecker) (m } } + // Add an indexer to get pods by their IP address. This is used when validating that an agent + // connection is coming from the right place. + var podIPIndexFunc client.IndexerFunc = index.PodIPIndexFunc + if err := controller.AddIndex( + context.Background(), + mgr.GetFieldIndexer(), + &apiv1.Pod{}, + "status.podIP", + podIPIndexFunc, + ); err != nil { + return nil, fmt.Errorf("error adding pod IP indexer: %w", err) + } + return mgr, nil } diff --git a/internal/mode/static/nginx/agent/agent.go b/internal/mode/static/nginx/agent/agent.go index 1d839c3bc9..7bc0818214 100644 --- a/internal/mode/static/nginx/agent/agent.go +++ b/internal/mode/static/nginx/agent/agent.go @@ -46,6 +46,7 @@ func NewNginxUpdater( logger logr.Logger, reader client.Reader, statusQueue *status.Queue, + resetConnChan <-chan struct{}, plus bool, ) *NginxUpdaterImpl { connTracker := agentgrpc.NewConnectionsTracker() @@ -57,6 +58,7 @@ func NewNginxUpdater( nginxDeployments, connTracker, statusQueue, + resetConnChan, ) fileService := newFileService(logger.WithName("fileService"), nginxDeployments, connTracker) diff --git a/internal/mode/static/nginx/agent/agent_test.go b/internal/mode/static/nginx/agent/agent_test.go index b159d5be5c..3266003981 100644 --- a/internal/mode/static/nginx/agent/agent_test.go +++ b/internal/mode/static/nginx/agent/agent_test.go @@ -51,7 +51,7 @@ func TestUpdateConfig(t *testing.T) { fakeBroadcaster.SendReturns(test.configApplied) plus := false - updater := NewNginxUpdater(logr.Discard(), fake.NewFakeClient(), &status.Queue{}, plus) + updater := NewNginxUpdater(logr.Discard(), fake.NewFakeClient(), &status.Queue{}, nil, plus) deployment := &Deployment{ broadcaster: fakeBroadcaster, podStatuses: make(map[string]error), @@ -142,7 +142,7 @@ func TestUpdateUpstreamServers(t *testing.T) { fakeBroadcaster := &broadcastfakes.FakeBroadcaster{} fakeBroadcaster.SendReturns(test.configApplied) - updater := NewNginxUpdater(logr.Discard(), fake.NewFakeClient(), &status.Queue{}, test.plus) + updater := NewNginxUpdater(logr.Discard(), fake.NewFakeClient(), &status.Queue{}, nil, test.plus) updater.retryTimeout = 0 deployment := &Deployment{ diff --git a/internal/mode/static/nginx/agent/command.go b/internal/mode/static/nginx/agent/command.go index d5be137cd4..8f694e581d 100644 --- a/internal/mode/static/nginx/agent/command.go +++ b/internal/mode/static/nginx/agent/command.go @@ -36,6 +36,7 @@ type commandService struct { pb.CommandServiceServer nginxDeployments *DeploymentStore statusQueue *status.Queue + resetConnChan <-chan struct{} connTracker agentgrpc.ConnectionsTracker k8sReader client.Reader logger logr.Logger @@ -48,6 +49,7 @@ func newCommandService( depStore *DeploymentStore, connTracker agentgrpc.ConnectionsTracker, statusQueue *status.Queue, + resetConnChan <-chan struct{}, ) *commandService { return &commandService{ connectionTimeout: connectionWaitTimeout, @@ -56,6 +58,7 @@ func newCommandService( connTracker: connTracker, nginxDeployments: depStore, statusQueue: statusQueue, + resetConnChan: resetConnChan, } } @@ -158,7 +161,7 @@ func (cs *commandService) Subscribe(in pb.CommandService_SubscribeServer) error // `updateNginxConfig`. The entire transaction (as described in above in the function comment) // must be locked to prevent the deployment files from changing during the transaction. // This means that the lock is held until we receive either an error or response from agent - // (via msgr.Errors() or msgr.Mesages()) and respond back, finally returning to the event handler + // (via msgr.Errors() or msgr.Messages()) and respond back, finally returning to the event handler // which releases the lock. select { case <-ctx.Done(): @@ -167,6 +170,8 @@ func (cs *commandService) Subscribe(in pb.CommandService_SubscribeServer) error default: } return grpcStatus.Error(codes.Canceled, context.Cause(ctx).Error()) + case <-cs.resetConnChan: + return grpcStatus.Error(codes.Unavailable, "TLS files updated") case msg := <-channels.ListenCh: var req *pb.ManagementPlaneRequest switch msg.Type { diff --git a/internal/mode/static/nginx/agent/command_test.go b/internal/mode/static/nginx/agent/command_test.go index 714ffaafe5..167aab860a 100644 --- a/internal/mode/static/nginx/agent/command_test.go +++ b/internal/mode/static/nginx/agent/command_test.go @@ -208,6 +208,7 @@ func TestCreateConnection(t *testing.T) { NewDeploymentStore(&connTracker), &connTracker, status.NewQueue(), + nil, ) resp, err := cs.CreateConnection(test.ctx, test.request) @@ -304,6 +305,7 @@ func TestSubscribe(t *testing.T) { store, &connTracker, status.NewQueue(), + nil, ) broadcaster := &broadcastfakes.FakeBroadcaster{} @@ -412,6 +414,78 @@ func TestSubscribe(t *testing.T) { g.Expect(deployment.podStatuses).ToNot(HaveKey("nginx-pod")) } +func TestSubscribe_Reset(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + connTracker := agentgrpcfakes.FakeConnectionsTracker{} + conn := agentgrpc.Connection{ + Parent: types.NamespacedName{Namespace: "test", Name: "nginx-deployment"}, + PodName: "nginx-pod", + InstanceID: "nginx-id", + } + connTracker.GetConnectionReturns(conn) + + store := NewDeploymentStore(&connTracker) + resetChan := make(chan struct{}) + cs := newCommandService( + logr.Discard(), + fake.NewFakeClient(), + store, + &connTracker, + status.NewQueue(), + resetChan, + ) + + broadcaster := &broadcastfakes.FakeBroadcaster{} + responseCh := make(chan struct{}) + listenCh := make(chan broadcast.NginxAgentMessage, 2) + subChannels := broadcast.SubscriberChannels{ + ListenCh: listenCh, + ResponseCh: responseCh, + } + broadcaster.SubscribeReturns(subChannels) + + // set the initial files to be applied by the Subscription + deployment := store.StoreWithBroadcaster(conn.Parent, broadcaster) + files := []File{ + { + Meta: &pb.FileMeta{ + Name: "nginx.conf", + Hash: "12345", + }, + Contents: []byte("file contents"), + }, + } + deployment.SetFiles(files) + + ctx, cancel := createGrpcContextWithCancel() + defer cancel() + + mockServer := newMockSubscribeServer(ctx) + + // start the Subscriber + errCh := make(chan error) + go func() { + errCh <- cs.Subscribe(mockServer) + }() + + // ensure initial config is read to unblock read channel + mockServer.recvChan <- &pb.DataPlaneResponse{ + CommandResponse: &pb.CommandResponse{ + Status: pb.CommandResponse_COMMAND_STATUS_OK, + }, + } + + resetChan <- struct{}{} + + g.Eventually(func() error { + err := <-errCh + g.Expect(err).To(HaveOccurred()) + return err + }).Should(MatchError(ContainSubstring("TLS files updated"))) +} + func TestSubscribe_Errors(t *testing.T) { t.Parallel() @@ -465,6 +539,7 @@ func TestSubscribe_Errors(t *testing.T) { NewDeploymentStore(&connTracker), &connTracker, status.NewQueue(), + nil, ) if test.setup != nil { @@ -580,6 +655,7 @@ func TestSetInitialConfig_Errors(t *testing.T) { NewDeploymentStore(&connTracker), &connTracker, status.NewQueue(), + nil, ) conn := &agentgrpc.Connection{ @@ -765,6 +841,7 @@ func TestGetPodOwner(t *testing.T) { NewDeploymentStore(nil), nil, status.NewQueue(), + nil, ) owner, err := cs.getPodOwner(test.podName) @@ -864,6 +941,7 @@ func TestUpdateDataPlaneStatus(t *testing.T) { NewDeploymentStore(&connTracker), &connTracker, status.NewQueue(), + nil, ) resp, err := cs.UpdateDataPlaneStatus(test.ctx, test.request) @@ -902,6 +980,7 @@ func TestUpdateDataPlaneHealth(t *testing.T) { NewDeploymentStore(&connTracker), &connTracker, status.NewQueue(), + nil, ) resp, err := cs.UpdateDataPlaneHealth(context.Background(), &pb.UpdateDataPlaneHealthRequest{}) diff --git a/internal/mode/static/nginx/agent/grpc/connections.go b/internal/mode/static/nginx/agent/grpc/connections.go index e0534a78f9..0bae634ccc 100644 --- a/internal/mode/static/nginx/agent/grpc/connections.go +++ b/internal/mode/static/nginx/agent/grpc/connections.go @@ -48,9 +48,6 @@ func NewConnectionsTracker() ConnectionsTracker { } // Track adds a connection to the tracking map. -// TODO(sberman): we need to handle the case when the token expires (once we support the token). -// This likely involves setting a callback to cancel a context when the token expires, which triggers -// the connection to be removed from the tracking list. func (c *AgentConnectionsTracker) Track(key string, conn Connection) { c.lock.Lock() defer c.lock.Unlock() diff --git a/internal/mode/static/nginx/agent/grpc/context/context.go b/internal/mode/static/nginx/agent/grpc/context/context.go index f8daf457eb..a3bb0d3642 100644 --- a/internal/mode/static/nginx/agent/grpc/context/context.go +++ b/internal/mode/static/nginx/agent/grpc/context/context.go @@ -6,6 +6,7 @@ import ( // GrpcInfo for storing identity information for the gRPC client. type GrpcInfo struct { + Token string `json:"token"` // auth token that was provided by the gRPC client IPAddress string `json:"ip_address"` // ip address of the agent } diff --git a/internal/mode/static/nginx/agent/grpc/filewatcher/doc.go b/internal/mode/static/nginx/agent/grpc/filewatcher/doc.go new file mode 100644 index 0000000000..cd54f18c44 --- /dev/null +++ b/internal/mode/static/nginx/agent/grpc/filewatcher/doc.go @@ -0,0 +1,4 @@ +/* +Package filewatcher contains the functions to watch for TLS file updates for the gRPC server. +*/ +package filewatcher diff --git a/internal/mode/static/nginx/agent/grpc/filewatcher/filewatcher.go b/internal/mode/static/nginx/agent/grpc/filewatcher/filewatcher.go new file mode 100644 index 0000000000..2d79f4047c --- /dev/null +++ b/internal/mode/static/nginx/agent/grpc/filewatcher/filewatcher.go @@ -0,0 +1,106 @@ +package filewatcher + +import ( + "context" + "fmt" + "sync/atomic" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/go-logr/logr" +) + +const monitoringInterval = 5 * time.Second + +var emptyEvent = fsnotify.Event{ + Name: "", + Op: 0, +} + +// FileWatcher watches for changes to files and notifies the channel when a change occurs. +type FileWatcher struct { + filesChanged *atomic.Bool + watcher *fsnotify.Watcher + notifyCh chan<- struct{} + logger logr.Logger + filesToWatch []string + interval time.Duration +} + +// NewFileWatcher creates a new FileWatcher instance. +func NewFileWatcher(logger logr.Logger, files []string, notifyCh chan<- struct{}) (*FileWatcher, error) { + filesChanged := &atomic.Bool{} + + watcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, fmt.Errorf("failed to initialize TLS file watcher: %w", err) + } + + return &FileWatcher{ + filesChanged: filesChanged, + watcher: watcher, + logger: logger, + filesToWatch: files, + notifyCh: notifyCh, + interval: monitoringInterval, + }, nil +} + +// Watch starts the watch for file changes. +func (w *FileWatcher) Watch(ctx context.Context) { + w.logger.V(1).Info("Starting file watcher") + + ticker := time.NewTicker(w.interval) + for _, file := range w.filesToWatch { + w.addWatcher(file) + } + + for { + select { + case <-ctx.Done(): + if err := w.watcher.Close(); err != nil { + w.logger.Error(err, "unable to close file watcher") + } + return + case event := <-w.watcher.Events: + w.handleEvent(event) + case <-ticker.C: + w.checkForUpdates() + case err := <-w.watcher.Errors: + w.logger.Error(err, "error watching file") + } + } +} + +func (w *FileWatcher) addWatcher(path string) { + if err := w.watcher.Add(path); err != nil { + w.logger.Error(err, "failed to watch file", "file", path) + } +} + +func (w *FileWatcher) handleEvent(event fsnotify.Event) { + if isEventSkippable(event) { + return + } + + if event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename) { + w.addWatcher(event.Name) + } + + w.filesChanged.Store(true) +} + +func (w *FileWatcher) checkForUpdates() { + if w.filesChanged.Load() { + w.logger.Info("TLS files changed, sending notification to reset nginx agent connections") + w.notifyCh <- struct{}{} + w.filesChanged.Store(false) + } +} + +func isEventSkippable(event fsnotify.Event) bool { + return event == emptyEvent || + event.Name == "" || + event.Has(fsnotify.Chmod) || + event.Has(fsnotify.Create) +} diff --git a/internal/mode/static/nginx/agent/grpc/filewatcher/filewatcher_test.go b/internal/mode/static/nginx/agent/grpc/filewatcher/filewatcher_test.go new file mode 100644 index 0000000000..1840e78849 --- /dev/null +++ b/internal/mode/static/nginx/agent/grpc/filewatcher/filewatcher_test.go @@ -0,0 +1,69 @@ +package filewatcher + +import ( + "context" + "os" + "path" + "testing" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/go-logr/logr" + . "github.com/onsi/gomega" +) + +func TestFileWatcher_Watch(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + notifyCh := make(chan struct{}, 1) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + file := path.Join(os.TempDir(), "test-file") + _, err := os.Create(file) + g.Expect(err).ToNot(HaveOccurred()) + defer os.Remove(file) + + w, err := NewFileWatcher(logr.Discard(), []string{file}, notifyCh) + g.Expect(err).ToNot(HaveOccurred()) + w.interval = 300 * time.Millisecond + + go w.Watch(ctx) + + w.watcher.Events <- fsnotify.Event{Name: file, Op: fsnotify.Write} + g.Eventually(func() bool { + return w.filesChanged.Load() + }).Should(BeTrue()) + + g.Eventually(notifyCh).Should(Receive()) +} + +func TestFileWatcher_handleEvent(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + w, err := NewFileWatcher(logr.Discard(), []string{"test-file"}, nil) + g.Expect(err).ToNot(HaveOccurred()) + + w.handleEvent(fsnotify.Event{Op: fsnotify.Write}) + g.Expect(w.filesChanged.Load()).To(BeFalse()) + + w.handleEvent(fsnotify.Event{Name: "test-chmod", Op: fsnotify.Chmod}) + g.Expect(w.filesChanged.Load()).To(BeFalse()) + + w.handleEvent(fsnotify.Event{Name: "test-create", Op: fsnotify.Create}) + g.Expect(w.filesChanged.Load()).To(BeFalse()) + + w.handleEvent(fsnotify.Event{Name: "test-write", Op: fsnotify.Write}) + g.Expect(w.filesChanged.Load()).To(BeTrue()) + w.filesChanged.Store(false) + + w.handleEvent(fsnotify.Event{Name: "test-remove", Op: fsnotify.Remove}) + g.Expect(w.filesChanged.Load()).To(BeTrue()) + w.filesChanged.Store(false) + + w.handleEvent(fsnotify.Event{Name: "test-rename", Op: fsnotify.Rename}) + g.Expect(w.filesChanged.Load()).To(BeTrue()) + w.filesChanged.Store(false) +} diff --git a/internal/mode/static/nginx/agent/grpc/grpc.go b/internal/mode/static/nginx/agent/grpc/grpc.go index a4f2a31268..f995756584 100644 --- a/internal/mode/static/nginx/agent/grpc/grpc.go +++ b/internal/mode/static/nginx/agent/grpc/grpc.go @@ -2,31 +2,41 @@ package grpc import ( "context" + "crypto/tls" + "crypto/x509" + "errors" "fmt" "net" + "os" "time" "github.com/go-logr/logr" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/status" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/filewatcher" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/interceptor" ) const ( keepAliveTime = 15 * time.Second keepAliveTimeout = 10 * time.Second + caCertPath = "/var/run/secrets/ngf/ca.crt" + tlsCertPath = "/var/run/secrets/ngf/tls.crt" + tlsKeyPath = "/var/run/secrets/ngf/tls.key" ) var ErrStatusInvalidConnection = status.Error(codes.Unauthenticated, "invalid connection") // Interceptor provides hooks to intercept the execution of an RPC on the server. type Interceptor interface { - Stream() grpc.StreamServerInterceptor - Unary() grpc.UnaryServerInterceptor + Stream(logr.Logger) grpc.StreamServerInterceptor + Unary(logr.Logger) grpc.UnaryServerInterceptor } // Server is a gRPC server for communicating with the nginx agent. @@ -35,6 +45,10 @@ type Server struct { interceptor Interceptor logger logr.Logger + + // resetConnChan is used by the filewatcher to trigger the Command service to + // reset any connections when TLS files are updated. + resetConnChan chan<- struct{} // RegisterServices is a list of functions to register gRPC services to the gRPC server. registerServices []func(*grpc.Server) // Port is the port that the server is listening on. @@ -42,12 +56,20 @@ type Server struct { port int } -func NewServer(logger logr.Logger, port int, registerSvcs []func(*grpc.Server)) *Server { +func NewServer( + logger logr.Logger, + port int, + registerSvcs []func(*grpc.Server), + k8sClient client.Client, + tokenAudience string, + resetConnChan chan<- struct{}, +) *Server { return &Server{ logger: logger, port: port, registerServices: registerSvcs, - interceptor: interceptor.NewContextSetter(), + interceptor: interceptor.NewContextSetter(k8sClient, tokenAudience), + resetConnChan: resetConnChan, } } @@ -58,6 +80,11 @@ func (g *Server) Start(ctx context.Context) error { return err } + tlsCredentials, err := getTLSConfig() + if err != nil { + return err + } + server := grpc.NewServer( grpc.KeepaliveParams( keepalive.ServerParameters{ @@ -71,14 +98,23 @@ func (g *Server) Start(ctx context.Context) error { PermitWithoutStream: true, }, ), - grpc.ChainStreamInterceptor(g.interceptor.Stream()), - grpc.ChainUnaryInterceptor(g.interceptor.Unary()), + grpc.ChainStreamInterceptor(g.interceptor.Stream(g.logger)), + grpc.ChainUnaryInterceptor(g.interceptor.Unary(g.logger)), + grpc.Creds(tlsCredentials), ) for _, registerSvc := range g.registerServices { registerSvc(server) } + tlsFiles := []string{caCertPath, tlsCertPath, tlsKeyPath} + fileWatcher, err := filewatcher.NewFileWatcher(g.logger.WithName("fileWatcher"), tlsFiles, g.resetConnChan) + if err != nil { + return err + } + + go fileWatcher.Watch(ctx) + go func() { <-ctx.Done() g.logger.Info("Shutting down GRPC Server") @@ -89,4 +125,30 @@ func (g *Server) Start(ctx context.Context) error { return server.Serve(listener) } +func getTLSConfig() (credentials.TransportCredentials, error) { + caPem, err := os.ReadFile(caCertPath) + if err != nil { + return nil, err + } + + certPool := x509.NewCertPool() + if !certPool.AppendCertsFromPEM(caPem) { + return nil, errors.New("error parsing CA PEM") + } + + getCertificateCallback := func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + serverCert, err := tls.LoadX509KeyPair(tlsCertPath, tlsKeyPath) + return &serverCert, err + } + + tlsConfig := &tls.Config{ + GetCertificate: getCertificateCallback, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: certPool, + MinVersion: tls.VersionTLS13, + } + + return credentials.NewTLS(tlsConfig), nil +} + var _ manager.Runnable = &Server{} diff --git a/internal/mode/static/nginx/agent/grpc/interceptor/interceptor.go b/internal/mode/static/nginx/agent/grpc/interceptor/interceptor.go index 3139da3cec..87517c5875 100644 --- a/internal/mode/static/nginx/agent/grpc/interceptor/interceptor.go +++ b/internal/mode/static/nginx/agent/grpc/interceptor/interceptor.go @@ -4,15 +4,29 @@ import ( "context" "fmt" "net" + "strings" + "time" + "github.com/go-logr/logr" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" "google.golang.org/grpc/status" + authv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" grpcContext "github.com/nginx/nginx-gateway-fabric/internal/mode/static/nginx/agent/grpc/context" ) +const ( + headerUUID = "uuid" + headerAuth = "authorization" +) + // streamHandler is a struct that implements StreamHandler, allowing the interceptor to replace the context. type streamHandler struct { grpc.ServerStream @@ -23,21 +37,28 @@ func (sh *streamHandler) Context() context.Context { return sh.ctx } -type ContextSetter struct{} +type ContextSetter struct { + k8sClient client.Client + audience string +} -func NewContextSetter() ContextSetter { - return ContextSetter{} +func NewContextSetter(k8sClient client.Client, audience string) ContextSetter { + return ContextSetter{ + k8sClient: k8sClient, + audience: audience, + } } -func (c ContextSetter) Stream() grpc.StreamServerInterceptor { +func (c ContextSetter) Stream(logger logr.Logger) grpc.StreamServerInterceptor { return func( - srv interface{}, + srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler, ) error { - ctx, err := setContext(ss.Context()) + ctx, err := c.validateConnection(ss.Context()) if err != nil { + logger.Error(err, "error validating connection") return err } return handler(srv, &streamHandler{ @@ -47,24 +68,48 @@ func (c ContextSetter) Stream() grpc.StreamServerInterceptor { } } -func (c ContextSetter) Unary() grpc.UnaryServerInterceptor { +func (c ContextSetter) Unary(logger logr.Logger) grpc.UnaryServerInterceptor { return func( ctx context.Context, - req interface{}, + req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler, - ) (resp interface{}, err error) { - if ctx, err = setContext(ctx); err != nil { + ) (resp any, err error) { + if ctx, err = c.validateConnection(ctx); err != nil { + logger.Error(err, "error validating connection") return nil, err } return handler(ctx, req) } } -// TODO(sberman): for now, we'll just use the IP address of the agent to link a Connection -// to a Subscription by setting it in the context. Once we support auth, we can likely change this -// interceptor to instead set the uuid. -func setContext(ctx context.Context) (context.Context, error) { +// validateConnection checks that the connection is valid and returns a new +// context containing information used by the gRPC command/file services. +func (c ContextSetter) validateConnection(ctx context.Context) (context.Context, error) { + gi, err := getGrpcInfo(ctx) + if err != nil { + return nil, err + } + + return c.validateToken(ctx, gi) +} + +func getGrpcInfo(ctx context.Context) (*grpcContext.GrpcInfo, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Error(codes.InvalidArgument, "no metadata") + } + + id := md.Get(headerUUID) + if len(id) == 0 { + return nil, status.Error(codes.Unauthenticated, "no identity") + } + + auths := md.Get(headerAuth) + if len(auths) == 0 { + return nil, status.Error(codes.Unauthenticated, "no authorization") + } + p, ok := peer.FromContext(ctx) if !ok { return nil, status.Error(codes.InvalidArgument, "no peer data") @@ -75,8 +120,76 @@ func setContext(ctx context.Context) (context.Context, error) { panic(fmt.Sprintf("address %q was not of type net.TCPAddr", p.Addr.String())) } - gi := &grpcContext.GrpcInfo{ + return &grpcContext.GrpcInfo{ + Token: auths[0], IPAddress: addr.IP.String(), + }, nil +} + +func (c ContextSetter) validateToken(ctx context.Context, gi *grpcContext.GrpcInfo) (context.Context, error) { + tokenReview := &authv1.TokenReview{ + Spec: authv1.TokenReviewSpec{ + Audiences: []string{c.audience}, + Token: gi.Token, + }, + } + + createCtx, createCancel := context.WithTimeout(ctx, 30*time.Second) + defer createCancel() + + if err := c.k8sClient.Create(createCtx, tokenReview); err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("error creating TokenReview: %v", err)) + } + + if !tokenReview.Status.Authenticated { + return nil, status.Error(codes.Unauthenticated, fmt.Sprintf("invalid authorization: %s", tokenReview.Status.Error)) + } + + usernameItems := strings.Split(tokenReview.Status.User.Username, ":") + if len(usernameItems) != 4 || usernameItems[0] != "system" || usernameItems[1] != "serviceaccount" { + msg := fmt.Sprintf( + "token username must be of the format 'system:serviceaccount:NAMESPACE:NAME': %s", + tokenReview.Status.User.Username, + ) + return nil, status.Error(codes.Unauthenticated, msg) + } + + getCtx, getCancel := context.WithTimeout(ctx, 30*time.Second) + defer getCancel() + + var podList corev1.PodList + opts := &client.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{"status.podIP": gi.IPAddress}), + } + + if err := c.k8sClient.List(getCtx, &podList, opts); err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("error listing pods: %s", err.Error())) + } + + if len(podList.Items) != 1 { + msg := fmt.Sprintf("expected one Pod to have IP address %s, found %d", gi.IPAddress, len(podList.Items)) + return nil, status.Error(codes.Internal, msg) + } + + podNS := podList.Items[0].GetNamespace() + if podNS != usernameItems[2] { + msg := fmt.Sprintf( + "token user namespace %q does not match namespace of requesting pod %q", usernameItems[2], podNS, + ) + return nil, status.Error(codes.Unauthenticated, msg) + } + + scName, ok := podList.Items[0].GetLabels()[controller.AppNameLabel] + if !ok { + msg := fmt.Sprintf("could not get app name from %q label; unable to authenticate token", controller.AppNameLabel) + return nil, status.Error(codes.Unauthenticated, msg) + } + + if scName != usernameItems[3] { + msg := fmt.Sprintf( + "token user name %q does not match service account name of requesting pod %q", usernameItems[3], scName, + ) + return nil, status.Error(codes.Unauthenticated, msg) } return grpcContext.NewGrpcContext(ctx, *gi), nil diff --git a/internal/mode/static/nginx/agent/grpc/interceptor/interceptor_test.go b/internal/mode/static/nginx/agent/grpc/interceptor/interceptor_test.go new file mode 100644 index 0000000000..04eda6ad50 --- /dev/null +++ b/internal/mode/static/nginx/agent/grpc/interceptor/interceptor_test.go @@ -0,0 +1,292 @@ +package interceptor + +import ( + "context" + "errors" + "net" + "testing" + + "github.com/go-logr/logr" + . "github.com/onsi/gomega" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + authv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nginx/nginx-gateway-fabric/internal/framework/controller" +) + +type mockServerStream struct { + grpc.ServerStream + ctx context.Context +} + +func (m *mockServerStream) Context() context.Context { + return m.ctx +} + +type mockClient struct { + client.Client + createErr, listErr error + username, appName, podNamespace string + authenticated bool +} + +func (m *mockClient) Create(_ context.Context, obj client.Object, _ ...client.CreateOption) error { + tr, ok := obj.(*authv1.TokenReview) + if !ok { + return errors.New("couldn't convert object to TokenReview") + } + tr.Status.Authenticated = m.authenticated + tr.Status.User = authv1.UserInfo{Username: m.username} + + return m.createErr +} + +func (m *mockClient) List(_ context.Context, obj client.ObjectList, _ ...client.ListOption) error { + podList, ok := obj.(*corev1.PodList) + if !ok { + return errors.New("couldn't convert object to PodList") + } + + var labels map[string]string + if m.appName != "" { + labels = map[string]string{ + controller.AppNameLabel: m.appName, + } + } + + podList.Items = []corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: m.podNamespace, + Labels: labels, + }, + }, + } + + return m.listErr +} + +func TestInterceptor(t *testing.T) { + t.Parallel() + + validMetadata := metadata.New(map[string]string{ + headerUUID: "test-uuid", + headerAuth: "test-token", + }) + validPeerData := &peer.Peer{ + Addr: &net.TCPAddr{IP: net.ParseIP("127.0.0.1")}, + } + + tests := []struct { + md metadata.MD + peer *peer.Peer + createErr error + listErr error + username string + appName string + podNamespace string + name string + expErrMsg string + authenticated bool + expErrCode codes.Code + }{ + { + name: "valid request", + md: validMetadata, + peer: validPeerData, + username: "system:serviceaccount:default:gateway-nginx", + appName: "gateway-nginx", + podNamespace: "default", + authenticated: true, + expErrCode: codes.OK, + }, + { + name: "missing metadata", + peer: validPeerData, + authenticated: true, + expErrCode: codes.InvalidArgument, + expErrMsg: "no metadata", + }, + { + name: "missing uuid", + md: metadata.New(map[string]string{ + headerAuth: "test-token", + }), + peer: validPeerData, + authenticated: true, + expErrCode: codes.Unauthenticated, + expErrMsg: "no identity", + }, + { + name: "missing authorization", + md: metadata.New(map[string]string{ + headerUUID: "test-uuid", + }), + peer: validPeerData, + authenticated: true, + createErr: nil, + expErrCode: codes.Unauthenticated, + expErrMsg: "no authorization", + }, + { + name: "missing peer data", + md: validMetadata, + authenticated: true, + expErrCode: codes.InvalidArgument, + expErrMsg: "no peer data", + }, + { + name: "tokenreview not created", + md: validMetadata, + peer: validPeerData, + authenticated: true, + createErr: errors.New("not created"), + expErrCode: codes.Internal, + expErrMsg: "error creating TokenReview", + }, + { + name: "tokenreview created and not authenticated", + md: validMetadata, + peer: validPeerData, + authenticated: false, + expErrCode: codes.Unauthenticated, + expErrMsg: "invalid authorization", + }, + { + name: "error listing pods", + md: validMetadata, + peer: validPeerData, + username: "system:serviceaccount:default:gateway-nginx", + appName: "gateway-nginx", + podNamespace: "default", + authenticated: true, + listErr: errors.New("can't list"), + expErrCode: codes.Internal, + expErrMsg: "error listing pods", + }, + { + name: "invalid username length", + md: validMetadata, + peer: validPeerData, + username: "serviceaccount:default:gateway-nginx", + appName: "gateway-nginx", + podNamespace: "default", + authenticated: true, + expErrCode: codes.Unauthenticated, + expErrMsg: "must be of the format", + }, + { + name: "missing system from username", + md: validMetadata, + peer: validPeerData, + username: "invalid:serviceaccount:default:gateway-nginx", + appName: "gateway-nginx", + podNamespace: "default", + authenticated: true, + expErrCode: codes.Unauthenticated, + expErrMsg: "must be of the format", + }, + { + name: "missing serviceaccount from username", + md: validMetadata, + peer: validPeerData, + username: "system:invalid:default:gateway-nginx", + appName: "gateway-nginx", + podNamespace: "default", + authenticated: true, + expErrCode: codes.Unauthenticated, + expErrMsg: "must be of the format", + }, + { + name: "mismatched namespace in username", + md: validMetadata, + peer: validPeerData, + username: "system:serviceaccount:invalid:gateway-nginx", + appName: "gateway-nginx", + podNamespace: "default", + authenticated: true, + expErrCode: codes.Unauthenticated, + expErrMsg: "does not match namespace", + }, + { + name: "mismatched name in username", + md: validMetadata, + peer: validPeerData, + username: "system:serviceaccount:default:invalid", + appName: "gateway-nginx", + podNamespace: "default", + authenticated: true, + expErrCode: codes.Unauthenticated, + expErrMsg: "does not match service account name", + }, + { + name: "missing app name label", + md: validMetadata, + peer: validPeerData, + username: "system:serviceaccount:default:gateway-nginx", + podNamespace: "default", + authenticated: true, + expErrCode: codes.Unauthenticated, + expErrMsg: "could not get app name", + }, + } + + streamHandler := func(_ any, _ grpc.ServerStream) error { + return nil + } + + unaryHandler := func(_ context.Context, _ any) (any, error) { + return nil, nil //nolint:nilnil // unit test + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + mockK8sClient := &mockClient{ + authenticated: test.authenticated, + createErr: test.createErr, + listErr: test.listErr, + username: test.username, + appName: test.appName, + podNamespace: test.podNamespace, + } + cs := NewContextSetter(mockK8sClient, "ngf-audience") + + ctx := context.Background() + if test.md != nil { + peerCtx := context.Background() + if test.peer != nil { + peerCtx = peer.NewContext(context.Background(), test.peer) + } + ctx = metadata.NewIncomingContext(peerCtx, test.md) + } + + stream := &mockServerStream{ctx: ctx} + + err := cs.Stream(logr.Discard())(nil, stream, nil, streamHandler) + if test.expErrCode != codes.OK { + g.Expect(status.Code(err)).To(Equal(test.expErrCode)) + g.Expect(err.Error()).To(ContainSubstring(test.expErrMsg)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + + _, err = cs.Unary(logr.Discard())(ctx, nil, nil, unaryHandler) + if test.expErrCode != codes.OK { + g.Expect(status.Code(err)).To(Equal(test.expErrCode)) + g.Expect(err.Error()).To(ContainSubstring(test.expErrMsg)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} diff --git a/internal/mode/static/provisioner/eventloop.go b/internal/mode/static/provisioner/eventloop.go index 6b50e79e38..e03a79ade8 100644 --- a/internal/mode/static/provisioner/eventloop.go +++ b/internal/mode/static/provisioner/eventloop.go @@ -29,11 +29,13 @@ func newEventLoop( selector metav1.LabelSelector, ngfNamespace string, dockerSecrets []string, + agentTLSSecret string, usageConfig *config.UsageReportConfig, ) (*events.EventLoop, error) { nginxResourceLabelPredicate := predicate.NginxLabelPredicate(selector) - secretsToWatch := make([]string, 0, len(dockerSecrets)+3) + secretsToWatch := make([]string, 0, len(dockerSecrets)+4) + secretsToWatch = append(secretsToWatch, agentTLSSecret) secretsToWatch = append(secretsToWatch, dockerSecrets...) if usageConfig != nil { diff --git a/internal/mode/static/provisioner/handler.go b/internal/mode/static/provisioner/handler.go index 7757cec043..5058d25771 100644 --- a/internal/mode/static/provisioner/handler.go +++ b/internal/mode/static/provisioner/handler.go @@ -171,7 +171,7 @@ func (h *eventHandler) provisionResources( gatewayNSName types.NamespacedName, ) error { resources := h.store.getNginxResourcesForGateway(gatewayNSName) - if resources.Gateway != nil { + if resources != nil && resources.Gateway != nil { resourceName := controller.CreateNginxResourceName(gatewayNSName.Name, h.gcName) if err := h.provisioner.provisionNginx( ctx, @@ -229,6 +229,13 @@ func (h *eventHandler) deprovisionSecretsForAllGateways(ctx context.Context, sec } switch { + case strings.HasSuffix(resources.AgentTLSSecret.Name, secret): + if err := h.provisioner.deleteSecret( + ctx, + controller.ObjectMetaToNamespacedName(resources.AgentTLSSecret), + ); err != nil { + allErrs = append(allErrs, err) + } case strings.HasSuffix(resources.PlusJWTSecret.Name, secret): if err := h.provisioner.deleteSecret( ctx, diff --git a/internal/mode/static/provisioner/handler_test.go b/internal/mode/static/provisioner/handler_test.go index fe1d63e9be..9fd0dcc8d1 100644 --- a/internal/mode/static/provisioner/handler_test.go +++ b/internal/mode/static/provisioner/handler_test.go @@ -23,7 +23,7 @@ func TestHandleEventBatch_Upsert(t *testing.T) { t.Parallel() g := NewWithT(t) - store := newStore([]string{dockerTestSecretName}, jwtTestSecretName, "", "") + store := newStore([]string{dockerTestSecretName}, "", jwtTestSecretName, "", "") provisioner, fakeClient, _ := defaultNginxProvisioner() provisioner.cfg.StatusQueue = status.NewQueue() @@ -196,7 +196,13 @@ func TestHandleEventBatch_Delete(t *testing.T) { t.Parallel() g := NewWithT(t) - store := newStore([]string{dockerTestSecretName}, jwtTestSecretName, caTestSecretName, clientTestSecretName) + store := newStore( + []string{dockerTestSecretName}, + agentTLSTestSecretName, + jwtTestSecretName, + caTestSecretName, + clientTestSecretName, + ) provisioner, fakeClient, _ := defaultNginxProvisioner() provisioner.cfg.StatusQueue = status.NewQueue() @@ -233,6 +239,14 @@ func TestHandleEventBatch_Delete(t *testing.T) { }, } + originalAgentTLSSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: agentTLSTestSecretName, + Namespace: ngfNamespace, + }, + } + g.Expect(fakeClient.Create(ctx, originalAgentTLSSecret)).To(Succeed()) + jwtSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "gw-nginx-" + jwtTestSecretName, @@ -311,6 +325,7 @@ func TestHandleEventBatch_Delete(t *testing.T) { g.Expect(fakeClient.Get(ctx, key, &corev1.Secret{})).ToNot(Succeed()) } + verifySecret(agentTLSTestSecretName, originalAgentTLSSecret) verifySecret(jwtTestSecretName, userJwtSecret) verifySecret(caTestSecretName, userCASecret) verifySecret(clientTestSecretName, userClientSSLSecret) diff --git a/internal/mode/static/provisioner/objects.go b/internal/mode/static/provisioner/objects.go index 5e9710efd5..b62dc3b1f4 100644 --- a/internal/mode/static/provisioner/objects.go +++ b/internal/mode/static/provisioner/objects.go @@ -50,6 +50,7 @@ func (p *NginxProvisioner) buildNginxResourceObjects( ngxIncludesConfigMapName := controller.CreateNginxResourceName(resourceName, nginxIncludesConfigMapNameSuffix) ngxAgentConfigMapName := controller.CreateNginxResourceName(resourceName, nginxAgentConfigMapNameSuffix) + agentTLSSecretName := controller.CreateNginxResourceName(resourceName, p.cfg.AgentTLSSecretName) var jwtSecretName, caSecretName, clientSSLSecretName string if p.cfg.Plus { @@ -98,6 +99,7 @@ func (p *NginxProvisioner) buildNginxResourceObjects( secrets, err := p.buildNginxSecrets( objectMeta, + agentTLSSecretName, dockerSecretNames, jwtSecretName, caSecretName, @@ -130,6 +132,7 @@ func (p *NginxProvisioner) buildNginxResourceObjects( ngxAgentConfigMapName, ports, selectorLabels, + agentTLSSecretName, dockerSecretNames, jwtSecretName, caSecretName, @@ -154,6 +157,7 @@ func (p *NginxProvisioner) buildNginxResourceObjects( func (p *NginxProvisioner) buildNginxSecrets( objectMeta metav1.ObjectMeta, + agentTLSSecretName string, dockerSecretNames map[string]string, jwtSecretName string, caSecretName string, @@ -162,6 +166,24 @@ func (p *NginxProvisioner) buildNginxSecrets( var secrets []client.Object var errs []error + if agentTLSSecretName != "" { + newSecret, err := p.getAndUpdateSecret( + p.cfg.AgentTLSSecretName, + metav1.ObjectMeta{ + Name: agentTLSSecretName, + Namespace: objectMeta.Namespace, + Labels: objectMeta.Labels, + Annotations: objectMeta.Annotations, + }, + corev1.SecretTypeTLS, + ) + if err != nil { + errs = append(errs, err) + } else { + secrets = append(secrets, newSecret) + } + } + for newName, origName := range dockerSecretNames { newSecret, err := p.getAndUpdateSecret( origName, @@ -171,6 +193,7 @@ func (p *NginxProvisioner) buildNginxSecrets( Labels: objectMeta.Labels, Annotations: objectMeta.Annotations, }, + corev1.SecretTypeDockerConfigJson, ) if err != nil { errs = append(errs, err) @@ -194,6 +217,7 @@ func (p *NginxProvisioner) buildNginxSecrets( Labels: objectMeta.Labels, Annotations: objectMeta.Annotations, }, + corev1.SecretTypeOpaque, ) if err != nil { errs = append(errs, err) @@ -211,6 +235,7 @@ func (p *NginxProvisioner) buildNginxSecrets( Labels: objectMeta.Labels, Annotations: objectMeta.Annotations, }, + corev1.SecretTypeOpaque, ) if err != nil { errs = append(errs, err) @@ -228,6 +253,7 @@ func (p *NginxProvisioner) buildNginxSecrets( Labels: objectMeta.Labels, Annotations: objectMeta.Annotations, }, + corev1.SecretTypeTLS, ) if err != nil { errs = append(errs, err) @@ -242,6 +268,7 @@ func (p *NginxProvisioner) buildNginxSecrets( func (p *NginxProvisioner) getAndUpdateSecret( name string, newObjectMeta metav1.ObjectMeta, + secretType corev1.SecretType, ) (*corev1.Secret, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -255,6 +282,7 @@ func (p *NginxProvisioner) getAndUpdateSecret( newSecret := &corev1.Secret{ ObjectMeta: newObjectMeta, Data: secret.Data, + Type: secretType, } return newSecret, nil @@ -402,6 +430,7 @@ func (p *NginxProvisioner) buildNginxDeployment( ngxAgentConfigMapName string, ports map[int32]struct{}, selectorLabels map[string]string, + agentTLSSecretName string, dockerSecretNames map[string]string, jwtSecretName string, caSecretName string, @@ -413,6 +442,7 @@ func (p *NginxProvisioner) buildNginxDeployment( ngxIncludesConfigMapName, ngxAgentConfigMapName, ports, + agentTLSSecretName, dockerSecretNames, jwtSecretName, caSecretName, @@ -420,7 +450,6 @@ func (p *NginxProvisioner) buildNginxDeployment( ) var object client.Object - // TODO(sberman): daemonset support deployment := &appsv1.Deployment{ ObjectMeta: objectMeta, Spec: appsv1.DeploymentSpec{ @@ -452,6 +481,7 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec( ngxIncludesConfigMapName string, ngxAgentConfigMapName string, ports map[int32]struct{}, + agentTLSSecretName string, dockerSecretNames map[string]string, jwtSecretName string, caSecretName string, @@ -491,6 +521,7 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec( }) image, pullPolicy := p.buildImage(nProxyCfg) + tokenAudience := fmt.Sprintf("%s.%s.svc", p.cfg.GatewayPodConfig.ServiceName, p.cfg.GatewayPodConfig.Namespace) spec := corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -509,7 +540,7 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec( Add: []corev1.Capability{"NET_BIND_SERVICE"}, Drop: []corev1.Capability{"ALL"}, }, - ReadOnlyRootFilesystem: helpers.GetPointer[bool](true), + ReadOnlyRootFilesystem: helpers.GetPointer(true), RunAsGroup: helpers.GetPointer[int64](1001), RunAsUser: helpers.GetPointer[int64](101), SeccompProfile: &corev1.SeccompProfile{ @@ -518,6 +549,8 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec( }, VolumeMounts: []corev1.VolumeMount{ {MountPath: "/etc/nginx-agent", Name: "nginx-agent"}, + {MountPath: "/var/run/secrets/ngf", Name: "nginx-agent-tls"}, + {MountPath: "/var/run/secrets/ngf/serviceaccount", Name: "token"}, {MountPath: "/var/log/nginx-agent", Name: "nginx-agent-log"}, {MountPath: "/etc/nginx/conf.d", Name: "nginx-conf"}, {MountPath: "/etc/nginx/stream-conf.d", Name: "nginx-stream-conf"}, @@ -562,7 +595,7 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec( Capabilities: &corev1.Capabilities{ Drop: []corev1.Capability{"ALL"}, }, - ReadOnlyRootFilesystem: helpers.GetPointer[bool](true), + ReadOnlyRootFilesystem: helpers.GetPointer(true), RunAsGroup: helpers.GetPointer[int64](1001), RunAsUser: helpers.GetPointer[int64](101), SeccompProfile: &corev1.SeccompProfile{ @@ -574,6 +607,21 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec( ImagePullSecrets: []corev1.LocalObjectReference{}, ServiceAccountName: objectMeta.Name, Volumes: []corev1.Volume{ + { + Name: "token", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Path: "token", + Audience: tokenAudience, + }, + }, + }, + }, + }, + }, {Name: "nginx-agent", VolumeSource: emptyDirVolumeSource}, { Name: "nginx-agent-config", @@ -585,6 +633,14 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec( }, }, }, + { + Name: "nginx-agent-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: agentTLSSecretName, + }, + }, + }, {Name: "nginx-agent-log", VolumeSource: emptyDirVolumeSource}, {Name: "nginx-conf", VolumeSource: emptyDirVolumeSource}, {Name: "nginx-stream-conf", VolumeSource: emptyDirVolumeSource}, @@ -790,6 +846,18 @@ func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName objects := []client.Object{deployment, service, serviceAccount, bootstrapCM, agentCM} + agentTLSSecretName := controller.CreateNginxResourceName( + deploymentNSName.Name, + p.cfg.AgentTLSSecretName, + ) + agentTLSSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: agentTLSSecretName, + Namespace: deploymentNSName.Namespace, + }, + } + objects = append(objects, agentTLSSecret) + for _, name := range p.cfg.NginxDockerSecretNames { newName := controller.CreateNginxResourceName(deploymentNSName.Name, name) secret := &corev1.Secret{ diff --git a/internal/mode/static/provisioner/objects_test.go b/internal/mode/static/provisioner/objects_test.go index f59f4ce253..907a7e9fad 100644 --- a/internal/mode/static/provisioner/objects_test.go +++ b/internal/mode/static/provisioner/objects_test.go @@ -26,19 +26,30 @@ func TestBuildNginxResourceObjects(t *testing.T) { t.Parallel() g := NewWithT(t) + agentTLSSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: agentTLSTestSecretName, + Namespace: ngfNamespace, + }, + Data: map[string][]byte{"tls.crt": []byte("tls")}, + } + fakeClient := fake.NewFakeClient(agentTLSSecret) + provisioner := &NginxProvisioner{ cfg: Config{ GatewayPodConfig: &config.GatewayPodConfig{ - Namespace: "default", + Namespace: ngfNamespace, Version: "1.0.0", Image: "ngf-image", }, + AgentTLSSecretName: agentTLSTestSecretName, }, baseLabelSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ "app": "nginx", }, }, + k8sClient: fakeClient, } gateway := &gatewayv1.Gateway{ @@ -83,7 +94,7 @@ func TestBuildNginxResourceObjects(t *testing.T) { objects, err := provisioner.buildNginxResourceObjects(resourceName, gateway, &graph.EffectiveNginxProxy{}) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(objects).To(HaveLen(5)) + g.Expect(objects).To(HaveLen(6)) validateLabelsAndAnnotations := func(obj client.Object) { g.Expect(obj.GetLabels()).To(Equal(expLabels)) @@ -95,7 +106,16 @@ func TestBuildNginxResourceObjects(t *testing.T) { validateLabelsAndAnnotations(obj) } - cmObj := objects[0] + secretObj := objects[0] + secret, ok := secretObj.(*corev1.Secret) + g.Expect(ok).To(BeTrue()) + g.Expect(secret.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, agentTLSTestSecretName))) + g.Expect(secret.GetLabels()).To(Equal(expLabels)) + g.Expect(secret.GetAnnotations()).To(Equal(expAnnotations)) + g.Expect(secret.Data).To(HaveKey("tls.crt")) + g.Expect(secret.Data["tls.crt"]).To(Equal([]byte("tls"))) + + cmObj := objects[1] cm, ok := cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) g.Expect(cm.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, nginxIncludesConfigMapNameSuffix))) @@ -103,7 +123,7 @@ func TestBuildNginxResourceObjects(t *testing.T) { g.Expect(cm.Data).To(HaveKey("main.conf")) g.Expect(cm.Data["main.conf"]).To(ContainSubstring("info")) - cmObj = objects[1] + cmObj = objects[2] cm, ok = cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) g.Expect(cm.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, nginxAgentConfigMapNameSuffix))) @@ -111,12 +131,12 @@ func TestBuildNginxResourceObjects(t *testing.T) { g.Expect(cm.Data).To(HaveKey("nginx-agent.conf")) g.Expect(cm.Data["nginx-agent.conf"]).To(ContainSubstring("command:")) - svcAcctObj := objects[2] + svcAcctObj := objects[3] svcAcct, ok := svcAcctObj.(*corev1.ServiceAccount) g.Expect(ok).To(BeTrue()) validateMeta(svcAcct) - svcObj := objects[3] + svcObj := objects[4] svc, ok := svcObj.(*corev1.Service) g.Expect(ok).To(BeTrue()) validateMeta(svc) @@ -142,7 +162,7 @@ func TestBuildNginxResourceObjects(t *testing.T) { }, })) - depObj := objects[4] + depObj := objects[5] dep, ok := depObj.(*appsv1.Deployment) g.Expect(ok).To(BeTrue()) validateMeta(dep) @@ -186,18 +206,29 @@ func TestBuildNginxResourceObjects_NginxProxyConfig(t *testing.T) { t.Parallel() g := NewWithT(t) + agentTLSSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: agentTLSTestSecretName, + Namespace: ngfNamespace, + }, + Data: map[string][]byte{"tls.crt": []byte("tls")}, + } + fakeClient := fake.NewFakeClient(agentTLSSecret) + provisioner := &NginxProvisioner{ cfg: Config{ GatewayPodConfig: &config.GatewayPodConfig{ - Namespace: "default", + Namespace: ngfNamespace, Version: "1.0.0", }, + AgentTLSSecretName: agentTLSTestSecretName, }, baseLabelSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ "app": "nginx", }, }, + k8sClient: fakeClient, } gateway := &gatewayv1.Gateway{ @@ -247,21 +278,21 @@ func TestBuildNginxResourceObjects_NginxProxyConfig(t *testing.T) { objects, err := provisioner.buildNginxResourceObjects(resourceName, gateway, nProxyCfg) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(objects).To(HaveLen(5)) + g.Expect(objects).To(HaveLen(6)) - cmObj := objects[0] + cmObj := objects[1] cm, ok := cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) g.Expect(cm.Data).To(HaveKey("main.conf")) g.Expect(cm.Data["main.conf"]).To(ContainSubstring("debug")) - cmObj = objects[1] + cmObj = objects[2] cm, ok = cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) g.Expect(cm.Data["nginx-agent.conf"]).To(ContainSubstring("level: debug")) g.Expect(cm.Data["nginx-agent.conf"]).To(ContainSubstring("port: 8080")) - svcObj := objects[3] + svcObj := objects[4] svc, ok := svcObj.(*corev1.Service) g.Expect(ok).To(BeTrue()) g.Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeNodePort)) @@ -269,7 +300,7 @@ func TestBuildNginxResourceObjects_NginxProxyConfig(t *testing.T) { g.Expect(svc.Spec.LoadBalancerIP).To(Equal("1.2.3.4")) g.Expect(svc.Spec.LoadBalancerSourceRanges).To(Equal([]string{"5.6.7.8"})) - depObj := objects[4] + depObj := objects[5] dep, ok := depObj.(*appsv1.Deployment) g.Expect(ok).To(BeTrue()) @@ -293,6 +324,13 @@ func TestBuildNginxResourceObjects_Plus(t *testing.T) { t.Parallel() g := NewWithT(t) + agentTLSSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: agentTLSTestSecretName, + Namespace: ngfNamespace, + }, + Data: map[string][]byte{"tls.crt": []byte("tls")}, + } jwtSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: jwtTestSecretName, @@ -315,7 +353,7 @@ func TestBuildNginxResourceObjects_Plus(t *testing.T) { Data: map[string][]byte{"tls.crt": []byte("tls")}, } - fakeClient := fake.NewFakeClient(jwtSecret, caSecret, clientSSLSecret) + fakeClient := fake.NewFakeClient(agentTLSSecret, jwtSecret, caSecret, clientSSLSecret) provisioner := &NginxProvisioner{ cfg: Config{ @@ -330,6 +368,7 @@ func TestBuildNginxResourceObjects_Plus(t *testing.T) { Endpoint: "test.com", SkipVerify: true, }, + AgentTLSSecretName: agentTLSTestSecretName, }, k8sClient: fakeClient, baseLabelSelector: metav1.LabelSelector{ @@ -360,7 +399,7 @@ func TestBuildNginxResourceObjects_Plus(t *testing.T) { objects, err := provisioner.buildNginxResourceObjects(resourceName, gateway, &graph.EffectiveNginxProxy{}) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(objects).To(HaveLen(8)) + g.Expect(objects).To(HaveLen(9)) expLabels := map[string]string{ "label": "value", @@ -372,7 +411,7 @@ func TestBuildNginxResourceObjects_Plus(t *testing.T) { "annotation": "value", } - secretObj := objects[0] + secretObj := objects[1] secret, ok := secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) g.Expect(secret.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, jwtTestSecretName))) @@ -381,7 +420,7 @@ func TestBuildNginxResourceObjects_Plus(t *testing.T) { g.Expect(secret.Data).To(HaveKey("license.jwt")) g.Expect(secret.Data["license.jwt"]).To(Equal([]byte("jwt"))) - secretObj = objects[1] + secretObj = objects[2] secret, ok = secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) g.Expect(secret.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, caTestSecretName))) @@ -390,7 +429,7 @@ func TestBuildNginxResourceObjects_Plus(t *testing.T) { g.Expect(secret.Data).To(HaveKey("ca.crt")) g.Expect(secret.Data["ca.crt"]).To(Equal([]byte("ca"))) - secretObj = objects[2] + secretObj = objects[3] secret, ok = secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) g.Expect(secret.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, clientTestSecretName))) @@ -399,7 +438,7 @@ func TestBuildNginxResourceObjects_Plus(t *testing.T) { g.Expect(secret.Data).To(HaveKey("tls.crt")) g.Expect(secret.Data["tls.crt"]).To(Equal([]byte("tls"))) - cmObj := objects[3] + cmObj := objects[4] cm, ok := cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) g.Expect(cm.Data).To(HaveKey("mgmt.conf")) @@ -409,13 +448,13 @@ func TestBuildNginxResourceObjects_Plus(t *testing.T) { g.Expect(cm.Data["mgmt.conf"]).To(ContainSubstring("ssl_certificate")) g.Expect(cm.Data["mgmt.conf"]).To(ContainSubstring("ssl_certificate_key")) - cmObj = objects[4] + cmObj = objects[5] cm, ok = cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) g.Expect(cm.Data).To(HaveKey("nginx-agent.conf")) g.Expect(cm.Data["nginx-agent.conf"]).To(ContainSubstring("api-action")) - depObj := objects[7] + depObj := objects[8] dep, ok := depObj.(*appsv1.Deployment) g.Expect(ok).To(BeTrue()) @@ -439,6 +478,14 @@ func TestBuildNginxResourceObjects_DockerSecrets(t *testing.T) { t.Parallel() g := NewWithT(t) + agentTLSSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: agentTLSTestSecretName, + Namespace: ngfNamespace, + }, + Data: map[string][]byte{"tls.crt": []byte("tls")}, + } + dockerSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: dockerTestSecretName, @@ -464,7 +511,7 @@ func TestBuildNginxResourceObjects_DockerSecrets(t *testing.T) { }, Data: map[string][]byte{"data": []byte("docker-registry2")}, } - fakeClient := fake.NewFakeClient(dockerSecret, dockerSecretRegistry1, dockerSecretRegistry2) + fakeClient := fake.NewFakeClient(agentTLSSecret, dockerSecret, dockerSecretRegistry1, dockerSecretRegistry2) provisioner := &NginxProvisioner{ cfg: Config{ @@ -472,6 +519,7 @@ func TestBuildNginxResourceObjects_DockerSecrets(t *testing.T) { Namespace: ngfNamespace, }, NginxDockerSecretNames: []string{dockerTestSecretName, dockerSecretRegistry1Name, dockerSecretRegistry2Name}, + AgentTLSSecretName: agentTLSTestSecretName, }, k8sClient: fakeClient, baseLabelSelector: metav1.LabelSelector{ @@ -492,7 +540,7 @@ func TestBuildNginxResourceObjects_DockerSecrets(t *testing.T) { objects, err := provisioner.buildNginxResourceObjects(resourceName, gateway, &graph.EffectiveNginxProxy{}) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(objects).To(HaveLen(8)) + g.Expect(objects).To(HaveLen(9)) expLabels := map[string]string{ "app": "nginx", @@ -500,27 +548,33 @@ func TestBuildNginxResourceObjects_DockerSecrets(t *testing.T) { "app.kubernetes.io/name": "gw-nginx", } - // the (docker-only) secret order in the object list is sorted by secret name - secretObj := objects[0] secret, ok := secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) + g.Expect(secret.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, agentTLSTestSecretName))) + g.Expect(secret.GetLabels()).To(Equal(expLabels)) + + // the (docker-only) secret order in the object list is sorted by secret name + + secretObj = objects[1] + secret, ok = secretObj.(*corev1.Secret) + g.Expect(ok).To(BeTrue()) g.Expect(secret.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, dockerTestSecretName))) g.Expect(secret.GetLabels()).To(Equal(expLabels)) - registry1SecretObj := objects[1] + registry1SecretObj := objects[2] secret, ok = registry1SecretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) g.Expect(secret.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, dockerSecretRegistry1Name))) g.Expect(secret.GetLabels()).To(Equal(expLabels)) - registry2SecretObj := objects[2] + registry2SecretObj := objects[3] secret, ok = registry2SecretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) g.Expect(secret.GetName()).To(Equal(controller.CreateNginxResourceName(resourceName, dockerSecretRegistry2Name))) g.Expect(secret.GetLabels()).To(Equal(expLabels)) - depObj := objects[7] + depObj := objects[8] dep, ok := depObj.(*appsv1.Deployment) g.Expect(ok).To(BeTrue()) @@ -553,10 +607,14 @@ func TestGetAndUpdateSecret_NotFound(t *testing.T) { k8sClient: fakeClient, } - _, err := provisioner.getAndUpdateSecret("non-existent-secret", metav1.ObjectMeta{ - Name: "new-secret", - Namespace: "default", - }) + _, err := provisioner.getAndUpdateSecret( + "non-existent-secret", + metav1.ObjectMeta{ + Name: "new-secret", + Namespace: "default", + }, + corev1.SecretTypeOpaque, + ) g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring("error getting secret")) @@ -575,7 +633,7 @@ func TestBuildNginxResourceObjectsForDeletion(t *testing.T) { objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) - g.Expect(objects).To(HaveLen(5)) + g.Expect(objects).To(HaveLen(6)) validateMeta := func(obj client.Object, name string) { g.Expect(obj.GetName()).To(Equal(name)) @@ -621,6 +679,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { ClientSSLSecretName: clientTestSecretName, }, NginxDockerSecretNames: []string{dockerTestSecretName}, + AgentTLSSecretName: agentTLSTestSecretName, }, } @@ -631,7 +690,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) - g.Expect(objects).To(HaveLen(9)) + g.Expect(objects).To(HaveLen(10)) validateMeta := func(obj client.Object, name string) { g.Expect(obj.GetName()).To(Equal(name)) @@ -668,7 +727,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( deploymentNSName.Name, - provisioner.cfg.NginxDockerSecretNames[0], + provisioner.cfg.AgentTLSSecretName, )) secretObj = objects[6] @@ -676,12 +735,20 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( deploymentNSName.Name, - provisioner.cfg.PlusUsageConfig.CASecretName, + provisioner.cfg.NginxDockerSecretNames[0], )) secretObj = objects[7] secret, ok = secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) + validateMeta(secret, controller.CreateNginxResourceName( + deploymentNSName.Name, + provisioner.cfg.PlusUsageConfig.CASecretName, + )) + + secretObj = objects[8] + secret, ok = secretObj.(*corev1.Secret) + g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( deploymentNSName.Name, provisioner.cfg.PlusUsageConfig.ClientSSLSecretName, diff --git a/internal/mode/static/provisioner/provisioner.go b/internal/mode/static/provisioner/provisioner.go index 71439f4b4e..e4a0ce7bee 100644 --- a/internal/mode/static/provisioner/provisioner.go +++ b/internal/mode/static/provisioner/provisioner.go @@ -42,7 +42,8 @@ type Provisioner interface { // Config is the configuration for the Provisioner. type Config struct { - GCName string + GCName string + AgentTLSSecretName string DeploymentStore agent.DeploymentStorer StatusQueue *status.Queue @@ -81,7 +82,13 @@ func NewNginxProvisioner( caSecretName = cfg.PlusUsageConfig.CASecretName clientSSLSecretName = cfg.PlusUsageConfig.ClientSSLSecretName } - store := newStore(cfg.NginxDockerSecretNames, jwtSecretName, caSecretName, clientSSLSecretName) + store := newStore( + cfg.NginxDockerSecretNames, + cfg.AgentTLSSecretName, + jwtSecretName, + caSecretName, + clientSSLSecretName, + ) selector := metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -114,6 +121,7 @@ func NewNginxProvisioner( selector, cfg.GatewayPodConfig.Namespace, cfg.NginxDockerSecretNames, + cfg.AgentTLSSecretName, cfg.PlusUsageConfig, ) if err != nil { @@ -354,6 +362,10 @@ func (p *NginxProvisioner) deprovisionNginx(ctx context.Context, gatewayNSName t // isUserSecret determines if the provided secret name is a special user secret, // for example an NGINX docker registry secret or NGINX Plus secret. func (p *NginxProvisioner) isUserSecret(name string) bool { + if name == p.cfg.AgentTLSSecretName { + return true + } + if slices.Contains(p.cfg.NginxDockerSecretNames, name) { return true } diff --git a/internal/mode/static/provisioner/provisioner_test.go b/internal/mode/static/provisioner/provisioner_test.go index 8ef7873386..987b835352 100644 --- a/internal/mode/static/provisioner/provisioner_test.go +++ b/internal/mode/static/provisioner/provisioner_test.go @@ -27,12 +27,13 @@ import ( "github.com/nginx/nginx-gateway-fabric/internal/mode/static/state/graph" ) -var ( - jwtTestSecretName = "jwt-secret" - caTestSecretName = "ca-secret" - clientTestSecretName = "client-secret" - dockerTestSecretName = "docker-secret" - ngfNamespace = "nginx-gateway" +const ( + agentTLSTestSecretName = "agent-tls-secret" + jwtTestSecretName = "jwt-secret" + caTestSecretName = "ca-secret" + clientTestSecretName = "client-secret" + dockerTestSecretName = "docker-secret" + ngfNamespace = "nginx-gateway" ) func createScheme() *runtime.Scheme { @@ -64,6 +65,12 @@ func expectResourcesToExist(g *WithT, k8sClient client.Client, nsName types.Name } g.Expect(k8sClient.Get(context.TODO(), agentCM, &corev1.ConfigMap{})).To(Succeed()) + agentTLSSecret := types.NamespacedName{ + Name: controller.CreateNginxResourceName(nsName.Name, agentTLSTestSecretName), + Namespace: nsName.Namespace, + } + g.Expect(k8sClient.Get(context.TODO(), agentTLSSecret, &corev1.Secret{})).To(Succeed()) + if !plus { return } @@ -112,6 +119,12 @@ func expectResourcesToNotExist(g *WithT, k8sClient client.Client, nsName types.N } g.Expect(k8sClient.Get(context.TODO(), agentCM, &corev1.ConfigMap{})).ToNot(Succeed()) + agentTLSSecret := types.NamespacedName{ + Name: controller.CreateNginxResourceName(nsName.Name, agentTLSTestSecretName), + Namespace: nsName.Namespace, + } + g.Expect(k8sClient.Get(context.TODO(), agentTLSSecret, &corev1.Secret{})).ToNot(Succeed()) + jwtSecret := types.NamespacedName{ Name: controller.CreateNginxResourceName(nsName.Name, jwtTestSecretName), Namespace: nsName.Namespace, @@ -144,7 +157,13 @@ func defaultNginxProvisioner( deploymentStore := &agentfakes.FakeDeploymentStorer{} return &NginxProvisioner{ - store: newStore([]string{dockerTestSecretName}, jwtTestSecretName, caTestSecretName, clientTestSecretName), + store: newStore( + []string{dockerTestSecretName}, + agentTLSTestSecretName, + jwtTestSecretName, + caTestSecretName, + clientTestSecretName, + ), k8sClient: fakeClient, cfg: Config{ DeploymentStore: deploymentStore, @@ -162,6 +181,7 @@ func defaultNginxProvisioner( ClientSSLSecretName: clientTestSecretName, }, NginxDockerSecretNames: []string{dockerTestSecretName}, + AgentTLSSecretName: agentTLSTestSecretName, }, leader: true, }, fakeClient, deploymentStore @@ -232,6 +252,12 @@ func TestRegisterGateway(t *testing.T) { objects := []client.Object{ gateway.Source, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: agentTLSTestSecretName, + Namespace: ngfNamespace, + }, + }, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: jwtTestSecretName, @@ -328,7 +354,14 @@ func TestProvisionerRestartsDeployment(t *testing.T) { } // provision everything first - provisioner, fakeClient, _ := defaultNginxProvisioner(gateway.Source) + agentTLSSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: agentTLSTestSecretName, + Namespace: ngfNamespace, + }, + Data: map[string][]byte{"tls.crt": []byte("tls")}, + } + provisioner, fakeClient, _ := defaultNginxProvisioner(gateway.Source, agentTLSSecret) provisioner.cfg.Plus = false provisioner.cfg.NginxDockerSecretNames = nil diff --git a/internal/mode/static/provisioner/store.go b/internal/mode/static/provisioner/store.go index ac63beb907..e28a42e103 100644 --- a/internal/mode/static/provisioner/store.go +++ b/internal/mode/static/provisioner/store.go @@ -23,6 +23,7 @@ type NginxResources struct { ServiceAccount metav1.ObjectMeta BootstrapConfigMap metav1.ObjectMeta AgentConfigMap metav1.ObjectMeta + AgentTLSSecret metav1.ObjectMeta PlusJWTSecret metav1.ObjectMeta PlusClientSSLSecret metav1.ObjectMeta PlusCASecret metav1.ObjectMeta @@ -37,7 +38,9 @@ type store struct { // nginxResources is a map of Gateway NamespacedNames and their associated nginx resources. nginxResources map[types.NamespacedName]*NginxResources - dockerSecretNames map[string]struct{} + dockerSecretNames map[string]struct{} + agentTLSSecretName string + // NGINX Plus secrets jwtSecretName string caSecretName string @@ -48,6 +51,7 @@ type store struct { func newStore( dockerSecretNames []string, + agentTLSSecretName, jwtSecretName, caSecretName, clientSSLSecretName string, @@ -61,6 +65,7 @@ func newStore( gateways: make(map[types.NamespacedName]*gatewayv1.Gateway), nginxResources: make(map[types.NamespacedName]*NginxResources), dockerSecretNames: dockerSecretNamesMap, + agentTLSSecretName: agentTLSSecretName, jwtSecretName: jwtSecretName, caSecretName: caSecretName, clientSSLSecretName: clientSSLSecretName, @@ -167,6 +172,7 @@ func (s *store) registerConfigMapInGatewayConfig(obj *corev1.ConfigMap, gatewayN } } +//nolint:gocyclo // will refactor at some point func (s *store) registerSecretInGatewayConfig(obj *corev1.Secret, gatewayNSName types.NamespacedName) { hasSuffix := func(str, suffix string) bool { return suffix != "" && strings.HasSuffix(str, suffix) @@ -174,6 +180,10 @@ func (s *store) registerSecretInGatewayConfig(obj *corev1.Secret, gatewayNSName if cfg, ok := s.nginxResources[gatewayNSName]; !ok { switch { + case hasSuffix(obj.GetName(), s.agentTLSSecretName): + s.nginxResources[gatewayNSName] = &NginxResources{ + AgentTLSSecret: obj.ObjectMeta, + } case hasSuffix(obj.GetName(), s.jwtSecretName): s.nginxResources[gatewayNSName] = &NginxResources{ PlusJWTSecret: obj.ObjectMeta, @@ -198,6 +208,8 @@ func (s *store) registerSecretInGatewayConfig(obj *corev1.Secret, gatewayNSName } } else { switch { + case hasSuffix(obj.GetName(), s.agentTLSSecretName): + cfg.AgentTLSSecret = obj.ObjectMeta case hasSuffix(obj.GetName(), s.jwtSecretName): cfg.PlusJWTSecret = obj.ObjectMeta case hasSuffix(obj.GetName(), s.caSecretName): @@ -284,6 +296,10 @@ func (s *store) gatewayExistsForResource(object client.Object, nsName types.Name } func secretResourceMatches(resources *NginxResources, nsName types.NamespacedName) bool { + if resourceMatches(resources.AgentTLSSecret, nsName) { + return true + } + for _, secret := range resources.DockerSecrets { if resourceMatches(secret, nsName) { return true diff --git a/internal/mode/static/provisioner/store_test.go b/internal/mode/static/provisioner/store_test.go index 079736fde0..814ef1bd79 100644 --- a/internal/mode/static/provisioner/store_test.go +++ b/internal/mode/static/provisioner/store_test.go @@ -22,10 +22,11 @@ func TestNewStore(t *testing.T) { t.Parallel() g := NewWithT(t) - store := newStore([]string{"docker-secret"}, "jwt-secret", "ca-secret", "client-ssl-secret") + store := newStore([]string{"docker-secret"}, "agent-tls-secret", "jwt-secret", "ca-secret", "client-ssl-secret") g.Expect(store).NotTo(BeNil()) g.Expect(store.dockerSecretNames).To(HaveKey("docker-secret")) + g.Expect(store.agentTLSSecretName).To(Equal("agent-tls-secret")) g.Expect(store.jwtSecretName).To(Equal("jwt-secret")) g.Expect(store.caSecretName).To(Equal("ca-secret")) g.Expect(store.clientSSLSecretName).To(Equal("client-ssl-secret")) @@ -35,7 +36,7 @@ func TestUpdateGateway(t *testing.T) { t.Parallel() g := NewWithT(t) - store := newStore(nil, "", "", "") + store := newStore(nil, "", "", "", "") gateway := &gatewayv1.Gateway{ ObjectMeta: metav1.ObjectMeta{ Name: "test-gateway", @@ -54,7 +55,7 @@ func TestDeleteGateway(t *testing.T) { t.Parallel() g := NewWithT(t) - store := newStore(nil, "", "", "") + store := newStore(nil, "", "", "", "") nsName := types.NamespacedName{Name: "test-gateway", Namespace: "default"} store.gateways[nsName] = &gatewayv1.Gateway{} @@ -68,7 +69,7 @@ func TestGetGateways(t *testing.T) { t.Parallel() g := NewWithT(t) - store := newStore(nil, "", "", "") + store := newStore(nil, "", "", "", "") gateway1 := &gatewayv1.Gateway{ ObjectMeta: metav1.ObjectMeta{ Name: "test-gateway-1", @@ -99,7 +100,7 @@ func TestRegisterResourceInGatewayConfig(t *testing.T) { t.Parallel() g := NewWithT(t) - store := newStore([]string{"docker-secret"}, "jwt-secret", "ca-secret", "client-ssl-secret") + store := newStore([]string{"docker-secret"}, "agent-tls-secret", "jwt-secret", "ca-secret", "client-ssl-secret") nsName := types.NamespacedName{Name: "test-gateway", Namespace: "default"} registerAndGetResources := func(obj interface{}) *NginxResources { @@ -198,6 +199,22 @@ func TestRegisterResourceInGatewayConfig(t *testing.T) { // clear out resources before next test store.deleteResourcesForGateway(nsName) + // Secret + agentTLSSecretMeta := metav1.ObjectMeta{ + Name: controller.CreateNginxResourceName(defaultMeta.Name, store.agentTLSSecretName), + Namespace: defaultMeta.Namespace, + } + agentTLSSecret := &corev1.Secret{ObjectMeta: agentTLSSecretMeta} + resources = registerAndGetResources(agentTLSSecret) + g.Expect(resources.AgentTLSSecret).To(Equal(agentTLSSecretMeta)) + + // Secret again, already exists + resources = registerAndGetResources(agentTLSSecret) + g.Expect(resources.AgentTLSSecret).To(Equal(agentTLSSecretMeta)) + + // clear out resources before next test + store.deleteResourcesForGateway(nsName) + // Secret jwtSecretMeta := metav1.ObjectMeta{ Name: controller.CreateNginxResourceName(defaultMeta.Name, store.jwtSecretName), @@ -361,7 +378,7 @@ func TestDeleteResourcesForGateway(t *testing.T) { t.Parallel() g := NewWithT(t) - store := newStore(nil, "", "", "") + store := newStore(nil, "", "", "", "") nsName := types.NamespacedName{Name: "test-gateway", Namespace: "default"} store.nginxResources[nsName] = &NginxResources{} @@ -373,7 +390,7 @@ func TestDeleteResourcesForGateway(t *testing.T) { func TestGatewayExistsForResource(t *testing.T) { t.Parallel() - store := newStore(nil, "", "", "") + store := newStore(nil, "", "", "", "") gateway := &graph.Gateway{} store.nginxResources[types.NamespacedName{Name: "test-gateway", Namespace: "default"}] = &NginxResources{ Gateway: gateway, @@ -397,6 +414,10 @@ func TestGatewayExistsForResource(t *testing.T) { Name: "test-agent-configmap", Namespace: "default", }, + AgentTLSSecret: metav1.ObjectMeta{ + Name: "test-agent-tls-secret", + Namespace: "default", + }, PlusJWTSecret: metav1.ObjectMeta{ Name: "test-jwt-secret", Namespace: "default", @@ -472,6 +493,16 @@ func TestGatewayExistsForResource(t *testing.T) { }, expected: gateway, }, + { + name: "Agent TLS Secret exists", + object: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-agent-tls-secret", + Namespace: "default", + }, + }, + expected: gateway, + }, { name: "JWT Secret exists", object: &corev1.Secret{ diff --git a/internal/mode/static/provisioner/templates.go b/internal/mode/static/provisioner/templates.go index 05e1b96623..e58ee3abec 100644 --- a/internal/mode/static/provisioner/templates.go +++ b/internal/mode/static/provisioner/templates.go @@ -33,6 +33,13 @@ const agentTemplateText = `command: server: host: {{ .ServiceName }}.{{ .Namespace }}.svc port: 443 + auth: + tokenpath: /var/run/secrets/ngf/serviceaccount/token + tls: + cert: /var/run/secrets/ngf/tls.crt + key: /var/run/secrets/ngf/tls.key + ca: /var/run/secrets/ngf/ca.crt + server_name: {{ .ServiceName }}.{{ .Namespace }}.svc allowed_directories: - /etc/nginx - /usr/share/nginx