diff --git a/.goreleaser.yml b/.goreleaser.yml index 2da49f57cf..8433bbc985 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -66,3 +66,6 @@ release: extra_files: - glob: ./build/out/crds.yaml - glob: ./deploy/manifests/nginx-gateway.yaml + - glob: ./deploy/manifests/nginx-plus-gateway.yaml + - glob: ./deploy/manifests/nginx-gateway-experimental.yaml + - glob: ./deploy/manifests/nginx-plus-gateway-experimental.yaml diff --git a/cmd/gateway/commands.go b/cmd/gateway/commands.go index 7265f5c4a7..3cd42ca3a1 100644 --- a/cmd/gateway/commands.go +++ b/cmd/gateway/commands.go @@ -57,6 +57,10 @@ func createStaticModeCommand() *cobra.Command { leaderElectionLockNameFlag = "leader-election-lock-name" plusFlag = "nginx-plus" gwAPIExperimentalFlag = "gateway-api-experimental-features" + usageReportSecretFlag = "usage-report-secret" + usageReportServerURLFlag = "usage-report-server-url" + usageReportSkipVerifyFlag = "usage-report-skip-verify" + usageReportClusterNameFlag = "usage-report-cluster-name" ) // flag values @@ -95,9 +99,17 @@ func createStaticModeCommand() *cobra.Command { value: "nginx-gateway-leader-election-lock", } - plus bool - gwExperimentalFeatures bool + + plus bool + usageReportSkipVerify bool + usageReportClusterName = stringValidatingValue{ + validator: validateQualifiedName, + } + usageReportSecretName = namespacedNameValue{} + usageReportServerURL = stringValidatingValue{ + validator: validateURL, + } ) cmd := &cobra.Command{ @@ -144,6 +156,20 @@ func createStaticModeCommand() *cobra.Command { gwNsName = &gateway.value } + var usageReportConfig *config.UsageReportConfig + if cmd.Flags().Changed(usageReportSecretFlag) { + if !plus { + return errors.New("usage-report arguments are only valid if using nginx-plus") + } + + usageReportConfig = &config.UsageReportConfig{ + SecretNsName: usageReportSecretName.value, + ServerURL: usageReportServerURL.value, + ClusterDisplayName: usageReportClusterName.value, + InsecureSkipVerify: usageReportSkipVerify, + } + } + conf := config.Config{ GatewayCtlrName: gatewayCtlrName.value, ConfigName: configName.String(), @@ -167,11 +193,12 @@ func createStaticModeCommand() *cobra.Command { Port: metricsListenPort.value, Secure: metricsSecure, }, - LeaderElection: config.LeaderElection{ + LeaderElection: config.LeaderElectionConfig{ Enabled: !disableLeaderElection, LockName: leaderElectionLockName.String(), Identity: podName, }, + UsageReportConfig: usageReportConfig, Plus: plus, TelemetryReportPeriod: period, Version: version, @@ -297,6 +324,33 @@ func createStaticModeCommand() *cobra.Command { "Requires the Gateway APIs installed from the experimental channel.", ) + cmd.Flags().Var( + &usageReportSecretName, + usageReportSecretFlag, + "The namespace/name of the Secret containing the credentials for NGINX Plus usage reporting.", + ) + + cmd.Flags().Var( + &usageReportServerURL, + usageReportServerURLFlag, + "The base server URL of the NGINX Plus usage reporting server.", + ) + + cmd.MarkFlagsRequiredTogether(usageReportSecretFlag, usageReportServerURLFlag) + + cmd.Flags().Var( + &usageReportClusterName, + usageReportClusterNameFlag, + "The display name of the Kubernetes cluster in the NGINX Plus usage reporting server.", + ) + + cmd.Flags().BoolVar( + &usageReportSkipVerify, + usageReportSkipVerifyFlag, + false, + "Disable client verification of the NGINX Plus usage reporting server certificate.", + ) + return cmd } diff --git a/cmd/gateway/commands_test.go b/cmd/gateway/commands_test.go index 80396eb888..05da529d49 100644 --- a/cmd/gateway/commands_test.go +++ b/cmd/gateway/commands_test.go @@ -155,6 +155,9 @@ func TestStaticModeCmdFlagValidation(t *testing.T) { "--health-disable", "--leader-election-lock-name=my-lock", "--leader-election-disable=false", + "--usage-report-secret=default/my-secret", + "--usage-report-server-url=https://my-api.com", + "--usage-report-cluster-name=my-cluster", }, wantErr: false, }, @@ -310,6 +313,66 @@ func TestStaticModeCmdFlagValidation(t *testing.T) { wantErr: true, expectedErrPrefix: `invalid argument "" for "--leader-election-disable" flag: strconv.ParseBool`, }, + { + name: "usage-report-secret is set to empty string", + args: []string{ + "--usage-report-secret=", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "" for "--usage-report-secret" flag: must be set`, + }, + { + name: "usage-report-secret is invalid", + args: []string{ + "--usage-report-secret=my-secret", // no namespace + }, + wantErr: true, + expectedErrPrefix: `invalid argument "my-secret" for "--usage-report-secret" flag: invalid format; ` + + "must be NAMESPACE/NAME", + }, + { + name: "usage-report-server-url is set to empty string", + args: []string{ + "--usage-report-server-url=", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "" for "--usage-report-server-url" flag: must be set`, + }, + { + name: "usage-report-server-url is an invalid url", + args: []string{ + "--usage-report-server-url=invalid", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "invalid" for "--usage-report-server-url" flag: "invalid" must be a valid URL`, + }, + { + name: "usage secret and server url not specified together", + args: []string{ + "--gateway-ctlr-name=gateway.nginx.org/nginx-gateway", + "--gatewayclass=nginx", + "--usage-report-server-url=http://example.com", + }, + wantErr: true, + expectedErrPrefix: "if any flags in the group [usage-report-secret usage-report-server-url] " + + "are set they must all be set", + }, + { + name: "usage-report-cluster-name is set to empty string", + args: []string{ + "--usage-report-cluster-name=", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "" for "--usage-report-cluster-name" flag: must be set`, + }, + { + name: "usage-report-cluster-name is invalid", + args: []string{ + "--usage-report-cluster-name=$invalid*(#)", + }, + wantErr: true, + expectedErrPrefix: `invalid argument "$invalid*(#)" for "--usage-report-cluster-name" flag: invalid format`, + }, } // common flags validation is tested separately diff --git a/cmd/gateway/validation.go b/cmd/gateway/validation.go index 103231071a..b31f8c6a2d 100644 --- a/cmd/gateway/validation.go +++ b/cmd/gateway/validation.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net" + "net/url" "regexp" "strings" @@ -89,6 +90,38 @@ func parseNamespacedResourceName(value string) (types.NamespacedName, error) { }, nil } +func validateQualifiedName(name string) error { + if len(name) == 0 { + return errors.New("must be set") + } + + messages := validation.IsQualifiedName(name) + if len(messages) > 0 { + msg := strings.Join(messages, "; ") + return fmt.Errorf("invalid format: %s", msg) + } + + return nil +} + +func validateURL(value string) error { + if len(value) == 0 { + return errors.New("must be set") + } + val, err := url.Parse(value) + if err != nil { + return fmt.Errorf("%q must be a valid URL: %w", value, err) + } + if val.Scheme == "" { + return fmt.Errorf("%q must be a valid URL: bad scheme", value) + } + if val.Host == "" { + return fmt.Errorf("%q must be a valid URL: bad host", value) + } + + return nil +} + func validateIP(ip string) error { if ip == "" { return errors.New("IP address must be set") diff --git a/cmd/gateway/validation_test.go b/cmd/gateway/validation_test.go index b3865a88b4..2e85a9d128 100644 --- a/cmd/gateway/validation_test.go +++ b/cmd/gateway/validation_test.go @@ -255,6 +255,130 @@ func TestParseNamespacedResourceName(t *testing.T) { } } +func TestValidateQualifiedName(t *testing.T) { + tests := []struct { + name string + value string + expErr bool + }{ + { + name: "valid", + value: "myName", + expErr: false, + }, + { + name: "valid with hyphen", + value: "my-name", + expErr: false, + }, + { + name: "valid with numbers", + value: "myName123", + expErr: false, + }, + { + name: "valid with '/'", + value: "my/name", + expErr: false, + }, + { + name: "valid with '.'", + value: "my.name", + expErr: false, + }, + { + name: "empty", + value: "", + expErr: true, + }, + { + name: "invalid character '$'", + value: "myName$", + expErr: true, + }, + { + name: "invalid character '^'", + value: "my^Name", + expErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewWithT(t) + + err := validateQualifiedName(test.value) + if test.expErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestValidateURL(t *testing.T) { + tests := []struct { + name string + url string + expErr bool + }{ + { + name: "valid", + url: "http://server.com", + expErr: false, + }, + { + name: "valid https", + url: "https://server.com", + expErr: false, + }, + { + name: "valid with port", + url: "http://server.com:8080", + expErr: false, + }, + { + name: "valid with ip address", + url: "http://10.0.0.1", + expErr: false, + }, + { + name: "valid with ip address and port", + url: "http://10.0.0.1:8080", + expErr: false, + }, + { + name: "invalid scheme", + url: "http//server.com", + expErr: true, + }, + { + name: "no scheme", + url: "server.com", + expErr: true, + }, + { + name: "no domain", + url: "http://", + expErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + err := validateURL(tc.url) + if !tc.expErr { + g.Expect(err).ToNot(HaveOccurred()) + } else { + g.Expect(err).To(HaveOccurred()) + } + }) + } +} + func TestValidateIP(t *testing.T) { tests := []struct { name string diff --git a/deploy/helm-chart/README.md b/deploy/helm-chart/README.md index 65bfd82e18..9032a23ca1 100644 --- a/deploy/helm-chart/README.md +++ b/deploy/helm-chart/README.md @@ -297,6 +297,10 @@ The following tables lists the configurable parameters of the NGINX Gateway Fabr | `nginx.image.tag` | The tag for the NGINX image. | edge | | `nginx.image.pullPolicy` | The `imagePullPolicy` for the NGINX image. | Always | | `nginx.plus` | Is NGINX Plus image being used | false | +| `nginx.usage.secretName` | The namespace/name of the Secret containing the credentials for NGINX Plus usage reporting. | | +| `nginx.usage.serverURL` | The base server URL of the NGINX Plus usage reporting server. | | +| `nginx.usage.clusterName` | The display name of the Kubernetes cluster in the NGINX Plus usage reporting server. | | +| `nginx.usage.insecureSkipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | false | | `nginx.lifecycle` | The `lifecycle` of the nginx container. | {} | | `nginx.extraVolumeMounts` | Extra `volumeMounts` for the nginx container. | {} | | `terminationGracePeriodSeconds` | The termination grace period of the NGINX Gateway Fabric pod. | 30 | diff --git a/deploy/helm-chart/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml index 79b0e242cb..12479708fe 100644 --- a/deploy/helm-chart/templates/deployment.yaml +++ b/deploy/helm-chart/templates/deployment.yaml @@ -55,6 +55,18 @@ spec: {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }} - --gateway-api-experimental-features {{- end }} + {{- if .Values.nginx.usage.secretName }} + - --usage-report-secret={{ .Values.nginx.usage.secretName }} + {{- end }} + {{- if .Values.nginx.usage.serverURL }} + - --usage-report-server-url={{ .Values.nginx.usage.serverURL }} + {{- end }} + {{- if .Values.nginx.usage.clusterName }} + - --usage-report-cluster-name={{ .Values.nginx.usage.clusterName }} + {{- end }} + {{- if .Values.nginx.usage.insecureSkipVerify }} + - --usage-report-skip-verify + {{- end }} env: - name: POD_IP valueFrom: diff --git a/deploy/helm-chart/templates/rbac.yaml b/deploy/helm-chart/templates/rbac.yaml index 16939ef6c9..962f279abe 100644 --- a/deploy/helm-chart/templates/rbac.yaml +++ b/deploy/helm-chart/templates/rbac.yaml @@ -36,6 +36,7 @@ rules: - configmaps {{- end }} verbs: + - get - list - watch # FIXME(bjee19): make nodes, pods, replicasets permission dependent on telemetry being enabled. @@ -65,6 +66,7 @@ rules: - replicasets verbs: - get + - list - apiGroups: - discovery.k8s.io resources: diff --git a/deploy/helm-chart/values.yaml b/deploy/helm-chart/values.yaml index 8154343043..a10d61a73e 100644 --- a/deploy/helm-chart/values.yaml +++ b/deploy/helm-chart/values.yaml @@ -66,6 +66,17 @@ nginx: ## Is NGINX Plus image being used plus: false + ## Configuration for NGINX Plus usage reporting. + usage: + ## The namespace/name of the Secret containing the credentials for NGINX Plus usage reporting. + secretName: "" + ## The base server URL of the NGINX Plus usage reporting server. + serverURL: "" + ## The display name of the Kubernetes cluster in the NGINX Plus usage reporting server. + clusterName: "" + ## Disable client verification of the NGINX Plus usage reporting server certificate. + insecureSkipVerify: false + ## The lifecycle of the nginx container. lifecycle: {} diff --git a/deploy/manifests/nginx-gateway-experimental.yaml b/deploy/manifests/nginx-gateway-experimental.yaml index 0625b4b5e2..a6480764de 100644 --- a/deploy/manifests/nginx-gateway-experimental.yaml +++ b/deploy/manifests/nginx-gateway-experimental.yaml @@ -34,6 +34,7 @@ rules: - secrets - configmaps verbs: + - get - list - watch # FIXME(bjee19): make nodes, pods, replicasets permission dependent on telemetry being enabled. @@ -63,6 +64,7 @@ rules: - replicasets verbs: - get + - list - apiGroups: - discovery.k8s.io resources: diff --git a/deploy/manifests/nginx-gateway.yaml b/deploy/manifests/nginx-gateway.yaml index 578e4950b3..37fbd2dab2 100644 --- a/deploy/manifests/nginx-gateway.yaml +++ b/deploy/manifests/nginx-gateway.yaml @@ -33,6 +33,7 @@ rules: - services - secrets verbs: + - get - list - watch # FIXME(bjee19): make nodes, pods, replicasets permission dependent on telemetry being enabled. @@ -62,6 +63,7 @@ rules: - replicasets verbs: - get + - list - apiGroups: - discovery.k8s.io resources: diff --git a/deploy/manifests/nginx-plus-gateway-experimental.yaml b/deploy/manifests/nginx-plus-gateway-experimental.yaml index 49d099b894..e2e5027dd2 100644 --- a/deploy/manifests/nginx-plus-gateway-experimental.yaml +++ b/deploy/manifests/nginx-plus-gateway-experimental.yaml @@ -34,6 +34,7 @@ rules: - secrets - configmaps verbs: + - get - list - watch # FIXME(bjee19): make nodes, pods, replicasets permission dependent on telemetry being enabled. @@ -63,6 +64,7 @@ rules: - replicasets verbs: - get + - list - apiGroups: - discovery.k8s.io resources: diff --git a/deploy/manifests/nginx-plus-gateway.yaml b/deploy/manifests/nginx-plus-gateway.yaml index 23759bdf02..a34a38d083 100644 --- a/deploy/manifests/nginx-plus-gateway.yaml +++ b/deploy/manifests/nginx-plus-gateway.yaml @@ -33,6 +33,7 @@ rules: - services - secrets verbs: + - get - list - watch # FIXME(bjee19): make nodes, pods, replicasets permission dependent on telemetry being enabled. @@ -62,6 +63,7 @@ rules: - replicasets verbs: - get + - list - apiGroups: - discovery.k8s.io resources: diff --git a/internal/mode/static/config/config.go b/internal/mode/static/config/config.go index 95c2f1377d..5ab8c811b4 100644 --- a/internal/mode/static/config/config.go +++ b/internal/mode/static/config/config.go @@ -20,6 +20,8 @@ type Config struct { GatewayPodConfig GatewayPodConfig // Logger is the Zap Logger used by all components. Logger logr.Logger + // UsageReportConfig specifies the NGINX Plus usage reporting config. + UsageReportConfig *UsageReportConfig // GatewayCtlrName is the name of this controller. GatewayCtlrName string // ConfigName is the name of the NginxGateway resource for this controller. @@ -27,7 +29,7 @@ type Config struct { // GatewayClassName is the name of the GatewayClass resource that the Gateway will use. GatewayClassName string // LeaderElection contains the configuration for leader election. - LeaderElection LeaderElection + LeaderElection LeaderElectionConfig // MetricsConfig specifies the metrics config. MetricsConfig MetricsConfig // HealthConfig specifies the health probe config. @@ -72,8 +74,8 @@ type HealthConfig struct { Enabled bool } -// LeaderElection contains the configuration for leader election. -type LeaderElection struct { +// LeaderElectionConfig contains the configuration for leader election. +type LeaderElectionConfig struct { // LockName holds the name of the leader election lock. LockName string // Identity is the unique name of the controller used for identifying the leader. @@ -81,3 +83,15 @@ type LeaderElection struct { // Enabled indicates whether leader election is enabled. Enabled bool } + +// UsageReportConfig contains the configuration for NGINX Plus usage reporting. +type UsageReportConfig struct { + // SecretNsName is the namespaced name of the Secret containing the server credentials. + SecretNsName types.NamespacedName + // ServerURL is the base URL of the reporting server. + ServerURL string + // ClusterDisplayName is the display name of the cluster. Optional. + ClusterDisplayName string + // InsecureSkipVerify controls whether the client verifies the server cert. + InsecureSkipVerify bool +} diff --git a/internal/mode/static/handler.go b/internal/mode/static/handler.go index 74828760b3..0d10522a72 100644 --- a/internal/mode/static/handler.go +++ b/internal/mode/static/handler.go @@ -35,12 +35,26 @@ type handlerMetricsCollector interface { ObserveLastEventBatchProcessTime(time.Duration) } +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 . secretStorer + +// secretStorer should store the usage Secret that contains the credentials for NGINX Plus usage reporting. +type secretStorer interface { + // Set stores the updated Secret. + Set(*v1.Secret) + // Delete nullifies the Secret value. + Delete() +} + // eventHandlerConfig holds configuration parameters for eventHandlerImpl. type eventHandlerConfig struct { // k8sClient is a Kubernetes API client k8sClient client.Client // gatewayPodConfig contains information about this Pod. gatewayPodConfig ngfConfig.GatewayPodConfig + // usageReportConfig contains the configuration for NGINX Plus usage reporting. + usageReportConfig *config.UsageReportConfig + // usageSecret contains the Secret for the NGINX Plus reporting credentials. + usageSecret secretStorer // processor is the state ChangeProcessor. processor state.ChangeProcessor // serviceResolver resolves Services to Endpoints. @@ -67,23 +81,72 @@ type eventHandlerConfig struct { version int } +// filterKey is the `kind_namespace_name" of an object being filtered. +type filterKey string + +// objectFilter contains callbacks for an object that should be treated differently by the handler instead of +// just using the typical Capture() call. +type objectFilter struct { + upsert func(context.Context, logr.Logger, client.Object) + delete func(context.Context, logr.Logger, types.NamespacedName) + captureChangeInGraph bool +} + // eventHandlerImpl implements EventHandler. // eventHandlerImpl is responsible for: // (1) Reconciling the Gateway API and Kubernetes built-in resources with the NGINX configuration. // (2) Keeping the statuses of the Gateway API resources updated. // (3) Updating control plane configuration. +// (4) Tracks the NGINX Plus usage reporting Secret (if applicable). type eventHandlerImpl struct { // latestConfiguration is the latest Configuration generation. latestConfiguration *dataplane.Configuration - cfg eventHandlerConfig - lock sync.Mutex + + // objectFilters contains all created objectFilters, with the key being a filterKey + objectFilters map[filterKey]objectFilter + + cfg eventHandlerConfig + lock sync.Mutex } // newEventHandlerImpl creates a new eventHandlerImpl. func newEventHandlerImpl(cfg eventHandlerConfig) *eventHandlerImpl { - return &eventHandlerImpl{ + handler := &eventHandlerImpl{ cfg: cfg, } + + handler.objectFilters = map[filterKey]objectFilter{ + // NginxGateway CRD + objectFilterKey(&ngfAPI.NginxGateway{}, handler.cfg.controlConfigNSName): { + upsert: handler.nginxGatewayCRDUpsert, + delete: handler.nginxGatewayCRDDelete, + captureChangeInGraph: false, + }, + // NGF-fronting Service + objectFilterKey( + &v1.Service{}, + types.NamespacedName{ + Name: handler.cfg.gatewayPodConfig.ServiceName, + Namespace: handler.cfg.gatewayPodConfig.Namespace, + }, + ): { + upsert: handler.nginxGatewayServiceUpsert, + delete: handler.nginxGatewayServiceDelete, + captureChangeInGraph: true, + }, + } + + if handler.cfg.usageReportConfig != nil { + // N+ usage reporting Secret + nsName := handler.cfg.usageReportConfig.SecretNsName + handler.objectFilters[objectFilterKey(&v1.Secret{}, nsName)] = objectFilter{ + upsert: handler.nginxPlusUsageSecretUpsert, + delete: handler.nginxPlusUsageSecretDelete, + captureChangeInGraph: true, + } + } + + return handler } func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Logger, batch events.EventBatch) { @@ -100,7 +163,7 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log }() for _, event := range batch { - h.handleEvent(ctx, logger, event) + h.parseAndCaptureEvent(ctx, logger, event) } changeType, graph := h.cfg.processor.Process() @@ -158,42 +221,30 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log h.cfg.statusUpdater.Update(ctx, buildGatewayAPIStatuses(graph, gwAddresses, nginxReloadRes)) } -func (h *eventHandlerImpl) handleEvent(ctx context.Context, logger logr.Logger, event interface{}) { +func (h *eventHandlerImpl) parseAndCaptureEvent(ctx context.Context, logger logr.Logger, event interface{}) { switch e := event.(type) { case *events.UpsertEvent: - switch obj := e.Resource.(type) { - case *ngfAPI.NginxGateway: - h.updateControlPlaneAndSetStatus(ctx, logger, obj) - case *apiv1.Service: - podConfig := h.cfg.gatewayPodConfig - if obj.Name == podConfig.ServiceName && obj.Namespace == podConfig.Namespace { - gwAddresses, err := getGatewayAddresses(ctx, h.cfg.k8sClient, obj, h.cfg.gatewayPodConfig) - if err != nil { - logger.Error(err, "Setting GatewayStatusAddress to Pod IP Address") - } - h.cfg.statusUpdater.UpdateAddresses(ctx, gwAddresses) + filterKey := objectFilterKey(e.Resource, client.ObjectKeyFromObject(e.Resource)) + + if filter, ok := h.objectFilters[filterKey]; ok { + filter.upsert(ctx, logger, e.Resource) + if !filter.captureChangeInGraph { + return } - h.cfg.processor.CaptureUpsertChange(e.Resource) - default: - h.cfg.processor.CaptureUpsertChange(e.Resource) } + + h.cfg.processor.CaptureUpsertChange(e.Resource) case *events.DeleteEvent: - switch e.Type.(type) { - case *ngfAPI.NginxGateway: - h.updateControlPlaneAndSetStatus(ctx, logger, nil) - case *apiv1.Service: - podConfig := h.cfg.gatewayPodConfig - if e.NamespacedName.Name == podConfig.ServiceName && e.NamespacedName.Namespace == podConfig.Namespace { - gwAddresses, err := getGatewayAddresses(ctx, h.cfg.k8sClient, nil, h.cfg.gatewayPodConfig) - if err != nil { - logger.Error(err, "Setting GatewayStatusAddress to Pod IP Address") - } - h.cfg.statusUpdater.UpdateAddresses(ctx, gwAddresses) + filterKey := objectFilterKey(e.Type, e.NamespacedName) + + if filter, ok := h.objectFilters[filterKey]; ok { + filter.delete(ctx, logger, e.NamespacedName) + if !filter.captureChangeInGraph { + return } - h.cfg.processor.CaptureDeleteChange(e.Type, e.NamespacedName) - default: - h.cfg.processor.CaptureDeleteChange(e.Type, e.NamespacedName) } + + h.cfg.processor.CaptureDeleteChange(e.Type, e.NamespacedName) default: panic(fmt.Errorf("unknown event type %T", e)) } @@ -412,3 +463,77 @@ func (h *eventHandlerImpl) setLatestConfiguration(cfg *dataplane.Configuration) h.latestConfiguration = cfg } + +func objectFilterKey(obj client.Object, nsName types.NamespacedName) filterKey { + return filterKey(fmt.Sprintf("%T_%s_%s", obj, nsName.Namespace, nsName.Name)) +} + +/* + +Handler Callback functions + +These functions are provided as callbacks to the handler. They are for objects that need special +treatment other than the typical Capture() call that leads to generating nginx config. + +*/ + +func (h *eventHandlerImpl) nginxGatewayCRDUpsert(ctx context.Context, logger logr.Logger, obj client.Object) { + cfg, ok := obj.(*ngfAPI.NginxGateway) + if !ok { + panic(fmt.Errorf("obj type mismatch: got %T, expected %T", obj, &ngfAPI.NginxGateway{})) + } + + h.updateControlPlaneAndSetStatus(ctx, logger, cfg) +} + +func (h *eventHandlerImpl) nginxGatewayCRDDelete( + ctx context.Context, + logger logr.Logger, + _ types.NamespacedName, +) { + h.updateControlPlaneAndSetStatus(ctx, logger, nil) +} + +func (h *eventHandlerImpl) nginxGatewayServiceUpsert(ctx context.Context, logger logr.Logger, obj client.Object) { + svc, ok := obj.(*v1.Service) + if !ok { + panic(fmt.Errorf("obj type mismatch: got %T, expected %T", svc, &v1.Service{})) + } + + gwAddresses, err := getGatewayAddresses(ctx, h.cfg.k8sClient, svc, h.cfg.gatewayPodConfig) + if err != nil { + logger.Error(err, "Setting GatewayStatusAddress to Pod IP Address") + } + + h.cfg.statusUpdater.UpdateAddresses(ctx, gwAddresses) +} + +func (h *eventHandlerImpl) nginxGatewayServiceDelete( + ctx context.Context, + logger logr.Logger, + _ types.NamespacedName, +) { + gwAddresses, err := getGatewayAddresses(ctx, h.cfg.k8sClient, nil, h.cfg.gatewayPodConfig) + if err != nil { + logger.Error(err, "Setting GatewayStatusAddress to Pod IP Address") + } + + h.cfg.statusUpdater.UpdateAddresses(ctx, gwAddresses) +} + +func (h *eventHandlerImpl) nginxPlusUsageSecretUpsert(_ context.Context, _ logr.Logger, obj client.Object) { + secret, ok := obj.(*v1.Secret) + if !ok { + panic(fmt.Errorf("obj type mismatch: got %T, expected %T", obj, &v1.Secret{})) + } + + h.cfg.usageSecret.Set(secret) +} + +func (h *eventHandlerImpl) nginxPlusUsageSecretDelete( + _ context.Context, + _ logr.Logger, + _ types.NamespacedName, +) { + h.cfg.usageSecret.Delete() +} diff --git a/internal/mode/static/handler_test.go b/internal/mode/static/handler_test.go index 682c2bc48c..9b8a60fc52 100644 --- a/internal/mode/static/handler_test.go +++ b/internal/mode/static/handler_test.go @@ -13,6 +13,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" ctlrZap "sigs.k8s.io/controller-runtime/pkg/log/zap" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -34,6 +35,7 @@ import ( "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/graph" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/statefakes" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/staticfakes" ) var _ = Describe("eventHandler", func() { @@ -237,7 +239,15 @@ var _ = Describe("eventHandler", func() { }) It("handles a deleted config", func() { - batch := []interface{}{&events.DeleteEvent{Type: &ngfAPI.NginxGateway{}}} + batch := []interface{}{ + &events.DeleteEvent{ + Type: &ngfAPI.NginxGateway{}, + NamespacedName: types.NamespacedName{ + Namespace: namespace, + Name: configName, + }, + }, + } handler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) Expect(handler.GetLatestConfiguration()).To(BeNil()) @@ -304,6 +314,95 @@ var _ = Describe("eventHandler", func() { }) }) + When("receiving usage Secret updates", func() { + var fakeSecretStore *staticfakes.FakeSecretStorer + var usageSecret *v1.Secret + + BeforeEach(func() { + fakeSecretStore = &staticfakes.FakeSecretStorer{} + usageSecret = &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "usage", + Namespace: "nginx-gateway", + }, + } + }) + + It("should not set the N+ usage secret if not initialized", func() { + handler.cfg.usageSecret = fakeSecretStore + + e := &events.UpsertEvent{ + Resource: usageSecret, + } + batch := []interface{}{e} + + handler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) + Expect(fakeSecretStore.SetCallCount()).To(Equal(0)) + Expect(fakeProcessor.CaptureUpsertChangeCallCount()).To(Equal(1)) + }) + + Context("usage secret is initialized", func() { + var usageSecretHandler *eventHandlerImpl + BeforeEach(func() { + usageCfg := &config.UsageReportConfig{ + SecretNsName: client.ObjectKeyFromObject(usageSecret), + } + usageSecretHandler = newEventHandlerImpl(eventHandlerConfig{ + k8sClient: fake.NewFakeClient(), + processor: fakeProcessor, + nginxConfiguredOnStartChecker: newNginxConfiguredOnStartChecker(), + controlConfigNSName: types.NamespacedName{Namespace: namespace, Name: configName}, + usageReportConfig: usageCfg, + usageSecret: fakeSecretStore, + gatewayPodConfig: config.GatewayPodConfig{ + ServiceName: "nginx-gateway", + Namespace: "nginx-gateway", + }, + metricsCollector: collectors.NewControllerNoopCollector(), + }) + }) + + It("should not set the N+ usage secret if processing a normal secret", func() { + e := &events.UpsertEvent{ + Resource: &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "not-usage", + Namespace: "nginx-gateway", + }, + }, + } + batch := []interface{}{e} + + usageSecretHandler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) + Expect(fakeSecretStore.SetCallCount()).To(Equal(0)) + Expect(fakeProcessor.CaptureUpsertChangeCallCount()).To(Equal(1)) + }) + + It("should set the N+ usage secret when upserted", func() { + e := &events.UpsertEvent{ + Resource: usageSecret, + } + batch := []interface{}{e} + + usageSecretHandler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) + Expect(fakeSecretStore.SetCallCount()).To(Equal(1)) + Expect(fakeProcessor.CaptureUpsertChangeCallCount()).To(Equal(1)) + }) + + It("should remove the N+ usage secret when deleted", func() { + e := &events.DeleteEvent{ + Type: &v1.Secret{}, + NamespacedName: client.ObjectKeyFromObject(usageSecret), + } + batch := []interface{}{e} + + usageSecretHandler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) + Expect(fakeSecretStore.DeleteCallCount()).To(Equal(1)) + Expect(fakeProcessor.CaptureDeleteChangeCallCount()).To(Equal(1)) + }) + }) + }) + When("receiving an EndpointsOnlyChange update", func() { e := &events.UpsertEvent{Resource: &discoveryV1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index 6d63d61626..99f97b8ebf 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -2,6 +2,7 @@ package static import ( "context" + "errors" "fmt" "time" @@ -51,6 +52,7 @@ import ( "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/resolver" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/validation" "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/telemetry" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/usage" ) const ( @@ -139,11 +141,28 @@ func StartManager(cfg config.Config) error { ) var ngxPlusClient *ngxclient.NginxClient + var usageSecret *usage.Secret if cfg.Plus { ngxPlusClient, err = ngxruntime.CreatePlusClient() if err != nil { return fmt.Errorf("error creating NGINX plus client: %w", err) } + + if cfg.UsageReportConfig != nil { + usageSecret = usage.NewUsageSecret() + reporter, err := createUsageReporterJob(mgr.GetAPIReader(), cfg, usageSecret, nginxChecker.getReadyCh()) + if err != nil { + return fmt.Errorf("error creating usage reporter job") + } + + if err = mgr.Add(reporter); err != nil { + return fmt.Errorf("cannot register usage reporter: %w", err) + } + } else { + if err = mgr.Add(createUsageWarningJob(cfg, nginxChecker.getReadyCh())); err != nil { + return fmt.Errorf("cannot register usage warning job: %w", err) + } + } } if cfg.MetricsConfig.Enabled { @@ -198,6 +217,8 @@ func StartManager(cfg config.Config) error { controlConfigNSName: controlConfigNSName, gatewayPodConfig: cfg.GatewayPodConfig, metricsCollector: handlerCollector, + usageReportConfig: cfg.UsageReportConfig, + usageSecret: usageSecret, }) objects, objectLists := prepareFirstEventBatchPreparerArgs( @@ -434,6 +455,10 @@ func registerControllers( return nil } +// 10 min jitter is enough per telemetry destination recommendation +// For the default period of 24 hours, jitter will be 10min /(24*60)min = 0.0069 +const telemetryJitterFactor = 10.0 / (24 * 60) // added jitter is bound by jitterFactor * period + func createTelemetryJob( cfg config.Config, dataCollector telemetry.DataCollector, @@ -442,25 +467,65 @@ func createTelemetryJob( logger := cfg.Logger.WithName("telemetryJob") exporter := telemetry.NewLoggingExporter(cfg.Logger.WithName("telemetryExporter").V(1 /* debug */)) - worker := telemetry.CreateTelemetryJobWorker(logger, exporter, dataCollector) - - // 10 min jitter is enough per telemetry destination recommendation - // For the default period of 24 hours, jitter will be 10min /(24*60)min = 0.0069 - jitterFactor := 10.0 / (24 * 60) // added jitter is bound by jitterFactor * period - return &runnables.Leader{ Runnable: runnables.NewCronJob( runnables.CronJobConfig{ - Worker: worker, + Worker: telemetry.CreateTelemetryJobWorker(logger, exporter, dataCollector), Logger: logger, Period: cfg.TelemetryReportPeriod, - JitterFactor: jitterFactor, + JitterFactor: telemetryJitterFactor, ReadyCh: readyCh, }, ), } } +func createUsageReporterJob( + k8sClient client.Reader, + cfg config.Config, + usageSecret *usage.Secret, + readyCh <-chan struct{}, +) (*runnables.Leader, error) { + logger := cfg.Logger.WithName("usageReporter") + reporter, err := usage.NewNIMReporter( + usageSecret, + cfg.UsageReportConfig.ServerURL, + cfg.UsageReportConfig.InsecureSkipVerify, + ) + if err != nil { + return nil, err + } + + return &runnables.Leader{ + Runnable: runnables.NewCronJob(runnables.CronJobConfig{ + Worker: usage.CreateUsageJobWorker(logger, k8sClient, reporter, cfg), + Logger: logger, + Period: cfg.TelemetryReportPeriod, + JitterFactor: telemetryJitterFactor, + ReadyCh: readyCh, + }), + }, nil +} + +func createUsageWarningJob(cfg config.Config, readyCh <-chan struct{}) *runnables.LeaderOrNonLeader { + logger := cfg.Logger.WithName("usageReporter") + worker := func(_ context.Context) { + logger.Error( + errors.New("usage reporting not enabled"), + "Usage reporting must be enabled when using NGINX Plus; redeploy with usage reporting enabled", + ) + } + + return &runnables.LeaderOrNonLeader{ + Runnable: runnables.NewCronJob(runnables.CronJobConfig{ + Worker: worker, + Logger: logger, + Period: 1 * time.Hour, + ReadyCh: readyCh, + }), + } +} + func prepareFirstEventBatchPreparerArgs( gcName string, gwNsName *types.NamespacedName, diff --git a/internal/mode/static/staticfakes/fake_secret_storer.go b/internal/mode/static/staticfakes/fake_secret_storer.go new file mode 100644 index 0000000000..e7bf3e70b5 --- /dev/null +++ b/internal/mode/static/staticfakes/fake_secret_storer.go @@ -0,0 +1,104 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package staticfakes + +import ( + "sync" + + v1 "k8s.io/api/core/v1" +) + +type FakeSecretStorer struct { + DeleteStub func() + deleteMutex sync.RWMutex + deleteArgsForCall []struct { + } + SetStub func(*v1.Secret) + setMutex sync.RWMutex + setArgsForCall []struct { + arg1 *v1.Secret + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeSecretStorer) Delete() { + fake.deleteMutex.Lock() + fake.deleteArgsForCall = append(fake.deleteArgsForCall, struct { + }{}) + stub := fake.DeleteStub + fake.recordInvocation("Delete", []interface{}{}) + fake.deleteMutex.Unlock() + if stub != nil { + fake.DeleteStub() + } +} + +func (fake *FakeSecretStorer) DeleteCallCount() int { + fake.deleteMutex.RLock() + defer fake.deleteMutex.RUnlock() + return len(fake.deleteArgsForCall) +} + +func (fake *FakeSecretStorer) DeleteCalls(stub func()) { + fake.deleteMutex.Lock() + defer fake.deleteMutex.Unlock() + fake.DeleteStub = stub +} + +func (fake *FakeSecretStorer) Set(arg1 *v1.Secret) { + fake.setMutex.Lock() + fake.setArgsForCall = append(fake.setArgsForCall, struct { + arg1 *v1.Secret + }{arg1}) + stub := fake.SetStub + fake.recordInvocation("Set", []interface{}{arg1}) + fake.setMutex.Unlock() + if stub != nil { + fake.SetStub(arg1) + } +} + +func (fake *FakeSecretStorer) SetCallCount() int { + fake.setMutex.RLock() + defer fake.setMutex.RUnlock() + return len(fake.setArgsForCall) +} + +func (fake *FakeSecretStorer) SetCalls(stub func(*v1.Secret)) { + fake.setMutex.Lock() + defer fake.setMutex.Unlock() + fake.SetStub = stub +} + +func (fake *FakeSecretStorer) SetArgsForCall(i int) *v1.Secret { + fake.setMutex.RLock() + defer fake.setMutex.RUnlock() + argsForCall := fake.setArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeSecretStorer) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.deleteMutex.RLock() + defer fake.deleteMutex.RUnlock() + fake.setMutex.RLock() + defer fake.setMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeSecretStorer) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} diff --git a/internal/mode/static/telemetry/collector.go b/internal/mode/static/telemetry/collector.go index 4aa0b0cbae..e7dc692a7b 100644 --- a/internal/mode/static/telemetry/collector.go +++ b/internal/mode/static/telemetry/collector.go @@ -7,7 +7,7 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -86,7 +86,7 @@ func NewDataCollectorImpl( // Collect collects and returns telemetry Data. func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) { - nodeCount, err := collectNodeCount(ctx, c.cfg.K8sClientReader) + nodeCount, err := CollectNodeCount(ctx, c.cfg.K8sClientReader) if err != nil { return Data{}, fmt.Errorf("failed to collect node count: %w", err) } @@ -102,7 +102,7 @@ func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) { } var clusterID string - if clusterID, err = collectClusterID(ctx, c.cfg.K8sClientReader); err != nil { + if clusterID, err = CollectClusterID(ctx, c.cfg.K8sClientReader); err != nil { return Data{}, fmt.Errorf("failed to collect clusterID: %w", err) } @@ -120,7 +120,8 @@ func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) { return data, nil } -func collectNodeCount(ctx context.Context, k8sClient client.Reader) (int, error) { +// CollectNodeCount returns the number of nodes in the cluster. +func CollectNodeCount(ctx context.Context, k8sClient client.Reader) (int, error) { var nodes v1.NodeList if err := k8sClient.List(ctx, &nodes); err != nil { return 0, fmt.Errorf("failed to get NodeList: %w", err) @@ -202,9 +203,10 @@ func collectNGFReplicaCount(ctx context.Context, k8sClient client.Reader, podNSN return int(*replicaSet.Spec.Replicas), nil } -func collectClusterID(ctx context.Context, k8sClient client.Reader) (string, error) { +// CollectClusterID gets the UID of the kube-system namespace. +func CollectClusterID(ctx context.Context, k8sClient client.Reader) (string, error) { key := types.NamespacedName{ - Name: meta.NamespaceSystem, + Name: metav1.NamespaceSystem, } var kubeNamespace v1.Namespace if err := k8sClient.Get(ctx, key, &kubeNamespace); err != nil { diff --git a/internal/mode/static/telemetry/collector_test.go b/internal/mode/static/telemetry/collector_test.go index 56f5fc3df6..8b9827c90a 100644 --- a/internal/mode/static/telemetry/collector_test.go +++ b/internal/mode/static/telemetry/collector_test.go @@ -10,7 +10,6 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -111,7 +110,7 @@ var _ = Describe("Collector", Ordered, func() { kubeNamespace = &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: meta.NamespaceSystem, + Name: metav1.NamespaceSystem, UID: "test-uid", }, } diff --git a/internal/mode/static/usage/doc.go b/internal/mode/static/usage/doc.go new file mode 100644 index 0000000000..d700daf0f7 --- /dev/null +++ b/internal/mode/static/usage/doc.go @@ -0,0 +1,4 @@ +/* +Package usage is responsible for reporting NGINX Plus usage data. +*/ +package usage diff --git a/internal/mode/static/usage/job_worker.go b/internal/mode/static/usage/job_worker.go new file mode 100644 index 0000000000..3f039181e3 --- /dev/null +++ b/internal/mode/static/usage/job_worker.go @@ -0,0 +1,81 @@ +package usage + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/telemetry" +) + +func CreateUsageJobWorker( + logger logr.Logger, + k8sClient client.Reader, + reporter Reporter, + cfg config.Config, +) func(ctx context.Context) { + return func(ctx context.Context) { + nodeCount, err := telemetry.CollectNodeCount(ctx, k8sClient) + if err != nil { + logger.Error(err, "Failed to collect node count") + } + + podCount, err := GetTotalNGFPodCount(ctx, k8sClient) + if err != nil { + logger.Error(err, "Failed to collect replica count") + } + + clusterUID, err := telemetry.CollectClusterID(ctx, k8sClient) + if err != nil { + logger.Error(err, "Failed to collect cluster UID") + } + + clusterDetails := ClusterDetails{ + Metadata: Metadata{ + DisplayName: cfg.UsageReportConfig.ClusterDisplayName, + UID: clusterUID, + }, + NodeCount: int64(nodeCount), + PodDetails: PodDetails{ + CurrentPodCounts: CurrentPodsCount{ + DosCount: int64(0), + PodCount: int64(podCount), + WafCount: int64(0), + }, + }, + } + + if err := reporter.Report(ctx, clusterDetails); err != nil { + logger.Error(err, "Failed to report NGINX Plus usage") + } + } +} + +// GetTotalNGFPodCount returns the total count of NGF Pods in the cluster. +// Uses the "app.kubernetes.io/name" label with either value of "nginx-gateway" or "nginx-gateway-fabric". +func GetTotalNGFPodCount(ctx context.Context, k8sClient client.Reader) (int, error) { + labelKey := "app.kubernetes.io/name" + labelVals := map[string]struct{}{ + "nginx-gateway-fabric": {}, + "nginx-gateway": {}, + } + + var rsList appsv1.ReplicaSetList + if err := k8sClient.List(ctx, &rsList, client.HasLabels{labelKey}); err != nil { + return 0, fmt.Errorf("failed to list replicasets: %w", err) + } + + var count int + for _, rs := range rsList.Items { + val := rs.Labels[labelKey] + if _, ok := labelVals[val]; ok && rs.Spec.Replicas != nil { + count += int(*rs.Spec.Replicas) + } + } + + return count, nil +} diff --git a/internal/mode/static/usage/job_worker_test.go b/internal/mode/static/usage/job_worker_test.go new file mode 100644 index 0000000000..d7272ae28a --- /dev/null +++ b/internal/mode/static/usage/job_worker_test.go @@ -0,0 +1,158 @@ +package usage_test + +import ( + "context" + "testing" + "time" + + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/config" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/usage" + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/usage/usagefakes" +) + +func TestCreateUsageJobWorker(t *testing.T) { + g := NewWithT(t) + + replicas := int32(1) + ngfReplicaSet := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "nginx-gateway", + Name: "ngf-replicaset", + Labels: map[string]string{ + "app.kubernetes.io/name": "nginx-gateway", + }, + }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: &replicas, + }, + } + + ngfPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "nginx-gateway", + Name: "ngf-pod", + OwnerReferences: []metav1.OwnerReference{ + { + Kind: "ReplicaSet", + Name: "ngf-replicaset", + }, + }, + }, + } + + kubeSystem := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: metav1.NamespaceSystem, + UID: "1234abcd", + }, + } + + k8sClient := fake.NewFakeClient(&v1.Node{}, ngfReplicaSet, ngfPod, kubeSystem) + reporter := &usagefakes.FakeReporter{} + + worker := usage.CreateUsageJobWorker( + zap.New(), + k8sClient, + reporter, + config.Config{ + GatewayPodConfig: config.GatewayPodConfig{ + Namespace: "nginx-gateway", + Name: "ngf-pod", + }, + UsageReportConfig: &config.UsageReportConfig{ + ClusterDisplayName: "my-cluster", + }, + }, + ) + + expData := usage.ClusterDetails{ + Metadata: usage.Metadata{ + UID: "1234abcd", + DisplayName: "my-cluster", + }, + NodeCount: 1, + PodDetails: usage.PodDetails{ + CurrentPodCounts: usage.CurrentPodsCount{ + PodCount: 1, + }, + }, + } + + timeout := 10 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + worker(ctx) + _, data := reporter.ReportArgsForCall(0) + g.Expect(data).To(Equal(expData)) +} + +func TestGetTotalNGFPodCount(t *testing.T) { + g := NewWithT(t) + + rs1Replicas := int32(1) + rs1 := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "nginx-gateway", + Name: "ngf-replicaset1", + Labels: map[string]string{ + "app.kubernetes.io/name": "nginx-gateway", + }, + }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: &rs1Replicas, + }, + } + + rs2Replicas := int32(3) + rs2 := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "nginx-gateway-2", + Name: "ngf-replicaset2", + Labels: map[string]string{ + "app.kubernetes.io/name": "nginx-gateway-fabric", + }, + }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: &rs2Replicas, + }, + } + + rs3Replicas := int32(5) + rs3 := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "not-ngf", + }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: &rs3Replicas, + }, + } + + rs4 := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "nginx-gateway-3", + Name: "ngf-replicaset-nil", + Labels: map[string]string{ + "app.kubernetes.io/name": "nginx-gateway-fabric", + }, + }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: nil, + }, + } + + k8sClient := fake.NewFakeClient(rs1, rs2, rs3, rs4) + + expCount := 4 + count, err := usage.GetTotalNGFPodCount(context.Background(), k8sClient) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(count).To(Equal(expCount)) +} diff --git a/internal/mode/static/usage/reporter.go b/internal/mode/static/usage/reporter.go new file mode 100644 index 0000000000..461b3b8404 --- /dev/null +++ b/internal/mode/static/usage/reporter.go @@ -0,0 +1,144 @@ +package usage + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" +) + +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 . credentialsGetter +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 . Reporter + +const apiBasePath = "/api/platform/v1/k8s-usage" + +// ClusterDetails are the k8s usage details for the cluster. +type ClusterDetails struct { + // Metadata contains the cluster metadata. + Metadata Metadata `json:"metadata"` + // PodDetails contain the details about the NGF Pod. + PodDetails PodDetails `json:"pod_details"` + // NodeCount is the count of Nodes in the cluster. + NodeCount int64 `json:"node_count"` +} + +// Metadata contains the cluster metadata. +type Metadata struct { + // DisplayName is a user friendly resource name. It can be used to define + // a longer, and less constrained, name for a resource. + DisplayName string `json:"displayName"` + // UID is the unique identifier for the cluster. + UID string `json:"uid"` +} + +// PodDetails contain the details about the NGF Pod. +type PodDetails struct { + // CurrentPodsCount is the total count of NGF NGINX Plus Pods in the cluster. + CurrentPodCounts CurrentPodsCount `json:"current_pod_counts"` +} + +// CurrentPodsCount is the total count of NGF NGINX Plus Pods in the cluster. +type CurrentPodsCount struct { + // PodCount is the current count of NGF NGINX Plus Pods in the cluster. + PodCount int64 `json:"pod_count"` + // DosCount is the count of Pods with NAP DOS enabled in the cluster. Not applicable for NGF, + // but required as part of the payload. + DosCount int64 `json:"dos_count"` + // WafCount is the count of Pods with NAP WAF enabled in the cluster. Not applicable for NGF, + // but required as part of the payload. + WafCount int64 `json:"waf_count"` +} + +// credentialsGetter get the credentials for NGINX Plus usage reporting. +type credentialsGetter interface { + // GetCredentials returns the base64 encoded username and password from the Secret. + GetCredentials() ([]byte, []byte) +} + +// Reporter reports the NGINX Plus usage info to the provided collector. +type Reporter interface { + Report(context.Context, ClusterDetails) error +} + +// NIMReporter reports the NGINX Plus usage info to NGINX Instance Manager. +type NIMReporter struct { + // credentials contains the credentials for the usage collector. + credentials credentialsGetter + // baseURL is the base server URL of the usage collector. + baseURL *url.URL + // insecureSkipVerify controls whether the client verifies the server cert. Used in testing. + insecureSkipVerify bool +} + +// NewNIMReporter creates a new NIM usage reporter. +func NewNIMReporter( + credentials credentialsGetter, + baseURL string, + insecureSkipVerify bool, +) (*NIMReporter, error) { + serverURL, err := url.Parse(baseURL) + if err != nil { + return nil, fmt.Errorf("error parsing usage server URL: %w", err) + } + + return &NIMReporter{ + credentials: credentials, + baseURL: serverURL, + insecureSkipVerify: insecureSkipVerify, + }, nil +} + +// Report sends a PUT request with the provided data to the API endpoint configured in the Reporter. +// The clusterUID is used as the name in the API path. +func (r *NIMReporter) Report(ctx context.Context, data ClusterDetails) error { + buf, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("error marshaling usage data: %w", err) + } + + queryURL := r.baseURL.JoinPath(apiBasePath, data.Metadata.UID) + bodyReader := bytes.NewReader(buf) + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, queryURL.String(), bodyReader) + if err != nil { + return fmt.Errorf("error creating usage API HTTP request: %w", err) + } + + req.Header.Add("Content-Type", "application/json") + username, password := r.credentials.GetCredentials() + if username == nil || password == nil { + return errors.New("username or password not set for NGINX Plus usage reporting; unable to send reports. " + + "Ensure that the usage Secret exists and the username and password are set.") + } + req.SetBasicAuth(string(username), string(password)) + + client := http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: r.insecureSkipVerify, //nolint:gosec // used for testing + }, + }, + } + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("error sending usage report request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read the response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("non-200 response: %v; response body: %v", resp.StatusCode, string(body)) + } + + return nil +} diff --git a/internal/mode/static/usage/reporter_test.go b/internal/mode/static/usage/reporter_test.go new file mode 100644 index 0000000000..5099dfb591 --- /dev/null +++ b/internal/mode/static/usage/reporter_test.go @@ -0,0 +1,108 @@ +package usage + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" +) + +func TestReport(t *testing.T) { + g := NewWithT(t) + + data := ClusterDetails{ + Metadata: Metadata{ + UID: "12345abcde", + DisplayName: "my-cluster", + }, + NodeCount: 9, + PodDetails: PodDetails{ + CurrentPodCounts: CurrentPodsCount{ + PodCount: 12, + }, + }, + } + + secret := &v1.Secret{ + Data: map[string][]byte{ + "username": []byte("user"), + "password": []byte("pass"), + }, + } + + store := NewUsageSecret() + store.Set(secret) + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var reqData ClusterDetails + g.Expect(json.NewDecoder(r.Body).Decode(&reqData)).To(Succeed()) + g.Expect(reqData).To(Equal(data)) + + g.Expect(r.URL.Path).To(Equal(fmt.Sprintf("%s/%s", apiBasePath, data.Metadata.UID))) + g.Expect(r.Method).To(Equal(http.MethodPut)) + user, pass, ok := r.BasicAuth() + g.Expect(ok).To(BeTrue()) + g.Expect(user).To(Equal("user")) + g.Expect(pass).To(Equal("pass")) + + contentType, ok := r.Header["Content-Type"] + g.Expect(ok).To(BeTrue()) + g.Expect(contentType[0]).To(Equal("application/json")) + + w.WriteHeader(200) + }), + ) + defer server.Close() + + insecureSkipVerify := false + reporter, err := NewNIMReporter(store, server.URL, insecureSkipVerify) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(reporter.Report(context.Background(), data)).To(Succeed()) +} + +func TestReport_NoCredentials(t *testing.T) { + g := NewWithT(t) + insecureSkipVerify := false + reporter, err := NewNIMReporter(NewUsageSecret(), "", insecureSkipVerify) + g.Expect(err).ToNot(HaveOccurred()) + + err = reporter.Report(context.Background(), ClusterDetails{}) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("username or password not set")) +} + +func TestReport_ServerError(t *testing.T) { + g := NewWithT(t) + + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(500) + }), + ) + defer server.Close() + + secret := &v1.Secret{ + Data: map[string][]byte{ + "username": []byte("user"), + "password": []byte("pass"), + }, + } + + store := NewUsageSecret() + store.Set(secret) + + insecureSkipVerify := false + reporter, err := NewNIMReporter(store, server.URL, insecureSkipVerify) + g.Expect(err).ToNot(HaveOccurred()) + + err = reporter.Report(context.Background(), ClusterDetails{}) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("non-200 response")) +} diff --git a/internal/mode/static/usage/secret.go b/internal/mode/static/usage/secret.go new file mode 100644 index 0000000000..01d333cbdd --- /dev/null +++ b/internal/mode/static/usage/secret.go @@ -0,0 +1,48 @@ +package usage + +import ( + "sync" + + v1 "k8s.io/api/core/v1" +) + +// Secret implements the SecretStorer interface. +type Secret struct { + secret *v1.Secret + lock *sync.Mutex +} + +// NewUsageSecret creates a new Secret wrapper. +func NewUsageSecret() *Secret { + return &Secret{ + lock: &sync.Mutex{}, + } +} + +// Set stores the updated Secre. +func (s *Secret) Set(secret *v1.Secret) { + s.lock.Lock() + defer s.lock.Unlock() + + s.secret = secret +} + +// Delete nullifies the Secret value. +func (s *Secret) Delete() { + s.lock.Lock() + defer s.lock.Unlock() + + s.secret = nil +} + +// GetCredentials returns the base64 encoded username and password from the Secret. +func (s *Secret) GetCredentials() ([]byte, []byte) { + s.lock.Lock() + defer s.lock.Unlock() + + if s.secret != nil { + return s.secret.Data["username"], s.secret.Data["password"] + } + + return nil, nil +} diff --git a/internal/mode/static/usage/secret_test.go b/internal/mode/static/usage/secret_test.go new file mode 100644 index 0000000000..0320c72c16 --- /dev/null +++ b/internal/mode/static/usage/secret_test.go @@ -0,0 +1,68 @@ +package usage + +import ( + "testing" + + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestSet(t *testing.T) { + store := NewUsageSecret() + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret", + Namespace: "custom", + }, + } + + g := NewWithT(t) + g.Expect(store.secret).To(BeNil()) + + store.Set(secret) + g.Expect(store.secret).To(Equal(secret)) +} + +func TestDelete(t *testing.T) { + store := NewUsageSecret() + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret", + Namespace: "custom", + }, + } + + g := NewWithT(t) + store.Set(secret) + g.Expect(store.secret).To(Equal(secret)) + + store.Delete() + g.Expect(store.secret).To(BeNil()) +} + +func TestGetCredentials(t *testing.T) { + store := NewUsageSecret() + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret", + Namespace: "custom", + }, + Data: map[string][]byte{ + "username": []byte("user"), + "password": []byte("pass"), + }, + } + + g := NewWithT(t) + + user, pass := store.GetCredentials() + g.Expect(user).To(BeNil()) + g.Expect(pass).To(BeNil()) + + store.Set(secret) + + user, pass = store.GetCredentials() + g.Expect(user).To(Equal([]byte("user"))) + g.Expect(pass).To(Equal([]byte("pass"))) +} diff --git a/internal/mode/static/usage/usagefakes/fake_credentials_getter.go b/internal/mode/static/usage/usagefakes/fake_credentials_getter.go new file mode 100644 index 0000000000..317565732a --- /dev/null +++ b/internal/mode/static/usage/usagefakes/fake_credentials_getter.go @@ -0,0 +1,103 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package usagefakes + +import ( + "sync" +) + +type FakeCredentialsGetter struct { + GetCredentialsStub func() ([]byte, []byte) + getCredentialsMutex sync.RWMutex + getCredentialsArgsForCall []struct { + } + getCredentialsReturns struct { + result1 []byte + result2 []byte + } + getCredentialsReturnsOnCall map[int]struct { + result1 []byte + result2 []byte + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeCredentialsGetter) GetCredentials() ([]byte, []byte) { + fake.getCredentialsMutex.Lock() + ret, specificReturn := fake.getCredentialsReturnsOnCall[len(fake.getCredentialsArgsForCall)] + fake.getCredentialsArgsForCall = append(fake.getCredentialsArgsForCall, struct { + }{}) + stub := fake.GetCredentialsStub + fakeReturns := fake.getCredentialsReturns + fake.recordInvocation("GetCredentials", []interface{}{}) + fake.getCredentialsMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeCredentialsGetter) GetCredentialsCallCount() int { + fake.getCredentialsMutex.RLock() + defer fake.getCredentialsMutex.RUnlock() + return len(fake.getCredentialsArgsForCall) +} + +func (fake *FakeCredentialsGetter) GetCredentialsCalls(stub func() ([]byte, []byte)) { + fake.getCredentialsMutex.Lock() + defer fake.getCredentialsMutex.Unlock() + fake.GetCredentialsStub = stub +} + +func (fake *FakeCredentialsGetter) GetCredentialsReturns(result1 []byte, result2 []byte) { + fake.getCredentialsMutex.Lock() + defer fake.getCredentialsMutex.Unlock() + fake.GetCredentialsStub = nil + fake.getCredentialsReturns = struct { + result1 []byte + result2 []byte + }{result1, result2} +} + +func (fake *FakeCredentialsGetter) GetCredentialsReturnsOnCall(i int, result1 []byte, result2 []byte) { + fake.getCredentialsMutex.Lock() + defer fake.getCredentialsMutex.Unlock() + fake.GetCredentialsStub = nil + if fake.getCredentialsReturnsOnCall == nil { + fake.getCredentialsReturnsOnCall = make(map[int]struct { + result1 []byte + result2 []byte + }) + } + fake.getCredentialsReturnsOnCall[i] = struct { + result1 []byte + result2 []byte + }{result1, result2} +} + +func (fake *FakeCredentialsGetter) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.getCredentialsMutex.RLock() + defer fake.getCredentialsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeCredentialsGetter) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} diff --git a/internal/mode/static/usage/usagefakes/fake_reporter.go b/internal/mode/static/usage/usagefakes/fake_reporter.go new file mode 100644 index 0000000000..b33a8759aa --- /dev/null +++ b/internal/mode/static/usage/usagefakes/fake_reporter.go @@ -0,0 +1,114 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package usagefakes + +import ( + "context" + "sync" + + "github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/usage" +) + +type FakeReporter struct { + ReportStub func(context.Context, usage.ClusterDetails) error + reportMutex sync.RWMutex + reportArgsForCall []struct { + arg1 context.Context + arg2 usage.ClusterDetails + } + reportReturns struct { + result1 error + } + reportReturnsOnCall map[int]struct { + result1 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeReporter) Report(arg1 context.Context, arg2 usage.ClusterDetails) error { + fake.reportMutex.Lock() + ret, specificReturn := fake.reportReturnsOnCall[len(fake.reportArgsForCall)] + fake.reportArgsForCall = append(fake.reportArgsForCall, struct { + arg1 context.Context + arg2 usage.ClusterDetails + }{arg1, arg2}) + stub := fake.ReportStub + fakeReturns := fake.reportReturns + fake.recordInvocation("Report", []interface{}{arg1, arg2}) + fake.reportMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeReporter) ReportCallCount() int { + fake.reportMutex.RLock() + defer fake.reportMutex.RUnlock() + return len(fake.reportArgsForCall) +} + +func (fake *FakeReporter) ReportCalls(stub func(context.Context, usage.ClusterDetails) error) { + fake.reportMutex.Lock() + defer fake.reportMutex.Unlock() + fake.ReportStub = stub +} + +func (fake *FakeReporter) ReportArgsForCall(i int) (context.Context, usage.ClusterDetails) { + fake.reportMutex.RLock() + defer fake.reportMutex.RUnlock() + argsForCall := fake.reportArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeReporter) ReportReturns(result1 error) { + fake.reportMutex.Lock() + defer fake.reportMutex.Unlock() + fake.ReportStub = nil + fake.reportReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeReporter) ReportReturnsOnCall(i int, result1 error) { + fake.reportMutex.Lock() + defer fake.reportMutex.Unlock() + fake.ReportStub = nil + if fake.reportReturnsOnCall == nil { + fake.reportReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.reportReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeReporter) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.reportMutex.RLock() + defer fake.reportMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeReporter) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ usage.Reporter = new(FakeReporter) diff --git a/site/content/how-to/monitoring/troubleshooting.md b/site/content/how-to/monitoring/troubleshooting.md index 10cd3606e3..e7911d01f8 100644 --- a/site/content/how-to/monitoring/troubleshooting.md +++ b/site/content/how-to/monitoring/troubleshooting.md @@ -14,7 +14,9 @@ This topic describes possible issues users might encounter when using NGINX Gate #### Description -Depending on your environment's configuration, the control plane may not have the proper permissions to reload NGINX. The NGINX configuration will not be applied and you will see the following error in the _nginx-gateway_ logs: `failed to reload NGINX: failed to send the HUP signal to NGINX main: operation not permitted` +Depending on your environment's configuration, the control plane may not have the proper permissions to reload NGINX. The NGINX configuration will not be applied and you will see the following error in the _nginx-gateway_ logs: + +`failed to reload NGINX: failed to send the HUP signal to NGINX main: operation not permitted` #### Resolution @@ -22,3 +24,15 @@ To resolve this issue you will need to set `allowPrivilegeEscalation` to `true`. - If using Helm, you can set the `nginxGateway.securityContext.allowPrivilegeEscalation` value. - If using the manifests directly, you can update this field under the `nginx-gateway` container's `securityContext`. + +### Usage Reporting errors + +#### Description + +If using NGINX Gateway Fabric with NGINX Plus as the data plane, you will see the following error in the _nginx-gateway_ logs if you have not enabled Usage Reporting: + +`usage reporting not enabled` + +#### Resolution + +To resolve this issue, enable Usage Reporting by following the [Usage Reporting]({{< relref "installation/usage-reporting.md" >}}) guide. diff --git a/site/content/installation/installing-ngf/helm.md b/site/content/installation/installing-ngf/helm.md index a7ffd02300..7ed6eaba28 100644 --- a/site/content/installation/installing-ngf/helm.md +++ b/site/content/installation/installing-ngf/helm.md @@ -33,22 +33,22 @@ To complete this guide, you'll need to install: To install the latest stable release of NGINX Gateway Fabric in the **nginx-gateway** namespace, run the following command: -- For NGINX: +##### For NGINX ```shell helm install ngf oci://ghcr.io/nginxinc/charts/nginx-gateway-fabric --create-namespace -n nginx-gateway ``` -- For NGINX Plus: +##### For NGINX Plus - {{< note >}}Replace `private-registry.nginx.com` with the proper registry for your NGINX Plus image, and if applicable, replace `nginx-plus-registry-secret` with your Secret name containing the registry credentials. {{< /note >}} + {{< note >}}Replace `private-registry.nginx.com` with the proper registry for your NGINX Plus image, and if applicable, replace `nginx-plus-registry-secret` with your Secret name containing the registry credentials.{{< /note >}} + + {{< important >}}Ensure that you [Enable Usage Reporting]({{< relref "installation/usage-reporting.md" >}}) when installing.{{< /important >}} ```shell helm install ngf oci://ghcr.io/nginxinc/charts/nginx-gateway-fabric --set nginx.image.repository=private-registry.nginx.com/nginx-gateway-fabric/nginx-plus --set nginx.plus=true --set serviceAccount.imagePullSecret=nginx-plus-registry-secret --create-namespace -n nginx-gateway ``` - - `ngf` is the name of the release, and can be changed to any name you want. This name is added as a prefix to the Deployment name. If the namespace already exists, you can omit the optional `--create-namespace` flag. If you want the latest version from the **main** branch, add `--version 0.0.0-edge` to your install command. diff --git a/site/content/installation/installing-ngf/manifests.md b/site/content/installation/installing-ngf/manifests.md index 94ca6ab1aa..67c91c7261 100644 --- a/site/content/installation/installing-ngf/manifests.md +++ b/site/content/installation/installing-ngf/manifests.md @@ -33,7 +33,7 @@ Deploying NGINX Gateway Fabric with Kubernetes manifests takes only a few steps. #### Stable release ```shell - kubectl apply -f https://github.com/nginxinc/nginx-gateway-fabric/releases/download/v1.0.0/crds.yaml + kubectl apply -f https://github.com/nginxinc/nginx-gateway-fabric/releases/download/v1.1.0/crds.yaml ``` #### Edge version @@ -53,19 +53,35 @@ Deploying NGINX Gateway Fabric with Kubernetes manifests takes only a few steps. #### Stable release +##### For NGINX + ```shell - kubectl apply -f https://github.com/nginxinc/nginx-gateway-fabric/releases/download/v1.0.0/nginx-gateway.yaml + kubectl apply -f https://github.com/nginxinc/nginx-gateway-fabric/releases/download/v1.1.0/nginx-gateway.yaml + ``` + +##### For NGINX Plus + + Download the [deployment YAML](https://github.com/nginxinc/nginx-gateway-fabric/releases/download/v1.2.0/nginx-plus-gateway.yaml). + + Update the `nginx-plus-gateway.yaml` file to include your chosen NGINX Plus image from the F5 Container registry or your custom image. + + {{< important >}}Ensure that you [Enable Usage Reporting]({{< relref "installation/usage-reporting.md" >}}) before applying.{{< /important >}} + + ```shell + kubectl apply -f nginx-plus-gateway.yaml ``` #### Edge version -- For NGINX: +##### For NGINX ```shell kubectl apply -f deploy/manifests/nginx-gateway.yaml ``` -- For NGINX Plus +##### For NGINX Plus + + {{< important >}}Ensure that you [Enable Usage Reporting]({{< relref "installation/usage-reporting.md" >}}) before applying.{{< /important >}} ```shell kubectl apply -f deploy/manifests/nginx-plus-gateway.yaml @@ -77,13 +93,13 @@ Deploying NGINX Gateway Fabric with Kubernetes manifests takes only a few steps. We support a subset of the additional features provided by the Gateway API experimental channel. To enable the experimental features of Gateway API which are supported by NGINX Gateway Fabric: -- For NGINX: +##### For NGINX ```shell kubectl apply -f deploy/manifests/nginx-gateway-experimental.yaml ``` -- For NGINX Plus +##### For NGINX Plus ```shell kubectl apply -f deploy/manifests/nginx-plus-gateway-experimental.yaml @@ -148,14 +164,14 @@ To upgrade NGINX Gateway Fabric and get the latest features and improvements, ta - To upgrade the Custom Resource Definitions (CRDs), run: ```shell - kubectl apply -f https://github.com/nginxinc/nginx-gateway-fabric/releases/download/v1.0.0/crds.yaml + kubectl apply -f https://github.com/nginxinc/nginx-gateway-fabric/releases/download/v1.1.0/crds.yaml ``` 1. **Upgrade NGINX Gateway Fabric deployment:** - To upgrade the deployment, run: ```shell - kubectl apply -f https://github.com/nginxinc/nginx-gateway-fabric/releases/download/v1.0.0/nginx-gateway.yaml + kubectl apply -f https://github.com/nginxinc/nginx-gateway-fabric/releases/download/v1.1.0/nginx-gateway.yaml ``` @@ -218,11 +234,11 @@ Follow these steps to uninstall NGINX Gateway Fabric and Gateway API from your K - To remove NGINX Gateway Fabric and its custom resource definitions (CRDs), run: ```shell - kubectl delete -f https://github.com/nginxinc/nginx-gateway-fabric/releases/download/v1.0.0/nginx-gateway.yaml + kubectl delete -f https://github.com/nginxinc/nginx-gateway-fabric/releases/download/v1.1.0/nginx-gateway.yaml ``` ```shell - kubectl delete -f https://github.com/nginxinc/nginx-gateway-fabric/releases/download/v1.0.0/crds.yaml + kubectl delete -f https://github.com/nginxinc/nginx-gateway-fabric/releases/download/v1.1.0/crds.yaml ``` 1. **Remove the Gateway API resources:** diff --git a/site/content/installation/usage-reporting.md b/site/content/installation/usage-reporting.md new file mode 100644 index 0000000000..a3f1e5c96e --- /dev/null +++ b/site/content/installation/usage-reporting.md @@ -0,0 +1,184 @@ +--- +title: "Enabling Usage Reporting for NGINX Plus" +description: "This page outlines how to enable Usage Reporting for NGINX Gateway Fabric and how to view the usage data through the API." +weight: 1000 +toc: true +docs: "DOCS-000" +--- + +## Overview + +Usage Reporting connects to the NGINX Instance Manager and reports the number of Nodes and NGINX Gateway Fabric Pods in the cluster. + +To use Usage Reporting, you must have access to [NGINX Instance Manager](https://www.nginx.com/products/nginx-management-suite/instance-manager). Usage Reporting is a requirement of the new Flexible Consumption Program for NGINX Gateway Fabric, used to calculate costs. **This only applies if using NGINX Plus as the data plane.** Usage is reported every 24 hours. + +## Requirements + +Usage Reporting needs to be configured when deploying NGINX Gateway Fabric. + +To enable Usage Reporting, you must have the following: + +- NGINX Gateway Fabric 1.2.0 or later +- [NGINX Instance Manager 2.11](https://docs.nginx.com/nginx-management-suite) or later + +In addition to the software requirements, you will need: + +- Access to an NGINX Instance Manager username and password for basic authentication. You will also need the URL of your NGINX Instance Manager system. The Usage Reporting user account must have access to the `/api/platform/v1/k8s-usage` endpoint. +- Access to the Kubernetes cluster where NGINX Gateway Fabric is deployed, with the ability to deploy a Kubernetes Secret. + +## Adding a User Account to NGINX Instance Manager + +1. Create a role following the steps in the [Create Roles](https://docs.nginx.com/nginx-management-suite/admin-guides/rbac/create-roles/) section of the NGINX Instance Manager documentation. Select these permissions in step 6 for the role: + +- Module: Instance Manager +- Feature: NGINX Plus Usage +- Access: CRUD + +1. Create a user account following the steps in the [Create New Users](https://docs.nginx.com/nginx-management-suite/admin-guides/authentication/basic-authentication/#create-users) section of the NGINX Instance Manager documentation. In step 6, assign the user to the role created above. Note that currently only "Basic Auth" authentication is supported for Usage Reporting purposes. + +## Enabling Usage Reporting in NGINX Gateway Fabric + +### Adding Credentials to a Kubernetes Secret + +To make the credentials available to NGINX Gateway Fabric to connect to the NGINX Instance Manager, we need to create a Kubernetes Secret. + +1. The username and password should be base64 encoded and stored in the Secret. In the following example, the username is `foo` and the password is `bar`. Run a similar command to generate the base64 encoded strings of your username and password: + + ```shell + echo -n 'foo' | base64 + # Zm9v + echo -n 'bar' | base64 + # YmFy + ``` + +1. Create the following Secret in your Kubernetes cluster, replacing the username and password with your generated strings. You can rename the Secret if desired, and create it in any Namespace you want. + + ```yaml + apiVersion: v1 + kind: Secret + metadata: + name: ngf-usage-auth + namespace: nginx-gateway + type: kubernetes.io/basic-auth + data: + username: Zm9v # base64 representation of 'foo' obtained in step 1 + password: YmFy # base64 representation of 'bar' obtained in step 1 + ``` + + If you need to update the basic-auth credentials at any time, update the `username` and `password` fields and apply the changes. NGINX Gateway Fabric will automatically detect the changes and use the new username and password without redeployment. + +### Install NGINX Gateway Fabric with Usage Reporting enabled + +When installing NGINX Gateway Fabric, a few configuration options need to be specified in order to enable Usage Reporting. You should follow the normal [installation](https://docs.nginx.com/nginx-gateway-fabric/installation/) steps using your preferred method, but ensure you include the following options: + +If using Helm, the `nginx.usage` values should be set as necessary: + +- `secretName` should be the `namespace/name` of the credentials Secret you created. Using our example, it would be `nginx-gateway/ngf-usage-auth`. This field is required. +- `serverURL` is the base server URL of the NGINX Instance Manager. This field is required. +- `clusterName` is an optional display name in the API for the usage data object. + +If using manifests, the following command-line options should be set as necessary on the `nginx-gateway` container: + +- `--usage-report-secret` should be the `namespace/name` of the credentials Secret you created. Using our example, it would be `nginx-gateway/ngf-usage-auth`. This field is required. +- `--usage-report-server-url` is the base server URL of the NGINX Instance Manager. This field is required. +- `--usage-report-cluster-name` is an optional display name in the API for the usage data object. + +Your NGINX Gateway Fabric Pods should also have one of the following labels: + +- `app.kubernetes.io/name=nginx-gateway` +- `app.kubernetes.io/name=nginx-gateway-fabric` + +{{< note >}}The default installation of NGINX Gateway Fabric already includes at least one of these labels.{{< /note >}} + +## Viewing Usage Data from the NGINX Instance Manager API + +NGINX Gateway Fabric sends the number of its instances and nodes in the cluster to NGINX Instance Manager every 24 hours. To view the usage data, query the NGINX Instance Manager API. The usage data is available at the following endpoint (replace `nim.example.com` with your server URL, and set the proper credentials in the `--user` field): + +```shell +curl --user "foo:bar" https://nim.example.com/api/platform/v1/k8s-usage +``` + +```json +{ + "items": [ + { + "max_node_count": 5, + "metadata": { + "createTime": "2023-01-27T09:12:33.001Z", + "displayName": "my-cluster", + "monthReturned": "May", + "uid": "d290f1ee-6c54-4b01-90e6-d701748f0851", + "updateTime": "2023-01-29T10:12:33.001Z" + }, + "node_count": 4, + "pod_details": { + "current_pod_counts": { + "dos_count": 0, + "pod_count": 15, + "waf_count": 0 + }, + "max_pod_counts": { + "max_dos_count": 0, + "max_pod_count": 25, + "max_waf_count": 0 + } + } + }, + { + "max_node_count": 3, + "metadata": { + "createTime": "2023-01-25T09:12:33.001Z", + "displayName": "my-cluster2", + "monthReturned": "May", + "uid": "12tgb8ug-g8ik-bs7h-gj3j-hjitk672946hb", + "updateTime": "2023-01-26T10:12:33.001Z" + }, + "node_count": 3, + "pod_details": { + "current_pod_counts": { + "dos_count": 0, + "pod_count": 5, + "waf_count": 0 + }, + "max_pod_counts": { + "max_dos_count": 0, + "max_pod_count": 15, + "max_waf_count": 0 + } + } + } + ] +} +``` + +You can also query the usage data for a specific cluster by specifying the cluster uid in the endpoint, for example: + +```shell +curl --user "foo:bar" https://nim.example.com/api/platform/v1/k8s-usage/d290f1ee-6c54-4b01-90e6-d701748f0851 +``` + +```json +{ + "max_node_count": 5, + "metadata": { + "createTime": "2023-01-27T09:12:33.001Z", + "displayName": "my-cluster", + "monthReturned": "May", + "uid": "d290f1ee-6c54-4b01-90e6-d701748f0851", + "updateTime": "2023-01-29T10:12:33.001Z" + }, + "node_count": 4, + "pod_details": { + "current_pod_counts": { + "dos_count": 0, + "pod_count": 15, + "waf_count": 0 + }, + "max_pod_counts": { + "max_dos_count": 0, + "max_pod_count": 25, + "max_waf_count": 0 + } + } +} +``` diff --git a/site/content/reference/cli-help.md b/site/content/reference/cli-help.md index 1c7c661e55..c99940bd1a 100644 --- a/site/content/reference/cli-help.md +++ b/site/content/reference/cli-help.md @@ -36,6 +36,10 @@ _Usage_: | _health-port_ | _int_ | Set the port where the health probe server is exposed. An integer between 1024 - 65535 (Default: `8081`). | | _leader-election-disable_ | _bool_ | Disable leader election, which is used to avoid multiple replicas of the NGINX Gateway Fabric reporting the status of the Gateway API resources. If disabled, all replicas of NGINX Gateway Fabric will update the statuses of the Gateway API resources (Default: `false`). | | _leader-election-lock-name_ | _string_ | The name of the leader election lock. A lease object with this name will be created in the same namespace as the controller (Default: `"nginx-gateway-leader-election-lock"`). | +| _usage-report-secret_ | _string_ | The namespace/name of the Secret containing the credentials for NGINX Plus usage reporting. | +| _usage-report-server-url_ | _string_ | The base server URL of the NGINX Plus usage reporting server. | +| _usage-report-cluster-name_ | _string_ | The display name of the Kubernetes cluster in the NGINX Plus usage reporting server. | +| _usage-report-skip-verify_ | _bool_ | Disable client verification of the NGINX Plus usage reporting server certificate. | {{% /bootstrap-table %}} ## Sleep