Integrate telemetry exporter (nginx#1656)

Problem:
Integrate the exporter library
https://github.com/nginxinc/telemetry-exporter so that it is possible
to send product telemetry data to an endpoint.

Solution:
- Integrate the exporter library.
- Update existing telemetry data struct to use the common data struct
  defined in the exporter library.
- Refactor existing telemetry data structs to adhere to the exporter
  library requirements.
- Generate scheme and Attributes for data structs.
- Allow configuring telemetry endpoint params via build flags. If the
  telemetry endpoint is not specified, NGF will log data points to the
  debug log (existing behavior).
- Add root CA certs to NGF image so that NGF can verify the cert of
  the telemetry service.

Testing:
- Unit tests
- Manual testing of build flags validation.
- Manual testing that NGF sends data to an OTel collector.
- Manual testing that NGF sends data to a dev F5 telemetry service.

CLOSES -- nginx#1377

Author: pleshakov

.goreleaser.yml

@@ -15,7 +15,7 @@ builds:
     asmflags:
       - all=-trimpath={{.Env.GOPATH}}
     ldflags:
-      - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} -X main.telemetryReportPeriod=24h
+      - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} -X main.telemetryReportPeriod=24h -X main.telemetryEndpointInsecure=false
     main: ./cmd/gateway/
     binary: gateway
 

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ repos:
     rev: v4.5.0
     hooks:
       - id: trailing-whitespace
-        exclude: (^tests/results/)
+        exclude: (^tests/results/|\.avdl$|_generated.go$)
       - id: end-of-file-fixer
       - id: check-yaml
         args: [--allow-multiple-documents]

Makefile

@@ -9,12 +9,19 @@ NJS_DIR = internal/mode/static/nginx/modules/src
 NGINX_DOCKER_BUILD_PLUS_ARGS = --secret id=nginx-repo.crt,src=nginx-repo.crt --secret id=nginx-repo.key,src=nginx-repo.key
 BUILD_AGENT=local
 TELEMETRY_REPORT_PERIOD = 24h # also configured in goreleaser.yml
+
+# FIXME(pleshakov) - TELEMETRY_ENDPOINT will have the default value of F5 telemetry service once we're ready
+# to report. https://github.com/nginxinc/nginx-gateway-fabric/issues/1563
+# Also, we will need to set it in goreleaser.yml
+TELEMETRY_ENDPOINT =# if empty, NGF will report telemetry in its logs at debug level.
+
+TELEMETRY_ENDPOINT_INSECURE = false # also configured in goreleaser.yml
 GW_API_VERSION = 1.0.0
 INSTALL_WEBHOOK = false
 NODE_VERSION = $(shell cat .nvmrc)
 
 # go build flags - should not be overridden by the user
-GO_LINKER_FlAGS_VARS = -X main.version=${VERSION} -X main.commit=${GIT_COMMIT} -X main.date=${DATE} -X main.telemetryReportPeriod=${TELEMETRY_REPORT_PERIOD}
+GO_LINKER_FlAGS_VARS = -X main.version=${VERSION} -X main.commit=${GIT_COMMIT} -X main.date=${DATE} -X main.telemetryReportPeriod=${TELEMETRY_REPORT_PERIOD} -X main.telemetryEndpoint=${TELEMETRY_ENDPOINT} -X main.telemetryEndpointInsecure=${TELEMETRY_ENDPOINT_INSECURE}
 GO_LINKER_FLAGS_OPTIMIZATIONS = -s -w
 GO_LINKER_FLAGS = $(GO_LINKER_FLAGS_OPTIMIZATIONS) $(GO_LINKER_FlAGS_VARS)
 

build/Dockerfile

@@ -26,6 +26,9 @@ COPY dist/gateway_linux_$TARGETARCH*/gateway /usr/bin/
 RUN setcap 'cap_kill=+ep' /usr/bin/gateway
 
 FROM scratch as common
+# CA certs are needed for telemetry report and NGINX Plus usage report features, so that
+# NGF can verify the server's certificate.
+COPY --from=builder --link /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 USER 102:1001
 ARG BUILD_AGENT
 ENV BUILD_AGENT=${BUILD_AGENT}

cmd/gateway/commands.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strconv"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -160,6 +161,17 @@ func createStaticModeCommand() *cobra.Command {
 				return fmt.Errorf("error parsing telemetry report period: %w", err)
 			}
 
+			if telemetryEndpoint != "" {
+				if err := validateEndpoint(telemetryEndpoint); err != nil {
+					return fmt.Errorf("error validating telemetry endpoint: %w", err)
+				}
+			}
+
+			telemetryEndpointInsecure, err := strconv.ParseBool(telemetryEndpointInsecure)
+			if err != nil {
+				return fmt.Errorf("error parsing telemetry endpoint insecure: %w", err)
+			}
+
 			var gwNsName *types.NamespacedName
 			if cmd.Flags().Changed(gatewayFlag) {
 				gwNsName = &gateway.value
@@ -211,8 +223,10 @@ func createStaticModeCommand() *cobra.Command {
 				},
 				UsageReportConfig: usageReportConfig,
 				ProductTelemetryConfig: config.ProductTelemetryConfig{
-					TelemetryReportPeriod: period,
-					Enabled:               !disableProductTelemetry,
+					ReportPeriod:     period,
+					Enabled:          !disableProductTelemetry,
+					Endpoint:         telemetryEndpoint,
+					EndpointInsecure: telemetryEndpointInsecure,
 				},
 				Plus:                 plus,
 				Version:              version,

cmd/gateway/main.go

@@ -13,6 +13,10 @@ var (
 
 	// telemetryReportPeriod is the period at which telemetry reports are sent.
 	telemetryReportPeriod string
+	// telemetryEndpoint is the endpoint to which telemetry reports are sent.
+	telemetryEndpoint string
+	// telemetryEndpointInsecure controls whether TLS should be used when sending telemetry reports.
+	telemetryEndpointInsecure string
 )
 
 func main() {

cmd/gateway/validation.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"net/url"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"k8s.io/apimachinery/pkg/types"
@@ -133,6 +134,35 @@ func validateIP(ip string) error {
 	return nil
 }
 
+// validateEndpoint validates an endpoint, which is <host>:<port> where host is either a hostname or an IP address.
+func validateEndpoint(endpoint string) error {
+	host, port, err := net.SplitHostPort(endpoint)
+	if err != nil {
+		return fmt.Errorf("%q must be in the format <host>:<port>: %w", endpoint, err)
+	}
+
+	portVal, err := strconv.ParseInt(port, 10, 16)
+	if err != nil {
+		return fmt.Errorf("port must be a valid number: %w", err)
+	}
+
+	if portVal < 1 || portVal > 65535 {
+		return fmt.Errorf("port outside of valid port range [1 - 65535]: %v", port)
+	}
+
+	if err := validateIP(host); err == nil {
+		return nil
+	}
+
+	if errs := validation.IsDNS1123Subdomain(host); len(errs) == 0 {
+		return nil
+	}
+
+	// we don't know if the user intended to use a hostname or an IP address,
+	// so we return a generic error message
+	return fmt.Errorf("%q must be in the format <host>:<port>", endpoint)
+}
+
 // validatePort makes sure a given port is inside the valid port range for its usage
 func validatePort(port int) error {
 	if port < 1024 || port > 65535 {

cmd/gateway/validation_test.go

@@ -419,6 +419,73 @@ func TestValidateIP(t *testing.T) {
 	}
 }
 
+func TestValidateEndpoint(t *testing.T) {
+	tests := []struct {
+		name   string
+		endp   string
+		expErr bool
+	}{
+		{
+			name:   "valid endpoint with hostname",
+			endp:   "localhost:8080",
+			expErr: false,
+		},
+		{
+			name:   "valid endpoint with IPv4",
+			endp:   "1.2.3.4:8080",
+			expErr: false,
+		},
+		{
+			name:   "valid endpoint with IPv6",
+			endp:   "[::1]:8080",
+			expErr: false,
+		},
+		{
+			name:   "invalid port - 1",
+			endp:   "localhost:0",
+			expErr: true,
+		},
+		{
+			name:   "invalid port - 2",
+			endp:   "localhost:65536",
+			expErr: true,
+		},
+		{
+			name:   "missing port with hostname",
+			endp:   "localhost",
+			expErr: true,
+		},
+		{
+			name:   "missing port with IPv4",
+			endp:   "1.2.3.4",
+			expErr: true,
+		},
+		{
+			name:   "missing port with IPv6",
+			endp:   "[::1]",
+			expErr: true,
+		},
+		{
+			name:   "invalid hostname or IP",
+			endp:   "loc@lhost:8080",
+			expErr: true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			err := validateEndpoint(tc.endp)
+			if !tc.expErr {
+				g.Expect(err).ToNot(HaveOccurred())
+			} else {
+				g.Expect(err).To(HaveOccurred())
+			}
+		})
+	}
+}
+
 func TestValidatePort(t *testing.T) {
 	tests := []struct {
 		name   string

docs/developer/implementing-a-feature.md

@@ -59,7 +59,11 @@ practices to ensure a successful feature development process.
     different reviewer in mind, you can request them as well. Refer to
     the [pull request](/docs/developer/pull-request.md) documentation for expectations and guidelines.
 14. **Obtain the necessary approvals**: Work with code reviewers to maintain the required number of approvals.
-15. **Squash and merge**: Squash your commits locally, or use the GitHub UI to squash and merge. Only one commit per
+15. **Ensure the product telemetry works**. If you made any changes to the product telemetry data points, it is
+    necessary to push the generated scheme (`.avdl`, generated in Step 12) to the scheme registry. After that, manually
+    verify that the product telemetry data is successfully pushed to the telemetry service by confirming that the data
+    has been received.
+16. **Squash and merge**: Squash your commits locally, or use the GitHub UI to squash and merge. Only one commit per
     pull request should be merged. Make sure the first line of the final commit message includes the pull request
     number. For example, Fix supported gateway conditions in compatibility doc (#674).
     > **Note**:

docs/developer/quickstart.md

@@ -214,14 +214,16 @@ Run the following make command from the project's root directory to lint the Hel
 make lint-helm
 ```
 
-## Run go generate
+## Run Code Generation
 
 To ensure all the generated code is up to date, run the following make command from the project's root directory:
 
 ```shell
 make generate
 ```
 
+That command also will generate the avro scheme (`.avdl`) for product telemetry data points.
+
 ## Update Generated Manifests
 
 To update the generated manifests, run the following make command from the project's root directory: