Add ephemeral debugging guide (nginx#1202)

miledxz · miledxz · commit d8e6bbf47d3d · 2024-01-04T11:50:27.000-07:00
Problem: No documentation on how to debug NGF in Kubernetes

Solution: Add a developer guide with step-by-step instructions 
on how to inject an ephemeral dlv debug container into NGF while
it's running in a cluster. Includes makefile targets to make the 
process easier and a dockerfile for the dlv debug image.
diff --git a/Makefile b/Makefile
@@ -8,6 +8,8 @@ NGINX_CONF_DIR = internal/mode/static/nginx/conf
 NJS_DIR = internal/mode/static/nginx/modules/src
 NGINX_DOCKER_BUILD_PLUS_ARGS = --secret id=nginx-repo.crt,src=nginx-repo.crt --secret id=nginx-repo.key,src=nginx-repo.key
 BUILD_AGENT=local
+GW_API_VERSION = 1.0.0
+INSTALL_WEBHOOK = false
 
 # go build flags - should not be overridden by the user
 GO_LINKER_FlAGS_VARS = -X main.version=${VERSION} -X main.commit=${GIT_COMMIT} -X main.date=${DATE}
@@ -147,13 +149,59 @@ njs-unit-test: ## Run unit tests for the njs httpmatches module
 lint-helm: ## Run the helm chart linter
 	helm lint $(CHART_DIR)
 
+.PHONY: load-images
+load-images: ## Load NGF and NGINX images on configured kind cluster.
+	kind load docker-image $(PREFIX):$(TAG) $(NGINX_PREFIX):$(TAG)
+
+.PHONY: load-images-with-plus
+load-images-with-plus: ## Load NGF and NGINX Plus images on configured kind cluster.
+	kind load docker-image $(PREFIX):$(TAG) $(NGINX_PLUS_PREFIX):$(TAG)
+
+.PHONY: install-ngf-local-build
+install-ngf-local-build: build-images load-images helm-install-local ## Install NGF from local build on configured kind cluster.
+
+.PHONY: install-ngf-local-build-with-plus
+install-ngf-local-build-with-plus: build-images-with-plus load-images-with-plus helm-install-local-with-plus ## Install NGF with NGINX Plus from local build on configured kind cluster.
+
+.PHONY: helm-install-local
+helm-install-local: ## Helm install NGF on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build.
+	./conformance/scripts/install-gateway.sh $(GW_API_VERSION) $(INSTALL_WEBHOOK)
+	helm install dev ./deploy/helm-chart --create-namespace --wait --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never -n nginx-gateway
+
+.PHONY: helm-install-local-with-plus
+helm-install-local-with-plus: ## Helm install NGF with NGINX Plus on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build-with-plus.
+	./conformance/scripts/install-gateway.sh $(GW_API_VERSION) $(INSTALL_WEBHOOK)
+	helm install dev ./deploy/helm-chart --create-namespace --wait --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginx.plus=true -n nginx-gateway
+
+# Debug Targets
 .PHONY: debug-build
 debug-build: GO_LINKER_FLAGS=$(GO_LINKER_FlAGS_VARS)
 debug-build: ADDITIONAL_GO_BUILD_FLAGS=-gcflags "all=-N -l"
 debug-build: build ## Build binary with debug info, symbols, and no optimizations
 
-.PHONY: build-ngf-debug-image
-build-ngf-debug-image: debug-build build-ngf-image ## Build NGF image with debug binary
+.PHONY: debug-build-dlv-image
+debug-build-dlv-image: check-for-docker ## Build the dlv debugger image.
+	docker build --platform linux/$(GOARCH) -f debug/Dockerfile -t dlv-debug:edge .
+
+.PHONY: debug-build-images
+debug-build-images: debug-build build-ngf-image build-nginx-image debug-build-dlv-image ## Build all images used in debugging.
+
+.PHONY: debug-build-images-with-plus
+debug-build-images-with-plus: debug-build build-ngf-image build-nginx-plus-image debug-build-dlv-image ## Build all images with NGINX plus used in debugging.
+
+.PHONY: debug-load-images
+debug-load-images: load-images ## Load all images used in debugging to kind cluster.
+	kind load docker-image dlv-debug:edge
+
+.PHONY: debug-load-images-with-plus
+debug-load-images-with-plus: load-images-with-plus ## Load all images with NGINX Plus used in debugging to kind cluster.
+	kind load docker-image dlv-debug:edge
+
+.PHONY: debug-install-local-build
+debug-install-local-build: debug-build-images debug-load-images helm-install-local ## Install NGF from local build using debug NGF binary on configured kind cluster.
+
+.PHONY: debug-install-local-build-with-plus
+debug-install-local-build-with-plus: debug-build-images-with-plus debug-load-images-with-plus helm-install-local-with-plus ## Install NGF with NGINX Plus from local build using debug NGF binary on configured kind cluster.
 
 .PHONY: dev-all
 dev-all: deps fmt njs-fmt vet lint unit-test njs-unit-test ## Run all the development checks
diff --git a/conformance/Makefile b/conformance/Makefile
@@ -44,7 +44,7 @@ build-images: ## Build NGF and nginx images
 
 .PHONY: load-images
 load-images: ## Load NGF and NGINX images on configured kind cluster
-	kind load docker-image $(PREFIX):$(TAG) $(NGINX_PREFIX):$(TAG)
+	cd .. && make PREFIX=$(PREFIX) TAG=$(TAG) load-images
 
 .PHONY: prepare-ngf-dependencies
 prepare-ngf-dependencies: update-ngf-manifest ## Install NGF dependencies on configured kind cluster
diff --git a/debug/Dockerfile b/debug/Dockerfile
@@ -0,0 +1,12 @@
+# syntax=docker/dockerfile:1.5
+# This Dockerfile builds an image with the dlv debugger. See the debugging guide in the developer docs for details
+# on how to use it.
+FROM golang:1.21-alpine AS builder
+
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
+
+FROM alpine:latest
+
+COPY --from=builder /go/bin/dlv /usr/bin/
+
+CMD ["sh"]
diff --git a/docs/developer/debugging.md b/docs/developer/debugging.md
@@ -0,0 +1,127 @@
+# Debugging
+
+## Debugging NGF Remotely in Kubernetes
+
+This section will walk you through how to attach an ephemeral [dlv](https://github.com/go-delve/delve) debugger
+container to NGF while it's running in Kubernetes. This will allow you to remotely debug NGF running in Kubernetes
+using your IDE.
+
+- Create a `kind` cluster:
+
+   ```console
+   make create-kind-cluster
+   ```
+
+- Set GOARCH environment variable:
+
+  The [Makefile](/Makefile) uses the GOARCH variable to build the binary and container images. The default value of GOARCH is `amd64`.
+
+  If you are deploying NGINX Gateway Fabric on a kind cluster, and the architecture of your machine is not `amd64`, you will want to set the GOARCH variable to the architecture of your local machine. You can find the value of GOARCH by running `go env`. Export the GOARCH variable in your `~/.zshrc` or `~/.bashrc`.
+
+  ```console
+  echo "export GOARCH=< Your architecture (e.g. arm64 or amd64) >" >> ~/.bashrc
+  source ~/.bashrc
+  ```
+
+  or for zsh:
+
+  ```console
+  echo "export GOARCH=< Your architecture (e.g. arm64 or amd64) >" >> ~/.zshrc
+  source ~/.zshrc
+  ```
+
+- Build debug images and install NGF on your kind cluster:
+
+  - **For NGINX OSS:**
+
+    ```console
+    make GOARCH=$GOARCH debug-install-local-build
+    ```
+
+  - **For NGINX Plus:**
+
+    ```console
+    make GOARCH=$GOARCH debug-install-local-build-with-plus
+    ```
+
+  > Note: The default value of GOARCH in the [Makefile](/Makefile) is `amd64`. If you try and debug an amd64 container on an ARM machine you will see the following error in the dlv container logs: `could not attach to pid <pid>: function not implemented`.
+  > This is a known issue and the only workaround is to create an arm64 image by specifying `GOARCH=arm64` the above commands.
+  > For more information, see this [issue](https://github.com/docker/for-mac/issues/5191)
+
+- Start kubectl proxy in the background:
+
+  ```console
+  kubectl proxy &
+  ```
+
+- Save the NGF Pod name:
+
+  ```console
+  POD_NAME=<NGF Pod>
+  ```
+
+- Run the following curl command to create an ephemeral debug container:
+
+  ```console
+  curl --location --request PATCH 127.0.0.1:8001/api/v1/namespaces/nginx-gateway/pods/$POD_NAME/ephemeralcontainers \
+  --header 'Content-Type: application/strategic-merge-patch+json' \
+  --data '{
+      "spec":
+      {
+          "ephemeralContainers":
+          [
+              {
+                  "name": "dlv",
+                  "command": [
+                      "/bin/sh",
+                      "-c",
+                      "PID=$(pgrep -f /usr/bin/gateway) && dlv attach $PID --headless --listen 127.0.0.1:40000 --api-version=2 --accept-multiclient --only-same-user=false"
+                      ],
+                  "image": "dlv-debug:edge",
+                  "imagePullPolicy": "Never",
+                  "targetContainerName": "nginx-gateway",
+                  "stdin": true,
+                  "tty": true,
+                  "securityContext": {
+                      "capabilities": {
+                          "add": [
+                              "SYS_PTRACE"
+                          ]
+                      },
+                      "runAsNonRoot":false
+                  }
+              }
+          ]
+      }
+  }'
+  ```
+
+- Verify that the dlv API server is running:
+
+  ```console
+  kubectl logs -n nginx-gateway $POD_NAME -c dlv
+  ```
+
+  you should see the following log:
+
+  ```text
+  API server listening at: 127.0.0.1:40000
+  ```
+
+- Kill the kubectl proxy process:
+
+  ```console
+  kill <kubectl proxy PID>
+  ```
+
+- Port-forward the dlv API server port on the NGF Pod:
+
+  ```console
+  kubectl port-forward -n nginx-gateway $POD_NAME 40000
+  ```
+
+- Connect to the remote dlv API server through your IDE:
+  - [jetbrains instructions](https://www.jetbrains.com/help/go/attach-to-running-go-processes-with-debugger.html)
+  - [vscode instructions](https://github.com/golang/vscode-go/blob/master/docs/debugging.md)
+
+- Debug!
diff --git a/docs/developer/quickstart.md b/docs/developer/quickstart.md
@@ -11,6 +11,7 @@ Follow these steps to set up your development environment.
 1. Install:
     - [Go](https://golang.org/doc/install) v1.21.0+
     - [Docker](https://docs.docker.com/get-docker/) v18.09+
+    - [Kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
     - [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/)
     - [Helm](https://helm.sh/docs/intro/quickstart/#install-helm)
     - [git](https://git-scm.com/)
@@ -50,12 +51,30 @@ Follow these steps to set up your development environment.
 
 ## Build the Binary and Images
 
+### Setting GOARCH
+
+The [Makefile](/Makefile) uses the GOARCH variable to build the binary and container images. The default value of GOARCH is `amd64`.
+
+If you are deploying NGINX Gateway Fabric on a kind cluster, and the architecture of your machine is not `amd64`, you will want to set the GOARCH variable to the architecture of your local machine. You can find the value of GOARCH by running `go env`. Export the GOARCH variable in your `~/.zshrc` or `~/.bashrc`.
+
+```shell
+echo "export GOARCH=< Your architecture (e.g. arm64 or amd64) >" >> ~/.bashrc
+source ~/.bashrc
+```
+
+or for zsh:
+
+```shell
+echo "export GOARCH=< Your architecture (e.g. arm64 or amd64) >" >> ~/.zshrc
+source ~/.zshrc
+```
+
 ### Build the Binary
 
 To build the binary, run the make build command from the project's root directory:
 
 ```makefile
-make build
+make GOARCH=$GOARCH build
 ```
 
 This command will build the binary and output it to the `/build/.out` directory.
@@ -65,7 +84,7 @@ This command will build the binary and output it to the `/build/.out` directory.
 To build the NGINX Gateway Fabric and NGINX container images from source run the following make command:
 
 ```makefile
-make TAG=$(whoami) build-images
+make GOARCH=$GOARCH TAG=$(whoami) build-images
 ```
 
 This will build the docker images `nginx-gateway-fabric:<your-user>` and `nginx-gateway-fabric/nginx:<your-user>`.
