From deaa29db8d8767833d97ffb57d50ed3ea9d443ea Mon Sep 17 00:00:00 2001 From: Kate Osborn Date: Tue, 18 Jul 2023 11:25:14 -0600 Subject: [PATCH 1/5] Add linting for markdown files --- .markdownlint.json | 22 +++ .pre-commit-config.yaml | 8 + CODE_OF_CONDUCT.md | 20 +- CONTRIBUTING.md | 24 +-- ISSUE_LIFECYCLE.md | 18 +- README.md | 79 +++++--- SECURITY.md | 9 +- conformance/README.md | 58 +++--- conformance/provisioner/README.md | 9 +- design/resource-validation.md | 36 ++-- docs/architecture.md | 20 +- docs/building-the-image.md | 4 +- docs/cli-help.md | 14 +- docs/developer/branching-and-workflow.md | 8 + docs/developer/go-style-guide.md | 7 +- docs/developer/implementing-a-feature.md | 6 +- docs/developer/pull-request.md | 27 +-- docs/developer/quickstart.md | 7 +- docs/developer/testing.md | 12 +- docs/gateway-api-compatibility.md | 195 ++++++++++--------- docs/installation.md | 22 ++- docs/proposals/README.md | 18 +- docs/proposals/template.md | 4 +- docs/release-process.md | 4 +- docs/resource-validation.md | 16 +- docs/running-on-kind.md | 8 +- examples/advanced-routing/README.md | 29 ++- examples/cafe-example/README.md | 16 +- examples/cross-namespace-routing/README.md | 13 +- examples/http-header-filter/README.md | 6 +- examples/https-termination/README.md | 20 +- examples/traffic-splitting/README.md | 15 +- internal/mode/static/nginx/modules/README.md | 41 ++-- 33 files changed, 491 insertions(+), 304 deletions(-) create mode 100644 .markdownlint.json diff --git a/.markdownlint.json b/.markdownlint.json new file mode 100644 index 0000000000..a118a707c8 --- /dev/null +++ b/.markdownlint.json @@ -0,0 +1,22 @@ +{ + "MD003": false, + "MD004": { + "style": "dash" + }, + "MD009": false, + "MD010": false, + "MD012": false, + "MD013": { + "line_length": 120, + "code_blocks": false, + "tables": false + }, + "MD022": false, + "MD024": { + "siblings_only": true + }, + "MD033": false, + "MD034": false, + "MD041": false, + "MD046": false +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e0938e48be..0094ee5f3a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,5 +48,13 @@ repos: - id: golangci-lint args: [--new-from-patch=/tmp/diff.patch] + # Rules are in .markdownlint.json file + # See https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md for rule descriptions + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: v0.35.0 + hooks: + - id: markdownlint-fix + args: [--ignore=CHANGELOG.md, --ignore=design/, --ignore=.github/] + ci: skip: [golang-diff, golangci-lint, prettier] diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index bc3c7d3617..60d1e3ede5 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -16,21 +16,21 @@ appearance, race, religion, or sexual identity and orientation. Examples of behavior that contributes to creating a positive environment include: -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members Examples of unacceptable behavior by participants include: -* The use of sexualized language or imagery and unwelcome sexual attention or +- The use of sexualized language or imagery and unwelcome sexual attention or advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic +- Trolling, insulting/derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or electronic address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a +- Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ceb9bd67a2..750be944cf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,7 @@ The following is a set of guidelines for contributing to NGINX Kubernetes Gateway. We really appreciate that you are considering contributing! -#### Table Of Contents +## Table Of Contents [Ask a Question](#ask-a-question) @@ -11,8 +11,8 @@ considering contributing! [Contributing](#contributing) -* [Issues and Discussions](#issues-and-discussions) -* [Development Guide](#development-guide) +- [Issues and Discussions](#issues-and-discussions) +- [Development Guide](#development-guide) [Code of Conduct](CODE_OF_CONDUCT.md) @@ -33,15 +33,15 @@ Follow our [Installation Instructions](/docs/installation.md) to get the NGINX K ### Project Structure -* NGINX Kubernetes Gateway is written in Go and uses the open source NGINX software as the data plane. -* The project follows a standard Go project layout - * The main code is found at `cmd/gateway/` - * The internal code is found at `internal/` - * Build files for Docker are found under `build/` - * Deployment yaml files are found at `deploy/` - * External APIs, clients, and SDKs can be found under `pkg/` -* We use [Go Modules](https://github.com/golang/go/wiki/Modules) for managing dependencies. -* We use [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) for our BDD style unit +- NGINX Kubernetes Gateway is written in Go and uses the open source NGINX software as the data plane. +- The project follows a standard Go project layout + - The main code is found at `cmd/gateway/` + - The internal code is found at `internal/` + - Build files for Docker are found under `build/` + - Deployment yaml files are found at `deploy/` + - External APIs, clients, and SDKs can be found under `pkg/` +- We use [Go Modules](https://github.com/golang/go/wiki/Modules) for managing dependencies. +- We use [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) for our BDD style unit tests. ## Contributing diff --git a/ISSUE_LIFECYCLE.md b/ISSUE_LIFECYCLE.md index 0be3af4c12..d5fab04257 100644 --- a/ISSUE_LIFECYCLE.md +++ b/ISSUE_LIFECYCLE.md @@ -7,25 +7,24 @@ lifecycle.) 1. New issue created by community member. - 2. Assign issue owner: All new issues are assigned an owner on the NGINX engineering team. This owner shepherds the issue through the subsequent stages in the issue lifecycle. - 3. Determine issue type: This is done with automation where possible, and manually by the owner where necessary. The associated label is applied to the issue. - #### Possible Issue Types - `needs more info`: The owner should use the issue to request information from the creator. If we don't receive the - needed information within 7 days, automation closes the issue. - `bug`: The implementation of a feature is not correct. + Possible Issue Types: - `enhancement`: An enhancement, tackling technical debt, documentation changes, or improving existing features. + - `needs more info`: The owner should use the issue to request information from the creator. If we don't receive the + needed information within 7 days, automation closes the issue. - `enhancement-proposal`: Enhancements that require an [Enhancement Proposal](/docs/proposals/README.md). + - `bug`: The implementation of a feature is not correct. - `question`: The owner converts the issue to a github discussion and engages the creator. + - `enhancement`: An enhancement, tackling technical debt, documentation changes, or improving existing features. + - `enhancement-proposal`: Enhancements that require an [Enhancement Proposal](/docs/proposals/README.md). + + - `question`: The owner converts the issue to a github discussion and engages the creator. 4. Determine milestone: The owner, in collaboration with the wider team (product management & engineering), determines what milestone to attach to an issue. Generally, milestones correspond to product releases - however there are two @@ -43,7 +42,6 @@ lifecycle.) `backlog` issues can be labeled by the owner as `help wanted` and/or `good first issue` as appropriate. - 5. Promotion of `backlog candidate` issue to `backlog` issue: If an issue labelled `backlog candidate` receives more than 30 upvotes within 60 days, we promote the issue by applying the `backlog` label. While issues promoted in this manner have not been committed to a particular release, we welcome PRs from the community on them. diff --git a/README.md b/README.md index 6befe5b155..b33c7847ca 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,19 @@ -[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/nginxinc/nginx-kubernetes-gateway/badge)](https://api.securityscorecards.dev/projects/github.com/nginxinc/nginx-kubernetes-gateway) [![FOSSA Status](https://app.fossa.com/api/projects/custom%2B5618%2Fgithub.com%2Fnginxinc%2Fnginx-kubernetes-gateway.svg?type=shield)](https://app.fossa.com/projects/custom%2B5618%2Fgithub.com%2Fnginxinc%2Fnginx-kubernetes-gateway?ref=badge_shield) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/nginxinc/nginx-kubernetes-gateway/badge)](https://api.securityscorecards.dev/projects/github.com/nginxinc/nginx-kubernetes-gateway) +[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B5618%2Fgithub.com%2Fnginxinc%2Fnginx-kubernetes-gateway.svg?type=shield)](https://app.fossa.com/projects/custom%2B5618%2Fgithub.com%2Fnginxinc%2Fnginx-kubernetes-gateway?ref=badge_shield) # NGINX Kubernetes Gateway -NGINX Kubernetes Gateway is an open-source project that provides an implementation of the [Gateway API](https://gateway-api.sigs.k8s.io/) using [NGINX](https://nginx.org/) as the data plane. The goal of this project is to implement the core Gateway APIs -- `Gateway`, `GatewayClass`, `HTTPRoute`, `TCPRoute`, `TLSRoute`, and `UDPRoute` -- to configure an HTTP or TCP/UDP load balancer, reverse-proxy, or API gateway for applications running on Kubernetes. NGINX Kubernetes Gateway is currently under development and supports a subset of the Gateway API. +NGINX Kubernetes Gateway is an open-source project that provides an implementation of +the [Gateway API](https://gateway-api.sigs.k8s.io/) using [NGINX](https://nginx.org/) as the data plane. The goal of +this project is to implement the core Gateway APIs -- `Gateway`, `GatewayClass`, `HTTPRoute`, `TCPRoute`, `TLSRoute`, +and `UDPRoute` -- to configure an HTTP or TCP/UDP load balancer, reverse-proxy, or API gateway for applications running +on Kubernetes. NGINX Kubernetes Gateway is currently under development and supports a subset of the Gateway API. -For a list of supported Gateway API resources and features, see the [Gateway API Compatibility](docs/gateway-api-compatibility.md) doc. +For a list of supported Gateway API resources and features, see +the [Gateway API Compatibility](docs/gateway-api-compatibility.md) doc. -> Warning: This project is actively in development (beta feature state) and should not be deployed in a production environment. -> All APIs, SDKs, designs, and packages are subject to change. +> Warning: This project is actively in development (beta feature state) and should not be deployed in a +> production environment. All APIs, SDKs, designs, and packages are subject to change. Learn about our [design principles](/docs/developer/design-principles.md) and [architecture](/docs/architecture.md). @@ -15,45 +21,53 @@ Learn about our [design principles](/docs/developer/design-principles.md) and [a 1. [Quick Start on a kind cluster](docs/running-on-kind.md). 2. [Install](docs/installation.md) NGINX Kubernetes Gateway. -3. [Build](docs/building-the-image.md) an NGINX Kubernetes Gateway container image from source or use a pre-built image available on [GitHub Container Registry](https://github.com/nginxinc/nginx-kubernetes-gateway/pkgs/container/nginx-kubernetes-gateway). +3. [Build](docs/building-the-image.md) an NGINX Kubernetes Gateway container image from source or use a pre-built image + available + on [GitHub Container Registry](https://github.com/nginxinc/nginx-kubernetes-gateway/pkgs/container/nginx-kubernetes-gateway). 4. Deploy various [examples](examples). ## NGINX Kubernetes Gateway Releases -We publish NGINX Kubernetes Gateway releases on GitHub. See our [releases page](https://github.com/nginxinc/nginx-kubernetes-gateway/releases). +We publish NGINX Kubernetes Gateway releases on GitHub. See +our [releases page](https://github.com/nginxinc/nginx-kubernetes-gateway/releases). The latest release is [0.5.0](https://github.com/nginxinc/nginx-kubernetes-gateway/releases/tag/v0.5.0). -The edge version is useful for experimenting with new features that are not yet published in a release. To use, choose the *edge* version built from the [latest commit](https://github.com/nginxinc/nginx-kubernetes-gateway/commits/main) from the main branch. +The edge version is useful for experimenting with new features that are not yet published in a release. To use, choose +the *edge* version built from the [latest commit](https://github.com/nginxinc/nginx-kubernetes-gateway/commits/main) +from the main branch. To use NGINX Kubernetes Gateway, you need to have access to: -* An NGINX Kubernetes Gateway image. -* Installation manifests. -* Documentation and examples. + +- An NGINX Kubernetes Gateway image. +- Installation manifests. +- Documentation and examples. It is important that the versions of those things above match. -The table below summarizes the options regarding the images, manifests, documentation and examples and gives your links to the correct versions: +The table below summarizes the options regarding the images, manifests, documentation and examples and gives your links +to the correct versions: -| Version | Description | Image | Installation Manifests | Documentation and Examples | -|-|-|-|-|-| -| Latest release | For experimental use | Use the 0.5.0 image from [GitHub](https://github.com/nginxinc/nginx-kubernetes-gateway/pkgs/container/nginx-kubernetes-gateway) | [Manifests](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/v0.5.0/deploy). | [Documentation](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/v0.5.0/docs). [Examples](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/v0.5.0/examples). | -| Edge| For experimental use and latest features | Use the edge image from [GitHub](https://github.com/nginxinc/nginx-kubernetes-gateway/pkgs/container/nginx-kubernetes-gateway) | [Manifests](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/main/deploy). | [Documentation](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/main/docs). [Examples](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/main/examples). | +| Version | Description | Image | Installation Manifests | Documentation and Examples | +|----------------|------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Latest release | For experimental use | Use the 0.5.0 image from [GitHub](https://github.com/nginxinc/nginx-kubernetes-gateway/pkgs/container/nginx-kubernetes-gateway) | [Manifests](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/v0.5.0/deploy). | [Documentation](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/v0.5.0/docs). [Examples](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/v0.5.0/examples). | +| Edge | For experimental use and latest features | Use the edge image from [GitHub](https://github.com/nginxinc/nginx-kubernetes-gateway/pkgs/container/nginx-kubernetes-gateway) | [Manifests](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/main/deploy). | [Documentation](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/main/docs). [Examples](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/main/examples). | ## Technical Specifications The following table lists the software versions NGINX Kubernetes Gateway supports. | NGINX Kubernetes Gateway | Gateway API | Kubernetes | NGINX OSS | -|-|-|-|-| -| Edge | 0.7.1 | 1.21+ | 1.25.x *| -| 0.5.0 | 0.7.1 | 1.21+ | 1.25.x *| -| 0.4.0 | 0.7.1 | 1.21+ | 1.25.x *| -| 0.3.0 | 0.6.2 | 1.21+ | 1.23.x *| -| 0.2.0 | 0.5.1 | 1.21+ | 1.21.x *| -| 0.1.0 | 0.5.0 | 1.19+ | 1.21.3 | +|--------------------------|-------------|------------|-----------| +| Edge | 0.7.1 | 1.21+ | 1.25.x * | +| 0.5.0 | 0.7.1 | 1.21+ | 1.25.x * | +| 0.4.0 | 0.7.1 | 1.21+ | 1.25.x * | +| 0.3.0 | 0.6.2 | 1.21+ | 1.23.x * | +| 0.2.0 | 0.5.1 | 1.21+ | 1.21.x * | +| 0.1.0 | 0.5.0 | 1.19+ | 1.21.3 | -\*the installation manifests use the minor version of NGINX container image (e.g. 1.25) and the patch version is not specified. This means that the latest available patch version is used. +\*the installation manifests use the minor version of NGINX container image (e.g. 1.25) and the patch version is not +specified. This means that the latest available patch version is used. ## SBOM (Software Bill of Materials) @@ -61,13 +75,19 @@ We generate SBOMs for the binaries and the Docker image. ### Binaries -The SBOMs for the binaries are available in the releases page. The SBOMs are generated using [syft](https://github.com/anchore/syft) and are available in SPDX format. +The SBOMs for the binaries are available in the releases page. The SBOMs are generated +using [syft](https://github.com/anchore/syft) and are available in SPDX format. ### Docker Images -The SBOM for the Docker image is available in the [GitHub Container](https://github.com/nginxinc/nginx-kubernetes-gateway/pkgs/container/nginx-kubernetes-gateway) repository. The SBOM is generated using [syft](https://github.com/anchore/syft) and stored as an attestation in the image manifest. +The SBOM for the Docker image is available in +the [GitHub Container](https://github.com/nginxinc/nginx-kubernetes-gateway/pkgs/container/nginx-kubernetes-gateway) +repository. The SBOM is generated using [syft](https://github.com/anchore/syft) and stored as an attestation in the +image manifest. + +For example to retrieve the SBOM for `linux/amd64` and analyze it using [grype](https://github.com/anchore/grype) you +can run the following command: -For example to retrieve the SBOM for `linux/amd64` and analyze it using [grype](https://github.com/anchore/grype) you can run the following command: ```shell docker buildx imagetools inspect ghcr.io/nginxinc/nginx-kubernetes-gateway:edge --format '{{ json (index .SBOM "linux/amd64").SPDX }}' | grype ``` @@ -76,11 +96,14 @@ docker buildx imagetools inspect ghcr.io/nginxinc/nginx-kubernetes-gateway:edge We’d like to hear your feedback! If you experience issues with our Gateway Controller, please [open a bug][bug] in GitHub. If you have any suggestions or enhancement requests, please [open an idea][idea] on GitHub discussions. You can -contact us directly via kubernetes@nginx.com or on the [NGINX Community Slack][slack] in the `#nginx-kubernetes-gateway` +contact us directly via kubernetes@nginx.com or on the [NGINX Community Slack][slack] in +the `#nginx-kubernetes-gateway` channel. [bug]:https://github.com/nginxinc/nginx-kubernetes-gateway/issues/new?assignees=&labels=&projects=&template=bug_report.md&title= + [idea]:https://github.com/nginxinc/nginx-kubernetes-gateway/discussions/categories/ideas + [slack]: https://nginxcommunity.slack.com/channels/nginx-kubernetes-gateway ## Contributing diff --git a/SECURITY.md b/SECURITY.md index cf8b3968a4..5f54cbd90a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,12 +2,15 @@ ## Supported Versions -We advise users to run the most recent release of the NGINX Kubernetes Gateway. A table with the supported versions can be found in the [README](README.md#technical-specifications). +We advise users to run the most recent release of the NGINX Kubernetes Gateway. A table with the supported versions can +be found in the [README](README.md#technical-specifications). ## Reporting a Vulnerability -The F5 Security Incident Response Team (F5 SIRT) has an email alias that makes it easy to report potential security vulnerabilities. +The F5 Security Incident Response Team (F5 SIRT) has an email alias that makes it easy to report potential security +vulnerabilities. -Please report any potential or current instances of security vulnerabilities with any F5 product to the F5 Security Incident Response Team at F5SIRT@f5.com +Please report any potential or current instances of security vulnerabilities with any F5 product to the F5 Security +Incident Response Team at F5SIRT@f5.com For more information visit https://www.f5.com/services/support/report-a-vulnerability diff --git a/conformance/README.md b/conformance/README.md index c74d264b64..98182b4c6c 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -1,11 +1,11 @@ # Running [Gateway Conformance Tests](https://gateway-api.sigs.k8s.io/concepts/conformance/#3-conformance-tests) in kind -## Prerequisites: +## Prerequisites -* [kind](https://kind.sigs.k8s.io/). -* Docker. -* Golang. -* [yq](https://github.com/mikefarah/yq/#install) +- [kind](https://kind.sigs.k8s.io/). +- Docker. +- Golang. +- [yq](https://github.com/mikefarah/yq/#install) **Note**: all commands in steps below are executed from the ```conformance``` directory @@ -15,7 +15,7 @@ List available commands: make ``` -``` +```text build-nkg-image Build NKG container and load it and NGINX container on configured kind cluster build-test-runner-image Build conformance test runner image cleanup-conformance-tests Clean up conformance tests fixtures @@ -37,40 +37,47 @@ update-nkg-manifest Update the NKG deployment manifest image name and **Note:** The following variables are configurable when running the below `make` commands: -| Variable | Default | Description | -| ------------- | ------------- | ------------- | -| TAG | latest | The tag for the conformance test image | -| PREFIX | conformance-test-runner | The prefix for the conformance test image | -| NKG_TAG | edge | The tag for the locally built NKG image | -| NKG_PREFIX | nginx-kubernetes-gateway | The prefix for the locally built NKG image | -| KIND_KUBE_CONFIG | ~/.kube/kind/config | The location of the kubeconfig | -| GATEWAY_CLASS | nginx | The gateway class that should be used for the tests | -| SUPPORTED_FEATURES | HTTPRoute,HTTPRouteQueryParamMatching, HTTPRouteMethodMatching,HTTPRoutePortRedirect, HTTPRouteSchemeRedirect | The supported features that should be tested by the conformance tests. Ensure the list is comma separated with no spaces. | -| EXEMPT_FEATURES | ReferenceGrant | The features that should not be tested by the conformance tests | -| NGINX_IMAGE | as defined in the ../deploy/manifests/deployment.yaml file | The NGINX image for the NKG deployments | -| NKG_DEPLOYMENT_MANIFEST | ../deploy/manifests/deployment.yaml | The location of the NKG deployment manifest | +| Variable | Default | Description | +|-------------------------|---------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------| +| TAG | latest | The tag for the conformance test image | +| PREFIX | conformance-test-runner | The prefix for the conformance test image | +| NKG_TAG | edge | The tag for the locally built NKG image | +| NKG_PREFIX | nginx-kubernetes-gateway | The prefix for the locally built NKG image | +| KIND_KUBE_CONFIG | ~/.kube/kind/config | The location of the kubeconfig | +| GATEWAY_CLASS | nginx | The gateway class that should be used for the tests | +| SUPPORTED_FEATURES | HTTPRoute,HTTPRouteQueryParamMatching, HTTPRouteMethodMatching,HTTPRoutePortRedirect, HTTPRouteSchemeRedirect | The supported features that should be tested by the conformance tests. Ensure the list is comma separated with no spaces. | +| EXEMPT_FEATURES | ReferenceGrant | The features that should not be tested by the conformance tests | +| NGINX_IMAGE | as defined in the ../deploy/manifests/deployment.yaml file | The NGINX image for the NKG deployments | +| NKG_DEPLOYMENT_MANIFEST | ../deploy/manifests/deployment.yaml | The location of the NKG deployment manifest | ### Step 1 - Create a kind Cluster ```makefile make create-kind-cluster ``` + ### Step 2 - Install Nginx Kubernetes Gateway to configured kind cluster #### *Option 1* Build and install Nginx Kubernetes Gateway from local to configured kind cluster + ```makefile make install-nkg-local-build ``` + #### *Option 2* Install Nginx Kubernetes Gateway from local already built image to configured kind cluster + ```makefile make install-nkg-local-no-build ``` + **Note:** You can optionally skip the actual *build* step. However, if choosing this option, the following step *must* be completed manually *before* the build step: - * Set NKG_PREFIX= NKG_TAG= to preferred values. - * Navigate to `deploy/manifests` and update values in `deployment.yaml` as specified in below code-block. - * Save the changes. - ``` + +- Set NKG_PREFIX= NKG_TAG= to preferred values. +- Navigate to `deploy/manifests` and update values in `deployment.yaml` as specified in below code-block. +- Save the changes. + + ```text . .. containers: @@ -79,6 +86,7 @@ this option, the following step *must* be completed manually *before* the build .. . ``` + #### *Option 3* Install Nginx Kubernetes Gateway from edge to configured kind cluster You can also skip the build NKG image step and prepare the environment to instead use the `edge` image @@ -87,19 +95,23 @@ make install-nkg-edge ``` ### Step 3 - Build conformance test runner image + ```makefile make build-test-runner-image ``` ### Step 4 - Run Gateway conformance tests + ```makefile make run-conformance-tests ``` ### Step 5 - Cleanup the conformance test fixtures and uninstall Nginx Kubernetes Gateway + ```makefile make cleanup-conformance-tests ``` + ```makefile make uninstall-nkg ``` @@ -107,11 +119,13 @@ make uninstall-nkg ### Step 6 - Revert changes to the NKG deployment manifest **Optional** Not required if using `edge` image **Warning**: `make undo-image-update` will hard reset changes to the deploy/manifests/deployment.yaml file! + ```makefile make undo-image-update ``` ### Step 7 - Delete kind cluster + ```makefile make delete-kind-cluster ``` diff --git a/conformance/provisioner/README.md b/conformance/provisioner/README.md index 2dd4bc1c7d..7610fd6f7f 100644 --- a/conformance/provisioner/README.md +++ b/conformance/provisioner/README.md @@ -3,7 +3,7 @@ Provisioner implements data plane provisioning for NGINX Kubernetes Gateway (NKG): it creates an NKG static mode Deployment for each Gateway that belongs to the provisioner GatewayClass. -``` +```text Usage: gateway provisioner-mode [flags] @@ -18,7 +18,7 @@ Global Flags: > Note: Provisioner is not ready for production yet (see this issue for more details https://github.com/nginxinc/nginx-kubernetes-gateway/issues/634). However, it can be used in the Gateway API conformance tests, which expect a Gateway API implementation to provision an independent data plane per Gateway. - +> > Note: Provisioner uses [this manifest](/deploy/manifests/deployment.yaml) to create an NKG static mode Deployment. This manifest gets included into the NKG binary during the NKG build. To customize the Deployment, modify the manifest and **re-build** NKG. @@ -28,14 +28,19 @@ How to deploy: 1. Follow the [installation](/docs/installation.md) instructions up until the Deploy the NGINX Kubernetes Gateway Step to deploy prerequisites for both the static mode Deployments and the provisioner. 1. Deploy provisioner: + ```shell kubectl apply -f conformance/provisioner/provisioner.yaml ``` + 1. Confirm the provisioner is running in nginx-gateway namespace: + ```shell kubectl get pods -n nginx-gateway ``` + ```text + NAME READY STATUS RESTARTS AGE nginx-gateway-provisioner-6c9d9fdcb8-b2pf8 1/1 Running 0 11m ``` diff --git a/design/resource-validation.md b/design/resource-validation.md index f245556713..ccbe654e26 100644 --- a/design/resource-validation.md +++ b/design/resource-validation.md @@ -26,21 +26,21 @@ its validity, which is important for the following reasons: To help the implementations with the validation, the Gateway API already includes: -* *The OpenAPI scheme with validation rules in the Gateway API CRDs*. It enforces the structure (for example, the field +- *The OpenAPI scheme with validation rules in the Gateway API CRDs*. It enforces the structure (for example, the field X must be a string) and the contents of the fields (for example, field Y only allows values 'a' and 'b'). Additionally, it enforces the limits like max lengths on field values. Note: Kubernetes API server enforces this validation. To bypass it, a user needs to change the CRDs. -* *The webhook validation*. This validation is written in go and run as part of the webhook, which is included in the +- *The webhook validation*. This validation is written in go and run as part of the webhook, which is included in the Gateway API installation files. The validation covers additional logic, not possible to implement in the CRDs. It does not repeat the validation from the CRDs. Note: a user can bypass this validation if the webhook is not installed. However, the built-in validation rules do not cover all validation needs of NKG: -* The rules are not enough for NGINX. For example, the validation rule for the +- The rules are not enough for NGINX. For example, the validation rule for the `value` of the path in a path-based routing rule allows symbols like `;`, `{` and `}`, which can break NGINX configuration for the corresponding [location](https://nginx.org/en/docs/http/ngx_http_core_module.html#location) block. -* The rules don't cover unsupported field cases. For example, the webhook does not know which filters are implemented by +- The rules don't cover unsupported field cases. For example, the webhook does not know which filters are implemented by NKG, thus it cannot generate an appropriate error for NKG. Additionally, as mentioned in [GEP-922](https://gateway-api.sigs.k8s.io/geps/gep-922/#implementers), @@ -53,9 +53,9 @@ Design a validation mechanism for Gateway API resources. ### Personas -* *Cluster admin* who installs Gateway API (the CRDs and Webhook), installs NKG, creates Gateway and GatewayClass +- *Cluster admin* who installs Gateway API (the CRDs and Webhook), installs NKG, creates Gateway and GatewayClass resources. -* *Application developer* who creates HTTPRoutes and other routes. +- *Application developer* who creates HTTPRoutes and other routes. ### User Stories @@ -66,16 +66,16 @@ Design a validation mechanism for Gateway API resources. ### Goals -* Ensure that NKG continues to work and/or fails predictably in the face of invalid input. -* Ensure that both cluster admin and application developers can see the validation errors reported about the resource +- Ensure that NKG continues to work and/or fails predictably in the face of invalid input. +- Ensure that both cluster admin and application developers can see the validation errors reported about the resource they create (own). -* For the best UX, minimize the feedback loop: users should be able to see most of the validation errors reported by a +- For the best UX, minimize the feedback loop: users should be able to see most of the validation errors reported by a Kubernetes API server during a CRUD operation on a resource. -* Ensure that the validation mechanism conforms to the Gateway API spec. +- Ensure that the validation mechanism conforms to the Gateway API spec. ### Non-Goals -* Validation of non-Gateway API resources: Secrets, EndpointSlices. (For example, a TLS Secret resource might include a +- Validation of non-Gateway API resources: Secrets, EndpointSlices. (For example, a TLS Secret resource might include a non-valid TLS cert that will make NGINX fail to reload). ## Design @@ -95,17 +95,17 @@ validation in our code. If a resource is invalid: -* NKG will not process it -- it will treat it as if the resource didn't exist. This also means that if the resource was +- NKG will not process it -- it will treat it as if the resource didn't exist. This also means that if the resource was updated from a valid to an invalid state, NKG will also ignore any previous valid state. For example, it will remove the generation configuration for an HTTPRoute resource. -* NKG will report the validation error as a +- NKG will report the validation error as a Warning [Event](https://kubernetes.io/docs/reference/kubernetes-api/cluster-resources/event-v1/) for that resource. The Event message will describe the error and explain that the resource was ignored. We chose to report an Event instead of updating the status, because to update the status, NKG first needs to look inside the resource to determine whether it belongs to it or not. However, since the webhook validation applies to all parts of the spec of resource, it means NKG has to look inside the invalid resource and parse potentially invalid parts. To avoid that, NKG will report an Event. The owner of the resource will be able to see the Event. -* NGK will also report the validation error in the NGK logs. +- NGK will also report the validation error in the NGK logs. ### NKG-specific validation @@ -139,9 +139,9 @@ following methods in order of their appearance in the table. Notes: -* The amount and the extent of the validation should allow multiple application developers to share a single NKG (User +- The amount and the extent of the validation should allow multiple application developers to share a single NKG (User story 1). -* We expect that most of the validation problems will be caught by CRD and webhook validation and reported quickly to +- We expect that most of the validation problems will be caught by CRD and webhook validation and reported quickly to users as a response to a Kubernetes API call (User story 2). ### Evolution @@ -171,9 +171,9 @@ to release a new version like supporting a new field. See also NGK processes two kinds of transactions: -* *Data plane transactions*. NGINX handles requests from clients that want to connect to applications exposed through +- *Data plane transactions*. NGINX handles requests from clients that want to connect to applications exposed through NKG. -* *Control plane transactions*. NKG handles configuration requests (ex. a new HTTPRoute is created) from NKG users. +- *Control plane transactions*. NKG handles configuration requests (ex. a new HTTPRoute is created) from NKG users. Invalid user input makes NGINX config invalid, which means NGINX will fail to reload, which will prevent any new control plane transactions until that invalid value is fixed or removed. The proposed design addresses this issue by preventing diff --git a/docs/architecture.md b/docs/architecture.md index e72520b4cf..cd081b39b6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -3,8 +3,8 @@ This document provides an overview of the architecture and design principles of the NGINX Kubernetes Gateway. The target audience includes the following groups: -* *Cluster Operators* who would like to know how the software works and also better understand how it can fail. -* *Developers* who would like to [contribute][contribute] to the project. +- *Cluster Operators* who would like to know how the software works and also better understand how it can fail. +- *Developers* who would like to [contribute][contribute] to the project. We assume that the reader is familiar with core Kubernetes concepts, such as Pods, Deployments, Services, and Endpoints. Additionally, we recommend reading [this blog post][blog] for an overview of the NGINX architecture. @@ -35,25 +35,25 @@ internet: The figure shows: -* A *Kubernetes cluster*. -* Users *Cluster Operator*, *Application Developer A* and *Application Developer B*. These users interact with the +- A *Kubernetes cluster*. +- Users *Cluster Operator*, *Application Developer A* and *Application Developer B*. These users interact with the cluster through the Kubernetes API by creating Kubernetes objects. -* *Clients A* and *Clients B* connect to *Applications A* and *B*, respectively. This applications have been deployed by +- *Clients A* and *Clients B* connect to *Applications A* and *B*, respectively. This applications have been deployed by the corresponding users. -* The *NKG Pod*, [deployed by *Cluster Operator*](/docs/installation.md) in the Namespace *nginx-gateway*. This Pod +- The *NKG Pod*, [deployed by *Cluster Operator*](/docs/installation.md) in the Namespace *nginx-gateway*. This Pod consists of two containers: `NGINX` and `NKG`. The *NKG* container interacts with the Kubernetes API to retrieve the most up-to-date Gateway API resources created within the cluster. It then dynamically configures the *NGINX* container based on these resources, ensuring proper alignment between the cluster state and the NGINX configuration. -* *Gateway AB*, created by *Cluster Operator*, requests a point where traffic can be translated to Services within the +- *Gateway AB*, created by *Cluster Operator*, requests a point where traffic can be translated to Services within the cluster. This Gateway includes a listener with a hostname `*.example.com`. Application Developers have the ability to attach their application's routes to this Gateway if their application's hostname matches `*.example.com`. -* *Application A* with two Pods deployed in the *applications* Namespace by *Application Developer A*. To expose the +- *Application A* with two Pods deployed in the *applications* Namespace by *Application Developer A*. To expose the application to its clients (*Clients A*) via the host `a.example.com`, *Application Developer A* creates *HTTPRoute A* and attaches it to `Gateway AB`. -* *Application B* with one Pod deployed in the *applications* Namespace by *Application Developer B*. To expose the +- *Application B* with one Pod deployed in the *applications* Namespace by *Application Developer B*. To expose the application to its clients (*Clients B*) via the host `b.example.com`, *Application Developer B* creates *HTTPRoute B* and attaches it to `Gateway AB`. -* *Public Endpoint*, which fronts the *NKG* Pod. This is typically a TCP load balancer (cloud, software, or hardware) +- *Public Endpoint*, which fronts the *NKG* Pod. This is typically a TCP load balancer (cloud, software, or hardware) or a combination of such load balancer with a NodePort Service. *Clients A* and *B* connect to their applications via the *Public Endpoint*. diff --git a/docs/building-the-image.md b/docs/building-the-image.md index 40c2dfca3e..8f63e895e6 100644 --- a/docs/building-the-image.md +++ b/docs/building-the-image.md @@ -3,6 +3,7 @@ ## Prerequisites Before you can build the NGINX Kubernetes Gateway, make sure you have the following software installed on your machine: + - [git](https://git-scm.com/) - [GNU Make](https://www.gnu.org/software/software.html) - [Docker](https://www.docker.com/) v18.09+ @@ -22,7 +23,8 @@ Before you can build the NGINX Kubernetes Gateway, make sure you have the follow make PREFIX=myregistry.example.com/nginx-kubernetes-gateway container ``` - Set the `PREFIX` variable to the name of the registry you'd like to push the image to. By default, the image will be named `nginx-kubernetes-gateway:edge`. + Set the `PREFIX` variable to the name of the registry you'd like to push the image to. By default, the image will be + named `nginx-kubernetes-gateway:edge`. 1. Push the image to your container registry: diff --git a/docs/cli-help.md b/docs/cli-help.md index 0ba6b040d8..18fa7a300b 100644 --- a/docs/cli-help.md +++ b/docs/cli-help.md @@ -8,15 +8,15 @@ This command configures NGINX in the scope of a single Gateway resource. Usage: -``` +```text gateway static-mode [flags] ``` Flags: -| Name | Type | Description | -|-|-|-| -| `gateway-ctlr-name` | `string` | The name of the Gateway controller. The controller name must be of the form: `DOMAIN/PATH`. The controller's domain is `k8s-gateway.nginx.org`. | -| `gatewayclass` | `string` | The name of the GatewayClass resource. Every NGINX Gateway must have a unique corresponding GatewayClass resource. | -| `gateway` | `string` | The namespaced name of the Gateway resource to use. Must be of the form: `NAMESPACE/NAME`. If not specified, the control plane will process all Gateways for the configured GatewayClass. However, among them, it will choose the oldest resource by creation timestamp. If the timestamps are equal, it will choose the resource that appears first in alphabetical order by {namespace}/{name}. | -| `update-gatewayclass-status` | `bool` | Update the status of the GatewayClass resource. (default true) | +| Name | Type | Description | +|------------------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `gateway-ctlr-name` | `string` | The name of the Gateway controller. The controller name must be of the form: `DOMAIN/PATH`. The controller's domain is `k8s-gateway.nginx.org`. | +| `gatewayclass` | `string` | The name of the GatewayClass resource. Every NGINX Gateway must have a unique corresponding GatewayClass resource. | +| `gateway` | `string` | The namespaced name of the Gateway resource to use. Must be of the form: `NAMESPACE/NAME`. If not specified, the control plane will process all Gateways for the configured GatewayClass. However, among them, it will choose the oldest resource by creation timestamp. If the timestamps are equal, it will choose the resource that appears first in alphabetical order by {namespace}/{name}. | +| `update-gatewayclass-status` | `bool` | Update the status of the GatewayClass resource. (default true) | diff --git a/docs/developer/branching-and-workflow.md b/docs/developer/branching-and-workflow.md index aec096b540..a86c8c9366 100644 --- a/docs/developer/branching-and-workflow.md +++ b/docs/developer/branching-and-workflow.md @@ -11,27 +11,35 @@ Below is an example of following the merge and fork process. Developer Alice: - Forks `github.com/nginxinc/nginx-kubernetes-gateway` → `github.com//nginx-kubernetes-gateway` - Adds upstream: + ```shell git remote add upstream git@github.com:nginxinc/nginx-kubernetes-gateway.git ``` + - Alice lists all of her configured remotes: + ```shell git remote -v ``` + Which shows the following: + ```text origin git@github.com:/nginx-kubernetes-gateway.git (fetch) origin git@github.com:/nginx-kubernetes-gateway.git (push) upstream git@github.com:nginxinc/nginx-kubernetes-gateway.git (fetch) upstream git@github.com:nginxinc/nginx-kubernetes-gateway.git (push) ``` + - Alice develops a feature or bugfix - using as many ephemeral branches as she needs. - Alice creates a PR `github.com//nginx-kubernetes-gateway:feature/some-feature` → `github.com/nginxinc/nginx-kubernetes-gateway:main` - Alice keeps her fork up to date by running: + ```shell git pull upstream main ``` + This will sync her local main branch with the main branch of the project's repo. ## Branch Naming Conventions diff --git a/docs/developer/go-style-guide.md b/docs/developer/go-style-guide.md index b624d79013..12ac809143 100644 --- a/docs/developer/go-style-guide.md +++ b/docs/developer/go-style-guide.md @@ -345,6 +345,7 @@ Panics should be used in the following cases: example, if the wrong type is passed or a go template is passed an invalid value. When using panics, pass an error as the argument. For example: + ```go panic(fmt.Errorf("unknown event type %T", e)) ``` @@ -370,9 +371,9 @@ Below are some general guidelines to follow for writing concurrent code: reentrant, you _must_ document that in the comments. Make it clear and obvious. - **Don't leak goroutines**: Goroutines are not garbage collected by the runtime, so every goroutine you start must also be cleaned up. Here's a couple of related principles: - - "If a goroutine is responsible for creating a goroutine, it is also responsible for ensuring it can stop the + - "If a goroutine is responsible for creating a goroutine, it is also responsible for ensuring it can stop the goroutine." -- [Concurrency in Go][cig] - - "Before you start a goroutine, always know when, and how, it will stop." -- [Concurrency Made Easy][cheney]. + - "Before you start a goroutine, always know when, and how, it will stop." -- [Concurrency Made Easy][cheney]. - **Blocking operations within a goroutine must be preemptable**: This allows goroutines to be cancelled and prevents goroutine leaks. - **Leverage contexts**: Contexts allow you to enforce deadlines and send cancellation signals to multiple goroutines. @@ -423,7 +424,7 @@ These recommendations are generally related to performance and efficiency but wi The `-gcflags '-m'` can be used to analyze escape analysis and estimate logging costs. -### Reduce the number of stored pointers. Structures should store instances whenever possible. +### Reduce the number of stored pointers. Structures should store instances whenever possible DO NOT use pointers to avoid copying. Pass by value. Ancillary benefit is reduction of nil checks. Fewer pointers helps garbage collection and can indicate memory regions that can be skipped. It reduces de-referencing and bounds checking in diff --git a/docs/developer/implementing-a-feature.md b/docs/developer/implementing-a-feature.md index 2e086d195c..51c2651776 100644 --- a/docs/developer/implementing-a-feature.md +++ b/docs/developer/implementing-a-feature.md @@ -5,7 +5,8 @@ practices to ensure a successful feature development process. > **Note** > -> If you’d like to implement a new feature, please open a discussion about the feature before creating an issue or opening a PR. +> If you’d like to implement a new feature, please open a discussion about the feature +> before creating an issue or opening a PR. 1. **Assign yourself to the GitHub issue for the feature**: Each feature must have a corresponding GitHub issue to track its progress. @@ -57,7 +58,8 @@ practices to ensure a successful feature development process. number. For example, Fix supported gateway conditions in compatibility doc (#674). > **Note**: When you squash commits, make sure to not include any commit messages related to the code review - (for example, Fixed a typo). If you changed the code as a result of the code review in way that the original commit message no longer describes it well, make sure to update the message. + (for example, Fixed a typo). If you changed the code as a result of the code review in way that the + > original commit message no longer describes it well, make sure to update the message. ## Fixing a Bug diff --git a/docs/developer/pull-request.md b/docs/developer/pull-request.md index 6b00f3f8c6..ce2d79cfa5 100644 --- a/docs/developer/pull-request.md +++ b/docs/developer/pull-request.md @@ -26,8 +26,8 @@ - When pushing code review changes, it's important to provide clear information to the reviewer. Here are a couple of guidelines to follow: - - Commit each code review change individually. - - Use descriptive commit messages that accurately describe the change made, making it easy for the reviewer to + - Commit each code review change individually. + - Use descriptive commit messages that accurately describe the change made, making it easy for the reviewer to understand the purpose of the change. - When approved, and all review comments have been resolved; squash commits to a single commit. Follow the [Commit Message Format](#commit-message-format) guidelines when writing the final commit. @@ -42,17 +42,17 @@ - Always review for: design, API semantics, [DRY](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself) and maintainability practices, bugs and quality, efficiency and correctness. - Code review checklist: - - Do any changes need to made in documentation? - - Do public docs need updating? - - Have internal workflow and informational docs been impacted? - - Are there unit tests for the change? - - If the change is a bug fix, has a unit test been added or modified that catches the bug? All bug fixes should + - Do any changes need to made in documentation? + - Do public docs need updating? + - Have internal workflow and informational docs been impacted? + - Are there unit tests for the change? + - If the change is a bug fix, has a unit test been added or modified that catches the bug? All bug fixes should be reproduced with a unit test before submitting any code. Then the code should be fixed so that the unit test passes. - - If the change is a feature or new functionality. The expectation is to include a reasonable set of unit tests + - If the change is a feature or new functionality. The expectation is to include a reasonable set of unit tests which exercise positive and negative cases. - - Is this change backwards compatible? Is it causing breaking behavior? - - Are there any [Comment Tags](#comment-tags) in the change? + - Is this change backwards compatible? Is it causing breaking behavior? + - Are there any [Comment Tags](#comment-tags) in the change? ## Commit Message Format @@ -86,11 +86,14 @@ Added unit tests for events up to 1MB. ``` > **Note** -> Do not put any customer identifying information in message. Say “a customer found…”, NOT “ACME Corp found…”. If customer generated data is included it must be redacted for any PII (or identifying information). This includes IPs, ports, names of applications and objects, and URL paths. If customers volunteered this information, do not propagate it externally. +> Do not put any customer identifying information in message. Say “a customer found…”, NOT “ACME Corp found…”. +> If customer generated data is included it must be redacted for any PII (or identifying information). +> This includes IPs, ports, names of applications and objects, and URL paths. +> If customers volunteered this information, do not propagate it externally. For additional help on writing good commit messages, see this [article](https://cbea.ms/git-commit/). -# Comment Tags +## Comment Tags We use the comment tags FIXME and TODO. diff --git a/docs/developer/quickstart.md b/docs/developer/quickstart.md index 3508b668e9..5fc16f1c20 100644 --- a/docs/developer/quickstart.md +++ b/docs/developer/quickstart.md @@ -20,15 +20,19 @@ Follow these steps to set up your development environment. ```shell go install golang.org/x/tools/go/analysis/passes/fieldalignment/cmd/fieldalignment@latest ``` - - [pre-commit](https://pre-commit.com/#install): + + - [pre-commit](https://pre-commit.com/#install): ```shell brew install pre-commit ``` + and then run + ```shell pre-commit install ``` + in the project root directory to install the git hooks. 2. [Fork the project repository](https://github.com/nginxinc/nginx-kubernetes-gateway/fork) @@ -38,6 +42,7 @@ Follow these steps to set up your development environment. git clone https://github.com//nginx-kubernetes-gateway.git cd nginx-kubernetes-gateway ``` + ```makefile make deps ``` diff --git a/docs/developer/testing.md b/docs/developer/testing.md index 452133a020..d7f4f50296 100644 --- a/docs/developer/testing.md +++ b/docs/developer/testing.md @@ -65,27 +65,37 @@ Follow the steps below for manual testing: 1. Follow the instructions to [deploy on kind](/docs/developer/quickstart.md#deploy-on-kind). 2. Test your changes. Make sure to check the following: - Logs of the `nginx-gateway` container. Look out for unexpected error logs or panics. + ```shell kubectl logs -n nginx-gateway -l app=nginx-gateway ``` + - Logs of the `nginx` container. Look for unexpected error logs and verify the access logs are correct. + ```shell kubectl logs -n nginx-gateway -l app=nginx ``` + - The generated nginx config. Make sure it's correct. + ```shell kubectl exec -it -n nginx-gateway -c nginx -- nginx -T ``` + - The statuses of the Gateway API Resources. Make sure they look correct. + ```shell kubectl describe ``` + - NGINX proxies traffic successfully (when applicable). - [Examples](/examples) work correctly. This will ensure that your changes have not introduced any regressions. > **Note** > -> Don't limit yourself to happy path testing. Make an effort to cover various scenarios, including edge cases and potential error conditions. By testing a wide range of scenarios, you can uncover hidden issues and ensure the robustness of your changes. +> Don't limit yourself to happy path testing. Make an effort to cover various scenarios, +> including edge cases and potential error conditions. By testing a wide range of scenarios, +> you can uncover hidden issues and ensure the robustness of your changes. Performing manual testing helps guarantee the stability, reliability, and effectiveness of your changes before submitting them for review and integration into the project. diff --git a/docs/gateway-api-compatibility.md b/docs/gateway-api-compatibility.md index 36a60304fd..0c1f867bd6 100644 --- a/docs/gateway-api-compatibility.md +++ b/docs/gateway-api-compatibility.md @@ -27,8 +27,9 @@ resource field: - *Not supported*. The resource or field is not yet supported. It will become partially or fully supported in future releases. -> Note: it might be possible that NGINX Kubernetes Gateway will never support some resources and/or fields of the Gateway API. We will document these decisions on a case by case basis. - +> Note: it might be possible that NGINX Kubernetes Gateway will never support some resources +> and/or fields of the Gateway API. We will document these decisions on a case by case basis. +> > NGINX Kubernetes Gateway doesn't support any features from the experimental release channel. ## Resources @@ -41,6 +42,7 @@ the [Gateway API documentation](https://gateway-api.sigs.k8s.io/references/spec/ ### GatewayClass > Support Levels: +> > - Core: Supported. > - Extended: Not supported. > - Implementation-specific: Not supported. @@ -50,20 +52,21 @@ the [static-mode](./cli-help.md#static-mode) command. Fields: -* `spec` - * `controllerName` - supported. - * `parametersRef` - not supported. - * `description` - supported. -* `status` - * `conditions` - supported (Condition/Status/Reason): - * `Accepted/True/Accepted` - * `Accepted/False/InvalidParameters` - * `Accepted/False/GatewayClassConflict`: Custom reason for when the GatewayClass references this controller, but +- `spec` + - `controllerName` - supported. + - `parametersRef` - not supported. + - `description` - supported. +- `status` + - `conditions` - supported (Condition/Status/Reason): + - `Accepted/True/Accepted` + - `Accepted/False/InvalidParameters` + - `Accepted/False/GatewayClassConflict`: Custom reason for when the GatewayClass references this controller, but a different GatewayClass name is provided to the controller via the command-line argument. ### Gateway > Support Levels: +> > - Core: Supported. > - Extended: Not supported. > - Implementation-specific: Not supported. @@ -73,122 +76,124 @@ Gateway's corresponding GatewayClass. See [static-mode](./cli-help.md#static-mod Fields: -* `spec` - * `gatewayClassName` - supported. - * `listeners` - * `name` - supported. - * `hostname` - supported. - * `port` - supported. - * `protocol` - partially supported. Allowed values: `HTTP`, `HTTPS`. - * `tls` - * `mode` - partially supported. Allowed value: `Terminate`. - * `certificateRefs` - The TLS certificate and key must be stored in a Secret resource of +- `spec` + - `gatewayClassName` - supported. + - `listeners` + - `name` - supported. + - `hostname` - supported. + - `port` - supported. + - `protocol` - partially supported. Allowed values: `HTTP`, `HTTPS`. + - `tls` + - `mode` - partially supported. Allowed value: `Terminate`. + - `certificateRefs` - The TLS certificate and key must be stored in a Secret resource of type `kubernetes.io/tls`. Only a single reference is supported. - * `options` - not supported. - * `allowedRoutes` - supported. - * `addresses` - not supported. -* `status` - * `addresses` - Pod IPAddress supported. - * `conditions` - supported (Condition/Status/Reason): - * `Accepted/True/Accepted` - * `Accepted/True/ListenersNotValid` - * `Accepted/False/ListenersNotValid` - * `Accepted/False/Invalid` - * `Accepted/False/UnsupportedValue`- custom reason for when a value of a field in a Gateway is invalid or not + - `options` - not supported. + - `allowedRoutes` - supported. + - `addresses` - not supported. +- `status` + - `addresses` - Pod IPAddress supported. + - `conditions` - supported (Condition/Status/Reason): + - `Accepted/True/Accepted` + - `Accepted/True/ListenersNotValid` + - `Accepted/False/ListenersNotValid` + - `Accepted/False/Invalid` + - `Accepted/False/UnsupportedValue`- custom reason for when a value of a field in a Gateway is invalid or not supported. - * `Accepted/False/GatewayConflict`- custom reason for when the Gateway is ignored due to a conflicting Gateway. + - `Accepted/False/GatewayConflict`- custom reason for when the Gateway is ignored due to a conflicting Gateway. NKG only supports a single Gateway. - * `Programmed/True/Programmed` - * `Programmed/False/Invalid` - * `Programmed/False/GatewayConflict`- custom reason for when the Gateway is ignored due to a conflicting + - `Programmed/True/Programmed` + - `Programmed/False/Invalid` + - `Programmed/False/GatewayConflict`- custom reason for when the Gateway is ignored due to a conflicting Gateway. NKG only supports a single Gateway. - * `listeners` - * `name` - supported. - * `supportedKinds` - supported. - * `attachedRoutes` - supported. - * `conditions` - supported (Condition/Status/Reason): - * `Accepted/True/Accepted` - * `Accepted/False/UnsupportedProtocol` - * `Accepted/False/InvalidCertificateRef` - * `Accepted/False/ProtocolConflict` - * `Accepted/False/UnsupportedValue`- custom reason for when a value of a field in a Listener is invalid or + - `listeners` + - `name` - supported. + - `supportedKinds` - supported. + - `attachedRoutes` - supported. + - `conditions` - supported (Condition/Status/Reason): + - `Accepted/True/Accepted` + - `Accepted/False/UnsupportedProtocol` + - `Accepted/False/InvalidCertificateRef` + - `Accepted/False/ProtocolConflict` + - `Accepted/False/UnsupportedValue`- custom reason for when a value of a field in a Listener is invalid or not supported. - * `Accepted/False/GatewayConflict` - custom reason for when the Gateway is ignored due to a conflicting + - `Accepted/False/GatewayConflict` - custom reason for when the Gateway is ignored due to a conflicting Gateway. NKG only supports a single Gateway. - * `Programmed/True/Programmed` - * `Programmed/False/Invalid` - * `ResolvedRefs/True/ResolvedRefs` - * `ResolvedRefs/False/InvalidCertificateRef` - * `ResolvedRefs/False/InvalidRouteKinds` - * `Conflicted/True/ProtocolConflict` - * `Conflicted/False/NoConflicts` + - `Programmed/True/Programmed` + - `Programmed/False/Invalid` + - `ResolvedRefs/True/ResolvedRefs` + - `ResolvedRefs/False/InvalidCertificateRef` + - `ResolvedRefs/False/InvalidRouteKinds` + - `Conflicted/True/ProtocolConflict` + - `Conflicted/False/NoConflicts` ### HTTPRoute > Support Levels: +> > - Core: Supported. > - Extended: Partially supported. > - Implementation-specific: Not supported. Fields: -* `spec` - * `parentRefs` - partially supported. Port not supported. - * `hostnames` - supported. - * `rules` - * `matches` - * `path` - partially supported. Only `PathPrefix` and `Exact` types. - * `headers` - partially supported. Only `Exact` type. - * `queryParams` - partially supported. Only `Exact` type. - * `method` - supported. - * `filters` - * `type` - supported. - * `requestRedirect` - supported except for the experimental `path` field. If multiple filters +- `spec` + - `parentRefs` - partially supported. Port not supported. + - `hostnames` - supported. + - `rules` + - `matches` + - `path` - partially supported. Only `PathPrefix` and `Exact` types. + - `headers` - partially supported. Only `Exact` type. + - `queryParams` - partially supported. Only `Exact` type. + - `method` - supported. + - `filters` + - `type` - supported. + - `requestRedirect` - supported except for the experimental `path` field. If multiple filters with `requestRedirect` are configured, NGINX Kubernetes Gateway will choose the first one and ignore the rest. - * `requestHeaderModifier` - supported. If multiple filters with `requestHeaderModifier` are configured, + - `requestHeaderModifier` - supported. If multiple filters with `requestHeaderModifier` are configured, NGINX Kubernetes Gateway will choose the first one and ignore the rest. - * `responseHeaderModifier`, `requestMirror`, `urlRewrite`, `extensionRef` - not supported. - * `backendRefs` - partially supported. Backend ref `filters` are not supported. -* `status` - * `parents` - * `parentRef` - supported. - * `controllerName` - supported. - * `conditions` - partially supported. Supported (Condition/Status/Reason): - * `Accepted/True/Accepted` - * `Accepted/False/NoMatchingListenerHostname` - * `Accepted/False/NoMatchingParent` - * `Accepted/False/NotAllowedByListeners` - * `Accepted/False/UnsupportedValue` - custom reason for when the HTTPRoute includes an invalid or + - `responseHeaderModifier`, `requestMirror`, `urlRewrite`, `extensionRef` - not supported. + - `backendRefs` - partially supported. Backend ref `filters` are not supported. +- `status` + - `parents` + - `parentRef` - supported. + - `controllerName` - supported. + - `conditions` - partially supported. Supported (Condition/Status/Reason): + - `Accepted/True/Accepted` + - `Accepted/False/NoMatchingListenerHostname` + - `Accepted/False/NoMatchingParent` + - `Accepted/False/NotAllowedByListeners` + - `Accepted/False/UnsupportedValue` - custom reason for when the HTTPRoute includes an invalid or unsupported value. - * `Accepted/False/InvalidListener` - custom reason for when the HTTPRoute references an invalid listener. - * `Accepted/False/GatewayNotProgrammed` - custom reason for when the Gateway is not Programmed. HTTPRoute + - `Accepted/False/InvalidListener` - custom reason for when the HTTPRoute references an invalid listener. + - `Accepted/False/GatewayNotProgrammed` - custom reason for when the Gateway is not Programmed. HTTPRoute may be valid and configured, but will maintain this status as long as the Gateway is not Programmed. - * `ResolvedRefs/True/ResolvedRefs` - * `ResolvedRefs/False/InvalidKind` - * `ResolvedRefs/False/RefNotPermitted` - * `ResolvedRefs/False/BackendNotFound` - * `ResolvedRefs/False/UnsupportedValue` - custom reason for when one of the HTTPRoute rules has a backendRef + - `ResolvedRefs/True/ResolvedRefs` + - `ResolvedRefs/False/InvalidKind` + - `ResolvedRefs/False/RefNotPermitted` + - `ResolvedRefs/False/BackendNotFound` + - `ResolvedRefs/False/UnsupportedValue` - custom reason for when one of the HTTPRoute rules has a backendRef with an unsupported value. ### ReferenceGrant > Support Levels: +> > - Core: Supported. > - Extended: N/A. > - Implementation-specific: N/A Fields: -* `spec` - * `to` - * `group` - supported. - * `kind` - supports `Secret` and `Service`. - * `name`- supported. - * `from` - * `group` - supported. - * `kind` - supports `Gateway` and `HTTPRoute`. - * `namespace`- supported. +- `spec` + - `to` + - `group` - supported. + - `kind` - supports `Secret` and `Service`. + - `name`- supported. + - `from` + - `group` - supported. + - `kind` - supports `Gateway` and `HTTPRoute`. + - `namespace`- supported. ### TLSRoute diff --git a/docs/installation.md b/docs/installation.md index 900435b324..8bb3b4bed3 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -8,7 +8,8 @@ This guide walks you through how to install NGINX Kubernetes Gateway on a generi ## Deploy NGINX Kubernetes Gateway -> Note: NGINX Kubernetes Gateway can only run in the `nginx-gateway` namespace. This limitation will be addressed in the future releases. +> Note: NGINX Kubernetes Gateway can only run in the `nginx-gateway` namespace. +> This limitation will be addressed in the future releases. 1. Clone the repo and change into the `nginx-kubernetes-gateway` directory: @@ -64,6 +65,7 @@ This guide walks you through how to install NGINX Kubernetes Gateway on a generi ```shell kubectl get pods -n nginx-gateway ``` + ```text NAME READY STATUS RESTARTS AGE nginx-gateway-5d4f4c7db7-xk2kq 2/2 Running 0 112s @@ -75,7 +77,9 @@ You can gain access to NGINX Kubernetes Gateway by creating a `NodePort` Service > Important > -> The Service manifests expose NGINX Kubernetes Gateway on ports 80 and 443, which exposes any Gateway [Listener](https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.Listener) configured for those ports. If you'd like to use different ports in your listeners, +> The Service manifests expose NGINX Kubernetes Gateway on ports 80 and 443, which exposes any +> Gateway [Listener](https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.Listener) +> configured for those ports. If you'd like to use different ports in your listeners, > update the manifests accordingly. ### Create a NodePort Service @@ -86,7 +90,8 @@ Create a Service with type `NodePort`: kubectl apply -f deploy/manifests/service/nodeport.yaml ``` -A `NodePort` Service will randomly allocate one port on every Node of the cluster. To access NGINX Kubernetes Gateway, use an IP address of any Node in the cluster along with the allocated port. +A `NodePort` Service will randomly allocate one port on every Node of the cluster. To access NGINX Kubernetes Gateway, +use an IP address of any Node in the cluster along with the allocated port. ### Create a LoadBalancer Service @@ -98,13 +103,14 @@ Create a Service with type `LoadBalancer` using the appropriate manifest for you kubectl apply -f deploy/manifests/service/loadbalancer.yaml ``` - Lookup the public IP of the load balancer, which is reported in the `EXTERNAL-IP` column in the output of the following command: + Lookup the public IP of the load balancer, which is reported in the `EXTERNAL-IP` column in the output of the + following command: ```shell kubectl get svc nginx-gateway -n nginx-gateway ``` - Use the public IP of the load balancer to access NGINX Kubernetes Gateway. + Use the public IP of the load balancer to access NGINX Kubernetes Gateway. - For AWS: @@ -112,13 +118,15 @@ Create a Service with type `LoadBalancer` using the appropriate manifest for you kubectl apply -f deploy/manifests/service/loadbalancer-aws-nlb.yaml ``` - In AWS, the NLB DNS name will be reported by Kubernetes in lieu of a public IP in the `EXTERNAL-IP` column. To get the DNS name run: + In AWS, the NLB DNS name will be reported by Kubernetes in lieu of a public IP in the `EXTERNAL-IP` column. To get the + DNS name run: ```shell kubectl get svc nginx-gateway -n nginx-gateway ``` - In general, you should rely on the NLB DNS name, however for testing purposes you can resolve the DNS name to get the IP address of the load balancer: + In general, you should rely on the NLB DNS name, however for testing purposes you can resolve the DNS name to get the + IP address of the load balancer: ```shell nslookup diff --git a/docs/proposals/README.md b/docs/proposals/README.md index 192a06ec76..dcc0e5e0c3 100644 --- a/docs/proposals/README.md +++ b/docs/proposals/README.md @@ -16,9 +16,9 @@ You should only write an Enhancement Proposal if a maintainer has requested one requests should start as an idea on [GitHub Discussions][discussion]. Not all enhancement requests will require an Enhancement Proposal. For example, here are some examples of requests that may not need an Enhancement Proposal: -* Gateway API fields. However, some larger Gateway API fields may require Enhancement Proposals if they require +- Gateway API fields. However, some larger Gateway API fields may require Enhancement Proposals if they require significant changes to the architecture of the code. -* Small changes (validation, documentation, fixups). It is always possible that the maintainers will determine a "small" +- Small changes (validation, documentation, fixups). It is always possible that the maintainers will determine a "small" change ends up requiring a Enhancement Proposal. [discussion]: https://github.com/nginxinc/nginx-kubernetes-gateway/discussions @@ -87,16 +87,16 @@ with the changes and update the status field of the corresponding Enhancement Pr Each Enhancement Proposal has a status field that defines its current state. Each transition will require a PR to update the Enhancement Proposal. -* **Provisional:** The goals described by this Enhancement Proposal have consensus but implementation details have not +- **Provisional:** The goals described by this Enhancement Proposal have consensus but implementation details have not been agreed to yet. -* **Implementable:** The goals and implementation details described by this Enhancement Proposal have consensus but have +- **Implementable:** The goals and implementation details described by this Enhancement Proposal have consensus but have not been fully implemented yet. -* **Completed:** This Enhancement Proposal has been implemented. +- **Completed:** This Enhancement Proposal has been implemented. Although less common, some Enhancement Proposals may end up in one of the following states: -* **Deferred:** We do not currently have bandwidth to handle this Enhancement Proposal, it may be revisited in the +- **Deferred:** We do not currently have bandwidth to handle this Enhancement Proposal, it may be revisited in the future. -* **Rejected:** This proposal was considered but ultimately rejected. -* **Replaced:** This proposal was considered but ultimately replaced by a newer proposal. -* **Withdrawn:** This proposal was considered but ultimately withdrawn by the author. +- **Rejected:** This proposal was considered but ultimately rejected. +- **Replaced:** This proposal was considered but ultimately replaced by a newer proposal. +- **Withdrawn:** This proposal was considered but ultimately withdrawn by the author. diff --git a/docs/proposals/template.md b/docs/proposals/template.md index 9041996a9c..79a1aa9fda 100644 --- a/docs/proposals/template.md +++ b/docs/proposals/template.md @@ -1,8 +1,8 @@ # Enhancement Proposal-\: Enhancement Proposal Template -* Issue: \ -* Status: Provisional|Implementable|Completed|Deferred|Rejected|Withdrawn|Replaced +- Issue: \ +- Status: Provisional|Implementable|Completed|Deferred|Rejected|Withdrawn|Replaced (See status definitions [here](README.md#status).) diff --git a/docs/release-process.md b/docs/release-process.md index 130601ea60..8cc31206cd 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -44,8 +44,8 @@ To create a new release, follow these steps: changes to NKG. This is in contrast with the autogenerated full changelog, which is created in the next step. Use the previous changelog entries for formatting and content guidance. 7. Create and push the release tag in the format `vX.Y.Z`. As a result, the CI/CD pipeline will: - * Build NKG container images with the release tag `X.Y.Z` and push it to the registry. - * Create a GitHub release with an autogenerated changelog and attached release artifacts. + - Build NKG container images with the release tag `X.Y.Z` and push it to the registry. + - Create a GitHub release with an autogenerated changelog and attached release artifacts. 8. Prepare and merge a PR into the main branch to update the [README](../README.md) to include the information about the latest release and also the [changelog](../CHANGELOG.md). 9. Close the issue created in Step 1. diff --git a/docs/resource-validation.md b/docs/resource-validation.md index 337cbe9150..dc210121ff 100644 --- a/docs/resource-validation.md +++ b/docs/resource-validation.md @@ -23,7 +23,8 @@ A Gateway API resource (a new resource or an update for the existing one) is val To confirm that a resource is valid and accepted by NKG, check that the `Accepted` condition in the resource status has the Status field set to `True`. For example, in a status of a valid HTTPRoute, if NKG accepts a parentRef, the status of that parentRef will look like this: -``` + +```text Status: Parents: Conditions: @@ -55,6 +56,7 @@ with the following error: ```shell kubectl apply -f coffee-route.yaml ``` + ```text The HTTPRoute "coffee" is invalid: spec.hostnames[0]: Invalid value: "cafe.!@#$%example.com": spec.hostnames[0] in body should match '^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$' ``` @@ -72,6 +74,7 @@ following error: ```shell kubectl apply -f prod-gateway.yaml ``` + ```text Error from server: error when creating "prod-gateway.yaml": admission webhook "validate.gateway.networking.k8s.io" denied the request: spec.listeners[1].hostname: Forbidden: should be empty for protocol TCP ``` @@ -91,6 +94,7 @@ hostname) with a Kubernetes event: ```shell kubectl describe gateway prod-gateway ``` + ```text . . . Events: @@ -100,7 +104,6 @@ Events: ``` > This validation step always runs and cannot be bypassed. - > NKG will ignore any resources that fail the webhook validation, like in the example above. > If the resource previously existed, NKG will remove any existing NGINX configuration for that resource. @@ -108,14 +111,14 @@ Events: This step catches the following cases of invalid values: -* Valid values from the Gateway API perspective but not supported by NKG yet. For example, a feature in an +- Valid values from the Gateway API perspective but not supported by NKG yet. For example, a feature in an HTTPRoute routing rule. Note: for the list of supported features, see [Gateway API Compatibility](gateway-api-compatibility.md) doc. -* Valid values from the Gateway API perspective, but invalid for NGINX, because NGINX has stricter validation +- Valid values from the Gateway API perspective, but invalid for NGINX, because NGINX has stricter validation requirements for certain fields. Such values will cause NGINX to fail to reload or operate erroneously. -* Invalid values (both from the Gateway API and NGINX perspectives) that were not rejected because Step 1 was bypassed. +- Invalid values (both from the Gateway API and NGINX perspectives) that were not rejected because Step 1 was bypassed. Similarly to the previous case, such values will cause NGINX to fail to reload or operate erroneously. -* Malicious values that inject unrestricted NGINX config into the NGINX configuration (similar to an SQL injection +- Malicious values that inject unrestricted NGINX config into the NGINX configuration (similar to an SQL injection attack). Below is an example of how NGK rejects an invalid resource. The validation error is reported via the status: @@ -123,6 +126,7 @@ Below is an example of how NGK rejects an invalid resource. The validation error ```shell kubectl describe httproutes.gateway.networking.k8s.io coffee ``` + ```text . . . Status: diff --git a/docs/running-on-kind.md b/docs/running-on-kind.md index 8734a9b2b2..efa90c53e4 100644 --- a/docs/running-on-kind.md +++ b/docs/running-on-kind.md @@ -9,7 +9,9 @@ This guide walks you through how to run NGINX Kubernetes Gateway on a [kind](htt ## Prepare Cluster -Create a cluster with `kind`. You can follow their [instructions](https://kind.sigs.k8s.io/docs/user/quick-start/#creating-a-cluster), or run the following make command at the root of the repository: +Create a cluster with `kind`. You can follow +their [instructions](https://kind.sigs.k8s.io/docs/user/quick-start/#creating-a-cluster), or run the following make +command at the root of the repository: ```makefile make create-kind-cluster @@ -27,7 +29,9 @@ Forward local ports 8080 and 8443 to ports 80 and 443 of the nginx-gateway Pod: kubectl -n nginx-gateway port-forward 8080:80 8443:443 ``` -> Note: NGINX will not listen on any ports until you configure a [Gateway](https://gateway-api.sigs.k8s.io/api-types/gateway/#gateway) resource with a valid listener. +> Note: NGINX will not listen on any ports until you configure a +> [Gateway](https://gateway-api.sigs.k8s.io/api-types/gateway/#gateway) resource with a valid listener. ## Use NGINX Kubernetes Gateway + To get started, follow the tutorials in the [examples](../examples) directory. diff --git a/examples/advanced-routing/README.md b/examples/advanced-routing/README.md index 412a8ceb6f..0dd76a39c1 100644 --- a/examples/advanced-routing/README.md +++ b/examples/advanced-routing/README.md @@ -1,11 +1,16 @@ # Advanced Routing -In this example we will deploy NGINX Kubernetes Gateway and configure advanced routing rules for a simple cafe application. -We will use HTTPRoute resources to route traffic to the cafe application based on a combination of the request method, headers, and query parameters. +In this example we will deploy NGINX Kubernetes Gateway and configure advanced routing rules for a simple cafe +application. We will use HTTPRoute resources to route traffic to the cafe application based on a combination of the +request method, headers, and query parameters. -The cafe application consists of four Services: `coffee-v1-svc`, `coffee-v2-svc`, `tea-svc`, and `tea-post-svc`. In the next section we will create the following routing rules for the cafe application: -- For the path `/coffee` route requests with the header `version` set to `v2` or with the query param `TEST` set to `v2` to `coffee-v2-svc`, and all other requests to `coffee-v1-svc`. -- For the path `/tea` route POST requests to `tea-post-svc`, and all other requests, such as `GET` requests, to `tea-svc`. +The cafe application consists of four Services: `coffee-v1-svc`, `coffee-v2-svc`, `tea-svc`, and `tea-post-svc`. In the +next section we will create the following routing rules for the cafe application: + +- For the path `/coffee` route requests with the header `version` set to `v2` or with the query param `TEST` set to `v2` + to `coffee-v2-svc`, and all other requests to `coffee-v1-svc`. +- For the path `/tea` route POST requests to `tea-post-svc`, and all other requests, such as `GET` requests, + to `tea-svc`. ## Running the Example @@ -15,13 +20,13 @@ The cafe application consists of four Services: `coffee-v1-svc`, `coffee-v2-svc` 1. Save the public IP address of NGINX Kubernetes Gateway into a shell variable: - ``` + ```text GW_IP=XXX.YYY.ZZZ.III ``` 1. Save the port of NGINX Kubernetes Gateway: - ``` + ```text GW_PORT= ``` @@ -38,6 +43,7 @@ The cafe application consists of four Services: `coffee-v1-svc`, `coffee-v2-svc` ```shell kubectl -n default get pods ``` + ```text NAME READY STATUS RESTARTS AGE coffee-v1-75869cf7ff-vlfpq 1/1 Running 0 17m @@ -71,6 +77,7 @@ Send a request with the header `version:v2` and confirm that the response comes ```shell curl --resolve cafe.example.com:$GW_PORT:$GW_IP http://cafe.example.com:$GW_PORT/coffee -H "version:v2" ``` + ```text Server address: 10.116.2.67:8080 Server name: coffee-v2-67499ff985-gw6vt @@ -82,6 +89,7 @@ Send a request with the query parameter `TEST=v2` and confirm that the response ```shell curl --resolve cafe.example.com:$GW_PORT:$GW_IP http://cafe.example.com:$GW_PORT/coffee?TEST=v2 ``` + ```text Server address: 10.116.2.67:8080 Server name: coffee-v2-67499ff985-gw6vt @@ -93,6 +101,7 @@ Send a request without the header or the query parameter and confirm the respons ```shell curl --resolve cafe.example.com:$GW_PORT:$GW_IP http://cafe.example.com:$GW_PORT/coffee ``` + ```text Server address: 10.116.2.70:8080 Server name: coffee-v1-75869cf7ff-vlfpq @@ -106,6 +115,7 @@ Send a POST request and confirm that the response comes from `tea-post-svc`: ```shell curl --resolve cafe.example.com:$GW_PORT:$GW_IP http://cafe.example.com:$GW_PORT/tea -X POST ``` + ```text Server address: 10.116.2.72:8080 Server name: tea-post-648dfcdd6c-2rlqb @@ -117,19 +127,22 @@ Send a GET request and confirm that the response comes from `tea-svc`: ```shell curl --resolve cafe.example.com:$GW_PORT:$GW_IP http://cafe.example.com:$GW_PORT/tea ``` + ```text Server address: 10.116.3.30:8080 Server name: tea-6fb46d899f-hjzwr ... ``` -The `/tea` endpoint has routing rules configured for GET and POST requests. If you send a request with a different method, NGINX Kubernetes Gateway will return a 404. +The `/tea` endpoint has routing rules configured for GET and POST requests. If you send a request with a different +method, NGINX Kubernetes Gateway will return a 404. Send a PUT request and confirm the 404 Not Found response: ```shell curl --resolve cafe.example.com:$GW_PORT:$GW_IP http://cafe.example.com:$GW_PORT/tea -X PUT ``` + ```text 404 Not Found diff --git a/examples/cafe-example/README.md b/examples/cafe-example/README.md index f517b95572..28f60ea76f 100644 --- a/examples/cafe-example/README.md +++ b/examples/cafe-example/README.md @@ -1,6 +1,7 @@ # Example -In this example we deploy NGINX Kubernetes Gateway, a simple web application, and then configure NGINX Kubernetes Gateway to route traffic to that application using HTTPRoute resources. +In this example we deploy NGINX Kubernetes Gateway, a simple web application, and then configure NGINX Gateway to route +traffic to that application using HTTPRoute resources. ## Running the Example @@ -10,13 +11,13 @@ In this example we deploy NGINX Kubernetes Gateway, a simple web application, an 1. Save the public IP address of NGINX Kubernetes Gateway into a shell variable: - ``` + ```text GW_IP=XXX.YYY.ZZZ.III ``` 1. Save the port of NGINX Kubernetes Gateway: - ``` + ```text GW_PORT= ``` @@ -33,6 +34,7 @@ In this example we deploy NGINX Kubernetes Gateway, a simple web application, an ```shell kubectl -n default get pods ``` + ```text NAME READY STATUS RESTARTS AGE coffee-6f4b79b975-2sb28 1/1 Running 0 12s @@ -62,6 +64,7 @@ To get coffee: ```shell curl --resolve cafe.example.com:$GW_PORT:$GW_IP http://cafe.example.com:$GW_PORT/coffee ``` + ```text Server address: 10.12.0.18:80 Server name: coffee-7586895968-r26zn @@ -72,6 +75,7 @@ To get tea: ```shell curl --resolve cafe.example.com:$GW_PORT:$GW_IP http://cafe.example.com:$GW_PORT/tea ``` + ```text Server address: 10.12.0.19:80 Server name: tea-7cd44fcb4d-xfw2x @@ -88,9 +92,11 @@ For example, run the following command to open your editor and change the HTTPRo kubectl -n default edit httproute tea ``` -Once changed, update the `curl` command above for the `tea` service to use the new hostname. Traffic should still pass successfully. +Once changed, update the `curl` command above for the `tea` service to use the new hostname. Traffic should still pass +successfully. -Likewise, if you change the Gateway listener's hostname to something else, you can prevent the HTTPRoute's traffic from passing successfully. +Likewise, if you change the Gateway listener's hostname to something else, you can prevent the HTTPRoute's traffic from +passing successfully. For example, run the following to open your editor and change the Gateway listener's hostname to `bar.example.com`: diff --git a/examples/cross-namespace-routing/README.md b/examples/cross-namespace-routing/README.md index 2181f5408f..4f689a7c26 100644 --- a/examples/cross-namespace-routing/README.md +++ b/examples/cross-namespace-routing/README.md @@ -11,13 +11,13 @@ in a different namespace from our HTTPRoutes. 1. Save the public IP address of NGINX Kubernetes Gateway into a shell variable: - ``` + ```text GW_IP=XXX.YYY.ZZZ.III ``` 1. Save the port of NGINX Kubernetes Gateway: - ``` + ```text GW_PORT= ``` @@ -34,6 +34,7 @@ in a different namespace from our HTTPRoutes. ```shell kubectl -n cafe get pods ``` + ```text NAME READY STATUS RESTARTS AGE coffee-6f4b79b975-2sb28 1/1 Running 0 12s @@ -53,11 +54,13 @@ in a different namespace from our HTTPRoutes. ```shell kubectl apply -f cafe-routes.yaml ``` + 1. Create the ReferenceGrant: ```shell kubectl apply -f reference-grant.yaml ``` + This ReferenceGrant allows all HTTPRoutes in the `default` Namespace to reference all Services in the `cafe` Namespace. @@ -70,6 +73,7 @@ To get coffee: ```shell curl --resolve cafe.example.com:$GW_PORT:$GW_IP http://cafe.example.com:$GW_PORT/coffee ``` + ```text Server address: 10.12.0.18:80 Server name: coffee-7586895968-r26zn @@ -80,6 +84,7 @@ To get tea: ```shell curl --resolve cafe.example.com:$GW_PORT:$GW_IP http://cafe.example.com:$GW_PORT/tea ``` + ```text Server address: 10.12.0.19:80 Server name: tea-7cd44fcb4d-xfw2x @@ -95,9 +100,11 @@ kubectl delete -f reference-grant.yaml ``` Now, if we try to access the application over HTTP, we will get an internal server error: + ```shell curl --resolve cafe.example.com:$GW_PORT:$GW_IP http://cafe.example.com:$GW_PORT/tea ``` + ```text 500 Internal Server Error @@ -113,6 +120,7 @@ You can also check the conditions of the HTTPRoutes `coffee` and `tea` to verify ```shell kubectl describe httproute coffee ``` + ```text Condtions: Message: Backend ref to Service cafe/coffee not permitted by any ReferenceGrant @@ -126,6 +134,7 @@ Condtions: ```shell kubectl describe httproute tea ``` + ```text Condtions: Message: Backend ref to Service cafe/tea not permitted by any ReferenceGrant diff --git a/examples/http-header-filter/README.md b/examples/http-header-filter/README.md index 3d88c235a3..33b1687844 100644 --- a/examples/http-header-filter/README.md +++ b/examples/http-header-filter/README.md @@ -11,13 +11,13 @@ headers to the request. 1. Save the public IP address of NGINX Kubernetes Gateway into a shell variable: - ``` + ```text GW_IP=XXX.YYY.ZZZ.III ``` 1. Save the port of NGINX Kubernetes Gateway: - ``` + ```text GW_PORT= ``` @@ -34,6 +34,7 @@ headers to the request. ```shell kubectl -n default get pods ``` + ```text NAME READY STATUS RESTARTS AGE headers-6f4b79b975-2sb28 1/1 Running 0 12s @@ -63,6 +64,7 @@ is absent. ```shell curl -s --resolve echo.example.com:$GW_PORT:$GW_IP http://echo.example.com:$GW_PORT/headers -H "My-Cool-Header:my-client-value" -H "My-Overwrite-Header:dont-see-this" ``` + ```text Headers: header 'Accept-Encoding' is 'compress' diff --git a/examples/https-termination/README.md b/examples/https-termination/README.md index 0abc14515e..c6d6a8b63f 100644 --- a/examples/https-termination/README.md +++ b/examples/https-termination/README.md @@ -12,13 +12,13 @@ reference a Secret in a different Namespace. 1. Save the public IP address of NGINX Kubernetes Gateway into a shell variable: - ``` + ```text GW_IP=XXX.YYY.ZZZ.III ``` 1. Save the ports of NGINX Kubernetes Gateway: - ``` + ```text GW_HTTP_PORT= GW_HTTPS_PORT= ``` @@ -36,6 +36,7 @@ reference a Secret in a different Namespace. ```shell kubectl -n default get pods ``` + ```text NAME READY STATUS RESTARTS AGE coffee-6f4b79b975-2sb28 1/1 Running 0 12s @@ -45,6 +46,7 @@ reference a Secret in a different Namespace. ## 3. Configure HTTPS Termination and Routing 1. Create the Namespace `certificate` and a Secret with a TLS certificate and key: + ```shell kubectl apply -f certificate-ns-and-cafe-secret.yaml ``` @@ -53,6 +55,7 @@ reference a Secret in a different Namespace. > **Important**: This certificate and key are for demo purposes only. 1. Create the ReferenceGrant: + ```shell kubectl apply -f reference-grant.yaml ``` @@ -61,15 +64,17 @@ reference a Secret in a different Namespace. the `certificate` Namespace. 1. Create the Gateway resource: + ```shell kubectl apply -f gateway.yaml ``` This [Gateway](./gateway.yaml) configures: - * `http` listener for HTTP traffic - * `https` listener for HTTPS traffic. It terminates TLS connections using the `cafe-secret` we created in step 1. + - `http` listener for HTTP traffic + - `https` listener for HTTPS traffic. It terminates TLS connections using the `cafe-secret` we created in step 1. 1. Create the HTTPRoute resources: + ```shell kubectl apply -f cafe-routes.yaml ``` @@ -110,6 +115,7 @@ To get a redirect for coffee: ```shell curl --resolve cafe.example.com:$GW_HTTP_PORT:$GW_IP http://cafe.example.com:$GW_HTTP_PORT/coffee --include ``` + ```text HTTP/1.1 302 Moved Temporarily ... @@ -122,6 +128,7 @@ To get a redirect for tea: ```shell curl --resolve cafe.example.com:$GW_HTTP_PORT:$GW_IP http://cafe.example.com:$GW_HTTP_PORT/tea --include ``` + ```text HTTP/1.1 302 Moved Temporarily ... @@ -139,6 +146,7 @@ To get coffee: ```shell curl --resolve cafe.example.com:$GW_HTTPS_PORT:$GW_IP https://cafe.example.com:$GW_HTTPS_PORT/coffee --insecure ``` + ```text Server address: 10.12.0.18:80 Server name: coffee-7586895968-r26zn @@ -149,6 +157,7 @@ To get tea: ```shell curl --resolve cafe.example.com:$GW_HTTPS_PORT:$GW_IP https://cafe.example.com:$GW_HTTPS_PORT/tea --insecure ``` + ```text Server address: 10.12.0.19:80 Server name: tea-7cd44fcb4d-xfw2x @@ -164,9 +173,11 @@ kubectl delete -f reference-grant.yaml ``` Now, if we try to access the application over HTTPS, we will get a connection refused error: + ```shell curl --resolve cafe.example.com:$GW_HTTPS_PORT:$GW_IP https://cafe.example.com:$GW_HTTPS_PORT/coffee --insecure -vvv ``` + ```text ... curl: (7) Failed to connect to cafe.example.com port 443 after 0 ms: Connection refused @@ -178,6 +189,7 @@ You can also check the conditions of the Gateway `https` Listener to verify the ```shell kubectl describe gateway gateway ``` + ```text Name: https Conditions: diff --git a/examples/traffic-splitting/README.md b/examples/traffic-splitting/README.md index 70c65d7b43..c84fcba5d5 100644 --- a/examples/traffic-splitting/README.md +++ b/examples/traffic-splitting/README.md @@ -1,7 +1,9 @@ # Example In this example we will deploy NGINX Kubernetes Gateway and configure traffic splitting for a simple cafe application. -We will use HTTPRoute resources to split traffic between two versions of the application -- `coffee-v1` and `coffee-v2`. +We will use HTTPRoute resources to split traffic between two versions of the application -- `coffee-v1` +and `coffee-v2`. + ## Running the Example @@ -11,13 +13,13 @@ We will use HTTPRoute resources to split traffic between two versions of the app 1. Save the public IP address of NGINX Kubernetes Gateway into a shell variable: - ``` + ```text GW_IP=XXX.YYY.ZZZ.III ``` 1. Save the port of NGINX Kubernetes Gateway: - ``` + ```text GW_PORT= ``` @@ -34,6 +36,7 @@ We will use HTTPRoute resources to split traffic between two versions of the app ```shell kubectl -n default get pods ``` + ```text NAME READY STATUS RESTARTS AGE coffee-v1-7c57c576b-rfjsh 1/1 Running 0 21m @@ -54,9 +57,9 @@ We will use HTTPRoute resources to split traffic between two versions of the app kubectl apply -f cafe-route.yaml ``` -This HTTPRoute resource defines a route for the path `/coffee` that sends 80% of the requests to `coffee-v1` and 20% to `coffee-v2`. -In this example, we use 80 and 20; however, the weights are calculated proportionally and do not need to sum to 100. -For example, the weights of 8 and 2, 16 and 4, or 32 and 8 all evaluate to the same relative proportions. +This HTTPRoute resource defines a route for the path `/coffee` that sends 80% of the requests to `coffee-v1` and 20% +to `coffee-v2`. In this example, we use 80 and 20; however, the weights are calculated proportionally and do not need to +sum to 100. For example, the weights of 8 and 2, 16 and 4, or 32 and 8 all evaluate to the same relative proportions. ## 4. Test the Application diff --git a/internal/mode/static/nginx/modules/README.md b/internal/mode/static/nginx/modules/README.md index 398a43a681..d75409df83 100644 --- a/internal/mode/static/nginx/modules/README.md +++ b/internal/mode/static/nginx/modules/README.md @@ -9,31 +9,43 @@ We recommend using [nvm](https://github.com/nvm-sh/nvm/blob/master/README.md) to - [Node.js](https://nodejs.org/en/) (version 1.18) - [npm](https://docs.npmjs.com/) -Once you've installed Node.js and npm, run `npm install` in this directory to install the rest of the project's dependencies. +Once you've installed Node.js and npm, run `npm install` in this directory to install the rest of the project's +dependencies. ## Modules -- [httpmatches](./src/httpmatches.js): a location handler for HTTP requests. It redirects requests to an internal location block based on the request's headers, arguments, and method. +- [httpmatches](./src/httpmatches.js): a location handler for HTTP requests. It redirects requests to an internal + location block based on the request's headers, arguments, and method. ### Helpful Resources for Module Development -When developing njs modules, it's important to remember that njs is a subset of JavaScript, and its compliance with ECMAScript is still evolving. -Not all JavaScript functionality is available in njs, and njs is not fully compatible with ECMAScript. The following docs are helpful development resources: +When developing njs modules, it's important to remember that njs is a subset of JavaScript, and its compliance with +ECMAScript is still evolving. Not all JavaScript functionality is available in njs, and njs is not fully compatible with +ECMAScript. The following docs are helpful development resources: - [HTTP njs module](https://nginx.org/en/docs/http/ngx_http_js_module.html) - [List of njs properties that are compatible with ECMAScript](http://nginx.org/en/docs/njs/compatibility.html) - [List of njs properties, methods, and objects that are not compatible with ECMAScript](http://nginx.org/en/docs/njs/reference.html) -**Note**: You must use the [default export statement](https://developer.mozilla.org/en-US/docs/web/javascript/reference/statements/export) to export functions in an njs module. +**Note**: You must use +the [default export statement](https://developer.mozilla.org/en-US/docs/web/javascript/reference/statements/export) to +export functions in an njs module. ## Unit Tests -This project uses the [Mocha](https://mochajs.org/) test framework and the [Chai](https://www.chaijs.com/) assertion library to write BDD-style unit tests. Tests for the modules are placed in the `/tests` directory and named as `.test.js`. +This project uses the [Mocha](https://mochajs.org/) test framework and the [Chai](https://www.chaijs.com/) assertion +library to write BDD-style unit tests. Tests for the modules are placed in the `/tests` directory and named +as `.test.js`. To run unit tests against the [httpmatches](./src/httpmatches.js) modules you must: -- Use the [default import statement](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#importing_defaults) to import the module. + +- Use + the [default import statement](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#importing_defaults) + to import the module. - Run mocha with the `--require esm` option. -- Mock the [NGINX HTTP Request Object](http://nginx.org/en/docs/njs/reference.html#http) and pass it to the exported function. Not all functions and fields on the HTTP request object need to be mocked, just the ones that are used in the module. +- Mock the [NGINX HTTP Request Object](http://nginx.org/en/docs/njs/reference.html#http) and pass it to the exported + function. Not all functions and fields on the HTTP request object need to be mocked, just the ones that are used in + the module. ### Run Unit Tests @@ -45,10 +57,12 @@ npm test ## Debugging -#### Debug Unit Tests +### Debug Unit Tests To debug on the command-line: -- Set a breakpoint using the [debugger](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger) statement. + +- Set a breakpoint using + the [debugger](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger) statement. - Run the tests with the inspect argument: ```shell @@ -58,16 +72,19 @@ npx mocha inspect -r esm If you are using JetBrains or VSCode for development, you can debug the unit tests in your IDE. For JetBrains: + - [Create a run/debug configuration for mocha](https://www.jetbrains.com/help/idea/run-debug-configuration-mocha.html). - Add `--require esm` to the `Extra Mocha Options` field in your run/debug configuration. For VSCode: + - [Create a debug configuration for mocha](https://dev.to/wakeupmh/debugging-mocha-tests-in-vscode-468a). - Add `--require esm` to the configuration args. -#### Log Statements +### Log Statements -You can add log statements to debug njs code at runtime. The following log functions are available on the [NGINX HTTP Request Object](http://nginx.org/en/docs/njs/reference.html#http): +You can add log statements to debug njs code at runtime. The following log functions are available on +the [NGINX HTTP Request Object](http://nginx.org/en/docs/njs/reference.html#http): Log at error level: From 852280c74db61414a5675d038495c674b6835e2f Mon Sep 17 00:00:00 2001 From: Kate Osborn Date: Thu, 20 Jul 2023 12:23:09 -0600 Subject: [PATCH 2/5] Use markdownlint-cli2; remove changelog and design from ignore --- .markdownlint-cli2.yaml | 23 +++ .markdownlint.json | 22 --- .pre-commit-config.yaml | 7 +- CHANGELOG.md | 153 ++++++++++------ design/archive/gateway-evaluation.md | 169 +++++++++++++----- .../control-data-plane-separation/design.md | 111 ++++++------ 6 files changed, 302 insertions(+), 183 deletions(-) create mode 100644 .markdownlint-cli2.yaml delete mode 100644 .markdownlint.json diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml new file mode 100644 index 0000000000..0503c1fec0 --- /dev/null +++ b/.markdownlint-cli2.yaml @@ -0,0 +1,23 @@ +# Rule configuration. +# For rule descriptions and how to fix: https://github.com/DavidAnson/markdownlint/tree/main#rules--aliases +config: + ul-style: + style: dash + no-hard-tabs: false + no-multiple-blanks: false + line-length: + line_length: 120 + code_blocks: false + tables: false + blanks-around-headers: false + no-duplicate-heading: + siblings_only: true + no-inline-html: false + no-bare-urls: false + no-emphasis-as-heading: false + first-line-h1: false + code-block-style: false + +# Define glob expressions to ignore +ignores: + - ".github/" diff --git a/.markdownlint.json b/.markdownlint.json deleted file mode 100644 index a118a707c8..0000000000 --- a/.markdownlint.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "MD003": false, - "MD004": { - "style": "dash" - }, - "MD009": false, - "MD010": false, - "MD012": false, - "MD013": { - "line_length": 120, - "code_blocks": false, - "tables": false - }, - "MD022": false, - "MD024": { - "siblings_only": true - }, - "MD033": false, - "MD034": false, - "MD041": false, - "MD046": false -} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0094ee5f3a..cfd66b95d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,11 +50,10 @@ repos: # Rules are in .markdownlint.json file # See https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md for rule descriptions - - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.35.0 + - repo: https://github.com/DavidAnson/markdownlint-cli2 + rev: v0.8.1 hooks: - - id: markdownlint-fix - args: [--ignore=CHANGELOG.md, --ignore=design/, --ignore=.github/] + - id: markdownlint-cli2-fix ci: skip: [golang-diff, golangci-lint, prettier] diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e8d3b8a6e..ebb73780da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog -This document includes a curated changelog for each release. We also publish a changelog as the description of a [GitHub release](https://github.com/nginxinc/nginx-kubernetes-gateway/releases), which, by contrast, is auto-generated and includes links to all PRs that went into the release. +This document includes a curated changelog for each release. We also publish a changelog as the description of +a [GitHub release](https://github.com/nginxinc/nginx-kubernetes-gateway/releases), which, by contrast, is auto-generated +and includes links to all PRs that went into the release. ## Release 0.5.0 @@ -9,16 +11,19 @@ This document includes a curated changelog for each release. We also publish a c This release completes all v1beta1 Core features of the Gateway API resources. See the [Gateway Compatibility doc](https://github.com/nginxinc/nginx-kubernetes-gateway/blob/v0.5.0/docs/gateway-api-compatibility.md) FEATURES: -* Support cross-namespace BackendRefs in HTTPRoutes. [PR-806](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/806) -* Support dynamic certificate rotation with Kubernetes Secrets. [PR-807](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/807) -* Support SupportedKinds in ListenerStatus. [PR-809](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/809) + +- Support cross-namespace BackendRefs in HTTPRoutes. [PR-806](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/806) +- Support dynamic certificate rotation with Kubernetes Secrets. [PR-807](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/807) +- Support SupportedKinds in ListenerStatus. [PR-809](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/809) BUG FIXES: -* Set redirect port in location header according to the scheme. [PR-801](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/801) -* Set proxy host header to the exact value of the request host header. [PR-827](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/827) -* Ensure Prefix matching requires trailing slash. [PR-817](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/817) + +- Set redirect port in location header according to the scheme. [PR-801](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/801) +- Set proxy host header to the exact value of the request host header. [PR-827](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/827) +- Ensure Prefix matching requires trailing slash. [PR-817](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/817) COMPATIBILITY: + - The Gateway API version: `0.7.1` - NGINX version: `1.25.x` * - Kubernetes version: `1.21+` @@ -26,6 +31,7 @@ COMPATIBILITY: \*the installation manifests use the `nginx:1.25` image, which always points to the latest version of 1.25.x releases. CONTAINER IMAGES: + - Control plane: `ghcr.io/nginxinc/nginx-kubernetes-gateway:0.5.0` ## Release 0.4.0 @@ -33,6 +39,7 @@ CONTAINER IMAGES: *July 6, 2023* This release brings: + - Support for more features of the Gateway API resources. See the [Gateway Compatibility doc](https://github.com/nginxinc/nginx-kubernetes-gateway/blob/v0.4.0/docs/gateway-api-compatibility.md) - Support for running the conformance test suite. See the [Conformance tests README](https://github.com/nginxinc/nginx-kubernetes-gateway/blob/v0.4.0/conformance/README.md). - Defined Enhancement Proposal process for NGINX Kubernetes Gateway project. See the [Enhancement Proposal README](https://github.com/nginxinc/nginx-kubernetes-gateway/blob/v0.4.0/docs/proposals/README.md). @@ -41,37 +48,41 @@ This release brings: - Miscellaneous enhancements and bug fixes. FEATURES: -* Allow empty sectionName in HTTPRoute parentRef. [PR-626](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/626) -* Exact PathMatch support for HTTPRoutes. [PR-603](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/603) -* Set ResolvedRefs condition to true on HTTPRoutes. [PR-645](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/645) -* Set gateway Pod IP as GatewayStatus address. [PR-638](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/638) -* Set Accepted condition type on Gateway status. [PR-633](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/633) -* Drop unrequired capabilities from containers. [PR-677](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/677) -* Update route condition where listener is not found. [PR-675](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/675) -* Set Gateway Programmed condition. [PR-658](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/658) -* AllowedRoutes support for Listeners. [PR-721](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/721) -* Support custom listener ports. [PR-745](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/745) -* Add support for RequestHeaderModifier for HTTPRouteRule objects. [PR-717](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/717) -* Add wildcard hostname support. [PR-769](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/769) -* Add Programmed status for listener. [PR-786](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/786) -* ReferenceGrant from Gateway to Secret. [PR-791](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/791) + +- Allow empty sectionName in HTTPRoute parentRef. [PR-626](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/626) +- Exact PathMatch support for HTTPRoutes. [PR-603](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/603) +- Set ResolvedRefs condition to true on HTTPRoutes. [PR-645](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/645) +- Set gateway Pod IP as GatewayStatus address. [PR-638](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/638) +- Set Accepted condition type on Gateway status. [PR-633](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/633) +- Drop unrequired capabilities from containers. [PR-677](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/677) +- Update route condition where listener is not found. [PR-675](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/675) +- Set Gateway Programmed condition. [PR-658](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/658) +- AllowedRoutes support for Listeners. [PR-721](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/721) +- Support custom listener ports. [PR-745](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/745) +- Add support for RequestHeaderModifier for HTTPRouteRule objects. [PR-717](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/717) +- Add wildcard hostname support. [PR-769](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/769) +- Add Programmed status for listener. [PR-786](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/786) +- ReferenceGrant from Gateway to Secret. [PR-791](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/791) BUG FIXES: -* Set upstream zone size to 512k. [PR-609](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/609) -* Allow empty HTTPRoute hostnames. [PR-650](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/650) -* Allow long server names. [PR-651](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/651) -* Add in required capabilities for writing TLS secrets. [PR-718](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/718) -* Fix binding to multiple listeners with empty section name. [PR-730](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/730) -* Add timeout and retry logic for finding NGINX PID file. [PR-676](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/676) -* Prioritize method matching. [PR-789](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/789) -* Add NewListenerInvalidRouteKinds condition. [PR-799](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/799) -* Set ResolvedRefs/False/InvalidKind condition on the HTTPRoute if a BackendRef specifies an unknown kind. [PR-800](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/800) -* Set GatewayClass status for ignored GatewayClasses. [PR-804](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/804) + +- Set upstream zone size to 512k. [PR-609](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/609) +- Allow empty HTTPRoute hostnames. [PR-650](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/650) +- Allow long server names. [PR-651](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/651) +- Add in required capabilities for writing TLS secrets. [PR-718](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/718) +- Fix binding to multiple listeners with empty section name. [PR-730](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/730) +- Add timeout and retry logic for finding NGINX PID file. [PR-676](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/676) +- Prioritize method matching. [PR-789](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/789) +- Add NewListenerInvalidRouteKinds condition. [PR-799](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/799) +- Set ResolvedRefs/False/InvalidKind condition on the HTTPRoute if a BackendRef specifies an unknown kind. [PR-800](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/800) +- Set GatewayClass status for ignored GatewayClasses. [PR-804](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/804) DEPENDENCIES: -* Bump sigs.k8s.io/gateway-api from 0.7.0 to 0.7.1. [PR-711](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/711) + +- Bump sigs.k8s.io/gateway-api from 0.7.0 to 0.7.1. [PR-711](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/711) COMPATIBILITY: + - The Gateway API version: `0.7.1` - NGINX version: `1.25.x` * - Kubernetes version: `1.21+` @@ -79,6 +90,7 @@ COMPATIBILITY: \*the installation manifests use the `nginx:1.25` image, which always points to the latest version of 1.25.x releases. CONTAINER IMAGES: + - Control plane: `ghcr.io/nginxinc/nginx-kubernetes-gateway:0.4.0` ## Release 0.3.0 @@ -86,31 +98,37 @@ CONTAINER IMAGES: *April 24, 2023* This release brings: + - Extensive validation of Gateway API resources for robustness, security and correctness. See the [validation doc](https://github.com/nginxinc/nginx-kubernetes-gateway/blob/v0.3.0/docs/resource-validation.md) for more details. - Defined open-source development process for NGINX Kubernetes Gateway project. See the [Issue lifecycle doc](https://github.com/nginxinc/nginx-kubernetes-gateway/blob/v0.3.0/ISSUE_LIFECYCLE.md). - Miscellaneous enhancements and bug fixes. FEATURES: -* Report proper Conditions in status of HTTPRoute and Gateway when GatewayClass is invalid or doesn't exist. [PR-576](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/576) -* Implement NKG-specific field validation for GatewayClasses. [PR-295](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/495) -* Implement NKG-specific field validation for HTTPRoutes. [PR-455](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/455) -* Implement NKG-specific field validation for Gateways. [PR-407](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/407) -* Run webhook validation rules inside NKG control plane. [PR-388](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/388) -* Make NGINX error log visible in NGINX container logs. [PR-319](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/319) -* Always generate a root "/" location block in NGINX config to handle unmatched requests with 404 response. [PR-356](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/356) + +- Report proper Conditions in status of HTTPRoute and Gateway when GatewayClass is invalid or doesn't exist. [PR-576](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/576) +- Implement NKG-specific field validation for GatewayClasses. [PR-295](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/495) +- Implement NKG-specific field validation for HTTPRoutes. [PR-455](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/455) +- Implement NKG-specific field validation for Gateways. [PR-407](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/407) +- Run webhook validation rules inside NKG control plane. [PR-388](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/388) +- Make NGINX error log visible in NGINX container logs. [PR-319](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/319) +- Always generate a root "/" location block in NGINX config to handle unmatched requests with 404 response. [PR-356](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/356) BUG FIXES: -* Fix HTTPRoute section name related bugs. [PR-568](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/568) -* Fix Observed Generation for Gateway Status. [PR-351](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/351) -* Fix status for parentRef with invalid listener in HTTPRoute. [PR-350](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/350) -* Fix initContainer failure during pod restart. [PR-337](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/337). Thanks to [Tom Plant](https://github.com/pl4nty) -* Generate default http server in NGINX if http listener exists in Gateway. [PR-320](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/320) + +- Fix HTTPRoute section name related bugs. [PR-568](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/568) +- Fix Observed Generation for Gateway Status. [PR-351](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/351) +- Fix status for parentRef with invalid listener in HTTPRoute. [PR-350](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/350) +- Fix initContainer failure during pod restart. [PR-337](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/337). + Thanks to [Tom Plant](https://github.com/pl4nty) +- Generate default http server in NGINX if http listener exists in Gateway. [PR-320](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/320) DEPENDENCIES: -* Bump sigs.k8s.io/gateway-api from 0.6.0 to 0.6.2. [PR-471](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/471) + +- Bump sigs.k8s.io/gateway-api from 0.6.0 to 0.6.2. [PR-471](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/471) COMPATIBILITY: + - The Gateway API version: `0.6.2` - NGINX version: `1.23.x` * - Kubernetes version: `1.21+` @@ -118,6 +136,7 @@ COMPATIBILITY: \*the installation manifests use the `nginx:1.23` image, which always points to the latest version of 1.23.x releases. CONTAINER IMAGES: + - Control plane: `ghcr.io/nginxinc/nginx-kubernetes-gateway:0.3.0` ## Release 0.2.0 @@ -127,22 +146,28 @@ CONTAINER IMAGES: This release extends the support of the features of the Gateway API resources. FEATURES: -* Support the Pod IPs instead of the virtual IP of a Service in the NGINX upstream. Additionally, NGINX Kubernetes Gateway will pick up any changes to the Pod IPs and update the NGINX upstream accordingly. [PR-221](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/221) -* Support the redirect filter in an HTTPRoute rule. [PR-218](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/218) -* Support weights in backendRefs in the HTTPRoute (traffic splitting). [PR-261](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/261) -* Support the ObservedGeneration field in the HTTPRoute status. [PR-254](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/254) + +- Support the Pod IPs instead of the virtual IP of a Service in the NGINX upstream. Additionally, NGINX Kubernetes + Gateway will pick up any changes to the Pod IPs and update the NGINX upstream + accordingly. [PR-221](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/221) +- Support the redirect filter in an HTTPRoute rule. [PR-218](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/218) +- Support weights in backendRefs in the HTTPRoute (traffic splitting). [PR-261](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/261) +- Support the ObservedGeneration field in the HTTPRoute status. [PR-254](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/254) BUG FIXES: -* Do not require the namespace in the `--gateway-ctlr-name` cli argument. [PR-235](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/235) -* Ensure NGINX Kubernetes Gateway exits gracefully during shutdown. [PR-250](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/250) -* Handle query param names in case-sensitive way. [PR-220](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/220) + +- Do not require the namespace in the `--gateway-ctlr-name` cli argument. [PR-235](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/235) +- Ensure NGINX Kubernetes Gateway exits gracefully during shutdown. [PR-250](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/250) +- Handle query param names in case-sensitive way. [PR-220](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/220) DEPENDENCIES: -* Use the latest NGINX 1.23 image. [PR-275](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/275) -* Bump sigs.k8s.io/gateway-api from 0.5.0 to 0.5.1 [PR-251](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/251) + +- Use the latest NGINX 1.23 image. [PR-275](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/275) +- Bump sigs.k8s.io/gateway-api from 0.5.0 to 0.5.1 [PR-251](https://github.com/nginxinc/nginx-kubernetes-gateway/pull/251) COMPATIBILITY: + - The Gateway API version: `0.5.1` - NGINX version: `1.21.x` * - Kubernetes version: `1.21+` @@ -150,6 +175,7 @@ COMPATIBILITY: \*the installation manifests use the `nginx:1.21` image, which always points to the latest version of 1.21.x releases. CONTAINER IMAGES: + - Control plane: `ghcr.io/nginxinc/nginx-kubernetes-gateway:0.2.0` ## Release 0.1.0 @@ -159,21 +185,30 @@ CONTAINER IMAGES: This is an initial release of NGINX Kubernetes Gateway project. The release includes: -- A control plane agent (a Kubernetes controller) that updates date plane (NGINX) configuration based on the state of the resources in the cluster. + +- A control plane agent (a Kubernetes controller) that updates date plane (NGINX) configuration based on the state of + the resources in the cluster. - Support for NGINX as a data plane. -- Kubernetes manifests for a Deployment with a single Pod with the control plane and data plane containers as well as Services to enable external connectivity to that Pod. +- Kubernetes manifests for a Deployment with a single Pod with the control plane and data plane containers as well as + Services to enable external connectivity to that Pod. - Support for a subset of features of GatewayClass, Gateway and HTTPRoute resources (see the [Gateway API Compatibility doc](https://github.com/nginxinc/nginx-kubernetes-gateway/blob/v0.1.0/README.md)). -We expect that the architecture of NGINX Kubernetes Gateway -- the number of pods and containers and their interaction -- will change as the project evolves. +We expect that the architecture of NGINX Kubernetes Gateway -- the number of pods and containers and their +interaction -- will change as the project evolves. -NGINX Kubernetes Gateway is ready for experimental usage. We included the [docs](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/v0.1.0/docs) as well as [examples](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/v0.1.0/examples). +NGINX Kubernetes Gateway is ready for experimental usage. We included +the [docs](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/v0.1.0/docs) as well +as [examples](https://github.com/nginxinc/nginx-kubernetes-gateway/tree/v0.1.0/examples). -If you'd like to give us feedback or get involved, see the [README](https://github.com/nginxinc/nginx-kubernetes-gateway) to learn how. +If you'd like to give us feedback or get involved, see +the [README](https://github.com/nginxinc/nginx-kubernetes-gateway) to learn how. COMPATIBILITY: + - The Gateway API version: `0.5.0` - NGINX version: `1.21.3` - Kubernetes version: `1.19+` CONTAINER IMAGES: + - Control plane: `ghcr.io/nginxinc/nginx-kubernetes-gateway:0.1.0` diff --git a/design/archive/gateway-evaluation.md b/design/archive/gateway-evaluation.md index bfaf2ea708..d3da2f1801 100644 --- a/design/archive/gateway-evaluation.md +++ b/design/archive/gateway-evaluation.md @@ -1,61 +1,99 @@ # Gateway Evaluation + STATUS: Archived -This document captures the design of the initial experimental work to implement the [Gateway API](https://gateway-api.sigs.k8s.io/) using the NGINX data plane. Throughout this document we will refer to this work as "Gateway evaluation". +This document captures the design of the initial experimental work to implement +the [Gateway API](https://gateway-api.sigs.k8s.io/) using the NGINX data plane. Throughout this document we will refer +to this work as "Gateway evaluation". ## Motivation -The SIG-NETWORK group has begun a mission to improve traffic control to and from Kubernetes clusters. The Gateway API aims to improve service networking by formalizing user personas and developing a decoupled and composable collection of resources for provisioning and configuring a data plane (load balancer). The intent is to create an extensible and expressive API that lends itself naturally to role-based access control methods. +The SIG-NETWORK group has begun a mission to improve traffic control to and from Kubernetes clusters. The Gateway API +aims to improve service networking by formalizing user personas and developing a decoupled and composable collection of +resources for provisioning and configuring a data plane (load balancer). The intent is to create an extensible and +expressive API that lends itself naturally to role-based access control methods. ## Requirements ### Personas -Gateway API focuses on three main personas: Infrastructure Provider (Infra), Cluster Operator (Admin), and Application Developer (AppDev); for more details see [Roles and personas](https://gateway-api.sigs.k8s.io/concepts/security-model/#roles-and-personas). +Gateway API focuses on three main personas: Infrastructure Provider (Infra), Cluster Operator (Admin), and Application +Developer (AppDev); for more details +see [Roles and personas](https://gateway-api.sigs.k8s.io/concepts/security-model/#roles-and-personas). -The Infra role focuses on provider tier assets: compute (e.g. virtual machines, base-metal server/node hardware), storage (e.g. external arrays, SANs), and networking (e.g. VPNs, subnet isolation, routing). +The Infra role focuses on provider tier assets: compute (e.g. virtual machines, base-metal server/node hardware), +storage (e.g. external arrays, SANs), and networking (e.g. VPNs, subnet isolation, routing). -NGINX does not provide Infra services or a platform product as it pertains to this role. For the purposes of this evaluation we intend to *focus on the **Admin** and **AppDev** personas*. The Gateway evaluation will require (and in some cases assume) a correct infrastructure and network topology is in place; cloud network routing, SDN control, DNS, and L4 load balancing are out of scope. +NGINX does not provide Infra services or a platform product as it pertains to this role. For the purposes of this +evaluation we intend to *focus on the **Admin** and **AppDev** personas*. The Gateway evaluation will require (and in +some cases assume) a correct infrastructure and network topology is in place; cloud network routing, SDN control, DNS, +and L4 load balancing are out of scope. ### Resources The Gateway evaluation intends to solve for a subset of L7 traffic management use-cases. -Three main API kinds will be supported for chosen Core features with Optional and Extended features to be considered later: -* GatewayClass: Defines a set of gateways with a common configuration and behavior. -* Gateway: Requests a point where traffic can be translated to Services within the cluster. -* Routes: Describe how traffic coming via the Gateway maps to the Services. +Three main API kinds will be supported for chosen Core features with Optional and Extended features to be considered +later: + +- GatewayClass: Defines a set of gateways with a common configuration and behavior. +- Gateway: Requests a point where traffic can be translated to Services within the cluster. +- Routes: Describe how traffic coming via the Gateway maps to the Services. The Gateway evaluation will employ each in the following manner: -- GatewayClass: The Admin controls the resource lifecycle and is responsible for creating, updating, and deleting a GatewayClass for NGINX. The GatewayClass is used by the Admin to provide global configuration; by way of ParameterRefs, to the subordinate Gateway resources. -- Gateway: The Admin controls the resource lifecycle and is responsible for creating, updating, and deleting a Gateway for NGINX. The Gateway resource is a representation and description of an NGINX data plane ready to be configured with Routes. -- Routes: The Gateway API describes multiple typed Routes: HTTP, TLS, TCP, and UDP. The AppDev controls the resource lifecycle and is responsible for creating, updating, and deleting a Route. The Gateway evaluation will only support HTTPRoute resources, and only an initial subset of features and not the entire Core set. -Gateway evaluation requires supplemental resources; other Core API resources are needed to complete traffic routing between an ingress point and the final backend Pod endpoints. Required supplemental resources are: -* Services: The AppDev controls the resource lifecycle and is responsible for creating, updating, and deleting Service resources. Kubernetes Service resources describe a logical collection of like processes and act as a load balancing primitive. AppDevs create Service resources to describe their product subsystems, Services act as an abstraction for a set of real backend processes that can serve requests. -* Endpoints: Kubernetes controllers manage Endpoint resources, Endpoints provide the association between Service abstractions and real servers. Gateway evaluation will watch Endpoints to discover upstream addresses. When AppDevs create Service and HTTPRoute objects, Gateway evaluation uses references in the HTTPRoute (via a HTTPRouteRules and HTTPBackendRefs (see [HTTP Routing](https://gateway-api.sigs.k8s.io/guides/http-routing/) for more detail)) to discover the Pod IP endpoints, i.e. Gateway evaluation uses Endpoints resources, referred to by HTTPRoutes, to link routing rules to upstream addresses. +- GatewayClass: The Admin controls the resource lifecycle and is responsible for creating, updating, and deleting a + GatewayClass for NGINX. The GatewayClass is used by the Admin to provide global configuration; by way of + ParameterRefs, to the subordinate Gateway resources. +- Gateway: The Admin controls the resource lifecycle and is responsible for creating, updating, and deleting a Gateway + for NGINX. The Gateway resource is a representation and description of an NGINX data plane ready to be configured with + Routes. +- Routes: The Gateway API describes multiple typed Routes: HTTP, TLS, TCP, and UDP. The AppDev controls the resource + lifecycle and is responsible for creating, updating, and deleting a Route. The Gateway evaluation will only support + HTTPRoute resources, and only an initial subset of features and not the entire Core set. + +Gateway evaluation requires supplemental resources; other Core API resources are needed to complete traffic routing +between an ingress point and the final backend Pod endpoints. Required supplemental resources are: + +- Services: The AppDev controls the resource lifecycle and is responsible for creating, updating, and deleting Service + resources. Kubernetes Service resources describe a logical collection of like processes and act as a load balancing + primitive. AppDevs create Service resources to describe their product subsystems, Services act as an abstraction for a + set of real backend processes that can serve requests. +- Endpoints: Kubernetes controllers manage Endpoint resources, Endpoints provide the association between Service + abstractions and real servers. Gateway evaluation will watch Endpoints to discover upstream addresses. When AppDevs + create Service and HTTPRoute objects, Gateway evaluation uses references in the HTTPRoute (via a HTTPRouteRules and + HTTPBackendRefs (see [HTTP Routing](https://gateway-api.sigs.k8s.io/guides/http-routing/) for more detail)) to + discover the Pod IP endpoints, i.e. Gateway evaluation uses Endpoints resources, referred to by HTTPRoutes, to link + routing rules to upstream addresses. ### Goals + - Vet a minimal feature set of Core resources for their applicability and conformance to NGINX configuration models. - Support L7 HTTPRoute resources; host routing and path routing only. - Support GatewayClass, Gateway, and HTTPRoute Status subobject requirements. - Develop expertise in Gateway API and refine a compatible, extensible, and flexible architecture. ### Non-Goals -- Replace or supplant current [NGINX Ingress Controller](https://github.com/nginxinc/kubernetes-ingress) (NIC) use-cases and will not work to achieve feature parity. -- The Gateway evaluation will not support Ingress v1 APIs or the set of NIC CRDs (e.g. VirtualServer and VirtualServerRoute). + +- Replace or supplant current [NGINX Ingress Controller](https://github.com/nginxinc/kubernetes-ingress) (NIC) use-cases + and will not work to achieve feature parity. +- The Gateway evaluation will not support Ingress v1 APIs or the set of NIC CRDs (e.g. VirtualServer and + VirtualServerRoute). - The Gateway will only support Service resources as a proper HTTPBackendRef. ## Design + Gateway evaluation is a decoupled (two containers) control plane and data plane. -The data plane is NGINX OSS run as a container within a Kubernetes Pod with `.spec.shareProcessNamespace` set to true. The control plane will interact with the data plane to provide configuration and limited process control. +The data plane is NGINX OSS run as a container within a Kubernetes Pod with `.spec.shareProcessNamespace` set to true. +The control plane will interact with the data plane to provide configuration and limited process control. -The control plane manages Kubernetes controllers; each controller follows the [Controller pattern](https://kubernetes.io/docs/concepts/architecture/controller/#controller-pattern). +The control plane manages Kubernetes controllers; each controller follows +the [Controller pattern](https://kubernetes.io/docs/concepts/architecture/controller/#controller-pattern). General dataflow: -``` +```text ----------- --------------- --------------- --------------------------- ----------------------- | K8S API | -> | Controllers | -> | Reconcilers | -> | Configuration Subsystem | -> | NGINX Data Plane | ----------- --------------- --------------- | Conf Graph / AST / IR | | Configuration Write | @@ -65,12 +103,20 @@ General dataflow: |------- Process Signal -------| ``` -The above shows a general data flow for the Gateway evaluation. Admins and AppDevs interact directly with the Kubernetes API server and the Gateway evaluation provides no additional network APIs or out-of-band control mechanisms for user interactions (excluding environment and CLI startup arguments). Using the controller pattern, Gateway evaluation reacts to and reconciles configuration updates. The Gateway API primitives are translated to compatible internal data model representations of NGINX data plane configurations. The resultant configurations are sent to data plane components and actuated via data plane natively supported means (writing to disk, API calls, process signaling, etc.). +The above shows a general data flow for the Gateway evaluation. Admins and AppDevs interact directly with the Kubernetes +API server and the Gateway evaluation provides no additional network APIs or out-of-band control mechanisms for user +interactions (excluding environment and CLI startup arguments). Using the controller pattern, Gateway evaluation reacts +to and reconciles configuration updates. The Gateway API primitives are translated to compatible internal data model +representations of NGINX data plane configurations. The resultant configurations are sent to data plane components and +actuated via data plane natively supported means (writing to disk, API calls, process signaling, etc.). -Gateway evaluation will be deployed inside the Kubernetes cluster using Deployments, DaemonSets, or unmanaged Pods and must be a pair of containers - the data plane container and the control plane container. (TBD: leader election, controller-runtime provides facilities but requirements need defining.) +Gateway evaluation will be deployed inside the Kubernetes cluster using Deployments, DaemonSets, or unmanaged Pods and +must be a pair of containers - the data plane container and the control plane container. (TBD: leader election, +controller-runtime provides facilities but requirements need defining.) Expected deployment: -``` + +```text External | Cluster Network | Network | -------------- @@ -94,32 +140,47 @@ Network | Network ``` ### Prerequisites -- Infra persona has enabled external access to the cluster. For example, configured an external network device which has connectivity to the cluster Node underlay network. + +- Infra persona has enabled external access to the cluster. For example, configured an external network device which has + connectivity to the cluster Node underlay network. - Admin has knowledge of the cluster external DNS Hostname and listen Port. ### Deployment Requirements + - Admin provisions GatewayClass referencing Gateway object name. - Admin provisions Gateway using name referenced from GatewayClass. -- Admin provisions Gateway using hostname and port: `.spec.listeners[].{ hostname, port, protocol }` where protocol MUST be "HTTP". -- Admin provisions cluster access to NGINX data planes via Service abstractions: Service `.spec.type=LoadBalancer` or `.spec.type=NodePort`. -- Admin provisions Gateway evaluation deployment via workload abstractions, Deployment or DaemonSet. Admin MAY provision single Pod via an unmanaged Pod configuration. +- Admin provisions Gateway using hostname and port: `.spec.listeners[].{ hostname, port, protocol }` where protocol MUST + be "HTTP". +- Admin provisions cluster access to NGINX data planes via Service abstractions: Service `.spec.type=LoadBalancer` + or `.spec.type=NodePort`. +- Admin provisions Gateway evaluation deployment via workload abstractions, Deployment or DaemonSet. Admin MAY provision + single Pod via an unmanaged Pod configuration. ### Operation > ***NOTE*** > -> *Where applicable Gateway evaluation will write status updates to Gateway API resources according to specification requirements. Each update is not recorded in these flows.* +> *Where applicable Gateway evaluation will write status updates to Gateway API resources according to specification +requirements. Each update is not recorded in these flows.* #### Startup -At process startup, the Gateway evaluation must discover the "state of the world". Until startup constraints have been satisified Gateway evaluation data plane instances will be unable to proxy data. Data plane instances MUST be inaccessible until GatewayClass, Gateway resources are properly resolved, i.e., GatewayClass and Gateway resources gate data flow. Gateway evaluation MAY use a combination of Kubernetes readinessProbes and NGINX configuration to prevent traffic. + +At process startup, the Gateway evaluation must discover the "state of the world". Until startup constraints have been +satisified Gateway evaluation data plane instances will be unable to proxy data. Data plane instances MUST be +inaccessible until GatewayClass, Gateway resources are properly resolved, i.e., GatewayClass and Gateway resources gate +data flow. Gateway evaluation MAY use a combination of Kubernetes readinessProbes and NGINX configuration to prevent +traffic. Gateway evaluation will, -- initialize with a minimal configuration; reading an environment variable or command-line argument referencing the Gateway's controller name (the controller name will match the Gateway.Name and the GatewayClass.Controller fields). + +- initialize with a minimal configuration; reading an environment variable or command-line argument referencing the + Gateway's controller name (the controller name will match the Gateway.Name and the GatewayClass.Controller fields). - find its GatewayClass. - list GatewayClass resources. - match GatewayClass.Controller to Gateway evaluation's controller name (conflict resolution: oldest wins). - watch and wait for GatewayClass creation and updates. - - log its state and subsequent state change (GatewayClass found or not found, GatewayClass discovered and process proceeding). + - log its state and subsequent state change (GatewayClass found or not found, GatewayClass discovered and process + proceeding). - watch for ParametersRef: `.spec.parametersRef`. If not found, continue processing with default runtime configuration. - find its Gateway. - list Gateway resources. @@ -133,15 +194,27 @@ Gateway evaluation will, - enter runtime loop. #### Runtime -The Gateway evaluation is primarily a set of control loops watching for changes in the declared configuration of the Gateway API. -Process runtime configuration updates may be delivered by updates to GatewayClass, Gateway, and ParametersRef. Gateway evaluation will watch these resources for updates, reconciling their changes, and resolving or rejecting based on calculated conflicts. +The Gateway evaluation is primarily a set of control loops watching for changes in the declared configuration of the +Gateway API. -Data plane traffic decisions and configuration are primarily conveyed via HTTPRoute updates. Gateway evaluation will establish an HTTPRoute controller reconciling configuration updates, and resolving or rejecting based on calculated conflicts or validation errors. HTTPRoute resources describe the server side proxy configuration using a set of rules applied to backend references. For the Gateway evaluation, only Service references will be supported. Gateway evaluation will establish additional controllers for Service and Endpoints updates. Updates (HTTPRoute and Endpoints) are translated into an intermediate representation describing the NGINX server, location, and upstreams; data structure specifics are yet to determined. Properties desired: sparse, directed, acyclic, relational. +Process runtime configuration updates may be delivered by updates to GatewayClass, Gateway, and ParametersRef. Gateway +evaluation will watch these resources for updates, reconciling their changes, and resolving or rejecting based on +calculated conflicts. -Once validated for conflicts and errors, the intermediate representation is exported to the data plane as NGINX configuration directives. +Data plane traffic decisions and configuration are primarily conveyed via HTTPRoute updates. Gateway evaluation will +establish an HTTPRoute controller reconciling configuration updates, and resolving or rejecting based on calculated +conflicts or validation errors. HTTPRoute resources describe the server side proxy configuration using a set of rules +applied to backend references. For the Gateway evaluation, only Service references will be supported. Gateway evaluation +will establish additional controllers for Service and Endpoints updates. Updates (HTTPRoute and Endpoints) are +translated into an intermediate representation describing the NGINX server, location, and upstreams; data structure +specifics are yet to determined. Properties desired: sparse, directed, acyclic, relational. + +Once validated for conflicts and errors, the intermediate representation is exported to the data plane as NGINX +configuration directives. Loop cycle: + - Event notification: Upsert, Delete - Reconcilation: - Service/Endpoints, HTTPRoute: @@ -172,7 +245,11 @@ Loop cycle: - Reconfigure control plane and data plane elements. ### Security Considerations -Gateway evaluation will not support ReferencePolicy resources and will not support cross namespace references. All references are to be validated against the parent resource's namespace and only allowed when the resources reside in the same isolation boundary. See [Object references](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#object-references). + +Gateway evaluation will not support ReferencePolicy resources and will not support cross namespace references. All +references are to be validated against the parent resource's namespace and only allowed when the resources reside in the +same isolation boundary. +See [Object references](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#object-references). ## Appendix A @@ -181,7 +258,8 @@ Gateway evaluation will not support ReferencePolicy resources and will not suppo > *This is not a design requirement and may not be met as a delivery constraint.* Controller design: -``` + +```text --------------- ------------------ ----------------------- ---------------------- | Controllers | -> | SDK Reconciler | -> | SDK Impl. Reconcile | -> | Graph / AST / Conf | --------------- ------------------ ----------------------- ---------------------- @@ -192,12 +270,17 @@ Controller design: ------- ------------- ``` -Controllers can be defined using Go interfaces and a modified Pimpl idiom (which Go naturally supports via structural typing and the interface paradigm). +Controllers can be defined using Go interfaces and a modified Pimpl idiom (which Go naturally supports via structural +typing and the interface paradigm). -With this approach controllers are built with a public set of interfaces and are responsible for all event watch operations. Implementations are separated into individual packages which satisfy the public interfaces. In the above diagram, SDK represents a package which provides access to versioned implementations, public interfaces, and the core network machinery of a controller reconciliation loop. +With this approach controllers are built with a public set of interfaces and are responsible for all event watch +operations. Implementations are separated into individual packages which satisfy the public interfaces. In the above +diagram, SDK represents a package which provides access to versioned implementations, public interfaces, and the core +network machinery of a controller reconciliation loop. E.g.) -``` + +```go package sdk type V1Alpha2Impl interface{ @@ -218,14 +301,16 @@ func (s sdk) V1Alpha2() V1Alpha2Impl { ``` SDK controllers reconcile and call through to backend implementations: -``` + +```go func (c controller) Reconcile(...) { sdk.V1Alpha2().Upsert(...) } ``` where an implementer will satisfy the interface and register their implementation during startup: -``` + +```go type myImpl { } diff --git a/design/control-data-plane-separation/design.md b/design/control-data-plane-separation/design.md index 0bf0c9f28a..4c1630c8ed 100644 --- a/design/control-data-plane-separation/design.md +++ b/design/control-data-plane-separation/design.md @@ -6,7 +6,7 @@ Issue #292: https://github.com/nginxinc/nginx-kubernetes-gateway/issues/292 > Note: I use data plane and agent interchangeably in this document. -# Background +## Background NKG composes its control and data plane containers into a single Kubernetes Pod. The control plane uses OS signals and a shared file system to configure and reload nginx. This architecture is problematic because the same RBAC policies govern @@ -14,7 +14,7 @@ the control and data planes and share CVE potential. A compromised control plane Kubernetes API server may be affected if the data plane is compromised. In addition to security concerns, this architecture does not allow the control plane and data plane to scale independently. -# Goals +## Goals - Data plane and control plane containers run in separate Pods - The communication channel between the control and data planes can be encrypted @@ -24,32 +24,32 @@ architecture does not allow the control plane and data plane to scale independen Kubernetes API server. - RBAC policy for control plane follows the principle of least privilege. -# Non-Goals +## Non-Goals - Control plane can scale - Support for multiple control planes per GatewayClass or Namespace. -# Design +## Design Since the choice of a data plane agent will inform the rest of the design, I will start by proposing the agent. -## Nginx Agent +### Nginx Agent I propose using the [nginx agent](https://github.com/nginx/agent) as our data plane agent. -### Evaluation of Requirements +#### Evaluation of Requirements The following list outlines all of NKG's requirements for an agent and whether the nginx agent meets them: - [x] It is open source. - [x] It supports both OSS and Plus versions of nginx. - [x] It can be deployed in Kubernetes. - - [ ] It supports readiness and liveness probes. - - [x] It supports logging to stderr/stdout. - - [x] It handles SIGTERM gracefully. - - [ ] Its container base image is scratch or something minimal. - - [x] It supports a read-only root file system. - - [x] It can run as a non-root user. + - [ ] It supports readiness and liveness probes. + - [x] It supports logging to stderr/stdout. + - [x] It handles SIGTERM gracefully. + - [ ] Its container base image is scratch or something minimal. + - [x] It supports a read-only root file system. + - [x] It can run as a non-root user. - [x] It can configure an nginx instance running in the same Pod. - [x] It is loosely coupled to the nginx version. The Nginx version can be updated independently of the agent. - [x] A user can update it independently of the control plane. @@ -81,8 +81,8 @@ Features needed (in priority order, more or less): - Add an option to refresh server token from a file - Add readiness and liveness endpoints - Produce a container image as a release artifact - - This image should be non-root - - This image should be as minimal as possible + - This image should be non-root + - This image should be as minimal as possible - Allow the control plane to access the N+ API to configure upstreams and the key-value store. - Add support for metrics enrichment. Metrics can be enriched with Kubernetes meta-information such as namespace, pod name, etc. @@ -92,11 +92,11 @@ Agent features/plugins that we'd like to disable: - Metrics service client - Data plane status updates - Config upload feature - - This is the feature that uploads the config to the control plane + - This is the feature that uploads the config to the control plane - The nginx-counting feature - The activity-events feature -### Benefits +#### Benefits Using the nginx agent has the following benefits: @@ -111,22 +111,22 @@ Using the nginx agent has the following benefits: - It is built to be modular and configurable. We should be able to disable most or all of the features we don’t need in the future. -### Drawbacks +#### Drawbacks Using the nginx agent has the following drawbacks: - It is not custom-built for our use case. It contains more features that we need. - It was not built to run in Kubernetes. - - Violates some of the best practices for running in Kubernetes. For example, it runs two processes in a single + - Violates some of the best practices for running in Kubernetes. For example, it runs two processes in a single container. - - Metrics do not include Kubernetes meta-information. + - Metrics do not include Kubernetes meta-information. - It is a dependency that we do not control. - It does not support dynamic configuration of upstreams and the key-value store. - We may need to replace it in the future if we run into performance issues or encounter a blocker. -## Alternatives +### Alternatives -### Write our own agent +#### Write our own agent We could write our own agent. This would give us the most control over the design and implementation. There are few different approaches we could take to design an agent. @@ -185,7 +185,7 @@ Drawbacks: - Translating xDS to nginx config is not a trivial task. Some features will not map easily to nginx config. - We would either need to completely re-write our control plane or adopt an xDS control plane -### Modify nginx-agent +#### Modify nginx-agent We could modify the nginx agent to meet our needs. This would require us to maintain a fork of the nginx agent. I see this option as a last resort that we can decide to adopt if we hit a roadblock with the nginx agent. @@ -198,20 +198,20 @@ this option as a last resort that we can decide to adopt if we hit a roadblock w [xds-wg]: https://github.com/cncf/xds -## Deployment Architecture +### Deployment Architecture ![Deployment architecture](deployment-architecture.png) -* _Control Plane Deployment_: The control plane is a Kubernetes Deployment with one container running the NKG +- _Control Plane Deployment_: The control plane is a Kubernetes Deployment with one container running the NKG controller. Initially, the control plane will be limited to a single Pod. Once we add leader election, the control plane will be able to scale. The control plane will perform the same functions as it does today, but instead of configuring nginx by writing files to a shared volume, it will send the configuration to the agent via gRPC. -* _Control Plane Service_: Exposes the control plane via a Kubernetes Service of type `ClusterIP`. The data plane will +- _Control Plane Service_: Exposes the control plane via a Kubernetes Service of type `ClusterIP`. The data plane will use the DNS name of the Service to connect to the control plane. -* _Data Plane DaemonSet/Deployment_: A user can deploy the data plane as either a DaemonSet or Deployment. The data +- _Data Plane DaemonSet/Deployment_: A user can deploy the data plane as either a DaemonSet or Deployment. The data plane contains a single container running both the agent and nginx processes. The agent will download the configuration from the control plane over a streaming RPC. -* _NGINX Service_: Exposes nginx via a Kubernetes Service of type `LoadBalancer .`This is the entry point for the +- _NGINX Service_: Exposes nginx via a Kubernetes Service of type `LoadBalancer .`This is the entry point for the customer’s traffic. Initially, this Service will only expose ports 80 and 443. In the future, if we add support for additional listener ports, this Service will expose all the listener ports. Note that this Service should not expose any of the agent’s ports. @@ -230,7 +230,7 @@ controllers will be able to coexist in the same cluster as long as they each hav resource. In this case, each installation of NKG will contain a unique `GatewayClass` resource, a control plane Deployment, and a data plane Deployment/DaemonSet. -## Communication Channels +### Communication Channels The control plane and agent will communicate over gRPC. The agent will establish a gRPC connection to the control plane on start-up. The agent will gracefully retry to connect to the control plane, so the start order of the containers is @@ -292,9 +292,9 @@ this [file](https://github.com/nginx/agent/blob/main/sdk/proto/command_svc.proto Command Messages have the following structure: -* Metadata - contains details about the sender and the message. -* Type - contains information about the type of data the message carries. -* Data - the message payload. +- Metadata - contains details about the sender and the message. +- Type - contains information about the type of data the message carries. +- Data - the message payload. Command messages act as envelopes, but they make use of both the type field and the `oneof` feature of gRPC to embed different types of payloads. @@ -377,14 +377,14 @@ message NginxConfig { } ``` -* `action`: what action the agent should take with the nginx config: apply, test, rollback, return, or force. -* `config_data`: contains metadata on the agent and nginx instance we are configuring. -* `zconfig`: a zipped file with all nginx `.conf` files. -* `zaux`: a zipped file with all nginx auxiliary files, such as njs modules, static html files, etc. -* `access_logs`: meta-information about the access logs. -* `error_logs`: meta-information about the error logs. -* `ssl`: meta-information about the SSL certificates stored on the data plane. -* `directory_map`:meta-information about the nginx configuration files. The agent uses this for synchronization (i.e., +- `action`: what action the agent should take with the nginx config: apply, test, rollback, return, or force. +- `config_data`: contains metadata on the agent and nginx instance we are configuring. +- `zconfig`: a zipped file with all nginx `.conf` files. +- `zaux`: a zipped file with all nginx auxiliary files, such as njs modules, static html files, etc. +- `access_logs`: meta-information about the access logs. +- `error_logs`: meta-information about the error logs. +- `ssl`: meta-information about the SSL certificates stored on the data plane. +- `directory_map`:meta-information about the nginx configuration files. The agent uses this for synchronization (i.e., comparing configuration against previous deployments) and to interrogate the file system before applying the configuration. @@ -421,7 +421,7 @@ well. [tls-config]: https://pkg.go.dev/crypto/tls#Config -## Authorization +### Authorization The agent will use a Kubernetes ServiceAccount token to authenticate with the control plane. The control plane will authenticate the token by sending a request to the Kubernetes [TokenReview API][token-review]. @@ -439,7 +439,7 @@ Upon successful connection, the agent will register by sending an `AgentConnecti to the control plane over the `CommandChannel.` This message is used by the control plane to associate the agent with internal resources correctly. See the [Registration](#agent-registration) section for more information. -### Long-lived token v/s bound token +#### Long-lived token v/s bound token Long-lived tokens are JWT tokens for a ServiceAccount that are valid for the lifetime of the ServiceAccount. They are stored in Secrets and can be mounted to a Pod as a file or an environment variable. We can use the TokenReview API to @@ -509,7 +509,7 @@ For a good comparison of long-lived and bound tokens, see [this blog post][bound [projected-volume]: https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#bound-service-account-token-volume -## Agent Registration +### Agent Registration ![Agent Connect Response](./connect-response.png) @@ -531,9 +531,9 @@ the enabled features, extensions, tags, log configuration, and alias for the age [response]: https://github.com/nginx/agent/blob/ea3a1b4df5d7ecf95bd3d9297d26e420f5e1dd57/sdk/proto/agent.pb.go#L226 -## Configuration Download +### Configuration Download -### Building the NginxConfig message +#### Building the NginxConfig message Currently, NKG configures nginx by translating the Gateway API resources into an [internal representation of the nginx config][internal-config], executing a template with this data to generate the @@ -560,14 +560,14 @@ Note that we must send the entire nginx configuration to the agent on each confi checksum if we want to reduce the number of configuration updates sent to the agent. By storing the last checksum in the control plane, we can avoid sending the configuration to the agent if it hasn’t changed. -### Static Configuration Files +#### Static Configuration Files Static configuration files, such as njs modules, do not need to be sent to the agent on every configuration update. Instead, we will mount these files to the agent’s container using a `ConfigMap .`This will also require us to specify the path to the module in the agent’s configuration file and the `DirectoryMap` of the `NginxConfig` message to prevent the agent from removing them from the filesystem. -### Handling User’s Secret Data +#### Handling User’s Secret Data The TLS certificates and keys specified in the [`GatewayTLSConfig`][gw-tls-config] field of `Listeners` are references to Kubernetes Secrets. In the future, we will support other forms of authentication data, such as JWT tokens and @@ -654,7 +654,7 @@ Drawbacks: > My preference is option 1, as it is the simplest and requires the least changes. -### Sending the NginxConfig message +#### Sending the NginxConfig message ![Download](./download-config.png) @@ -682,9 +682,8 @@ the `CommandChannel` in a `NginxConfigResponse` message. [host-path]: https://kubernetes.io/docs/concepts/storage/volumes/#hostpath -[config-resp]: https://github.com/nginx/agent/blob/ea3a1b4df5d7ecf95bd3d9297d26e420f5e1dd57/sdk/proto/command.pb.go#L891 -## Agent Configuration +### Agent Configuration We can configure the agent through a YAML file, command-line flags, or environment variables on start-up. The agent interprets configuration in the following priorities (from highest to lowest): @@ -779,13 +778,13 @@ ok command-line-arguments 17.727s - NKG can handle frequent configuration changes (1 change per second) - NKG can handle large configurations: - - 5000 server blocks - - 64 TLS certs/keys - - 50 JWT keys - - 50 TLS cert/keys for egress - - 50 CA certs - - 50 basic auth files - - 50 OIDC secrets + - 5000 server blocks + - 64 TLS certs/keys + - 50 JWT keys + - 50 TLS cert/keys for egress + - 50 CA certs + - 50 basic auth files + - 50 OIDC secrets - NKG can scale to X number of data plane pods (we need to figure out what X is) [performance]: https://github.com/nginx/agent/blob/main/test/performance/user_workflow_test.go From 97f53905b479fad3e657109cb3fb2a934766b55b Mon Sep 17 00:00:00 2001 From: Kate Osborn Date: Thu, 20 Jul 2023 12:37:35 -0600 Subject: [PATCH 3/5] Add lint check to workflow --- .github/workflows/lint.yml | 12 ++++++++++++ .pre-commit-config.yaml | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b7a802ff50..e9142aab70 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -76,3 +76,15 @@ jobs: - uses: reviewdog/action-actionlint@7485c2136bd093d2317a854c72910eebaee35238 # v1.37.1 with: actionlint_flags: -shellcheck "" + + markdown-lint: + name: Markdown Lint + runs-on: ubuntu-22.04 + steps: + - name: Checkout Repository + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + + - uses: DavidAnson/markdownlint-cli2-action@8f3516061301755c97ff833a8e933f09282cc5b5 # v11.0.0 + with: + config: ${{ github.workspace }}.markdownlint-cli2.yaml + globs: '**/*.md' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cfd66b95d9..f16467419c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,4 +56,4 @@ repos: - id: markdownlint-cli2-fix ci: - skip: [golang-diff, golangci-lint, prettier] + skip: [golang-diff, golangci-lint, prettier, markdownlint-cli2-fix] From 74a1fa41d7edb97b825f2bb8109c1aaed23082cf Mon Sep 17 00:00:00 2001 From: Kate Osborn Date: Thu, 20 Jul 2023 12:38:58 -0600 Subject: [PATCH 4/5] Fix path --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e9142aab70..213e1a77ad 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -86,5 +86,5 @@ jobs: - uses: DavidAnson/markdownlint-cli2-action@8f3516061301755c97ff833a8e933f09282cc5b5 # v11.0.0 with: - config: ${{ github.workspace }}.markdownlint-cli2.yaml + config: ${{ github.workspace }}/.markdownlint-cli2.yaml globs: '**/*.md' From fc5c04c18c3fdfa3caee0b5789cca380d496198c Mon Sep 17 00:00:00 2001 From: Kate Osborn Date: Thu, 20 Jul 2023 15:17:01 -0600 Subject: [PATCH 5/5] Fix file name --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f16467419c..a4a3a678fd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,7 +48,7 @@ repos: - id: golangci-lint args: [--new-from-patch=/tmp/diff.patch] - # Rules are in .markdownlint.json file + # Rules are in .markdownlint-cli2.yaml file # See https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md for rule descriptions - repo: https://github.com/DavidAnson/markdownlint-cli2 rev: v0.8.1