From ca165d9a5dc50c4c998dc9140e23050cb86c8fd5 Mon Sep 17 00:00:00 2001 From: Saylor Berman Date: Thu, 17 Aug 2023 16:05:52 -0600 Subject: [PATCH] Fix CRD manifest installation Problem: Deploying a CRD definition and an instantiation of that CRD in the same kubectl command will fail due to the kubectl cache not being updated for the existence of the CRD definition. They need to be run in separate commands. Solution: Separate the CRDs into a different file and install prior to installing the rest of the gateway resources. --- .yamllint.yaml | 2 + Makefile | 3 +- conformance/Makefile | 2 +- .../crds/gateway.nginx.org_nginxgateways.yaml | 134 +++++++++++++++++ deploy/manifests/nginx-gateway.yaml | 137 ------------------ docs/developer/quickstart.md | 1 + docs/installation.md | 10 ++ 7 files changed, 150 insertions(+), 139 deletions(-) create mode 100644 deploy/manifests/crds/gateway.nginx.org_nginxgateways.yaml diff --git a/.yamllint.yaml b/.yamllint.yaml index a846ba0179..c199e193b1 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -30,6 +30,7 @@ rules: check-multi-line-strings: true ignore: | deploy/manifests/nginx-gateway.yaml + deploy/manifests/crds key-duplicates: enable key-ordering: disable line-length: @@ -39,6 +40,7 @@ rules: ignore: | .github/workflows/ deploy/manifests/nginx-gateway.yaml + deploy/manifests/crds new-line-at-end-of-file: enable new-lines: enable octal-values: disable diff --git a/Makefile b/Makefile index 76ba88b924..eeb61356e1 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config## The location of the kind kubeconfi OUT_DIR ?= $(shell pwd)/build/out## The folder where the binary will be stored ARCH ?= amd64## The architecture of the image and/or binary. For example: amd64 or arm64 override HELM_TEMPLATE_COMMON_ARGS += --set creator=template --set nameOverride=nginx-gateway## The common options for the Helm template command. -override HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE += --include-crds --set service.create=false## The options to be passed to the full Helm templating command only. +override HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE += --set service.create=false## The options to be passed to the full Helm templating command only. override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) .DEFAULT_GOAL := help @@ -133,6 +133,7 @@ build-nkg-debug-image: debug-build build-nkg-image ## Build NKG image with debug .PHONY: generate-manifests generate-manifests: ## Generate manifests using Helm. + cp $(CHART_DIR)/crds/* $(MANIFEST_DIR)/crds/ helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) $(HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE) -n nginx-gateway | cat $(strip $(MANIFEST_DIR))/namespace.yaml - > $(strip $(MANIFEST_DIR))/nginx-gateway.yaml helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) -n nginx-gateway -s templates/deployment.yaml > conformance/provisioner/static-deployment.yaml helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) -n nginx-gateway -s templates/service.yaml > $(strip $(MANIFEST_DIR))/service/loadbalancer.yaml diff --git a/conformance/Makefile b/conformance/Makefile index d081f454ba..702ec0c6a7 100644 --- a/conformance/Makefile +++ b/conformance/Makefile @@ -8,7 +8,7 @@ KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config TAG = latest PREFIX = conformance-test-runner NKG_MANIFEST=../deploy/manifests/nginx-gateway.yaml -CRDS=../deploy/helm-chart/crds/ +CRDS=../deploy/manifests/crds/ SERVICE_MANIFEST=../deploy/manifests/service/nodeport.yaml STATIC_MANIFEST=provisioner/static-deployment.yaml PROVISIONER_MANIFEST=provisioner/provisioner.yaml diff --git a/deploy/manifests/crds/gateway.nginx.org_nginxgateways.yaml b/deploy/manifests/crds/gateway.nginx.org_nginxgateways.yaml new file mode 100644 index 0000000000..a5829d407c --- /dev/null +++ b/deploy/manifests/crds/gateway.nginx.org_nginxgateways.yaml @@ -0,0 +1,134 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: nginxgateways.gateway.nginx.org +spec: + group: gateway.nginx.org + names: + kind: NginxGateway + listKind: NginxGatewayList + plural: nginxgateways + singular: nginxgateway + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NginxGateway represents the dynamic configuration for an NGINX + Kubernetes Gateway control plane. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NginxGatewaySpec defines the desired state of the NginxGateway. + properties: + logging: + description: Logging defines logging related settings for the control + plane. + properties: + level: + default: info + description: Level defines the logging level. + enum: + - info + - debug + - error + type: string + type: object + type: object + status: + description: NginxGatewayStatus defines the state of the NginxGateway. + properties: + conditions: + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + \n type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge + // +listType=map // +listMapKey=type Conditions []metav1.Condition + `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/manifests/nginx-gateway.yaml b/deploy/manifests/nginx-gateway.yaml index be614b8200..2036234985 100644 --- a/deploy/manifests/nginx-gateway.yaml +++ b/deploy/manifests/nginx-gateway.yaml @@ -3,143 +3,6 @@ kind: Namespace metadata: name: nginx-gateway --- -# Source: crds/gateway.nginx.org_nginxgateways.yaml ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.13.0 - name: nginxgateways.gateway.nginx.org -spec: - group: gateway.nginx.org - names: - kind: NginxGateway - listKind: NginxGatewayList - plural: nginxgateways - singular: nginxgateway - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: NginxGateway represents the dynamic configuration for an NGINX - Kubernetes Gateway control plane. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NginxGatewaySpec defines the desired state of the NginxGateway. - properties: - logging: - description: Logging defines logging related settings for the control - plane. - properties: - level: - default: info - description: Level defines the logging level. - enum: - - info - - debug - - error - type: string - type: object - type: object - status: - description: NginxGatewayStatus defines the state of the NginxGateway. - properties: - conditions: - items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - maxItems: 8 - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} - ---- # Source: nginx-kubernetes-gateway/templates/rbac.yaml apiVersion: v1 kind: ServiceAccount diff --git a/docs/developer/quickstart.md b/docs/developer/quickstart.md index 71f3b38368..7c27218468 100644 --- a/docs/developer/quickstart.md +++ b/docs/developer/quickstart.md @@ -104,6 +104,7 @@ This will build the docker images `nginx-kubernetes-gateway:` and `ng ```shell make generate-manifests HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=nginx-kubernetes-gateway --set nginxGateway.image.tag=$(whoami) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=nginx-kubernetes-gateway/nginx --set nginx.image.tag=$(whoami) --set nginx.image.pullPolicy=Never" + kubectl apply -f deploy/manifests/crds kubectl apply -f deploy/manifests/nginx-gateway.yaml kubectl apply -f deploy/manifests/service/nodeport.yaml ``` diff --git a/docs/installation.md b/docs/installation.md index b1ce91af94..11f540bfd7 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -30,6 +30,12 @@ page. kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v0.7.1/standard-install.yaml ``` +1. Deploy the NGINX Kubernetes Gateway CRDs: + + ```shell + kubectl apply -f deploy/manifests/crds + ``` + 1. Deploy the NGINX Kubernetes Gateway: ```shell @@ -122,6 +128,10 @@ To get started, follow the tutorials in the [examples](../examples) directory. kubectl delete -f deploy/manifests/nginx-gateway.yaml ``` + ```shell + kubectl delete -f deploy/manifests/crds + ``` + 1. Uninstall the Gateway API resources from the standard channel (the CRDs and the validating webhook): >**Warning: This command will delete all the corresponding custom resources in your cluster across all namespaces!