diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 32265936e032d..f60c3dc68bb62 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -371,7 +371,6 @@ func TestMinimalGCE(t *testing.T) { withAddons( dnsControllerAddon, gcpCCMAddon, - gcpPDCSIAddon, ). runTestTerraformGCE(t) } diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 4a46eaadbfcfb..b0348cd98c9bf 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -482,9 +482,18 @@ spec: gcpPDCSIDriver: description: GCPPDCSIDriver is the config for the GCP PD CSI driver properties: + defaultStorageClassName: + description: Default StorageClassName is the name of the default + StorageClass created for GCP PD CSI driver. + type: string enabled: description: Enabled enables the GCP PD CSI driver type: boolean + version: + description: |- + Version is the container image tag used. + Default: The latest stable release which is compatible with your Kubernetes version + type: string type: object manageStorageClasses: description: |- diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index b79e86add3daf..c4baed039442f 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -984,6 +984,11 @@ type EBSCSIDriverSpec struct { type PDCSIDriver struct { // Enabled enables the GCP PD CSI driver Enabled *bool `json:"enabled,omitempty"` + // Version is the container image tag used. + // Default: The latest stable release which is compatible with your Kubernetes version + Version *string `json:"version,omitempty"` + // Default StorageClassName is the name of the default StorageClass created for GCP PD CSI driver. + DefaultStorageClassName *string `json:"defaultStorageClassName,omitempty"` } // SnapshotControllerConfig is the config for the CSI Snapshot Controller diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index f576c0f3d39e3..513db503ef1d4 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -1054,6 +1054,11 @@ type EBSCSIDriverSpec struct { type PDCSIDriver struct { // Enabled enables the GCP PD CSI driver Enabled *bool `json:"enabled,omitempty"` + // Version is the container image tag used. + // Default: The latest stable release which is compatible with your Kubernetes version + Version *string `json:"version,omitempty"` + // Default StorageClassName is the name of the default StorageClass created for GCP PD CSI driver. + DefaultStorageClassName *string `json:"defaultStorageClassName,omitempty"` } // SnapshotControllerConfig is the config for the CSI Snapshot Controller diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 00740ac9871a7..d4104d59c68dc 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -7258,6 +7258,8 @@ func Convert_kops_OpenstackSpec_To_v1alpha2_OpenstackSpec(in *kops.OpenstackSpec func autoConvert_v1alpha2_PDCSIDriver_To_kops_PDCSIDriver(in *PDCSIDriver, out *kops.PDCSIDriver, s conversion.Scope) error { out.Enabled = in.Enabled + out.Version = in.Version + out.DefaultStorageClassName = in.DefaultStorageClassName return nil } @@ -7268,6 +7270,8 @@ func Convert_v1alpha2_PDCSIDriver_To_kops_PDCSIDriver(in *PDCSIDriver, out *kops func autoConvert_kops_PDCSIDriver_To_v1alpha2_PDCSIDriver(in *kops.PDCSIDriver, out *PDCSIDriver, s conversion.Scope) error { out.Enabled = in.Enabled + out.Version = in.Version + out.DefaultStorageClassName = in.DefaultStorageClassName return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 3992cebbf1aa1..ec15596d494cc 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -5578,6 +5578,16 @@ func (in *PDCSIDriver) DeepCopyInto(out *PDCSIDriver) { *out = new(bool) **out = **in } + if in.Version != nil { + in, out := &in.Version, &out.Version + *out = new(string) + **out = **in + } + if in.DefaultStorageClassName != nil { + in, out := &in.DefaultStorageClassName, &out.DefaultStorageClassName + *out = new(string) + **out = **in + } return } diff --git a/pkg/apis/kops/v1alpha3/componentconfig.go b/pkg/apis/kops/v1alpha3/componentconfig.go index f9c9f94219859..1ed120337cbf2 100644 --- a/pkg/apis/kops/v1alpha3/componentconfig.go +++ b/pkg/apis/kops/v1alpha3/componentconfig.go @@ -980,6 +980,11 @@ type EBSCSIDriverSpec struct { type PDCSIDriver struct { // Enabled enables the GCP PD CSI driver Enabled *bool `json:"enabled,omitempty"` + // Version is the container image tag used. + // Default: The latest stable release which is compatible with your Kubernetes version + Version *string `json:"version,omitempty"` + // Default StorageClassName is the name of the default StorageClass created for GCP PD CSI driver. + DefaultStorageClassName *string `json:"defaultStorageClassName,omitempty"` } // SnapshotControllerConfig is the config for the CSI Snapshot Controller diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go index db9d5850b906a..2468c95064abc 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go @@ -7547,6 +7547,8 @@ func Convert_kops_OpenstackSpec_To_v1alpha3_OpenstackSpec(in *kops.OpenstackSpec func autoConvert_v1alpha3_PDCSIDriver_To_kops_PDCSIDriver(in *PDCSIDriver, out *kops.PDCSIDriver, s conversion.Scope) error { out.Enabled = in.Enabled + out.Version = in.Version + out.DefaultStorageClassName = in.DefaultStorageClassName return nil } @@ -7557,6 +7559,8 @@ func Convert_v1alpha3_PDCSIDriver_To_kops_PDCSIDriver(in *PDCSIDriver, out *kops func autoConvert_kops_PDCSIDriver_To_v1alpha3_PDCSIDriver(in *kops.PDCSIDriver, out *PDCSIDriver, s conversion.Scope) error { out.Enabled = in.Enabled + out.Version = in.Version + out.DefaultStorageClassName = in.DefaultStorageClassName return nil } diff --git a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go index 4f8c834062ee0..49a64a3e18dc3 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go @@ -5530,6 +5530,16 @@ func (in *PDCSIDriver) DeepCopyInto(out *PDCSIDriver) { *out = new(bool) **out = **in } + if in.Version != nil { + in, out := &in.Version, &out.Version + *out = new(string) + **out = **in + } + if in.DefaultStorageClassName != nil { + in, out := &in.DefaultStorageClassName, &out.DefaultStorageClassName + *out = new(string) + **out = **in + } return } diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index f23437c4689de..f7f60122f7564 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -5805,6 +5805,16 @@ func (in *PDCSIDriver) DeepCopyInto(out *PDCSIDriver) { *out = new(bool) **out = **in } + if in.Version != nil { + in, out := &in.Version, &out.Version + *out = new(string) + **out = **in + } + if in.DefaultStorageClassName != nil { + in, out := &in.DefaultStorageClassName, &out.DefaultStorageClassName + *out = new(string) + **out = **in + } return } diff --git a/pkg/model/components/gcppdcsidriver.go b/pkg/model/components/gcppdcsidriver.go index a055a39daa5a3..6ea8fe4a3fffa 100644 --- a/pkg/model/components/gcppdcsidriver.go +++ b/pkg/model/components/gcppdcsidriver.go @@ -37,8 +37,15 @@ func (b *GCPPDCSIDriverOptionsBuilder) BuildOptions(o *kops.Cluster) error { if gce.PDCSIDriver == nil { gce.PDCSIDriver = &kops.PDCSIDriver{ - Enabled: fi.PtrTo(true), + Enabled: fi.PtrTo(true), + DefaultStorageClassName: fi.PtrTo("balanced-csi"), } + } else if gce.PDCSIDriver.DefaultStorageClassName != nil && gce.PDCSIDriver.Enabled == nil { + gce.PDCSIDriver.Enabled = fi.PtrTo(true) + } + + if gce.PDCSIDriver.Version == nil { + gce.PDCSIDriver.Version = fi.PtrTo("v1.22.1") } return nil diff --git a/tests/integration/update_cluster/ha_gce/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/ha_gce/data/aws_s3_object_cluster-completed.spec_content index 9ae6323c68d97..b7670a981892d 100644 --- a/tests/integration/update_cluster/ha_gce/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/ha_gce/data/aws_s3_object_cluster-completed.spec_content @@ -12,7 +12,9 @@ spec: cloudConfig: gceServiceAccount: default gcpPDCSIDriver: + defaultStorageClassName: balanced-csi enabled: true + version: v1.22.1 manageStorageClasses: true multizone: true nodeTags: ha-gce-example-com-k8s-io-role-node diff --git a/tests/integration/update_cluster/ha_gce/data/aws_s3_object_ha-gce.example.com-addons-bootstrap_content b/tests/integration/update_cluster/ha_gce/data/aws_s3_object_ha-gce.example.com-addons-bootstrap_content index 170e9098b5e54..8197d4aaa7c78 100644 --- a/tests/integration/update_cluster/ha_gce/data/aws_s3_object_ha-gce.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/ha_gce/data/aws_s3_object_ha-gce.example.com-addons-bootstrap_content @@ -47,7 +47,7 @@ spec: version: 9.99.0 - id: k8s-1.23 manifest: gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml - manifestHash: 7e2de94f2da0fcc49117a4bba5e90718c1ebd802b4e59a3ae7160f0e141f971f + manifestHash: b0d1b75a405846ae0716a86d9d38eb5da6bc2b708ef8fead88bf8e55fdac5da3 name: gcp-pd-csi-driver.addons.k8s.io selector: k8s-addon: gcp-pd-csi-driver.addons.k8s.io diff --git a/tests/integration/update_cluster/ha_gce/data/aws_s3_object_ha-gce.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/ha_gce/data/aws_s3_object_ha-gce.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content index 684efe994bcc6..0aca79bc38dc4 100644 --- a/tests/integration/update_cluster/ha_gce/data/aws_s3_object_ha-gce.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content +++ b/tests/integration/update_cluster/ha_gce/data/aws_s3_object_ha-gce.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content @@ -37,6 +37,26 @@ volumeBindingMode: WaitForFirstConsumer --- +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: balanced-storage +parameters: + provisioned-iops-on-create: "5000" + provisioned-throughput-on-create: 250Mi + type: hyperdisk-balanced +provisioner: pd.csi.storage.gke.io +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer + +--- + allowVolumeExpansion: true apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -583,16 +603,6 @@ spec: values: - linux containers: - - args: - - --v=5 - - --endpoint=unix:/csi/csi.sock - - --extra-labels=k8s-io-cluster-name=ha-gce-example-com - env: [] - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.1 - name: gce-pd-driver - volumeMounts: - - mountPath: /csi - name: socket-dir - args: - --v=5 - --csi-address=/csi/csi.sock @@ -609,7 +619,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v6.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -638,7 +648,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-attacher:v4.2.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.8.1 livenessProbe: failureThreshold: 1 httpGet: @@ -667,7 +677,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0 + image: registry.k8s.io/sig-storage/csi-resizer:v2.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -691,18 +701,34 @@ spec: - --leader-election - --leader-election-namespace=$(PDCSI_NAMESPACE) - --timeout=300s + - --retry-interval-max=60s env: - name: PDCSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 + image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.1 name: csi-snapshotter volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=5 + - --endpoint=unix:/csi/csi.sock + - --extra-labels=k8s-io-cluster-name=ha-gce-example-com + - --supports-dynamic-iops-provisioning=hyperdisk-balanced,hyperdisk-extreme + - --supports-dynamic-throughput-provisioning=hyperdisk-balanced,hyperdisk-throughput,hyperdisk-ml + - --enable-data-cache + - --run-node-service=false + env: [] + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 + name: gce-pd-driver + volumeMounts: + - mountPath: /csi + name: socket-dir hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-controller serviceAccountName: csi-gce-pd-controller-sa tolerations: @@ -754,7 +780,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0 name: csi-driver-registrar volumeMounts: - mountPath: /csi @@ -765,7 +791,8 @@ spec: - --v=5 - --endpoint=unix:/csi/csi.sock - --run-controller-service=false - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.13 + - --enable-data-cache + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 name: gce-pd-driver securityContext: privileged: true @@ -786,7 +813,8 @@ spec: - mountPath: /sys name: sys hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-node serviceAccountName: csi-gce-pd-node-sa tolerations: diff --git a/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_cluster-completed.spec_content index 17f02368bd828..9d787c278095a 100644 --- a/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_cluster-completed.spec_content @@ -13,7 +13,9 @@ spec: channel: stable cloudConfig: gcpPDCSIDriver: + defaultStorageClassName: balanced-csi enabled: true + version: v1.22.1 manageStorageClasses: true multizone: true nodeTags: minimal-example-com-k8s-io-role-node diff --git a/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index 09f73622dbfc1..497e61a88fb7d 100644 --- a/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -118,7 +118,7 @@ spec: version: 9.99.0 - id: k8s-1.23 manifest: gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml - manifestHash: b565f0905b27e405ac3855bed4231f766e0068833c62ad7abcb408612ca19943 + manifestHash: 6feff18c6ec4651da2eccc5a8ed76408cf1fcd78d76dc2f3ffdce3d8c983df09 name: gcp-pd-csi-driver.addons.k8s.io selector: k8s-addon: gcp-pd-csi-driver.addons.k8s.io diff --git a/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content index 513d77088b7a9..6a30d1b10b2d9 100644 --- a/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content +++ b/tests/integration/update_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content @@ -37,6 +37,26 @@ volumeBindingMode: WaitForFirstConsumer --- +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: balanced-storage +parameters: + provisioned-iops-on-create: "5000" + provisioned-throughput-on-create: 250Mi + type: hyperdisk-balanced +provisioner: pd.csi.storage.gke.io +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer + +--- + allowVolumeExpansion: true apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -583,16 +603,6 @@ spec: values: - linux containers: - - args: - - --v=5 - - --endpoint=unix:/csi/csi.sock - - --extra-labels=k8s-io-cluster-name=minimal-example-com - env: [] - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.1 - name: gce-pd-driver - volumeMounts: - - mountPath: /csi - name: socket-dir - args: - --v=5 - --csi-address=/csi/csi.sock @@ -609,7 +619,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v6.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -638,7 +648,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-attacher:v4.2.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.8.1 livenessProbe: failureThreshold: 1 httpGet: @@ -667,7 +677,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0 + image: registry.k8s.io/sig-storage/csi-resizer:v2.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -691,18 +701,34 @@ spec: - --leader-election - --leader-election-namespace=$(PDCSI_NAMESPACE) - --timeout=300s + - --retry-interval-max=60s env: - name: PDCSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 + image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.1 name: csi-snapshotter volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=5 + - --endpoint=unix:/csi/csi.sock + - --extra-labels=k8s-io-cluster-name=minimal-example-com + - --supports-dynamic-iops-provisioning=hyperdisk-balanced,hyperdisk-extreme + - --supports-dynamic-throughput-provisioning=hyperdisk-balanced,hyperdisk-throughput,hyperdisk-ml + - --enable-data-cache + - --run-node-service=false + env: [] + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 + name: gce-pd-driver + volumeMounts: + - mountPath: /csi + name: socket-dir hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-controller serviceAccountName: csi-gce-pd-controller-sa tolerations: @@ -754,7 +780,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0 name: csi-driver-registrar volumeMounts: - mountPath: /csi @@ -765,7 +791,8 @@ spec: - --v=5 - --endpoint=unix:/csi/csi.sock - --run-controller-service=false - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.13 + - --enable-data-cache + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 name: gce-pd-driver securityContext: privileged: true @@ -786,7 +813,8 @@ spec: - mountPath: /sys name: sys hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-node serviceAccountName: csi-gce-pd-node-sa tolerations: diff --git a/tests/integration/update_cluster/minimal_gce/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/minimal_gce/data/aws_s3_object_cluster-completed.spec_content index 53b62cca7ad08..3485596d024c2 100644 --- a/tests/integration/update_cluster/minimal_gce/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/minimal_gce/data/aws_s3_object_cluster-completed.spec_content @@ -12,7 +12,8 @@ spec: cloudConfig: gceServiceAccount: default gcpPDCSIDriver: - enabled: true + enabled: false + version: v1.22.1 manageStorageClasses: true multizone: true nodeTags: minimal-gce-example-com-k8s-io-role-node diff --git a/tests/integration/update_cluster/minimal_gce/data/aws_s3_object_minimal-gce.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal_gce/data/aws_s3_object_minimal-gce.example.com-addons-bootstrap_content index d0ecc0613e491..49ad2d6a878a0 100644 --- a/tests/integration/update_cluster/minimal_gce/data/aws_s3_object_minimal-gce.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal_gce/data/aws_s3_object_minimal-gce.example.com-addons-bootstrap_content @@ -40,18 +40,11 @@ spec: version: 9.99.0 - id: v1.7.0 manifest: storage-gce.addons.k8s.io/v1.7.0.yaml - manifestHash: 4f73c7d683f04e61d60e90053a6db1fd82272dc0c13790390740f5e207181c77 + manifestHash: 6e9ece81d6a7890599d0ce7e56c4f415863a1b64b69849a119c96d1c47670620 name: storage-gce.addons.k8s.io selector: k8s-addon: storage-gce.addons.k8s.io version: 9.99.0 - - id: k8s-1.23 - manifest: gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml - manifestHash: 4c036acd18f0d7ee26eaeec7730fe5a80bc8756f29917550b569b65a22594afa - name: gcp-pd-csi-driver.addons.k8s.io - selector: - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - version: 9.99.0 - id: k8s-1.23 manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml manifestHash: dd48ce99e79111f6fa61e057cc3740b802c63e5fee4af1cd55b39fe1651298c6 diff --git a/tests/integration/update_cluster/minimal_gce/data/aws_s3_object_minimal-gce.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/minimal_gce/data/aws_s3_object_minimal-gce.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content deleted file mode 100644 index 67fcb7c5c3624..0000000000000 --- a/tests/integration/update_cluster/minimal_gce/data/aws_s3_object_minimal-gce.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content +++ /dev/null @@ -1,840 +0,0 @@ -allowVolumeExpansion: true -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - annotations: - storageclass.kubernetes.io/is-default-class: "false" - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - kubernetes.io/cluster-service: "true" - name: standard-csi -parameters: - type: pd-standard -provisioner: pd.csi.storage.gke.io -volumeBindingMode: WaitForFirstConsumer - ---- - -allowVolumeExpansion: true -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - annotations: - storageclass.kubernetes.io/is-default-class: "true" - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - kubernetes.io/cluster-service: "true" - name: balanced-csi -parameters: - type: pd-balanced -provisioner: pd.csi.storage.gke.io -reclaimPolicy: Delete -volumeBindingMode: WaitForFirstConsumer - ---- - -allowVolumeExpansion: true -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - kubernetes.io/cluster-service: "true" - name: ssd-csi -parameters: - type: pd-ssd -provisioner: pd.csi.storage.gke.io -reclaimPolicy: Delete -volumeBindingMode: WaitForFirstConsumer - ---- - -apiVersion: v1 -kind: Namespace -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: gce-pd-csi-driver - ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver - ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-node-sa - namespace: gce-pd-csi-driver - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - k8s-app: gcp-compute-persistent-disk-csi-driver - name: csi-gce-pd-leaderelection-role - namespace: gce-pd-csi-driver -rules: -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - watch - - list - - delete - - update - - create - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-attacher-role -rules: -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - update - - patch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - storage.k8s.io - resources: - - csinodes - verbs: - - get - - list - - watch -- apiGroups: - - storage.k8s.io - resources: - - volumeattachments - verbs: - - get - - list - - watch - - update - - patch -- apiGroups: - - storage.k8s.io - resources: - - volumeattachments/status - verbs: - - patch - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-provisioner-role -rules: -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - create - - delete -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - get - - list - - watch - - update -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - list - - watch - - create - - update - - patch -- apiGroups: - - storage.k8s.io - resources: - - csinodes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshots - verbs: - - get - - list -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshotcontents - verbs: - - get - - list -- apiGroups: - - storage.k8s.io - resources: - - volumeattachments - verbs: - - get - - list - - watch - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-resizer-role -rules: -- apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - update - - patch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - persistentvolumeclaims/status - verbs: - - update - - patch -- apiGroups: - - "" - resources: - - events - verbs: - - list - - watch - - create - - update - - patch -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-snapshotter-role -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - list - - watch - - create - - update - - patch -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshotclasses - verbs: - - get - - list - - watch -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshotcontents - verbs: - - create - - get - - list - - watch - - update - - delete - - patch -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshotcontents/status - verbs: - - update - - patch - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - k8s-app: gcp-compute-persistent-disk-csi-driver - name: csi-gce-pd-controller-leaderelection-binding - namespace: gce-pd-csi-driver -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: csi-gce-pd-leaderelection-role -subjects: -- kind: ServiceAccount - name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-controller -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: csi-gce-pd-node-deploy -subjects: -- kind: ServiceAccount - name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-controller-attacher-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: csi-gce-pd-attacher-role -subjects: -- kind: ServiceAccount - name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-controller-deploy -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: csi-gce-pd-controller-deploy -subjects: -- kind: ServiceAccount - name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-controller-provisioner-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: csi-gce-pd-provisioner-role -subjects: -- kind: ServiceAccount - name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-controller-snapshotter-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: csi-gce-pd-snapshotter-role -subjects: -- kind: ServiceAccount - name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: csi-gce-pd-node-deploy -subjects: -- kind: ServiceAccount - name: csi-gce-pd-node-sa - namespace: gce-pd-csi-driver - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-resizer-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: csi-gce-pd-resizer-role -subjects: -- kind: ServiceAccount - name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver - ---- - -apiVersion: scheduling.k8s.io/v1 -description: This priority class should be used for the GCE PD CSI driver controller - deployment only. -globalDefault: false -kind: PriorityClass -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-controller -value: 900000000 - ---- - -apiVersion: scheduling.k8s.io/v1 -description: This priority class should be used for the GCE PD CSI driver node deployment - only. -globalDefault: false -kind: PriorityClass -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-node -value: 900001000 - ---- - -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-controller - namespace: gce-pd-csi-driver -spec: - replicas: 1 - selector: - matchLabels: - app: gcp-compute-persistent-disk-csi-driver - template: - metadata: - labels: - app: gcp-compute-persistent-disk-csi-driver - kops.k8s.io/managed-by: kops - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - - key: kubernetes.io/os - operator: In - values: - - linux - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - key: kubernetes.io/os - operator: In - values: - - linux - containers: - - args: - - --v=5 - - --endpoint=unix:/csi/csi.sock - - --extra-labels=k8s-io-cluster-name=minimal-gce-example-com - env: [] - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.1 - name: gce-pd-driver - volumeMounts: - - mountPath: /csi - name: socket-dir - - args: - - --v=5 - - --csi-address=/csi/csi.sock - - --feature-gates=Topology=true - - --http-endpoint=:22011 - - --leader-election-namespace=$(PDCSI_NAMESPACE) - - --timeout=250s - - --extra-create-metadata - - --leader-election - - --default-fstype=ext4 - - --controller-publish-readonly=true - env: - - name: PDCSI_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.0 - livenessProbe: - failureThreshold: 1 - httpGet: - path: /healthz/leader-election - port: http-endpoint - initialDelaySeconds: 10 - periodSeconds: 20 - timeoutSeconds: 10 - name: csi-provisioner - ports: - - containerPort: 22011 - name: http-endpoint - protocol: TCP - volumeMounts: - - mountPath: /csi - name: socket-dir - - args: - - --v=5 - - --csi-address=/csi/csi.sock - - --http-endpoint=:22012 - - --leader-election - - --leader-election-namespace=$(PDCSI_NAMESPACE) - - --timeout=250s - env: - - name: PDCSI_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-attacher:v4.2.0 - livenessProbe: - failureThreshold: 1 - httpGet: - path: /healthz/leader-election - port: http-endpoint - initialDelaySeconds: 10 - periodSeconds: 20 - timeoutSeconds: 10 - name: csi-attacher - ports: - - containerPort: 22012 - name: http-endpoint - protocol: TCP - volumeMounts: - - mountPath: /csi - name: socket-dir - - args: - - --v=5 - - --csi-address=/csi/csi.sock - - --http-endpoint=:22013 - - --leader-election - - --leader-election-namespace=$(PDCSI_NAMESPACE) - - --handle-volume-inuse-error=false - env: - - name: PDCSI_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0 - livenessProbe: - failureThreshold: 1 - httpGet: - path: /healthz/leader-election - port: http-endpoint - initialDelaySeconds: 10 - periodSeconds: 20 - timeoutSeconds: 10 - name: csi-resizer - ports: - - containerPort: 22013 - name: http-endpoint - protocol: TCP - volumeMounts: - - mountPath: /csi - name: socket-dir - - args: - - --v=5 - - --csi-address=/csi/csi.sock - - --metrics-address=:22014 - - --leader-election - - --leader-election-namespace=$(PDCSI_NAMESPACE) - - --timeout=300s - env: - - name: PDCSI_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 - name: csi-snapshotter - volumeMounts: - - mountPath: /csi - name: socket-dir - hostNetwork: true - nodeSelector: null - priorityClassName: csi-gce-pd-controller - serviceAccountName: csi-gce-pd-controller-sa - tolerations: - - effect: NoSchedule - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - volumes: - - emptyDir: {} - name: socket-dir - ---- - -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: csi-gce-pd-node - namespace: gce-pd-csi-driver -spec: - selector: - matchLabels: - app: gcp-compute-persistent-disk-csi-driver - template: - metadata: - labels: - app: gcp-compute-persistent-disk-csi-driver - kops.k8s.io/managed-by: kops - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - linux - containers: - - args: - - --v=5 - - --csi-address=/csi/csi.sock - - --kubelet-registration-path=/var/lib/kubelet/plugins/pd.csi.storage.gke.io/csi.sock - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0 - name: csi-driver-registrar - volumeMounts: - - mountPath: /csi - name: plugin-dir - - mountPath: /registration - name: registration-dir - - args: - - --v=5 - - --endpoint=unix:/csi/csi.sock - - --run-controller-service=false - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.13 - name: gce-pd-driver - securityContext: - privileged: true - volumeMounts: - - mountPath: /var/lib/kubelet - mountPropagation: Bidirectional - name: kubelet-dir - - mountPath: /csi - name: plugin-dir - - mountPath: /dev - name: device-dir - - mountPath: /etc/udev - name: udev-rules-etc - - mountPath: /lib/udev - name: udev-rules-lib - - mountPath: /run/udev - name: udev-socket - - mountPath: /sys - name: sys - hostNetwork: true - nodeSelector: null - priorityClassName: csi-gce-pd-node - serviceAccountName: csi-gce-pd-node-sa - tolerations: - - operator: Exists - volumes: - - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - name: registration-dir - - hostPath: - path: /var/lib/kubelet - type: Directory - name: kubelet-dir - - hostPath: - path: /var/lib/kubelet/plugins/pd.csi.storage.gke.io/ - type: DirectoryOrCreate - name: plugin-dir - - hostPath: - path: /dev - type: Directory - name: device-dir - - hostPath: - path: /etc/udev - type: Directory - name: udev-rules-etc - - hostPath: - path: /lib/udev - type: Directory - name: udev-rules-lib - - hostPath: - path: /run/udev - type: Directory - name: udev-socket - - hostPath: - path: /sys - type: Directory - name: sys - ---- - -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - labels: - addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io - app.kubernetes.io/managed-by: kops - k8s-addon: gcp-pd-csi-driver.addons.k8s.io - name: pd.csi.storage.gke.io -spec: - attachRequired: true - podInfoOnMount: false diff --git a/tests/integration/update_cluster/minimal_gce/data/aws_s3_object_minimal-gce.example.com-addons-storage-gce.addons.k8s.io-v1.7.0_content b/tests/integration/update_cluster/minimal_gce/data/aws_s3_object_minimal-gce.example.com-addons-storage-gce.addons.k8s.io-v1.7.0_content index b5b781d7554ae..e109954baeefa 100644 --- a/tests/integration/update_cluster/minimal_gce/data/aws_s3_object_minimal-gce.example.com-addons-storage-gce.addons.k8s.io-v1.7.0_content +++ b/tests/integration/update_cluster/minimal_gce/data/aws_s3_object_minimal-gce.example.com-addons-storage-gce.addons.k8s.io-v1.7.0_content @@ -1,6 +1,8 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "true" labels: addon.kops.k8s.io/name: storage-gce.addons.k8s.io addonmanager.kubernetes.io/mode: EnsureExists diff --git a/tests/integration/update_cluster/minimal_gce/in-v1alpha2.yaml b/tests/integration/update_cluster/minimal_gce/in-v1alpha2.yaml index 9a49973323a2d..cdd1b578726e0 100644 --- a/tests/integration/update_cluster/minimal_gce/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/minimal_gce/in-v1alpha2.yaml @@ -12,27 +12,29 @@ spec: cloudProvider: gce configBase: memfs://tests/minimal-gce.example.com etcdClusters: - - cpuRequest: 200m - etcdMembers: - - instanceGroup: master-us-test1-a - name: a - memoryRequest: 100Mi - name: main - - cpuRequest: 100m - etcdMembers: - - instanceGroup: master-us-test1-a - name: a - memoryRequest: 100Mi - name: events + - cpuRequest: 200m + etcdMembers: + - instanceGroup: master-us-test1-a + name: a + memoryRequest: 100Mi + name: main + - cpuRequest: 100m + etcdMembers: + - instanceGroup: master-us-test1-a + name: a + memoryRequest: 100Mi + name: events cloudConfig: gceServiceAccount: default + gcpPDCSIDriver: + enabled: false iam: legacy: false kubelet: anonymousAuth: false kubernetesApiAccess: - - 0.0.0.0/0 - - ::/0 + - 0.0.0.0/0 + - ::/0 kubernetesVersion: v1.32.0 masterPublicName: api.minimal-gce.example.com networking: @@ -40,18 +42,17 @@ spec: nonMasqueradeCIDR: 100.64.0.0/10 project: testproject sshAccess: - - 0.0.0.0/0 - - ::/0 + - 0.0.0.0/0 + - ::/0 subnets: - - name: us-test1 - region: us-test1 - type: Public + - name: us-test1 + region: us-test1 + type: Public topology: dns: type: Public --- - apiVersion: kops.k8s.io/v1alpha2 kind: InstanceGroup metadata: @@ -66,12 +67,11 @@ spec: minSize: 1 role: Master subnets: - - us-test1 + - us-test1 zones: - - us-test1-a + - us-test1-a --- - apiVersion: kops.k8s.io/v1alpha2 kind: InstanceGroup metadata: @@ -86,6 +86,6 @@ spec: minSize: 2 role: Node subnets: - - us-test1 + - us-test1 zones: - - us-test1-a + - us-test1-a diff --git a/tests/integration/update_cluster/minimal_gce/kubernetes.tf b/tests/integration/update_cluster/minimal_gce/kubernetes.tf index 35ea8951986dd..3cc206ee153eb 100644 --- a/tests/integration/update_cluster/minimal_gce/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gce/kubernetes.tf @@ -114,14 +114,6 @@ resource "aws_s3_object" "minimal-gce-example-com-addons-gcp-cloud-controller-ad server_side_encryption = "AES256" } -resource "aws_s3_object" "minimal-gce-example-com-addons-gcp-pd-csi-driver-addons-k8s-io-k8s-1-23" { - bucket = "testingBucket" - content = file("${path.module}/data/aws_s3_object_minimal-gce.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content") - key = "tests/minimal-gce.example.com/addons/gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml" - provider = aws.files - server_side_encryption = "AES256" -} - resource "aws_s3_object" "minimal-gce-example-com-addons-kops-controller-addons-k8s-io-k8s-1-16" { bucket = "testingBucket" content = file("${path.module}/data/aws_s3_object_minimal-gce.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content") diff --git a/tests/integration/update_cluster/minimal_gce_dns-none/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/minimal_gce_dns-none/data/aws_s3_object_cluster-completed.spec_content index c8b428f6df99f..6c06404c8af5c 100644 --- a/tests/integration/update_cluster/minimal_gce_dns-none/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/minimal_gce_dns-none/data/aws_s3_object_cluster-completed.spec_content @@ -15,7 +15,9 @@ spec: channel: stable cloudConfig: gcpPDCSIDriver: + defaultStorageClassName: balanced-csi enabled: true + version: v1.22.1 manageStorageClasses: true multizone: true nodeTags: minimal-gce-example-com-k8s-io-role-node diff --git a/tests/integration/update_cluster/minimal_gce_dns-none/data/aws_s3_object_minimal-gce.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal_gce_dns-none/data/aws_s3_object_minimal-gce.example.com-addons-bootstrap_content index 65810132c6d4d..b9164a46934a1 100644 --- a/tests/integration/update_cluster/minimal_gce_dns-none/data/aws_s3_object_minimal-gce.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal_gce_dns-none/data/aws_s3_object_minimal-gce.example.com-addons-bootstrap_content @@ -40,7 +40,7 @@ spec: version: 9.99.0 - id: k8s-1.23 manifest: gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml - manifestHash: 4c036acd18f0d7ee26eaeec7730fe5a80bc8756f29917550b569b65a22594afa + manifestHash: 56cdfca56f471b0f045f0769e786ecda8ca56b1ab3cc5b2d9f5ccf6cff7dcd0d name: gcp-pd-csi-driver.addons.k8s.io selector: k8s-addon: gcp-pd-csi-driver.addons.k8s.io diff --git a/tests/integration/update_cluster/minimal_gce_dns-none/data/aws_s3_object_minimal-gce.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/minimal_gce_dns-none/data/aws_s3_object_minimal-gce.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content index 67fcb7c5c3624..183292e4f0a54 100644 --- a/tests/integration/update_cluster/minimal_gce_dns-none/data/aws_s3_object_minimal-gce.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content +++ b/tests/integration/update_cluster/minimal_gce_dns-none/data/aws_s3_object_minimal-gce.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content @@ -37,6 +37,26 @@ volumeBindingMode: WaitForFirstConsumer --- +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: balanced-storage +parameters: + provisioned-iops-on-create: "5000" + provisioned-throughput-on-create: 250Mi + type: hyperdisk-balanced +provisioner: pd.csi.storage.gke.io +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer + +--- + allowVolumeExpansion: true apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -583,16 +603,6 @@ spec: values: - linux containers: - - args: - - --v=5 - - --endpoint=unix:/csi/csi.sock - - --extra-labels=k8s-io-cluster-name=minimal-gce-example-com - env: [] - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.1 - name: gce-pd-driver - volumeMounts: - - mountPath: /csi - name: socket-dir - args: - --v=5 - --csi-address=/csi/csi.sock @@ -609,7 +619,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v6.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -638,7 +648,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-attacher:v4.2.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.8.1 livenessProbe: failureThreshold: 1 httpGet: @@ -667,7 +677,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0 + image: registry.k8s.io/sig-storage/csi-resizer:v2.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -691,18 +701,34 @@ spec: - --leader-election - --leader-election-namespace=$(PDCSI_NAMESPACE) - --timeout=300s + - --retry-interval-max=60s env: - name: PDCSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 + image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.1 name: csi-snapshotter volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=5 + - --endpoint=unix:/csi/csi.sock + - --extra-labels=k8s-io-cluster-name=minimal-gce-example-com + - --supports-dynamic-iops-provisioning=hyperdisk-balanced,hyperdisk-extreme + - --supports-dynamic-throughput-provisioning=hyperdisk-balanced,hyperdisk-throughput,hyperdisk-ml + - --enable-data-cache + - --run-node-service=false + env: [] + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 + name: gce-pd-driver + volumeMounts: + - mountPath: /csi + name: socket-dir hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-controller serviceAccountName: csi-gce-pd-controller-sa tolerations: @@ -754,7 +780,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0 name: csi-driver-registrar volumeMounts: - mountPath: /csi @@ -765,7 +791,8 @@ spec: - --v=5 - --endpoint=unix:/csi/csi.sock - --run-controller-service=false - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.13 + - --enable-data-cache + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 name: gce-pd-driver securityContext: privileged: true @@ -786,7 +813,8 @@ spec: - mountPath: /sys name: sys hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-node serviceAccountName: csi-gce-pd-node-sa tolerations: diff --git a/tests/integration/update_cluster/minimal_gce_ilb/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/minimal_gce_ilb/data/aws_s3_object_cluster-completed.spec_content index 9d69d087c34c4..bd1cf4565db4a 100644 --- a/tests/integration/update_cluster/minimal_gce_ilb/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/minimal_gce_ilb/data/aws_s3_object_cluster-completed.spec_content @@ -16,7 +16,9 @@ spec: cloudConfig: gceServiceAccount: default gcpPDCSIDriver: + defaultStorageClassName: balanced-csi enabled: true + version: v1.22.1 manageStorageClasses: true multizone: true nodeTags: minimal-gce-ilb-example-com-k8s-io-role-node diff --git a/tests/integration/update_cluster/minimal_gce_ilb/data/aws_s3_object_minimal-gce-ilb.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal_gce_ilb/data/aws_s3_object_minimal-gce-ilb.example.com-addons-bootstrap_content index 73791545f4f30..fe22f3c6b267d 100644 --- a/tests/integration/update_cluster/minimal_gce_ilb/data/aws_s3_object_minimal-gce-ilb.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal_gce_ilb/data/aws_s3_object_minimal-gce-ilb.example.com-addons-bootstrap_content @@ -47,7 +47,7 @@ spec: version: 9.99.0 - id: k8s-1.23 manifest: gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml - manifestHash: bf17f20a2a21bfb570ec14b24c2beb43cce108784cd94ed1f96789a53a79098e + manifestHash: 9180b3f2f2361f3c0897a45c78adf3a82be6c4319be1418acaf51a9b4e0077f4 name: gcp-pd-csi-driver.addons.k8s.io selector: k8s-addon: gcp-pd-csi-driver.addons.k8s.io diff --git a/tests/integration/update_cluster/minimal_gce_ilb/data/aws_s3_object_minimal-gce-ilb.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/minimal_gce_ilb/data/aws_s3_object_minimal-gce-ilb.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content index 50e39b4c28c26..f4e61ec9e0514 100644 --- a/tests/integration/update_cluster/minimal_gce_ilb/data/aws_s3_object_minimal-gce-ilb.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content +++ b/tests/integration/update_cluster/minimal_gce_ilb/data/aws_s3_object_minimal-gce-ilb.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content @@ -37,6 +37,26 @@ volumeBindingMode: WaitForFirstConsumer --- +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: balanced-storage +parameters: + provisioned-iops-on-create: "5000" + provisioned-throughput-on-create: 250Mi + type: hyperdisk-balanced +provisioner: pd.csi.storage.gke.io +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer + +--- + allowVolumeExpansion: true apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -583,16 +603,6 @@ spec: values: - linux containers: - - args: - - --v=5 - - --endpoint=unix:/csi/csi.sock - - --extra-labels=k8s-io-cluster-name=minimal-gce-ilb-example-com - env: [] - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.1 - name: gce-pd-driver - volumeMounts: - - mountPath: /csi - name: socket-dir - args: - --v=5 - --csi-address=/csi/csi.sock @@ -609,7 +619,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v6.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -638,7 +648,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-attacher:v4.2.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.8.1 livenessProbe: failureThreshold: 1 httpGet: @@ -667,7 +677,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0 + image: registry.k8s.io/sig-storage/csi-resizer:v2.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -691,18 +701,34 @@ spec: - --leader-election - --leader-election-namespace=$(PDCSI_NAMESPACE) - --timeout=300s + - --retry-interval-max=60s env: - name: PDCSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 + image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.1 name: csi-snapshotter volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=5 + - --endpoint=unix:/csi/csi.sock + - --extra-labels=k8s-io-cluster-name=minimal-gce-ilb-example-com + - --supports-dynamic-iops-provisioning=hyperdisk-balanced,hyperdisk-extreme + - --supports-dynamic-throughput-provisioning=hyperdisk-balanced,hyperdisk-throughput,hyperdisk-ml + - --enable-data-cache + - --run-node-service=false + env: [] + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 + name: gce-pd-driver + volumeMounts: + - mountPath: /csi + name: socket-dir hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-controller serviceAccountName: csi-gce-pd-controller-sa tolerations: @@ -754,7 +780,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0 name: csi-driver-registrar volumeMounts: - mountPath: /csi @@ -765,7 +791,8 @@ spec: - --v=5 - --endpoint=unix:/csi/csi.sock - --run-controller-service=false - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.13 + - --enable-data-cache + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 name: gce-pd-driver securityContext: privileged: true @@ -786,7 +813,8 @@ spec: - mountPath: /sys name: sys hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-node serviceAccountName: csi-gce-pd-node-sa tolerations: diff --git a/tests/integration/update_cluster/minimal_gce_ilb_longclustername/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/minimal_gce_ilb_longclustername/data/aws_s3_object_cluster-completed.spec_content index a3f4fae52ba2d..530a601adb33b 100644 --- a/tests/integration/update_cluster/minimal_gce_ilb_longclustername/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/minimal_gce_ilb_longclustername/data/aws_s3_object_cluster-completed.spec_content @@ -16,7 +16,9 @@ spec: cloudConfig: gceServiceAccount: default gcpPDCSIDriver: + defaultStorageClassName: balanced-csi enabled: true + version: v1.22.1 manageStorageClasses: true multizone: true nodeTags: minimal-gce-with-a-very-very-very-very--96dqvi-k8s-io-role-node diff --git a/tests/integration/update_cluster/minimal_gce_ilb_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal_gce_ilb_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-bootstrap_content index e73d67c5dfc9e..efe56ea3e32a6 100644 --- a/tests/integration/update_cluster/minimal_gce_ilb_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal_gce_ilb_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-bootstrap_content @@ -47,7 +47,7 @@ spec: version: 9.99.0 - id: k8s-1.23 manifest: gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml - manifestHash: e1f5166c3ef5d62b244f1d74e22c396956ac5cf85de4828617a729313a9c49b7 + manifestHash: 234c31cbc7c906d1a2df8067ccbe85172ec4659da1d1a9ee2833eb2723cbced0 name: gcp-pd-csi-driver.addons.k8s.io selector: k8s-addon: gcp-pd-csi-driver.addons.k8s.io diff --git a/tests/integration/update_cluster/minimal_gce_ilb_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/minimal_gce_ilb_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content index d12604a82fefb..195db1b6ab5fb 100644 --- a/tests/integration/update_cluster/minimal_gce_ilb_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content +++ b/tests/integration/update_cluster/minimal_gce_ilb_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content @@ -37,6 +37,26 @@ volumeBindingMode: WaitForFirstConsumer --- +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: balanced-storage +parameters: + provisioned-iops-on-create: "5000" + provisioned-throughput-on-create: 250Mi + type: hyperdisk-balanced +provisioner: pd.csi.storage.gke.io +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer + +--- + allowVolumeExpansion: true apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -583,16 +603,6 @@ spec: values: - linux containers: - - args: - - --v=5 - - --endpoint=unix:/csi/csi.sock - - --extra-labels=k8s-io-cluster-name=minimal-gce-with-a-very-very-very-very-very-long-name-example-com - env: [] - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.1 - name: gce-pd-driver - volumeMounts: - - mountPath: /csi - name: socket-dir - args: - --v=5 - --csi-address=/csi/csi.sock @@ -609,7 +619,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v6.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -638,7 +648,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-attacher:v4.2.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.8.1 livenessProbe: failureThreshold: 1 httpGet: @@ -667,7 +677,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0 + image: registry.k8s.io/sig-storage/csi-resizer:v2.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -691,18 +701,34 @@ spec: - --leader-election - --leader-election-namespace=$(PDCSI_NAMESPACE) - --timeout=300s + - --retry-interval-max=60s env: - name: PDCSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 + image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.1 name: csi-snapshotter volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=5 + - --endpoint=unix:/csi/csi.sock + - --extra-labels=k8s-io-cluster-name=minimal-gce-with-a-very-very-very-very-very-long-name-example-com + - --supports-dynamic-iops-provisioning=hyperdisk-balanced,hyperdisk-extreme + - --supports-dynamic-throughput-provisioning=hyperdisk-balanced,hyperdisk-throughput,hyperdisk-ml + - --enable-data-cache + - --run-node-service=false + env: [] + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 + name: gce-pd-driver + volumeMounts: + - mountPath: /csi + name: socket-dir hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-controller serviceAccountName: csi-gce-pd-controller-sa tolerations: @@ -754,7 +780,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0 name: csi-driver-registrar volumeMounts: - mountPath: /csi @@ -765,7 +791,8 @@ spec: - --v=5 - --endpoint=unix:/csi/csi.sock - --run-controller-service=false - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.13 + - --enable-data-cache + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 name: gce-pd-driver securityContext: privileged: true @@ -786,7 +813,8 @@ spec: - mountPath: /sys name: sys hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-node serviceAccountName: csi-gce-pd-node-sa tolerations: diff --git a/tests/integration/update_cluster/minimal_gce_longclustername/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/minimal_gce_longclustername/data/aws_s3_object_cluster-completed.spec_content index 2737f0d3b648d..b1b58b35e8412 100644 --- a/tests/integration/update_cluster/minimal_gce_longclustername/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/minimal_gce_longclustername/data/aws_s3_object_cluster-completed.spec_content @@ -13,7 +13,9 @@ spec: gceServiceAccount: default gceUseStartupScript: true gcpPDCSIDriver: + defaultStorageClassName: balanced-csi enabled: true + version: v1.22.1 manageStorageClasses: true multizone: true nodeTags: minimal-gce-with-a-very-very-very-very--96dqvi-k8s-io-role-node diff --git a/tests/integration/update_cluster/minimal_gce_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal_gce_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-bootstrap_content index e73d67c5dfc9e..efe56ea3e32a6 100644 --- a/tests/integration/update_cluster/minimal_gce_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal_gce_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-bootstrap_content @@ -47,7 +47,7 @@ spec: version: 9.99.0 - id: k8s-1.23 manifest: gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml - manifestHash: e1f5166c3ef5d62b244f1d74e22c396956ac5cf85de4828617a729313a9c49b7 + manifestHash: 234c31cbc7c906d1a2df8067ccbe85172ec4659da1d1a9ee2833eb2723cbced0 name: gcp-pd-csi-driver.addons.k8s.io selector: k8s-addon: gcp-pd-csi-driver.addons.k8s.io diff --git a/tests/integration/update_cluster/minimal_gce_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/minimal_gce_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content index d12604a82fefb..195db1b6ab5fb 100644 --- a/tests/integration/update_cluster/minimal_gce_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content +++ b/tests/integration/update_cluster/minimal_gce_longclustername/data/aws_s3_object_minimal-gce-with-a-very-very-very-very-very-long-name.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content @@ -37,6 +37,26 @@ volumeBindingMode: WaitForFirstConsumer --- +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: balanced-storage +parameters: + provisioned-iops-on-create: "5000" + provisioned-throughput-on-create: 250Mi + type: hyperdisk-balanced +provisioner: pd.csi.storage.gke.io +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer + +--- + allowVolumeExpansion: true apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -583,16 +603,6 @@ spec: values: - linux containers: - - args: - - --v=5 - - --endpoint=unix:/csi/csi.sock - - --extra-labels=k8s-io-cluster-name=minimal-gce-with-a-very-very-very-very-very-long-name-example-com - env: [] - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.1 - name: gce-pd-driver - volumeMounts: - - mountPath: /csi - name: socket-dir - args: - --v=5 - --csi-address=/csi/csi.sock @@ -609,7 +619,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v6.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -638,7 +648,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-attacher:v4.2.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.8.1 livenessProbe: failureThreshold: 1 httpGet: @@ -667,7 +677,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0 + image: registry.k8s.io/sig-storage/csi-resizer:v2.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -691,18 +701,34 @@ spec: - --leader-election - --leader-election-namespace=$(PDCSI_NAMESPACE) - --timeout=300s + - --retry-interval-max=60s env: - name: PDCSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 + image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.1 name: csi-snapshotter volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=5 + - --endpoint=unix:/csi/csi.sock + - --extra-labels=k8s-io-cluster-name=minimal-gce-with-a-very-very-very-very-very-long-name-example-com + - --supports-dynamic-iops-provisioning=hyperdisk-balanced,hyperdisk-extreme + - --supports-dynamic-throughput-provisioning=hyperdisk-balanced,hyperdisk-throughput,hyperdisk-ml + - --enable-data-cache + - --run-node-service=false + env: [] + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 + name: gce-pd-driver + volumeMounts: + - mountPath: /csi + name: socket-dir hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-controller serviceAccountName: csi-gce-pd-controller-sa tolerations: @@ -754,7 +780,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0 name: csi-driver-registrar volumeMounts: - mountPath: /csi @@ -765,7 +791,8 @@ spec: - --v=5 - --endpoint=unix:/csi/csi.sock - --run-controller-service=false - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.13 + - --enable-data-cache + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 name: gce-pd-driver securityContext: privileged: true @@ -786,7 +813,8 @@ spec: - mountPath: /sys name: sys hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-node serviceAccountName: csi-gce-pd-node-sa tolerations: diff --git a/tests/integration/update_cluster/minimal_gce_plb/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/minimal_gce_plb/data/aws_s3_object_cluster-completed.spec_content index 7e5d430b678be..477980d2476b2 100644 --- a/tests/integration/update_cluster/minimal_gce_plb/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/minimal_gce_plb/data/aws_s3_object_cluster-completed.spec_content @@ -16,7 +16,9 @@ spec: cloudConfig: gceServiceAccount: default gcpPDCSIDriver: + defaultStorageClassName: balanced-storage enabled: true + version: v1.22.1 manageStorageClasses: true multizone: true nodeTags: minimal-gce-plb-example-com-k8s-io-role-node diff --git a/tests/integration/update_cluster/minimal_gce_plb/data/aws_s3_object_minimal-gce-plb.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal_gce_plb/data/aws_s3_object_minimal-gce-plb.example.com-addons-bootstrap_content index 72eefb9a34fd9..230d5aed8c7e8 100644 --- a/tests/integration/update_cluster/minimal_gce_plb/data/aws_s3_object_minimal-gce-plb.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal_gce_plb/data/aws_s3_object_minimal-gce-plb.example.com-addons-bootstrap_content @@ -47,7 +47,7 @@ spec: version: 9.99.0 - id: k8s-1.23 manifest: gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml - manifestHash: afc1eb3f194bfc3fd4c571d2b8b7ed10c06564935036093a4a5645e8f3974882 + manifestHash: c897bee1857fa152f1abac83f90d55d0f2e500e92dde185fecb3e4394025217e name: gcp-pd-csi-driver.addons.k8s.io selector: k8s-addon: gcp-pd-csi-driver.addons.k8s.io diff --git a/tests/integration/update_cluster/minimal_gce_plb/data/aws_s3_object_minimal-gce-plb.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/minimal_gce_plb/data/aws_s3_object_minimal-gce-plb.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content index e16a69eae462b..10897ac0252dd 100644 --- a/tests/integration/update_cluster/minimal_gce_plb/data/aws_s3_object_minimal-gce-plb.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content +++ b/tests/integration/update_cluster/minimal_gce_plb/data/aws_s3_object_minimal-gce-plb.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content @@ -21,8 +21,6 @@ allowVolumeExpansion: true apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - annotations: - storageclass.kubernetes.io/is-default-class: "true" labels: addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io app.kubernetes.io/managed-by: kops @@ -37,6 +35,28 @@ volumeBindingMode: WaitForFirstConsumer --- +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "true" + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: balanced-storage +parameters: + provisioned-iops-on-create: "5000" + provisioned-throughput-on-create: 250Mi + type: hyperdisk-balanced +provisioner: pd.csi.storage.gke.io +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer + +--- + allowVolumeExpansion: true apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -583,16 +603,6 @@ spec: values: - linux containers: - - args: - - --v=5 - - --endpoint=unix:/csi/csi.sock - - --extra-labels=k8s-io-cluster-name=minimal-gce-plb-example-com - env: [] - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.1 - name: gce-pd-driver - volumeMounts: - - mountPath: /csi - name: socket-dir - args: - --v=5 - --csi-address=/csi/csi.sock @@ -609,7 +619,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v6.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -638,7 +648,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-attacher:v4.2.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.8.1 livenessProbe: failureThreshold: 1 httpGet: @@ -667,7 +677,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0 + image: registry.k8s.io/sig-storage/csi-resizer:v2.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -691,18 +701,34 @@ spec: - --leader-election - --leader-election-namespace=$(PDCSI_NAMESPACE) - --timeout=300s + - --retry-interval-max=60s env: - name: PDCSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 + image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.1 name: csi-snapshotter volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=5 + - --endpoint=unix:/csi/csi.sock + - --extra-labels=k8s-io-cluster-name=minimal-gce-plb-example-com + - --supports-dynamic-iops-provisioning=hyperdisk-balanced,hyperdisk-extreme + - --supports-dynamic-throughput-provisioning=hyperdisk-balanced,hyperdisk-throughput,hyperdisk-ml + - --enable-data-cache + - --run-node-service=false + env: [] + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 + name: gce-pd-driver + volumeMounts: + - mountPath: /csi + name: socket-dir hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-controller serviceAccountName: csi-gce-pd-controller-sa tolerations: @@ -754,7 +780,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0 name: csi-driver-registrar volumeMounts: - mountPath: /csi @@ -765,7 +791,8 @@ spec: - --v=5 - --endpoint=unix:/csi/csi.sock - --run-controller-service=false - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.13 + - --enable-data-cache + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 name: gce-pd-driver securityContext: privileged: true @@ -786,7 +813,8 @@ spec: - mountPath: /sys name: sys hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-node serviceAccountName: csi-gce-pd-node-sa tolerations: diff --git a/tests/integration/update_cluster/minimal_gce_plb/in-v1alpha2.yaml b/tests/integration/update_cluster/minimal_gce_plb/in-v1alpha2.yaml index 6f0e2ad594a8b..e1d6cad9696ab 100644 --- a/tests/integration/update_cluster/minimal_gce_plb/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/minimal_gce_plb/in-v1alpha2.yaml @@ -35,6 +35,8 @@ spec: type: hyperdisk-balanced cloudConfig: gceServiceAccount: default + gcpPDCSIDriver: + defaultStorageClassName: balanced-storage iam: legacy: false kubelet: diff --git a/tests/integration/update_cluster/minimal_gce_private/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/minimal_gce_private/data/aws_s3_object_cluster-completed.spec_content index 82bd4826fe410..367cc0c8b25d6 100644 --- a/tests/integration/update_cluster/minimal_gce_private/data/aws_s3_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/minimal_gce_private/data/aws_s3_object_cluster-completed.spec_content @@ -12,7 +12,9 @@ spec: cloudConfig: gceServiceAccount: default gcpPDCSIDriver: + defaultStorageClassName: balanced-csi enabled: true + version: v1.22.1 manageStorageClasses: true multizone: true nodeTags: minimal-gce-private-example-com-k8s-io-role-node diff --git a/tests/integration/update_cluster/minimal_gce_private/data/aws_s3_object_minimal-gce-private.example.com-addons-bootstrap_content b/tests/integration/update_cluster/minimal_gce_private/data/aws_s3_object_minimal-gce-private.example.com-addons-bootstrap_content index 6dfb43a8455bd..8e009f359c66f 100644 --- a/tests/integration/update_cluster/minimal_gce_private/data/aws_s3_object_minimal-gce-private.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/minimal_gce_private/data/aws_s3_object_minimal-gce-private.example.com-addons-bootstrap_content @@ -47,7 +47,7 @@ spec: version: 9.99.0 - id: k8s-1.23 manifest: gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml - manifestHash: 8d5e209ab30933b56dc4136983d524b79d769a2ac5bd6627fcb4ad99c5e02133 + manifestHash: 85c4b3880ef53b45b4adc1db7ebcbfefbb0c5b82c007a57532020fe6c511d8a7 name: gcp-pd-csi-driver.addons.k8s.io selector: k8s-addon: gcp-pd-csi-driver.addons.k8s.io diff --git a/tests/integration/update_cluster/minimal_gce_private/data/aws_s3_object_minimal-gce-private.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/minimal_gce_private/data/aws_s3_object_minimal-gce-private.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content index ac1cace9d66ec..7cbcf9228cdaf 100644 --- a/tests/integration/update_cluster/minimal_gce_private/data/aws_s3_object_minimal-gce-private.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content +++ b/tests/integration/update_cluster/minimal_gce_private/data/aws_s3_object_minimal-gce-private.example.com-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content @@ -37,6 +37,26 @@ volumeBindingMode: WaitForFirstConsumer --- +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: balanced-storage +parameters: + provisioned-iops-on-create: "5000" + provisioned-throughput-on-create: 250Mi + type: hyperdisk-balanced +provisioner: pd.csi.storage.gke.io +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer + +--- + allowVolumeExpansion: true apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -583,16 +603,6 @@ spec: values: - linux containers: - - args: - - --v=5 - - --endpoint=unix:/csi/csi.sock - - --extra-labels=k8s-io-cluster-name=minimal-gce-private-example-com - env: [] - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.1 - name: gce-pd-driver - volumeMounts: - - mountPath: /csi - name: socket-dir - args: - --v=5 - --csi-address=/csi/csi.sock @@ -609,7 +619,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v6.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -638,7 +648,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-attacher:v4.2.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.8.1 livenessProbe: failureThreshold: 1 httpGet: @@ -667,7 +677,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0 + image: registry.k8s.io/sig-storage/csi-resizer:v2.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -691,18 +701,34 @@ spec: - --leader-election - --leader-election-namespace=$(PDCSI_NAMESPACE) - --timeout=300s + - --retry-interval-max=60s env: - name: PDCSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 + image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.1 name: csi-snapshotter volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=5 + - --endpoint=unix:/csi/csi.sock + - --extra-labels=k8s-io-cluster-name=minimal-gce-private-example-com + - --supports-dynamic-iops-provisioning=hyperdisk-balanced,hyperdisk-extreme + - --supports-dynamic-throughput-provisioning=hyperdisk-balanced,hyperdisk-throughput,hyperdisk-ml + - --enable-data-cache + - --run-node-service=false + env: [] + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 + name: gce-pd-driver + volumeMounts: + - mountPath: /csi + name: socket-dir hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-controller serviceAccountName: csi-gce-pd-controller-sa tolerations: @@ -754,7 +780,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0 name: csi-driver-registrar volumeMounts: - mountPath: /csi @@ -765,7 +791,8 @@ spec: - --v=5 - --endpoint=unix:/csi/csi.sock - --run-controller-service=false - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.13 + - --enable-data-cache + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 name: gce-pd-driver securityContext: privileged: true @@ -786,7 +813,8 @@ spec: - mountPath: /sys name: sys hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-node serviceAccountName: csi-gce-pd-node-sa tolerations: diff --git a/upup/models/cloudup/resources/addons/gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml.template b/upup/models/cloudup/resources/addons/gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml.template index 0b5eeb109b849..a5de5bb788363 100644 --- a/upup/models/cloudup/resources/addons/gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml.template +++ b/upup/models/cloudup/resources/addons/gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml.template @@ -1,6 +1,7 @@ # https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver/tree/master/deploy/kubernetes/overlays/noauth {{ if WithDefaultBool .CloudConfig.ManageStorageClasses true }} +{{ with .CloudProvider.GCE.PDCSIDriver }} apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: @@ -22,8 +23,10 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: balanced-csi + {{ if eq "balanced-csi" (GetString .DefaultStorageClassName) }} annotations: storageclass.kubernetes.io/is-default-class: "true" + {{ end }} labels: kubernetes.io/cluster-service: "true" k8s-addon: gcp-pd-csi-driver.addons.k8s.io @@ -36,6 +39,28 @@ allowVolumeExpansion: true --- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: balanced-storage + {{ if eq "balanced-storage" (GetString .DefaultStorageClassName) }} + annotations: + storageclass.kubernetes.io/is-default-class: "true" + {{ end }} + labels: + kubernetes.io/cluster-service: "true" + k8s-addon: gcp-pd-csi-driver.addons.k8s.io +provisioner: pd.csi.storage.gke.io +reclaimPolicy: Delete +parameters: + type: hyperdisk-balanced + provisioned-throughput-on-create: "250Mi" + provisioned-iops-on-create: "5000" +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true + +--- + apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: @@ -50,6 +75,7 @@ reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer allowVolumeExpansion: true +{{ end }} {{ end }} --- @@ -472,16 +498,6 @@ spec: values: - linux containers: - - args: - - --v=5 - - --endpoint=unix:/csi/csi.sock - - --extra-labels=k8s-io-cluster-name={{ replace ClusterName "." "-" }} - env: [] - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.1 - name: gce-pd-driver - volumeMounts: - - mountPath: /csi - name: socket-dir - args: - --v=5 - --csi-address=/csi/csi.sock @@ -498,7 +514,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-provisioner:v3.4.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v6.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -527,7 +543,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-attacher:v4.2.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.8.1 livenessProbe: failureThreshold: 1 httpGet: @@ -556,7 +572,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0 + image: registry.k8s.io/sig-storage/csi-resizer:v2.0.0 livenessProbe: failureThreshold: 1 httpGet: @@ -580,18 +596,34 @@ spec: - --leader-election - --leader-election-namespace=$(PDCSI_NAMESPACE) - --timeout=300s + - --retry-interval-max=60s env: - name: PDCSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 + image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.1 name: csi-snapshotter volumeMounts: - mountPath: /csi name: socket-dir + - args: + - --v=5 + - --endpoint=unix:/csi/csi.sock + - --extra-labels=k8s-io-cluster-name={{ replace ClusterName "." "-" }} + - "--supports-dynamic-iops-provisioning=hyperdisk-balanced,hyperdisk-extreme" + - "--supports-dynamic-throughput-provisioning=hyperdisk-balanced,hyperdisk-throughput,hyperdisk-ml" + - --enable-data-cache + - --run-node-service=false + env: [] + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:{{ .CloudProvider.GCE.PDCSIDriver.Version }} + name: gce-pd-driver + volumeMounts: + - mountPath: /csi + name: socket-dir hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-controller serviceAccountName: csi-gce-pd-controller-sa tolerations: @@ -636,7 +668,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.7.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0 name: csi-driver-registrar volumeMounts: - mountPath: /csi @@ -647,7 +679,8 @@ spec: - --v=5 - --endpoint=unix:/csi/csi.sock - --run-controller-service=false - image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.10.13 + - --enable-data-cache + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:{{ .CloudProvider.GCE.PDCSIDriver.Version }} name: gce-pd-driver securityContext: privileged: true @@ -672,7 +705,8 @@ spec: mountPath: /etc/selinux {{ end }} hostNetwork: true - nodeSelector: null + nodeSelector: + kubernetes.io/os: linux priorityClassName: csi-gce-pd-node serviceAccountName: csi-gce-pd-node-sa tolerations: diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index fb7538319cf32..99d540dffe32e 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -112,6 +112,12 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS } return defaultValue } + dest["GetString"] = func(v *string) string { + if v != nil { + return *v + } + return "" + } dest["GetCloudProvider"] = cluster.GetCloudProvider dest["GetInstanceGroup"] = tf.GetInstanceGroup