From ae2b6d1a73bdcbd62c714c93aae42c57f5f9907c Mon Sep 17 00:00:00 2001 From: Jesus Perez Rey Date: Wed, 20 Dec 2023 20:09:25 +0000 Subject: [PATCH 1/4] GCP: Add support for specifying the backups storage bucket. Signed-off-by: Jesus Perez Rey --- roles/common/defaults/main.yml | 1 + roles/infrastructure/defaults/main.yml | 8 +++++--- roles/infrastructure/tasks/initialize_gcp.yml | 5 +++-- roles/platform/defaults/main.yml | 2 ++ roles/platform/tasks/setup_gcp_authz.yml | 2 ++ roles/platform/tasks/setup_gcp_env.yml | 1 + 6 files changed, 14 insertions(+), 5 deletions(-) diff --git a/roles/common/defaults/main.yml b/roles/common/defaults/main.yml index a1a3dbb9..fde2e9d7 100644 --- a/roles/common/defaults/main.yml +++ b/roles/common/defaults/main.yml @@ -104,6 +104,7 @@ common__ml_path: "{{ infra.storage.path.ml | default('d common__de_path: "{{ infra.storage.path.de | default('dataeng') }}" common__logs_path: "{{ infra.storage.path.logs | default(common__logs_suffix) }}" common__data_path: "{{ infra.storage.path.data | default(common__data_suffix) }}" +common__backups_path: "{{ infra.storage.path.backups | default(common__logs_path) }}" common__ranger_audit_path: "{{ infra.storage.path.ranger_audit | default('ranger/audit') }}" # AWS Infra diff --git a/roles/infrastructure/defaults/main.yml b/roles/infrastructure/defaults/main.yml index 4d113d5b..00dbb4c4 100644 --- a/roles/infrastructure/defaults/main.yml +++ b/roles/infrastructure/defaults/main.yml @@ -68,6 +68,7 @@ infra__storage_name: "{{ common__storage_name }}" infra__logs_path: "{{ common__logs_path }}" infra__data_path: "{{ common__data_path }}" +infra__backups_path: "{{ common__backups_path }}" infra__ranger_audit_path: "{{ common__ranger_audit_path }}" infra__public_key_path: "{{ globals.ssh.key_path | default('~/.ssh') }}" @@ -133,8 +134,9 @@ infra__aws_private_endpoints: "{{ infra.aws.vpc.private_endpoints | defaul # GCP infra__gcp_project: "{{ common__gcp_project }}" -infra__gcp_storage_location_data: "{{ infra.gcp.storage.path.data | default([infra__storage_name, infra__data_path] | join('-')) }}" -infra__gcp_storage_location_logs: "{{ infra.gcp.storage.path.logs | default([infra__storage_name, infra__logs_path] | join('-')) }}" +infra__gcp_storage_location_data: "{{ infra.gcp.storage.path.data | default([infra__storage_name, infra__data_path] | join('-')) }}" +infra__gcp_storage_location_logs: "{{ infra.gcp.storage.path.logs | default([infra__storage_name, infra__logs_path] | join('-')) }}" +infra__gcp_storage_location_backups: "{{ infra.gcp.storage.path.backups | default(infra__gcp_storage_location_logs) }}" infra__gcp_cloud_router_name_suffix: "{{ infra.gcp.network.router.name_suffix | default('router') }}" infra__gcp_cloud_router_name: "{{ infra.gcp.network.router.name | default([infra__namespace, infra__gcp_cloud_router_name_suffix] | join('-')) }}" @@ -173,4 +175,4 @@ infra__utlity_bucket_name: "{{ globals.utility_bucket_name | default( # Teardown infra__force_teardown: "{{ common__force_teardown }}" -infra__env_name: "{{ common__env_name }}" # Used for purge lookups \ No newline at end of file +infra__env_name: "{{ common__env_name }}" # Used for purge lookups diff --git a/roles/infrastructure/tasks/initialize_gcp.yml b/roles/infrastructure/tasks/initialize_gcp.yml index 3aa49566..a04e2f04 100644 --- a/roles/infrastructure/tasks/initialize_gcp.yml +++ b/roles/infrastructure/tasks/initialize_gcp.yml @@ -38,11 +38,12 @@ fail_msg: "Gcloud Collection failed to retrieve resources, you may need to run 'gcloud auth login' or 'gcloud init': {{ __gcp_vpc_info }}" quiet: yes -- name: Set fact for Log and Data locations +- name: Set fact for Log, Backups and Data locations ansible.builtin.set_fact: infra__gcp_storage_locations: "{{ infra__gcp_storage_locations | default([]) | union([__gcp_storage_location_item]) }}" loop_control: loop_var: __gcp_storage_location_item loop: - "{{ infra__gcp_storage_location_data }}" - - "{{ infra__gcp_storage_location_logs }}" \ No newline at end of file + - "{{ infra__gcp_storage_location_logs }}" + - "{{ infra__gcp_storage_location_backups }}" diff --git a/roles/platform/defaults/main.yml b/roles/platform/defaults/main.yml index d7100966..97f00a52 100644 --- a/roles/platform/defaults/main.yml +++ b/roles/platform/defaults/main.yml @@ -52,6 +52,7 @@ plat__storage_name: "{{ common__storage_name }}" plat__logs_path: "{{ common__logs_path }}" plat__data_path: "{{ common__data_path }}" +plat__data_path: "{{ common__backups_path }}" plat__public_key_id: "{{ common__public_key_id }}" plat__public_key_text: "{{ common__public_key_text }}" @@ -168,6 +169,7 @@ plat__gcp_idbroker_identity_name: "{{ env.gcp.role.name.idbroker | d plat__gcp_storage_location_data: "{{ env.gcp.storage.path.data | default([plat__storage_name, plat__data_path] | join('-')) }}" plat__gcp_storage_location_logs: "{{ env.gcp.storage.path.logs | default([plat__storage_name, plat__logs_path] | join('-')) }}" +plat__gcp_storage_location_backups: "{{ env.gcp.storage.path.backups | default(plat__gcp_storage_location_logs) }}" plat__gcp_xaccount_policy_bindings: "{{ env.gcp.bindings.cross_account | default(plat__gcp_xaccount_policy_bindings_default) }}" plat__gcp_log_role_perms: "{{ env.gcp.bindings.logs | default(plat__gcp_log_policy_bindings_default) }}" diff --git a/roles/platform/tasks/setup_gcp_authz.yml b/roles/platform/tasks/setup_gcp_authz.yml index 1681bbc6..fb34320e 100644 --- a/roles/platform/tasks/setup_gcp_authz.yml +++ b/roles/platform/tasks/setup_gcp_authz.yml @@ -167,6 +167,8 @@ loop: - account: "serviceAccount:{{ plat__gcp_log_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com:admin" bucket: "{{ plat__gcp_storage_location_logs }}" + - account: "serviceAccount:{{ plat__gcp_log_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com:admin" + bucket: "{{ plat__gcp_storage_location_backups }}" - account: "serviceAccount:{{ plat__gcp_datalakeadmin_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com:admin" bucket: "{{ plat__gcp_storage_location_data }}" - account: "serviceAccount:{{ plat__gcp_ranger_audit_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com:admin" diff --git a/roles/platform/tasks/setup_gcp_env.yml b/roles/platform/tasks/setup_gcp_env.yml index 3c67727b..799f76e3 100644 --- a/roles/platform/tasks/setup_gcp_env.yml +++ b/roles/platform/tasks/setup_gcp_env.yml @@ -25,6 +25,7 @@ public_ip: "{{ plat__use_public_ip }}" log_location: "gs://{{ plat__gcp_storage_location_logs }}" log_identity: "{{ plat__gcp_log_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com" + backups_location: "gs://{{ plat__gcp_storage_location_backups }}" vpc_id: "{{ plat__vpc_name }}" subnet_ids: - "{{ plat__gcp_subnet_id if plat__gcp_subnet_id else plat__gcp_subnets_discovered[0].name }}" # TODO - Check in validation_gcp.yml -- CDP on GCP only supports a single subnet deployment From 27b0f15d6aac6a1c95469110b00fd1a2f530f0c9 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Wed, 20 Dec 2023 20:15:16 +0000 Subject: [PATCH 2/4] Update backup_location parameter name for GCP env Signed-off-by: Jim Enright --- roles/platform/tasks/setup_gcp_env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/platform/tasks/setup_gcp_env.yml b/roles/platform/tasks/setup_gcp_env.yml index 799f76e3..aac400c0 100644 --- a/roles/platform/tasks/setup_gcp_env.yml +++ b/roles/platform/tasks/setup_gcp_env.yml @@ -25,7 +25,7 @@ public_ip: "{{ plat__use_public_ip }}" log_location: "gs://{{ plat__gcp_storage_location_logs }}" log_identity: "{{ plat__gcp_log_identity_name }}@{{ plat__gcp_project }}.iam.gserviceaccount.com" - backups_location: "gs://{{ plat__gcp_storage_location_backups }}" + backup_location: "gs://{{ plat__gcp_storage_location_backups }}" vpc_id: "{{ plat__vpc_name }}" subnet_ids: - "{{ plat__gcp_subnet_id if plat__gcp_subnet_id else plat__gcp_subnets_discovered[0].name }}" # TODO - Check in validation_gcp.yml -- CDP on GCP only supports a single subnet deployment From eb0493575513047cc4207f48c0a2c265d04323bd Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 21 Dec 2023 11:45:31 +0000 Subject: [PATCH 3/4] Remove duplicate plat__data_path variable Signed-off-by: Jim Enright --- roles/platform/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/platform/defaults/main.yml b/roles/platform/defaults/main.yml index 97f00a52..cce4ab39 100644 --- a/roles/platform/defaults/main.yml +++ b/roles/platform/defaults/main.yml @@ -52,7 +52,7 @@ plat__storage_name: "{{ common__storage_name }}" plat__logs_path: "{{ common__logs_path }}" plat__data_path: "{{ common__data_path }}" -plat__data_path: "{{ common__backups_path }}" +plat__backup_path: "{{ common__backups_path }}" plat__public_key_id: "{{ common__public_key_id }}" plat__public_key_text: "{{ common__public_key_text }}" From 0785306b5b8dd3b8cf33247134355eaaa68f2151 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 21 Dec 2023 14:29:20 +0000 Subject: [PATCH 4/4] Update configuation parameter docs Signed-off-by: Jim Enright --- docs/configuration.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/configuration.yml b/docs/configuration.yml index 8354cffb..af937ff8 100644 --- a/docs/configuration.yml +++ b/docs/configuration.yml @@ -268,6 +268,7 @@ env: suffix: storage: path: + backups: data: logs: suffix: @@ -419,6 +420,7 @@ infra: region: storage: path: + backups: data: logs: security_group: @@ -434,6 +436,7 @@ infra: storage: name: path: + backups: data: de: logs: