Move DFX Beta implementation to GA process

roles/infrastructure/tasks/teardown_aws_compute.yml

@@ -42,6 +42,19 @@
   poll: 0
   register: __eks_teardowns_info
 
+- name: Ensure RDS Instance deletion protection is disabled if discovered
+  when:
+    - infra__force_teardown | bool
+    - __infra_aws_rds_instances is defined
+    - __infra_aws_rds_instances | length > 0
+  command: >
+    aws rds modify-db-instance
+    --db-instance-identifier {{ __infra_rds_protect_item.db_instance_identifier }}
+    --no-deletion-protection
+  loop: "{{ __infra_aws_rds_instances }}"
+  loop_control:
+    loop_var: __infra_rds_protect_item
+
 - name: Handle RDS Instance removal if discovered
   when:
     - infra__force_teardown | bool
@@ -55,6 +68,7 @@
   loop: "{{ __infra_aws_rds_instances }}"
   loop_control:
     loop_var: __infra_rds_remove_item
+    label: "{{ __infra_rds_remove_item.db_instance_identifier }}"
   async: 3600 # 1 hour timeout
   poll: 0
   register: __rds_teardowns_info
@@ -71,7 +85,7 @@
   loop: "{{ __rds_teardowns_info.results }}"
   register: __rds_teardowns_async
   until: __rds_teardowns_async.finished
-  retries: 360
+  retries: 3600
   delay: 10
 
 - name: Wait for EKS teardowns to complete

roles/platform/vars/main.yml

@@ -39,6 +39,9 @@ plat__cdp_iam_identities:
   dw_user: DWUser
   ml_admin: MLAdmin
   ml_user: MLUser
+  df_admin: DFAdmin
+  df_flow_admin: DFFlowAdmin
+  df_flow_user: DFFlowUser
 
 plat__cdp_iam_admin_group_resource_roles_default:
   - "{{ plat__cdp_iam_identities.env_admin }}"
@@ -47,11 +50,16 @@ plat__cdp_iam_admin_group_resource_roles_default:
   - "{{ plat__cdp_iam_identities.dw_user }}"
   - "{{ plat__cdp_iam_identities.ml_admin }}"
   - "{{ plat__cdp_iam_identities.ml_user }}"
+  - "{{ plat__cdp_iam_identities.df_admin }}"
+  - "{{ plat__cdp_iam_identities.df_flow_admin }}"
+  - "{{ plat__cdp_iam_identities.df_flow_user }}"
 
 plat__cdp_iam_user_group_resource_roles_default:
   - "{{ plat__cdp_iam_identities.env_user }}"
   - "{{ plat__cdp_iam_identities.dw_user }}"
   - "{{ plat__cdp_iam_identities.ml_user }}"
+  - "{{ plat__cdp_iam_identities.df_flow_admin }}"
+  - "{{ plat__cdp_iam_identities.df_flow_user }}"
 
 plat__gcp_xaccount_policy_bindings_default:
   - "roles/compute.instanceAdmin.v1"

roles/runtime/defaults/main.yml

@@ -71,9 +71,13 @@ run__dw_force_delete:               "{{ dw.force_delete | default (run__force_te
 run__df_nodes_min:                  "{{ df.min_k8s_nodes | default(3) }}"
 run__df_nodes_max:                  "{{ df.max_k8s_nodes | default(5) }}"
 run__df_public_loadbalancer:        "{{ df.public_loadbalancer | default(run__public_endpoint_access) }}"
-run__df_ip_ranges:                  "{{ df.ip_ranges | default([]) }}"
+run__df_lb_ip_ranges:               "{{ df.loadbalancer_ip_ranges | default([]) }}"
+run__df_kube_ip_ranges:             "{{ df.kube_ip_ranges | default([]) }}"
+run__df_cluster_subnets:            "{{ df.cluster_subnets | default([]) }}"
+run__df_lb_subnets:                 "{{ df.loadbalancer_subnets | default([]) }}"
 run__df_persist:                    "{{ df.teardown.persist | default(False) }}"
-run__df_force_delete:               "{{ df.force_delete | default (run__force_teardown) }}"
+run__df_force_delete:               "{{ df.force_delete | default(run__force_teardown) }}"
+run__df_terminate_deployments:      "{{ df.terminate_deployments | default(run__force_teardown) }}"
 
 # Deploy
 run__include_ml:                     "{{ common__include_ml }}"

roles/runtime/tasks/initialize_teardown.yml

@@ -25,7 +25,7 @@
   ansible.builtin.include_tasks: "initialize_base.yml"
 
 - name: Discover CDP DF Deployments
-  register: run__df_env_info
+  register: run__df_service_info
   when: run__include_df
   cloudera.cloud.df_info:
     name: "{{ run__env_name }}"

roles/runtime/tasks/setup_base.yml

@@ -76,12 +76,15 @@
 
 - name: Execute CDP Dataflow setup
   when: run__include_df
-  cloudera.cloud.df:
-    name: "{{ run__cdp_env_crn }}"
+  cloudera.cloud.df_service:
+    env_crn: "{{ run__cdp_env_crn }}"
     nodes_min: "{{ run__df_nodes_min }}"
     nodes_max: "{{ run__df_nodes_max }}"
     public_loadbalancer: "{{ run__df_public_loadbalancer }}"
-    ip_ranges: "{{ run__df_ip_ranges }}"
+    loadbalancer_ip_ranges: "{{ run__df_lb_ip_ranges }}"
+    kube_ip_ranges: "{{ run__df_kube_ip_ranges }}"
+    cluster_subnets: "{{ run__df_cluster_subnets }}"
+    loadbalancer_subnets: "{{ run__df_lb_subnets }}"
     state: present
     wait: no
 
@@ -126,6 +129,6 @@
 
 - name: Wait for CDP Dataflow deployment to complete
   when: run__include_df
-  cloudera.cloud.df:
-    name: "{{ run__cdp_env_crn }}"
+  cloudera.cloud.df_service:
+    env_crn: "{{ run__cdp_env_crn }}"
     wait: yes

roles/runtime/tasks/teardown_base.yml

@@ -48,17 +48,18 @@
   register: __df_teardown_info
   when:
     - run__include_df or run__force_teardown | bool
-    - run__df_env_info is defined and run__df_env_info.services is defined
-    - run__df_env_info.services | length > 0
-  cloudera.cloud.df:
-    name: "{{ __df_teardown_req_item.crn }}"
+    - run__df_service_info is defined and run__df_service_info.services is defined
+    - run__df_service_info.services | length > 0
+  cloudera.cloud.df_service:
+    df_crn: "{{ __df_teardown_req_item.crn }}"
     persist: "{{ run__df_persist }}"
     force: "{{ run__df_force_delete }}"
+    terminate: "{{ run__df_terminate_deployments }}"
     state: absent
     wait: no
   loop_control:
     loop_var: __df_teardown_req_item
-  loop: "{{ run__df_env_info.services }}"
+  loop: "{{ run__df_service_info.services }}"
 
 - name: Execute CDP ML Workspace teardown
   when:
@@ -158,13 +159,13 @@
 - name: Wait for CDP Dataflow deployment to decommission
   when:
     - run__include_df
-    - run__df_env_info.services | length > 0
-  cloudera.cloud.df:
-    name: "{{ __df_teardown_wait_item.crn }}"
+    - run__df_service_info.services | length > 0
+  cloudera.cloud.df_service:
+    df_crn: "{{ __df_teardown_wait_item.crn }}"
     persist: "{{ run__df_persist }}"
     force: "{{ run__df_force_delete }}"
     state: absent
     wait: yes
   loop_control:
     loop_var: __df_teardown_wait_item
-  loop: "{{ run__df_env_info.services }}"
+  loop: "{{ run__df_service_info.services }}"