Add support for CDE (#58)

* Add DE definitions
* Clean up CDE roles 
* Add Wait statement for DE setup
* Improve defaults for DE setup
* Include vars in docs/configuration.yml

Signed-off-by: Curtis Howard <[email protected]>
Co-authored-by: Alan Silva <[email protected]>

Author: curtishoward

docs/configuration.yml

@@ -15,6 +15,10 @@ datahub:
   suffix:
   tags:
 de:
+  definitions:
+  suffix:
+  tags:
+  force_delete:
 df:
   suffix:
   min_k8s_nodes:
@@ -189,6 +193,7 @@ globals:
     admin:
     app:
     cml:
+    cde:
     credential:
     cross_account:
     data:
@@ -361,4 +366,4 @@ data:
       suffix:
   teardown:
       delete_policies:
-      delete_roles: 
+      delete_roles: 

roles/common/defaults/main.yml

@@ -45,6 +45,7 @@ common__external_data_suffix:             "{{ globals.labels.external_data | def
 common__datalake_admin_suffix:            "{{ globals.labels.datalake_admin | default('dladmin') }}"
 common__ranger_audit_suffix:              "{{ globals.labels.ranger_audit | default('audit') }}"
 common__cml_suffix:                       "{{ globals.labels.cml | default('cml') }}"
+common__cde_suffix:                       "{{ globals.labels.cde | default('cde') }}"
 common__igw_suffix:                       "{{ globals.labels.internet_gateway | default('igw') }}"
 common__app_suffix:                       "{{ globals.labels.app | default('app') }}"
 common__group_suffix:                     "{{ globals.labels.group | default('group') }}"
@@ -141,4 +142,4 @@ common__include_datahub:                   "{{ datahub is defined | bool }}"
 common__include_opdb:                      "{{ opdb is defined | bool }}"
 
 # Teardown
-common__force_teardown:                    "{{ globals.force_teardown | default(False) }}"  # WARNING: This will purge your namespace and anything related to it, use with extreme caution
+common__force_teardown:                    "{{ globals.force_teardown | default(False) }}"  # WARNING: This will purge your namespace and anything related to it, use with extreme caution

roles/info/tasks/main.yml

@@ -34,7 +34,7 @@
         - name: Set fact for the artifacts directory path
           ansible.builtin.set_fact:
             __artifacts_directory_path: "{{ __artifacts_directory_create.path }}"
-      
+
     - name: Validate artifacts directory if it does exist
       when: __artifacts_directory.stat.exists
       block:
@@ -70,6 +70,11 @@
     env: "{{ info__env_name }}"
   register: __ml_info
 
+- name: Query CDP DE Services
+  cloudera.cloud.de_info:
+    env: "{{ info__env_name }}"
+  register: __de_info
+
 - name: Query CDP Operational DBs
   cloudera.cloud.opdb_info:
     env:  "{{ info__env_name }}"
@@ -82,11 +87,12 @@
       datalake: "{{ __datalake_info.datalakes | first | default({}) }}"
       datahubs: "{{ __datahubs_info.datahubs }}"
       workspaces: "{{ __ml_info.workspaces }}"
+      services: "{{ __de_info.services }}"
       operational_dbs: "{{ __opdb_info.databases }}"
 
 - name: Save the CDP deployment details locally
   when: info__create_deployment_details
   ansible.builtin.copy:
     content: "{{ { 'deployment': deployment } | to_nice_yaml(indent=2) }}"
     dest: "{{ __artifacts_directory_path | default('.') }}/deployment_info.yml"
-  delegate_to: localhost
+  delegate_to: localhost

roles/platform/tasks/initialize_teardown.yml

@@ -28,11 +28,12 @@
       - plat__env_info.environments[0].descendants.df | length == 0
       - plat__env_info.environments[0].descendants.dw | length == 0
       - plat__env_info.environments[0].descendants.opdb | length == 0
+      - plat__env_info.environments[0].descendants.de | length == 0
     fail_msg: "Environment {{ plat__env_name }} has one or more child services registered, please check and try again"
     quiet: yes
 
 - name: Include provider-specific initialization base
   ansible.builtin.include_tasks: "initialize_{{ plat__infra_type | lower }}.yml"
 
 - name: Include provider-specific resources for CDP Public Environment teardown
-  ansible.builtin.include_tasks: "initialize_teardown_{{ plat__infra_type | lower }}.yml"
+  ansible.builtin.include_tasks: "initialize_teardown_{{ plat__infra_type | lower }}.yml"

roles/platform/vars/main.yml

@@ -42,6 +42,8 @@ plat__cdp_iam_identities:
   df_admin: DFAdmin
   df_flow_admin: DFFlowAdmin
   df_flow_user: DFFlowUser
+  de_admin: DEAdmin
+  de_user: DEUser
 
 plat__cdp_iam_admin_group_resource_roles_default:
   - "{{ plat__cdp_iam_identities.env_admin }}"
@@ -53,13 +55,16 @@ plat__cdp_iam_admin_group_resource_roles_default:
   - "{{ plat__cdp_iam_identities.df_admin }}"
   - "{{ plat__cdp_iam_identities.df_flow_admin }}"
   - "{{ plat__cdp_iam_identities.df_flow_user }}"
+  - "{{ plat__cdp_iam_identities.de_admin }}"
+  - "{{ plat__cdp_iam_identities.de_user }}"
 
 plat__cdp_iam_user_group_resource_roles_default:
   - "{{ plat__cdp_iam_identities.env_user }}"
   - "{{ plat__cdp_iam_identities.dw_user }}"
   - "{{ plat__cdp_iam_identities.ml_user }}"
   - "{{ plat__cdp_iam_identities.df_flow_admin }}"
   - "{{ plat__cdp_iam_identities.df_flow_user }}"
+  - "{{ plat__cdp_iam_identities.de_user }}"
 
 plat__gcp_xaccount_policy_bindings_default:
   - "roles/compute.instanceAdmin.v1"
@@ -81,4 +86,4 @@ plat__azure_roles:
   storown: 'Storage Blob Data Owner'
   storcnt: 'Storage Blob Data Contributor'
   contrib: 'Contributor'
-  owner: 'Owner'
+  owner: 'Owner'

roles/runtime/defaults/main.yml

@@ -64,6 +64,11 @@ run__ml_force_delete:               "{{ ml.force_delete | default (run__force_te
 run__ml_remove_storage:             "{{ ml.remove_storage | default (run__force_teardown) }}"
 run__ml_public_loadbalancer:        "{{ ml.public_loadbalancer | default(run__public_endpoint_access) }}"
 
+run__de_definitions:                "{{ de.definitions | default([{}]) }}"
+run__de_suffix:                     "{{ de.suffix | default('de') }}"
+run__de_tags:                       "{{ de.tags | default(common__tags) }}"
+run__de_force_delete:               "{{ de.force_delete | default (run__force_teardown) }}"
+
 run__dw_definitions:                "{{ dw.definitions | default([{}]) }}"
 run__dw_suffix:                     "{{ dw.suffix | default('dw') }}"
 run__dw_force_delete:               "{{ dw.force_delete | default (run__force_teardown) }}"
@@ -85,4 +90,4 @@ run__include_dw:                     "{{ common__include_dw }}"
 run__include_de:                     "{{ common__include_de }}"
 run__include_df:                     "{{ common__include_df }}"
 run__include_datahub:                "{{ common__include_datahub }}"
-run__include_opdb:                   "{{ common__include_opdb }}"
+run__include_opdb:                   "{{ common__include_opdb }}"

roles/runtime/tasks/initialize_base.yml

@@ -161,3 +161,20 @@
       loop_control:
         loop_var: __ml_config
         label: "{{ config.name }}"
+
+- name: Prepare for CDP DE Service experiences
+  when: run__include_de
+  block:
+    - name: Construct CDP DE Service configurations
+      ansible.builtin.set_fact:
+        run__de_configs: "{{ run__de_configs | default([]) | union([config]) }}"
+      vars:
+        include: "{{ lookup('template', __de_config.include | default('experiences_config_placeholder.j2')) | from_yaml }}"
+        overlay_tags: "{{ __de_config.tags | default(include.tags) | default(run__de_tags) }}"
+        config:
+          name: "{{ __de_config.name | default([run__namespace_cdp, __de_config.suffix | default(include.suffix) | default(run__de_suffix)] | join('-')) }}"
+          raw: "{{ __de_config }}"
+      loop: "{{ run__de_definitions }}"
+      loop_control:
+        loop_var: __de_config
+        label: "{{ config.name }}"

roles/runtime/tasks/initialize_teardown.yml

@@ -68,4 +68,16 @@
       loop: "{{ run__env_info.environments[0].descendants.ml }}"
       loop_control:
         loop_var: __ml_config
-        label: "{{ __ml_config.instanceName }}"
+        label: "{{ __ml_config.instanceName }}"
+
+    - name: Prepare teardown list of all DE Services in Environment
+      ansible.builtin.set_fact:
+        run__de_configs: "{{ run__de_configs | default([]) | union([config]) }}"
+      vars:
+        config:
+          name: "{{ __de_config.name }}"
+          env: "{{ __de_config.environmentName }}"
+      loop: "{{ run__env_info.environments[0].descendants.de }}"
+      loop_control:
+        loop_var: __de_config
+        label: "{{ __de_config.name }}"

roles/runtime/tasks/setup_base.yml

@@ -59,6 +59,34 @@
   poll: 0
   register: __ml_builds
 
+- name: Execute CDP DE Service experiences setup
+  when: run__include_de
+  cloudera.cloud.de:
+    name: "{{ __de_config_item.name }}"
+    env: "{{ run__env_name }}"
+    instance_type: "{{ __de_config_item.raw.instance_type | default(run__de_compute_default[run__infra_type]) }}"
+    minimum_instances: "{{ __de_config_item.raw.minimum_instances | default(omit) }}"
+    maximum_instances: "{{ __de_config_item.raw.maximum_instances | default(omit) }}"
+    minimum_spot_instances: "{{ __de_config_item.raw.minimum_spot_instances | default(omit) }}"
+    maximum_spot_instances: "{{ __de_config_item.raw.maximum_spot_instances | default(omit) }}"
+    chart_value_overrides: "{{ __de_config_item.raw.chart_value_overrides | default(omit) }}"
+    enable_public_endpoint: "{{ __de_config_item.raw.enable_public_endpoint | default(omit) }}"
+    enable_workload_analytics: "{{ __de_config_item.raw.enable_workload_analytics | default(omit) }}"
+    initial_instances: "{{ __de_config_item.raw.initial_instances | default(omit) }}"
+    initial_spot_instances: "{{ __de_config_item.raw.initial_spot_instances | default(omit) }}"
+    root_volume_size: "{{ __de_config_item.raw.root_volume_size | default(omit) }}"
+    skip_validation: "{{ __de_config_item.raw.skip_validation | default(omit) }}"
+    tags: "{{ __de_config_item.raw.tags | default(omit) }}"
+    use_ssd: "{{ __de_config_item.raw.use_ssd | default(omit) }}"
+    whitelist_ips: "{{ __de_config_item.raw.whitelist_ips | default(omit) }}"
+  loop_control:
+    loop_var: __de_config_item
+    label: "{{ __de_config_item.name | default ('None') }}"
+  loop: "{{ run__de_configs }}"
+  async: 7200 # 2 hour timeout
+  poll: 0
+  register: __de_builds
+
 - name: Execute CDP OpDB setup
   when: run__include_opdb
   cloudera.cloud.opdb:
@@ -114,6 +142,19 @@
   retries: 120
   delay: 30
 
+- name: Wait for CDP DE Service experiences to complete
+  when: run__include_de
+  ansible.builtin.async_status:
+    jid: "{{ __de_build.ansible_job_id }}"
+  loop_control:
+    loop_var: __de_build
+    label: "{{ __de_build.__de_config_item.name | default ('None') }}"
+  loop: "{{ __de_builds.results }}"
+  register: __de_builds_async
+  until: __de_builds_async.finished
+  retries: 120
+  delay: 60
+
 - name: Wait for CDP OpDB deployments to complete
   when: run__include_opdb
   ansible.builtin.async_status:

roles/runtime/tasks/teardown_base.yml

@@ -80,6 +80,24 @@
   poll: 0
   register: __ml_teardowns_info
 
+- name: Execute CDP DE Service teardown
+  when:
+    - run__include_de or run__force_teardown | bool
+    - run__de_configs is defined
+    - run__de_configs | length > 0
+  cloudera.cloud.de:
+    name: "{{ __de_config_item.name }}"
+    env: "{{ run__env_name }}"
+    state: absent
+    force_delete: "{{ run__de_force_delete }}"
+  loop_control:
+    loop_var: __de_config_item
+    label: "{{ __de_config_item.name | default('de') }}"
+  loop: "{{ run__de_configs }}"
+  async: 7200 # 2 hour timeout
+  poll: 0
+  register: __de_teardowns_info
+
 - name: Execute CDP Datahub teardown
   when:
     - run__include_datahub or run__force_teardown | bool
@@ -114,6 +132,22 @@
   retries: 120
   delay: 30
 
+- name: Wait for CDP DE Service deployments to decommission
+  when:
+    - __de_teardowns_info is defined
+    - __de_teardowns_info.results is defined
+    - __de_teardowns_info.results | length > 0
+  ansible.builtin.async_status:
+    jid: "{{ __de_teardown.ansible_job_id }}"
+  loop_control:
+    loop_var: __de_teardown
+    label: "{{ __de_teardown.__de_config_item.name | default('de') }}"
+  loop: "{{ __de_teardowns_info.results }}"
+  register: __de_teardowns_async
+  until: __de_teardowns_async.finished
+  retries: 120
+  delay: 60
+
 - name: Wait for CDP Datahub deployments to decommission
   when:
     - __datahub_teardowns_info is defined
@@ -168,4 +202,4 @@
     wait: yes
   loop_control:
     loop_var: __df_teardown_wait_item
-  loop: "{{ run__df_service_info.services }}"
+  loop: "{{ run__df_service_info.services }}"