Skip to content

Commit 26ee4c8

Browse files
wmudgewmudge
authored
Add lookup plugins for DL and DH (#98)
* Add parse_environment, move parse_services (removing CdpServiceLookupModule) * Convert to use parse_services directly * Create datahub_definition lookup * Create datahub_template lookup * Create datahub_instance lookup * Create datalake_runtime lookup * Add note for OSX error Signed-off-by: Webster Mudge <[email protected]>
1 parent e02f279 commit 26ee4c8

File tree

7 files changed

+481
-24
lines changed

7 files changed

+481
-24
lines changed

‎plugins/lookup/cdp_service.py‎

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -12,25 +12,45 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15+
import re
16+
1517
from ansible.errors import AnsibleError
1618
from ansible.plugins.lookup import LookupBase
1719
from ansible.module_utils.common.text.converters import to_text, to_native
1820
from ansible.utils.display import Display
1921

22+
from cdpy.cdpy import Cdpy
23+
2024
display = Display()
25+
SEMVER = re.compile("(\d+\.[.\d]*\d+)")
26+
27+
def parse_services(terms:list, name:str, entity:dict, service:str, knox:bool, default:any):
28+
lookup = 'knoxService' if knox else 'serviceName'
29+
results = []
30+
try:
31+
for term in LookupBase._flatten(terms):
32+
display.vvv("%s_service lookup connecting to '%s[%s]'" % (service, name, term))
33+
services = [s for s in entity['endpoints']['endpoints'] if s[lookup] == term and 'serviceUrl' in s]
34+
if services:
35+
results.append([to_text(s['serviceUrl']) for s in services])
36+
else:
37+
results.append(default)
38+
return results
39+
except KeyError as e:
40+
raise AnsibleError("Error parsing result for '%s':'" % name, to_native(e))
41+
42+
def parse_environment(environment:str):
43+
env = Cdpy().datalake.describe_all_datalakes(environment)
44+
45+
if not env:
46+
raise AnsibleError("No Datalake found for Environment '%s'" % environment)
47+
elif len(env) > 1:
48+
raise AnsibleError("Multiple Datalakes found for Enviroment '%s'" % environment)
49+
50+
raw_version = env[0]['productVersions'][0]['version']
51+
52+
match = SEMVER.match(raw_version)
53+
if not match:
54+
raise AnsibleError("Unable to parse runtime version for Environment '%s': %s" % (environment, raw_version))
2155

22-
class CdpServiceLookupModule(LookupBase):
23-
def parse_services(self, terms:list, name:str, entity:dict, service:str):
24-
lookup = 'knoxService' if self.get_option('knox_service') else 'serviceName'
25-
results = []
26-
try:
27-
for term in LookupBase._flatten(terms):
28-
display.vvv("%s_service lookup connecting to '%s[%s]'" % (service, name, term))
29-
services = [s for s in entity['endpoints']['endpoints'] if s[lookup] == term and 'serviceUrl' in s]
30-
if services:
31-
results.append([to_text(s['serviceUrl']) for s in services])
32-
else:
33-
results.append(self.get_option('default'))
34-
return results
35-
except KeyError as e:
36-
raise AnsibleError("Error parsing result for '%s':'" % (name, to_native(e)))
56+
return env[0]['cloudPlatform'], raw_version, match.group(0)

‎plugins/lookup/datahub_definition.py‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
# Copyright 2023 Cloudera, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from __future__ import (absolute_import, division, print_function)
16+
__metaclass__ = type
17+
18+
DOCUMENTATION = '''
19+
lookup: datahub_definition
20+
author: Webster Mudge (@wmudge) <[email protected]>
21+
short_description: Get a Datahub definition for a CDP Public Cloud Environment
22+
description:
23+
- Allows you to retrieve the Datahub definition matching the Datalake CDH cloud platform and Runtime for one or more CDP Public Cloud Environments.
24+
- If an Environment is not found or is ambigious, the lookup will return an error.
25+
options:
26+
_terms:
27+
description:
28+
- A CDP Public Cloud Environment name
29+
type: string
30+
required: True
31+
definition:
32+
description:
33+
- Name (substring) of the Datahub definition to filter the resulting matches
34+
required: False
35+
type: string
36+
detailed:
37+
description:
38+
- Whether to return the full entry for the matching Datahub definitions
39+
required: False
40+
type: boolean
41+
default: False
42+
type:
43+
description:
44+
- Category of the Datahub definition to filter
45+
choices:
46+
- OPERATIONALDATABASE
47+
- FLOW_MANAGEMENT
48+
- DATAMART
49+
- DISCOVERY_DATA_AND_EXPLORATION
50+
- DATAENGINEERING
51+
- STREAMING
52+
- OTHER
53+
required: False
54+
type: string
55+
notes:
56+
- Requires C(cdpy).
57+
- If you encounter I(worker found in a dead state) and are running OSX, set the environment variable, C(OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES).
58+
seealso:
59+
- module: cloudera.cloud.datahub_definition_info
60+
description: Cloudera CDP Public Cloud Datahub definition module
61+
'''
62+
63+
EXAMPLES = '''
64+
- name: Retrieve the Datahub definition for a single CDP Public Cloud Environment
65+
ansible.builtin.debug:
66+
msg: "{{ query('cloudera.cloud.datahub_definition', 'example-env') }}"
67+
68+
- name: Retrieve the Datahub definition that match the given substring
69+
ansible.builtin.debug:
70+
msg: "{{ query('cloudera.cloud.datahub_definition', 'example-env', definition='Flow Management Light Duty') }}"
71+
72+
- name: Retrieve the only streaming Datahub definition
73+
ansible.builtin.debug:
74+
msg: "{{ query('cloudera.cloud.datahub_definition', 'example-env', type='STREAMING') }}"
75+
76+
- name: Retrieve the full details for the Datahub definition
77+
ansible.builtin.debug:
78+
msg: "{{ query('cloudera.cloud.datahub_definition', 'example-env', detailed=True) }}"
79+
80+
- name: Retrieve the Datahub definition details for multiple CDP Public Cloud Environments
81+
ansible.builtin.debug:
82+
msg: "{{ lookup('cloudera.cloud.datahub_definition', ['example-env', 'another-env'], wantlist=True) }}"
83+
'''
84+
85+
RETURN = '''
86+
_list:
87+
description: List of lists of Datahub definition
88+
type: list
89+
elements: complex
90+
'''
91+
92+
from ansible.errors import AnsibleError
93+
from ansible.plugins.lookup import LookupBase
94+
from ansible.module_utils.common.text.converters import to_native
95+
from ansible.utils.display import Display
96+
97+
from cdpy.cdpy import Cdpy
98+
from cdpy.common import CdpError
99+
100+
from ansible_collections.cloudera.cloud.plugins.lookup.cdp_service import parse_environment
101+
102+
display = Display()
103+
104+
class LookupModule(LookupBase):
105+
def run(self, terms, variables=None, **kwargs):
106+
self.set_options(var_options=variables, direct=kwargs)
107+
108+
try:
109+
all_definitions = Cdpy().datahub.list_cluster_definitions()
110+
results = []
111+
for term in terms:
112+
cloud_platform, raw_version, semantic_version = parse_environment(term)
113+
display.vvv("Filtering definitions for %s[%s][%s]" % (term, cloud_platform, semantic_version))
114+
115+
for d in all_definitions:
116+
if d['cloudPlatform'] == cloud_platform and d['productVersion'] == 'CDH %s' % semantic_version:
117+
if (self.get_option('type') is not None and self.get_option('type') != d['type']) or \
118+
(self.get_option('definition') is not None and self.get_option('definition') not in d['clusterDefinitionName']):
119+
continue
120+
results.append([d if self.get_option('detailed') else d['clusterDefinitionName']])
121+
return results
122+
except KeyError as e:
123+
raise AnsibleError("Error parsing result: %s" % to_native(e))
124+
except CdpError as e:
125+
raise AnsibleError("Error connecting to CDP: %s" % to_native(e))

‎plugins/lookup/datahub_instance.py‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
# Copyright 2023 Cloudera, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from __future__ import (absolute_import, division, print_function)
16+
__metaclass__ = type
17+
18+
DOCUMENTATION = '''
19+
lookup: datahub_instance
20+
author: Webster Mudge (@wmudge) <[email protected]>
21+
short_description: Get the instances for a CDP Public Cloud Datahub
22+
description:
23+
- Allows you to retrieve the instances by one or more instance groups for a CDP Public Cloud Datahub.
24+
- If the Datahub is not found or is ambigious, the lookup will return an error.
25+
- If the instance group is not found, the lookup will return the C(default) value.
26+
options:
27+
_terms:
28+
description:
29+
- Instance group name
30+
type: string
31+
required: True
32+
datahub:
33+
description:
34+
- Name of the Datahub
35+
required: True
36+
type: string
37+
detailed:
38+
description:
39+
- Whether to return the full entry for the matching Datahub instance group
40+
required: False
41+
type: boolean
42+
default: False
43+
default:
44+
description: What return when the instance group is not found on the Datahub
45+
type: any
46+
default: []
47+
notes:
48+
- Requires C(cdpy).
49+
seealso:
50+
- module: cloudera.cloud.datahub_cluster_info
51+
description: Cloudera CDP Public Cloud Datahub cluster module
52+
'''
53+
54+
EXAMPLES = '''
55+
- name: Retrieve the instances for the NiFi instance group for a CDP Public Cloud Flow Management datahub
56+
ansible.builtin.debug:
57+
msg: "{{ query('cloudera.cloud.datahub_instance', 'nifi', datahub='example-flow-dh') }}"
58+
59+
- name: Retrieve the full details for the instance
60+
ansible.builtin.debug:
61+
msg: "{{ query('cloudera.cloud.datahub_instance', 'nifi', datahub='example-flow-dh', detailed=True) }}"
62+
63+
- name: Retrieve the instance details for multiple instance groups
64+
ansible.builtin.debug:
65+
msg: "{{ lookup('cloudera.cloud.datahub_instance', ['nifi', 'management'], datahub='example-flow-dh', wantlist=True) }}"
66+
'''
67+
68+
RETURN = '''
69+
_list:
70+
description: List of lists of instances
71+
type: list
72+
elements: complex
73+
'''
74+
75+
from ansible.errors import AnsibleError
76+
from ansible.plugins.lookup import LookupBase
77+
from ansible.module_utils.common.text.converters import to_native
78+
from ansible.utils.display import Display
79+
80+
from cdpy.cdpy import Cdpy
81+
from cdpy.common import CdpError
82+
83+
from ansible_collections.cloudera.cloud.plugins.lookup.cdp_service import parse_environment
84+
85+
display = Display()
86+
87+
class LookupModule(LookupBase):
88+
def run(self, terms, variables=None, **kwargs):
89+
self.set_options(var_options=variables, direct=kwargs)
90+
91+
try:
92+
datahub = Cdpy().datahub.describe_cluster(self.get_option('datahub'))
93+
if datahub is None:
94+
raise AnsibleError("No Datahub found for '%s'" % self.get_option('datahub'))
95+
96+
all_instance_groups = {ig['name']:ig for ig in datahub['instanceGroups']}
97+
results = []
98+
99+
for term in LookupBase._flatten(terms):
100+
display.vvv("Filtering instance groups for %s[%s]" % (self.get_option('datahub'), term))
101+
if term in all_instance_groups:
102+
if self.get_option('detailed'):
103+
results.append(all_instance_groups[term]['instances'])
104+
else:
105+
results.append([i['fqdn'] for i in all_instance_groups[term]['instances']])
106+
else:
107+
results.append(self.get_option('default'))
108+
return results
109+
except KeyError as e:
110+
raise AnsibleError("Error parsing result: %s" % to_native(e))
111+
except CdpError as e:
112+
raise AnsibleError("Error connecting to CDP: %s" % to_native(e))

‎plugins/lookup/datahub_service.py‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -85,24 +85,22 @@
8585
'''
8686

8787
from ansible.errors import AnsibleError
88+
from ansible.plugins.lookup import LookupBase
8889
from ansible.module_utils.common.text.converters import to_native
8990

9091
from cdpy.cdpy import Cdpy
9192
from cdpy.common import CdpError
9293

93-
from ansible_collections.cloudera.cloud.plugins.lookup.cdp_service import CdpServiceLookupModule
94+
from ansible_collections.cloudera.cloud.plugins.lookup.cdp_service import parse_services
9495

9596

96-
class LookupModule(CdpServiceLookupModule):
97+
class LookupModule(LookupBase):
9798
def run(self, terms, variables=None, **kwargs):
9899
self.set_options(var_options=variables, direct=kwargs)
99-
100100
try:
101101
datahub = Cdpy().datahub.describe_cluster(self.get_option('datahub'))
102-
103102
if datahub is None:
104103
raise AnsibleError("No Datahub found for '%s'" % self.get_option('datahub'))
105-
106-
return self.parse_services(terms, self.get_option('datahub'), datahub, 'datahub')
104+
return parse_services(terms, self.get_option('datahub'), datahub, 'datahub', self.get_option('knox_service'), self.get_option('default'))
107105
except CdpError as e:
108106
raise AnsibleError("Error connecting to service '%s': %s" % (self.get_option('datahub'), to_native(e)))

0 commit comments

Comments
 (0)