diff --git a/README/inputs.conf.spec b/README/inputs.conf.spec index 868421f..4d80549 100644 --- a/README/inputs.conf.spec +++ b/README/inputs.conf.spec @@ -9,6 +9,8 @@ gitTempDir = * location where to store the output of the script on the filesystem (note this directory will be deleted/re-created but the parent dir must exist) gitRepoURL = * git repository URL to store the objects (SSH URL only) +sslVerify = +* Set to 'true' or 'false' to enable/disable SSL verification for REST requests to `srcUrl`. Set to a path to specify a file with valid CA. (https://2.python-requests.org/en/master/user/advanced/#ssl-cert-verification) noPrivate = * disable the backup of user level / private objects (true/false), default false noDisabled = @@ -47,6 +49,9 @@ gitTempDir = * location where to store the output of the script on the filesystem (note this directory will be deleted/re-created but the parent dir must exist) gitRepoURL = * git repository URL to store the objects (SSH URL only) +sslVerify = +* Set to 'true' or 'false' to enable/disable SSL verification for REST requests to `srcUrl`. Set to a path to specify a file with valid CA. (https://2.python-requests.org/en/master/user/advanced/#ssl-cert-verification) + auditLogsLookupBackTime = * This is how far back the audit logs will be checked to ensure that a restore entry is valid, this should be set to your interval time or slightly more, defaults to -1h (use Splunk format) debugMode = diff --git a/bin/postversioncontrolrestore.py b/bin/postversioncontrolrestore.py index 18afd92..f6ca563 100644 --- a/bin/postversioncontrolrestore.py +++ b/bin/postversioncontrolrestore.py @@ -10,8 +10,35 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib")) from splunklib.searchcommands import dispatch, GeneratingCommand, Configuration, Option +from splunklib.searchcommands.validators import Validator, Boolean from splunklib.binding import HTTPError +class OrValidator(Validator): + def __init__(self, a, b): + self.a = a + self.b = b + def __call__(self, value): + """Returns b if a raises an exception otherwise a.""" + try: + return self.a.__call__(value) + except ValueError: + return self.b.__call__(value) + + def format(self, value): + """Returns b if a raises an exception otherwise a.""" + try: + return self.a.format(value) + except: + return self.b.format(value) + +class Filename(Validator): + # TODO Validate file path + def __call__(self, value): + return value + + def format(self, value): + return value + splunkLogsDir = os.environ['SPLUNK_HOME'] + "/var/log/splunk" #Setup the logging logging_config = dict( @@ -60,6 +87,8 @@ class SVCPostRestore(GeneratingCommand): restoreAsUser = Option(require=True) scope = Option(require=True) timeout = Option(require=True) + sslVerify = Option(require=False, default=False, validate=OrValidator(Boolean(), Filename())) + requestingAddress = Option(require=False, default=False) def generate(self): """ @@ -87,6 +116,8 @@ def generate(self): body['restoreAsUser'] = self.restoreAsUser body['scope'] = self.scope body['timeout'] = self.timeout + if self.requestingAddress: + body['requestingAddress'] = self.requestingAddress logger.info("Attempting POST request to url=%s with body=\"%s\"" % (url, body)) @@ -94,7 +125,7 @@ def generate(self): logger.debug("Using token %s" % (body['Authorization'])) - attempt = requests.post(url, verify=False, data=body) + attempt = requests.post(url, verify=self.sslVerify, data=body) if attempt.status_code != 200: logger.error("POST request failed with status_code=%s, reason=%s, text=%s on url=%s" % (attempt.status_code, attempt.reason, attempt.text, url)) yield {'result': 'Unknown failure, received a non-200 response code of %s on the url %s, reason %s, text result is %s' % (attempt.status_code, url, attempt.reason, attempt.text)} diff --git a/bin/splunkversioncontrol_backup.py b/bin/splunkversioncontrol_backup.py index b795ac7..96479b2 100644 --- a/bin/splunkversioncontrol_backup.py +++ b/bin/splunkversioncontrol_backup.py @@ -47,6 +47,12 @@ gitRepoURL git repository URL to store the objects (SSH URL only) + + sslVerify + Set to 'true' or 'false' to enable/disable SSL verification for REST requests to `srcUrl`. Set to a path to specify a file with valid CA. (https://2.python-requests.org/en/master/user/advanced/#ssl-cert-verification) + is_bool('sslVerify') + false + noPrivate disable the backup of user level / private objects (true/false), default false @@ -203,10 +209,13 @@ def validate_arguments(): else: ssh_command = "ssh" + sslVerify = False + if 'sslVerify' in val_data: + sslVerify = val_data['sslVerify'] + #Run a sanity check and make sure we can connect into the remote Splunk instance if not useLocalAuth: url = val_data['srcURL'] + "/servicesNS/nobody/%s/search/jobs/export?search=makeresults" % (appName) - #Verify=false is hardcoded to workaround local SSL issues srcUsername = val_data['srcUsername'] srcPassword = val_data['srcPassword'] if srcPassword.find("password:") == 0: @@ -224,7 +233,7 @@ def validate_arguments(): try: logger.debug("Running query against URL %s with username %s proxies_length=%s" % (url, srcUsername, len(proxies))) - res = requests.get(url, auth=(srcUsername, srcPassword), verify=False, proxies=proxies) + res = requests.get(url, auth=(srcUsername, srcPassword), verify=self.sslVerify, proxies=proxies) logger.debug("End query against URL %s with username %s" % (url, srcUsername)) if (res.status_code != requests.codes.ok): diff --git a/bin/splunkversioncontrol_backup_class.py b/bin/splunkversioncontrol_backup_class.py index 74254be..b2a54ea 100644 --- a/bin/splunkversioncontrol_backup_class.py +++ b/bin/splunkversioncontrol_backup_class.py @@ -77,6 +77,7 @@ class SplunkVersionControlBackup: gitRepoURL = None stanzaName = None lastRunEpoch = None + sslVerify = False # read XML configuration passed from splunkd def get_config(self): @@ -136,8 +137,7 @@ def getAllAppsList(self): else: auth = HTTPBasicAuth(self.srcUsername, self.srcPassword) - #Verify=false is hardcoded to workaround local SSL issues - res = requests.get(url, auth=auth, headers=headers, verify=False, proxies=self.proxies) + res = requests.get(url, auth=auth, headers=headers, verify=self.sslVerify, proxies=self.proxies) if (res.status_code != requests.codes.ok): logger.fatal("i=\"%s\" Could not obtain a list of all apps, URL=%s statuscode=%s reason=%s, response=\"%s\"" % (self.stanzaName, url, res.status_code, res.reason, res.text)) sys.exit(-1) @@ -190,8 +190,7 @@ def runQueries(self, app, endpoint, type, fieldIgnoreList, aliasAttributes={}, v else: auth = HTTPBasicAuth(self.srcUsername, self.srcPassword) - #Verify=false is hardcoded to workaround local SSL issues - res = requests.get(url, auth=auth, headers=headers, verify=False, proxies=self.proxies) + res = requests.get(url, auth=auth, headers=headers, verify=self.sslVerify, proxies=self.proxies) if (res.status_code != requests.codes.ok): logger.error("i=\"%s\" URL=%s in app=%s statuscode=%s reason=%s response=\"%s\"" % (self.stanzaName, url, app, res.status_code, res.reason, res.text)) @@ -433,8 +432,7 @@ def macros(self, app): else: auth = HTTPBasicAuth(self.srcUsername, self.srcPassword) - #Verify=false is hardcoded to workaround local SSL issues - res = requests.get(url, auth=auth, headers=headers, verify=False, proxies=self.proxies) + res = requests.get(url, auth=auth, headers=headers, verify=self.sslVerify, proxies=self.proxies) if (res.status_code != requests.codes.ok): logger.error("i=\"%s\" Type macro in app=%s, URL=%s statuscode=%s reason=%s, response=\"%s\"" % (self.stanzaName, app, url, res.status_code, res.reason, res.text)) @@ -749,7 +747,7 @@ def runSearchJob(self, query): headers = {'Authorization': 'Splunk %s' % self.session_key } else: auth = HTTPBasicAuth(self.srcUsername, self.srcPassword) - res = requests.post(url, auth=auth, headers=headers, verify=False, data=data, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=data, proxies=self.proxies) if (res.status_code != requests.codes.ok): logger.error("i=\"%s\" URL=%s statuscode=%s reason=%s response=\"%s\"" % (self.stanzaName, url, res.status_code, res.reason, res.text)) res = json.loads(res.text) @@ -1022,6 +1020,9 @@ def run_script(self): self.proxies = proxies + if 'sslVerify' in config: + self.sslVerify = config['sslVerify'] + #From server self.splunk_rest = config['srcURL'] excludedList = [ "srcPassword", "session_key" ] @@ -1035,7 +1036,6 @@ def run_script(self): headers={'Authorization': 'Splunk %s' % config['session_key']} - #Verify=false is hardcoded to workaround local SSL issues url = 'https://localhost:8089/services/shcluster/captain/info?output_mode=json' res = requests.get(url, headers=headers, verify=False) if (res.status_code == 503): diff --git a/bin/splunkversioncontrol_rest_restore.py b/bin/splunkversioncontrol_rest_restore.py index ac2563d..ce9eb35 100644 --- a/bin/splunkversioncontrol_rest_restore.py +++ b/bin/splunkversioncontrol_rest_restore.py @@ -59,6 +59,29 @@ class SVCRestore(splunk.rest.BaseRestHandler): + def query_back_for_user_and_permissions(self, requestingAddress, authorization_token, sslVerify): + headers = { "Authorization" : authorization_token } + + #Run a query back against the source system to check the username/role + if requestingAddress: + remoteAddr = requestingAddress + else: + remoteAddr = "https://" + self.request['remoteAddr'] + ":8089" + url = remoteAddr + "/services/authentication/current-context?output_mode=json" + logger.info("Received remote request checking username and role related to the token on url=%s" % (url)) + logger.debug("token=%s" % (authorization_token)) + + res = self.runHttpRequest(url, headers, None, "get", "checking username with token (based on incoming ip address)", sslVerify=sslVerify) + if not res: + return + + json_dict = json.loads(res.text) + #self.response.write(str(json_dict) + "\n\n\n") + username = json_dict['entry'][0]['content']['username'] + roles = json_dict['entry'][0]['content']['roles'] + return username, roles + + def handle_POST(self): starttime = calendar.timegm(time.gmtime()) payload = six.moves.urllib.parse.parse_qs(self.request['payload']) @@ -75,26 +98,6 @@ def handle_POST(self): self.response.write("Received remote call but attr=%s was missing from arguments, received=\"%s\"" % (attr, payload)) return - headers = { "Authorization" : payload['Authorization'][0] } - - #Run a query back against the source system to check the username/role - remoteAddr = self.request['remoteAddr'] - url = "https://" + remoteAddr + ":8089/services/authentication/current-context?output_mode=json" - logger.info("Received remote request checking username and role related to the token on url=%s" % (url)) - logger.debug("token=%s" % (payload['Authorization'][0])) - - res = self.runHttpRequest(url, headers, None, "get", "checking username with token (based on incoming ip address)") - if not res: - return - - json_dict = json.loads(res.text) - #self.response.write(str(json_dict) + "\n\n\n") - username = json_dict['entry'][0]['content']['username'] - roles = json_dict['entry'][0]['content']['roles'] - - #self.response.write(username + "\n") - #self.response.write(str(roles) + "\n") - logger.info("username=%s roles=%s" % (username, roles)) splunk_vc_name = payload['splunk_vc_name'][0] #self.response.write(splunk_vc_name + "\n\n\n") @@ -104,7 +107,7 @@ def handle_POST(self): url = "https://localhost:8089/servicesNS/-/-/data/inputs/splunkversioncontrol_restore/" + six.moves.urllib.parse.quote(splunk_vc_name) + "?output_mode=json" logger.debug("Now running query against url=%s to obtain config information" % (url)) - res = self.runHttpRequest(url, headers, None, "get", "querying the inputs for splunkversioncontrol_restore with name %s" % (splunk_vc_name)) + res = self.runHttpRequest(url, headers, None, "get", "querying the inputs for splunkversioncontrol_restore with name %s" % (splunk_vc_name), sslVerify=False) if not res: return @@ -134,6 +137,10 @@ def handle_POST(self): destURL = json_dict['destURL'] + sslVerify = False + if 'sslVerify' in json_dict: + sslVerify = json_dict['sslVerify'] + headers = {} auth = None @@ -156,6 +163,14 @@ def handle_POST(self): else: time_wait = 600 + if 'requestingAddress' in payload: + requestingAddress = payload['requestingAddress'][0] + else: + requestingAddress = None + + username, roles = self.query_back_for_user_and_permissions(requestingAddress, payload['Authorization'][0], sslVerify=sslVerify) + logger.info("username=%s roles=%s" % (username, roles)) + app = payload['app'][0] type = payload['type'][0] obj_name = payload['obj_name'][0] @@ -178,7 +193,7 @@ def handle_POST(self): starttime = starttime-60-timeout - json_res = self.runSearchJob(destURL, remoteAppName, headers, auth, username, starttime) + json_res = self.runSearchJob(destURL, remoteAppName, headers, auth, username, starttime, sslVerify=sslVerify) if 'error' in json_res: self.response.write("An error occurred: %s" % (json_res['error'])) @@ -200,7 +215,7 @@ def handle_POST(self): headers = { "Authorization" : "Splunk " + self.request['systemAuth'] } curtime = calendar.timegm(time.gmtime()) url = "https://localhost:8089/servicesNS/nobody/SplunkVersionControl/storage/collections/data/splunkversioncontrol_rest_restore_status" - res = self.runHttpRequest(url, headers, None, "get", "checking kvstore collection splunkversioncontrol_rest_restore_status") + res = self.runHttpRequest(url, headers, None, "get", "checking kvstore collection splunkversioncontrol_rest_restore_status", sslVerify=False) if not res: return @@ -209,14 +224,14 @@ def handle_POST(self): if not len(res) == 0: if not 'start_time' in res[0]: logger.warn("Warning invalid kvstore data, will wipe it and continue in collection splunkversioncontrol_rest_restore_status on url=%s, value returned res=\"%s\"" % (url, payload)) - self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status') + self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status', sslVerify=False) else: kvstore_start_time = res[0]['start_time'] target_time = curtime - time_wait if kvstore_start_time < target_time: logger.warn("Found existing entry from %s but time is %s, this is past the limit of current time minus %s (%s)" % (kvstore_start_time, curtime, time_wait, target_time)) #More than 10 minutes ago, delete the entry and move on - self.runHttpRequest(url, headers, None, "delete", "wiping kvstore splunkversioncontrol_rest_restore_status due to record %s older than %s time period" % (kvstore_start_time, target_time)) + self.runHttpRequest(url, headers, None, "delete", "wiping kvstore splunkversioncontrol_rest_restore_status due to record %s older than %s time period" % (kvstore_start_time, target_time), sslVerify=False) else: removal_target = kvstore_start_time + time_wait + 1 logger.warn("Attempted to run but found a running restore instance with time=%s and current_time=%s, will delete and move on after current_time_minus=%s seconds (override_time=%s)" % (kvstore_start_time, curtime, time_wait, removal_target)) @@ -227,7 +242,7 @@ def handle_POST(self): payload = json.dumps({ 'start_time': curtime }) headers['Content-Type'] = 'application/json' #update kvstore with runtime - res = self.runHttpRequest(url, headers, payload, 'post', 'updating kvstore collection splunkversioncontrol_rest_restore_status') + res = self.runHttpRequest(url, headers, payload, 'post', 'updating kvstore collection splunkversioncontrol_rest_restore_status', sslVerify=False) if not res: return res @@ -239,16 +254,16 @@ def handle_POST(self): self.response.write("Restore has failed to complete successfully in app %s, object of type %s, with name %s, from tag %s, scope %s with restoreAsUser %s and your username of %s. Message is %s" % (app, type, obj_name, tag, scope, restoreAsUser, username, message)) logger.warn("Restore has failed to complete successfully in app=%s, object of type=%s, with name=%s, from tag=%s, scope=%s with restoreAsUser=%s and requested by username=%s, message=%s" % (app, type, obj_name, tag, scope, restoreAsUser, username, message)) - self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status after completed run') + self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status after completed run', sslVerify=False) #Run a Splunk query via the search/jobs endpoint - def runSearchJob(self, url, appname, headers, auth, username, earliest_time): + def runSearchJob(self, url, appname, headers, auth, username, earliest_time, sslVerify=False): url = url + "/servicesNS/-/%s/search/jobs" % (appname) query = "savedsearch \"Splunk Version Control Audit Query POST\" username=\"%s\" | stats count | where count>0" % (username) logger.debug("Running requests.post() on url=%s query=\"%s\"" % (url, query)) data = { "search" : query, "output_mode" : "json", "exec_mode" : "oneshot", "earliest_time" : earliest_time } - res = requests.post(url, auth=auth, headers=headers, verify=False, data=data) + res = requests.post(url, auth=auth, headers=headers, verify=sslVerify, data=data) if (res.status_code != requests.codes.ok): logger.error("url=%s status_code=%s reason=%s, response=\"%s\"" % (url, res.status_code, res.reason, res.text)) return { "error": "url=%s status_code=%s reason=%s, response=\"%s\"" % (url, res.status_code, res.reason, res.text) } @@ -264,17 +279,17 @@ def runSearchJob(self, url, appname, headers, auth, username, earliest_time): logger.warn("messages from query=\"%s\" were messages=\"%s\"" % (query, res["messages"])) return res - def runHttpRequest(self, url, headers, data, type, text): + def runHttpRequest(self, url, headers, data, type, text, sslVerify=False): if type == "delete": - res = requests.delete(url, headers=headers, verify=False) + res = requests.delete(url, headers=headers, verify=sslVerify) elif type == "post": - res = requests.post(url, headers=headers, verify=False, data=data) + res = requests.post(url, headers=headers, verify=sslVerify, data=data) elif type == "get": - res = requests.get(url, headers=headers, verify=False) + res = requests.get(url, headers=headers, verify=sslVerify) if (res.status_code != requests.codes.ok and res.status_code != 201): - logger.error("Unexpected response code while %s, on url=%s, statuscode=%s reason=%s, response=\"%s\", payload=\"%s\"" % (text, url, res.status_code, res.reason, res.text, data)) - self.response.write("Error unexpected response code while %s, on url %s, statuscode %s reason %s, response \"%s\", payload=\"%s\"" % (text, url, res.status_code, res.reason, res.text, data)) + logger.error("Unexpected response code while %s, on url=%s, statuscode=%s reason=%s, response=\"%s\", payload=\"%s\", verify=\"%s\"" % (text, url, res.status_code, res.reason, res.text, data, sslVerify)) + self.response.write("Error unexpected response code while %s, on url %s, statuscode %s reason %s, response \"%s\", payload=\"%s\", verify=\"%s\"" % (text, url, res.status_code, res.reason, res.text, data, sslVerify)) return return res diff --git a/bin/splunkversioncontrol_restore.py b/bin/splunkversioncontrol_restore.py index 0e31ef7..6c490c1 100644 --- a/bin/splunkversioncontrol_restore.py +++ b/bin/splunkversioncontrol_restore.py @@ -49,6 +49,13 @@ gitRepoURL git repository URL to store the objects (SSH URL only) + + sslVerify + Set to 'true' or 'false' to enable/disable SSL verification for REST requests to `srcUrl`. Set to a path to specify a file with valid CA. (https://2.python-requests.org/en/master/user/advanced/#ssl-cert-verification) + is_bool('sslVerify') + false + + auditLogsLookupBackTime This is how far back the audit logs will be checked to ensure that a restore entry is valid, this should be set to your interval time or slightly more, defaults to -1h (use Splunk format) @@ -176,12 +183,15 @@ def validate_arguments(): print_error("Unable to convert timeout field to a valid value, this must be an integer value in seconds, value provided was %s" % (val_data['timewait'])) sys.exit(1) + sslVerify = False + if 'sslVerify' in val_data: + sslVerify = val_data['sslVerify'] + session_key = val_data['session_key'] #Run a sanity check and make sure we can connect into the remote Splunk instance if not useLocalAuth: url = val_data['destURL'] + "/servicesNS/nobody/%s/search/jobs/export?search=makeresults" % (appName) - #Verify=false is hardcoded to workaround local SSL issues destUsername = val_data['destUsername'] destPassword = val_data['destPassword'] @@ -199,7 +209,7 @@ def validate_arguments(): try: logger.debug("Running query against URL %s with username %s proxies_length=%s" % (url, destUsername, len(proxies))) - res = requests.get(url, auth=(destUsername, destPassword), verify=False, proxies=proxies) + res = requests.get(url, auth=(destUsername, destPassword), verify=sslVerify, proxies=proxies) logger.debug("End query against URL %s with username %s" % (url, destUsername)) if (res.status_code != requests.codes.ok): print_error("Attempt to validate access to Splunk failed with code %s, reason %s, text %s on URL %s" % (res.status_code, res.reason, res.text, url)) diff --git a/bin/splunkversioncontrol_restore_class.py b/bin/splunkversioncontrol_restore_class.py index 5ab04ad..7cce8ff 100644 --- a/bin/splunkversioncontrol_restore_class.py +++ b/bin/splunkversioncontrol_restore_class.py @@ -69,6 +69,7 @@ class SplunkVersionControlRestore: appName = "SplunkVersionControl" gitRepoURL = None stanzaName = None + sslVerify = False # read XML configuration passed from splunkd def get_config(self): @@ -161,7 +162,7 @@ def runQueries(self, app, endpoint, type, name, scope, user, restoreAsUser, admi res_result = False #Verify=false is hardcoded to workaround local SSL issues - res = requests.get(url, auth=auth, headers=headers, verify=False, proxies=self.proxies) + res = requests.get(url, auth=auth, headers=headers, verify=self.sslVerify, proxies=self.proxies) objExists = False #If we get 404 it definitely does not exist or it has a name override @@ -348,7 +349,7 @@ def runRestore(self, config, type, endpoint, app, name, user, restoreAsUser, adm objExistsURL = "%s/%s?output_mode=json" % (url, origName) logger.debug("i=\"%s\" URL=%s re-checking object exists URL due to name override from %s to original name of %s proxies_length=%s" % (self.stanzaName, objExistsURL, name, origName, len(self.proxies))) #Verify=false is hardcoded to workaround local SSL issues - res = requests.get(objExistsURL, auth=auth, headers=headers, verify=False, proxies=self.proxies) + res = requests.get(objExistsURL, auth=auth, headers=headers, verify=self.sslVerify, proxies=self.proxies) #If we get 404 it definitely does not exist or it has a name override if (res.status_code == 404): @@ -399,14 +400,14 @@ def runRestore(self, config, type, endpoint, app, name, user, restoreAsUser, adm del payload['disabled'] logger.debug("i=\"%s\" Attempting to %s type=%s with name=%s on URL=%s with payload=\"%s\" in app=%s proxies_length=%s" % (self.stanzaName, createOrUpdate, type, name, url, payload, app, len(self.proxies))) - res = requests.post(url, auth=auth, headers=headers, verify=False, data=payload, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=payload, proxies=self.proxies) if (res.status_code != requests.codes.ok and res.status_code != 201): logger.error("i=\"%s\" user=%s, name=%s of type=%s with URL=%s statuscode=%s reason=%s, response=\"%s\", in app=%s, owner=%s" % (self.stanzaName, user, name, type, url, res.status_code, res.reason, res.text, app, owner)) #Saved Searches sometimes fail due to the VSID field, auto-retry in case that solves the problem... if type=="savedsearches": if 'vsid' in payload: del payload['vsid'] - res = requests.post(url, auth=auth, headers=headers, verify=False, data=payload, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=payload, proxies=self.proxies) if (res.status_code != requests.codes.ok and res.status_code != 201): logger.error("i=\"%s\" user=%s, re-attempted without vsid but result for name=%s of type=%s with URL=%s statuscode=%s reason=%s, response=\"%s\", in app=%s, owner=%s" % (self.stanzaName, user, name, type, url, res.status_code, res.reason, res.text, app, owner)) result = False @@ -442,7 +443,7 @@ def runRestore(self, config, type, endpoint, app, name, user, restoreAsUser, adm url = "%s/acl" % (objURL) payload = { "owner": owner, "sharing" : sharing } logger.info("i=\"%s\" Attempting to change ownership of type=%s with name=%s via URL=%s to owner=%s in app=%s with sharing=%s" % (self.stanzaName, type, name, url, owner, app, sharing)) - res = requests.post(url, auth=auth, headers=headers, verify=False, data=payload, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=payload, proxies=self.proxies) #If re-own fails log this for investigation if (res.status_code != requests.codes.ok): @@ -502,7 +503,7 @@ def runRestoreMacro(self, config, app, name, username, restoreAsUser, adminLevel #servicesNS/-/search/properties/macros #__stanza = - res = requests.post(url, auth=auth, headers=headers, verify=False, data=payload, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=payload, proxies=self.proxies) if (res.status_code != requests.codes.ok and res.status_code != 201): message = "name=%s of type=macro in app=%s with URL=%s statuscode=%s reason=%s, response=\"%s\", owner=%s" % (name, app, url, res.status_code, res.reason, res.text, owner) logger.error("i=\"" + self.stanzaName + "\"" + message) @@ -529,7 +530,7 @@ def runRestoreMacro(self, config, app, name, username, restoreAsUser, adminLevel payload = config logger.debug("i=\"%s\" Attempting to modify type=macro name=%s on URL=%s with payload=\"%s\" in app=%s proxies_length=%s" % (self.stanzaName, name, url, payload, app, len(self.proxies))) - res = requests.post(url, auth=auth, headers=headers, verify=False, data=payload, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=payload, proxies=self.proxies) if (res.status_code != requests.codes.ok and res.status_code != 201): logger.error("i=\"%s\" name=%s of type=macro in app=%s with URL=%s statuscode=%s reason=%s, response=\"%s\"" % (self.stanzaName, name, app, url, res.status_code, res.reason, res.text)) result = False @@ -538,7 +539,7 @@ def runRestoreMacro(self, config, app, name, username, restoreAsUser, adminLevel url = "%s/servicesNS/%s/%s/configs/conf-macros/%s/acl" % (self.splunk_rest, owner, app, name) payload = { "owner": owner, "sharing" : sharing } logger.info("i=\"%s\" Attempting to change ownership of type=macro name=%s via URL=%s to owner=%s in app=%s with sharing=%s" % (self.stanzaName, name, url, owner, app, sharing)) - res = requests.post(url, auth=auth, headers=headers, verify=False, data=payload, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=payload, proxies=self.proxies) if (res.status_code != requests.codes.ok): logger.error("i=\"%s\" name=%s of type=macro in app=%s with URL=%s statuscode=%s reason=%s, response=\"%s\", owner=%s sharing=%s" % (self.stanzaName, name, app, url, res.status_code, res.reason, res.text, owner, sharing)) else: @@ -581,7 +582,7 @@ def macros(self, app, name, scope, user, restoreAsUser, adminLevel): auth = HTTPBasicAuth(self.destUsername, self.destPassword) #Verify=false is hardcoded to workaround local SSL issues - res = requests.get(url, auth=auth, headers=headers, verify=False, proxies=self.proxies) + res = requests.get(url, auth=auth, headers=headers, verify=self.sslVerify, proxies=self.proxies) objExists = False if (res.status_code == 404): logger.debug("i=\"%s\" URL=%s is throwing a 404, assuming new object creation" % (self.stanzaName, url)) @@ -843,7 +844,7 @@ def runSearchJob(self, query, earliest_time="-1h"): else: auth = HTTPBasicAuth(self.destUsername, self.destPassword) - res = requests.post(url, auth=auth, headers=headers, verify=False, data=data, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=data, proxies=self.proxies) if (res.status_code != requests.codes.ok): logger.error("i=\"%s\" URL=%s statuscode=%s reason=%s, response=\"%s\"" % (self.stanzaName, url, res.status_code, res.reason, res.text)) res = json.loads(res.text) @@ -949,6 +950,9 @@ def run_script(self, restlist_override=None, config_override=None): self.proxies = proxies + if 'sslVerify' in config: + self.sslVerify = config['sslVerify'] + dirExists = os.path.isdir(self.gitTempDir) if dirExists and len(os.listdir(self.gitTempDir)) != 0: if not ".git" in os.listdir(self.gitTempDir): diff --git a/bin/splunkversioncontrol_utility.py b/bin/splunkversioncontrol_utility.py index bed452a..97818df 100644 --- a/bin/splunkversioncontrol_utility.py +++ b/bin/splunkversioncontrol_utility.py @@ -36,7 +36,7 @@ def runOSProcess(command, logger, timeout=60, shell=False): return str(stdout), str(stderr), res # use the password endpoint to obtain the clear_password passed in, start with the context of this app and then try all contexts -def get_password(password, session_key, logger): +def get_password(password, session_key, logger, sslVerify=False): #TODO move this into shared function to obtain passwords: context = os.path.dirname(os.path.dirname(__file__)) diff --git a/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml b/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml index 977a12a..839c909 100644 --- a/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml +++ b/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml @@ -129,7 +129,7 @@ SplunkVersionControl Results (please wait a few minutes for results) - | postversioncontrolrestore app="$app$" obj_name="$name$" restoreAsUser="$restoreAsUser$" scope="$scope$" splunk_vc_name=`splunk_vc_name` tag="$tag$" url=`splunk_vc_url` type="$type$" timeout=`splunk_vc_timeout` + | postversioncontrolrestore app="$app$" obj_name="$name$" restoreAsUser="$restoreAsUser$" scope="$scope$" splunk_vc_name=`splunk_vc_name` tag="$tag$" url=`splunk_vc_url` type="$type$" timeout=`splunk_vc_timeout` sslVerify=`sslVerify` requestingAddress=`requestingAddress` -5m now 1 diff --git a/default/macros.conf b/default/macros.conf index fe69706..31c7670 100644 --- a/default/macros.conf +++ b/default/macros.conf @@ -18,3 +18,12 @@ iseval = 0 [splunk_vc_timeout] definition = 30 iseval = 0 + +[sslVerify] +definition = false +iseval = 0 + +[requestingAddress] +definition = false +iseval = 0 + diff --git a/test/bkp_default.yml b/test/bkp_default.yml new file mode 100644 index 0000000..8876445 --- /dev/null +++ b/test/bkp_default.yml @@ -0,0 +1,38 @@ +--- +#ansible_post_tasks: file:///tmp/install_git.yml +splunk: + conf: + - key: inputs + value: + directory: /opt/splunk/etc/apps/SplunkVersionControl/local + content: + splunkversioncontrol_backup://SH: + srcURL: https://splunk_sh:8089 + srcUsername: admin + srcPassword: g8nGXBQBF + gitTempDir: /opt/splunk/vcs/git_tmp_backup + gitRepoURL: file:///opt/splunk/vcs/backup.git + sslVerify: /cert_dir/ca.crt + debugMode: true + useLocalAuth: false + interval: 60 + splunkversioncontrol_restore://SH: + destURL: https://splunk_sh:8089 + destUsername: admin + destPassword: g8nGXBQBF + gitTempDir: /opt/splunk/vcs/git_tmp_restore + gitRepoURL: file:///opt/splunk/vcs/backup.git + sslVerify: /cert_dir/ca.crt + debugMode: true + useLocalAuth: false + auditLogsLookupBackTime: -2h + timewait: 30 + - key: server + value: + directory: /opt/splunk/etc/system/local/ + content: + sslConfig: + enableSplunkdSSL: true + serverCert: /cert_dir/splunk_moc.pem + sslRootCAPath: /cert_dir/ca.crt + sslPassword: password diff --git a/test/certificates/.gitignore b/test/certificates/.gitignore new file mode 100644 index 0000000..423a60f --- /dev/null +++ b/test/certificates/.gitignore @@ -0,0 +1,5 @@ +*.crt +*.key +*.pem +*.csr +castuff diff --git a/test/certificates/Dockerfile b/test/certificates/Dockerfile new file mode 100644 index 0000000..411a7c2 --- /dev/null +++ b/test/certificates/Dockerfile @@ -0,0 +1,9 @@ +FROM alpine:latest + +RUN apk update && \ + apk add --no-cache openssl bash && \ + rm -rf "/var/cache/apk/*" + +COPY createca.sh createcerts.sh entrypoint.sh /scripts/ + +ENTRYPOINT [ "bash", "/scripts/entrypoint.sh" ] diff --git a/test/certificates/createca.sh b/test/certificates/createca.sh new file mode 100644 index 0000000..9491f1d --- /dev/null +++ b/test/certificates/createca.sh @@ -0,0 +1,13 @@ +set -e +CA_KEY=ca.key +CA_CRT=ca.crt +SUBJ="/C=CH/ST=Bern/L=Bern/O=Example Company/CN=CA Domain1" + +if [ ! -f $CA_KEY ]; then + openssl genrsa -out $CA_KEY 4096 +fi + +if [ ! -f $CA_CRT ]; then + openssl req -new -x509 -key ${CA_KEY} -out ${CA_CRT} -subj "$SUBJ" -days 3000 +fi + diff --git a/test/certificates/createcerts.sh b/test/certificates/createcerts.sh new file mode 100644 index 0000000..3c0a2cc --- /dev/null +++ b/test/certificates/createcerts.sh @@ -0,0 +1,130 @@ +set -e + +if [ $# -ne 3 ] + then + cat < $dir/serial +[ ! -f $dir/index.txt ] && touch $dir/index.txt +# SIGN Request +openssl ca \ + -batch \ + -in ${CSR} \ + -out ${CERT} \ + -extensions ${CRT_TYPE} \ + -config <( +cat <<-EOF +[ ca ] +default_ca = CA_default +[ CA_default ] +copy_extensions = copy +dir = . +certs = $dir/certs +crl_dir = $dir/crl +new_certs_dir = $dir/newcerts +database = $dir/index.txt +serial = $dir/serial +RANDFILE = $dir/private/.rand + +private_key = ./ca.key +certificate = ./ca.crt + +crlnumber = $dir/crlnumber +crl = $dir/crl/ca.crl.pem +crl_extensions = crl_ext +default_crl_days = 30 + +# SHA-1 is deprecated, so use SHA-2 instead. +default_md = sha256 + +name_opt = ca_default +cert_opt = ca_default +default_days = 375 +preserve = no +policy = policy_loose +unique_subject = no + +[ policy_loose ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +[ usr_cert ] +basicConstraints = CA:FALSE +nsCertType = client +nsComment = "Client Certificate" +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer +keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, emailProtection + +[ server_cert ] +basicConstraints = CA:FALSE +nsCertType = server +nsComment = "OpenSSL Generated Server Certificate" +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer:always +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth + +EOF +) + +cat ${CERT} ${KEYOUT} > ${COMBINED} +#rm ${CERT} ${CSR} ${KEYOUT} diff --git a/test/certificates/entrypoint.sh b/test/certificates/entrypoint.sh new file mode 100755 index 0000000..ede1288 --- /dev/null +++ b/test/certificates/entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -e +echo "Starting Certificate Creation" +mkdir -p /cert_dir +cd /cert_dir +sh /scripts/createca.sh +while read -r line; do + # Get the string before = (the var name) + name="${line%=*}" + eval value="\$$name" + if [[ $name = 'CERTIFICATE'* ]] + then + echo "name: ${name}, value: ${value}" + bash /scripts/createcerts.sh ${value} + fi +done <