diff --git a/.flake8 b/.flake8 index 1e7127dc..87a7d626 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,6 @@ [flake8] -ignore = E501, F821 +ignore = E501, F821, W504, W605, E303 show-source = True - +count = True +statistics = True +max-line-length=127 \ No newline at end of file diff --git a/.github/workflows/ci-pull-request.yml b/.github/workflows/ci-pull-request.yml index e7aabeca..6eb28e1c 100644 --- a/.github/workflows/ci-pull-request.yml +++ b/.github/workflows/ci-pull-request.yml @@ -42,10 +42,9 @@ jobs: run: poetry install - name: Lint - continue-on-error: true run: | # stop the build if there are Python syntax errors or undefined names - poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + poetry run flake8 # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics diff --git a/doc/conf.py b/doc/conf.py index d17cadb3..2c6e609e 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -18,8 +18,8 @@ # import os import sys -sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath('..')) # -- General configuration ------------------------------------------------ @@ -30,7 +30,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.linkcode' ] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.linkcode'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -76,7 +76,6 @@ # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False - # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for @@ -95,13 +94,11 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] - # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. htmlhelp_basename = 'python-sdc-clientdoc' - # -- Options for LaTeX output --------------------------------------------- latex_elements = { @@ -130,7 +127,6 @@ u'Sysdig Inc.', 'manual'), ] - # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples @@ -140,7 +136,6 @@ [author], 1) ] - # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples @@ -159,13 +154,12 @@ def find_line(): for part in info['fullname'].split('.'): obj = getattr(obj, part) import inspect - fn = inspect.getsourcefile(obj) source, lineno = inspect.findsource(obj) return lineno + 1 if domain != 'py' or not info['module']: return None - #tag = 'master' if 'dev' in release else ('v' + release) + # tag = 'master' if 'dev' in release else ('v' + release) url = "https://github.com/draios/python-sdc-client/blob/master/sdcclient/_client.py" try: return url + '#L%d' % find_line() diff --git a/examples/dashboard_backup_v1_restore_v2.py b/examples/dashboard_backup_v1_restore_v2.py index f33d9ba0..eb9f7ef5 100755 --- a/examples/dashboard_backup_v1_restore_v2.py +++ b/examples/dashboard_backup_v1_restore_v2.py @@ -12,12 +12,8 @@ # Parse arguments # if len(sys.argv) != 5: - print(( - 'usage: %s ' - % sys.argv[0])) - print( - 'You can find your token at https://app.sysdigcloud.com/#/settings/user' - ) + print(f'usage: {sys.argv[0]} ') + print('You can find your token at https://app.sysdigcloud.com/#/settings/user') sys.exit(1) sdc_v1_url = sys.argv[1] diff --git a/examples/delete_event.py b/examples/delete_event.py index 44b6fdab..898315f5 100755 --- a/examples/delete_event.py +++ b/examples/delete_event.py @@ -6,12 +6,12 @@ import getopt import sys -from sdcclient import SdcClient, SdMonitorClient +from sdcclient import SdMonitorClient + # # Parse arguments # -from sdcclient.monitor import EventsClientV2 def usage(): diff --git a/examples/get_data_simple.py b/examples/get_data_simple.py index 2da2ffb6..3a9fabad 100755 --- a/examples/get_data_simple.py +++ b/examples/get_data_simple.py @@ -92,9 +92,9 @@ # # Print table headers # - dataToPrint = ' '.join([str(x['id']).ljust(colLen) if len(str(x['id'])) < colLen else str(x['id'])[ - :(colLen - 3)].ljust( - colLen - 3) + '...' for x in metrics]) + dataToPrint = ' '.join( + [str(x['id']).ljust(colLen) if len(str(x['id'])) < colLen else str(x['id'])[:(colLen - 3)].ljust( + colLen - 3) + '...' for x in metrics]) print(('%s %s' % ('timestamp'.ljust(colLen), dataToPrint) if sampling > 0 else dataToPrint)) print('') diff --git a/examples/get_secure_policy_events_old.py b/examples/get_secure_policy_events_old.py index b6250b78..5799256f 100755 --- a/examples/get_secure_policy_events_old.py +++ b/examples/get_secure_policy_events_old.py @@ -14,18 +14,18 @@ # UNSUPPORTED: This script is unsupported as it is only an example from an old API version. # =========================================================================================== -import os -import sys +import getopt import json import operator import re -import getopt -sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) +import sys + from sdcclient.secure import PolicyEventsClientOld def usage(): - print('usage: %s [-s|--summarize] [-l|--limit ] [| ]' % sys.argv[0]) + print('usage: %s [-s|--summarize] [-l|--limit ] [| ]' % + sys.argv[0]) print('-s|--summarize: group policy events by sanitized output and print by frequency') print('-l|--limit: with -s, only print the first outputs') print('You can find your token at https://secure.sysdig.com/#/settings/user') diff --git a/examples/list_access_keys.py b/examples/list_access_keys.py index 36139a77..9af0cdfa 100755 --- a/examples/list_access_keys.py +++ b/examples/list_access_keys.py @@ -4,9 +4,8 @@ # have Admin rights. # -import os import sys -sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) + from sdcclient import SdcClient # diff --git a/examples/restore_alerts.py b/examples/restore_alerts.py index fe0ad0fc..a50966a7 100755 --- a/examples/restore_alerts.py +++ b/examples/restore_alerts.py @@ -87,5 +87,5 @@ print(res) sys.exit(1) -print(('All Alerts in ' + alerts_dump_file + ' restored successfully (' - + str(created_count) + ' created, ' + str(updated_count) + ' updated)')) +print(f'All Alerts in {alerts_dump_file} restored successfully ' + f'({str(created_count)} created, {str(updated_count)} updated)') diff --git a/poetry.lock b/poetry.lock index 9215c715..77a391b0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -85,11 +85,14 @@ optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" [package.dependencies] -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} mccabe = ">=0.6.0,<0.7.0" pycodestyle = ">=2.6.0a1,<2.7.0" pyflakes = ">=2.2.0,<2.3.0" +[package.dependencies.importlib-metadata] +version = "*" +python = "<3.8" + [[package]] name = "idna" version = "2.10" @@ -105,17 +108,18 @@ description = "Read metadata from Python packages" category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[package.dependencies] -zipp = ">=0.5" +marker = "python_version < \"3.8\"" [package.extras] docs = ["sphinx", "rst.linker"] testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] +[package.dependencies] +zipp = ">=0.5" + [[package]] name = "mamba" -version = "0.11.1" +version = "0.11.2" description = "The definitive testing tool for Python. Born under the banner of Behavior Driven Development." category = "dev" optional = false @@ -184,16 +188,16 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[package.extras] +security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] +socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] + [package.dependencies] certifi = ">=2017.4.17" chardet = ">=3.0.2,<4" idna = ">=2.5,<3" urllib3 = ">=1.21.1,<1.27" -[package.extras] -security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] -socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] - [[package]] name = "requests-toolbelt" version = "0.9.1" @@ -220,6 +224,7 @@ description = "TatSu takes a grammar in a variation of EBNF as input, and output category = "main" optional = false python-versions = "*" +marker = "python_version < \"3.8\"" [package.extras] future-regex = ["regex"] @@ -231,6 +236,7 @@ description = "TatSu takes a grammar in a variation of EBNF as input, and output category = "main" optional = false python-versions = ">=3.8" +marker = "python_version >= \"3.8\" and python_version < \"4.0\"" [package.extras] future-regex = ["regex"] @@ -255,13 +261,14 @@ description = "Backport of pathlib-compatible object wrapper for zip files" category = "dev" optional = false python-versions = ">=3.6" +marker = "python_version < \"3.8\"" [package.extras] docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] [metadata] -lock-version = "1.1" +lock-version = "1.0" python-versions = "^3.6" content-hash = "fb28d0db08cb771d219781fbfa925110552065fc4f3349c061fe9dc2aa894f4c" @@ -338,7 +345,7 @@ importlib-metadata = [ {file = "importlib_metadata-2.0.0.tar.gz", hash = "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da"}, ] mamba = [ - {file = "mamba-0.11.1.tar.gz", hash = "sha256:f976735949bc9a8731cc0876aaea2720949bd3d1554b0e94004c91a4f61abecb"}, + {file = "mamba-0.11.2.tar.gz", hash = "sha256:75cfc6dfd287dcccaf86dd753cf48e0a7337487c7c3fafda05a6a67ded6da496"}, ] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, diff --git a/sdcclient/__init__.py b/sdcclient/__init__.py index f11bda4d..c61b16b6 100644 --- a/sdcclient/__init__.py +++ b/sdcclient/__init__.py @@ -1,9 +1,11 @@ -from sdcclient._monitor import SdcClient -from sdcclient._monitor import SdMonitorClient +import sdcclient.monitor as monitor +import sdcclient.secure as secure +from sdcclient._monitor import SdMonitorClient, SdcClient from sdcclient._monitor_v1 import SdMonitorClientV1 +from sdcclient._scanning import SdScanningClient from sdcclient._secure import SdSecureClient from sdcclient._secure_v1 import SdSecureClientV1 -from sdcclient._scanning import SdScanningClient from sdcclient.ibm_auth_helper import IbmAuthHelper -import sdcclient.secure -import sdcclient.monitor \ No newline at end of file + +__all__ = ["SdMonitorClient", "SdcClient", "SdMonitorClientV1", "SdScanningClient", "SdSecureClient", + "SdSecureClientV1", "IbmAuthHelper", "monitor", "secure"] diff --git a/sdcclient/_common.py b/sdcclient/_common.py index 72f50066..b68a4742 100644 --- a/sdcclient/_common.py +++ b/sdcclient/_common.py @@ -46,7 +46,7 @@ def __init__(self, token="", sdc_url='https://app.sysdigcloud.com', ssl_verify=T self.hdrs = self.__get_headers(custom_headers) self.url = os.environ.get("SDC_URL", sdc_url).rstrip('/') self.ssl_verify = os.environ.get("SDC_SSL_VERIFY", None) - if self.ssl_verify == None: + if self.ssl_verify is None: self.ssl_verify = ssl_verify else: if self.ssl_verify.lower() in ['true', 'false']: @@ -256,7 +256,7 @@ def create_email_notification_channel(self, channel_name, email_recipients): } res = self.http.post(self.url + '/api/notificationChannels', headers=self.hdrs, data=json.dumps(channel_json), - verify=self.ssl_verify) + verify=self.ssl_verify) return self._request_result(res) def create_notification_channel(self, channel): @@ -269,12 +269,13 @@ def create_notification_channel(self, channel): } res = self.http.post(self.url + '/api/notificationChannels', headers=self.hdrs, data=json.dumps(channel_json), - verify=self.ssl_verify) + verify=self.ssl_verify) return self._request_result(res) def get_notification_channel(self, id): - res = self.http.get(self.url + '/api/notificationChannels/' + str(id), headers=self.hdrs, verify=self.ssl_verify) + res = self.http.get(self.url + '/api/notificationChannels/' + str(id), headers=self.hdrs, + verify=self.ssl_verify) if not self._checkResponse(res): return False, self.lasterr @@ -285,7 +286,7 @@ def update_notification_channel(self, channel): return [False, "Invalid channel format"] res = self.http.put(self.url + '/api/notificationChannels/' + str(channel['id']), headers=self.hdrs, - data=json.dumps({"notificationChannel": channel}), verify=self.ssl_verify) + data=json.dumps({"notificationChannel": channel}), verify=self.ssl_verify) return self._request_result(res) def delete_notification_channel(self, channel): @@ -293,7 +294,7 @@ def delete_notification_channel(self, channel): return [False, "Invalid channel format"] res = self.http.delete(self.url + '/api/notificationChannels/' + str(channel['id']), headers=self.hdrs, - verify=self.ssl_verify) + verify=self.ssl_verify) if not self._checkResponse(res): return False, self.lasterr return True, None @@ -387,7 +388,7 @@ def get_topology_map(self, grouping_hierarchy, time_window_s, sampling_time_s): # Fire the request # res = self.http.post(self.url + '/api/data?format=map', headers=self.hdrs, - data=json.dumps(req_json), verify=self.ssl_verify) + data=json.dumps(req_json), verify=self.ssl_verify) return self._request_result(res) def get_data(self, metrics, start_ts, end_ts=0, sampling_s=0, @@ -436,7 +437,7 @@ def get_data(self, metrics, start_ts, end_ts=0, sampling_s=0, reqbody['sampling'] = sampling_s res = self.http.post(self.url + '/api/data/', headers=self.hdrs, data=json.dumps(reqbody), - verify=self.ssl_verify) + verify=self.ssl_verify) return self._request_result(res) def get_sysdig_captures(self, from_sec=None, to_sec=None, scope_filter=None): @@ -581,7 +582,7 @@ def create_user_invite(self, user_email, first_name=None, last_name=None, system user_json = {k: v for k, v in options.items() if v is not None} res = self.http.post(self.url + '/api/users', headers=self.hdrs, data=json.dumps(user_json), - verify=self.ssl_verify) + verify=self.ssl_verify) return self._request_result(res) def delete_user(self, user_email): @@ -594,10 +595,10 @@ def delete_user(self, user_email): **Example** `examples/user_team_mgmt.py `_ ''' - res = self.get_user_ids([user_email]) - if res[0] == False: - return res - userid = res[1][0] + ok, res = self.get_user_ids([user_email]) + if not ok: + return ok, res + userid = res[0] res = self.http.delete(self.url + '/api/users/' + str(userid), headers=self.hdrs, verify=self.ssl_verify) if not self._checkResponse(res): return [False, self.lasterr] @@ -625,10 +626,10 @@ def get_users(self): return [True, res.json()['users']] def edit_user(self, user_email, firstName=None, lastName=None, systemRole=None): - res = self.get_user(user_email) - if res[0] == False: - return res - user = res[1] + ok, user = self.get_user(user_email) + if not ok: + return ok, user + reqbody = { 'systemRole': systemRole if systemRole else user['systemRole'], 'username': user_email, @@ -636,18 +637,18 @@ def edit_user(self, user_email, firstName=None, lastName=None, systemRole=None): 'version': user['version'] } - if firstName == None: + if firstName is None: reqbody['firstName'] = user['firstName'] if 'firstName' in list(user.keys()) else '' else: reqbody['firstName'] = firstName - if lastName == None: + if lastName is None: reqbody['lastName'] = user['lastName'] if 'lastName' in list(user.keys()) else '' else: reqbody['lastName'] = lastName res = self.http.put(self.url + '/api/users/' + str(user['id']), headers=self.hdrs, data=json.dumps(reqbody), - verify=self.ssl_verify) + verify=self.ssl_verify) if not self._checkResponse(res): return [False, self.lasterr] return [True, 'Successfully edited user'] @@ -681,12 +682,12 @@ def get_team(self, name): **Example** `examples/user_team_mgmt.py `_ ''' - res = self.get_teams(name) - if res[0] == False: - return res - for t in res[1]: - if t['name'] == name: - return [True, t] + ok, res = self.get_teams(name) + if not ok: + return ok, res + for team in res: + if team['name'] == name: + return [True, team] return [False, 'Could not find team'] def get_team_ids(self, teams): @@ -711,11 +712,11 @@ def _get_id_user_dict(self, user_ids): return [True, dict((user['id'], user['username']) for user in u)] def get_user_ids(self, users): - res = self._get_user_id_dict(users) - if res[0] == False: - return res + ok, res = self._get_user_id_dict(users) + if not ok: + return ok, res else: - return [True, list(res[1].values())] + return [True, list(res.values())] def create_team(self, name, memberships=None, filter='', description='', show='host', theme='#7BB0B2', perm_capture=False, perm_custom_events=False, perm_aws_data=False): @@ -751,16 +752,16 @@ def create_team(self, name, memberships=None, filter='', description='', show='h } # Map user-names to IDs - if memberships != None and len(memberships) != 0: - res = self._get_user_id_dict(list(memberships.keys())) - if res[0] == False: + if memberships: + ok, res = self._get_user_id_dict(list(memberships.keys())) + if not ok: return [False, 'Could not fetch IDs for user names'] reqbody['userRoles'] = [ { 'userId': user_id, 'role': memberships[user_name] } - for (user_name, user_id) in res[1].items() + for (user_name, user_id) in res.items() ] else: reqbody['users'] = [] @@ -769,7 +770,7 @@ def create_team(self, name, memberships=None, filter='', description='', show='h reqbody['filter'] = filter res = self.http.post(self.url + '/api/teams', headers=self.hdrs, data=json.dumps(reqbody), - verify=self.ssl_verify) + verify=self.ssl_verify) return self._request_result(res) def edit_team(self, name, memberships=None, filter=None, description=None, show=None, theme=None, @@ -795,53 +796,52 @@ def edit_team(self, name, memberships=None, filter=None, description=None, show= **Example** `examples/user_team_mgmt.py `_ ''' - res = self.get_team(name) - if res[0] == False: - return res + ok, team = self.get_team(name) + if not ok: + return ok, team - t = res[1] reqbody = { 'name': name, - 'theme': theme if theme else t['theme'], - 'show': show if show else t['show'], - 'canUseSysdigCapture': perm_capture if perm_capture else t['canUseSysdigCapture'], - 'canUseCustomEvents': perm_custom_events if perm_custom_events else t['canUseCustomEvents'], - 'canUseAwsMetrics': perm_aws_data if perm_aws_data else t['canUseAwsMetrics'], - 'id': t['id'], - 'version': t['version'] + 'theme': theme if theme else team['theme'], + 'show': show if show else team['show'], + 'canUseSysdigCapture': perm_capture if perm_capture else team['canUseSysdigCapture'], + 'canUseCustomEvents': perm_custom_events if perm_custom_events else team['canUseCustomEvents'], + 'canUseAwsMetrics': perm_aws_data if perm_aws_data else team['canUseAwsMetrics'], + 'id': team['id'], + 'version': team['version'] } # Handling team description if description is not None: reqbody['description'] = description - elif 'description' in list(t.keys()): - reqbody['description'] = t['description'] + elif 'description' in list(team.keys()): + reqbody['description'] = team['description'] # Handling for users to map (user-name, team-role) pairs to memberships - if memberships != None: - res = self._get_user_id_dict(list(memberships.keys())) - if res[0] == False: + if memberships is not None: + ok, res = self._get_user_id_dict(list(memberships.keys())) + if not res: return [False, 'Could not convert user names to IDs'] reqbody['userRoles'] = [ { 'userId': user_id, 'role': memberships[user_name] } - for (user_name, user_id) in res[1].items() + for (user_name, user_id) in res.items() ] - elif 'userRoles' in list(t.keys()): - reqbody['userRoles'] = t['userRoles'] + elif 'userRoles' in list(team.keys()): + reqbody['userRoles'] = team['userRoles'] else: reqbody['userRoles'] = [] # Special handling for filters since we don't support blank filters - if filter != None: + if filter is not None: reqbody['filter'] = filter - elif 'filter' in list(t.keys()): - reqbody['filter'] = t['filter'] + elif 'filter' in list(team.keys()): + reqbody['filter'] = team['filter'] - res = self.http.put(self.url + '/api/teams/' + str(t['id']), headers=self.hdrs, data=json.dumps(reqbody), - verify=self.ssl_verify) + res = self.http.put(self.url + '/api/teams/' + str(team['id']), headers=self.hdrs, data=json.dumps(reqbody), + verify=self.ssl_verify) return self._request_result(res) def delete_team(self, name): @@ -854,12 +854,11 @@ def delete_team(self, name): **Example** `examples/user_team_mgmt.py `_ ''' - res = self.get_team(name) - if res[0] == False: - return res + ok, team = self.get_team(name) + if not ok: + return ok, team - t = res[1] - res = self.http.delete(self.url + '/api/teams/' + str(t['id']), headers=self.hdrs, verify=self.ssl_verify) + res = self.http.delete(self.url + '/api/teams/' + str(team['id']), headers=self.hdrs, verify=self.ssl_verify) if not self._checkResponse(res): return [False, self.lasterr] return [True, None] @@ -878,20 +877,18 @@ def list_memberships(self, team): **Example** `examples/user_team_mgmt_extended.py `_ ''' - res = self.get_team(team) - if res[0] == False: - return res + ok, res = self.get_team(team) + if not ok: + return ok, res - raw_memberships = res[1]['userRoles'] + raw_memberships = res['userRoles'] user_ids = [m['userId'] for m in raw_memberships] - res = self._get_id_user_dict(user_ids) - if res[0] == False: + ok, res = self._get_id_user_dict(user_ids) + if not ok: return [False, 'Could not fetch IDs for user names'] else: - id_user_dict = res[1] - - return [True, dict([(id_user_dict[m['userId']], m['role']) for m in raw_memberships])] + return [True, dict([(res[m['userId']], m['role']) for m in raw_memberships])] def save_memberships(self, team, memberships): ''' @@ -986,7 +983,7 @@ def disable_access_key(self, access_key): The access keys object ''' res = self.http.post(self.url + '/api/customer/accessKeys/' + access_key + "/disable/", headers=self.hdrs, - verify=self.ssl_verify) + verify=self.ssl_verify) return self._request_result(res) def enable_access_key(self, access_key): @@ -1001,7 +998,7 @@ def enable_access_key(self, access_key): The access keys object ''' res = self.http.post(self.url + '/api/customer/accessKeys/' + access_key + "/enable/", headers=self.hdrs, - verify=self.ssl_verify) + verify=self.ssl_verify) return self._request_result(res) def get_agents_config(self): @@ -1013,7 +1010,7 @@ def get_agents_config(self): def set_agents_config(self, config): res = self.http.put(self.url + '/api/agents/config', headers=self.hdrs, data=json.dumps(config), - verify=self.ssl_verify) + verify=self.ssl_verify) return self._request_result(res) def clear_agents_config(self): @@ -1021,14 +1018,12 @@ def clear_agents_config(self): return self.set_agents_config(data) def get_user_api_token(self, username, teamname): - res = self.get_team(teamname) - if res[0] == False: - return res + ok, team = self.get_team(teamname) + if not ok: + return ok, team - t = res[1] - - res = self.http.get(self.url + '/api/token/%s/%d' % (username, t['id']), headers=self.hdrs, - verify=self.ssl_verify) + res = self.http.get(self.url + '/api/token/%s/%d' % (username, team['id']), headers=self.hdrs, + verify=self.ssl_verify) if not self._checkResponse(res): return [False, self.lasterr] data = res.json() diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index fd73bfb6..056d799f 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -11,7 +11,6 @@ def __init__(self, token="", sdc_url='https://app.sysdigcloud.com', ssl_verify=T super(SdMonitorClient, self).__init__(token, sdc_url, ssl_verify, custom_headers) self.product = "SDC" - def get_alerts(self): '''**Description** Retrieve the list of alerts configured by the user. @@ -80,7 +79,8 @@ def update_notification_resolution(self, notification, resolved): notification['resolved'] = resolved data = {'notification': notification} - res = self.http.put(self.url + '/api/notifications/' + str(notification['id']), headers=self.hdrs, data=json.dumps(data), verify=self.ssl_verify) + res = self.http.put(self.url + '/api/notifications/' + str(notification['id']), headers=self.hdrs, + data=json.dumps(data), verify=self.ssl_verify) return self._request_result(res) def create_alert(self, name=None, description=None, severity=None, for_atleast_s=None, condition=None, @@ -127,7 +127,8 @@ def create_alert(self, name=None, description=None, severity=None, for_atleast_s if alert_obj is None: if None in (name, description, severity, for_atleast_s, condition): - return [False, 'Must specify a full Alert object or all parameters: name, description, severity, for_atleast_s, condition'] + return [False, + 'Must specify a full Alert object or all parameters: name, description, severity, for_atleast_s, condition'] else: # # Populate the alert information @@ -145,14 +146,14 @@ def create_alert(self, name=None, description=None, severity=None, for_atleast_s } } - if segmentby != None and segmentby != []: + if segmentby: alert_json['alert']['segmentBy'] = segmentby alert_json['alert']['segmentCondition'] = {'type': segment_condition} - if annotations != None and annotations != {}: + if annotations: alert_json['alert']['annotations'] = annotations - if notify != None: + if notify is not None: alert_json['alert']['notificationChannelIds'] = notify else: # The REST API enforces "Alert ID and version must be null", so remove them if present, @@ -166,7 +167,8 @@ def create_alert(self, name=None, description=None, severity=None, for_atleast_s # # Create the new alert # - res = self.http.post(self.url + '/api/alerts', headers=self.hdrs, data=json.dumps(alert_json), verify=self.ssl_verify) + res = self.http.post(self.url + '/api/alerts', headers=self.hdrs, data=json.dumps(alert_json), + verify=self.ssl_verify) return self._request_result(res) def update_alert(self, alert): @@ -185,7 +187,8 @@ def update_alert(self, alert): if 'id' not in alert: return [False, "Invalid alert format"] - res = self.http.put(self.url + '/api/alerts/' + str(alert['id']), headers=self.hdrs, data=json.dumps({"alert": alert}), verify=self.ssl_verify) + res = self.http.put(self.url + '/api/alerts/' + str(alert['id']), headers=self.hdrs, + data=json.dumps({"alert": alert}), verify=self.ssl_verify) return self._request_result(res) def delete_alert(self, alert): @@ -259,13 +262,12 @@ def set_explore_grouping_hierarchy(self, new_hierarchy): body['groups'][0]['groupBy'].append({'metric': item}) res = self.http.put(self.url + '/api/groupConfigurations/explore', headers=self.hdrs, - data=json.dumps(body), verify=self.ssl_verify) + data=json.dumps(body), verify=self.ssl_verify) if not self._checkResponse(res): return [False, self.lasterr] else: return [True, None] - def get_metrics(self): '''**Description** Return the metric list that can be used for data requests/alerts/dashboards. @@ -295,7 +297,8 @@ def convert_scope_string_to_expression(scope): expressions = [] string_expressions = scope.strip(' \t\n\r').split(' and ') - expression_re = re.compile('^(?Pnot )?(?P[^ ]+) (?P=|!=|in|contains|starts with) (?P(:?"[^"]+"|\'[^\']+\'|\(.+\)|.+))$') + expression_re = re.compile( + '^(?Pnot )?(?P[^ ]+) (?P=|!=|in|contains|starts with) (?P(:?"[^"]+"|\'[^\']+\'|(.+)|.+))$') for string_expression in string_expressions: matches = expression_re.match(string_expression) diff --git a/sdcclient/_monitor_v1.py b/sdcclient/_monitor_v1.py index 9d78a549..a9d3387f 100644 --- a/sdcclient/_monitor_v1.py +++ b/sdcclient/_monitor_v1.py @@ -20,9 +20,10 @@ def __init__(self, token="", sdc_url='https://app.sysdigcloud.com', ssl_verify=T self._dashboards_api_endpoint = '/ui/dashboards' self._default_dashboards_api_endpoint = '/api/defaultDashboards' - def create_dashboard_from_template(self, dashboard_name, template, scope, shared=False, public=False, annotations={}): + def create_dashboard_from_template(self, dashboard_name, template, scope, shared=False, public=False, + annotations={}): if scope is not None: - if isinstance(scope, basestring) == False: + if not isinstance(scope, basestring): return [False, 'Invalid scope format: Expected a string'] # @@ -37,11 +38,13 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared template['publicToken'] = None # set dashboard scope to the specific parameter - scopeExpression = self.convert_scope_string_to_expression(scope) - if scopeExpression[0] == False: - return scopeExpression + ok, scope_expression = self.convert_scope_string_to_expression(scope) + if not ok: + return ok, scope_expression template['filterExpression'] = scope - template['scopeExpressionList'] = map(lambda ex: {'operand':ex['operand'], 'operator':ex['operator'],'value':ex['value'],'displayName':'', 'isVariable':False}, scopeExpression[1]) + template['scopeExpressionList'] = map( + lambda ex: {'operand': ex['operand'], 'operator': ex['operator'], 'value': ex['value'], 'displayName': '', + 'isVariable': False}, scope_expression) if 'widgets' in template and template['widgets'] is not None: # Default dashboards (aka Explore views) specify panels with the property `widgets`, @@ -55,7 +58,7 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared if 'overrideFilter' not in chart: chart['overrideFilter'] = False - if chart['overrideFilter'] == False: + if not chart['overrideFilter']: # patch frontend bug to hide scope override warning even when it's not really overridden chart['scope'] = scope @@ -73,7 +76,8 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared # # Create the new dashboard # - res = self.http.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': template}), verify=self.ssl_verify) + res = self.http.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, + data=json.dumps({'dashboard': template}), verify=self.ssl_verify) return self._request_result(res) def create_dashboard(self, name): @@ -99,11 +103,13 @@ def create_dashboard(self, name): # # Create the new dashboard # - res = self.http.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}), - verify=self.ssl_verify) + res = self.http.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, + data=json.dumps({'dashboard': dashboard_configuration}), + verify=self.ssl_verify) return self._request_result(res) - def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, sort_by=None, limit=None, layout=None): + def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, sort_by=None, limit=None, + layout=None): """**Description** Adds a panel to the dashboard. A panel can be a time series, or a top chart (i.e. bar chart), or a number panel. @@ -176,7 +182,8 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, panel_configuration['scope'] = scope # if chart scope is equal to dashboard scope, set it as non override - panel_configuration['overrideFilter'] = ('scope' in dashboard and dashboard['scope'] != scope) or ('scope' not in dashboard and scope != None) + panel_configuration['overrideFilter'] = ('scope' in dashboard and dashboard['scope'] != scope) or \ + ('scope' not in dashboard and scope is not None) # # Configure panel type @@ -185,7 +192,7 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, panel_configuration['showAs'] = 'timeSeries' panel_configuration['showAsType'] = 'line' - if limit != None: + if limit is not None: panel_configuration['paging'] = { 'from': 0, 'to': limit - 1 @@ -223,7 +230,7 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, # # Configure layout # - if layout != None: + if layout is not None: panel_configuration['gridConfiguration'] = layout # @@ -240,8 +247,9 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, # # Update dashboard # - res = self.http.put(self.url + self._dashboards_api_endpoint + '/' + str(dashboard['id']), headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}), - verify=self.ssl_verify) + res = self.http.put(self.url + self._dashboards_api_endpoint + '/' + str(dashboard['id']), headers=self.hdrs, + data=json.dumps({'dashboard': dashboard_configuration}), + verify=self.ssl_verify) return self._request_result(res) def remove_dashboard_panel(self, dashboard, panel_name): @@ -281,8 +289,9 @@ def filter_fn(panel): # # Update dashboard # - res = self.http.put(self.url + self._dashboards_api_endpoint + '/' + str(dashboard['id']), headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}), - verify=self.ssl_verify) + res = self.http.put(self.url + self._dashboards_api_endpoint + '/' + str(dashboard['id']), + headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}), + verify=self.ssl_verify) return self._request_result(res) else: return [False, 'Not found'] diff --git a/sdcclient/_scanning.py b/sdcclient/_scanning.py index 83bce0b8..f9cfad28 100644 --- a/sdcclient/_scanning.py +++ b/sdcclient/_scanning.py @@ -1059,7 +1059,7 @@ def get_vulnerability_details(self, id): if id is None: return [False, "No vulnerability ID provided"] - url = self.url + f"/api/scanning/v1/anchore/query/vulnerabilities" + url = f"{self.url}/api/scanning/v1/anchore/query/vulnerabilities" params = { "id": id, @@ -1079,7 +1079,7 @@ def add_vulnerability_exception_bundle(self, name, comment=""): if not name: return [False, "A name is required for the exception bundle"] - url = self.url + f"/api/scanning/v1/vulnexceptions" + url = f"{self.url}/api/scanning/v1/vulnexceptions" params = { "version": "1_0", "name": name, @@ -1104,7 +1104,7 @@ def delete_vulnerability_exception_bundle(self, id): return [True, None] def list_vulnerability_exception_bundles(self): - url = self.url + f"/api/scanning/v1/vulnexceptions" + url = f"{self.url}/api/scanning/v1/vulnexceptions" params = { "bundleId": "default", @@ -1214,9 +1214,7 @@ def download_cve_report_csv(self, vuln_type="os", scope_type="static"): "tag": "" }, "runtimeScope": {}, - "imageQueryFilter": { - "vType": vuln_type - }, + "imageQueryFilter": {"vType": vuln_type}, "offset": 0, "limit": 100000 } diff --git a/sdcclient/_secure.py b/sdcclient/_secure.py index b570a7a8..e8262d0c 100644 --- a/sdcclient/_secure.py +++ b/sdcclient/_secure.py @@ -1,19 +1,17 @@ import json -import os -import shutil import time -import yaml - from sdcclient._common import _SdcCommon -from sdcclient.secure import PolicyEventsClientV1, PolicyEventsClientOld +from sdcclient.secure import FalcoRulesFilesClientOld, PolicyEventsClientV1, PolicyEventsClientOld -class SdSecureClient(PolicyEventsClientV1, PolicyEventsClientOld, _SdcCommon): +class SdSecureClient(FalcoRulesFilesClientOld, + PolicyEventsClientV1, + PolicyEventsClientOld, + _SdcCommon): def __init__(self, token="", sdc_url='https://secure.sysdig.com', ssl_verify=True, custom_headers=None): super(SdSecureClient, self).__init__(token, sdc_url, ssl_verify, custom_headers) - self.customer_id = None self.product = "SDS" self._policy_v2 = None @@ -27,393 +25,6 @@ def policy_v2(self): self._policy_v2 = res.status_code != 404 return self._policy_v2 - def _get_falco_rules(self, kind): - res = self.http.get(self.url + '/api/settings/falco/{}RulesFile'.format(kind), headers=self.hdrs, - verify=self.ssl_verify) - if not self._checkResponse(res): - return [False, self.lasterr] - data = res.json() - return [True, data] - - def get_system_falco_rules(self): - '''**Description** - Get the system falco rules file in use for this customer. See the `Falco wiki `_ for documentation on the falco rules format. - - **Arguments** - - None - - **Success Return Value** - The contents of the system falco rules file. - - **Example** - `examples/get_secure_system_falco_rules.py `_ - ''' - - return self._get_falco_rules("system") - - def get_user_falco_rules(self): - '''**Description** - Get the user falco rules file in use for this customer. See the `Falco wiki `_ for documentation on the falco rules format. - - **Arguments** - - None - - **Success Return Value** - The contents of the user falco rules file. - - **Example** - `examples/get_secure_user_falco_rules.py `_ - ''' - ok, res = self._get_user_falco_rules() - if not ok: - return [False, res] - - local_rules_file = [file - for file in res["customFalcoRulesFiles"]["files"] - if file["name"] == "falco_rules_local.yaml"] - if len(local_rules_file) == 0: - return [False, "Expected falco_rules_local.yaml file, but no file found"] - - return [True, local_rules_file[0]["variants"][0]["content"]] - - def _get_user_falco_rules(self): - res = self.http.get(self.url + '/api/settings/falco/customRulesFiles', headers=self.hdrs, - verify=self.ssl_verify) - - if not self._checkResponse(res): - return [False, self.lasterr] - - return [True, (res.json())] - - def _set_falco_rules(self, kind, rules_content): - payload = self._get_falco_rules(kind) - - if not payload[0]: - return payload - - payload[1]["{}RulesFile".format(kind)]["content"] = rules_content # pylint: disable=unsubscriptable-object - - res = self.http.put(self.url + '/api/settings/falco/{}RulesFile'.format(kind), headers=self.hdrs, - data=json.dumps(payload[1]), verify=self.ssl_verify) - if not self._checkResponse(res): - return [False, self.lasterr] - return [True, res.json()] - - def set_system_falco_rules(self, rules_content): - '''**Description** - Set the system falco rules file in use for this customer. NOTE: This API endpoint can *only* be used in on-premise deployments. Generally the system falco rules file is only modified in conjunction with Sysdig support. See the `Falco wiki `_ for documentation on the falco rules format. - - **Arguments** - - A string containing the system falco rules. - - **Success Return Value** - The contents of the system falco rules file that were just updated. - - **Example** - `examples/set_secure_system_falco_rules.py `_ - - ''' - return self._set_falco_rules("system", rules_content) - - def set_user_falco_rules(self, rules_content): - '''**Description** - Set the user falco rules file in use for this customer. See the `Falco wiki `_ for documentation on the falco rules format. - - **Arguments** - - A string containing the user falco rules. - - **Success Return Value** - The contents of the user falco rules file that were just updated. - - **Example** - `examples/set_secure_user_falco_rules.py `_ - - ''' - ok, res = self._get_user_falco_rules() - - if not ok: - return res - - local_rules_file = [file - for file in res["customFalcoRulesFiles"]["files"] - if file["name"] == "falco_rules_local.yaml"] - if len(local_rules_file) == 0: - return [False, "Expected falco_rules_local.yaml file, but no file found"] - - local_rules_file[0]["variants"][0]["content"] = rules_content - - res = self.http.put(self.url + '/api/settings/falco/customRulesFiles', headers=self.hdrs, - data=json.dumps(res), verify=self.ssl_verify) - - if not self._checkResponse(res): - return [False, self.lasterr] - res_json = res.json() - return [True, res_json["customFalcoRulesFiles"]["files"][0]["variants"][0]["content"]] - - - # Only one kind for now called "default", but might add a "custom" kind later. - def _get_falco_rules_files(self, kind): - - res = self.http.get(self.url + '/api/settings/falco/{}RulesFiles'.format(kind), headers=self.hdrs, - verify=self.ssl_verify) - if not self._checkResponse(res): - return [False, self.lasterr] - data = res.json() - - return [True, data] - - def get_default_falco_rules_files(self): - '''**Description** - Get the set of falco rules files from the backend. The _files programs and endpoints are a - replacement for the system_file endpoints and allow for publishing multiple files instead - of a single file as well as publishing multiple variants of a given file that are compatible - with different agent versions. - - **Arguments** - - None - - **Success Return Value** - A dict with the following keys: - - tag: A string used to uniquely identify this set of rules. It is recommended that this tag change every time the set of rules is updated. - - files: An array of dicts. Each dict has the following keys: - - name: the name of the file - - variants: An array of dicts with the following keys: - - requiredEngineVersion: the minimum falco engine version that can read this file - - content: the falco rules content - An example would be: - {'tag': 'v1.5.9', - 'files': [ - { - 'name': 'falco_rules.yaml', - 'variants': [ - { - 'content': '- required_engine_version: 29\n\n- list: foo\n', - 'requiredEngineVersion': 29 - }, - { - 'content': '- required_engine_version: 1\n\n- list: foo\n', - 'requiredEngineVersion': 1 - } - ] - }, - { - 'name': 'k8s_audit_rules.yaml', - 'variants': [ - { - 'content': '# some comment\n', - 'requiredEngineVersion': 0 - } - ] - } - ] - } - - **Example** - `examples/get_default_falco_rules_files.py `_ - ''' - - res = self._get_falco_rules_files("default") - - if not res[0]: - return res - else: - res_obj = res[1]["defaultFalcoRulesFiles"] - - # Copy only the tag and files over - ret = {} - - if "tag" in res_obj: - ret["tag"] = res_obj["tag"] - - if "files" in res_obj: - ret["files"] = res_obj["files"] - - if "defaultPolicies" in res_obj: - ret["defaultPolicies"] = res_obj["defaultPolicies"] - - return [True, ret] - - def save_default_falco_rules_files(self, fsobj, save_dir): - '''**Description** - Given a dict returned from get_default_falco_rules_files, save those files to a set of files below save_dir. - The first level below save_dir is a directory with the tag name and an optional default_policies.yaml file, - which groups rules into recommended default policies. The second level is a directory per file. - The third level is a directory per variant. Finally the files are at the lowest level, in a file called "content". - For example, using the example dict in get_default_falco_rules_files(), the directory layout would look like: - save_dir/ - default_policies.yaml - v1.5.9/ - falco_rules.yaml/ - 29/ - content: a file containing "- required_engine_version: 29\n\n- list: foo\n" - 1/ - content: a file containing "- required_engine_version: 1\n\n- list: foo\n" - k8s_audit_rules.yaml/ - 0/ - content: a file containing "# some comment" - **Arguments** - - fsobj: a python dict matching the structure returned by get_default_falco_rules_files() - - save_dir: a directory path under which to save the files. If the path already exists, it will be removed first. - - **Success Return Value** - - None - - **Example** - `examples/get_default_falco_rules_files.py `_ - ''' - if os.path.exists(save_dir): - try: - if os.path.isdir(save_dir): - shutil.rmtree(save_dir) - else: - os.unlink(save_dir) - except Exception as e: - return [False, "Could not remove existing save dir {}: {}".format(save_dir, str(e))] - - prefix = os.path.join(save_dir, fsobj["tag"]) - try: - os.makedirs(prefix) - except Exception as e: - return [False, "Could not create tag directory {}: {}".format(prefix, str(e))] - - if "defaultPolicies" in fsobj: - with open(os.path.join(save_dir, "default_policies.yaml"), 'w') as outfile: - yaml.safe_dump(fsobj["defaultPolicies"], outfile) - - if "files" in fsobj: - for fobj in fsobj["files"]: - fprefix = os.path.join(prefix, fobj["name"]) - try: - os.makedirs(fprefix) - except Exception as e: - return [False, "Could not create file directory {}: {}".format(fprefix, str(e))] - for variant in fobj["variants"]: - vprefix = os.path.join(fprefix, str(variant["requiredEngineVersion"])) - try: - os.makedirs(vprefix) - except Exception as e: - return [False, "Could not create variant directory {}: {}".format(vprefix, str(e))] - cpath = os.path.join(vprefix, "content") - try: - with open(cpath, "w") as cfile: - cfile.write(variant["content"]) - except Exception as e: - return [False, "Could not write content to {}: {}".format(cfile, str(e))] - - return [True, None] - - # Only One kind for now, but might add a "custom" kind later. - def _set_falco_rules_files(self, kind, rules_files): - - payload = self._get_falco_rules_files(kind) - - if not payload[0]: - return payload - - obj = payload[1]["{}FalcoRulesFiles".format(kind)] # pylint: disable=unsubscriptable-object - - obj["tag"] = rules_files["tag"] - obj["files"] = rules_files["files"] - if "defaultPolicies" in rules_files: - obj["defaultPolicies"] = rules_files["defaultPolicies"] - - res = self.http.put(self.url + '/api/settings/falco/{}RulesFiles'.format(kind), headers=self.hdrs, - data=json.dumps(payload[1]), verify=self.ssl_verify) - if not self._checkResponse(res): - return [False, self.lasterr] - return [True, res.json()] - - def set_default_falco_rules_files(self, rules_files): - '''**Description** - Update the set of falco rules files to the provided set of files. See the `Falco wiki `_ for documentation on the falco rules format. - The _files programs and endpoints are a replacement for the system_file endpoints and - allow for publishing multiple files instead of a single file as well as publishing - multiple variants of a given file that are compatible with different agent versions. - - **Arguments** - - rules_files: a dict with the same structure as returned by get_default_falco_rules_files. - - **Success Return Value** - The contents of the default falco rules files that were just updated. - - **Example** - `examples/set_default_falco_rules_files.py `_ - - ''' - - return self._set_falco_rules_files("default", rules_files) - - def load_default_falco_rules_files(self, save_dir): - '''**Description** - Given a file and directory layout as described in save_default_falco_rules_files(), load those files and - return a dict representing the contents. This dict is suitable for passing to set_default_falco_rules_files(). - - **Arguments** - - save_dir: a directory path from which to load the files. - - **Success Return Value** - - A dict matching the format described in get_default_falco_rules_files. - - **Example** - `examples/set_default_falco_rules_files.py `_ - ''' - - tags = os.listdir(save_dir) - - try: - tags.remove("default_policies.yaml") - except ValueError: - # Do nothing, it wasn't in the list of files - pass - - if len(tags) != 1: - return [False, "Directory {} did not contain exactly 1 entry".format(save_dir)] - - tpath = os.path.join(save_dir, tags[0]) - - if not os.path.isdir(tpath): - return [False, "Tag path {} is not a directory".format(tpath)] - - defjson = [] - defpath = os.path.join(save_dir, "default_policies.yaml") - if os.path.exists(defpath): - try: - with open(defpath, "r") as infile: - defjson = yaml.safe_load(infile) - except Exception as exc: - return [False, "Could not load default_policies.yaml: " + exc] - - ret = {"tag": os.path.basename(tpath), "files": [], "defaultPolicies": defjson} - - for fdir in os.listdir(tpath): - fpath = os.path.join(tpath, fdir) - if not os.path.isdir(fpath): - return [False, "File path {} is not a directory".format(fpath)] - fobj = {"name": os.path.basename(fpath), "variants": []} - for vdir in os.listdir(fpath): - vpath = os.path.join(fpath, vdir) - if not os.path.isdir(vpath): - return [False, "Variant path {} is not a directory".format(vpath)] - cpath = os.path.join(vpath, "content") - try: - with open(cpath, 'r') as content_file: - try: - required_engine_version = int(os.path.basename(vpath)) - if int(os.path.basename(vpath)) < 0: - return [False, "Variant directory {} must be a positive number".format(vpath)] - fobj["variants"].append({ - "requiredEngineVersion": required_engine_version, - "content": content_file.read() - }) - except ValueError: - return [False, "Variant directory {} must be a number".format(vpath)] - except Exception as e: - return [False, "Could not read content at {}: {}".format(cpath, str(e))] - - ret["files"].append(fobj) - - return [True, ret] - def create_default_policies(self): '''**Description** Create new policies based on the currently available set of rules. For now, this only covers Falco rules, but we might extend @@ -1285,19 +896,19 @@ def __get_matched_profileIDs(self, requested_profile, profile_list): ''' **Description** Helper function for retrieving the list of matching profile - + **Arguments** - the requested profile Id (string) - List of dictionary, where each dictionary contains the profile information - + **Success Return Value** List of dictionary, where each dictionary represents a profile with the ID prefix substring matching the requested one - + **Content structure of the profile_list parameter** This array of profiles contains all the relevant information. For the purposes of this function, only the profileId field is relevant. - + [ { "profileGroupId": 0, @@ -1373,7 +984,7 @@ def __get_matched_profileIDs(self, requested_profile, profile_list): request_len = len(requested_profile) for profile in profile_list: - # get the length of the substring to match + # get the length of the substring to match str_len_match = min(len(profile), request_len) if profile['profileId'][0:str_len_match] == requested_profile[0:str_len_match]: diff --git a/sdcclient/monitor/__init__.py b/sdcclient/monitor/__init__.py index a4ff11e0..01f5a735 100644 --- a/sdcclient/monitor/__init__.py +++ b/sdcclient/monitor/__init__.py @@ -1,4 +1,6 @@ -from ._dashboards_v3 import DashboardsClientV3 from ._dashboards_v2 import DashboardsClientV2 +from ._dashboards_v3 import DashboardsClientV3 from ._events_v1 import EventsClientV1 from ._events_v2 import EventsClientV2 + +__all__ = ["DashboardsClientV3", "DashboardsClientV2", "EventsClientV1", "EventsClientV2"] diff --git a/sdcclient/monitor/_dashboards_v2.py b/sdcclient/monitor/_dashboards_v2.py index cc563507..61fd9b7a 100644 --- a/sdcclient/monitor/_dashboards_v2.py +++ b/sdcclient/monitor/_dashboards_v2.py @@ -16,7 +16,7 @@ def __init__(self, token="", sdc_url='https://app.sysdigcloud.com', ssl_verify=T def get_views_list(self): res = self.http.get(self.url + self._default_dashboards_api_endpoint, headers=self.hdrs, - verify=self.ssl_verify) + verify=self.ssl_verify) if not self._checkResponse(res): return [False, self.lasterr] return [True, res.json()] @@ -39,7 +39,7 @@ def get_view(self, name): return [False, 'view ' + name + ' not found'] res = self.http.get(self.url + self._default_dashboards_api_endpoint + '/' + id, headers=self.hdrs, - verify=self.ssl_verify) + verify=self.ssl_verify) return self._request_result(res) def get_dashboards(self): @@ -66,7 +66,7 @@ def update_dashboard(self, dashboard_data): `examples/dashboard_basic_crud.py `_ ''' res = self.http.put(self.url + self._dashboards_api_endpoint + "/" + str(dashboard_data['id']), - headers=self.hdrs, verify=self.ssl_verify, data=json.dumps({'dashboard': dashboard_data})) + headers=self.hdrs, verify=self.ssl_verify, data=json.dumps({'dashboard': dashboard_data})) return self._request_result(res) def find_dashboard_by(self, name=None): @@ -104,8 +104,8 @@ def create_dashboard_with_configuration(self, configuration): del configuration_clone['version'] res = self.http.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, - data=json.dumps({'dashboard': configuration_clone}), - verify=self.ssl_verify) + data=json.dumps({'dashboard': configuration_clone}), + verify=self.ssl_verify) return self._request_result(res) def create_dashboard(self, name): @@ -135,8 +135,8 @@ def create_dashboard(self, name): # Create the new dashboard # res = self.http.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, - data=json.dumps({'dashboard': dashboard_configuration}), - verify=self.ssl_verify) + data=json.dumps({'dashboard': dashboard_configuration}), + verify=self.ssl_verify) return self._request_result(res) # TODO COVER @@ -214,8 +214,8 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, panel_configuration['scope'] = scope # if chart scope is equal to dashboard scope, set it as non override - panel_configuration['overrideScope'] = ('scope' in dashboard and dashboard['scope'] != scope) or ( - 'scope' not in dashboard and scope != None) + panel_configuration['overrideScope'] = ('scope' in dashboard and dashboard['scope'] != scope) or \ + ('scope' not in dashboard and scope is not None) if 'custom_display_options' not in panel_configuration: panel_configuration['custom_display_options'] = { @@ -246,7 +246,7 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, if panel_type == 'timeSeries': panel_configuration['showAs'] = 'timeSeries' - if limit != None: + if limit is not None: panel_configuration['custom_display_options']['valueLimit'] = { 'count': limit, 'direction': 'desc' @@ -257,7 +257,7 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, elif panel_type == 'top': panel_configuration['showAs'] = 'top' - if limit != None: + if limit is not None: panel_configuration['custom_display_options']['valueLimit'] = { 'count': limit, 'direction': sort_direction @@ -266,7 +266,7 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, # # Configure layout # - if layout != None: + if layout is not None: panel_configuration['gridConfiguration'] = layout # @@ -283,8 +283,8 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, # Update dashboard # res = self.http.put(self.url + self._dashboards_api_endpoint + '/' + str(dashboard['id']), headers=self.hdrs, - data=json.dumps({'dashboard': dashboard_configuration}), - verify=self.ssl_verify) + data=json.dumps({'dashboard': dashboard_configuration}), + verify=self.ssl_verify) return self._request_result(res) # TODO COVER @@ -324,9 +324,10 @@ def filter_fn(panel): # # Update dashboard # - res = self.http.put(self.url + self._dashboards_api_endpoint + '/' + str(dashboard['id']), headers=self.hdrs, - data=json.dumps({'dashboard': dashboard_configuration}), - verify=self.ssl_verify) + res = self.http.put(self.url + self._dashboards_api_endpoint + '/' + str(dashboard['id']), + headers=self.hdrs, + data=json.dumps({'dashboard': dashboard_configuration}), + verify=self.ssl_verify) return self._request_result(res) else: return [False, 'Not found'] @@ -371,7 +372,7 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared if 'overrideScope' not in chart: chart['overrideScope'] = False - if chart['overrideScope'] == False: + if not chart['overrideScope']: # patch frontend bug to hide scope override warning even when it's not really overridden chart['scope'] = scope @@ -387,7 +388,7 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared # Create the new dashboard # res = self.http.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, - data=json.dumps({'dashboard': template}), verify=self.ssl_verify) + data=json.dumps({'dashboard': template}), verify=self.ssl_verify) return self._request_result(res) @@ -411,11 +412,11 @@ def create_dashboard_from_view(self, newdashname, viewname, filter, shared=False # # Find our template view # - gvres = self.get_view(viewname) - if gvres[0] is False: - return gvres + ok, gvres = self.get_view(viewname) + if not ok: + return ok, gvres - view = gvres[1]['defaultDashboard'] + view = gvres['defaultDashboard'] view['timeMode'] = {'mode': 1} view['time'] = {'last': 2 * 60 * 60 * 1000000, 'sampling': 2 * 60 * 60 * 1000000} @@ -436,7 +437,7 @@ def get_dashboard(self, dashboard_id): `examples/dashboard_basic_crud.py `_ ''' res = self.http.get(self.url + self._dashboards_api_endpoint + "/" + str(dashboard_id), headers=self.hdrs, - verify=self.ssl_verify) + verify=self.ssl_verify) return self._request_result(res) def create_dashboard_from_dashboard(self, newdashname, templatename, filter, shared=False, public=False): @@ -580,7 +581,7 @@ def delete_dashboard(self, dashboard): return [False, "Invalid dashboard format"] res = self.http.delete(self.url + self._dashboards_api_endpoint + '/' + str(dashboard['id']), headers=self.hdrs, - verify=self.ssl_verify) + verify=self.ssl_verify) if not self._checkResponse(res): return [False, self.lasterr] diff --git a/sdcclient/monitor/_dashboards_v3.py b/sdcclient/monitor/_dashboards_v3.py index 0425dc34..5f43f6d2 100644 --- a/sdcclient/monitor/_dashboards_v3.py +++ b/sdcclient/monitor/_dashboards_v3.py @@ -289,7 +289,7 @@ def create_dashboard_from_template(self, dashboard_name, template, scope=None, s if 'overrideScope' not in chart: chart['overrideScope'] = False - if chart['overrideScope'] == False: + if not chart['overrideScope']: # patch frontend bug to hide scope override warning even when it's not really overridden chart['scope'] = scope diff --git a/sdcclient/monitor/_events_v1.py b/sdcclient/monitor/_events_v1.py index ebfbc251..dc342e78 100644 --- a/sdcclient/monitor/_events_v1.py +++ b/sdcclient/monitor/_events_v1.py @@ -65,7 +65,7 @@ def post_event(self, name, description=None, severity=None, event_filter=None, t 'event': {k: v for k, v in options.items() if v is not None} } res = self.http.post(self.url + '/api/events/', headers=self.hdrs, data=json.dumps(edata), - verify=self.ssl_verify) + verify=self.ssl_verify) return self._request_result(res) def delete_event(self, event): diff --git a/sdcclient/monitor/dashboard_converters/__init__.py b/sdcclient/monitor/dashboard_converters/__init__.py index 508873eb..246a404b 100644 --- a/sdcclient/monitor/dashboard_converters/__init__.py +++ b/sdcclient/monitor/dashboard_converters/__init__.py @@ -1,2 +1,4 @@ +from ._dashboard_scope import convert_scope_string_to_expression from ._dashboard_versions import convert_dashboard_between_versions -from ._dashboard_scope import convert_scope_string_to_expression \ No newline at end of file + +__all__ = ["convert_dashboard_between_versions", "convert_scope_string_to_expression"] diff --git a/sdcclient/monitor/dashboard_converters/_dashboard_scope.py b/sdcclient/monitor/dashboard_converters/_dashboard_scope.py index 966d4334..f35e2703 100644 --- a/sdcclient/monitor/dashboard_converters/_dashboard_scope.py +++ b/sdcclient/monitor/dashboard_converters/_dashboard_scope.py @@ -1,7 +1,7 @@ import tatsu -def convert_scope_string_to_expression(scope = None): +def convert_scope_string_to_expression(scope=None): if scope is None or not scope: return [True, []] @@ -10,8 +10,8 @@ def convert_scope_string_to_expression(scope = None): start = expression $ ; - expression - = + expression + = | operand simple_operator word | operand multiple_operator multiple_value ; @@ -34,8 +34,8 @@ def convert_scope_string_to_expression(scope = None): operand = /[a-zA-Z0-9_\-\.]+/ ; - multiple_value - = + multiple_value + = | '[' word_array ']' | word ; @@ -46,8 +46,8 @@ def convert_scope_string_to_expression(scope = None): | word ; - word = - | /[a-zA-Z0-9-_\-\.]+/ + word = + | /[a-zA-Z0-9-_\-\.]+/ | '"' /[a-zA-Z0-9-_\-\.]+/ '"' | "'" /[a-zA-Z0-9-_\-\.]+/ "'" ; diff --git a/sdcclient/monitor/dashboard_converters/_dashboard_versions.py b/sdcclient/monitor/dashboard_converters/_dashboard_versions.py index a5ebf3db..90f5b428 100644 --- a/sdcclient/monitor/dashboard_converters/_dashboard_versions.py +++ b/sdcclient/monitor/dashboard_converters/_dashboard_versions.py @@ -254,6 +254,7 @@ def convert_property_name(prop_name, old_metric, new_metric): } } + def convert_dashboard_between_versions(dashboard, version_from, version_to): ''' **Description** diff --git a/sdcclient/secure/__init__.py b/sdcclient/secure/__init__.py index 098e7de6..a0dbb464 100644 --- a/sdcclient/secure/__init__.py +++ b/sdcclient/secure/__init__.py @@ -1,2 +1,5 @@ +from ._falco_rules_files_old import FalcoRulesFilesClientOld from ._policy_events_old import PolicyEventsClientOld -from ._policy_events_v1 import PolicyEventsClientV1 \ No newline at end of file +from ._policy_events_v1 import PolicyEventsClientV1 + +__all__ = ["PolicyEventsClientOld", "PolicyEventsClientV1", "FalcoRulesFilesClientOld"] diff --git a/sdcclient/secure/_falco_rules_files_old.py b/sdcclient/secure/_falco_rules_files_old.py new file mode 100644 index 00000000..a6da3ac1 --- /dev/null +++ b/sdcclient/secure/_falco_rules_files_old.py @@ -0,0 +1,407 @@ +import json +import os +import shutil + +import yaml + +from sdcclient._common import _SdcCommon + + +class FalcoRulesFilesClientOld(_SdcCommon): + def __init__(self, token="", sdc_url='https://secure.sysdig.com', ssl_verify=True, custom_headers=None): + super(FalcoRulesFilesClientOld, self).__init__(token, sdc_url, ssl_verify, custom_headers) + self.product = "SDS" + + # TODO: Remove this one, deprecated + def _get_falco_rules(self, kind): + res = self.http.get(self.url + '/api/settings/falco/{}RulesFile'.format(kind), headers=self.hdrs, + verify=self.ssl_verify) + if not self._checkResponse(res): + return [False, self.lasterr] + data = res.json() + return [True, data] + + # TODO: Change this one to use newestDefaultRulesFiles endpoint + def get_system_falco_rules(self): + '''**Description** + Get the system falco rules file in use for this customer. See the `Falco wiki `_ for documentation on the falco rules format. + + **Arguments** + - None + + **Success Return Value** + The contents of the system falco rules file. + + **Example** + `examples/get_secure_system_falco_rules.py `_ + ''' + + return self._get_falco_rules("system") + + def get_user_falco_rules(self): + '''**Description** + Get the user falco rules file in use for this customer. See the `Falco wiki `_ for documentation on the falco rules format. + + **Arguments** + - None + + **Success Return Value** + The contents of the user falco rules file. + + **Example** + `examples/get_secure_user_falco_rules.py `_ + ''' + ok, res = self._get_user_falco_rules() + if not ok: + return [False, res] + + local_rules_file = [file + for file in res["customFalcoRulesFiles"]["files"] + if file["name"] == "falco_rules_local.yaml"] + if len(local_rules_file) == 0: + return [False, "Expected falco_rules_local.yaml file, but no file found"] + + return [True, local_rules_file[0]["variants"][0]["content"]] + + def _get_user_falco_rules(self): + res = self.http.get(self.url + '/api/settings/falco/customRulesFiles', headers=self.hdrs, + verify=self.ssl_verify) + + if not self._checkResponse(res): + return [False, self.lasterr] + + return [True, (res.json())] + + # TODO: Remove this + def _set_falco_rules(self, kind, rules_content): + payload = self._get_falco_rules(kind) + + if not payload[0]: + return payload + + payload[1]["{}RulesFile".format(kind)]["content"] = rules_content # pylint: disable=unsubscriptable-object + + res = self.http.put(self.url + '/api/settings/falco/{}RulesFile'.format(kind), headers=self.hdrs, + data=json.dumps(payload[1]), verify=self.ssl_verify) + if not self._checkResponse(res): + return [False, self.lasterr] + return [True, res.json()] + + def set_system_falco_rules(self, rules_content): + '''**Description** + Set the system falco rules file in use for this customer. NOTE: This API endpoint can *only* be used in on-premise deployments. Generally the system falco rules file is only modified in conjunction with Sysdig support. See the `Falco wiki `_ for documentation on the falco rules format. + + **Arguments** + - A string containing the system falco rules. + + **Success Return Value** + The contents of the system falco rules file that were just updated. + + **Example** + `examples/set_secure_system_falco_rules.py `_ + + ''' + return self._set_falco_rules("system", rules_content) + + def set_user_falco_rules(self, rules_content): + '''**Description** + Set the user falco rules file in use for this customer. See the `Falco wiki `_ for documentation on the falco rules format. + + **Arguments** + - A string containing the user falco rules. + + **Success Return Value** + The contents of the user falco rules file that were just updated. + + **Example** + `examples/set_secure_user_falco_rules.py `_ + + ''' + ok, res = self._get_user_falco_rules() + + if not ok: + return res + + local_rules_file = [file + for file in res["customFalcoRulesFiles"]["files"] + if file["name"] == "falco_rules_local.yaml"] + if len(local_rules_file) == 0: + return [False, "Expected falco_rules_local.yaml file, but no file found"] + + local_rules_file[0]["variants"][0]["content"] = rules_content + + res = self.http.put(self.url + '/api/settings/falco/customRulesFiles', headers=self.hdrs, + data=json.dumps(res), verify=self.ssl_verify) + + if not self._checkResponse(res): + return [False, self.lasterr] + res_json = res.json() + return [True, res_json["customFalcoRulesFiles"]["files"][0]["variants"][0]["content"]] + + # get_falco_syscall_rules() + + # get_falco_ka_rules() + + # Only one kind for now called "default", but might add a "custom" kind later. + # TODO Remove this one + def _get_falco_rules_files(self, kind): + + res = self.http.get(self.url + '/api/settings/falco/{}RulesFiles'.format(kind), headers=self.hdrs, + verify=self.ssl_verify) + if not self._checkResponse(res): + return [False, self.lasterr] + data = res.json() + + return [True, data] + + def get_default_falco_rules_files(self): + '''**Description** + Get the set of falco rules files from the backend. The _files programs and endpoints are a + replacement for the system_file endpoints and allow for publishing multiple files instead + of a single file as well as publishing multiple variants of a given file that are compatible + with different agent versions. + + **Arguments** + - None + + **Success Return Value** + A dict with the following keys: + - tag: A string used to uniquely identify this set of rules. It is recommended that this tag change every time the set of rules is updated. + - files: An array of dicts. Each dict has the following keys: + - name: the name of the file + - variants: An array of dicts with the following keys: + - requiredEngineVersion: the minimum falco engine version that can read this file + - content: the falco rules content + An example would be: + {'tag': 'v1.5.9', + 'files': [ + { + 'name': 'falco_rules.yaml', + 'variants': [ + { + 'content': '- required_engine_version: 29\n\n- list: foo\n', + 'requiredEngineVersion': 29 + }, + { + 'content': '- required_engine_version: 1\n\n- list: foo\n', + 'requiredEngineVersion': 1 + } + ] + }, + { + 'name': 'k8s_audit_rules.yaml', + 'variants': [ + { + 'content': '# some comment\n', + 'requiredEngineVersion': 0 + } + ] + } + ] + } + + **Example** + `examples/get_default_falco_rules_files.py `_ + ''' + + res = self._get_falco_rules_files("default") + + if not res[0]: + return res + else: + res_obj = res[1]["defaultFalcoRulesFiles"] + + # Copy only the tag and files over + ret = {} + + if "tag" in res_obj: + ret["tag"] = res_obj["tag"] + + if "files" in res_obj: + ret["files"] = res_obj["files"] + + if "defaultPolicies" in res_obj: + ret["defaultPolicies"] = res_obj["defaultPolicies"] + + return [True, ret] + + def save_default_falco_rules_files(self, fsobj, save_dir): + '''**Description** + Given a dict returned from get_default_falco_rules_files, save those files to a set of files below save_dir. + The first level below save_dir is a directory with the tag name and an optional default_policies.yaml file, + which groups rules into recommended default policies. The second level is a directory per file. + The third level is a directory per variant. Finally the files are at the lowest level, in a file called "content". + For example, using the example dict in get_default_falco_rules_files(), the directory layout would look like: + save_dir/ + default_policies.yaml + v1.5.9/ + falco_rules.yaml/ + 29/ + content: a file containing "- required_engine_version: 29\n\n- list: foo\n" + 1/ + content: a file containing "- required_engine_version: 1\n\n- list: foo\n" + k8s_audit_rules.yaml/ + 0/ + content: a file containing "# some comment" + **Arguments** + - fsobj: a python dict matching the structure returned by get_default_falco_rules_files() + - save_dir: a directory path under which to save the files. If the path already exists, it will be removed first. + + **Success Return Value** + - None + + **Example** + `examples/get_default_falco_rules_files.py `_ + ''' + if os.path.exists(save_dir): + try: + if os.path.isdir(save_dir): + shutil.rmtree(save_dir) + else: + os.unlink(save_dir) + except Exception as e: + return [False, "Could not remove existing save dir {}: {}".format(save_dir, str(e))] + + prefix = os.path.join(save_dir, fsobj["tag"]) + try: + os.makedirs(prefix) + except Exception as e: + return [False, "Could not create tag directory {}: {}".format(prefix, str(e))] + + if "defaultPolicies" in fsobj: + with open(os.path.join(save_dir, "default_policies.yaml"), 'w') as outfile: + yaml.safe_dump(fsobj["defaultPolicies"], outfile) + + if "files" in fsobj: + for fobj in fsobj["files"]: + fprefix = os.path.join(prefix, fobj["name"]) + try: + os.makedirs(fprefix) + except Exception as e: + return [False, "Could not create file directory {}: {}".format(fprefix, str(e))] + for variant in fobj["variants"]: + vprefix = os.path.join(fprefix, str(variant["requiredEngineVersion"])) + try: + os.makedirs(vprefix) + except Exception as e: + return [False, "Could not create variant directory {}: {}".format(vprefix, str(e))] + cpath = os.path.join(vprefix, "content") + try: + with open(cpath, "w") as cfile: + cfile.write(variant["content"]) + except Exception as e: + return [False, "Could not write content to {}: {}".format(cfile, str(e))] + + return [True, None] + + # Only One kind for now, but might add a "custom" kind later. + def _set_falco_rules_files(self, kind, rules_files): + + payload = self._get_falco_rules_files(kind) + + if not payload[0]: + return payload + + obj = payload[1]["{}FalcoRulesFiles".format(kind)] # pylint: disable=unsubscriptable-object + + obj["tag"] = rules_files["tag"] + obj["files"] = rules_files["files"] + if "defaultPolicies" in rules_files: + obj["defaultPolicies"] = rules_files["defaultPolicies"] + + res = self.http.put(self.url + '/api/settings/falco/{}RulesFiles'.format(kind), headers=self.hdrs, + data=json.dumps(payload[1]), verify=self.ssl_verify) + if not self._checkResponse(res): + return [False, self.lasterr] + return [True, res.json()] + + def set_default_falco_rules_files(self, rules_files): + '''**Description** + Update the set of falco rules files to the provided set of files. See the `Falco wiki `_ for documentation on the falco rules format. + The _files programs and endpoints are a replacement for the system_file endpoints and + allow for publishing multiple files instead of a single file as well as publishing + multiple variants of a given file that are compatible with different agent versions. + + **Arguments** + - rules_files: a dict with the same structure as returned by get_default_falco_rules_files. + + **Success Return Value** + The contents of the default falco rules files that were just updated. + + **Example** + `examples/set_default_falco_rules_files.py `_ + + ''' + + return self._set_falco_rules_files("default", rules_files) + + def load_default_falco_rules_files(self, save_dir): + '''**Description** + Given a file and directory layout as described in save_default_falco_rules_files(), load those files and + return a dict representing the contents. This dict is suitable for passing to set_default_falco_rules_files(). + + **Arguments** + - save_dir: a directory path from which to load the files. + + **Success Return Value** + - A dict matching the format described in get_default_falco_rules_files. + + **Example** + `examples/set_default_falco_rules_files.py `_ + ''' + + tags = os.listdir(save_dir) + + try: + tags.remove("default_policies.yaml") + except ValueError: + # Do nothing, it wasn't in the list of files + pass + + if len(tags) != 1: + return [False, "Directory {} did not contain exactly 1 entry".format(save_dir)] + + tpath = os.path.join(save_dir, tags[0]) + + if not os.path.isdir(tpath): + return [False, "Tag path {} is not a directory".format(tpath)] + + defjson = [] + defpath = os.path.join(save_dir, "default_policies.yaml") + if os.path.exists(defpath): + try: + with open(defpath, "r") as infile: + defjson = yaml.safe_load(infile) + except Exception as exc: + return [False, "Could not load default_policies.yaml: " + exc] + + ret = {"tag": os.path.basename(tpath), "files": [], "defaultPolicies": defjson} + + for fdir in os.listdir(tpath): + fpath = os.path.join(tpath, fdir) + if not os.path.isdir(fpath): + return [False, "File path {} is not a directory".format(fpath)] + fobj = {"name": os.path.basename(fpath), "variants": []} + for vdir in os.listdir(fpath): + vpath = os.path.join(fpath, vdir) + if not os.path.isdir(vpath): + return [False, "Variant path {} is not a directory".format(vpath)] + cpath = os.path.join(vpath, "content") + try: + with open(cpath, 'r') as content_file: + try: + required_engine_version = int(os.path.basename(vpath)) + if int(os.path.basename(vpath)) < 0: + return [False, "Variant directory {} must be a positive number".format(vpath)] + fobj["variants"].append({ + "requiredEngineVersion": required_engine_version, + "content": content_file.read() + }) + except ValueError: + return [False, "Variant directory {} must be a number".format(vpath)] + except Exception as e: + return [False, "Could not read content at {}: {}".format(cpath, str(e))] + + ret["files"].append(fobj) + + return [True, ret] diff --git a/sdcclient/secure/_policy_events_old.py b/sdcclient/secure/_policy_events_old.py index d6bd792c..a5a316d1 100644 --- a/sdcclient/secure/_policy_events_old.py +++ b/sdcclient/secure/_policy_events_old.py @@ -8,10 +8,7 @@ class PolicyEventsClientOld(_SdcCommon): def __init__(self, token="", sdc_url='https://secure.sysdig.com', ssl_verify=True, custom_headers=None): super(PolicyEventsClientOld, self).__init__(token, sdc_url, ssl_verify, custom_headers) - - self.customer_id = None self.product = "SDS" - self._policy_v2 = None def _get_policy_events_int(self, ctx): warn("The PolicyEventsClientOld class is deprecated in favour of PolicyEventsClientV1; use it only if you have " diff --git a/sdcclient/secure/_policy_events_v1.py b/sdcclient/secure/_policy_events_v1.py index 8746cf56..79e38c6f 100644 --- a/sdcclient/secure/_policy_events_v1.py +++ b/sdcclient/secure/_policy_events_v1.py @@ -6,10 +6,7 @@ class PolicyEventsClientV1(_SdcCommon): def __init__(self, token="", sdc_url='https://secure.sysdig.com', ssl_verify=True, custom_headers=None): super(PolicyEventsClientV1, self).__init__(token, sdc_url, ssl_verify, custom_headers) - - self.customer_id = None self.product = "SDS" - self._policy_v2 = None def _get_policy_events_int(self, ctx): limit = ctx.get("limit", 50) diff --git a/specs/_common/agent_spec.py b/specs/_common/agent_spec.py index 22ea08bf..e0734ab1 100644 --- a/specs/_common/agent_spec.py +++ b/specs/_common/agent_spec.py @@ -1,10 +1,9 @@ import os -import time from expects import expect, have_key, have_keys from expects.matchers import _Or from expects.matchers.built_in import be_empty, contain, be_above_or_equal -from mamba import before, it, description, after +from mamba import before, it, description from sdcclient import SdcClient from specs import be_successful_api_call diff --git a/specs/monitor/dashboard_converters/dashboard_scope_spec.py b/specs/monitor/dashboard_converters/dashboard_scope_spec.py index 3ede5be6..e07aa4ad 100644 --- a/specs/monitor/dashboard_converters/dashboard_scope_spec.py +++ b/specs/monitor/dashboard_converters/dashboard_scope_spec.py @@ -1,5 +1,5 @@ -from expects import * -from mamba import * +from expects import equal, expect, be_false, start_with +from mamba import description, it from sdcclient.monitor.dashboard_converters import convert_scope_string_to_expression diff --git a/specs/monitor/dashboards_v2_spec.py b/specs/monitor/dashboards_v2_spec.py index b01ffc38..a75466db 100644 --- a/specs/monitor/dashboards_v2_spec.py +++ b/specs/monitor/dashboards_v2_spec.py @@ -3,7 +3,7 @@ import tempfile from expects import expect, have_key, have_keys, contain, equal, start_with -from expects.matchers.built_in import be_false, be_empty +from expects.matchers.built_in import be_false from mamba import before, it, context, after, description from sdcclient.monitor import DashboardsClientV2 diff --git a/specs/secure/policy_v1_spec.py b/specs/secure/policy_v1_spec.py index e9365f1b..c33b47f1 100644 --- a/specs/secure/policy_v1_spec.py +++ b/specs/secure/policy_v1_spec.py @@ -27,34 +27,37 @@ "isLimitedToContainer": True } ] + + def policy_json(): - return """\ -{ - "name": "%s", - "description": "%s", - "notificationChannelIds": [], - "severity": 0, - "hostScope": true, - "enabled": true, - "actions": %s, - "falcoConfiguration": { - "fields": [], - "ruleNameRegEx": "%s", + return f"""\ +{{ + "name": "{_POLICY_NAME}", + "description": "{_POLICY_DESCRIPTION}", + "notificationChannelIds": [], + "severity": 0, + "hostScope": true, + "enabled": true, + "actions": {json.dumps(_POLICY_ACTIONS)}, + "falcoConfiguration": {{ + "fields": [], + "ruleNameRegEx": "{_POLICY_RULES_REGEX}", "onDefault": "DEFAULT_MATCH_EFFECT_NEXT" - }, - "policyEventsCount": 0, - "isManual": true, - "isBuiltin": true, - "containerScope": true, - "modifiedOn": 1597646118000, + }}, + "policyEventsCount": 0, + "isManual": true, + "isBuiltin": true, + "containerScope": true, + "modifiedOn": 1597646118000, "createdOn": 1597646118000 -} -""" % (_POLICY_NAME, _POLICY_DESCRIPTION, json.dumps(_POLICY_ACTIONS), _POLICY_RULES_REGEX) +}} +""" + with description("Policies v1", "integration") as self: with before.all: self.clientV1 = SdSecureClientV1(sdc_url=os.getenv("SDC_SECURE_URL", "https://secure.sysdig.com"), - token=os.getenv("SDC_SECURE_TOKEN")) + token=os.getenv("SDC_SECURE_TOKEN")) with after.each: self.cleanup_policies() diff --git a/specs/secure/policy_v2_spec.py b/specs/secure/policy_v2_spec.py index b67a4c63..d9385534 100644 --- a/specs/secure/policy_v2_spec.py +++ b/specs/secure/policy_v2_spec.py @@ -47,7 +47,6 @@ def policy_json(): with after.each: self.cleanup_policies() - def cleanup_policies(self): _, res = self.client.list_policies() for policy in res: @@ -55,7 +54,6 @@ def cleanup_policies(self): ok, res = self.client.delete_policy_id(policy["id"]) expect((ok, res)).to(be_successful_api_call) - with it("is able to list all existing policies"): ok, res = self.client.list_policies() expect((ok, res)).to(be_successful_api_call) diff --git a/specs/secure/scanning/scanning_cve_report_spec.py b/specs/secure/scanning/scanning_cve_report_spec.py index 850cadc8..9ce1e9a5 100644 --- a/specs/secure/scanning/scanning_cve_report_spec.py +++ b/specs/secure/scanning/scanning_cve_report_spec.py @@ -1,7 +1,7 @@ import os -from expects import * -from mamba import * +from expects import expect, start_with +from mamba import before, context, description, it from sdcclient import SdScanningClient from specs import be_successful_api_call diff --git a/specs/secure/scanning/scanning_vulnerability_exceptions_spec.py b/specs/secure/scanning/scanning_vulnerability_exceptions_spec.py index ee1b549b..3bf8d2dc 100644 --- a/specs/secure/scanning/scanning_vulnerability_exceptions_spec.py +++ b/specs/secure/scanning/scanning_vulnerability_exceptions_spec.py @@ -97,15 +97,17 @@ def clean_bundles(self): expect((ok, res)).to(be_successful_api_call) expect(res).to( - have_keys(id=equal(self.created_exception_bundle), - items=contain( - have_keys( - id=equal(self.created_exception), - gate=equal("vulnerabilities"), - trigger_id=equal(self.created_exception_cve), - enabled=be_true, - ) - )) + have_keys( + id=equal(self.created_exception_bundle), + items=contain( + have_keys( + id=equal(self.created_exception), + gate=equal("vulnerabilities"), + trigger_id=equal(self.created_exception_cve), + enabled=be_true, + ) + ) + ) ) with it("is able to remove them"): diff --git a/specs/secure/scanning/scanning_vulnerability_spec.py b/specs/secure/scanning/scanning_vulnerability_spec.py index 13feb0f4..8d992adf 100644 --- a/specs/secure/scanning/scanning_vulnerability_spec.py +++ b/specs/secure/scanning/scanning_vulnerability_spec.py @@ -34,4 +34,4 @@ ok, res = self.client.get_vulnerability_details(id=non_existing_vuln_id) expect((ok, res)).to_not(be_successful_api_call) - expect(res).to(equal(f"No vulnerability ID provided")) + expect(res).to(equal("No vulnerability ID provided")) diff --git a/utils/sync_pagerduty_policies.py b/utils/sync_pagerduty_policies.py index 669d94ff..eb720fd1 100644 --- a/utils/sync_pagerduty_policies.py +++ b/utils/sync_pagerduty_policies.py @@ -5,28 +5,22 @@ import argparse import copy import json -import os import sys from functools import reduce import requests -sys.path.insert( - 0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) - from sdcclient import SdMonitorClient - # # Parse arguments # -parser = argparse.ArgumentParser( - description='Synchronize PagerDuty escalation policies with Sysdig, to make sure each escalation policy has a notification channel enabled in Sysdig') +parser = argparse.ArgumentParser(description='Synchronize PagerDuty escalation policies with Sysdig, ' + 'to make sure each escalation policy has a notification ' + 'channel enabled in Sysdig') parser.add_argument('sysdig-token', nargs=1, help='Sysdig API token') -parser.add_argument( - 'pagerduty-account-id', nargs=1, help='PagerDuty account ID') -parser.add_argument( - 'pagerduty-access-key', nargs=1, help='PagerDuty API access key') +parser.add_argument('pagerduty-account-id', nargs=1, help='PagerDuty account ID') +parser.add_argument('pagerduty-access-key', nargs=1, help='PagerDuty API access key') parser.add_argument( '--link', action='store_true', @@ -69,6 +63,7 @@ def run(sysdig_token, pager_duty_id, pager_duty_token, link, unlink, dry_run): pager_duty_channels = [channel for channel in res['notificationChannels'] if channel['type'] == 'PAGER_DUTY'] print('Found {} PagerDuty notification {} configured in Sysdig'.format( len(pager_duty_channels), pluralize('channel', len(pager_duty_channels)))) + # print(json.dumps(pager_duty_channels, sort_keys=True, indent=4)) # Build map of notification channel -> integration key @@ -119,7 +114,9 @@ def get_integration_map(acc, channel): service_integration_keys = {} for service in services: service['sysdig_integrations'] = [integration for integration in service['integrations'] - if 'vendor' in integration and integration['vendor'] and integration['vendor']['id'] == sysdig_vendor['id']] + if + 'vendor' in integration and integration['vendor'] and integration['vendor'][ + 'id'] == sysdig_vendor['id']] for integration in service['sysdig_integrations']: service_integration_keys[integration['integration_key']] = { @@ -190,7 +187,9 @@ def get_integration_map(acc, channel): disconnected_services = [] for service in sysdig_services: for integration in service['integrations']: - if integration['vendor'] and integration['vendor']['id'] == sysdig_vendor['id'] and integration['integration_key'] not in integration_keys: + if integration['vendor'] and \ + integration['vendor']['id'] == sysdig_vendor['id'] and \ + integration['integration_key'] not in integration_keys: disconnected_services.append({ 'service': service, 'integration': integration @@ -218,23 +217,29 @@ def get_integration_map(acc, channel): else: for service in sysdig_services: for integration in service['integrations']: - if integration['vendor'] and integration['vendor']['id'] == sysdig_vendor['id'] and integration['integration_key'] in integration_keys: + if integration['vendor'] and \ + integration['vendor']['id'] == sysdig_vendor['id'] and \ + integration['integration_key'] in integration_keys: channel = integration_keys[integration['integration_key']] if channel['name'] != policy['name']: # # rename channel to match new policy name # actions.append({ - 'info': 'Rename notification channel "{}" to policy name "{}"'.format(channel['name'], policy['name']), - 'fn': actions_factory.rename_notification_channel(channel, policy['name'], service_name) + 'info': 'Rename notification channel "{}" to policy name "{}"'.format( + channel['name'], policy['name']), + 'fn': actions_factory.rename_notification_channel(channel, policy['name'], + service_name) }) elif channel['options']['serviceName'] != service_name: # # rename channel service to service name # actions.append({ - 'info': 'Rename channel service "{}" to service name "{}"'.format(service['name'], service_name), - 'fn': actions_factory.rename_notification_channel(channel, policy['name'], service_name) + 'info': 'Rename channel service "{}" to service name "{}"'.format(service['name'], + service_name), + 'fn': actions_factory.rename_notification_channel(channel, policy['name'], + service_name) }) if len(service['integrations']) == 1 and service['name'] != service_name: @@ -481,6 +486,5 @@ def pluralize(term, count, plural=None): # let's get started! print('') - run(args['sysdig-token'][0], args['pagerduty-account-id'][0], args['pagerduty-access-key'][0], args['link'], args['unlink'], args['dry_run'])