From d35fd0a73082681e70b03bbb47f8a1f60847ff93 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Wed, 27 Mar 2019 20:54:25 -0700 Subject: [PATCH 01/48] Make example more generic --- examples/create_dashboard.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/create_dashboard.py b/examples/create_dashboard.py index 6ba642f0..7c65ec9a 100755 --- a/examples/create_dashboard.py +++ b/examples/create_dashboard.py @@ -29,7 +29,7 @@ def usage(): usage() # Name for the dashboard to create -dashboardName = "API test - cassandra in prod" +dashboardName = "Overview by Process" for opt, arg in opts: if opt in ("-d", "--dashboard"): dashboardName = arg @@ -55,8 +55,7 @@ def usage(): # in Sysdig Cloud Explore page. # You can also refer to AWS tags by using "cloudProvider.tag.*" metadata or # agent tags by using "agent.tag.*" metadata -dashboardFilter = "kubernetes.namespace.name = prod and proc.name = cassandra" - +dashboardFilter = "kubernetes.namespace.name = prod" print('Creating dashboard from view') ok, res = sdclient.create_dashboard_from_view(dashboardName, viewName, dashboardFilter) # @@ -74,9 +73,9 @@ def usage(): # # Name of the dashboard to copy -dashboardCopy = "Copy Of {}".format(dashboardName) +dashboardCopy = "Copy of {}".format(dashboardName) # Filter to apply to the new dashboard. Same as above. -dashboardFilter = "kubernetes.namespace.name = dev and proc.name = cassandra" +dashboardFilter = "kubernetes.namespace.name != prod" print('Creating dashboard from dashboard') ok, res = sdclient.create_dashboard_from_dashboard(dashboardCopy, dashboardName, dashboardFilter) From 49467dcaa996aa3930bcd3ddb3a313886c4dfb31 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Wed, 27 Mar 2019 20:54:57 -0700 Subject: [PATCH 02/48] Fix dashboard/panel scope definition --- sdcclient/_monitor.py | 98 ++++++++++++++++++++++++++++++------------- 1 file changed, 69 insertions(+), 29 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 4cc48914..1d077c99 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -1,6 +1,7 @@ import json import copy import requests +import re from sdcclient._common import _SdcCommon @@ -430,31 +431,10 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, 'groupAggregation': metric['aggregations']['group'] if 'aggregations' in metric else None, 'propertyName': property_name + str(i) }) - # - # Convert scope to format used by Sysdig Monitor - # - if scope != None: - filter_expressions = scope.strip(' \t\n\r?!.').split(" and ") - filters = [] - - for filter_expression in filter_expressions: - values = filter_expression.strip(' \t\n\r?!.').split("=") - if len(values) != 2: - return [False, "invalid scope format"] - filters.append({ - 'metric': values[0].strip(' \t\n\r?!.'), - 'op': '=', - 'value': values[1].strip(' \t\n\r"?!.'), - 'filters': None - }) - if len(filters) > 0: - panel_configuration['filter'] = { - 'filters': { - 'logic': 'and', - 'filters': filters - } - } + panel_configuration['scope'] = scope + # if chart scope is equal to dashboard scope, set it as non override + panel_configuration['overrideFilter'] = ('scope' in dashboard and dashboard['scope'] != scope) or ('scope' not in dashboard and scope != None) # # Configure panel type @@ -580,17 +560,31 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared template['isPublic'] = public template['publicToken'] = None - # # set dashboard scope to the specific parameter - # NOTE: Individual panels might override the dashboard scope, the override will NOT be reset - # + scopeExpression = self.convert_scope_string_to_expression(scope) + if scopeExpression[0] == False: + return scopeExpression template['filterExpression'] = scope + template['scopeExpressionList'] = map(lambda ex: {'operand':ex['operand'], 'operator':ex['operator'],'value':ex['value'],'displayName':'', 'isVariable':False}, scopeExpression[1]) - if 'items' in template: + if 'widgets' in template and template['widgets'] is not None: + # Default dashboards (aka Explore views) specify panels with the property `widgets`, + # while custom dashboards use `items` + template['items'] = list(template['widgets']) + del template['widgets'] + + # NOTE: Individual panels might override the dashboard scope, the override will NOT be reset + if 'items' in template and template['items'] is not None: for chart in template['items']: - if 'overrideFilter' in chart and chart['overrideFilter'] == False: + if 'overrideFilter' not in chart: + chart['overrideFilter'] = False + + if chart['overrideFilter'] == False: # patch frontend bug to hide scope override warning even when it's not really overridden chart['scope'] = scope + + # if chart scope is equal to dashboard scope, set it as non override + chart['overrideFilter'] = chart['scope'] != scope if 'annotations' in template: template['annotations'].update(annotations) @@ -754,6 +748,52 @@ def get_metrics(self): res = requests.get(self.url + '/api/data/metrics', headers=self.hdrs, verify=self.ssl_verify) return self._request_result(res) + def convert_scope_string_to_expression(self, scope): + if scope != None: + expressions = [] + string_expressions = scope.strip(' \t\n\r').split(' and ') + expression_re = re.compile('^(?Pnot )?(?P[^ ]+) (?P=|!=|in|contains|starts with) (?P.+)$') + + for string_expression in string_expressions: + matches = expression_re.match(string_expression) + + if matches == None: + return [False, 'invalid scope format'] + + is_not = matches.group('not') != None + + if matches.group('operator') == 'in': + list_value = matches.group('value').strip(' ()') + value_matches = re.findall('[^,]+', list_value) + + if len(value_matches) == 0: + return [False, 'invalid scope list format'] + + values = map(lambda v: v.strip(' "\''), value_matches) + else: + values = [matches.group('value').strip('"\'')] + + if matches.group('operator') == 'in': + operator = 'in' if is_not == False else 'notIn' + elif matches.group('operator') == '=': + operator = 'equals' + elif matches.group('operator') == '!=': + operator = 'notEquals' + elif matches.group('operator') == 'contains': + operator = 'contains' if is_not == False else 'notContains' + elif matches.group('operator') == 'starts with': + operator = 'startsWith' + + expressions.append({ + 'operand': matches.group('operand'), + 'operator': operator, + 'value': values + }) + + return [True, expressions] + else: + return [True, None] + # For backwards compatibility SdcClient = SdMonitorClient From 0fed9d92ff926b54dd4b84f96c12c930b376b097 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 28 Mar 2019 08:28:10 -0700 Subject: [PATCH 03/48] Add note and refactor name --- sdcclient/_monitor.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 1d077c99..a564eb38 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -561,7 +561,7 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared template['publicToken'] = None # set dashboard scope to the specific parameter - scopeExpression = self.convert_scope_string_to_expression(scope) + scopeExpression = self._convert_scope_string_to_expression(scope) if scopeExpression[0] == False: return scopeExpression template['filterExpression'] = scope @@ -748,7 +748,15 @@ def get_metrics(self): res = requests.get(self.url + '/api/data/metrics', headers=self.hdrs, verify=self.ssl_verify) return self._request_result(res) - def convert_scope_string_to_expression(self, scope): + def _convert_scope_string_to_expression(self, scope): + '''**Description** + Internal function to convert a filter string to a filter object to be used with dashboards. + ''' + # + # NOTE: The supported grammar is not perfectly aligned with the grammar supported by the Sysdig backend. + # Proper grammar implementation will happen soon. + # For practical purposes, the parsing will have equivalent results. + # if scope != None: expressions = [] string_expressions = scope.strip(' \t\n\r').split(' and ') From a7d34cdb86c78a417e44a812e48fdc993b5b500e Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 28 Mar 2019 09:19:46 -0700 Subject: [PATCH 04/48] Make API endpoints customizable --- sdcclient/_monitor.py | 22 ++++++++++++---------- sdcclient/_monitor_v1.py | 17 +++++++++++++++++ 2 files changed, 29 insertions(+), 10 deletions(-) create mode 100644 sdcclient/_monitor_v1.py diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index a564eb38..2eb28cb9 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -16,6 +16,8 @@ class SdMonitorClient(_SdcCommon): def __init__(self, token="", sdc_url='https://app.sysdigcloud.com', ssl_verify=True): super(SdMonitorClient, self).__init__(token, sdc_url, ssl_verify) self.product = "SDC" + self._dashboards_api_endpoint = '/ui/dashboards' + self._default_dashboards_api_endpoint = '/api/defaultDashboards' def get_alerts(self): '''**Description** @@ -263,7 +265,7 @@ def set_explore_grouping_hierarchy(self, new_hierarchy): return [True, None] def get_views_list(self): - res = requests.get(self.url + '/api/defaultDashboards', headers=self.hdrs, + res = requests.get(self.url + self._default_dashboards_api_endpoint, headers=self.hdrs, verify=self.ssl_verify) if not self._checkResponse(res): return [False, self.lasterr] @@ -286,7 +288,7 @@ def get_view(self, name): if not id: return [False, 'view ' + name + ' not found'] - res = requests.get(self.url + '/api/defaultDashboards/' + id, headers=self.hdrs, + res = requests.get(self.url + self._default_dashboards_api_endpoint + '/' + id, headers=self.hdrs, verify=self.ssl_verify) return self._request_result(res) @@ -300,7 +302,7 @@ def get_dashboards(self): **Example** `examples/list_dashboards.py `_ ''' - res = requests.get(self.url + '/ui/dashboards', headers=self.hdrs, verify=self.ssl_verify) + res = requests.get(self.url + self._dashboards_api_endpoint, headers=self.hdrs, verify=self.ssl_verify) return self._request_result(res) def find_dashboard_by(self, name=None): @@ -330,7 +332,7 @@ def create_item(configuration): return [True, dashboards] def create_dashboard_with_configuration(self, configuration): - res = requests.post(self.url + '/ui/dashboards', headers=self.hdrs, data=json.dumps({'dashboard': configuration}), + res = requests.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': configuration}), verify=self.ssl_verify) return self._request_result(res) @@ -357,7 +359,7 @@ def create_dashboard(self, name): # # Create the new dashboard # - res = requests.post(self.url + '/ui/dashboards', headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}), + res = requests.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}), verify=self.ssl_verify) return self._request_result(res) @@ -498,7 +500,7 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, # # Update dashboard # - res = requests.put(self.url + '/ui/dashboards/' + str(dashboard['id']), headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}), + res = requests.put(self.url + self._dashboards_api_endpoint + '/' + str(dashboard['id']), headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}), verify=self.ssl_verify) return self._request_result(res) @@ -538,7 +540,7 @@ def filter_fn(panel): # # Update dashboard # - res = requests.put(self.url + '/ui/dashboards/' + str(dashboard['id']), headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}), + res = requests.put(self.url + self._dashboards_api_endpoint + '/' + str(dashboard['id']), headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}), verify=self.ssl_verify) return self._request_result(res) else: @@ -596,7 +598,7 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared # # Create the new dashboard # - res = requests.post(self.url + '/ui/dashboards', headers=self.hdrs, data=json.dumps({'dashboard': template}), verify=self.ssl_verify) + res = requests.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': template}), verify=self.ssl_verify) return self._request_result(res) def create_dashboard_from_view(self, newdashname, viewname, filter, shared=False, public=False, annotations={}): @@ -655,7 +657,7 @@ def create_dashboard_from_dashboard(self, newdashname, templatename, filter, sha # # Get the list of dashboards from the server # - res = requests.get(self.url + '/ui/dashboards', headers=self.hdrs, verify=self.ssl_verify) + res = requests.get(self.url + self._dashboards_api_endpoint, headers=self.hdrs, verify=self.ssl_verify) if not self._checkResponse(res): return [False, self.lasterr] @@ -729,7 +731,7 @@ def delete_dashboard(self, dashboard): if 'id' not in dashboard: return [False, "Invalid dashboard format"] - res = requests.delete(self.url + '/ui/dashboards/' + str(dashboard['id']), headers=self.hdrs, verify=self.ssl_verify) + res = requests.delete(self.url + self._dashboards_api_endpoint + '/' + str(dashboard['id']), headers=self.hdrs, verify=self.ssl_verify) if not self._checkResponse(res): return [False, self.lasterr] diff --git a/sdcclient/_monitor_v1.py b/sdcclient/_monitor_v1.py new file mode 100644 index 00000000..85372379 --- /dev/null +++ b/sdcclient/_monitor_v1.py @@ -0,0 +1,17 @@ +from sdcclient._monitor import SdMonitorClient + +try: + basestring +except NameError: + basestring = str + + +class SdMonitorClientV1(SdMonitorClient): + '''**Description** + Handles dashboards version 1 (ie. up to February 2019). For later Sysdig Monitor versions, please use :class:`~SdMonitorClient` instead. + ''' + + def __init__(self, token="", sdc_url='https://app.sysdigcloud.com', ssl_verify=True): + super(SdMonitorClientV1, self).__init__(token, sdc_url, ssl_verify) + self._dashboards_api_endpoint = '/ui/dashboards' + self._default_dashboards_api_endpoint = '/api/defaultDashboards' From b5f0691db093c29229a07dbafd76bd551c48d6d5 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 28 Mar 2019 15:48:09 -0700 Subject: [PATCH 05/48] Move v1 to separate module --- sdcclient/__init__.py | 1 + sdcclient/_monitor_v1.py | 272 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 272 insertions(+), 1 deletion(-) diff --git a/sdcclient/__init__.py b/sdcclient/__init__.py index 2c176ac9..9081724c 100644 --- a/sdcclient/__init__.py +++ b/sdcclient/__init__.py @@ -1,4 +1,5 @@ from sdcclient._monitor import SdcClient from sdcclient._monitor import SdMonitorClient +from sdcclient._monitor_v1 import SdMonitorClientV1 from sdcclient._secure import SdSecureClient from sdcclient._scanning import SdScanningClient diff --git a/sdcclient/_monitor_v1.py b/sdcclient/_monitor_v1.py index 85372379..0aafb1a9 100644 --- a/sdcclient/_monitor_v1.py +++ b/sdcclient/_monitor_v1.py @@ -1,3 +1,8 @@ +import json +import copy +import requests +import re + from sdcclient._monitor import SdMonitorClient try: @@ -11,7 +16,272 @@ class SdMonitorClientV1(SdMonitorClient): Handles dashboards version 1 (ie. up to February 2019). For later Sysdig Monitor versions, please use :class:`~SdMonitorClient` instead. ''' - def __init__(self, token="", sdc_url='https://app.sysdigcloud.com', ssl_verify=True): + def __init__(self, token="", sdc_url='https://app-staging2.sysdigcloud.com', ssl_verify=True): super(SdMonitorClientV1, self).__init__(token, sdc_url, ssl_verify) self._dashboards_api_endpoint = '/ui/dashboards' self._default_dashboards_api_endpoint = '/api/defaultDashboards' + + def create_dashboard_from_template(self, dashboard_name, template, scope, shared=False, public=False, annotations={}): + if scope is not None: + if isinstance(scope, basestring) == False: + return [False, 'Invalid scope format: Expected a string'] + + # + # Clean up the dashboard we retireved so it's ready to be pushed + # + template['id'] = None + template['version'] = None + template['schema'] = 1 + template['name'] = dashboard_name + template['isShared'] = shared + template['isPublic'] = public + template['publicToken'] = None + + # set dashboard scope to the specific parameter + scopeExpression = self._convert_scope_string_to_expression(scope) + if scopeExpression[0] == False: + return scopeExpression + template['filterExpression'] = scope + template['scopeExpressionList'] = map(lambda ex: {'operand':ex['operand'], 'operator':ex['operator'],'value':ex['value'],'displayName':'', 'isVariable':False}, scopeExpression[1]) + + if 'widgets' in template and template['widgets'] is not None: + # Default dashboards (aka Explore views) specify panels with the property `widgets`, + # while custom dashboards use `items` + template['items'] = list(template['widgets']) + del template['widgets'] + + # NOTE: Individual panels might override the dashboard scope, the override will NOT be reset + if 'items' in template and template['items'] is not None: + for chart in template['items']: + if 'overrideFilter' not in chart: + chart['overrideFilter'] = False + + if chart['overrideFilter'] == False: + # patch frontend bug to hide scope override warning even when it's not really overridden + chart['scope'] = scope + + # if chart scope is equal to dashboard scope, set it as non override + chart['overrideFilter'] = chart['scope'] != scope + + if 'annotations' in template: + template['annotations'].update(annotations) + else: + template['annotations'] = annotations + + template['annotations']['createdByEngine'] = True + + # + # Create the new dashboard + # + res = requests.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': template}), verify=self.ssl_verify) + return self._request_result(res) + + def create_dashboard(self, name): + ''' + **Description** + Creates an empty dashboard. You can then add panels by using ``add_dashboard_panel``. + + **Arguments** + - **name**: the name of the dashboard that will be created. + + **Success Return Value** + A dictionary showing the details of the new dashboard. + + **Example** + `examples/dashboard.py `_ + ''' + dashboard_configuration = { + 'name': name, + 'schema': 2, + 'items': [] + } + + # + # Create the new dashboard + # + res = requests.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}), + verify=self.ssl_verify) + return self._request_result(res) + + def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, sort_by=None, limit=None, layout=None): + """**Description** + Adds a panel to the dashboard. A panel can be a time series, or a top chart (i.e. bar chart), or a number panel. + + **Arguments** + - **dashboard**: dashboard to edit + - **name**: name of the new panel + - **panel_type**: type of the new panel. Valid values are: ``timeSeries``, ``top``, ``number`` + - **metrics**: a list of dictionaries, specifying the metrics to show in the panel, and optionally, if there is only one metric, a grouping key to segment that metric by. A metric is any of the entries that can be found in the *Metrics* section of the Explore page in Sysdig Monitor. Metric entries require an *aggregations* section specifying how to aggregate the metric across time and groups of containers/hosts. A grouping key is any of the entries that can be found in the *Show* or *Segment By* sections of the Explore page in Sysdig Monitor. Refer to the examples section below for ready to use code snippets. Note, certain panels allow certain combinations of metrics and grouping keys: + - ``timeSeries``: 1 or more metrics OR 1 metric + 1 grouping key + - ``top``: 1 or more metrics OR 1 metric + 1 grouping key + - ``number``: 1 metric only + - **scope**: filter to apply to the panel; must be based on metadata available in Sysdig Monitor; Example: *kubernetes.namespace.name='production' and container.image='nginx'*. + - **sort_by**: Data sorting; The parameter is optional and it's a dictionary of ``metric`` and ``mode`` (it can be ``desc`` or ``asc``) + - **limit**: This parameter sets the limit on the number of lines/bars shown in a ``timeSeries`` or ``top`` panel. In the case of more entities being available than the limit, the top entities according to the sort will be shown. The default value is 10 for ``top`` panels (for ``timeSeries`` the default is defined by Sysdig Monitor itself). Note that increasing the limit above 10 is not officially supported and may cause performance and rendering issues + - **layout**: Size and position of the panel. The dashboard layout is defined by a grid of 12 columns, each row height is equal to the column height. For example, say you want to show 2 panels at the top: one panel might be 6 x 3 (half the width, 3 rows height) located in row 1 and column 1 (top-left corner of the viewport), the second panel might be 6 x 3 located in row 1 and position 7. The location is specified by a dictionary of ``row`` (row position), ``col`` (column position), ``size_x`` (width), ``size_y`` (height). + + **Success Return Value** + A dictionary showing the details of the edited dashboard. + + **Example** + `examples/dashboard.py `_ + """ + panel_configuration = { + 'name': name, + 'showAs': None, + 'showAsType': None, + 'metrics': [], + 'gridConfiguration': { + 'col': 1, + 'row': 1, + 'size_x': 12, + 'size_y': 6 + } + } + + if panel_type == 'timeSeries': + # + # In case of a time series, the current dashboard implementation + # requires the timestamp to be explicitly specified as "key". + # However, this function uses the same abstraction of the data API + # that doesn't require to specify a timestamp key (you only need to + # specify time window and sampling) + # + metrics = copy.copy(metrics) + metrics.insert(0, {'id': 'timestamp'}) + + # + # Convert list of metrics to format used by Sysdig Monitor + # + property_names = {} + k_count = 0 + v_count = 0 + for i, metric in enumerate(metrics): + property_name = 'v' if 'aggregations' in metric else 'k' + + if property_name == 'k': + i = k_count + k_count += 1 + else: + i = v_count + v_count += 1 + property_names[metric['id']] = property_name + str(i) + + panel_configuration['metrics'].append({ + 'metricId': metric['id'], + 'aggregation': metric['aggregations']['time'] if 'aggregations' in metric else None, + 'groupAggregation': metric['aggregations']['group'] if 'aggregations' in metric else None, + 'propertyName': property_name + str(i) + }) + + panel_configuration['scope'] = scope + # if chart scope is equal to dashboard scope, set it as non override + panel_configuration['overrideFilter'] = ('scope' in dashboard and dashboard['scope'] != scope) or ('scope' not in dashboard and scope != None) + + # + # Configure panel type + # + if panel_type == 'timeSeries': + panel_configuration['showAs'] = 'timeSeries' + panel_configuration['showAsType'] = 'line' + + if limit != None: + panel_configuration['paging'] = { + 'from': 0, + 'to': limit - 1 + } + + elif panel_type == 'number': + panel_configuration['showAs'] = 'summary' + panel_configuration['showAsType'] = 'summary' + elif panel_type == 'top': + panel_configuration['showAs'] = 'top' + panel_configuration['showAsType'] = 'bars' + + if sort_by is None: + panel_configuration['sorting'] = [{ + 'id': 'v0', + 'mode': 'desc' + }] + else: + panel_configuration['sorting'] = [{ + 'id': property_names[sort_by['metric']], + 'mode': sort_by['mode'] + }] + + if limit is None: + panel_configuration['paging'] = { + 'from': 0, + 'to': 10 + } + else: + panel_configuration['paging'] = { + 'from': 0, + 'to': limit - 1 + } + + # + # Configure layout + # + if layout != None: + panel_configuration['gridConfiguration'] = layout + + # + # Clone existing dashboard... + # + dashboard_configuration = copy.deepcopy(dashboard) + dashboard_configuration['id'] = None + + # + # ... and add the new panel + # + dashboard_configuration['items'].append(panel_configuration) + + # + # Update dashboard + # + res = requests.put(self.url + self._dashboards_api_endpoint + '/' + str(dashboard['id']), headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}), + verify=self.ssl_verify) + return self._request_result(res) + + def remove_dashboard_panel(self, dashboard, panel_name): + '''**Description** + Removes a panel from the dashboard. The panel to remove is identified by the specified ``name``. + + **Arguments** + - **name**: name of the panel to find and remove + + **Success Return Value** + A dictionary showing the details of the edited dashboard. + + **Example** + `examples/dashboard.py `_ + ''' + # + # Clone existing dashboard... + # + dashboard_configuration = copy.deepcopy(dashboard) + dashboard_configuration['id'] = None + + # + # ... find the panel + # + def filter_fn(panel): + return panel['name'] == panel_name + panels = list(filter(filter_fn, dashboard_configuration['items'])) + + if len(panels) > 0: + # + # ... and remove it + # + for panel in panels: + dashboard_configuration['items'].remove(panel) + + # + # Update dashboard + # + res = requests.put(self.url + self._dashboards_api_endpoint + '/' + str(dashboard['id']), headers=self.hdrs, data=json.dumps({'dashboard': dashboard_configuration}), + verify=self.ssl_verify) + return self._request_result(res) + else: + return [False, 'Not found'] From 394622c1c16256285195e3a366b4cb83215b1c71 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 28 Mar 2019 15:49:23 -0700 Subject: [PATCH 06/48] Support dashboards API v2 --- sdcclient/_monitor.py | 140 ++++++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 65 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 2eb28cb9..e70ec17c 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -16,8 +16,8 @@ class SdMonitorClient(_SdcCommon): def __init__(self, token="", sdc_url='https://app.sysdigcloud.com', ssl_verify=True): super(SdMonitorClient, self).__init__(token, sdc_url, ssl_verify) self.product = "SDC" - self._dashboards_api_endpoint = '/ui/dashboards' - self._default_dashboards_api_endpoint = '/api/defaultDashboards' + self._dashboards_api_endpoint = '/api/v2/dashboards' + self._default_dashboards_api_endpoint = '/api/v2/defaultDashboards' def get_alerts(self): '''**Description** @@ -332,7 +332,14 @@ def create_item(configuration): return [True, dashboards] def create_dashboard_with_configuration(self, configuration): - res = requests.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': configuration}), + # Remove id and version properties if already set + configuration_clone = copy.deepcopy(configuration) + if 'id' in configuration_clone: + del configuration_clone['id'] + if 'version' in configuration_clone: + del configuration_clone['version'] + + res = requests.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': configuration_clone}), verify=self.ssl_verify) return self._request_result(res) @@ -352,8 +359,8 @@ def create_dashboard(self, name): ''' dashboard_configuration = { 'name': name, - 'schema': 1, - 'items': [] + 'schema': 2, + 'widgets': [] } # @@ -363,7 +370,7 @@ def create_dashboard(self, name): verify=self.ssl_verify) return self._request_result(res) - def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, sort_by=None, limit=None, layout=None): + def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, sort_direction='desc', limit=None, layout=None): """**Description** Adds a panel to the dashboard. A panel can be a time series, or a top chart (i.e. bar chart), or a number panel. @@ -376,7 +383,7 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, - ``top``: 1 or more metrics OR 1 metric + 1 grouping key - ``number``: 1 metric only - **scope**: filter to apply to the panel; must be based on metadata available in Sysdig Monitor; Example: *kubernetes.namespace.name='production' and container.image='nginx'*. - - **sort_by**: Data sorting; The parameter is optional and it's a dictionary of ``metric`` and ``mode`` (it can be ``desc`` or ``asc``) + - **sort_direction**: Data sorting; The parameter is optional and it's a string identifying the sorting direction (it can be ``desc`` or ``asc``) - **limit**: This parameter sets the limit on the number of lines/bars shown in a ``timeSeries`` or ``top`` panel. In the case of more entities being available than the limit, the top entities according to the sort will be shown. The default value is 10 for ``top`` panels (for ``timeSeries`` the default is defined by Sysdig Monitor itself). Note that increasing the limit above 10 is not officially supported and may cause performance and rendering issues - **layout**: Size and position of the panel. The dashboard layout is defined by a grid of 12 columns, each row height is equal to the column height. For example, say you want to show 2 panels at the top: one panel might be 6 x 3 (half the width, 3 rows height) located in row 1 and column 1 (top-left corner of the viewport), the second panel might be 6 x 3 located in row 1 and position 7. The location is specified by a dictionary of ``row`` (row position), ``col`` (column position), ``size_x`` (width), ``size_y`` (height). @@ -389,7 +396,6 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, panel_configuration = { 'name': name, 'showAs': None, - 'showAsType': None, 'metrics': [], 'gridConfiguration': { 'col': 1, @@ -399,6 +405,14 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, } } + # + # Set unique ID (incremental from 1) + # + id = 1 + while len(filter(lambda w: w['id'] == id, dashboard['widgets'])) > 0: + id += 1 + panel_configuration['id'] = id + if panel_type == 'timeSeries': # # In case of a time series, the current dashboard implementation @@ -428,56 +442,60 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, property_names[metric['id']] = property_name + str(i) panel_configuration['metrics'].append({ - 'metricId': metric['id'], - 'aggregation': metric['aggregations']['time'] if 'aggregations' in metric else None, + 'id': metric['id'], + 'timeAggregation': metric['aggregations']['time'] if 'aggregations' in metric else None, 'groupAggregation': metric['aggregations']['group'] if 'aggregations' in metric else None, 'propertyName': property_name + str(i) }) panel_configuration['scope'] = scope # if chart scope is equal to dashboard scope, set it as non override - panel_configuration['overrideFilter'] = ('scope' in dashboard and dashboard['scope'] != scope) or ('scope' not in dashboard and scope != None) - + panel_configuration['overrideScope'] = ('scope' in dashboard and dashboard['scope'] != scope) or ('scope' not in dashboard and scope != None) + + if 'custom_display_options' not in panel_configuration: + panel_configuration['custom_display_options'] = { + 'valueLimit': { + 'count': 10, + 'direction': 'desc' + }, + 'histogram': { + 'numberOfBuckets': 10 + }, + 'yAxisScale': 'linear', + 'yAxisLeftDomain': { + 'from': 0, + 'to': None + }, + 'yAxisRightDomain': { + 'from': 0, + 'to': None + }, + 'xAxis': { + 'from': 0, + 'to': None + } + } # # Configure panel type # if panel_type == 'timeSeries': panel_configuration['showAs'] = 'timeSeries' - panel_configuration['showAsType'] = 'line' if limit != None: - panel_configuration['paging'] = { - 'from': 0, - 'to': limit - 1 + panel_configuration['custom_display_options']['valueLimit'] = { + 'count': limit, + 'direction': 'desc' } elif panel_type == 'number': panel_configuration['showAs'] = 'summary' - panel_configuration['showAsType'] = 'summary' elif panel_type == 'top': panel_configuration['showAs'] = 'top' - panel_configuration['showAsType'] = 'bars' - - if sort_by is None: - panel_configuration['sorting'] = [{ - 'id': 'v0', - 'mode': 'desc' - }] - else: - panel_configuration['sorting'] = [{ - 'id': property_names[sort_by['metric']], - 'mode': sort_by['mode'] - }] - if limit is None: - panel_configuration['paging'] = { - 'from': 0, - 'to': 10 - } - else: - panel_configuration['paging'] = { - 'from': 0, - 'to': limit - 1 + if limit != None: + panel_configuration['custom_display_options']['valueLimit'] = { + 'count': limit, + 'direction': sort_direction } # @@ -490,12 +508,11 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, # Clone existing dashboard... # dashboard_configuration = copy.deepcopy(dashboard) - dashboard_configuration['id'] = None # # ... and add the new panel # - dashboard_configuration['items'].append(panel_configuration) + dashboard_configuration['widgets'].append(panel_configuration) # # Update dashboard @@ -521,21 +538,21 @@ def remove_dashboard_panel(self, dashboard, panel_name): # Clone existing dashboard... # dashboard_configuration = copy.deepcopy(dashboard) - dashboard_configuration['id'] = None + del dashboard_configuration['id'] # # ... find the panel # def filter_fn(panel): return panel['name'] == panel_name - panels = list(filter(filter_fn, dashboard_configuration['items'])) + panels = list(filter(filter_fn, dashboard_configuration['widgets'])) if len(panels) > 0: # # ... and remove it # for panel in panels: - dashboard_configuration['items'].remove(panel) + dashboard_configuration['widgets'].remove(panel) # # Update dashboard @@ -556,44 +573,37 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared # template['id'] = None template['version'] = None - template['schema'] = 1 + template['schema'] = 2 template['name'] = dashboard_name - template['isShared'] = shared - template['isPublic'] = public + template['shared'] = shared + template['public'] = public template['publicToken'] = None # set dashboard scope to the specific parameter scopeExpression = self._convert_scope_string_to_expression(scope) if scopeExpression[0] == False: return scopeExpression - template['filterExpression'] = scope - template['scopeExpressionList'] = map(lambda ex: {'operand':ex['operand'], 'operator':ex['operator'],'value':ex['value'],'displayName':'', 'isVariable':False}, scopeExpression[1]) - - if 'widgets' in template and template['widgets'] is not None: - # Default dashboards (aka Explore views) specify panels with the property `widgets`, - # while custom dashboards use `items` - template['items'] = list(template['widgets']) - del template['widgets'] + template['scopeExpressionList'] = map(lambda ex: {'operand':ex['operand'], 'operator':ex['operator'],'value':ex['value'],'displayName':'', 'variable':False}, scopeExpression[1]) # NOTE: Individual panels might override the dashboard scope, the override will NOT be reset - if 'items' in template and template['items'] is not None: - for chart in template['items']: - if 'overrideFilter' not in chart: - chart['overrideFilter'] = False + if 'widgets' in template and template['widgets'] is not None: + for chart in template['widgets']: + if 'overrideScope' not in chart: + chart['overrideScope'] = False - if chart['overrideFilter'] == False: + if chart['overrideScope'] == False: # patch frontend bug to hide scope override warning even when it's not really overridden chart['scope'] = scope # if chart scope is equal to dashboard scope, set it as non override - chart['overrideFilter'] = chart['scope'] != scope + chart['overrideScope'] = chart['scope'] != scope - if 'annotations' in template: - template['annotations'].update(annotations) - else: - template['annotations'] = annotations + # if 'annotations' in template: + # template['annotations'].update(annotations) + # else: + # template['annotations'] = annotations - template['annotations']['createdByEngine'] = True + # template['annotations']['createdByEngine'] = True # # Create the new dashboard From b0f237abfa6bb2d4dc42d447ee736ffa3f7b6e09 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 28 Mar 2019 15:50:21 -0700 Subject: [PATCH 07/48] Update examples to work with dashboards API v2 --- examples/create_dashboard.py | 8 ++++---- examples/dashboard.py | 14 +++++++------- examples/dashboard_save_load.py | 8 ++++---- examples/delete_dashboard.py | 4 ++-- examples/download_dashboards.py | 6 +++--- examples/list_dashboards.py | 6 +++--- examples/restore_dashboards.py | 21 ++++----------------- 7 files changed, 27 insertions(+), 40 deletions(-) diff --git a/examples/create_dashboard.py b/examples/create_dashboard.py index 7c65ec9a..097e472d 100755 --- a/examples/create_dashboard.py +++ b/examples/create_dashboard.py @@ -10,7 +10,7 @@ import os import sys sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) -from sdcclient import SdcClient +from sdcclient import SdMonitorClient # @@ -42,7 +42,7 @@ def usage(): # # Instantiate the SDC client # -sdclient = SdcClient(sdc_token) +sdclient = SdMonitorClient(sdc_token) # # Create the new dashboard, applying to cassandra in production @@ -55,7 +55,7 @@ def usage(): # in Sysdig Cloud Explore page. # You can also refer to AWS tags by using "cloudProvider.tag.*" metadata or # agent tags by using "agent.tag.*" metadata -dashboardFilter = "kubernetes.namespace.name = prod" +dashboardFilter = 'proc.name = "cassandra"' print('Creating dashboard from view') ok, res = sdclient.create_dashboard_from_view(dashboardName, viewName, dashboardFilter) # @@ -75,7 +75,7 @@ def usage(): # Name of the dashboard to copy dashboardCopy = "Copy of {}".format(dashboardName) # Filter to apply to the new dashboard. Same as above. -dashboardFilter = "kubernetes.namespace.name != prod" +dashboardFilter = 'proc.name != "cassandra"' print('Creating dashboard from dashboard') ok, res = sdclient.create_dashboard_from_dashboard(dashboardCopy, dashboardName, dashboardFilter) diff --git a/examples/dashboard.py b/examples/dashboard.py index 79c5eb08..86fe4feb 100755 --- a/examples/dashboard.py +++ b/examples/dashboard.py @@ -8,7 +8,7 @@ import os import sys sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) -from sdcclient import SdcClient +from SdMonitorClient import SdMonitorClient # @@ -39,7 +39,7 @@ def usage(): # # Instantiate the SDC client # -sdclient = SdcClient(sdc_token) +sdclient = SdMonitorClient(sdc_token) # @@ -77,10 +77,10 @@ def usage(): panel_name = 'CPU Over Time' panel_type = 'timeSeries' metrics = [ - {'id': 'kubernetes.pod.name'}, + {'id': 'proc.name'}, {'id': 'cpu.used.percent', 'aggregations': {'time': 'avg', 'group': 'avg'}} ] -scope = 'kubernetes.namespace.name = "dev" and kubernetes.replicationController.name = "cassandra"' +scope = 'proc.name = "cassandra"' ok, res = sdclient.add_dashboard_panel(dashboard_configuration, panel_name, panel_type, metrics, scope=scope) # Check the result @@ -101,9 +101,9 @@ def usage(): {'id': 'host.hostName'}, {'id': 'cpu.used.percent', 'aggregations': {'time': 'avg', 'group': 'avg'}} ] -sort_by = {'metric': 'cpu.used.percent', 'mode': 'desc'} +sort_direction = 'desc' limit = 10 -ok, res = sdclient.add_dashboard_panel(dashboard_configuration, panel_name, panel_type, metrics, sort_by=sort_by, limit=limit) +ok, res = sdclient.add_dashboard_panel(dashboard_configuration, panel_name, panel_type, metrics, sort_direction=sort_direction, limit=limit) # Check the result if ok: @@ -137,7 +137,7 @@ def usage(): # # Remove a panel # -ok, res = sdclient.remove_dashboard_panel(dashboard_configuration, 'CPU') +ok, res = sdclient.remove_dashboard_panel(dashboard_configuration, 'CPU Over Time') # Check the result if ok: diff --git a/examples/dashboard_save_load.py b/examples/dashboard_save_load.py index b37c1f37..bc1494c9 100644 --- a/examples/dashboard_save_load.py +++ b/examples/dashboard_save_load.py @@ -7,7 +7,7 @@ import sys import json sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) -from sdcclient import SdcClient +from sdcclient import SdMonitorClient # # Parse arguments @@ -22,7 +22,7 @@ # # Instantiate the SDC client # -sdclient = SdcClient(sdc_token) +sdclient = SdMonitorClient(sdc_token) # # Serialize the first user dashboard to disk @@ -34,7 +34,7 @@ sys.exit(1) if len(res[u'dashboards']) > 0: - with open('dashboard.json', 'w') as outf: + with open('dashboard.json', 'w') as outf: json.dump(res[u'dashboards'][0], outf) else: print('the user has no dashboards. Exiting.') @@ -44,7 +44,7 @@ # Now create the dashboard from the file. We use a filter for the Cassandra process # as an example. # -dashboardFilter = "proc.name = cassandra" +dashboardFilter = 'proc.name = "cassandra"' ok, res = sdclient.create_dashboard_from_file('test dasboard from file', 'dashboard.json', dashboardFilter) diff --git a/examples/delete_dashboard.py b/examples/delete_dashboard.py index 6347f280..a5bbbedd 100755 --- a/examples/delete_dashboard.py +++ b/examples/delete_dashboard.py @@ -7,7 +7,7 @@ import os import sys sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) -from sdcclient import SdcClient +from sdcclient import SdMonitorClient # @@ -38,7 +38,7 @@ def usage(): # # Instantiate the SDC client # -sdclient = SdcClient(sdc_token) +sdclient = SdMonitorClient(sdc_token) # # List the dashboards diff --git a/examples/download_dashboards.py b/examples/download_dashboards.py index 1fa23c19..00f39c62 100644 --- a/examples/download_dashboards.py +++ b/examples/download_dashboards.py @@ -8,7 +8,7 @@ import zipfile import json sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) -from sdcclient import SdcClient +from sdcclient import SdMonitorClient def zipdir(path, ziph): @@ -52,7 +52,7 @@ def cleanup_dir(path): # # Instantiate the SDC client # -sdclient = SdcClient(sdc_token, sdc_url='https://app.sysdigcloud.com') +sdclient = SdMonitorClient(sdc_token) # # Fire the request. @@ -79,7 +79,7 @@ def cleanup_dir(path): file_path = os.path.join(sysdig_dashboard_dir, str(db['id'])) f = open(file_path, 'w') f.write(json.dumps(db)) - print("Name: %s, # Charts: %d" % (db['name'], len(db['items']))) + print("Name: %s, # Charts: %d" % (db['name'], len(db['widgets']))) f.close() zipf = zipfile.ZipFile(dashboard_state_file, 'w', zipfile.ZIP_DEFLATED) diff --git a/examples/list_dashboards.py b/examples/list_dashboards.py index c9c3a65b..b958b538 100755 --- a/examples/list_dashboards.py +++ b/examples/list_dashboards.py @@ -6,7 +6,7 @@ import os import sys sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) -from sdcclient import SdcClient +from sdcclient import SdMonitorClient # # Parse arguments @@ -21,7 +21,7 @@ # # Instantiate the SDC client # -sdclient = SdcClient(sdc_token) +sdclient = SdMonitorClient(sdc_token) # # Fire the request. @@ -36,4 +36,4 @@ sys.exit(1) for db in res['dashboards']: - print("Name: %s, # Charts: %d" % (db['name'], len(db['items'] if 'items' in db else []))) + print("Name: %s, # Charts: %d" % (db['name'], len(db['widgets'] if 'widgets' in db else []))) diff --git a/examples/restore_dashboards.py b/examples/restore_dashboards.py index 4d201487..01a22beb 100644 --- a/examples/restore_dashboards.py +++ b/examples/restore_dashboards.py @@ -8,7 +8,7 @@ import zipfile import json sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) -from sdcclient import SdcClient +from sdcclient import SdMonitorClient # # Parse arguments @@ -24,32 +24,19 @@ # # Instantiate the SDC client # -sdclient = SdcClient(sdc_token) +sdclient = SdMonitorClient(sdc_token) zipf = zipfile.ZipFile(dashboard_state_file, 'r') - -dashboard_conf_items = ['showAsType', 'filterRoot', 'linkMetrics', - 'singleTimeNavigation', 'gridConfiguration', 'responsive', - 'nodesNoiseFilter', 'compareWith', 'format', 'linksNoiseFilter', - 'filterProcesses', 'isLegendExpanded', 'inhertitTimeNavigation', - 'schema', 'sortAscending', 'mapDataLimit', 'metrics', 'filterExtNodes', - 'sorting', 'name', 'sourceExploreView', 'items', 'showAs', 'eventsFilter', - 'timeMode', 'isShared', 'sourceDrilldownView', 'filterExpression'] - for info in zipf.infolist(): data = zipf.read(info.filename) try: j = json.loads(data) except ValueError: - print('Non-JSON item found in ZIP: ' + info.filename + ' (skipping)') + print('Invalid JSON file found in ZIP file ' + info.filename + ': skipping') continue - k = {} - for item in j.keys(): - if item in dashboard_conf_items: - k[item] = j[item] - ok, res = sdclient.create_dashboard_with_configuration(k) + ok, res = sdclient.create_dashboard_with_configuration(j) if ok: print('Restored Dashboard named: ' + j['name']) else: From 74502515d38ea8d75140689d4c86a18fae33cd0a Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 28 Mar 2019 16:42:36 -0700 Subject: [PATCH 08/48] Fix import --- examples/dashboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/dashboard.py b/examples/dashboard.py index 86fe4feb..79adc255 100755 --- a/examples/dashboard.py +++ b/examples/dashboard.py @@ -8,7 +8,7 @@ import os import sys sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) -from SdMonitorClient import SdMonitorClient +from sdcclient import SdMonitorClient # From 56fc51841345da6e81d67cd8af3eab50fa834048 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 28 Mar 2019 17:43:05 -0700 Subject: [PATCH 09/48] Add versioning to dashboard backups --- examples/dashboard_save_load.py | 3 +- examples/download_dashboards.py | 6 ++-- sdcclient/_monitor.py | 54 +++++++++++++++++++++++++++------ sdcclient/_monitor_v1.py | 1 + 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/examples/dashboard_save_load.py b/examples/dashboard_save_load.py index bc1494c9..2b39ee61 100644 --- a/examples/dashboard_save_load.py +++ b/examples/dashboard_save_load.py @@ -34,8 +34,7 @@ sys.exit(1) if len(res[u'dashboards']) > 0: - with open('dashboard.json', 'w') as outf: - json.dump(res[u'dashboards'][0], outf) + sdclient.save_dashboard_to_file(res[u'dashboards'][0], 'dashboard.json') else: print('the user has no dashboards. Exiting.') sys.exit(0) diff --git a/examples/download_dashboards.py b/examples/download_dashboards.py index 00f39c62..a205320b 100644 --- a/examples/download_dashboards.py +++ b/examples/download_dashboards.py @@ -76,11 +76,9 @@ def cleanup_dir(path): for db in res['dashboards']: - file_path = os.path.join(sysdig_dashboard_dir, str(db['id'])) - f = open(file_path, 'w') - f.write(json.dumps(db)) + sdclient.save_dashboard_to_file(db, os.path.join(sysdig_dashboard_dir, str(db['id']))) + print("Name: %s, # Charts: %d" % (db['name'], len(db['widgets']))) - f.close() zipf = zipfile.ZipFile(dashboard_state_file, 'w', zipfile.ZIP_DEFLATED) zipdir(sysdig_dashboard_dir, zipf) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index e70ec17c..16620260 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -16,8 +16,9 @@ class SdMonitorClient(_SdcCommon): def __init__(self, token="", sdc_url='https://app.sysdigcloud.com', ssl_verify=True): super(SdMonitorClient, self).__init__(token, sdc_url, ssl_verify) self.product = "SDC" - self._dashboards_api_endpoint = '/api/v2/dashboards' - self._default_dashboards_api_endpoint = '/api/v2/defaultDashboards' + self._dashboards_api_version = 'v2' + self._dashboards_api_endpoint = '/api/{}/dashboards'.format(self._dashboards_api_version) + self._default_dashboards_api_endpoint = '/api/{}/defaultDashboards'.format(self._dashboards_api_version) def get_alerts(self): '''**Description** @@ -692,14 +693,20 @@ def create_dashboard_from_dashboard(self, newdashname, templatename, filter, sha # return self.create_dashboard_from_template(newdashname, dboard, filter, shared, public, annotations) - def create_dashboard_from_file(self, newdashname, filename, filter, shared=False, public=False, annotations={}): + def create_dashboard_from_file(self, dashboard_name, filename, filter, shared=False, public=False, annotations={}): ''' **Description** - Create a new dasboard using a dashboard template saved to disk. + Create a new dasboard using a dashboard template saved to disk. See :func:`~SdcClient.save_dashboard_to_file` to use the file to create a dashboard (usefl to create and restore backups). + + The file can contain the following JSON formats: + 1. dashboard object in the format of an array element returned by :func:`~SdcClient.get_dashboards` + 2. JSON object with the following properties: + * version: dashboards API version (e.g. 'v2') + * dashboard: dashboard object in the format of an array element returned by :func:`~SdcClient.get_dashboards` **Arguments** - - **newdashname**: the name of the dashboard that will be created. - - **filename**: name of a file containing a JSON object for a dashboard in the format of an array element returned by :func:`~SdcClient.get_dashboards` + - **dashboard_name**: the name of the dashboard that will be created. + - **filename**: name of a file containing a JSON object - **filter**: a boolean expression combining Sysdig Monitor segmentation criteria defines what the new dasboard will be applied to. For example: *kubernetes.namespace.name='production' and container.image='nginx'*. - **shared**: if set to True, the new dashboard will be a shared one. - **public**: if set to True, the new dashboard will be shared with public token. @@ -715,15 +722,42 @@ def create_dashboard_from_file(self, newdashname, filename, filter, shared=False # Load the Dashboard # with open(filename) as data_file: - dboard = json.load(data_file) + loaded_object = json.load(data_file) - dboard['timeMode'] = {'mode': 1} - dboard['time'] = {'last': 2 * 60 * 60 * 1000000, 'sampling': 2 * 60 * 60 * 1000000} + # + # Handle old files + # + if 'dashboard' not in loaded_object: + loaded_object = { + 'version': 'v1', + 'dashboard': loaded_object + } + + dashboard = loaded_object['dashboard'] # # Create the new dashboard # - return self.create_dashboard_from_template(newdashname, dboard, filter, shared, public, annotations) + return self.create_dashboard_from_template(dashboard_name, dashboard, filter, shared, public, annotations) + + def save_dashboard_to_file(self, dashboard, filename): + ''' + **Description** + Save a dashboard to disk. See :func:`~SdcClient.create_dashboard_from_file` to use the file to create a dashboard (usefl to create and restore backups). + + The file will contain a JSON object with the following properties: + * version: dashboards API version (e.g. 'v2') + * dashboard: dashboard object in the format of an array element returned by :func:`~SdcClient.get_dashboards` + + **Arguments** + - **dashboard**: dashboard object in the format of an array element returned by :func:`~SdcClient.get_dashboards` + - **filename**: name of a file that will contain a JSON object + + **Example** + `examples/dashboard_save_load.py `_ + ''' + with open(filename, 'w') as outf: + json.dump(dashboard, outf) def delete_dashboard(self, dashboard): '''**Description** diff --git a/sdcclient/_monitor_v1.py b/sdcclient/_monitor_v1.py index 0aafb1a9..5259bd0e 100644 --- a/sdcclient/_monitor_v1.py +++ b/sdcclient/_monitor_v1.py @@ -18,6 +18,7 @@ class SdMonitorClientV1(SdMonitorClient): def __init__(self, token="", sdc_url='https://app-staging2.sysdigcloud.com', ssl_verify=True): super(SdMonitorClientV1, self).__init__(token, sdc_url, ssl_verify) + self._dashboards_api_version = 'v1' self._dashboards_api_endpoint = '/ui/dashboards' self._default_dashboards_api_endpoint = '/api/defaultDashboards' From b11f777c511153daf4c26116ff158e8f28266dbb Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 28 Mar 2019 17:43:48 -0700 Subject: [PATCH 10/48] Fix URL --- sdcclient/_monitor_v1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdcclient/_monitor_v1.py b/sdcclient/_monitor_v1.py index 5259bd0e..a1f8516b 100644 --- a/sdcclient/_monitor_v1.py +++ b/sdcclient/_monitor_v1.py @@ -16,7 +16,7 @@ class SdMonitorClientV1(SdMonitorClient): Handles dashboards version 1 (ie. up to February 2019). For later Sysdig Monitor versions, please use :class:`~SdMonitorClient` instead. ''' - def __init__(self, token="", sdc_url='https://app-staging2.sysdigcloud.com', ssl_verify=True): + def __init__(self, token="", sdc_url='https://app.sysdigcloud.com', ssl_verify=True): super(SdMonitorClientV1, self).__init__(token, sdc_url, ssl_verify) self._dashboards_api_version = 'v1' self._dashboards_api_endpoint = '/ui/dashboards' From 37be90184e10c5682415595edacac50f3cfe8658 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 29 Mar 2019 10:49:03 -0700 Subject: [PATCH 11/48] Revert temporary local change --- sdcclient/_monitor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 16620260..15af8603 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -757,7 +757,10 @@ def save_dashboard_to_file(self, dashboard, filename): `examples/dashboard_save_load.py `_ ''' with open(filename, 'w') as outf: - json.dump(dashboard, outf) + json.dump({ + 'version': self._dashboards_api_version, + 'dashboard': loaded_object + }, outf) def delete_dashboard(self, dashboard): '''**Description** From 2604a247049c82741faf48d2a387c52967e20170 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 29 Mar 2019 11:20:44 -0700 Subject: [PATCH 12/48] Apply reviews --- sdcclient/_monitor.py | 90 ++++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index a564eb38..947c9d77 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -757,50 +757,52 @@ def _convert_scope_string_to_expression(self, scope): # Proper grammar implementation will happen soon. # For practical purposes, the parsing will have equivalent results. # - if scope != None: - expressions = [] - string_expressions = scope.strip(' \t\n\r').split(' and ') - expression_re = re.compile('^(?Pnot )?(?P[^ ]+) (?P=|!=|in|contains|starts with) (?P.+)$') - - for string_expression in string_expressions: - matches = expression_re.match(string_expression) - - if matches == None: - return [False, 'invalid scope format'] - - is_not = matches.group('not') != None - - if matches.group('operator') == 'in': - list_value = matches.group('value').strip(' ()') - value_matches = re.findall('[^,]+', list_value) - - if len(value_matches) == 0: - return [False, 'invalid scope list format'] - - values = map(lambda v: v.strip(' "\''), value_matches) - else: - values = [matches.group('value').strip('"\'')] - - if matches.group('operator') == 'in': - operator = 'in' if is_not == False else 'notIn' - elif matches.group('operator') == '=': - operator = 'equals' - elif matches.group('operator') == '!=': - operator = 'notEquals' - elif matches.group('operator') == 'contains': - operator = 'contains' if is_not == False else 'notContains' - elif matches.group('operator') == 'starts with': - operator = 'startsWith' - - expressions.append({ - 'operand': matches.group('operand'), - 'operator': operator, - 'value': values - }) - - return [True, expressions] - else: - return [True, None] + + if scope is None or not scope: + return [True, []] + + expressions = [] + string_expressions = scope.strip(' \t\n\r').split(' and ') + expression_re = re.compile('^(?Pnot )?(?P[^ ]+) (?P=|!=|in|contains|starts with) (?P.+)$') + + for string_expression in string_expressions: + matches = expression_re.match(string_expression) + + if matches is None: + return [False, 'invalid scope format'] + + is_not_operator = matches.group('not') is not None + + if matches.group('operator') == 'in': + list_value = matches.group('value').strip(' ()') + value_matches = re.findall('[^,]+', list_value) + + if len(value_matches) == 0: + return [False, 'invalid scope value list format'] + + values = map(lambda v: v.strip(' "\''), value_matches) + else: + values = [matches.group('value').strip('"\'')] + + operator_parse_dict = { + 'in': 'in' if not is_not_operator else 'notIn', + '=': 'equals', + '!=': 'notEquals', + 'contains': 'contains' if not is_not_operator else 'notContains', + 'starts with': 'startsWith' + } + + operator = operator_parse_dict.get(matches.group('operator'), None) + if operator is None: + return [False, 'invalid scope operator'] + + expressions.append({ + 'operand': matches.group('operand'), + 'operator': operator, + 'value': values + }) + + return [True, expressions] # For backwards compatibility From 62e11b937c397d62f9220b8a328cdc8213b6e6da Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 29 Mar 2019 11:21:42 -0700 Subject: [PATCH 13/48] Add show cases/tests --- examples/dashboard_scope.py | 51 +++++++++++++++++++++++++++++++++++++ test/test_monitor_apis.sh | 1 + 2 files changed, 52 insertions(+) create mode 100755 examples/dashboard_scope.py diff --git a/examples/dashboard_scope.py b/examples/dashboard_scope.py new file mode 100755 index 00000000..ecc75c3b --- /dev/null +++ b/examples/dashboard_scope.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# +# This example shows some examples of scope you can use for dashboards. +# + +import getopt +import os +import sys +sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) +from sdcclient import SdcClient + +# random token, since we're not going to use the API for real +sdc_token = "bb8754d5-025d-4b88-a815-358fba58c4be" + +# +# Instantiate the SDC client +# +sdclient = SdcClient(sdc_token) + +# +# Scopes can be passed to most of dashboard-related functions, e.g. create_dashboard_from_file. +# +# NOTE: _convert_scope_string_to_expression should never be used in a user script +# We're going to use it here just to demonstrate some scope options and some constraints +# +def evaluate(scope): + parsed_scope = sdclient._convert_scope_string_to_expression(scope) + print '{} is valid: {}'.format(scope, parsed_scope[0] == True) + +# simple example: tag = value +evaluate('proc.name = "cassandra"') + +# other operators +evaluate('proc.name != "cassandra"') +evaluate('proc.name starts with "cassandra"') +evaluate('proc.name contains "cassandra"') + +# list operators +evaluate('proc.name in ("cassandra", "mysql")') + +# not-ed expressions +evaluate('not proc.name starts with "cassandra"') +evaluate('not proc.name contains "cassandra"') +evaluate('not proc.name in ("cassandra", "mysql")') + +# you can combine multiple expressions; note that only and-ed scopes are currently supported +evaluate('kubernetes.service.name = "database" and proc.name = "cassandra"') + +# the scope can obviously be omitted in the dashboard configuration +evaluate('') +evaluate(None) \ No newline at end of file diff --git a/test/test_monitor_apis.sh b/test/test_monitor_apis.sh index f41b2331..b0f39997 100644 --- a/test/test_monitor_apis.sh +++ b/test/test_monitor_apis.sh @@ -21,6 +21,7 @@ date; $SCRIPTDIR/../examples/create_alert.py -a $ALERT_NAME $PYTHON_SDC_TEST_MON date; $SCRIPTDIR/../examples/update_alert.py -a $ALERT_NAME $PYTHON_SDC_TEST_MONITOR_API_TOKEN date; $SCRIPTDIR/../examples/delete_alert.py -a $ALERT_NAME $PYTHON_SDC_TEST_MONITOR_API_TOKEN date; $SCRIPTDIR/../examples/dashboard.py -d $DASHBOARD_1_NAME $PYTHON_SDC_TEST_MONITOR_API_TOKEN +date; $SCRIPTDIR/../examples/dashboard_scope.py date; $SCRIPTDIR/../examples/create_dashboard.py -d $DASHBOARD_2_NAME $PYTHON_SDC_TEST_MONITOR_API_TOKEN date; $SCRIPTDIR/../examples/delete_dashboard.py -p $SESSION_UUID $PYTHON_SDC_TEST_MONITOR_API_TOKEN date; $SCRIPTDIR/../examples/get_data_advanced.py $PYTHON_SDC_TEST_MONITOR_API_TOKEN $AGENT_HOSTNAME From 7211dfb4394b261c8cbe0d2e67b1aa5f6291311d Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 29 Mar 2019 11:46:55 -0700 Subject: [PATCH 14/48] Improve scope handling --- examples/dashboard_scope.py | 40 +++++++++++++++++++++++++------------ sdcclient/_monitor.py | 5 +++-- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/examples/dashboard_scope.py b/examples/dashboard_scope.py index ecc75c3b..de12ade4 100755 --- a/examples/dashboard_scope.py +++ b/examples/dashboard_scope.py @@ -23,29 +23,43 @@ # NOTE: _convert_scope_string_to_expression should never be used in a user script # We're going to use it here just to demonstrate some scope options and some constraints # -def evaluate(scope): +def evaluate(scope, expected): parsed_scope = sdclient._convert_scope_string_to_expression(scope) print '{} is valid: {}'.format(scope, parsed_scope[0] == True) + if parsed_scope[0] != expected: + print('Unexpected parsing result!') + sys.exit(1) + + # simple example: tag = value -evaluate('proc.name = "cassandra"') +evaluate('proc.name = "cassandra"', True) # other operators -evaluate('proc.name != "cassandra"') -evaluate('proc.name starts with "cassandra"') -evaluate('proc.name contains "cassandra"') +evaluate('proc.name != "cassandra"', True) +evaluate('proc.name starts with "cassandra"', True) +evaluate('proc.name contains "cassandra"', True) # list operators -evaluate('proc.name in ("cassandra", "mysql")') +evaluate('proc.name in ("cassandra", "mysql")', True) # not-ed expressions -evaluate('not proc.name starts with "cassandra"') -evaluate('not proc.name contains "cassandra"') -evaluate('not proc.name in ("cassandra", "mysql")') +evaluate('not proc.name starts with "cassandra"', True) +evaluate('not proc.name contains "cassandra"', True) +evaluate('not proc.name in ("cassandra", "mysql")', True) -# you can combine multiple expressions; note that only and-ed scopes are currently supported -evaluate('kubernetes.service.name = "database" and proc.name = "cassandra"') +# you can combine multiple expressions; note that only AND'd scopes are currently supported +evaluate('kubernetes.service.name = "database" and proc.name = "cassandra"', True) # the scope can obviously be omitted in the dashboard configuration -evaluate('') -evaluate(None) \ No newline at end of file +evaluate('', True) +evaluate(None, True) + +# invalid scopes will cause errors +evaluate('proc.name == "cassandra"', False) # invalid operator +evaluate('proc.name = "cassandra" or proc.name = "mysql"', False) # not AND'd expressions +evaluate('proc.name in ("cassandra\', \'mysql")', False) # mismatching quotes +evaluate('proc.name in ("cassandra", "mysql"', False) # missing parenthesis + +# currently, one space is required around operands and operators -- improvements will come soon +evaluate('proc.name="cassandra"', False) \ No newline at end of file diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 947c9d77..3560a857 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -763,7 +763,7 @@ def _convert_scope_string_to_expression(self, scope): expressions = [] string_expressions = scope.strip(' \t\n\r').split(' and ') - expression_re = re.compile('^(?Pnot )?(?P[^ ]+) (?P=|!=|in|contains|starts with) (?P.+)$') + expression_re = re.compile('^(?Pnot )?(?P[^ ]+) (?P=|!=|in|contains|starts with) (?P(:?"[^"]+"|\'[^\']+\'|\(.+\)))$') for string_expression in string_expressions: matches = expression_re.match(string_expression) @@ -775,11 +775,12 @@ def _convert_scope_string_to_expression(self, scope): if matches.group('operator') == 'in': list_value = matches.group('value').strip(' ()') - value_matches = re.findall('[^,]+', list_value) + value_matches = re.findall('(:?\'[^\',]+\')|(:?"[^",]+")', list_value) if len(value_matches) == 0: return [False, 'invalid scope value list format'] + value_matches = map(lambda v: v[0] if v[0] else v[1], value_matches) values = map(lambda v: v.strip(' "\''), value_matches) else: values = [matches.group('value').strip('"\'')] From b47139c3f87f60c7be78f6798fe2f513176bc2af Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 29 Mar 2019 11:53:08 -0700 Subject: [PATCH 15/48] Missing new line --- examples/dashboard_scope.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/dashboard_scope.py b/examples/dashboard_scope.py index de12ade4..36089562 100755 --- a/examples/dashboard_scope.py +++ b/examples/dashboard_scope.py @@ -62,4 +62,4 @@ def evaluate(scope, expected): evaluate('proc.name in ("cassandra", "mysql"', False) # missing parenthesis # currently, one space is required around operands and operators -- improvements will come soon -evaluate('proc.name="cassandra"', False) \ No newline at end of file +evaluate('proc.name="cassandra"', False) From 3ff5af813aa8b2effbe9684c569e04273514cd74 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 29 Mar 2019 11:53:18 -0700 Subject: [PATCH 16/48] Fix scope --- examples/create_dashboard.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/create_dashboard.py b/examples/create_dashboard.py index 7c65ec9a..7faaafda 100755 --- a/examples/create_dashboard.py +++ b/examples/create_dashboard.py @@ -55,7 +55,7 @@ def usage(): # in Sysdig Cloud Explore page. # You can also refer to AWS tags by using "cloudProvider.tag.*" metadata or # agent tags by using "agent.tag.*" metadata -dashboardFilter = "kubernetes.namespace.name = prod" +dashboardFilter = "kubernetes.namespace.name = 'prod'" print('Creating dashboard from view') ok, res = sdclient.create_dashboard_from_view(dashboardName, viewName, dashboardFilter) # @@ -75,7 +75,7 @@ def usage(): # Name of the dashboard to copy dashboardCopy = "Copy of {}".format(dashboardName) # Filter to apply to the new dashboard. Same as above. -dashboardFilter = "kubernetes.namespace.name != prod" +dashboardFilter = "kubernetes.namespace.name != 'prod'" print('Creating dashboard from dashboard') ok, res = sdclient.create_dashboard_from_dashboard(dashboardCopy, dashboardName, dashboardFilter) From 1ca46568ad4a7383f28ad14fd2a0f8ae2b392d64 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 29 Mar 2019 11:56:11 -0700 Subject: [PATCH 17/48] Revert "Fix scope" This reverts commit 3ff5af813aa8b2effbe9684c569e04273514cd74. --- examples/create_dashboard.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/create_dashboard.py b/examples/create_dashboard.py index 7faaafda..7c65ec9a 100755 --- a/examples/create_dashboard.py +++ b/examples/create_dashboard.py @@ -55,7 +55,7 @@ def usage(): # in Sysdig Cloud Explore page. # You can also refer to AWS tags by using "cloudProvider.tag.*" metadata or # agent tags by using "agent.tag.*" metadata -dashboardFilter = "kubernetes.namespace.name = 'prod'" +dashboardFilter = "kubernetes.namespace.name = prod" print('Creating dashboard from view') ok, res = sdclient.create_dashboard_from_view(dashboardName, viewName, dashboardFilter) # @@ -75,7 +75,7 @@ def usage(): # Name of the dashboard to copy dashboardCopy = "Copy of {}".format(dashboardName) # Filter to apply to the new dashboard. Same as above. -dashboardFilter = "kubernetes.namespace.name != 'prod'" +dashboardFilter = "kubernetes.namespace.name != prod" print('Creating dashboard from dashboard') ok, res = sdclient.create_dashboard_from_dashboard(dashboardCopy, dashboardName, dashboardFilter) From b2f4600cb6ed76b614df168bbe3576534fe1d5f6 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 29 Mar 2019 12:02:49 -0700 Subject: [PATCH 18/48] Accept some invalid scopes for now --- examples/dashboard_scope.py | 17 ++++++++++++++--- sdcclient/_monitor.py | 4 ++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/examples/dashboard_scope.py b/examples/dashboard_scope.py index 36089562..2c3db9c7 100755 --- a/examples/dashboard_scope.py +++ b/examples/dashboard_scope.py @@ -35,6 +35,10 @@ def evaluate(scope, expected): # simple example: tag = value evaluate('proc.name = "cassandra"', True) +# NOTE: For now you can still leave values without quotes. +# The API will be more strict, so please make sure you adopt the new format! +evaluate('proc.name = cassandra', True) + # other operators evaluate('proc.name != "cassandra"', True) evaluate('proc.name starts with "cassandra"', True) @@ -57,9 +61,16 @@ def evaluate(scope, expected): # invalid scopes will cause errors evaluate('proc.name == "cassandra"', False) # invalid operator -evaluate('proc.name = "cassandra" or proc.name = "mysql"', False) # not AND'd expressions -evaluate('proc.name in ("cassandra\', \'mysql")', False) # mismatching quotes -evaluate('proc.name in ("cassandra", "mysql"', False) # missing parenthesis # currently, one space is required around operands and operators -- improvements will come soon evaluate('proc.name="cassandra"', False) + +# +# The current grammer is unable to validate all errors -- in these cases, the API will fail! +# Improvements will come soon! +# +# Here some errors that will not be detected by the Python library, but the API will +# +evaluate('proc.name = "cassandra" or proc.name = "mysql"', True) # not AND'd expressions are supported +evaluate('proc.name in ("cassandra\', \'mysql")', True) # mismatching quotes +evaluate('proc.name in ("cassandra", "mysql"', True) # missing parenthesis diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 3560a857..2a8be3ea 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -763,7 +763,7 @@ def _convert_scope_string_to_expression(self, scope): expressions = [] string_expressions = scope.strip(' \t\n\r').split(' and ') - expression_re = re.compile('^(?Pnot )?(?P[^ ]+) (?P=|!=|in|contains|starts with) (?P(:?"[^"]+"|\'[^\']+\'|\(.+\)))$') + expression_re = re.compile('^(?Pnot )?(?P[^ ]+) (?P=|!=|in|contains|starts with) (?P(:?"[^"]+"|\'[^\']+\'|\(.+\)|.+))$') for string_expression in string_expressions: matches = expression_re.match(string_expression) @@ -775,7 +775,7 @@ def _convert_scope_string_to_expression(self, scope): if matches.group('operator') == 'in': list_value = matches.group('value').strip(' ()') - value_matches = re.findall('(:?\'[^\',]+\')|(:?"[^",]+")', list_value) + value_matches = re.findall('(:?\'[^\',]+\')|(:?"[^",]+")|(:?[,]+)', list_value) if len(value_matches) == 0: return [False, 'invalid scope value list format'] From 312ad7966107ef89ce75921301979ca2252578d4 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 29 Mar 2019 15:26:15 -0700 Subject: [PATCH 19/48] Fix dashboard save --- sdcclient/_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 15af8603..8c154762 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -759,7 +759,7 @@ def save_dashboard_to_file(self, dashboard, filename): with open(filename, 'w') as outf: json.dump({ 'version': self._dashboards_api_version, - 'dashboard': loaded_object + 'dashboard': dashboard }, outf) def delete_dashboard(self, dashboard): From 3bc52050e9a030d6af6840823d4899a6674d3e47 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 29 Mar 2019 15:26:49 -0700 Subject: [PATCH 20/48] Scaffolding dashboard conversions --- sdcclient/_monitor.py | 32 ++++++++++++++++++++++++++++++++ sdcclient/_monitor_v1.py | 7 +++++++ 2 files changed, 39 insertions(+) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 3f781535..60dce3c5 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -735,6 +735,15 @@ def create_dashboard_from_file(self, dashboard_name, filename, filter, shared=Fa dashboard = loaded_object['dashboard'] + if loaded_object['version'] != self._dashboards_api_version: + # + # Convert the dashboard (if possible) + # + conversion_result, dashboard = self._convert_dashboard_to_current_version(dashboard, loaded_object['version']) + + if conversion_result == False: + return conversion_result, dashboard + # # Create the new dashboard # @@ -854,6 +863,29 @@ def _convert_scope_string_to_expression(self, scope): return [True, expressions] + def _get_dashboard_converters(self): + return { + 'v2': { + # 'v1': _convert_dashboard_v1_to_v2 + } + } + + def _convert_dashboard_to_current_version(self, dashboard, version): + converters_to = self._get_dashboard_converters().get(self._dashboards_api_version, None) + if converters_to == None: + return False, 'unexpected error: no dashboard converters from version {} are supported'.format(self._dashboards_api_version) + + converter = converters_to.get(version, None) + + if converter == None: + return False, 'dashboard version {} cannot be converted to {}'.format(version, self._dashboards_api_version) + + return converter(dashboard) + + +def _convert_dashboard_v1_to_v2(dashboard): + return True, dashboard + # For backwards compatibility SdcClient = SdMonitorClient diff --git a/sdcclient/_monitor_v1.py b/sdcclient/_monitor_v1.py index a1f8516b..83645029 100644 --- a/sdcclient/_monitor_v1.py +++ b/sdcclient/_monitor_v1.py @@ -286,3 +286,10 @@ def filter_fn(panel): return self._request_result(res) else: return [False, 'Not found'] + + def _get_dashboard_converters(self): + '''**Description** + Internal function to return dashboard converters from one version to another one. + ''' + # There's not really a previous version... + return {} From 0c1f1d2475add35cf39dca36ac7b332b38de7d10 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Mon, 1 Apr 2019 13:09:41 -0700 Subject: [PATCH 21/48] Initial implementation of v1 => v2 migration --- sdcclient/_monitor.py | 143 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 2 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 60dce3c5..2fbf25ae 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -866,7 +866,7 @@ def _convert_scope_string_to_expression(self, scope): def _get_dashboard_converters(self): return { 'v2': { - # 'v1': _convert_dashboard_v1_to_v2 + 'v1': _convert_dashboard_v1_to_v2 } } @@ -884,7 +884,146 @@ def _convert_dashboard_to_current_version(self, dashboard, version): def _convert_dashboard_v1_to_v2(dashboard): - return True, dashboard + # + # Migrations + # + # Each converter function will take: + # 1. name of the v1 dashboard property + # 2. v1 dashboard configuration + # 3. v2 dashboard configuration + # + # Each converter will apply changes to v2 dashboard configuration according to v1 + # + def keep_as_is(prop_name, old_dashboard, new_dashboard): + new_dashboard[prop_name] = old_dashboard[prop_name] + + def drop_it(prop_name = None, old_dashboard = None, new_dashboard = None): + pass + + def rename_to(new_prop_name): + def rename(prop_name, old_dashboard, new_dashboard): + new_dashboard[new_prop_name] = old_dashboard[prop_name] + + return rename + + def convert_schema(prop_name, old_dashboard, new_dashboard): + new_dashboard[prop_name] = 2 + + def convert_scope(prop_name, old_dashboard, new_dashboard): + drop_it() + + def convert_events_filter(prop_name, old_dashboard, new_dashboard): + rename_to('eventsOverlaySettings')(prop_name, old_dashboard, new_dashboard) + + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameMetrics'] + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameScope'] + + def convert_items(prop_name, old_dashboard, new_dashboard): + def convert_color_coding(prop_name, old_widget, new_widget): + best_value = None + worst_value= None + for item in old_widget[prop_name]['thresholds']: + if item['color'] == 'best': + best_value = item['max'] if not item['max'] else item['min'] + elif item['color'] == 'worst': + worst_value = item['min'] if not item['min'] else item['max'] + + if best_value is not None and worst_value is not None: + new_widget[prop_name] = { + 'best': best_value, + 'worst': worst_value + } + + def convert_display_options(prop_name, old_widget, new_widget): + keep_as_is(prop_name, old_widget, new_widget) + + if 'yAxisScaleFactor' in new_widget[prop_name]: + del new_widget[prop_name]['yAxisScaleFactor'] + + def convert_group(prop_name, old_widget, new_widget): + group_by_metrics = old_widget[prop_name]['configuration']['groups'][0]['groupBy'] + + migrated = [] + for metric in group_by_metrics: + migrated.append({ 'labelId': metric['metric'] }) + + new_widget['groupingLabelsIds'] = migrated + + def convert_metrics(prop_name, old_widget, new_widget): + keep_as_is(prop_name, old_widget, new_widget) + + for metric in new_widget[prop_name]: + rename_to('id')('metricId', metric, metric) + if 'aggregation' in metric: + # timestamp metric doesn't have aggregations + rename_to('timeAggregation')('aggregation', metric, metric) + + widget_migrations = { + 'colorCoding': convert_color_coding, + 'compareToConfig': keep_as_is, + 'customDisplayOptions': convert_display_options, + 'gridConfiguration': keep_as_is, + 'group': convert_group, + 'hasTransparentBackground': rename_to('transparentBackground'), + 'limitToScope': keep_as_is, + 'isPanelTitleVisible': rename_to('panelTitleVisible'), + 'markdownSource': keep_as_is, + 'limitToScope': keep_as_is, + 'metrics': convert_metrics, + 'name': keep_as_is, + 'overrideFilter': rename_to('overrideScope'), + 'paging': drop_it, + 'scope': keep_as_is, + 'showAs': keep_as_is, + 'showAsType': drop_it, + 'sorting': drop_it, + 'textpanelTooltip': keep_as_is, + } + + migrated_widgets = [] + for old_widget in old_dashboard[prop_name]: + migrated_widget = {} + + for key in widget_migrations.keys(): + if key in old_widget: + widget_migrations[key](key, old_widget, migrated_widget) + + migrated_widgets.append(migrated_widget) + + + new_dashboard['widgets'] = migrated_widgets + + return migrated + + migrations = { + 'autoCreated': keep_as_is, + 'createdOn': keep_as_is, + 'eventsFilter': convert_events_filter, + 'filterExpression': convert_scope, + 'id': keep_as_is, + 'isPublic': rename_to('public'), + 'isShared': rename_to('shared'), + 'items': convert_items, + 'layout': keep_as_is, + 'modifiedOn': keep_as_is, + 'name': keep_as_is, + 'publicToken': drop_it, + 'schema': convert_schema, + 'scopeExpressionList': drop_it, + 'teamId': keep_as_is, + 'username': keep_as_is, + 'version': keep_as_is, + } + + # + # Apply migrations + # + migrated = {} + for key in migrations.keys(): + if key in dashboard: + migrations[key](key, dashboard, migrated) + + return True, migrated # For backwards compatibility From efe6a32437a83b62202095c0b38127e4944c491d Mon Sep 17 00:00:00 2001 From: davideschiera Date: Mon, 1 Apr 2019 13:15:17 -0700 Subject: [PATCH 22/48] Accept panels without scope --- sdcclient/_monitor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 2a8be3ea..8c921516 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -584,7 +584,8 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared chart['scope'] = scope # if chart scope is equal to dashboard scope, set it as non override - chart['overrideFilter'] = chart['scope'] != scope + chart_scope = chart['scope'] if 'scope' in chart else None + chart['overrideFilter'] = chart_scope != scope if 'annotations' in template: template['annotations'].update(annotations) From ce26a83223d2680c140edb073523df7209fe0553 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 29 Mar 2019 15:26:49 -0700 Subject: [PATCH 23/48] Scaffolding dashboard conversions --- sdcclient/_monitor.py | 32 ++++++++++++++++++++++++++++++++ sdcclient/_monitor_v1.py | 7 +++++++ 2 files changed, 39 insertions(+) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index d5ed8d55..becb2e6c 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -736,6 +736,15 @@ def create_dashboard_from_file(self, dashboard_name, filename, filter, shared=Fa dashboard = loaded_object['dashboard'] + if loaded_object['version'] != self._dashboards_api_version: + # + # Convert the dashboard (if possible) + # + conversion_result, dashboard = self._convert_dashboard_to_current_version(dashboard, loaded_object['version']) + + if conversion_result == False: + return conversion_result, dashboard + # # Create the new dashboard # @@ -855,6 +864,29 @@ def _convert_scope_string_to_expression(self, scope): return [True, expressions] + def _get_dashboard_converters(self): + return { + 'v2': { + # 'v1': _convert_dashboard_v1_to_v2 + } + } + + def _convert_dashboard_to_current_version(self, dashboard, version): + converters_to = self._get_dashboard_converters().get(self._dashboards_api_version, None) + if converters_to == None: + return False, 'unexpected error: no dashboard converters from version {} are supported'.format(self._dashboards_api_version) + + converter = converters_to.get(version, None) + + if converter == None: + return False, 'dashboard version {} cannot be converted to {}'.format(version, self._dashboards_api_version) + + return converter(dashboard) + + +def _convert_dashboard_v1_to_v2(dashboard): + return True, dashboard + # For backwards compatibility SdcClient = SdMonitorClient diff --git a/sdcclient/_monitor_v1.py b/sdcclient/_monitor_v1.py index a1f8516b..83645029 100644 --- a/sdcclient/_monitor_v1.py +++ b/sdcclient/_monitor_v1.py @@ -286,3 +286,10 @@ def filter_fn(panel): return self._request_result(res) else: return [False, 'Not found'] + + def _get_dashboard_converters(self): + '''**Description** + Internal function to return dashboard converters from one version to another one. + ''' + # There's not really a previous version... + return {} From 88dc25a92edebe386d4edb6cdf8499a27a323603 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Mon, 1 Apr 2019 13:09:41 -0700 Subject: [PATCH 24/48] Initial implementation of v1 => v2 migration --- sdcclient/_monitor.py | 143 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 2 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index becb2e6c..36f1ad67 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -867,7 +867,7 @@ def _convert_scope_string_to_expression(self, scope): def _get_dashboard_converters(self): return { 'v2': { - # 'v1': _convert_dashboard_v1_to_v2 + 'v1': _convert_dashboard_v1_to_v2 } } @@ -885,7 +885,146 @@ def _convert_dashboard_to_current_version(self, dashboard, version): def _convert_dashboard_v1_to_v2(dashboard): - return True, dashboard + # + # Migrations + # + # Each converter function will take: + # 1. name of the v1 dashboard property + # 2. v1 dashboard configuration + # 3. v2 dashboard configuration + # + # Each converter will apply changes to v2 dashboard configuration according to v1 + # + def keep_as_is(prop_name, old_dashboard, new_dashboard): + new_dashboard[prop_name] = old_dashboard[prop_name] + + def drop_it(prop_name = None, old_dashboard = None, new_dashboard = None): + pass + + def rename_to(new_prop_name): + def rename(prop_name, old_dashboard, new_dashboard): + new_dashboard[new_prop_name] = old_dashboard[prop_name] + + return rename + + def convert_schema(prop_name, old_dashboard, new_dashboard): + new_dashboard[prop_name] = 2 + + def convert_scope(prop_name, old_dashboard, new_dashboard): + drop_it() + + def convert_events_filter(prop_name, old_dashboard, new_dashboard): + rename_to('eventsOverlaySettings')(prop_name, old_dashboard, new_dashboard) + + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameMetrics'] + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameScope'] + + def convert_items(prop_name, old_dashboard, new_dashboard): + def convert_color_coding(prop_name, old_widget, new_widget): + best_value = None + worst_value= None + for item in old_widget[prop_name]['thresholds']: + if item['color'] == 'best': + best_value = item['max'] if not item['max'] else item['min'] + elif item['color'] == 'worst': + worst_value = item['min'] if not item['min'] else item['max'] + + if best_value is not None and worst_value is not None: + new_widget[prop_name] = { + 'best': best_value, + 'worst': worst_value + } + + def convert_display_options(prop_name, old_widget, new_widget): + keep_as_is(prop_name, old_widget, new_widget) + + if 'yAxisScaleFactor' in new_widget[prop_name]: + del new_widget[prop_name]['yAxisScaleFactor'] + + def convert_group(prop_name, old_widget, new_widget): + group_by_metrics = old_widget[prop_name]['configuration']['groups'][0]['groupBy'] + + migrated = [] + for metric in group_by_metrics: + migrated.append({ 'labelId': metric['metric'] }) + + new_widget['groupingLabelsIds'] = migrated + + def convert_metrics(prop_name, old_widget, new_widget): + keep_as_is(prop_name, old_widget, new_widget) + + for metric in new_widget[prop_name]: + rename_to('id')('metricId', metric, metric) + if 'aggregation' in metric: + # timestamp metric doesn't have aggregations + rename_to('timeAggregation')('aggregation', metric, metric) + + widget_migrations = { + 'colorCoding': convert_color_coding, + 'compareToConfig': keep_as_is, + 'customDisplayOptions': convert_display_options, + 'gridConfiguration': keep_as_is, + 'group': convert_group, + 'hasTransparentBackground': rename_to('transparentBackground'), + 'limitToScope': keep_as_is, + 'isPanelTitleVisible': rename_to('panelTitleVisible'), + 'markdownSource': keep_as_is, + 'limitToScope': keep_as_is, + 'metrics': convert_metrics, + 'name': keep_as_is, + 'overrideFilter': rename_to('overrideScope'), + 'paging': drop_it, + 'scope': keep_as_is, + 'showAs': keep_as_is, + 'showAsType': drop_it, + 'sorting': drop_it, + 'textpanelTooltip': keep_as_is, + } + + migrated_widgets = [] + for old_widget in old_dashboard[prop_name]: + migrated_widget = {} + + for key in widget_migrations.keys(): + if key in old_widget: + widget_migrations[key](key, old_widget, migrated_widget) + + migrated_widgets.append(migrated_widget) + + + new_dashboard['widgets'] = migrated_widgets + + return migrated + + migrations = { + 'autoCreated': keep_as_is, + 'createdOn': keep_as_is, + 'eventsFilter': convert_events_filter, + 'filterExpression': convert_scope, + 'id': keep_as_is, + 'isPublic': rename_to('public'), + 'isShared': rename_to('shared'), + 'items': convert_items, + 'layout': keep_as_is, + 'modifiedOn': keep_as_is, + 'name': keep_as_is, + 'publicToken': drop_it, + 'schema': convert_schema, + 'scopeExpressionList': drop_it, + 'teamId': keep_as_is, + 'username': keep_as_is, + 'version': keep_as_is, + } + + # + # Apply migrations + # + migrated = {} + for key in migrations.keys(): + if key in dashboard: + migrations[key](key, dashboard, migrated) + + return True, migrated # For backwards compatibility From 387e0119b0853b4b6c2b5d5eb8f7b17ffeb79d57 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Mon, 1 Apr 2019 15:51:43 -0700 Subject: [PATCH 25/48] Fix migration --- sdcclient/_monitor.py | 85 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 16 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 36f1ad67..55a2f056 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -895,11 +895,23 @@ def _convert_dashboard_v1_to_v2(dashboard): # # Each converter will apply changes to v2 dashboard configuration according to v1 # + def with_default(converter, default=None): + def fn(prop_name, old_dashboard, new_dashboard): + if prop_name not in old_dashboard: + old_dashboard[prop_name] = default + + converter(prop_name, old_dashboard, new_dashboard) + + return fn + def keep_as_is(prop_name, old_dashboard, new_dashboard): new_dashboard[prop_name] = old_dashboard[prop_name] def drop_it(prop_name = None, old_dashboard = None, new_dashboard = None): pass + + def ignore(prop_name = None, old_dashboard = None, new_dashboard = None): + pass def rename_to(new_prop_name): def rename(prop_name, old_dashboard, new_dashboard): @@ -911,13 +923,17 @@ def convert_schema(prop_name, old_dashboard, new_dashboard): new_dashboard[prop_name] = 2 def convert_scope(prop_name, old_dashboard, new_dashboard): - drop_it() + # # TODO! + + new_dashboard['scopeExpressionList'] = None def convert_events_filter(prop_name, old_dashboard, new_dashboard): rename_to('eventsOverlaySettings')(prop_name, old_dashboard, new_dashboard) - del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameMetrics'] - del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameScope'] + if 'showNotificationsDoNotFilterSameMetrics' in new_dashboard['eventsOverlaySettings']: + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameMetrics'] + if 'showNotificationsDoNotFilterSameScope' in new_dashboard['eventsOverlaySettings']: + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameScope'] def convert_items(prop_name, old_dashboard, new_dashboard): def convert_color_coding(prop_name, old_widget, new_widget): @@ -950,14 +966,46 @@ def convert_group(prop_name, old_widget, new_widget): new_widget['groupingLabelsIds'] = migrated - def convert_metrics(prop_name, old_widget, new_widget): + def convert_name(prop_name, old_widget, new_widget): keep_as_is(prop_name, old_widget, new_widget) - for metric in new_widget[prop_name]: - rename_to('id')('metricId', metric, metric) - if 'aggregation' in metric: - # timestamp metric doesn't have aggregations - rename_to('timeAggregation')('aggregation', metric, metric) + unique_id = 1 + name = old_widget[prop_name] + + for widget in old_dashboard['items']: + if widget == old_widget: + return + + if new_widget[prop_name] == widget[prop_name]: + new_widget[prop_name] = '{} ({})'.format(name, unique_id) + unique_id += 1 + + + def convert_metrics(prop_name, old_widget, new_widget): + def convert_property_name(prop_name, old_metric, new_metric): + keep_as_is(prop_name, old_metric, new_metric) + + if old_metric['metricId'] == 'timestamp': + return 'k0' + + metric_migrations = { + 'metricId': rename_to('id'), + 'aggregation': rename_to('timeAggregation'), + 'groupAggregation': rename_to('groupAggregation'), + 'propertyName': convert_property_name + } + + migrated_metrics = [] + for old_metric in old_widget[prop_name]: + migrated_metric = {} + + for key in metric_migrations.keys(): + if key in old_metric: + metric_migrations[key](key, old_metric, migrated_metric) + + migrated_metrics.append(migrated_metric) + + new_widget['metrics'] = migrated_metrics widget_migrations = { 'colorCoding': convert_color_coding, @@ -971,10 +1019,12 @@ def convert_metrics(prop_name, old_widget, new_widget): 'markdownSource': keep_as_is, 'limitToScope': keep_as_is, 'metrics': convert_metrics, - 'name': keep_as_is, + 'name': convert_name, 'overrideFilter': rename_to('overrideScope'), 'paging': drop_it, - 'scope': keep_as_is, + + 'scope': drop_it, # TODO !!!! + 'showAs': keep_as_is, 'showAsType': drop_it, 'sorting': drop_it, @@ -983,7 +1033,9 @@ def convert_metrics(prop_name, old_widget, new_widget): migrated_widgets = [] for old_widget in old_dashboard[prop_name]: - migrated_widget = {} + migrated_widget = { + 'id': len(migrated_widgets) + 1 + } for key in widget_migrations.keys(): if key in old_widget: @@ -999,8 +1051,11 @@ def convert_metrics(prop_name, old_widget, new_widget): migrations = { 'autoCreated': keep_as_is, 'createdOn': keep_as_is, - 'eventsFilter': convert_events_filter, + 'eventsFilter': with_default(convert_events_filter, { + 'filterNotificationsUserInputFilter': '' + }), 'filterExpression': convert_scope, + 'scopeExpressionList': ignore, # scope will be generated from 'filterExpression' 'id': keep_as_is, 'isPublic': rename_to('public'), 'isShared': rename_to('shared'), @@ -1010,7 +1065,6 @@ def convert_metrics(prop_name, old_widget, new_widget): 'name': keep_as_is, 'publicToken': drop_it, 'schema': convert_schema, - 'scopeExpressionList': drop_it, 'teamId': keep_as_is, 'username': keep_as_is, 'version': keep_as_is, @@ -1021,8 +1075,7 @@ def convert_metrics(prop_name, old_widget, new_widget): # migrated = {} for key in migrations.keys(): - if key in dashboard: - migrations[key](key, dashboard, migrated) + migrations[key](key, copy.deepcopy(dashboard), migrated) return True, migrated From 1b50dbe3fc88597d3a236a02286de49d5204451f Mon Sep 17 00:00:00 2001 From: davideschiera Date: Mon, 1 Apr 2019 15:52:05 -0700 Subject: [PATCH 26/48] Add migration example --- examples/dashboard_backup_v1_restore_v2.py | 59 ++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 examples/dashboard_backup_v1_restore_v2.py diff --git a/examples/dashboard_backup_v1_restore_v2.py b/examples/dashboard_backup_v1_restore_v2.py new file mode 100644 index 00000000..6e198299 --- /dev/null +++ b/examples/dashboard_backup_v1_restore_v2.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# +# Save the first user dashboard to file and then use create_dashboard_from_file() +# to apply the stored dasboard again with a different filter. +# +import os +import sys +import json +sys.path.insert( + 0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) +from sdcclient import SdMonitorClient +from sdcclient import SdMonitorClientV1 + +# +# Parse arguments +# +if len(sys.argv) != 2: + print('usage: %s ' % sys.argv[0]) + print( + 'You can find your token at https://app.sysdigcloud.com/#/settings/user' + ) + sys.exit(1) + +sdc_token = sys.argv[1] + +# +# Instantiate the SDC client +# +sdclient = SdMonitorClient(sdc_token, sdc_url='https://app.sysdigcloud.com') +sdclientV1 = SdMonitorClientV1( + sdc_token, sdc_url='https://app.sysdigcloud.com') + +# +# Serialize the first user dashboard to disk +# +ok, res = sdclientV1.get_dashboards() + +if not ok: + print(res) + sys.exit(1) + +for item in res['dashboards']: + file_name = '{}.json'.format(item['id']) + sdclientV1.save_dashboard_to_file(item, file_name) + + ok, res = sdclient.create_dashboard_from_file( + u'import of {}'.format(item['name']), + file_name, + None, + shared=item['isShared'], + public=item['isPublic']) + + if ok: + sdclient.delete_dashboard(res['dashboard']) + else: + print('=====') + print(res) + + print('=====') From 92d4f052d7d14c8e326258bad6480a5dd2269a5b Mon Sep 17 00:00:00 2001 From: davideschiera Date: Tue, 2 Apr 2019 11:58:51 -0700 Subject: [PATCH 27/48] Fixes and support for scope migration --- sdcclient/_monitor.py | 386 ++++++++++++++++++++++-------------------- 1 file changed, 204 insertions(+), 182 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 55a2f056..3b6ad052 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -584,7 +584,10 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared scopeExpression = self._convert_scope_string_to_expression(scope) if scopeExpression[0] == False: return scopeExpression - template['scopeExpressionList'] = map(lambda ex: {'operand':ex['operand'], 'operator':ex['operator'],'value':ex['value'],'displayName':'', 'variable':False}, scopeExpression[1]) + if scopeExpression[1]: + template['scopeExpressionList'] = map(lambda ex: {'operand': ex['operand'], 'operator': ex['operator'], 'value': ex['value'], 'displayName': '', 'variable': False}, scopeExpression[1]) + else: + template['scopeExpressionList'] = None # NOTE: Individual panels might override the dashboard scope, the override will NOT be reset if 'widgets' in template and template['widgets'] is not None: @@ -611,7 +614,15 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared # Create the new dashboard # res = requests.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': template}), verify=self.ssl_verify) - return self._request_result(res) + + result = self._request_result(res) + + if result[0] == False: + print json.dumps({'dashboard': template}) + # else: + # print json.dumps({'dashboard': template}) + + return result def create_dashboard_from_view(self, newdashname, viewname, filter, shared=False, public=False, annotations={}): '''**Description** @@ -867,7 +878,7 @@ def _convert_scope_string_to_expression(self, scope): def _get_dashboard_converters(self): return { 'v2': { - 'v1': _convert_dashboard_v1_to_v2 + 'v1': self._convert_dashboard_v1_to_v2 } } @@ -881,203 +892,214 @@ def _convert_dashboard_to_current_version(self, dashboard, version): if converter == None: return False, 'dashboard version {} cannot be converted to {}'.format(version, self._dashboards_api_version) - return converter(dashboard) - - -def _convert_dashboard_v1_to_v2(dashboard): - # - # Migrations - # - # Each converter function will take: - # 1. name of the v1 dashboard property - # 2. v1 dashboard configuration - # 3. v2 dashboard configuration - # - # Each converter will apply changes to v2 dashboard configuration according to v1 - # - def with_default(converter, default=None): - def fn(prop_name, old_dashboard, new_dashboard): - if prop_name not in old_dashboard: - old_dashboard[prop_name] = default - - converter(prop_name, old_dashboard, new_dashboard) + try: + return converter(dashboard) + except Exception as err: + return False, str(err) + + def _convert_dashboard_v1_to_v2(self, dashboard): + # + # Migrations + # + # Each converter function will take: + # 1. name of the v1 dashboard property + # 2. v1 dashboard configuration + # 3. v2 dashboard configuration + # + # Each converter will apply changes to v2 dashboard configuration according to v1 + # + def with_default(converter, default=None): + def fn(prop_name, old_dashboard, new_dashboard): + if prop_name not in old_dashboard: + old_dashboard[prop_name] = default + + converter(prop_name, old_dashboard, new_dashboard) + + return fn + + def keep_as_is(prop_name, old_dashboard, new_dashboard): + new_dashboard[prop_name] = old_dashboard[prop_name] - return fn + def drop_it(prop_name = None, old_dashboard = None, new_dashboard = None): + pass - def keep_as_is(prop_name, old_dashboard, new_dashboard): - new_dashboard[prop_name] = old_dashboard[prop_name] - - def drop_it(prop_name = None, old_dashboard = None, new_dashboard = None): - pass - - def ignore(prop_name = None, old_dashboard = None, new_dashboard = None): - pass - - def rename_to(new_prop_name): - def rename(prop_name, old_dashboard, new_dashboard): - new_dashboard[new_prop_name] = old_dashboard[prop_name] - - return rename - - def convert_schema(prop_name, old_dashboard, new_dashboard): - new_dashboard[prop_name] = 2 - - def convert_scope(prop_name, old_dashboard, new_dashboard): - # # TODO! - - new_dashboard['scopeExpressionList'] = None - - def convert_events_filter(prop_name, old_dashboard, new_dashboard): - rename_to('eventsOverlaySettings')(prop_name, old_dashboard, new_dashboard) - - if 'showNotificationsDoNotFilterSameMetrics' in new_dashboard['eventsOverlaySettings']: - del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameMetrics'] - if 'showNotificationsDoNotFilterSameScope' in new_dashboard['eventsOverlaySettings']: - del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameScope'] - - def convert_items(prop_name, old_dashboard, new_dashboard): - def convert_color_coding(prop_name, old_widget, new_widget): - best_value = None - worst_value= None - for item in old_widget[prop_name]['thresholds']: - if item['color'] == 'best': - best_value = item['max'] if not item['max'] else item['min'] - elif item['color'] == 'worst': - worst_value = item['min'] if not item['min'] else item['max'] - - if best_value is not None and worst_value is not None: - new_widget[prop_name] = { - 'best': best_value, - 'worst': worst_value - } + def ignore(prop_name = None, old_dashboard = None, new_dashboard = None): + pass - def convert_display_options(prop_name, old_widget, new_widget): - keep_as_is(prop_name, old_widget, new_widget) + def rename_to(new_prop_name): + def rename(prop_name, old_dashboard, new_dashboard): + new_dashboard[new_prop_name] = old_dashboard[prop_name] - if 'yAxisScaleFactor' in new_widget[prop_name]: - del new_widget[prop_name]['yAxisScaleFactor'] + return rename - def convert_group(prop_name, old_widget, new_widget): - group_by_metrics = old_widget[prop_name]['configuration']['groups'][0]['groupBy'] - - migrated = [] - for metric in group_by_metrics: - migrated.append({ 'labelId': metric['metric'] }) - - new_widget['groupingLabelsIds'] = migrated + def convert_schema(prop_name, old_dashboard, new_dashboard): + new_dashboard[prop_name] = 2 - def convert_name(prop_name, old_widget, new_widget): - keep_as_is(prop_name, old_widget, new_widget) + def convert_scope(prop_name, old_dashboard, new_dashboard): + # # TODO! - unique_id = 1 - name = old_widget[prop_name] + scope = old_dashboard[prop_name] + scope_conversion = self._convert_scope_string_to_expression(scope) - for widget in old_dashboard['items']: - if widget == old_widget: - return + if scope_conversion[0]: + if scope_conversion[1]: + new_dashboard['scopeExpressionList'] = scope_conversion[1] + else: + # the property can be either `null` or a non-empty array + new_dashboard['scopeExpressionList'] = None + else: + raise SyntaxError('scope not supported by the current grammar') + + def convert_events_filter(prop_name, old_dashboard, new_dashboard): + rename_to('eventsOverlaySettings')(prop_name, old_dashboard, new_dashboard) + + if 'showNotificationsDoNotFilterSameMetrics' in new_dashboard['eventsOverlaySettings']: + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameMetrics'] + if 'showNotificationsDoNotFilterSameScope' in new_dashboard['eventsOverlaySettings']: + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameScope'] + + def convert_items(prop_name, old_dashboard, new_dashboard): + def convert_color_coding(prop_name, old_widget, new_widget): + best_value = None + worst_value= None + for item in old_widget[prop_name]['thresholds']: + if item['color'] == 'best': + best_value = item['max'] if not item['max'] else item['min'] + elif item['color'] == 'worst': + worst_value = item['min'] if not item['min'] else item['max'] - if new_widget[prop_name] == widget[prop_name]: - new_widget[prop_name] = '{} ({})'.format(name, unique_id) - unique_id += 1 + if best_value is not None and worst_value is not None: + new_widget[prop_name] = { + 'best': best_value, + 'worst': worst_value + } + def convert_display_options(prop_name, old_widget, new_widget): + keep_as_is(prop_name, old_widget, new_widget) - def convert_metrics(prop_name, old_widget, new_widget): - def convert_property_name(prop_name, old_metric, new_metric): - keep_as_is(prop_name, old_metric, new_metric) - - if old_metric['metricId'] == 'timestamp': - return 'k0' - - metric_migrations = { - 'metricId': rename_to('id'), - 'aggregation': rename_to('timeAggregation'), - 'groupAggregation': rename_to('groupAggregation'), - 'propertyName': convert_property_name - } + if 'yAxisScaleFactor' in new_widget[prop_name]: + del new_widget[prop_name]['yAxisScaleFactor'] - migrated_metrics = [] - for old_metric in old_widget[prop_name]: - migrated_metric = {} - - for key in metric_migrations.keys(): - if key in old_metric: - metric_migrations[key](key, old_metric, migrated_metric) - - migrated_metrics.append(migrated_metric) - - new_widget['metrics'] = migrated_metrics - - widget_migrations = { - 'colorCoding': convert_color_coding, - 'compareToConfig': keep_as_is, - 'customDisplayOptions': convert_display_options, - 'gridConfiguration': keep_as_is, - 'group': convert_group, - 'hasTransparentBackground': rename_to('transparentBackground'), - 'limitToScope': keep_as_is, - 'isPanelTitleVisible': rename_to('panelTitleVisible'), - 'markdownSource': keep_as_is, - 'limitToScope': keep_as_is, - 'metrics': convert_metrics, - 'name': convert_name, - 'overrideFilter': rename_to('overrideScope'), - 'paging': drop_it, - - 'scope': drop_it, # TODO !!!! - - 'showAs': keep_as_is, - 'showAsType': drop_it, - 'sorting': drop_it, - 'textpanelTooltip': keep_as_is, - } + def convert_group(prop_name, old_widget, new_widget): + group_by_metrics = old_widget[prop_name]['configuration']['groups'][0]['groupBy'] + + migrated = [] + for metric in group_by_metrics: + migrated.append({ 'labelId': metric['metric'] }) + + new_widget['groupingLabelsIds'] = migrated + + def convert_name(prop_name, old_widget, new_widget): + keep_as_is(prop_name, old_widget, new_widget) + + unique_id = 1 + name = old_widget[prop_name] + + for widget in old_dashboard['items']: + if widget == old_widget: + return + + if new_widget[prop_name] == widget[prop_name]: + new_widget[prop_name] = '{} ({})'.format(name, unique_id) + unique_id += 1 + + + def convert_metrics(prop_name, old_widget, new_widget): + def convert_property_name(prop_name, old_metric, new_metric): + keep_as_is(prop_name, old_metric, new_metric) + + if old_metric['metricId'] == 'timestamp': + return 'k0' + + metric_migrations = { + 'metricId': rename_to('id'), + 'aggregation': rename_to('timeAggregation'), + 'groupAggregation': rename_to('groupAggregation'), + 'propertyName': convert_property_name + } - migrated_widgets = [] - for old_widget in old_dashboard[prop_name]: - migrated_widget = { - 'id': len(migrated_widgets) + 1 + migrated_metrics = [] + for old_metric in old_widget[prop_name]: + migrated_metric = {} + + for key in metric_migrations.keys(): + if key in old_metric: + metric_migrations[key](key, old_metric, migrated_metric) + + migrated_metrics.append(migrated_metric) + + new_widget['metrics'] = migrated_metrics + + widget_migrations = { + 'colorCoding': convert_color_coding, + 'compareToConfig': keep_as_is, + 'customDisplayOptions': convert_display_options, + 'gridConfiguration': keep_as_is, + 'group': convert_group, + 'hasTransparentBackground': rename_to('transparentBackground'), + 'limitToScope': keep_as_is, + 'isPanelTitleVisible': rename_to('panelTitleVisible'), + 'markdownSource': keep_as_is, + 'metrics': convert_metrics, + 'name': convert_name, + 'overrideFilter': rename_to('overrideScope'), + 'paging': drop_it, + + 'scope': keep_as_is, + + 'showAs': keep_as_is, + 'showAsType': drop_it, + 'sorting': drop_it, + 'textpanelTooltip': keep_as_is, } - for key in widget_migrations.keys(): - if key in old_widget: - widget_migrations[key](key, old_widget, migrated_widget) + migrated_widgets = [] + for old_widget in old_dashboard[prop_name]: + migrated_widget = { + 'id': len(migrated_widgets) + 1 + } + + for key in widget_migrations.keys(): + if key in old_widget: + widget_migrations[key](key, old_widget, migrated_widget) - migrated_widgets.append(migrated_widget) + migrated_widgets.append(migrated_widget) + + new_dashboard['widgets'] = migrated_widgets + + return migrated - new_dashboard['widgets'] = migrated_widgets - - return migrated - - migrations = { - 'autoCreated': keep_as_is, - 'createdOn': keep_as_is, - 'eventsFilter': with_default(convert_events_filter, { - 'filterNotificationsUserInputFilter': '' - }), - 'filterExpression': convert_scope, - 'scopeExpressionList': ignore, # scope will be generated from 'filterExpression' - 'id': keep_as_is, - 'isPublic': rename_to('public'), - 'isShared': rename_to('shared'), - 'items': convert_items, - 'layout': keep_as_is, - 'modifiedOn': keep_as_is, - 'name': keep_as_is, - 'publicToken': drop_it, - 'schema': convert_schema, - 'teamId': keep_as_is, - 'username': keep_as_is, - 'version': keep_as_is, - } - - # - # Apply migrations - # - migrated = {} - for key in migrations.keys(): - migrations[key](key, copy.deepcopy(dashboard), migrated) - - return True, migrated + migrations = { + 'autoCreated': keep_as_is, + 'createdOn': keep_as_is, + 'eventsFilter': with_default(convert_events_filter, { + 'filterNotificationsUserInputFilter': '' + }), + 'filterExpression': convert_scope, + 'scopeExpressionList': ignore, # scope will be generated from 'filterExpression' + 'id': keep_as_is, + 'isPublic': rename_to('public'), + 'isShared': rename_to('shared'), + 'items': convert_items, + 'layout': keep_as_is, + 'modifiedOn': keep_as_is, + 'name': keep_as_is, + 'publicToken': drop_it, + 'schema': convert_schema, + 'teamId': keep_as_is, + 'username': keep_as_is, + 'version': keep_as_is, + } + + # + # Apply migrations + # + migrated = {} + for key in migrations.keys(): + migrations[key](key, copy.deepcopy(dashboard), migrated) + + return True, migrated # For backwards compatibility From 49c4beef4de25640145ed21fbf3e77d3bb875e8c Mon Sep 17 00:00:00 2001 From: davideschiera Date: Tue, 2 Apr 2019 11:59:30 -0700 Subject: [PATCH 28/48] Temporary change --- sdcclient/_monitor.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 3b6ad052..08dfb302 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -615,14 +615,7 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared # res = requests.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': template}), verify=self.ssl_verify) - result = self._request_result(res) - - if result[0] == False: - print json.dumps({'dashboard': template}) - # else: - # print json.dumps({'dashboard': template}) - - return result + return self._request_result(res) def create_dashboard_from_view(self, newdashname, viewname, filter, shared=False, public=False, annotations={}): '''**Description** From aa6aeacf4d669ac728b49b0fa9011f7c056af6d6 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Tue, 2 Apr 2019 13:17:28 -0700 Subject: [PATCH 29/48] More fixes --- sdcclient/_monitor.py | 71 ++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 08dfb302..76ed8728 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -901,27 +901,34 @@ def _convert_dashboard_v1_to_v2(self, dashboard): # # Each converter will apply changes to v2 dashboard configuration according to v1 # + def when_set(converter): + def fn(prop_name, old_obj, new_obj): + if prop_name in old_obj and old_obj[prop_name] is not None: + converter(prop_name, old_obj, new_obj) + + return fn + def with_default(converter, default=None): - def fn(prop_name, old_dashboard, new_dashboard): - if prop_name not in old_dashboard: - old_dashboard[prop_name] = default + def fn(prop_name, old_obj, new_obj): + if prop_name not in old_obj: + old_obj[prop_name] = default - converter(prop_name, old_dashboard, new_dashboard) + converter(prop_name, old_obj, new_obj) return fn - def keep_as_is(prop_name, old_dashboard, new_dashboard): - new_dashboard[prop_name] = old_dashboard[prop_name] + def keep_as_is(prop_name, old_obj, new_obj): + new_obj[prop_name] = old_obj[prop_name] - def drop_it(prop_name = None, old_dashboard = None, new_dashboard = None): + def drop_it(prop_name = None, old_obj = None, new_obj = None): pass - def ignore(prop_name = None, old_dashboard = None, new_dashboard = None): + def ignore(prop_name = None, old_obj = None, new_obj = None): pass def rename_to(new_prop_name): - def rename(prop_name, old_dashboard, new_dashboard): - new_dashboard[new_prop_name] = old_dashboard[prop_name] + def rename(prop_name, old_obj, new_obj): + new_obj[new_prop_name] = old_obj[prop_name] return rename @@ -983,19 +990,21 @@ def convert_group(prop_name, old_widget, new_widget): new_widget['groupingLabelsIds'] = migrated def convert_name(prop_name, old_widget, new_widget): - keep_as_is(prop_name, old_widget, new_widget) - + # + # enforce unique name (on old dashboard, before migration) + # unique_id = 1 name = old_widget[prop_name] for widget in old_dashboard['items']: if widget == old_widget: - return + break - if new_widget[prop_name] == widget[prop_name]: - new_widget[prop_name] = '{} ({})'.format(name, unique_id) + if old_widget[prop_name] == widget[prop_name]: + old_widget[prop_name] = '{} ({})'.format(name, unique_id) unique_id += 1 + keep_as_is(prop_name, old_widget, new_widget) def convert_metrics(prop_name, old_widget, new_widget): def convert_property_name(prop_name, old_metric, new_metric): @@ -1024,26 +1033,26 @@ def convert_property_name(prop_name, old_metric, new_metric): new_widget['metrics'] = migrated_metrics widget_migrations = { - 'colorCoding': convert_color_coding, - 'compareToConfig': keep_as_is, - 'customDisplayOptions': convert_display_options, - 'gridConfiguration': keep_as_is, - 'group': convert_group, - 'hasTransparentBackground': rename_to('transparentBackground'), - 'limitToScope': keep_as_is, - 'isPanelTitleVisible': rename_to('panelTitleVisible'), - 'markdownSource': keep_as_is, - 'metrics': convert_metrics, - 'name': convert_name, + 'colorCoding': when_set(convert_color_coding), + 'compareToConfig': when_set(keep_as_is), + 'customDisplayOptions': with_default(convert_display_options, {}), + 'gridConfiguration': drop_it, + 'group': when_set(convert_group), + 'hasTransparentBackground': when_set(rename_to('transparentBackground')), + 'limitToScope': when_set(keep_as_is), + 'isPanelTitleVisible': when_set(rename_to('panelTitleVisible')), + 'markdownSource': when_set(keep_as_is), + 'metrics': with_default(convert_metrics, []), + 'name': with_default(convert_name, 'Panel'), 'overrideFilter': rename_to('overrideScope'), 'paging': drop_it, - 'scope': keep_as_is, + 'scope': with_default(keep_as_is, None), 'showAs': keep_as_is, 'showAsType': drop_it, 'sorting': drop_it, - 'textpanelTooltip': keep_as_is, + 'textpanelTooltip': when_set(keep_as_is), } migrated_widgets = [] @@ -1053,12 +1062,10 @@ def convert_property_name(prop_name, old_metric, new_metric): } for key in widget_migrations.keys(): - if key in old_widget: - widget_migrations[key](key, old_widget, migrated_widget) + widget_migrations[key](key, old_widget, migrated_widget) migrated_widgets.append(migrated_widget) - - + new_dashboard['widgets'] = migrated_widgets return migrated From 22f1baed7362cff98d38c5e1e447d7dbdffc56ec Mon Sep 17 00:00:00 2001 From: davideschiera Date: Tue, 2 Apr 2019 13:19:39 -0700 Subject: [PATCH 30/48] Comment --- sdcclient/_monitor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 76ed8728..b48ea56b 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -1046,9 +1046,7 @@ def convert_property_name(prop_name, old_metric, new_metric): 'name': with_default(convert_name, 'Panel'), 'overrideFilter': rename_to('overrideScope'), 'paging': drop_it, - 'scope': with_default(keep_as_is, None), - 'showAs': keep_as_is, 'showAsType': drop_it, 'sorting': drop_it, @@ -1058,6 +1056,7 @@ def convert_property_name(prop_name, old_metric, new_metric): migrated_widgets = [] for old_widget in old_dashboard[prop_name]: migrated_widget = { + # create unique ID 'id': len(migrated_widgets) + 1 } From 900ca4171d48aa291e16fdff86261d70c9710f61 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Mon, 8 Apr 2019 14:13:20 -0700 Subject: [PATCH 31/48] Support "not x = y" and "not x != y" --- sdcclient/_monitor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 8c921516..27916b1f 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -788,8 +788,8 @@ def _convert_scope_string_to_expression(self, scope): operator_parse_dict = { 'in': 'in' if not is_not_operator else 'notIn', - '=': 'equals', - '!=': 'notEquals', + '=': 'equals' if not is_not_operator else 'notEquals', + '!=': 'notEquals' if not is_not_operator else 'equals', 'contains': 'contains' if not is_not_operator else 'notContains', 'starts with': 'startsWith' } From 49a7274d7409c24e5ac101bf8e3a2f9ecfb7e89f Mon Sep 17 00:00:00 2001 From: davideschiera Date: Mon, 8 Apr 2019 14:22:38 -0700 Subject: [PATCH 32/48] Drop support for annotations in dashboards --- sdcclient/_monitor.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index e1987a7b..3221aed2 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -564,7 +564,7 @@ def filter_fn(panel): else: return [False, 'Not found'] - def create_dashboard_from_template(self, dashboard_name, template, scope, shared=False, public=False, annotations={}): + def create_dashboard_from_template(self, dashboard_name, template, scope, shared=False, public=False): if scope is not None: if isinstance(scope, basestring) == False: return [False, 'Invalid scope format: Expected a string'] @@ -600,20 +600,13 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared chart_scope = chart['scope'] if 'scope' in chart else None chart['overrideScope'] = chart_scope != scope - # if 'annotations' in template: - # template['annotations'].update(annotations) - # else: - # template['annotations'] = annotations - - # template['annotations']['createdByEngine'] = True - # # Create the new dashboard # res = requests.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': template}), verify=self.ssl_verify) return self._request_result(res) - def create_dashboard_from_view(self, newdashname, viewname, filter, shared=False, public=False, annotations={}): + def create_dashboard_from_view(self, newdashname, viewname, filter, shared=False, public=False): '''**Description** Create a new dasboard using one of the Sysdig Monitor views as a template. You will be able to define the scope of the new dashboard. @@ -623,7 +616,6 @@ def create_dashboard_from_view(self, newdashname, viewname, filter, shared=False - **filter**: a boolean expression combining Sysdig Monitor segmentation criteria that defines what the new dasboard will be applied to. For example: *kubernetes.namespace.name='production' and container.image='nginx'*. - **shared**: if set to True, the new dashboard will be a shared one. - **public**: if set to True, the new dashboard will be shared with public token. - - **annotations**: an optional dictionary of custom properties that you can associate to this dashboard for automation or management reasons **Success Return Value** A dictionary showing the details of the new dashboard. @@ -646,9 +638,9 @@ def create_dashboard_from_view(self, newdashname, viewname, filter, shared=False # # Create the new dashboard # - return self.create_dashboard_from_template(newdashname, view, filter, shared, public, annotations) + return self.create_dashboard_from_template(newdashname, view, filter, shared, public) - def create_dashboard_from_dashboard(self, newdashname, templatename, filter, shared=False, public=False, annotations={}): + def create_dashboard_from_dashboard(self, newdashname, templatename, filter, shared=False, public=False): '''**Description** Create a new dasboard using one of the existing dashboards as a template. You will be able to define the scope of the new dasboard. @@ -658,7 +650,6 @@ def create_dashboard_from_dashboard(self, newdashname, templatename, filter, sha - **filter**: a boolean expression combining Sysdig Monitor segmentation criteria defines what the new dasboard will be applied to. For example: *kubernetes.namespace.name='production' and container.image='nginx'*. - **shared**: if set to True, the new dashboard will be a shared one. - **public**: if set to True, the new dashboard will be shared with public token. - - **annotations**: an optional dictionary of custom properties that you can associate to this dashboard for automation or management reasons **Success Return Value** A dictionary showing the details of the new dashboard. @@ -692,9 +683,9 @@ def create_dashboard_from_dashboard(self, newdashname, templatename, filter, sha # # Create the dashboard # - return self.create_dashboard_from_template(newdashname, dboard, filter, shared, public, annotations) + return self.create_dashboard_from_template(newdashname, dboard, filter, shared, public) - def create_dashboard_from_file(self, dashboard_name, filename, filter, shared=False, public=False, annotations={}): + def create_dashboard_from_file(self, dashboard_name, filename, filter, shared=False, public=False): ''' **Description** Create a new dasboard using a dashboard template saved to disk. See :func:`~SdcClient.save_dashboard_to_file` to use the file to create a dashboard (usefl to create and restore backups). @@ -711,7 +702,6 @@ def create_dashboard_from_file(self, dashboard_name, filename, filter, shared=Fa - **filter**: a boolean expression combining Sysdig Monitor segmentation criteria defines what the new dasboard will be applied to. For example: *kubernetes.namespace.name='production' and container.image='nginx'*. - **shared**: if set to True, the new dashboard will be a shared one. - **public**: if set to True, the new dashboard will be shared with public token. - - **annotations**: an optional dictionary of custom properties that you can associate to this dashboard for automation or management reasons **Success Return Value** A dictionary showing the details of the new dashboard. @@ -739,7 +729,7 @@ def create_dashboard_from_file(self, dashboard_name, filename, filter, shared=Fa # # Create the new dashboard # - return self.create_dashboard_from_template(dashboard_name, dashboard, filter, shared, public, annotations) + return self.create_dashboard_from_template(dashboard_name, dashboard, filter, shared, public) def save_dashboard_to_file(self, dashboard, filename): ''' From 1e394761e02ff54e2468f83b12fb18dac7057fce Mon Sep 17 00:00:00 2001 From: davideschiera Date: Mon, 8 Apr 2019 14:24:22 -0700 Subject: [PATCH 33/48] Remove panel ID --- sdcclient/_monitor.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 3221aed2..f9a72c0b 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -406,14 +406,6 @@ def add_dashboard_panel(self, dashboard, name, panel_type, metrics, scope=None, } } - # - # Set unique ID (incremental from 1) - # - id = 1 - while len(filter(lambda w: w['id'] == id, dashboard['widgets'])) > 0: - id += 1 - panel_configuration['id'] = id - if panel_type == 'timeSeries': # # In case of a time series, the current dashboard implementation From fbaf5d94fe60e86c27fc54e0e7662739e7716f4e Mon Sep 17 00:00:00 2001 From: davideschiera Date: Mon, 8 Apr 2019 14:25:34 -0700 Subject: [PATCH 34/48] Remove panel ID --- sdcclient/_monitor.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index ea3f8c27..8986b761 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -1037,10 +1037,7 @@ def convert_property_name(prop_name, old_metric, new_metric): migrated_widgets = [] for old_widget in old_dashboard[prop_name]: - migrated_widget = { - # create unique ID - 'id': len(migrated_widgets) + 1 - } + migrated_widget = {} for key in widget_migrations.keys(): widget_migrations[key](key, old_widget, migrated_widget) From f75d491d054efebc4a0bf501fd7036df19893fa7 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Wed, 10 Apr 2019 10:13:24 -0400 Subject: [PATCH 35/48] Convert scope conversion function to public/static --- examples/dashboard_scope.py | 11 ++--------- sdcclient/_monitor.py | 5 +++-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/examples/dashboard_scope.py b/examples/dashboard_scope.py index 2c3db9c7..7f2d0ac9 100755 --- a/examples/dashboard_scope.py +++ b/examples/dashboard_scope.py @@ -9,22 +9,15 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) from sdcclient import SdcClient -# random token, since we're not going to use the API for real -sdc_token = "bb8754d5-025d-4b88-a815-358fba58c4be" - -# -# Instantiate the SDC client -# -sdclient = SdcClient(sdc_token) # # Scopes can be passed to most of dashboard-related functions, e.g. create_dashboard_from_file. # -# NOTE: _convert_scope_string_to_expression should never be used in a user script +# NOTE: convert_scope_string_to_expression should never be used in a user script # We're going to use it here just to demonstrate some scope options and some constraints # def evaluate(scope, expected): - parsed_scope = sdclient._convert_scope_string_to_expression(scope) + parsed_scope = SdcClient.convert_scope_string_to_expression(scope) print '{} is valid: {}'.format(scope, parsed_scope[0] == True) if parsed_scope[0] != expected: diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 27916b1f..0c21c7a3 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -561,7 +561,7 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared template['publicToken'] = None # set dashboard scope to the specific parameter - scopeExpression = self._convert_scope_string_to_expression(scope) + scopeExpression = self.convert_scope_string_to_expression(scope) if scopeExpression[0] == False: return scopeExpression template['filterExpression'] = scope @@ -749,7 +749,8 @@ def get_metrics(self): res = requests.get(self.url + '/api/data/metrics', headers=self.hdrs, verify=self.ssl_verify) return self._request_result(res) - def _convert_scope_string_to_expression(self, scope): + @staticmethod + def convert_scope_string_to_expression(scope): '''**Description** Internal function to convert a filter string to a filter object to be used with dashboards. ''' From 4ffa6a01ebfb28e62e88c71adbbb1914e4d044f7 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Wed, 10 Apr 2019 14:33:03 -0400 Subject: [PATCH 36/48] Fix fn name --- sdcclient/_monitor_v1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdcclient/_monitor_v1.py b/sdcclient/_monitor_v1.py index a1f8516b..0c962082 100644 --- a/sdcclient/_monitor_v1.py +++ b/sdcclient/_monitor_v1.py @@ -39,7 +39,7 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared template['publicToken'] = None # set dashboard scope to the specific parameter - scopeExpression = self._convert_scope_string_to_expression(scope) + scopeExpression = self.convert_scope_string_to_expression(scope) if scopeExpression[0] == False: return scopeExpression template['filterExpression'] = scope From dc89e1da0871275251b61e46de5a6c9991813039 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Wed, 10 Apr 2019 14:33:22 -0400 Subject: [PATCH 37/48] Fix fn name --- sdcclient/_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 207d0c8c..7c414200 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -922,7 +922,7 @@ def convert_scope(prop_name, old_dashboard, new_dashboard): # # TODO! scope = old_dashboard[prop_name] - scope_conversion = self._convert_scope_string_to_expression(scope) + scope_conversion = self.convert_scope_string_to_expression(scope) if scope_conversion[0]: if scope_conversion[1]: From 8512fa6825e2872764cf6c66ace378a9db694ddd Mon Sep 17 00:00:00 2001 From: davideschiera Date: Wed, 10 Apr 2019 14:38:31 -0400 Subject: [PATCH 38/48] Improve script output --- examples/dashboard_backup_v1_restore_v2.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/dashboard_backup_v1_restore_v2.py b/examples/dashboard_backup_v1_restore_v2.py index 6e198299..5f75ba31 100644 --- a/examples/dashboard_backup_v1_restore_v2.py +++ b/examples/dashboard_backup_v1_restore_v2.py @@ -41,8 +41,11 @@ for item in res['dashboards']: file_name = '{}.json'.format(item['id']) + print('Saving v1 dashboard {} to file {}...'.format( + item['name'], file_name)) sdclientV1.save_dashboard_to_file(item, file_name) + print('Importing dashboard to v2...') ok, res = sdclient.create_dashboard_from_file( u'import of {}'.format(item['name']), file_name, @@ -51,9 +54,10 @@ public=item['isPublic']) if ok: + print('Dashboard {} imported!'.format(item['name'])) sdclient.delete_dashboard(res['dashboard']) else: - print('=====') + print('Dashboard {} import failed:'.format(item['name'])) print(res) - print('=====') + print('\n') From 8292983a401aa97a90a6b93cbed92e8662d9e258 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Wed, 10 Apr 2019 15:02:22 -0400 Subject: [PATCH 39/48] Improve script --- examples/dashboard_backup_v1_restore_v2.py | 40 ++++++++++++---------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/examples/dashboard_backup_v1_restore_v2.py b/examples/dashboard_backup_v1_restore_v2.py index 5f75ba31..6bb3e6b3 100644 --- a/examples/dashboard_backup_v1_restore_v2.py +++ b/examples/dashboard_backup_v1_restore_v2.py @@ -14,50 +14,54 @@ # # Parse arguments # -if len(sys.argv) != 2: - print('usage: %s ' % sys.argv[0]) +if len(sys.argv) != 5: + print( + 'usage: %s ' + % sys.argv[0]) print( 'You can find your token at https://app.sysdigcloud.com/#/settings/user' ) sys.exit(1) -sdc_token = sys.argv[1] +sdc_v1_url = sys.argv[1] +sdc_v1_token = sys.argv[2] +sdc_v2_url = sys.argv[3] +sdc_v2_token = sys.argv[4] # # Instantiate the SDC client # -sdclient = SdMonitorClient(sdc_token, sdc_url='https://app.sysdigcloud.com') -sdclientV1 = SdMonitorClientV1( - sdc_token, sdc_url='https://app.sysdigcloud.com') +sdclient_v2 = SdMonitorClient(sdc_v2_token, sdc_url=sdc_v2_url) +sdclient_v1 = SdMonitorClientV1(sdc_v1_token, sdc_url=sdc_v1_url) # # Serialize the first user dashboard to disk # -ok, res = sdclientV1.get_dashboards() +ok, res = sdclient_v1.get_dashboards() if not ok: print(res) sys.exit(1) -for item in res['dashboards']: - file_name = '{}.json'.format(item['id']) +for dashboard in res['dashboards']: + file_name = '{}.json'.format(dashboard['id']) print('Saving v1 dashboard {} to file {}...'.format( - item['name'], file_name)) - sdclientV1.save_dashboard_to_file(item, file_name) + dashboard['name'], file_name)) + sdclient_v1.save_dashboard_to_file(dashboard, file_name) print('Importing dashboard to v2...') - ok, res = sdclient.create_dashboard_from_file( - u'import of {}'.format(item['name']), + ok, res = sdclient_v2.create_dashboard_from_file( + u'import of {}'.format(dashboard['name']), file_name, None, - shared=item['isShared'], - public=item['isPublic']) + shared=dashboard['isShared'], + public=dashboard['isPublic']) if ok: - print('Dashboard {} imported!'.format(item['name'])) - sdclient.delete_dashboard(res['dashboard']) + print('Dashboard {} imported!'.format(dashboard['name'])) + sdclient_v2.delete_dashboard(res['dashboard']) else: - print('Dashboard {} import failed:'.format(item['name'])) + print('Dashboard {} import failed:'.format(dashboard['name'])) print(res) print('\n') From 5c2992489f01694a8526627d30599a61e3f94db9 Mon Sep 17 00:00:00 2001 From: Julio J Date: Thu, 11 Apr 2019 12:47:18 +0200 Subject: [PATCH 40/48] Keep as is the GridConfiguration for panels --- sdcclient/_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 7c414200..db198457 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -1019,7 +1019,7 @@ def convert_property_name(prop_name, old_metric, new_metric): 'colorCoding': when_set(convert_color_coding), 'compareToConfig': when_set(keep_as_is), 'customDisplayOptions': with_default(convert_display_options, {}), - 'gridConfiguration': drop_it, + 'gridConfiguration': keep_as_is, 'group': when_set(convert_group), 'hasTransparentBackground': when_set(rename_to('transparentBackground')), 'limitToScope': when_set(keep_as_is), From 436b005490cfdfe00745e15c43f90caf7224dacf Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 11 Apr 2019 14:15:12 -0400 Subject: [PATCH 41/48] Merge changes from #84 to API v1 --- sdcclient/_monitor_v1.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdcclient/_monitor_v1.py b/sdcclient/_monitor_v1.py index 0c962082..ebad67b6 100644 --- a/sdcclient/_monitor_v1.py +++ b/sdcclient/_monitor_v1.py @@ -60,9 +60,10 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared if chart['overrideFilter'] == False: # patch frontend bug to hide scope override warning even when it's not really overridden chart['scope'] = scope - + # if chart scope is equal to dashboard scope, set it as non override - chart['overrideFilter'] = chart['scope'] != scope + chart_scope = chart['scope'] if 'scope' in chart else None + chart['overrideFilter'] = chart_scope != scope if 'annotations' in template: template['annotations'].update(annotations) @@ -269,6 +270,7 @@ def remove_dashboard_panel(self, dashboard, panel_name): # def filter_fn(panel): return panel['name'] == panel_name + panels = list(filter(filter_fn, dashboard_configuration['items'])) if len(panels) > 0: From df8aabb88e36f9f1c5f58752f2ef614fec368588 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 11 Apr 2019 14:41:03 -0400 Subject: [PATCH 42/48] Handle old files --- examples/restore_dashboards.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/restore_dashboards.py b/examples/restore_dashboards.py index 01a22beb..8ff55c22 100644 --- a/examples/restore_dashboards.py +++ b/examples/restore_dashboards.py @@ -36,6 +36,12 @@ print('Invalid JSON file found in ZIP file ' + info.filename + ': skipping') continue + # + # Handle old files + # + if 'dashboard' in j: + j = j['dashboard'] + ok, res = sdclient.create_dashboard_with_configuration(j) if ok: print('Restored Dashboard named: ' + j['name']) From 9dd861b48797b5aa8836b186027da4f34dee0e31 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 11 Apr 2019 16:58:33 -0400 Subject: [PATCH 43/48] Drop layout property --- sdcclient/_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index db198457..6c031a9d 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -1061,7 +1061,7 @@ def convert_property_name(prop_name, old_metric, new_metric): 'isPublic': rename_to('public'), 'isShared': rename_to('shared'), 'items': convert_items, - 'layout': keep_as_is, + 'layout': drop_it, 'modifiedOn': keep_as_is, 'name': keep_as_is, 'publicToken': drop_it, From 43c4dc8fdc64fdddd6e9c8b38d64c16a42d76bfa Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 11 Apr 2019 17:27:22 -0400 Subject: [PATCH 44/48] Print all errors, not just the first one --- sdcclient/_common.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/sdcclient/_common.py b/sdcclient/_common.py index a69b2830..af9f7f62 100644 --- a/sdcclient/_common.py +++ b/sdcclient/_common.py @@ -38,16 +38,18 @@ def _checkResponse(self, res): return False if 'errors' in j: - if 'message' in j['errors'][0]: - self.lasterr = j['errors'][0]['message'] + error_msgs = [] + for error in j['errors']: + error_msg = [] + if 'message' in error: + error_msg.append(error['message']) - if 'reason' in j['errors'][0]: - if self.lasterr is not None: - self.lasterr += ' ' - else: - self.lasrerr = '' + if 'reason' in error: + error_msg.append(error['reason']) - self.lasterr += j['errors'][0]['reason'] + error_msgs.append(': '.join(error_msg)) + + self.lasterr = '\n'.join(error_msgs) elif 'message' in j: self.lasterr = j['message'] else: From f98d049fe04977e7b4fce621ef2d2ae42c07383e Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 11 Apr 2019 17:55:09 -0400 Subject: [PATCH 45/48] Fix conversion of metrics and topology panels --- sdcclient/_monitor.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 6c031a9d..f3cba09f 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -972,6 +972,14 @@ def convert_group(prop_name, old_widget, new_widget): new_widget['groupingLabelsIds'] = migrated + def convert_override_filter(prop_name, old_widget, new_widget): + if old_widget['showAs'] == 'map': + # override scope always true if scope is set + new_widget['overrideScope'] = True + else: + new_widget['overrideScope'] = old_widget[prop_name] + + def convert_name(prop_name, old_widget, new_widget): # # enforce unique name (on old dashboard, before migration) @@ -1013,6 +1021,21 @@ def convert_property_name(prop_name, old_metric, new_metric): migrated_metrics.append(migrated_metric) + # Property name convention: + # timestamp: k0 (if present) + # other keys: k* (from 0 or 1, depending on timestamp) + # values: v* (from 0) + timestamp_key = filter(lambda m: m['id'] == 'timestamp' and not ('timeAggregation' in m and m['timeAggregation'] is not None), migrated_metrics) + no_timestamp_keys = filter(lambda m: m['id'] != 'timestamp' and not ('timeAggregation' in m and m['timeAggregation'] is not None), migrated_metrics) + values = filter(lambda m: 'timeAggregation' in m and m['timeAggregation'] is not None, migrated_metrics) + if timestamp_key: + timestamp_key[0]['propertyName'] = 'k0' + k_offset = 1 if timestamp_key else 0 + for i in range(0, len(no_timestamp_keys)): + no_timestamp_keys[i]['propertyName'] = 'k{}'.format(i + k_offset) + for i in range(0, len(values)): + values[i]['propertyName'] = 'v{}'.format(i) + new_widget['metrics'] = migrated_metrics widget_migrations = { @@ -1027,7 +1050,7 @@ def convert_property_name(prop_name, old_metric, new_metric): 'markdownSource': when_set(keep_as_is), 'metrics': with_default(convert_metrics, []), 'name': with_default(convert_name, 'Panel'), - 'overrideFilter': rename_to('overrideScope'), + 'overrideFilter': convert_override_filter, 'paging': drop_it, 'scope': with_default(keep_as_is, None), 'showAs': keep_as_is, From 4daa2fdb0f77a1f91fbd6bf129758c4182965f4b Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 12 Apr 2019 10:02:40 -0700 Subject: [PATCH 46/48] Fix import of topology panels --- sdcclient/_monitor.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index f3cba09f..29629cac 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -591,9 +591,13 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared # patch frontend bug to hide scope override warning even when it's not really overridden chart['scope'] = scope - # if chart scope is equal to dashboard scope, set it as non override - chart_scope = chart['scope'] if 'scope' in chart else None - chart['overrideScope'] = chart_scope != scope + if chart['showAs'] != 'map': + # if chart scope is equal to dashboard scope, set it as non override + chart_scope = chart['scope'] if 'scope' in chart else None + chart['overrideScope'] = chart_scope != scope + else: + # topology panels must override the scope + chart['overrideScope'] = True # # Create the new dashboard @@ -968,9 +972,9 @@ def convert_group(prop_name, old_widget, new_widget): migrated = [] for metric in group_by_metrics: - migrated.append({ 'labelId': metric['metric'] }) + migrated.append({ 'id': metric['metric'] }) - new_widget['groupingLabelsIds'] = migrated + new_widget['groupingLabelIds'] = migrated def convert_override_filter(prop_name, old_widget, new_widget): if old_widget['showAs'] == 'map': From c1b0a4f8e53e01163dea7f112df7ca85ca6cd6bf Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 12 Apr 2019 10:57:11 -0700 Subject: [PATCH 47/48] Fix import of default dashboards --- sdcclient/_monitor.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 29629cac..49f189ba 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -572,6 +572,13 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared template['public'] = public template['publicToken'] = None + # default dashboards don't have eventsOverlaySettings property + # make sure to add the default set if the template doesn't include it + if 'eventsOverlaySettings' not in template or not template['eventsOverlaySettings']: + template['eventsOverlaySettings'] = { + 'filterNotificationsUserInputFilter': '' + } + # set dashboard scope to the specific parameter scopeExpression = self.convert_scope_string_to_expression(scope) if scopeExpression[0] == False: From 87d1205bfa71c219d6799b38ffc97659e9164b7d Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 12 Apr 2019 11:05:38 -0700 Subject: [PATCH 48/48] Sort metrics --- sdcclient/_monitor.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 49f189ba..9de57a3c 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -1036,18 +1036,22 @@ def convert_property_name(prop_name, old_metric, new_metric): # timestamp: k0 (if present) # other keys: k* (from 0 or 1, depending on timestamp) # values: v* (from 0) + sorted_metrics = [] timestamp_key = filter(lambda m: m['id'] == 'timestamp' and not ('timeAggregation' in m and m['timeAggregation'] is not None), migrated_metrics) no_timestamp_keys = filter(lambda m: m['id'] != 'timestamp' and not ('timeAggregation' in m and m['timeAggregation'] is not None), migrated_metrics) values = filter(lambda m: 'timeAggregation' in m and m['timeAggregation'] is not None, migrated_metrics) if timestamp_key: timestamp_key[0]['propertyName'] = 'k0' + sorted_metrics.append(timestamp_key[0]) k_offset = 1 if timestamp_key else 0 for i in range(0, len(no_timestamp_keys)): no_timestamp_keys[i]['propertyName'] = 'k{}'.format(i + k_offset) + sorted_metrics.append(no_timestamp_keys[i]) for i in range(0, len(values)): values[i]['propertyName'] = 'v{}'.format(i) + sorted_metrics.append(values[i]) - new_widget['metrics'] = migrated_metrics + new_widget['metrics'] = sorted_metrics widget_migrations = { 'colorCoding': when_set(convert_color_coding),