From 3bc52050e9a030d6af6840823d4899a6674d3e47 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 29 Mar 2019 15:26:49 -0700 Subject: [PATCH 01/18] Scaffolding dashboard conversions --- sdcclient/_monitor.py | 32 ++++++++++++++++++++++++++++++++ sdcclient/_monitor_v1.py | 7 +++++++ 2 files changed, 39 insertions(+) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 3f781535..60dce3c5 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -735,6 +735,15 @@ def create_dashboard_from_file(self, dashboard_name, filename, filter, shared=Fa dashboard = loaded_object['dashboard'] + if loaded_object['version'] != self._dashboards_api_version: + # + # Convert the dashboard (if possible) + # + conversion_result, dashboard = self._convert_dashboard_to_current_version(dashboard, loaded_object['version']) + + if conversion_result == False: + return conversion_result, dashboard + # # Create the new dashboard # @@ -854,6 +863,29 @@ def _convert_scope_string_to_expression(self, scope): return [True, expressions] + def _get_dashboard_converters(self): + return { + 'v2': { + # 'v1': _convert_dashboard_v1_to_v2 + } + } + + def _convert_dashboard_to_current_version(self, dashboard, version): + converters_to = self._get_dashboard_converters().get(self._dashboards_api_version, None) + if converters_to == None: + return False, 'unexpected error: no dashboard converters from version {} are supported'.format(self._dashboards_api_version) + + converter = converters_to.get(version, None) + + if converter == None: + return False, 'dashboard version {} cannot be converted to {}'.format(version, self._dashboards_api_version) + + return converter(dashboard) + + +def _convert_dashboard_v1_to_v2(dashboard): + return True, dashboard + # For backwards compatibility SdcClient = SdMonitorClient diff --git a/sdcclient/_monitor_v1.py b/sdcclient/_monitor_v1.py index a1f8516b..83645029 100644 --- a/sdcclient/_monitor_v1.py +++ b/sdcclient/_monitor_v1.py @@ -286,3 +286,10 @@ def filter_fn(panel): return self._request_result(res) else: return [False, 'Not found'] + + def _get_dashboard_converters(self): + '''**Description** + Internal function to return dashboard converters from one version to another one. + ''' + # There's not really a previous version... + return {} From 0c1f1d2475add35cf39dca36ac7b332b38de7d10 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Mon, 1 Apr 2019 13:09:41 -0700 Subject: [PATCH 02/18] Initial implementation of v1 => v2 migration --- sdcclient/_monitor.py | 143 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 2 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 60dce3c5..2fbf25ae 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -866,7 +866,7 @@ def _convert_scope_string_to_expression(self, scope): def _get_dashboard_converters(self): return { 'v2': { - # 'v1': _convert_dashboard_v1_to_v2 + 'v1': _convert_dashboard_v1_to_v2 } } @@ -884,7 +884,146 @@ def _convert_dashboard_to_current_version(self, dashboard, version): def _convert_dashboard_v1_to_v2(dashboard): - return True, dashboard + # + # Migrations + # + # Each converter function will take: + # 1. name of the v1 dashboard property + # 2. v1 dashboard configuration + # 3. v2 dashboard configuration + # + # Each converter will apply changes to v2 dashboard configuration according to v1 + # + def keep_as_is(prop_name, old_dashboard, new_dashboard): + new_dashboard[prop_name] = old_dashboard[prop_name] + + def drop_it(prop_name = None, old_dashboard = None, new_dashboard = None): + pass + + def rename_to(new_prop_name): + def rename(prop_name, old_dashboard, new_dashboard): + new_dashboard[new_prop_name] = old_dashboard[prop_name] + + return rename + + def convert_schema(prop_name, old_dashboard, new_dashboard): + new_dashboard[prop_name] = 2 + + def convert_scope(prop_name, old_dashboard, new_dashboard): + drop_it() + + def convert_events_filter(prop_name, old_dashboard, new_dashboard): + rename_to('eventsOverlaySettings')(prop_name, old_dashboard, new_dashboard) + + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameMetrics'] + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameScope'] + + def convert_items(prop_name, old_dashboard, new_dashboard): + def convert_color_coding(prop_name, old_widget, new_widget): + best_value = None + worst_value= None + for item in old_widget[prop_name]['thresholds']: + if item['color'] == 'best': + best_value = item['max'] if not item['max'] else item['min'] + elif item['color'] == 'worst': + worst_value = item['min'] if not item['min'] else item['max'] + + if best_value is not None and worst_value is not None: + new_widget[prop_name] = { + 'best': best_value, + 'worst': worst_value + } + + def convert_display_options(prop_name, old_widget, new_widget): + keep_as_is(prop_name, old_widget, new_widget) + + if 'yAxisScaleFactor' in new_widget[prop_name]: + del new_widget[prop_name]['yAxisScaleFactor'] + + def convert_group(prop_name, old_widget, new_widget): + group_by_metrics = old_widget[prop_name]['configuration']['groups'][0]['groupBy'] + + migrated = [] + for metric in group_by_metrics: + migrated.append({ 'labelId': metric['metric'] }) + + new_widget['groupingLabelsIds'] = migrated + + def convert_metrics(prop_name, old_widget, new_widget): + keep_as_is(prop_name, old_widget, new_widget) + + for metric in new_widget[prop_name]: + rename_to('id')('metricId', metric, metric) + if 'aggregation' in metric: + # timestamp metric doesn't have aggregations + rename_to('timeAggregation')('aggregation', metric, metric) + + widget_migrations = { + 'colorCoding': convert_color_coding, + 'compareToConfig': keep_as_is, + 'customDisplayOptions': convert_display_options, + 'gridConfiguration': keep_as_is, + 'group': convert_group, + 'hasTransparentBackground': rename_to('transparentBackground'), + 'limitToScope': keep_as_is, + 'isPanelTitleVisible': rename_to('panelTitleVisible'), + 'markdownSource': keep_as_is, + 'limitToScope': keep_as_is, + 'metrics': convert_metrics, + 'name': keep_as_is, + 'overrideFilter': rename_to('overrideScope'), + 'paging': drop_it, + 'scope': keep_as_is, + 'showAs': keep_as_is, + 'showAsType': drop_it, + 'sorting': drop_it, + 'textpanelTooltip': keep_as_is, + } + + migrated_widgets = [] + for old_widget in old_dashboard[prop_name]: + migrated_widget = {} + + for key in widget_migrations.keys(): + if key in old_widget: + widget_migrations[key](key, old_widget, migrated_widget) + + migrated_widgets.append(migrated_widget) + + + new_dashboard['widgets'] = migrated_widgets + + return migrated + + migrations = { + 'autoCreated': keep_as_is, + 'createdOn': keep_as_is, + 'eventsFilter': convert_events_filter, + 'filterExpression': convert_scope, + 'id': keep_as_is, + 'isPublic': rename_to('public'), + 'isShared': rename_to('shared'), + 'items': convert_items, + 'layout': keep_as_is, + 'modifiedOn': keep_as_is, + 'name': keep_as_is, + 'publicToken': drop_it, + 'schema': convert_schema, + 'scopeExpressionList': drop_it, + 'teamId': keep_as_is, + 'username': keep_as_is, + 'version': keep_as_is, + } + + # + # Apply migrations + # + migrated = {} + for key in migrations.keys(): + if key in dashboard: + migrations[key](key, dashboard, migrated) + + return True, migrated # For backwards compatibility From ce26a83223d2680c140edb073523df7209fe0553 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Fri, 29 Mar 2019 15:26:49 -0700 Subject: [PATCH 03/18] Scaffolding dashboard conversions --- sdcclient/_monitor.py | 32 ++++++++++++++++++++++++++++++++ sdcclient/_monitor_v1.py | 7 +++++++ 2 files changed, 39 insertions(+) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index d5ed8d55..becb2e6c 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -736,6 +736,15 @@ def create_dashboard_from_file(self, dashboard_name, filename, filter, shared=Fa dashboard = loaded_object['dashboard'] + if loaded_object['version'] != self._dashboards_api_version: + # + # Convert the dashboard (if possible) + # + conversion_result, dashboard = self._convert_dashboard_to_current_version(dashboard, loaded_object['version']) + + if conversion_result == False: + return conversion_result, dashboard + # # Create the new dashboard # @@ -855,6 +864,29 @@ def _convert_scope_string_to_expression(self, scope): return [True, expressions] + def _get_dashboard_converters(self): + return { + 'v2': { + # 'v1': _convert_dashboard_v1_to_v2 + } + } + + def _convert_dashboard_to_current_version(self, dashboard, version): + converters_to = self._get_dashboard_converters().get(self._dashboards_api_version, None) + if converters_to == None: + return False, 'unexpected error: no dashboard converters from version {} are supported'.format(self._dashboards_api_version) + + converter = converters_to.get(version, None) + + if converter == None: + return False, 'dashboard version {} cannot be converted to {}'.format(version, self._dashboards_api_version) + + return converter(dashboard) + + +def _convert_dashboard_v1_to_v2(dashboard): + return True, dashboard + # For backwards compatibility SdcClient = SdMonitorClient diff --git a/sdcclient/_monitor_v1.py b/sdcclient/_monitor_v1.py index a1f8516b..83645029 100644 --- a/sdcclient/_monitor_v1.py +++ b/sdcclient/_monitor_v1.py @@ -286,3 +286,10 @@ def filter_fn(panel): return self._request_result(res) else: return [False, 'Not found'] + + def _get_dashboard_converters(self): + '''**Description** + Internal function to return dashboard converters from one version to another one. + ''' + # There's not really a previous version... + return {} From 88dc25a92edebe386d4edb6cdf8499a27a323603 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Mon, 1 Apr 2019 13:09:41 -0700 Subject: [PATCH 04/18] Initial implementation of v1 => v2 migration --- sdcclient/_monitor.py | 143 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 2 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index becb2e6c..36f1ad67 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -867,7 +867,7 @@ def _convert_scope_string_to_expression(self, scope): def _get_dashboard_converters(self): return { 'v2': { - # 'v1': _convert_dashboard_v1_to_v2 + 'v1': _convert_dashboard_v1_to_v2 } } @@ -885,7 +885,146 @@ def _convert_dashboard_to_current_version(self, dashboard, version): def _convert_dashboard_v1_to_v2(dashboard): - return True, dashboard + # + # Migrations + # + # Each converter function will take: + # 1. name of the v1 dashboard property + # 2. v1 dashboard configuration + # 3. v2 dashboard configuration + # + # Each converter will apply changes to v2 dashboard configuration according to v1 + # + def keep_as_is(prop_name, old_dashboard, new_dashboard): + new_dashboard[prop_name] = old_dashboard[prop_name] + + def drop_it(prop_name = None, old_dashboard = None, new_dashboard = None): + pass + + def rename_to(new_prop_name): + def rename(prop_name, old_dashboard, new_dashboard): + new_dashboard[new_prop_name] = old_dashboard[prop_name] + + return rename + + def convert_schema(prop_name, old_dashboard, new_dashboard): + new_dashboard[prop_name] = 2 + + def convert_scope(prop_name, old_dashboard, new_dashboard): + drop_it() + + def convert_events_filter(prop_name, old_dashboard, new_dashboard): + rename_to('eventsOverlaySettings')(prop_name, old_dashboard, new_dashboard) + + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameMetrics'] + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameScope'] + + def convert_items(prop_name, old_dashboard, new_dashboard): + def convert_color_coding(prop_name, old_widget, new_widget): + best_value = None + worst_value= None + for item in old_widget[prop_name]['thresholds']: + if item['color'] == 'best': + best_value = item['max'] if not item['max'] else item['min'] + elif item['color'] == 'worst': + worst_value = item['min'] if not item['min'] else item['max'] + + if best_value is not None and worst_value is not None: + new_widget[prop_name] = { + 'best': best_value, + 'worst': worst_value + } + + def convert_display_options(prop_name, old_widget, new_widget): + keep_as_is(prop_name, old_widget, new_widget) + + if 'yAxisScaleFactor' in new_widget[prop_name]: + del new_widget[prop_name]['yAxisScaleFactor'] + + def convert_group(prop_name, old_widget, new_widget): + group_by_metrics = old_widget[prop_name]['configuration']['groups'][0]['groupBy'] + + migrated = [] + for metric in group_by_metrics: + migrated.append({ 'labelId': metric['metric'] }) + + new_widget['groupingLabelsIds'] = migrated + + def convert_metrics(prop_name, old_widget, new_widget): + keep_as_is(prop_name, old_widget, new_widget) + + for metric in new_widget[prop_name]: + rename_to('id')('metricId', metric, metric) + if 'aggregation' in metric: + # timestamp metric doesn't have aggregations + rename_to('timeAggregation')('aggregation', metric, metric) + + widget_migrations = { + 'colorCoding': convert_color_coding, + 'compareToConfig': keep_as_is, + 'customDisplayOptions': convert_display_options, + 'gridConfiguration': keep_as_is, + 'group': convert_group, + 'hasTransparentBackground': rename_to('transparentBackground'), + 'limitToScope': keep_as_is, + 'isPanelTitleVisible': rename_to('panelTitleVisible'), + 'markdownSource': keep_as_is, + 'limitToScope': keep_as_is, + 'metrics': convert_metrics, + 'name': keep_as_is, + 'overrideFilter': rename_to('overrideScope'), + 'paging': drop_it, + 'scope': keep_as_is, + 'showAs': keep_as_is, + 'showAsType': drop_it, + 'sorting': drop_it, + 'textpanelTooltip': keep_as_is, + } + + migrated_widgets = [] + for old_widget in old_dashboard[prop_name]: + migrated_widget = {} + + for key in widget_migrations.keys(): + if key in old_widget: + widget_migrations[key](key, old_widget, migrated_widget) + + migrated_widgets.append(migrated_widget) + + + new_dashboard['widgets'] = migrated_widgets + + return migrated + + migrations = { + 'autoCreated': keep_as_is, + 'createdOn': keep_as_is, + 'eventsFilter': convert_events_filter, + 'filterExpression': convert_scope, + 'id': keep_as_is, + 'isPublic': rename_to('public'), + 'isShared': rename_to('shared'), + 'items': convert_items, + 'layout': keep_as_is, + 'modifiedOn': keep_as_is, + 'name': keep_as_is, + 'publicToken': drop_it, + 'schema': convert_schema, + 'scopeExpressionList': drop_it, + 'teamId': keep_as_is, + 'username': keep_as_is, + 'version': keep_as_is, + } + + # + # Apply migrations + # + migrated = {} + for key in migrations.keys(): + if key in dashboard: + migrations[key](key, dashboard, migrated) + + return True, migrated # For backwards compatibility From 387e0119b0853b4b6c2b5d5eb8f7b17ffeb79d57 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Mon, 1 Apr 2019 15:51:43 -0700 Subject: [PATCH 05/18] Fix migration --- sdcclient/_monitor.py | 85 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 16 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 36f1ad67..55a2f056 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -895,11 +895,23 @@ def _convert_dashboard_v1_to_v2(dashboard): # # Each converter will apply changes to v2 dashboard configuration according to v1 # + def with_default(converter, default=None): + def fn(prop_name, old_dashboard, new_dashboard): + if prop_name not in old_dashboard: + old_dashboard[prop_name] = default + + converter(prop_name, old_dashboard, new_dashboard) + + return fn + def keep_as_is(prop_name, old_dashboard, new_dashboard): new_dashboard[prop_name] = old_dashboard[prop_name] def drop_it(prop_name = None, old_dashboard = None, new_dashboard = None): pass + + def ignore(prop_name = None, old_dashboard = None, new_dashboard = None): + pass def rename_to(new_prop_name): def rename(prop_name, old_dashboard, new_dashboard): @@ -911,13 +923,17 @@ def convert_schema(prop_name, old_dashboard, new_dashboard): new_dashboard[prop_name] = 2 def convert_scope(prop_name, old_dashboard, new_dashboard): - drop_it() + # # TODO! + + new_dashboard['scopeExpressionList'] = None def convert_events_filter(prop_name, old_dashboard, new_dashboard): rename_to('eventsOverlaySettings')(prop_name, old_dashboard, new_dashboard) - del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameMetrics'] - del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameScope'] + if 'showNotificationsDoNotFilterSameMetrics' in new_dashboard['eventsOverlaySettings']: + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameMetrics'] + if 'showNotificationsDoNotFilterSameScope' in new_dashboard['eventsOverlaySettings']: + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameScope'] def convert_items(prop_name, old_dashboard, new_dashboard): def convert_color_coding(prop_name, old_widget, new_widget): @@ -950,14 +966,46 @@ def convert_group(prop_name, old_widget, new_widget): new_widget['groupingLabelsIds'] = migrated - def convert_metrics(prop_name, old_widget, new_widget): + def convert_name(prop_name, old_widget, new_widget): keep_as_is(prop_name, old_widget, new_widget) - for metric in new_widget[prop_name]: - rename_to('id')('metricId', metric, metric) - if 'aggregation' in metric: - # timestamp metric doesn't have aggregations - rename_to('timeAggregation')('aggregation', metric, metric) + unique_id = 1 + name = old_widget[prop_name] + + for widget in old_dashboard['items']: + if widget == old_widget: + return + + if new_widget[prop_name] == widget[prop_name]: + new_widget[prop_name] = '{} ({})'.format(name, unique_id) + unique_id += 1 + + + def convert_metrics(prop_name, old_widget, new_widget): + def convert_property_name(prop_name, old_metric, new_metric): + keep_as_is(prop_name, old_metric, new_metric) + + if old_metric['metricId'] == 'timestamp': + return 'k0' + + metric_migrations = { + 'metricId': rename_to('id'), + 'aggregation': rename_to('timeAggregation'), + 'groupAggregation': rename_to('groupAggregation'), + 'propertyName': convert_property_name + } + + migrated_metrics = [] + for old_metric in old_widget[prop_name]: + migrated_metric = {} + + for key in metric_migrations.keys(): + if key in old_metric: + metric_migrations[key](key, old_metric, migrated_metric) + + migrated_metrics.append(migrated_metric) + + new_widget['metrics'] = migrated_metrics widget_migrations = { 'colorCoding': convert_color_coding, @@ -971,10 +1019,12 @@ def convert_metrics(prop_name, old_widget, new_widget): 'markdownSource': keep_as_is, 'limitToScope': keep_as_is, 'metrics': convert_metrics, - 'name': keep_as_is, + 'name': convert_name, 'overrideFilter': rename_to('overrideScope'), 'paging': drop_it, - 'scope': keep_as_is, + + 'scope': drop_it, # TODO !!!! + 'showAs': keep_as_is, 'showAsType': drop_it, 'sorting': drop_it, @@ -983,7 +1033,9 @@ def convert_metrics(prop_name, old_widget, new_widget): migrated_widgets = [] for old_widget in old_dashboard[prop_name]: - migrated_widget = {} + migrated_widget = { + 'id': len(migrated_widgets) + 1 + } for key in widget_migrations.keys(): if key in old_widget: @@ -999,8 +1051,11 @@ def convert_metrics(prop_name, old_widget, new_widget): migrations = { 'autoCreated': keep_as_is, 'createdOn': keep_as_is, - 'eventsFilter': convert_events_filter, + 'eventsFilter': with_default(convert_events_filter, { + 'filterNotificationsUserInputFilter': '' + }), 'filterExpression': convert_scope, + 'scopeExpressionList': ignore, # scope will be generated from 'filterExpression' 'id': keep_as_is, 'isPublic': rename_to('public'), 'isShared': rename_to('shared'), @@ -1010,7 +1065,6 @@ def convert_metrics(prop_name, old_widget, new_widget): 'name': keep_as_is, 'publicToken': drop_it, 'schema': convert_schema, - 'scopeExpressionList': drop_it, 'teamId': keep_as_is, 'username': keep_as_is, 'version': keep_as_is, @@ -1021,8 +1075,7 @@ def convert_metrics(prop_name, old_widget, new_widget): # migrated = {} for key in migrations.keys(): - if key in dashboard: - migrations[key](key, dashboard, migrated) + migrations[key](key, copy.deepcopy(dashboard), migrated) return True, migrated From 1b50dbe3fc88597d3a236a02286de49d5204451f Mon Sep 17 00:00:00 2001 From: davideschiera Date: Mon, 1 Apr 2019 15:52:05 -0700 Subject: [PATCH 06/18] Add migration example --- examples/dashboard_backup_v1_restore_v2.py | 59 ++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 examples/dashboard_backup_v1_restore_v2.py diff --git a/examples/dashboard_backup_v1_restore_v2.py b/examples/dashboard_backup_v1_restore_v2.py new file mode 100644 index 00000000..6e198299 --- /dev/null +++ b/examples/dashboard_backup_v1_restore_v2.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# +# Save the first user dashboard to file and then use create_dashboard_from_file() +# to apply the stored dasboard again with a different filter. +# +import os +import sys +import json +sys.path.insert( + 0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) +from sdcclient import SdMonitorClient +from sdcclient import SdMonitorClientV1 + +# +# Parse arguments +# +if len(sys.argv) != 2: + print('usage: %s ' % sys.argv[0]) + print( + 'You can find your token at https://app.sysdigcloud.com/#/settings/user' + ) + sys.exit(1) + +sdc_token = sys.argv[1] + +# +# Instantiate the SDC client +# +sdclient = SdMonitorClient(sdc_token, sdc_url='https://app.sysdigcloud.com') +sdclientV1 = SdMonitorClientV1( + sdc_token, sdc_url='https://app.sysdigcloud.com') + +# +# Serialize the first user dashboard to disk +# +ok, res = sdclientV1.get_dashboards() + +if not ok: + print(res) + sys.exit(1) + +for item in res['dashboards']: + file_name = '{}.json'.format(item['id']) + sdclientV1.save_dashboard_to_file(item, file_name) + + ok, res = sdclient.create_dashboard_from_file( + u'import of {}'.format(item['name']), + file_name, + None, + shared=item['isShared'], + public=item['isPublic']) + + if ok: + sdclient.delete_dashboard(res['dashboard']) + else: + print('=====') + print(res) + + print('=====') From 92d4f052d7d14c8e326258bad6480a5dd2269a5b Mon Sep 17 00:00:00 2001 From: davideschiera Date: Tue, 2 Apr 2019 11:58:51 -0700 Subject: [PATCH 07/18] Fixes and support for scope migration --- sdcclient/_monitor.py | 386 ++++++++++++++++++++++-------------------- 1 file changed, 204 insertions(+), 182 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 55a2f056..3b6ad052 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -584,7 +584,10 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared scopeExpression = self._convert_scope_string_to_expression(scope) if scopeExpression[0] == False: return scopeExpression - template['scopeExpressionList'] = map(lambda ex: {'operand':ex['operand'], 'operator':ex['operator'],'value':ex['value'],'displayName':'', 'variable':False}, scopeExpression[1]) + if scopeExpression[1]: + template['scopeExpressionList'] = map(lambda ex: {'operand': ex['operand'], 'operator': ex['operator'], 'value': ex['value'], 'displayName': '', 'variable': False}, scopeExpression[1]) + else: + template['scopeExpressionList'] = None # NOTE: Individual panels might override the dashboard scope, the override will NOT be reset if 'widgets' in template and template['widgets'] is not None: @@ -611,7 +614,15 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared # Create the new dashboard # res = requests.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': template}), verify=self.ssl_verify) - return self._request_result(res) + + result = self._request_result(res) + + if result[0] == False: + print json.dumps({'dashboard': template}) + # else: + # print json.dumps({'dashboard': template}) + + return result def create_dashboard_from_view(self, newdashname, viewname, filter, shared=False, public=False, annotations={}): '''**Description** @@ -867,7 +878,7 @@ def _convert_scope_string_to_expression(self, scope): def _get_dashboard_converters(self): return { 'v2': { - 'v1': _convert_dashboard_v1_to_v2 + 'v1': self._convert_dashboard_v1_to_v2 } } @@ -881,203 +892,214 @@ def _convert_dashboard_to_current_version(self, dashboard, version): if converter == None: return False, 'dashboard version {} cannot be converted to {}'.format(version, self._dashboards_api_version) - return converter(dashboard) - - -def _convert_dashboard_v1_to_v2(dashboard): - # - # Migrations - # - # Each converter function will take: - # 1. name of the v1 dashboard property - # 2. v1 dashboard configuration - # 3. v2 dashboard configuration - # - # Each converter will apply changes to v2 dashboard configuration according to v1 - # - def with_default(converter, default=None): - def fn(prop_name, old_dashboard, new_dashboard): - if prop_name not in old_dashboard: - old_dashboard[prop_name] = default - - converter(prop_name, old_dashboard, new_dashboard) + try: + return converter(dashboard) + except Exception as err: + return False, str(err) + + def _convert_dashboard_v1_to_v2(self, dashboard): + # + # Migrations + # + # Each converter function will take: + # 1. name of the v1 dashboard property + # 2. v1 dashboard configuration + # 3. v2 dashboard configuration + # + # Each converter will apply changes to v2 dashboard configuration according to v1 + # + def with_default(converter, default=None): + def fn(prop_name, old_dashboard, new_dashboard): + if prop_name not in old_dashboard: + old_dashboard[prop_name] = default + + converter(prop_name, old_dashboard, new_dashboard) + + return fn + + def keep_as_is(prop_name, old_dashboard, new_dashboard): + new_dashboard[prop_name] = old_dashboard[prop_name] - return fn + def drop_it(prop_name = None, old_dashboard = None, new_dashboard = None): + pass - def keep_as_is(prop_name, old_dashboard, new_dashboard): - new_dashboard[prop_name] = old_dashboard[prop_name] - - def drop_it(prop_name = None, old_dashboard = None, new_dashboard = None): - pass - - def ignore(prop_name = None, old_dashboard = None, new_dashboard = None): - pass - - def rename_to(new_prop_name): - def rename(prop_name, old_dashboard, new_dashboard): - new_dashboard[new_prop_name] = old_dashboard[prop_name] - - return rename - - def convert_schema(prop_name, old_dashboard, new_dashboard): - new_dashboard[prop_name] = 2 - - def convert_scope(prop_name, old_dashboard, new_dashboard): - # # TODO! - - new_dashboard['scopeExpressionList'] = None - - def convert_events_filter(prop_name, old_dashboard, new_dashboard): - rename_to('eventsOverlaySettings')(prop_name, old_dashboard, new_dashboard) - - if 'showNotificationsDoNotFilterSameMetrics' in new_dashboard['eventsOverlaySettings']: - del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameMetrics'] - if 'showNotificationsDoNotFilterSameScope' in new_dashboard['eventsOverlaySettings']: - del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameScope'] - - def convert_items(prop_name, old_dashboard, new_dashboard): - def convert_color_coding(prop_name, old_widget, new_widget): - best_value = None - worst_value= None - for item in old_widget[prop_name]['thresholds']: - if item['color'] == 'best': - best_value = item['max'] if not item['max'] else item['min'] - elif item['color'] == 'worst': - worst_value = item['min'] if not item['min'] else item['max'] - - if best_value is not None and worst_value is not None: - new_widget[prop_name] = { - 'best': best_value, - 'worst': worst_value - } + def ignore(prop_name = None, old_dashboard = None, new_dashboard = None): + pass - def convert_display_options(prop_name, old_widget, new_widget): - keep_as_is(prop_name, old_widget, new_widget) + def rename_to(new_prop_name): + def rename(prop_name, old_dashboard, new_dashboard): + new_dashboard[new_prop_name] = old_dashboard[prop_name] - if 'yAxisScaleFactor' in new_widget[prop_name]: - del new_widget[prop_name]['yAxisScaleFactor'] + return rename - def convert_group(prop_name, old_widget, new_widget): - group_by_metrics = old_widget[prop_name]['configuration']['groups'][0]['groupBy'] - - migrated = [] - for metric in group_by_metrics: - migrated.append({ 'labelId': metric['metric'] }) - - new_widget['groupingLabelsIds'] = migrated + def convert_schema(prop_name, old_dashboard, new_dashboard): + new_dashboard[prop_name] = 2 - def convert_name(prop_name, old_widget, new_widget): - keep_as_is(prop_name, old_widget, new_widget) + def convert_scope(prop_name, old_dashboard, new_dashboard): + # # TODO! - unique_id = 1 - name = old_widget[prop_name] + scope = old_dashboard[prop_name] + scope_conversion = self._convert_scope_string_to_expression(scope) - for widget in old_dashboard['items']: - if widget == old_widget: - return + if scope_conversion[0]: + if scope_conversion[1]: + new_dashboard['scopeExpressionList'] = scope_conversion[1] + else: + # the property can be either `null` or a non-empty array + new_dashboard['scopeExpressionList'] = None + else: + raise SyntaxError('scope not supported by the current grammar') + + def convert_events_filter(prop_name, old_dashboard, new_dashboard): + rename_to('eventsOverlaySettings')(prop_name, old_dashboard, new_dashboard) + + if 'showNotificationsDoNotFilterSameMetrics' in new_dashboard['eventsOverlaySettings']: + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameMetrics'] + if 'showNotificationsDoNotFilterSameScope' in new_dashboard['eventsOverlaySettings']: + del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameScope'] + + def convert_items(prop_name, old_dashboard, new_dashboard): + def convert_color_coding(prop_name, old_widget, new_widget): + best_value = None + worst_value= None + for item in old_widget[prop_name]['thresholds']: + if item['color'] == 'best': + best_value = item['max'] if not item['max'] else item['min'] + elif item['color'] == 'worst': + worst_value = item['min'] if not item['min'] else item['max'] - if new_widget[prop_name] == widget[prop_name]: - new_widget[prop_name] = '{} ({})'.format(name, unique_id) - unique_id += 1 + if best_value is not None and worst_value is not None: + new_widget[prop_name] = { + 'best': best_value, + 'worst': worst_value + } + def convert_display_options(prop_name, old_widget, new_widget): + keep_as_is(prop_name, old_widget, new_widget) - def convert_metrics(prop_name, old_widget, new_widget): - def convert_property_name(prop_name, old_metric, new_metric): - keep_as_is(prop_name, old_metric, new_metric) - - if old_metric['metricId'] == 'timestamp': - return 'k0' - - metric_migrations = { - 'metricId': rename_to('id'), - 'aggregation': rename_to('timeAggregation'), - 'groupAggregation': rename_to('groupAggregation'), - 'propertyName': convert_property_name - } + if 'yAxisScaleFactor' in new_widget[prop_name]: + del new_widget[prop_name]['yAxisScaleFactor'] - migrated_metrics = [] - for old_metric in old_widget[prop_name]: - migrated_metric = {} - - for key in metric_migrations.keys(): - if key in old_metric: - metric_migrations[key](key, old_metric, migrated_metric) - - migrated_metrics.append(migrated_metric) - - new_widget['metrics'] = migrated_metrics - - widget_migrations = { - 'colorCoding': convert_color_coding, - 'compareToConfig': keep_as_is, - 'customDisplayOptions': convert_display_options, - 'gridConfiguration': keep_as_is, - 'group': convert_group, - 'hasTransparentBackground': rename_to('transparentBackground'), - 'limitToScope': keep_as_is, - 'isPanelTitleVisible': rename_to('panelTitleVisible'), - 'markdownSource': keep_as_is, - 'limitToScope': keep_as_is, - 'metrics': convert_metrics, - 'name': convert_name, - 'overrideFilter': rename_to('overrideScope'), - 'paging': drop_it, - - 'scope': drop_it, # TODO !!!! - - 'showAs': keep_as_is, - 'showAsType': drop_it, - 'sorting': drop_it, - 'textpanelTooltip': keep_as_is, - } + def convert_group(prop_name, old_widget, new_widget): + group_by_metrics = old_widget[prop_name]['configuration']['groups'][0]['groupBy'] + + migrated = [] + for metric in group_by_metrics: + migrated.append({ 'labelId': metric['metric'] }) + + new_widget['groupingLabelsIds'] = migrated + + def convert_name(prop_name, old_widget, new_widget): + keep_as_is(prop_name, old_widget, new_widget) + + unique_id = 1 + name = old_widget[prop_name] + + for widget in old_dashboard['items']: + if widget == old_widget: + return + + if new_widget[prop_name] == widget[prop_name]: + new_widget[prop_name] = '{} ({})'.format(name, unique_id) + unique_id += 1 + + + def convert_metrics(prop_name, old_widget, new_widget): + def convert_property_name(prop_name, old_metric, new_metric): + keep_as_is(prop_name, old_metric, new_metric) + + if old_metric['metricId'] == 'timestamp': + return 'k0' + + metric_migrations = { + 'metricId': rename_to('id'), + 'aggregation': rename_to('timeAggregation'), + 'groupAggregation': rename_to('groupAggregation'), + 'propertyName': convert_property_name + } - migrated_widgets = [] - for old_widget in old_dashboard[prop_name]: - migrated_widget = { - 'id': len(migrated_widgets) + 1 + migrated_metrics = [] + for old_metric in old_widget[prop_name]: + migrated_metric = {} + + for key in metric_migrations.keys(): + if key in old_metric: + metric_migrations[key](key, old_metric, migrated_metric) + + migrated_metrics.append(migrated_metric) + + new_widget['metrics'] = migrated_metrics + + widget_migrations = { + 'colorCoding': convert_color_coding, + 'compareToConfig': keep_as_is, + 'customDisplayOptions': convert_display_options, + 'gridConfiguration': keep_as_is, + 'group': convert_group, + 'hasTransparentBackground': rename_to('transparentBackground'), + 'limitToScope': keep_as_is, + 'isPanelTitleVisible': rename_to('panelTitleVisible'), + 'markdownSource': keep_as_is, + 'metrics': convert_metrics, + 'name': convert_name, + 'overrideFilter': rename_to('overrideScope'), + 'paging': drop_it, + + 'scope': keep_as_is, + + 'showAs': keep_as_is, + 'showAsType': drop_it, + 'sorting': drop_it, + 'textpanelTooltip': keep_as_is, } - for key in widget_migrations.keys(): - if key in old_widget: - widget_migrations[key](key, old_widget, migrated_widget) + migrated_widgets = [] + for old_widget in old_dashboard[prop_name]: + migrated_widget = { + 'id': len(migrated_widgets) + 1 + } + + for key in widget_migrations.keys(): + if key in old_widget: + widget_migrations[key](key, old_widget, migrated_widget) - migrated_widgets.append(migrated_widget) + migrated_widgets.append(migrated_widget) + + new_dashboard['widgets'] = migrated_widgets + + return migrated - new_dashboard['widgets'] = migrated_widgets - - return migrated - - migrations = { - 'autoCreated': keep_as_is, - 'createdOn': keep_as_is, - 'eventsFilter': with_default(convert_events_filter, { - 'filterNotificationsUserInputFilter': '' - }), - 'filterExpression': convert_scope, - 'scopeExpressionList': ignore, # scope will be generated from 'filterExpression' - 'id': keep_as_is, - 'isPublic': rename_to('public'), - 'isShared': rename_to('shared'), - 'items': convert_items, - 'layout': keep_as_is, - 'modifiedOn': keep_as_is, - 'name': keep_as_is, - 'publicToken': drop_it, - 'schema': convert_schema, - 'teamId': keep_as_is, - 'username': keep_as_is, - 'version': keep_as_is, - } - - # - # Apply migrations - # - migrated = {} - for key in migrations.keys(): - migrations[key](key, copy.deepcopy(dashboard), migrated) - - return True, migrated + migrations = { + 'autoCreated': keep_as_is, + 'createdOn': keep_as_is, + 'eventsFilter': with_default(convert_events_filter, { + 'filterNotificationsUserInputFilter': '' + }), + 'filterExpression': convert_scope, + 'scopeExpressionList': ignore, # scope will be generated from 'filterExpression' + 'id': keep_as_is, + 'isPublic': rename_to('public'), + 'isShared': rename_to('shared'), + 'items': convert_items, + 'layout': keep_as_is, + 'modifiedOn': keep_as_is, + 'name': keep_as_is, + 'publicToken': drop_it, + 'schema': convert_schema, + 'teamId': keep_as_is, + 'username': keep_as_is, + 'version': keep_as_is, + } + + # + # Apply migrations + # + migrated = {} + for key in migrations.keys(): + migrations[key](key, copy.deepcopy(dashboard), migrated) + + return True, migrated # For backwards compatibility From 49c4beef4de25640145ed21fbf3e77d3bb875e8c Mon Sep 17 00:00:00 2001 From: davideschiera Date: Tue, 2 Apr 2019 11:59:30 -0700 Subject: [PATCH 08/18] Temporary change --- sdcclient/_monitor.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 3b6ad052..08dfb302 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -615,14 +615,7 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared # res = requests.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': template}), verify=self.ssl_verify) - result = self._request_result(res) - - if result[0] == False: - print json.dumps({'dashboard': template}) - # else: - # print json.dumps({'dashboard': template}) - - return result + return self._request_result(res) def create_dashboard_from_view(self, newdashname, viewname, filter, shared=False, public=False, annotations={}): '''**Description** From aa6aeacf4d669ac728b49b0fa9011f7c056af6d6 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Tue, 2 Apr 2019 13:17:28 -0700 Subject: [PATCH 09/18] More fixes --- sdcclient/_monitor.py | 71 ++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 08dfb302..76ed8728 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -901,27 +901,34 @@ def _convert_dashboard_v1_to_v2(self, dashboard): # # Each converter will apply changes to v2 dashboard configuration according to v1 # + def when_set(converter): + def fn(prop_name, old_obj, new_obj): + if prop_name in old_obj and old_obj[prop_name] is not None: + converter(prop_name, old_obj, new_obj) + + return fn + def with_default(converter, default=None): - def fn(prop_name, old_dashboard, new_dashboard): - if prop_name not in old_dashboard: - old_dashboard[prop_name] = default + def fn(prop_name, old_obj, new_obj): + if prop_name not in old_obj: + old_obj[prop_name] = default - converter(prop_name, old_dashboard, new_dashboard) + converter(prop_name, old_obj, new_obj) return fn - def keep_as_is(prop_name, old_dashboard, new_dashboard): - new_dashboard[prop_name] = old_dashboard[prop_name] + def keep_as_is(prop_name, old_obj, new_obj): + new_obj[prop_name] = old_obj[prop_name] - def drop_it(prop_name = None, old_dashboard = None, new_dashboard = None): + def drop_it(prop_name = None, old_obj = None, new_obj = None): pass - def ignore(prop_name = None, old_dashboard = None, new_dashboard = None): + def ignore(prop_name = None, old_obj = None, new_obj = None): pass def rename_to(new_prop_name): - def rename(prop_name, old_dashboard, new_dashboard): - new_dashboard[new_prop_name] = old_dashboard[prop_name] + def rename(prop_name, old_obj, new_obj): + new_obj[new_prop_name] = old_obj[prop_name] return rename @@ -983,19 +990,21 @@ def convert_group(prop_name, old_widget, new_widget): new_widget['groupingLabelsIds'] = migrated def convert_name(prop_name, old_widget, new_widget): - keep_as_is(prop_name, old_widget, new_widget) - + # + # enforce unique name (on old dashboard, before migration) + # unique_id = 1 name = old_widget[prop_name] for widget in old_dashboard['items']: if widget == old_widget: - return + break - if new_widget[prop_name] == widget[prop_name]: - new_widget[prop_name] = '{} ({})'.format(name, unique_id) + if old_widget[prop_name] == widget[prop_name]: + old_widget[prop_name] = '{} ({})'.format(name, unique_id) unique_id += 1 + keep_as_is(prop_name, old_widget, new_widget) def convert_metrics(prop_name, old_widget, new_widget): def convert_property_name(prop_name, old_metric, new_metric): @@ -1024,26 +1033,26 @@ def convert_property_name(prop_name, old_metric, new_metric): new_widget['metrics'] = migrated_metrics widget_migrations = { - 'colorCoding': convert_color_coding, - 'compareToConfig': keep_as_is, - 'customDisplayOptions': convert_display_options, - 'gridConfiguration': keep_as_is, - 'group': convert_group, - 'hasTransparentBackground': rename_to('transparentBackground'), - 'limitToScope': keep_as_is, - 'isPanelTitleVisible': rename_to('panelTitleVisible'), - 'markdownSource': keep_as_is, - 'metrics': convert_metrics, - 'name': convert_name, + 'colorCoding': when_set(convert_color_coding), + 'compareToConfig': when_set(keep_as_is), + 'customDisplayOptions': with_default(convert_display_options, {}), + 'gridConfiguration': drop_it, + 'group': when_set(convert_group), + 'hasTransparentBackground': when_set(rename_to('transparentBackground')), + 'limitToScope': when_set(keep_as_is), + 'isPanelTitleVisible': when_set(rename_to('panelTitleVisible')), + 'markdownSource': when_set(keep_as_is), + 'metrics': with_default(convert_metrics, []), + 'name': with_default(convert_name, 'Panel'), 'overrideFilter': rename_to('overrideScope'), 'paging': drop_it, - 'scope': keep_as_is, + 'scope': with_default(keep_as_is, None), 'showAs': keep_as_is, 'showAsType': drop_it, 'sorting': drop_it, - 'textpanelTooltip': keep_as_is, + 'textpanelTooltip': when_set(keep_as_is), } migrated_widgets = [] @@ -1053,12 +1062,10 @@ def convert_property_name(prop_name, old_metric, new_metric): } for key in widget_migrations.keys(): - if key in old_widget: - widget_migrations[key](key, old_widget, migrated_widget) + widget_migrations[key](key, old_widget, migrated_widget) migrated_widgets.append(migrated_widget) - - + new_dashboard['widgets'] = migrated_widgets return migrated From 22f1baed7362cff98d38c5e1e447d7dbdffc56ec Mon Sep 17 00:00:00 2001 From: davideschiera Date: Tue, 2 Apr 2019 13:19:39 -0700 Subject: [PATCH 10/18] Comment --- sdcclient/_monitor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 76ed8728..b48ea56b 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -1046,9 +1046,7 @@ def convert_property_name(prop_name, old_metric, new_metric): 'name': with_default(convert_name, 'Panel'), 'overrideFilter': rename_to('overrideScope'), 'paging': drop_it, - 'scope': with_default(keep_as_is, None), - 'showAs': keep_as_is, 'showAsType': drop_it, 'sorting': drop_it, @@ -1058,6 +1056,7 @@ def convert_property_name(prop_name, old_metric, new_metric): migrated_widgets = [] for old_widget in old_dashboard[prop_name]: migrated_widget = { + # create unique ID 'id': len(migrated_widgets) + 1 } From fbaf5d94fe60e86c27fc54e0e7662739e7716f4e Mon Sep 17 00:00:00 2001 From: davideschiera Date: Mon, 8 Apr 2019 14:25:34 -0700 Subject: [PATCH 11/18] Remove panel ID --- sdcclient/_monitor.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index ea3f8c27..8986b761 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -1037,10 +1037,7 @@ def convert_property_name(prop_name, old_metric, new_metric): migrated_widgets = [] for old_widget in old_dashboard[prop_name]: - migrated_widget = { - # create unique ID - 'id': len(migrated_widgets) + 1 - } + migrated_widget = {} for key in widget_migrations.keys(): widget_migrations[key](key, old_widget, migrated_widget) From dc89e1da0871275251b61e46de5a6c9991813039 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Wed, 10 Apr 2019 14:33:22 -0400 Subject: [PATCH 12/18] Fix fn name --- sdcclient/_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 207d0c8c..7c414200 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -922,7 +922,7 @@ def convert_scope(prop_name, old_dashboard, new_dashboard): # # TODO! scope = old_dashboard[prop_name] - scope_conversion = self._convert_scope_string_to_expression(scope) + scope_conversion = self.convert_scope_string_to_expression(scope) if scope_conversion[0]: if scope_conversion[1]: From 8512fa6825e2872764cf6c66ace378a9db694ddd Mon Sep 17 00:00:00 2001 From: davideschiera Date: Wed, 10 Apr 2019 14:38:31 -0400 Subject: [PATCH 13/18] Improve script output --- examples/dashboard_backup_v1_restore_v2.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/dashboard_backup_v1_restore_v2.py b/examples/dashboard_backup_v1_restore_v2.py index 6e198299..5f75ba31 100644 --- a/examples/dashboard_backup_v1_restore_v2.py +++ b/examples/dashboard_backup_v1_restore_v2.py @@ -41,8 +41,11 @@ for item in res['dashboards']: file_name = '{}.json'.format(item['id']) + print('Saving v1 dashboard {} to file {}...'.format( + item['name'], file_name)) sdclientV1.save_dashboard_to_file(item, file_name) + print('Importing dashboard to v2...') ok, res = sdclient.create_dashboard_from_file( u'import of {}'.format(item['name']), file_name, @@ -51,9 +54,10 @@ public=item['isPublic']) if ok: + print('Dashboard {} imported!'.format(item['name'])) sdclient.delete_dashboard(res['dashboard']) else: - print('=====') + print('Dashboard {} import failed:'.format(item['name'])) print(res) - print('=====') + print('\n') From 8292983a401aa97a90a6b93cbed92e8662d9e258 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Wed, 10 Apr 2019 15:02:22 -0400 Subject: [PATCH 14/18] Improve script --- examples/dashboard_backup_v1_restore_v2.py | 40 ++++++++++++---------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/examples/dashboard_backup_v1_restore_v2.py b/examples/dashboard_backup_v1_restore_v2.py index 5f75ba31..6bb3e6b3 100644 --- a/examples/dashboard_backup_v1_restore_v2.py +++ b/examples/dashboard_backup_v1_restore_v2.py @@ -14,50 +14,54 @@ # # Parse arguments # -if len(sys.argv) != 2: - print('usage: %s ' % sys.argv[0]) +if len(sys.argv) != 5: + print( + 'usage: %s ' + % sys.argv[0]) print( 'You can find your token at https://app.sysdigcloud.com/#/settings/user' ) sys.exit(1) -sdc_token = sys.argv[1] +sdc_v1_url = sys.argv[1] +sdc_v1_token = sys.argv[2] +sdc_v2_url = sys.argv[3] +sdc_v2_token = sys.argv[4] # # Instantiate the SDC client # -sdclient = SdMonitorClient(sdc_token, sdc_url='https://app.sysdigcloud.com') -sdclientV1 = SdMonitorClientV1( - sdc_token, sdc_url='https://app.sysdigcloud.com') +sdclient_v2 = SdMonitorClient(sdc_v2_token, sdc_url=sdc_v2_url) +sdclient_v1 = SdMonitorClientV1(sdc_v1_token, sdc_url=sdc_v1_url) # # Serialize the first user dashboard to disk # -ok, res = sdclientV1.get_dashboards() +ok, res = sdclient_v1.get_dashboards() if not ok: print(res) sys.exit(1) -for item in res['dashboards']: - file_name = '{}.json'.format(item['id']) +for dashboard in res['dashboards']: + file_name = '{}.json'.format(dashboard['id']) print('Saving v1 dashboard {} to file {}...'.format( - item['name'], file_name)) - sdclientV1.save_dashboard_to_file(item, file_name) + dashboard['name'], file_name)) + sdclient_v1.save_dashboard_to_file(dashboard, file_name) print('Importing dashboard to v2...') - ok, res = sdclient.create_dashboard_from_file( - u'import of {}'.format(item['name']), + ok, res = sdclient_v2.create_dashboard_from_file( + u'import of {}'.format(dashboard['name']), file_name, None, - shared=item['isShared'], - public=item['isPublic']) + shared=dashboard['isShared'], + public=dashboard['isPublic']) if ok: - print('Dashboard {} imported!'.format(item['name'])) - sdclient.delete_dashboard(res['dashboard']) + print('Dashboard {} imported!'.format(dashboard['name'])) + sdclient_v2.delete_dashboard(res['dashboard']) else: - print('Dashboard {} import failed:'.format(item['name'])) + print('Dashboard {} import failed:'.format(dashboard['name'])) print(res) print('\n') From 5c2992489f01694a8526627d30599a61e3f94db9 Mon Sep 17 00:00:00 2001 From: Julio J Date: Thu, 11 Apr 2019 12:47:18 +0200 Subject: [PATCH 15/18] Keep as is the GridConfiguration for panels --- sdcclient/_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 7c414200..db198457 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -1019,7 +1019,7 @@ def convert_property_name(prop_name, old_metric, new_metric): 'colorCoding': when_set(convert_color_coding), 'compareToConfig': when_set(keep_as_is), 'customDisplayOptions': with_default(convert_display_options, {}), - 'gridConfiguration': drop_it, + 'gridConfiguration': keep_as_is, 'group': when_set(convert_group), 'hasTransparentBackground': when_set(rename_to('transparentBackground')), 'limitToScope': when_set(keep_as_is), From 9dd861b48797b5aa8836b186027da4f34dee0e31 Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 11 Apr 2019 16:58:33 -0400 Subject: [PATCH 16/18] Drop layout property --- sdcclient/_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index db198457..6c031a9d 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -1061,7 +1061,7 @@ def convert_property_name(prop_name, old_metric, new_metric): 'isPublic': rename_to('public'), 'isShared': rename_to('shared'), 'items': convert_items, - 'layout': keep_as_is, + 'layout': drop_it, 'modifiedOn': keep_as_is, 'name': keep_as_is, 'publicToken': drop_it, From 43c4dc8fdc64fdddd6e9c8b38d64c16a42d76bfa Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 11 Apr 2019 17:27:22 -0400 Subject: [PATCH 17/18] Print all errors, not just the first one --- sdcclient/_common.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/sdcclient/_common.py b/sdcclient/_common.py index a69b2830..af9f7f62 100644 --- a/sdcclient/_common.py +++ b/sdcclient/_common.py @@ -38,16 +38,18 @@ def _checkResponse(self, res): return False if 'errors' in j: - if 'message' in j['errors'][0]: - self.lasterr = j['errors'][0]['message'] + error_msgs = [] + for error in j['errors']: + error_msg = [] + if 'message' in error: + error_msg.append(error['message']) - if 'reason' in j['errors'][0]: - if self.lasterr is not None: - self.lasterr += ' ' - else: - self.lasrerr = '' + if 'reason' in error: + error_msg.append(error['reason']) - self.lasterr += j['errors'][0]['reason'] + error_msgs.append(': '.join(error_msg)) + + self.lasterr = '\n'.join(error_msgs) elif 'message' in j: self.lasterr = j['message'] else: From f98d049fe04977e7b4fce621ef2d2ae42c07383e Mon Sep 17 00:00:00 2001 From: davideschiera Date: Thu, 11 Apr 2019 17:55:09 -0400 Subject: [PATCH 18/18] Fix conversion of metrics and topology panels --- sdcclient/_monitor.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/sdcclient/_monitor.py b/sdcclient/_monitor.py index 6c031a9d..f3cba09f 100644 --- a/sdcclient/_monitor.py +++ b/sdcclient/_monitor.py @@ -972,6 +972,14 @@ def convert_group(prop_name, old_widget, new_widget): new_widget['groupingLabelsIds'] = migrated + def convert_override_filter(prop_name, old_widget, new_widget): + if old_widget['showAs'] == 'map': + # override scope always true if scope is set + new_widget['overrideScope'] = True + else: + new_widget['overrideScope'] = old_widget[prop_name] + + def convert_name(prop_name, old_widget, new_widget): # # enforce unique name (on old dashboard, before migration) @@ -1013,6 +1021,21 @@ def convert_property_name(prop_name, old_metric, new_metric): migrated_metrics.append(migrated_metric) + # Property name convention: + # timestamp: k0 (if present) + # other keys: k* (from 0 or 1, depending on timestamp) + # values: v* (from 0) + timestamp_key = filter(lambda m: m['id'] == 'timestamp' and not ('timeAggregation' in m and m['timeAggregation'] is not None), migrated_metrics) + no_timestamp_keys = filter(lambda m: m['id'] != 'timestamp' and not ('timeAggregation' in m and m['timeAggregation'] is not None), migrated_metrics) + values = filter(lambda m: 'timeAggregation' in m and m['timeAggregation'] is not None, migrated_metrics) + if timestamp_key: + timestamp_key[0]['propertyName'] = 'k0' + k_offset = 1 if timestamp_key else 0 + for i in range(0, len(no_timestamp_keys)): + no_timestamp_keys[i]['propertyName'] = 'k{}'.format(i + k_offset) + for i in range(0, len(values)): + values[i]['propertyName'] = 'v{}'.format(i) + new_widget['metrics'] = migrated_metrics widget_migrations = { @@ -1027,7 +1050,7 @@ def convert_property_name(prop_name, old_metric, new_metric): 'markdownSource': when_set(keep_as_is), 'metrics': with_default(convert_metrics, []), 'name': with_default(convert_name, 'Panel'), - 'overrideFilter': rename_to('overrideScope'), + 'overrideFilter': convert_override_filter, 'paging': drop_it, 'scope': with_default(keep_as_is, None), 'showAs': keep_as_is,