Allow dashboards created with API v1 to be uploaded to API v2

Merge from #86

Author: davideschiera

examples/dashboard_backup_v1_restore_v2.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+#
+# Save the first user dashboard to file and then use create_dashboard_from_file()
+# to apply the stored dasboard again with a different filter.
+#
+import os
+import sys
+import json
+sys.path.insert(
+    0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..'))
+from sdcclient import SdMonitorClient
+from sdcclient import SdMonitorClientV1
+
+#
+# Parse arguments
+#
+if len(sys.argv) != 5:
+    print(
+        'usage: %s <sysdig-v1-url> <sysdig-v1-token> <sysdig-v2-url> <sysdig-v2-token>'
+        % sys.argv[0])
+    print(
+        'You can find your token at https://app.sysdigcloud.com/#/settings/user'
+    )
+    sys.exit(1)
+
+sdc_v1_url = sys.argv[1]
+sdc_v1_token = sys.argv[2]
+sdc_v2_url = sys.argv[3]
+sdc_v2_token = sys.argv[4]
+
+#
+# Instantiate the SDC client
+#
+sdclient_v2 = SdMonitorClient(sdc_v2_token, sdc_url=sdc_v2_url)
+sdclient_v1 = SdMonitorClientV1(sdc_v1_token, sdc_url=sdc_v1_url)
+
+#
+# Serialize the first user dashboard to disk
+#
+ok, res = sdclient_v1.get_dashboards()
+
+if not ok:
+    print(res)
+    sys.exit(1)
+
+for dashboard in res['dashboards']:
+    file_name = '{}.json'.format(dashboard['id'])
+    print('Saving v1 dashboard {} to file {}...'.format(
+        dashboard['name'], file_name))
+    sdclient_v1.save_dashboard_to_file(dashboard, file_name)
+
+    print('Importing dashboard to v2...')
+    ok, res = sdclient_v2.create_dashboard_from_file(
+        u'import of {}'.format(dashboard['name']),
+        file_name,
+        None,
+        shared=dashboard['isShared'],
+        public=dashboard['isPublic'])
+
+    if ok:
+        print('Dashboard {} imported!'.format(dashboard['name']))
+        sdclient_v2.delete_dashboard(res['dashboard'])
+    else:
+        print('Dashboard {} import failed:'.format(dashboard['name']))
+        print(res)
+
+    print('\n')

sdcclient/_common.py

@@ -38,16 +38,18 @@ def _checkResponse(self, res):
                 return False
 
             if 'errors' in j:
-                if 'message' in j['errors'][0]:
-                    self.lasterr = j['errors'][0]['message']
+                error_msgs = []
+                for error in j['errors']:
+                    error_msg = []
+                    if 'message' in error:
+                        error_msg.append(error['message'])
 
-                if 'reason' in j['errors'][0]:
-                    if self.lasterr is not None:
-                        self.lasterr += ' '
-                    else:
-                        self.lasrerr = ''
+                    if 'reason' in error:
+                        error_msg.append(error['reason'])
 
-                    self.lasterr += j['errors'][0]['reason']
+                    error_msgs.append(': '.join(error_msg))
+
+                self.lasterr = '\n'.join(error_msgs)
             elif 'message' in j:
                 self.lasterr = j['message']
             else:

sdcclient/_monitor.py

@@ -576,7 +576,10 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared
         scopeExpression = self.convert_scope_string_to_expression(scope)
         if scopeExpression[0] == False:
             return scopeExpression
-        template['scopeExpressionList'] = map(lambda ex: {'operand':ex['operand'], 'operator':ex['operator'],'value':ex['value'],'displayName':'', 'variable':False}, scopeExpression[1])
+        if scopeExpression[1]:
+            template['scopeExpressionList'] = map(lambda ex: {'operand': ex['operand'], 'operator': ex['operator'], 'value': ex['value'], 'displayName': '', 'variable': False}, scopeExpression[1])
+        else:
+            template['scopeExpressionList'] = None
 
         # NOTE: Individual panels might override the dashboard scope, the override will NOT be reset
         if 'widgets' in template and template['widgets'] is not None:
@@ -596,6 +599,7 @@ def create_dashboard_from_template(self, dashboard_name, template, scope, shared
         # Create the new dashboard
         #
         res = requests.post(self.url + self._dashboards_api_endpoint, headers=self.hdrs, data=json.dumps({'dashboard': template}), verify=self.ssl_verify)
+            
         return self._request_result(res)
 
     def create_dashboard_from_view(self, newdashname, viewname, filter, shared=False, public=False):
@@ -718,6 +722,15 @@ def create_dashboard_from_file(self, dashboard_name, filename, filter, shared=Fa
 
         dashboard = loaded_object['dashboard']
 
+        if loaded_object['version'] != self._dashboards_api_version:
+            #
+            # Convert the dashboard (if possible)
+            #
+            conversion_result, dashboard = self._convert_dashboard_to_current_version(dashboard, loaded_object['version'])
+
+            if conversion_result == False:
+                return conversion_result, dashboard
+
         #
         # Create the new dashboard
         #
@@ -838,6 +851,258 @@ def convert_scope_string_to_expression(scope):
 
         return [True, expressions]
 
+    def _get_dashboard_converters(self):
+        return {
+            'v2': {
+                'v1': self._convert_dashboard_v1_to_v2
+            }
+        }
+
+    def _convert_dashboard_to_current_version(self, dashboard, version):
+        converters_to = self._get_dashboard_converters().get(self._dashboards_api_version, None)
+        if converters_to == None:
+            return False, 'unexpected error: no dashboard converters from version {} are supported'.format(self._dashboards_api_version)
+        
+        converter = converters_to.get(version, None)
+
+        if converter == None:
+            return False, 'dashboard version {} cannot be converted to {}'.format(version, self._dashboards_api_version)
+
+        try:
+            return converter(dashboard)
+        except Exception as err:
+            return False, str(err)
+
+    def _convert_dashboard_v1_to_v2(self, dashboard):
+        #
+        # Migrations
+        #
+        # Each converter function will take:
+        #   1. name of the v1 dashboard property
+        #   2. v1 dashboard configuration
+        #   3. v2 dashboard configuration
+        #
+        # Each converter will apply changes to v2 dashboard configuration according to v1
+        #
+        def when_set(converter):
+            def fn(prop_name, old_obj, new_obj):
+                if prop_name in old_obj and old_obj[prop_name] is not None:
+                    converter(prop_name, old_obj, new_obj)
+            
+            return fn
+
+        def with_default(converter, default=None):
+            def fn(prop_name, old_obj, new_obj):
+                if prop_name not in old_obj:
+                    old_obj[prop_name] = default
+
+                converter(prop_name, old_obj, new_obj)
+            
+            return fn
+            
+        def keep_as_is(prop_name, old_obj, new_obj):
+            new_obj[prop_name] = old_obj[prop_name]
+        
+        def drop_it(prop_name = None, old_obj = None, new_obj = None):
+            pass
+        
+        def ignore(prop_name = None, old_obj = None, new_obj = None):
+            pass
+
+        def rename_to(new_prop_name):
+            def rename(prop_name, old_obj, new_obj):
+                new_obj[new_prop_name] = old_obj[prop_name]
+
+            return rename
+
+        def convert_schema(prop_name, old_dashboard, new_dashboard):
+            new_dashboard[prop_name] = 2
+
+        def convert_scope(prop_name, old_dashboard, new_dashboard):
+            # # TODO!
+
+            scope = old_dashboard[prop_name]
+            scope_conversion = self.convert_scope_string_to_expression(scope)
+
+            if scope_conversion[0]:
+                if scope_conversion[1]:
+                    new_dashboard['scopeExpressionList'] = scope_conversion[1]
+                else:
+                    # the property can be either `null` or a non-empty array
+                    new_dashboard['scopeExpressionList'] = None
+            else:
+                raise SyntaxError('scope not supported by the current grammar')
+
+        def convert_events_filter(prop_name, old_dashboard, new_dashboard):
+            rename_to('eventsOverlaySettings')(prop_name, old_dashboard, new_dashboard)
+
+            if 'showNotificationsDoNotFilterSameMetrics' in new_dashboard['eventsOverlaySettings']:
+                del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameMetrics']
+            if 'showNotificationsDoNotFilterSameScope' in new_dashboard['eventsOverlaySettings']:
+                del new_dashboard['eventsOverlaySettings']['showNotificationsDoNotFilterSameScope']
+
+        def convert_items(prop_name, old_dashboard, new_dashboard):    
+            def convert_color_coding(prop_name, old_widget, new_widget):
+                best_value = None
+                worst_value= None
+                for item in old_widget[prop_name]['thresholds']:
+                    if item['color'] == 'best':
+                        best_value = item['max'] if not item['max'] else item['min']
+                    elif item['color'] == 'worst':
+                        worst_value = item['min'] if not item['min'] else item['max']
+                
+                if best_value is not None and worst_value is not None:
+                    new_widget[prop_name] = {
+                        'best': best_value,
+                        'worst': worst_value
+                    }
+
+            def convert_display_options(prop_name, old_widget, new_widget):
+                keep_as_is(prop_name, old_widget, new_widget)
+
+                if 'yAxisScaleFactor' in new_widget[prop_name]:
+                    del new_widget[prop_name]['yAxisScaleFactor']
+
+            def convert_group(prop_name, old_widget, new_widget):
+                group_by_metrics = old_widget[prop_name]['configuration']['groups'][0]['groupBy']
+                
+                migrated = []
+                for metric in group_by_metrics:
+                    migrated.append({ 'labelId': metric['metric'] })
+                
+                new_widget['groupingLabelsIds'] = migrated
+
+            def convert_override_filter(prop_name, old_widget, new_widget):
+                if old_widget['showAs'] == 'map':
+                    # override scope always true if scope is set
+                    new_widget['overrideScope'] = True
+                else:
+                    new_widget['overrideScope'] = old_widget[prop_name]
+
+
+            def convert_name(prop_name, old_widget, new_widget):
+                #
+                # enforce unique name (on old dashboard, before migration)
+                #
+                unique_id = 1
+                name = old_widget[prop_name]
+
+                for widget in old_dashboard['items']:
+                    if widget == old_widget:
+                        break
+                    
+                    if old_widget[prop_name] == widget[prop_name]:
+                        old_widget[prop_name] = '{} ({})'.format(name, unique_id)
+                        unique_id += 1
+
+                keep_as_is(prop_name, old_widget, new_widget)
+
+            def convert_metrics(prop_name, old_widget, new_widget):
+                def convert_property_name(prop_name, old_metric, new_metric):
+                    keep_as_is(prop_name, old_metric, new_metric)
+                    
+                    if old_metric['metricId'] == 'timestamp':
+                        return 'k0'
+
+                metric_migrations = {
+                    'metricId': rename_to('id'),
+                    'aggregation': rename_to('timeAggregation'),
+                    'groupAggregation': rename_to('groupAggregation'),
+                    'propertyName': convert_property_name
+                }
+
+                migrated_metrics = []
+                for old_metric in old_widget[prop_name]:
+                    migrated_metric = {}
+
+                    for key in metric_migrations.keys():
+                        if key in old_metric:
+                            metric_migrations[key](key, old_metric, migrated_metric)
+
+                    migrated_metrics.append(migrated_metric)
+
+                # Property name convention:
+                # timestamp: k0 (if present)
+                # other keys: k* (from 0 or 1, depending on timestamp)
+                # values: v* (from 0)
+                timestamp_key = filter(lambda m: m['id'] == 'timestamp' and not ('timeAggregation' in m and m['timeAggregation'] is not None), migrated_metrics)
+                no_timestamp_keys = filter(lambda m: m['id'] != 'timestamp' and not ('timeAggregation' in m and m['timeAggregation'] is not None), migrated_metrics)
+                values = filter(lambda m: 'timeAggregation' in m and m['timeAggregation'] is not None, migrated_metrics)
+                if timestamp_key:
+                    timestamp_key[0]['propertyName'] = 'k0'
+                k_offset = 1 if timestamp_key else 0
+                for i in range(0, len(no_timestamp_keys)):
+                    no_timestamp_keys[i]['propertyName'] = 'k{}'.format(i + k_offset)
+                for i in range(0, len(values)):
+                    values[i]['propertyName'] = 'v{}'.format(i)
+                    
+                new_widget['metrics'] = migrated_metrics
+
+            widget_migrations = {
+                'colorCoding': when_set(convert_color_coding),
+                'compareToConfig': when_set(keep_as_is),
+                'customDisplayOptions': with_default(convert_display_options, {}),
+                'gridConfiguration': keep_as_is,
+                'group': when_set(convert_group),
+                'hasTransparentBackground': when_set(rename_to('transparentBackground')),
+                'limitToScope': when_set(keep_as_is),
+                'isPanelTitleVisible': when_set(rename_to('panelTitleVisible')),
+                'markdownSource': when_set(keep_as_is),
+                'metrics': with_default(convert_metrics, []),
+                'name': with_default(convert_name, 'Panel'),
+                'overrideFilter': convert_override_filter,
+                'paging': drop_it,
+                'scope': with_default(keep_as_is, None),
+                'showAs': keep_as_is,
+                'showAsType': drop_it,
+                'sorting': drop_it,
+                'textpanelTooltip': when_set(keep_as_is),
+            }
+
+            migrated_widgets = []
+            for old_widget in old_dashboard[prop_name]:
+                migrated_widget = {}
+
+                for key in widget_migrations.keys():
+                    widget_migrations[key](key, old_widget, migrated_widget)
+
+                migrated_widgets.append(migrated_widget)
+       
+            new_dashboard['widgets'] = migrated_widgets
+            
+            return migrated
+        
+        migrations = {
+            'autoCreated': keep_as_is,
+            'createdOn': keep_as_is,
+            'eventsFilter': with_default(convert_events_filter, {
+                'filterNotificationsUserInputFilter': ''
+            }),
+            'filterExpression': convert_scope,
+            'scopeExpressionList': ignore, # scope will be generated from 'filterExpression'
+            'id': keep_as_is,
+            'isPublic': rename_to('public'),
+            'isShared': rename_to('shared'),
+            'items': convert_items,
+            'layout': drop_it,
+            'modifiedOn': keep_as_is,
+            'name': keep_as_is,
+            'publicToken': drop_it,
+            'schema': convert_schema,
+            'teamId': keep_as_is,
+            'username': keep_as_is,
+            'version': keep_as_is,
+        }
+
+        #
+        # Apply migrations
+        #
+        migrated = {}
+        for key in migrations.keys():
+            migrations[key](key, copy.deepcopy(dashboard), migrated)
+
+        return True, migrated
+
 
 # For backwards compatibility
 SdcClient = SdMonitorClient