Dataset Upload UI

uvdat/asgi.py

@@ -6,7 +6,7 @@
 from django.core.asgi import get_asgi_application
 from django.urls import path
 
-from uvdat.core.notifications import AnalyticsConsumer
+from uvdat.core.notifications import AnalyticsConsumer, ConversionConsumer
 
 os.environ['DJANGO_SETTINGS_MODULE'] = 'uvdat.settings'
 if not os.environ.get('DJANGO_CONFIGURATION'):
@@ -23,7 +23,12 @@
                         'ws/analytics/project/<int:project_id>/results/',
                         AnalyticsConsumer.as_asgi(),
                         name='analytics-ws',
-                    )
+                    ),
+                    path(
+                        'ws/conversion/',
+                        ConversionConsumer.as_asgi(),
+                        name='conversion-ws',
+                    ),
                 ]
             )
         ),

uvdat/core/notifications.py

@@ -1,4 +1,5 @@
 from asgiref.sync import async_to_sync
+from channels.exceptions import StopConsumer
 from channels.generic.websocket import JsonWebsocketConsumer
 
 
@@ -11,6 +12,21 @@ def connect(self):
 
     def disconnect(self, code):
         async_to_sync(self.channel_layer.group_discard)(self.group_name, self.channel_name)
+        raise StopConsumer()
+
+    def send_notification(self, event):
+        self.send_json(content=event['message'])
+
+
+class ConversionConsumer(JsonWebsocketConsumer):
+    def connect(self):
+        self.group_name = 'conversion'
+        async_to_sync(self.channel_layer.group_add)(self.group_name, self.channel_name)
+        self.accept()
+
+    def disconnect(self, code):
+        async_to_sync(self.channel_layer.group_discard)(self.group_name, self.channel_name)
+        raise StopConsumer()
 
     def send_notification(self, event):
         self.send_json(content=event['message'])

uvdat/core/tasks/conversion.py

@@ -32,6 +32,11 @@ def get_cog_path(file):
         source = large_image.open(file)
         if source.geospatial:
             raster_path = file
+            metadata = source.getMetadata()
+            if len(metadata.get('frames', [])) > 1:
+                # If multiframe, return early;
+                # large_image_converter is not multiframe-compatible yet
+                return raster_path
     except large_image.exceptions.TileSourceError:
         pass
 

uvdat/core/tasks/dataset.py

@@ -26,11 +26,11 @@ def create_layers_and_frames(dataset, layer_options=None):
             frames = []
             kwargs = dict(dataset=dataset)
             data_name = layer_info.get('data')
-            source_file_name = layer_info.get('source_file')
+            source_file_names = layer_info.get('source_files')
             if data_name is not None:
                 kwargs['name'] = data_name
-            if source_file_name is not None:
-                kwargs['source_file__name'] = source_file_name
+            if source_file_names is not None:
+                kwargs['source_file__name__in'] = source_file_names
 
             for layer_data in [
                 *VectorData.objects.filter(**kwargs).order_by('name').all(),
@@ -40,6 +40,7 @@ def create_layers_and_frames(dataset, layer_options=None):
                 additional_filters = layer_info.get('additional_filters', {})
                 metadata = layer_data.metadata or {}
                 bands = metadata.get('bands')
+                raster_frames = metadata.get('frames')
                 summary = layer_data.summary if hasattr(layer_data, 'summary') else {}
                 properties = summary.get('properties')
                 if properties and frame_property and frame_property in properties:
@@ -54,7 +55,7 @@ def create_layers_and_frames(dataset, layer_options=None):
                                         index=len(frames),
                                         data=layer_data.name,
                                         source_filters=dict(
-                                            frame_property=value, **additional_filters
+                                            **{frame_property: value}, **additional_filters
                                         ),
                                     )
                                 )
@@ -66,10 +67,23 @@ def create_layers_and_frames(dataset, layer_options=None):
                                         name=f'Frame {i}',
                                         index=len(frames),
                                         data=layer_data.name,
-                                        source_filters=dict(frame_property=i, **additional_filters),
+                                        source_filters=dict(
+                                            **{frame_property: i}, **additional_filters
+                                        ),
                                     )
                                 )
-                elif bands and len(bands) > 1:
+                elif frame_property == 'frame' and raster_frames and len(raster_frames) > 1:
+                    for raster_frame in raster_frames:
+                        i = raster_frame.get('Index')
+                        frames.append(
+                            dict(
+                                name=i,
+                                index=i,
+                                data=layer_data.name,
+                                source_filters=dict(frame=i),
+                            )
+                        )
+                elif frame_property == 'band' and bands and len(bands) > 1:
                     for band in bands:
                         frames.append(
                             dict(

web/components.d.ts

@@ -16,6 +16,7 @@ declare module 'vue' {
     DatasetList: typeof import('./src/components/DatasetList.vue')['default']
     DatasetSelect: typeof import('./src/components/projects/DatasetSelect.vue')['default']
     DatasetsPanel: typeof import('./src/components/sidebars/DatasetsPanel.vue')['default']
+    DatasetUpload: typeof import('./src/components/projects/DatasetUpload.vue')['default']
     DetailView: typeof import('./src/components/DetailView.vue')['default']
     FloatingPanel: typeof import('./src/components/sidebars/FloatingPanel.vue')['default']
     LayersPanel: typeof import('./src/components/sidebars/LayersPanel.vue')['default']

web/package.json

@@ -17,6 +17,7 @@
     "colormap": "^2.3.2",
     "core-js": "^3.8.3",
     "dayjs": "^1.11.13",
+    "django-s3-file-field": "^1.1.0",
     "git-describe": "^4.1.1",
     "html2canvas": "^1.4.1",
     "lodash": "^4.17.21",

web/src/App.vue

@@ -5,17 +5,19 @@ import Map from "./components/map/Map.vue";
 import SideBars from "./components/sidebars/SideBars.vue";
 import ControlsBar from "./components/ControlsBar.vue";
 
-import { useAppStore, usePanelStore, useProjectStore } from "@/store";
+import { useAppStore, usePanelStore, useProjectStore, useConversionStore } from "@/store";
 const appStore = useAppStore();
 const panelStore = usePanelStore();
 const projectStore = useProjectStore();
+const conversionStore = useConversionStore();
 
 const showError = computed(() => appStore.currentError !== undefined);
 
 function onReady() {
   if (appStore.currentUser) {
     projectStore.clearState();
     projectStore.loadProjects();
+    conversionStore.createWebSocket();
   }
 }
 

web/src/api/auth.js

@@ -1,5 +1,6 @@
 import axios from "axios";
 import OauthClient from "@resonant/oauth-client";
+import S3FileFieldClient from 'django-s3-file-field';
 import { useAppStore, useMapStore, useProjectStore } from "@/store";
 
 export const baseURL = `${import.meta.env.VITE_APP_API_ROOT}api/v1/`;
@@ -13,6 +14,12 @@ export const oauthClient = new OauthClient(
   { redirectUrl: window.location.origin }
 );
 
+let s3ffClient = undefined;
+
+export function getS3ffClient() {
+  return s3ffClient
+}
+
 export async function restoreLogin() {
   if (!oauthClient) {
     return;
@@ -24,6 +31,12 @@ export async function restoreLogin() {
         useAppStore().currentUser = response.data;
       }
     });
+    s3ffClient = new S3FileFieldClient({
+      baseUrl: baseURL + 's3-upload/',
+      apiConfig: {
+        headers: oauthClient.authHeaders
+      }
+    })
   }
 }
 

web/src/api/rest.ts

@@ -1,4 +1,4 @@
-import { apiClient } from "./auth";
+import { apiClient, getS3ffClient } from "./auth";
 import {
   User,
   Project,
@@ -93,6 +93,14 @@ export async function getDataset(datasetId: number): Promise<Dataset> {
   return (await apiClient.get(`datasets/${datasetId}`)).data;
 }
 
+export async function createDataset(data: any): Promise<Dataset> {
+  return (await apiClient.post('datasets/', data)).data;
+}
+
+export async function spawnDatasetConversion(datasetId: number, options: any): Promise<Dataset> {
+  return (await apiClient.post(`datasets/${datasetId}/convert/`, options || {})).data;
+}
+
 export async function getDatasetLayers(datasetId: number): Promise<Layer[]> {
   return (await apiClient.get(`datasets/${datasetId}/layers`)).data;
 }
@@ -113,6 +121,17 @@ export async function getFileDataObjects(fileId: number): Promise<(RasterData |
   return (await apiClient.get(`files/${fileId}/data`)).data;
 }
 
+export async function uploadFile(file:File): Promise<string> {
+  const s3ffClient = getS3ffClient()
+  return await s3ffClient.uploadFile(
+    file, 'core.FileItem.file',
+  )
+}
+
+export async function createFileItem(data: any): Promise<FileItem> {
+  return (await apiClient.post('files/', data)).data;
+}
+
 export async function getDatasetNetworks(datasetId: number): Promise<Network[]> {
   return (await apiClient.get(`datasets/${datasetId}/networks`)).data;
 }

web/src/components/DatasetList.vue

@@ -31,7 +31,7 @@ const datasetGroups = computed(() => {
 const expandedGroups = ref();
 
 function expandAllGroups() {
-  if (!expandedGroups.value && filteredDatasets.value) {
+  if (filteredDatasets.value) {
     expandedGroups.value = Object.keys(datasetGroups.value)
   }
 }