diff --git a/uvdat/asgi.py b/uvdat/asgi.py index e48ba3c3..e166ad2e 100644 --- a/uvdat/asgi.py +++ b/uvdat/asgi.py @@ -6,7 +6,7 @@ from django.core.asgi import get_asgi_application from django.urls import path -from uvdat.core.notifications import AnalyticsConsumer +from uvdat.core.notifications import AnalyticsConsumer, ConversionConsumer os.environ['DJANGO_SETTINGS_MODULE'] = 'uvdat.settings' if not os.environ.get('DJANGO_CONFIGURATION'): @@ -23,7 +23,12 @@ 'ws/analytics/project//results/', AnalyticsConsumer.as_asgi(), name='analytics-ws', - ) + ), + path( + 'ws/conversion/', + ConversionConsumer.as_asgi(), + name='conversion-ws', + ), ] ) ), diff --git a/uvdat/core/notifications.py b/uvdat/core/notifications.py index 00adc070..7cbaf3b3 100644 --- a/uvdat/core/notifications.py +++ b/uvdat/core/notifications.py @@ -1,4 +1,5 @@ from asgiref.sync import async_to_sync +from channels.exceptions import StopConsumer from channels.generic.websocket import JsonWebsocketConsumer @@ -11,6 +12,21 @@ def connect(self): def disconnect(self, code): async_to_sync(self.channel_layer.group_discard)(self.group_name, self.channel_name) + raise StopConsumer() + + def send_notification(self, event): + self.send_json(content=event['message']) + + +class ConversionConsumer(JsonWebsocketConsumer): + def connect(self): + self.group_name = 'conversion' + async_to_sync(self.channel_layer.group_add)(self.group_name, self.channel_name) + self.accept() + + def disconnect(self, code): + async_to_sync(self.channel_layer.group_discard)(self.group_name, self.channel_name) + raise StopConsumer() def send_notification(self, event): self.send_json(content=event['message']) diff --git a/uvdat/core/tasks/conversion.py b/uvdat/core/tasks/conversion.py index 71d0d531..7c2ba1ec 100644 --- a/uvdat/core/tasks/conversion.py +++ b/uvdat/core/tasks/conversion.py @@ -32,6 +32,11 @@ def get_cog_path(file): source = large_image.open(file) if source.geospatial: raster_path = file + metadata = source.getMetadata() + if len(metadata.get('frames', [])) > 1: + # If multiframe, return early; + # large_image_converter is not multiframe-compatible yet + return raster_path except large_image.exceptions.TileSourceError: pass diff --git a/uvdat/core/tasks/dataset.py b/uvdat/core/tasks/dataset.py index 163d776c..d7d276af 100644 --- a/uvdat/core/tasks/dataset.py +++ b/uvdat/core/tasks/dataset.py @@ -26,11 +26,11 @@ def create_layers_and_frames(dataset, layer_options=None): frames = [] kwargs = dict(dataset=dataset) data_name = layer_info.get('data') - source_file_name = layer_info.get('source_file') + source_file_names = layer_info.get('source_files') if data_name is not None: kwargs['name'] = data_name - if source_file_name is not None: - kwargs['source_file__name'] = source_file_name + if source_file_names is not None: + kwargs['source_file__name__in'] = source_file_names for layer_data in [ *VectorData.objects.filter(**kwargs).order_by('name').all(), @@ -40,6 +40,7 @@ def create_layers_and_frames(dataset, layer_options=None): additional_filters = layer_info.get('additional_filters', {}) metadata = layer_data.metadata or {} bands = metadata.get('bands') + raster_frames = metadata.get('frames') summary = layer_data.summary if hasattr(layer_data, 'summary') else {} properties = summary.get('properties') if properties and frame_property and frame_property in properties: @@ -54,7 +55,7 @@ def create_layers_and_frames(dataset, layer_options=None): index=len(frames), data=layer_data.name, source_filters=dict( - frame_property=value, **additional_filters + **{frame_property: value}, **additional_filters ), ) ) @@ -66,10 +67,23 @@ def create_layers_and_frames(dataset, layer_options=None): name=f'Frame {i}', index=len(frames), data=layer_data.name, - source_filters=dict(frame_property=i, **additional_filters), + source_filters=dict( + **{frame_property: i}, **additional_filters + ), ) ) - elif bands and len(bands) > 1: + elif frame_property == 'frame' and raster_frames and len(raster_frames) > 1: + for raster_frame in raster_frames: + i = raster_frame.get('Index') + frames.append( + dict( + name=i, + index=i, + data=layer_data.name, + source_filters=dict(frame=i), + ) + ) + elif frame_property == 'band' and bands and len(bands) > 1: for band in bands: frames.append( dict( diff --git a/web/components.d.ts b/web/components.d.ts index ab5a30fb..9dd65dfe 100644 --- a/web/components.d.ts +++ b/web/components.d.ts @@ -16,6 +16,7 @@ declare module 'vue' { DatasetList: typeof import('./src/components/DatasetList.vue')['default'] DatasetSelect: typeof import('./src/components/projects/DatasetSelect.vue')['default'] DatasetsPanel: typeof import('./src/components/sidebars/DatasetsPanel.vue')['default'] + DatasetUpload: typeof import('./src/components/projects/DatasetUpload.vue')['default'] DetailView: typeof import('./src/components/DetailView.vue')['default'] FloatingPanel: typeof import('./src/components/sidebars/FloatingPanel.vue')['default'] LayersPanel: typeof import('./src/components/sidebars/LayersPanel.vue')['default'] diff --git a/web/package-lock.json b/web/package-lock.json index 5246215b..3b93286b 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -1,12 +1,12 @@ { "name": "uvdat", - "version": "1.2.1", + "version": "1.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "uvdat", - "version": "1.2.1", + "version": "1.6.0", "dependencies": { "@mdi/font": "^7.2.96", "@resonant/oauth-client": "^0.9.0", @@ -16,6 +16,7 @@ "colormap": "^2.3.2", "core-js": "^3.8.3", "dayjs": "^1.11.13", + "django-s3-file-field": "^1.1.0", "git-describe": "^4.1.1", "html2canvas": "^1.4.1", "lodash": "^4.17.21", @@ -5137,6 +5138,18 @@ "node": ">=8" } }, + "node_modules/django-s3-file-field": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/django-s3-file-field/-/django-s3-file-field-1.1.0.tgz", + "integrity": "sha512-yksfBPSuwrFk5702r3ApqUkt/VUVF616mkoWMNDcPa5WnX1viQHNg+hG2+v8dB9OXFt/0IaF0lPBW+BcgNDNMg==", + "license": "Apache-2.0", + "dependencies": { + "axios": "^1.9.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", diff --git a/web/package.json b/web/package.json index 508521ff..8121d8dc 100644 --- a/web/package.json +++ b/web/package.json @@ -17,6 +17,7 @@ "colormap": "^2.3.2", "core-js": "^3.8.3", "dayjs": "^1.11.13", + "django-s3-file-field": "^1.1.0", "git-describe": "^4.1.1", "html2canvas": "^1.4.1", "lodash": "^4.17.21", diff --git a/web/src/App.vue b/web/src/App.vue index 98310172..e60974f3 100644 --- a/web/src/App.vue +++ b/web/src/App.vue @@ -5,10 +5,11 @@ import Map from "./components/map/Map.vue"; import SideBars from "./components/sidebars/SideBars.vue"; import ControlsBar from "./components/ControlsBar.vue"; -import { useAppStore, usePanelStore, useProjectStore } from "@/store"; +import { useAppStore, usePanelStore, useProjectStore, useConversionStore } from "@/store"; const appStore = useAppStore(); const panelStore = usePanelStore(); const projectStore = useProjectStore(); +const conversionStore = useConversionStore(); const showError = computed(() => appStore.currentError !== undefined); @@ -16,6 +17,7 @@ function onReady() { if (appStore.currentUser) { projectStore.clearState(); projectStore.loadProjects(); + conversionStore.createWebSocket(); } } diff --git a/web/src/api/auth.js b/web/src/api/auth.js index b93bf6e4..80a9f0d1 100644 --- a/web/src/api/auth.js +++ b/web/src/api/auth.js @@ -1,5 +1,6 @@ import axios from "axios"; import OauthClient from "@resonant/oauth-client"; +import S3FileFieldClient from 'django-s3-file-field'; import { useAppStore, useMapStore, useProjectStore } from "@/store"; export const baseURL = `${import.meta.env.VITE_APP_API_ROOT}api/v1/`; @@ -13,6 +14,12 @@ export const oauthClient = new OauthClient( { redirectUrl: window.location.origin } ); +let s3ffClient = undefined; + +export function getS3ffClient() { + return s3ffClient +} + export async function restoreLogin() { if (!oauthClient) { return; @@ -24,6 +31,12 @@ export async function restoreLogin() { useAppStore().currentUser = response.data; } }); + s3ffClient = new S3FileFieldClient({ + baseUrl: baseURL + 's3-upload/', + apiConfig: { + headers: oauthClient.authHeaders + } + }) } } diff --git a/web/src/api/rest.ts b/web/src/api/rest.ts index e9a63114..098bfdf5 100644 --- a/web/src/api/rest.ts +++ b/web/src/api/rest.ts @@ -1,4 +1,4 @@ -import { apiClient } from "./auth"; +import { apiClient, getS3ffClient } from "./auth"; import { User, Project, @@ -93,6 +93,14 @@ export async function getDataset(datasetId: number): Promise { return (await apiClient.get(`datasets/${datasetId}`)).data; } +export async function createDataset(data: any): Promise { + return (await apiClient.post('datasets/', data)).data; +} + +export async function spawnDatasetConversion(datasetId: number, options: any): Promise { + return (await apiClient.post(`datasets/${datasetId}/convert/`, options || {})).data; +} + export async function getDatasetLayers(datasetId: number): Promise { return (await apiClient.get(`datasets/${datasetId}/layers`)).data; } @@ -113,6 +121,17 @@ export async function getFileDataObjects(fileId: number): Promise<(RasterData | return (await apiClient.get(`files/${fileId}/data`)).data; } +export async function uploadFile(file:File): Promise { + const s3ffClient = getS3ffClient() + return await s3ffClient.uploadFile( + file, 'core.FileItem.file', + ) +} + +export async function createFileItem(data: any): Promise { + return (await apiClient.post('files/', data)).data; +} + export async function getDatasetNetworks(datasetId: number): Promise { return (await apiClient.get(`datasets/${datasetId}/networks`)).data; } diff --git a/web/src/components/DatasetList.vue b/web/src/components/DatasetList.vue index b3e32efc..911177f9 100644 --- a/web/src/components/DatasetList.vue +++ b/web/src/components/DatasetList.vue @@ -31,7 +31,7 @@ const datasetGroups = computed(() => { const expandedGroups = ref(); function expandAllGroups() { - if (!expandedGroups.value && filteredDatasets.value) { + if (filteredDatasets.value) { expandedGroups.value = Object.keys(datasetGroups.value) } } diff --git a/web/src/components/projects/DatasetSelect.vue b/web/src/components/projects/DatasetSelect.vue index 1ba6a071..971dd6fb 100644 --- a/web/src/components/projects/DatasetSelect.vue +++ b/web/src/components/projects/DatasetSelect.vue @@ -3,15 +3,18 @@ import { computed } from 'vue'; import DatasetList from '@/components/DatasetList.vue' import DetailView from '@/components/DetailView.vue' import { Dataset } from '@/types'; -import { useLayerStore } from '@/store'; +import { useLayerStore, useConversionStore } from '@/store'; const layerStore = useLayerStore(); +const conversionStore = useConversionStore(); const props = defineProps<{ datasets: Dataset[] | undefined; - selectedIds: number[] | undefined; + savingId: number | undefined; + addedIds?: number[] | undefined; + buttonIcon: string }>(); -const emit = defineEmits(["toggleDatasets"]); +const emit = defineEmits(["buttonClick"]); const datasetsWithLayers = computed(() => { return props.datasets?.map((dataset) => { @@ -21,10 +24,6 @@ const datasetsWithLayers = computed(() => { } }) }) - -function toggleSelected(items: Dataset[]) { - emit("toggleDatasets", items); -} + + diff --git a/web/src/components/projects/DatasetUpload.vue b/web/src/components/projects/DatasetUpload.vue new file mode 100644 index 00000000..6cdf17de --- /dev/null +++ b/web/src/components/projects/DatasetUpload.vue @@ -0,0 +1,393 @@ + + + + + diff --git a/web/src/components/projects/ProjectConfig.vue b/web/src/components/projects/ProjectConfig.vue index bdde60a1..d080a3b4 100644 --- a/web/src/components/projects/ProjectConfig.vue +++ b/web/src/components/projects/ProjectConfig.vue @@ -1,6 +1,7 @@