diff --git a/index.html b/index.html index a20467e4..5b3c4a49 100644 --- a/index.html +++ b/index.html @@ -27,7 +27,7 @@ - SimWrapper + AequilibraE Explore diff --git a/package.json b/package.json index 13133efb..ff65102f 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "blob-util": "^2.0.2", "buefy": "^0.9.29", "bulma": "^1.0.0", + "cartocolor": "^5.0.2", "color-string": "^1.9.1", "colormap": "^2.3.1", "comlink": "^4.4.2", @@ -119,6 +120,10 @@ "sax-wasm": "^2.2.4", "shallow-equal": "^1.2.1", "shapefile": "^0.6.6", + "spatialite": "^0.1.0", + "spl.js": "^0.1.2", + "sql.js": "^1.13.0", + "the-new-css-reset": "^1.7.3", "threads": "^1.7.0", "three": "^0.127.0", diff --git a/src/Globals.ts b/src/Globals.ts index dc9f52cf..8aee7f37 100644 --- a/src/Globals.ts +++ b/src/Globals.ts @@ -168,6 +168,7 @@ export interface FileSystemConfig { example?: boolean isGithub?: boolean isZIB?: boolean + isS3?: boolean // AWS S3 bucket with public access flask?: boolean // Flask filesystem supports OMX open matrix API - see https://github.com/simwrapper/omx-server } diff --git a/src/components/LegendColors.vue b/src/components/LegendColors.vue index 6f9cfc46..e733b4c2 100644 --- a/src/components/LegendColors.vue +++ b/src/components/LegendColors.vue @@ -1,11 +1,24 @@ @@ -32,6 +45,13 @@ export default defineComponent({ margin: 0; } +.legend-subtitle { + font-weight: bold; + margin-top: 0.5em; + margin-bottom: 0.25em; + display: block; +} + .item-label { margin: '0 0.5rem 0.0rem 0'; font-weight: 'bold'; diff --git a/src/dash-panels/_allPanels.ts b/src/dash-panels/_allPanels.ts index 0573d107..74391de7 100644 --- a/src/dash-panels/_allPanels.ts +++ b/src/dash-panels/_allPanels.ts @@ -28,6 +28,7 @@ export const panelLookup: { [key: string]: AsyncComponent } = { xml: defineAsyncComponent(() => import('./xml.vue')), // full-screen map visualizations: + aequilibrae: defineAsyncComponent(() => import('./aequilibrae.vue')), carriers: defineAsyncComponent(() => import('./carriers.vue')), flowmap: defineAsyncComponent(() => import('./flowmap.vue')), links: defineAsyncComponent(() => import('./links.vue')), diff --git a/src/dash-panels/aequilibrae.vue b/src/dash-panels/aequilibrae.vue new file mode 100644 index 00000000..be8054bf --- /dev/null +++ b/src/dash-panels/aequilibrae.vue @@ -0,0 +1,49 @@ + + + + + diff --git a/src/dash-panels/tile.vue b/src/dash-panels/tile.vue index e749104f..ed218b01 100644 --- a/src/dash-panels/tile.vue +++ b/src/dash-panels/tile.vue @@ -1,10 +1,14 @@ + + diff --git a/src/plugins/aequilibrae/useAequilibrae.ts b/src/plugins/aequilibrae/useAequilibrae.ts new file mode 100644 index 00000000..09c79c57 --- /dev/null +++ b/src/plugins/aequilibrae/useAequilibrae.ts @@ -0,0 +1,54 @@ +// Core utilities for AequilibraE plugin: YAML parsing and configuration handling + +import YAML from 'yaml' +import type { VizDetails, LayerConfig } from '../sqlite-map/types' +import { resolvePath, resolvePaths } from '../sqlite-map/utils' + +// Keep parseYamlConfig here as a convenience; feature builders moved to sqlite-map/feature-builder +export async function parseYamlConfig( + yamlText: string, + subfolder: string | null +): Promise { + const config = YAML.parse(yamlText) + const dbFile = config.database || config.file + if (!dbFile) throw new Error('No database field found in YAML config') + + const databasePath = resolvePath(dbFile, subfolder) + + // Process extraDatabases paths + let extraDatabases: Record | undefined + if (config.extraDatabases) { + extraDatabases = resolvePaths(config.extraDatabases, subfolder) + } + + return { + title: config.title || dbFile, + description: config.description || '', + database: databasePath, + extraDatabases, + view: config.view || '', + layers: config.layers || {}, + center: config.center, + zoom: config.zoom, + bearing: config.bearing, + pitch: config.pitch, + geometryLimit: config.geometryLimit, + coordinatePrecision: config.coordinatePrecision, + minimalProperties: config.minimalProperties, + defaults: config.defaults, + legend: config.legend, + } +} + +// Re-export moved builders from sqlite-map +export { buildTables, buildGeoFeatures } from '../sqlite-map/feature-builder' + +// Re-export database functions from sqlite-map for convenience +export { openDb, releaseDb, getCachedJoinData } from '../sqlite-map/db' +export { + initSql, + releaseSql, + acquireLoadingSlot, + mapLoadingComplete, + getTotalMapsLoading, +} from '../sqlite-map/loader' diff --git a/src/plugins/plotly/PlotlyDiagram.vue b/src/plugins/plotly/PlotlyDiagram.vue index 5f401671..d0f1ae16 100644 --- a/src/plugins/plotly/PlotlyDiagram.vue +++ b/src/plugins/plotly/PlotlyDiagram.vue @@ -196,6 +196,9 @@ const MyComponent = defineComponent({ // merge user-supplied layout with SimWrapper layout defaults if (this.vizDetails.layout) this.mergeLayouts() + // Apply top-level axis range settings (xMin/xMax/yMin/yMax) even without a layout property + this.applyAxisRangeSettings() + if (this.vizDetails.fixedRatio) { this.vizDetails.layout.xaxis = Object.assign(this.vizDetails.layout.xaxis, { constrain: 'domain', @@ -259,6 +262,16 @@ const MyComponent = defineComponent({ // Update the 'minXValue' if the minimum value in the 'x' array of the current trace is less than the current 'minXValue'. if (xAxisMin <= this.minXValue) this.minXValue = xAxisMin + + // optionally, if a max/min is set for the traces, collect the greatest maxes and least mins to build the layout ranges + if (this.traces[i].xaxis_max !== undefined && this.traces[i].xaxis_max > this.maxXValue) + this.maxXValue = this.traces[i].xaxis_max + if (this.traces[i].yaxis_max !== undefined && this.traces[i].yaxis_max > this.maxYValue) + this.maxYValue = this.traces[i].yaxis_max + if (this.traces[i].xaxis_min !== undefined && this.traces[i].xaxis_min < this.minXValue) + this.minXValue = this.traces[i].xaxis_min + if (this.traces[i].yaxis_min !== undefined && this.traces[i].yaxis_min < this.minYValue) + this.minYValue = this.traces[i].yaxis_min } // Set the x-axis and y-axis ranges in the layout based on the calculated 'minXValue', 'maxXValue', 'minYValue', and 'maxYValue'. @@ -283,6 +296,8 @@ const MyComponent = defineComponent({ // this.maxYValue + // ']' // ) + + }, changeDimensions(dim: any) { if (dim?.height && dim?.width) { @@ -308,20 +323,56 @@ const MyComponent = defineComponent({ delete mergedLayout.height delete mergedLayout.width + // Apply top-level xMin/xMax/yMin/yMax if provided + const { xMin, xMax, yMin, yMax } = this.vizDetails + const hasXRange = xMin !== undefined || xMax !== undefined + const hasYRange = yMin !== undefined || yMax !== undefined + // be selective about these: if (mergedLayout.xaxis) { mergedLayout.xaxis.automargin = true - mergedLayout.xaxis.autorange = true mergedLayout.xaxis.animate = true + // Only set autorange if no range is specified (via xMin/xMax or layout.xaxis.range) + const xRangeFromLayout = mergedLayout.xaxis.range && mergedLayout.xaxis.range.length === 2 + if (hasXRange) { + mergedLayout.xaxis.range = [ + xMin !== undefined ? xMin : null, + xMax !== undefined ? xMax : null + ] + mergedLayout.xaxis.autorange = false + } else if (xRangeFromLayout) { + mergedLayout.xaxis.autorange = false + } else { + mergedLayout.xaxis.autorange = true + } if (!mergedLayout.xaxis.title) mergedLayout.xaxis.title = this.layout.xaxis.title } else { - mergedLayout.xaxis = this.layout.xaxis + mergedLayout.xaxis = { ...this.layout.xaxis } + if (hasXRange) { + mergedLayout.xaxis.range = [ + xMin !== undefined ? xMin : null, + xMax !== undefined ? xMax : null + ] + mergedLayout.xaxis.autorange = false + } } if (mergedLayout.yaxis) { mergedLayout.yaxis.automargin = true - mergedLayout.yaxis.autorange = true mergedLayout.yaxis.animate = true + // Only set autorange if no range is specified (via yMin/yMax or layout.yaxis.range) + const yRangeFromLayout = mergedLayout.yaxis.range && mergedLayout.yaxis.range.length === 2 + if (hasYRange) { + mergedLayout.yaxis.range = [ + yMin !== undefined ? yMin : null, + yMax !== undefined ? yMax : null + ] + mergedLayout.yaxis.autorange = false + } else if (yRangeFromLayout) { + mergedLayout.yaxis.autorange = false + } else { + mergedLayout.yaxis.autorange = true + } // bug #357: scatterplots fail if rangemode is set if (!this.traces.find(a => a?.type == 'scatter')) { @@ -329,7 +380,14 @@ const MyComponent = defineComponent({ } if (!mergedLayout.yaxis.title) mergedLayout.yaxis.title = this.layout.yaxis.title } else { - mergedLayout.yaxis = this.layout.yaxis + mergedLayout.yaxis = { ...this.layout.yaxis } + if (hasYRange) { + mergedLayout.yaxis.range = [ + yMin !== undefined ? yMin : null, + yMax !== undefined ? yMax : null + ] + mergedLayout.yaxis.autorange = false + } } if (mergedLayout.yaxis2) { @@ -345,6 +403,32 @@ const MyComponent = defineComponent({ this.layout = mergedLayout }, + /** + * Apply top-level axis range settings (xMin, xMax, yMin, yMax) from vizDetails. + * This handles the case where user doesn't provide a layout property but still wants axis ranges. + */ + applyAxisRangeSettings() { + const { xMin, xMax, yMin, yMax } = this.vizDetails + const hasXRange = xMin !== undefined || xMax !== undefined + const hasYRange = yMin !== undefined || yMax !== undefined + + if (hasXRange) { + this.layout.xaxis.range = [ + xMin !== undefined ? xMin : null, + xMax !== undefined ? xMax : null + ] + this.layout.xaxis.autorange = false + } + + if (hasYRange) { + this.layout.yaxis.range = [ + yMin !== undefined ? yMin : null, + yMax !== undefined ? yMax : null + ] + this.layout.yaxis.autorange = false + } + }, + // This method checks if facet_col and/or facet_row are defined in the traces createFacets() { if (this.traces[0].facet_col == undefined && this.traces[0].facet_row == undefined) return diff --git a/src/plugins/pluginRegistry.ts b/src/plugins/pluginRegistry.ts index 742dc9c8..489b8668 100644 --- a/src/plugins/pluginRegistry.ts +++ b/src/plugins/pluginRegistry.ts @@ -143,6 +143,11 @@ const plugins = [ ], component: defineAsyncComponent(() => import('./logistics/LogisticsViewer.vue')), }, + { + kebabName: "aeq-reader", + filePatterns: ['**/aeqviz-*.y?(a)ml'], + component: defineAsyncComponent(() => import('./aequilibrae/AequilibraEReader.vue')), + }, ] export const pluginComponents: { [key: string]: AsyncComponent } = {} diff --git a/src/plugins/shape-file/DeckMapComponent.vue b/src/plugins/shape-file/DeckMapComponent.vue index 77728fdf..2486dd5c 100644 --- a/src/plugins/shape-file/DeckMapComponent.vue +++ b/src/plugins/shape-file/DeckMapComponent.vue @@ -48,6 +48,8 @@ export default defineComponent({ redraw: { type: Number, required: true }, screenshot: { type: Number, required: true }, viewId: { type: Number, required: true }, + lineWidthUnits: { type: String, required: false, default: 'pixels' }, + pointRadiusUnits: { type: String, required: false, default: 'pixels' }, }, data() { @@ -322,15 +324,15 @@ export default defineComponent({ this.highlightedLinkIndex == -1 ? null : this.highlightedLinkIndex, autoHighlight: true, highlightColor: [255, 255, 255, 160], - lineWidthUnits: 'pixels', + lineWidthUnits: this.lineWidthUnits, lineWidthScale: 1, lineWidthMinPixels: 0, // typeof lineWidths === 'number' ? 0 : 1, lineWidthMaxPixels: 50, getOffset: OFFSET_DIRECTION.RIGHT, opacity: this.opacity, pickable: true, - pointRadiusUnits: 'pixels', - pointRadiusMinPixels: 2, + pointRadiusUnits: this.pointRadiusUnits, + pointRadiusMinPixels: 0, // pointRadiusMaxPixels: 50, stroked: this.isStroked, // useDevicePixels: this.isTakingScreenshot, @@ -386,7 +388,7 @@ export default defineComponent({ this.highlightedLinkIndex == -1 ? null : this.highlightedLinkIndex, highlightColor: [255, 255, 255, 160], // [255, 0, 204, 255], opacity: 1, - widthUnits: 'pixels', + widthUnits: this.lineWidthUnits, widthMinPixels: 1, offsetDirection: OFFSET_DIRECTION.RIGHT, transitions: { diff --git a/src/plugins/sqlite-map/SqliteReader.vue b/src/plugins/sqlite-map/SqliteReader.vue new file mode 100644 index 00000000..f42c2de9 --- /dev/null +++ b/src/plugins/sqlite-map/SqliteReader.vue @@ -0,0 +1,304 @@ + + + + + diff --git a/src/plugins/sqlite-map/color-utils.ts b/src/plugins/sqlite-map/color-utils.ts new file mode 100644 index 00000000..83844d72 --- /dev/null +++ b/src/plugins/sqlite-map/color-utils.ts @@ -0,0 +1,99 @@ +// Color and encoder utilities extracted from styling.ts + +import * as cartoColors from 'cartocolor' +import type { RGBA, RGB } from './types' + +export const hexToRgbA = (hex: string, alpha: number = 1): RGBA => { + const bytes = hex.replace('#', '').match(/.{1,2}/g) || ['80', '80', '80'] + return [ + parseInt(bytes[0], 16), + parseInt(bytes[1], 16), + parseInt(bytes[2], 16), + Math.round(alpha * 255), + ] +} + +export const hexToRgb = (hex: string): RGB => { + const rgba = hexToRgbA(hex, 1) + return [rgba[0], rgba[1], rgba[2]] +} + +export const hexToRgba = (hex: string, alpha: number = 1): RGBA => hexToRgbA(hex, alpha) + +export const getPaletteColors = (name: string, numColors: number): string[] => { + const palette = (cartoColors as any)[name || 'YlGn'] + if (!palette) return Array(numColors).fill('#808080') + const sizes = Object.keys(palette) + .map(Number) + .filter(n => n > 0) + .sort((a, b) => a - b) + const size = sizes.find(s => s >= numColors) || sizes[sizes.length - 1] + return palette[size] || Array(numColors).fill('#808080') +} + +// Small numeric helpers used by encoders +export const toNumber = (value: any): number | null => { + const num = Number(value) + return isNaN(num) ? null : num +} + +const safeMin = (arr: number[]): number => { + if (arr.length === 0) return 0 + let min = arr[0] + for (let i = 1; i < arr.length; i++) { + if (arr[i] < min) min = arr[i] + } + return min +} + +const safeMax = (arr: number[]): number => { + if (arr.length === 0) return 1 + let max = arr[0] + for (let i = 1; i < arr.length; i++) { + if (arr[i] > max) max = arr[i] + } + return max +} + +export const buildColorEncoder = ( + values: any[], + style: any, + dataRange: [number, number] | null = null +) => { + if (dataRange) { + values = values.map(v => { + const num = toNumber(v) + if (num === null) return null + return Math.max(dataRange[0], Math.min(dataRange[1], num)) + }) + } + + const nums = values.map(toNumber).filter((v): v is number => v !== null) + const s = style || {} + const [min, max] = s.range ? s.range : [safeMin(nums), safeMax(nums)] + + const paletteName = s.palette || 'YlGn' + const numColors = s.numColors || 7 + const colors = getPaletteColors(paletteName, numColors).map((h: string) => hexToRgba(h, 1)) + + const scale = max === min ? 0 : (numColors - 1) / (max - min) + + return (value: any) => { + const num = toNumber(value) ?? min + const idx = Math.round((num - min) * scale) + return colors[Math.max(0, Math.min(numColors - 1, idx))] + } +} + +export const buildCategoryEncoder = ( + colors: Record, + defaultColor: string = '#808080' +) => { + const colorMap = new Map() + for (const [key, hex] of Object.entries(colors)) { + colorMap.set(String(key), hexToRgbA(hex, 1)) + } + const defaultRgba = hexToRgbA(defaultColor, 1) + + return (value: any) => colorMap.get(String(value)) || defaultRgba +} diff --git a/src/plugins/sqlite-map/db.ts b/src/plugins/sqlite-map/db.ts new file mode 100644 index 00000000..9a855a14 --- /dev/null +++ b/src/plugins/sqlite-map/db.ts @@ -0,0 +1,622 @@ +/** + * Database utilities for AequilibraE plugin + * + * This module provides helper functions for interacting with SQLite databases + * containing spatial data and results. It includes functions for schema inspection, + * data querying, joining operations, and memory-optimized GeoJSON extraction. + * + * @fileoverview Database Utility Functions for AequilibraE + * @author SimWrapper Development Team + */ + +import proj4 from 'proj4' +import type { JoinConfig, GeoFeature, SqliteDb, SPL } from './types' +import { + ESSENTIAL_SPATIAL_COLUMNS, + isGeometryColumn, + getUsedColumns, + simplifyCoordinates, + createJoinCacheKey, +} from './utils' + +/** + * Retrieves all table names from a SQLite database + * + * @param db - SQLite database connection object + * @returns Promise Array of table names + */ +export async function getTableNames(db: SqliteDb): Promise { + const result = await db.exec("SELECT name FROM sqlite_master WHERE type='table';").get.objs + return result.map((row: any) => row.name) +} + +/** + * Gets the schema (column information) for a specific table + * + * @param db - SQLite database connection object + * @param tableName - Name of the table to inspect + * @returns Promise with array of column metadata (name, type, nullable) + */ +export async function getTableSchema( + db: SqliteDb, + tableName: string +): Promise<{ name: string; type: string; nullable: boolean }[]> { + const result = await db.exec(`PRAGMA table_info("${tableName}");`).get.objs + return result.map((row: any) => ({ + name: row.name, + type: row.type, + nullable: row.notnull === 0, + })) +} + +/** + * Counts the number of rows in a table + * + * @param db - SQLite database connection object + * @param tableName - Name of the table to count + * @returns Promise Number of rows in the table + */ +export async function getRowCount(db: SqliteDb, tableName: string): Promise { + const result = await db.exec(`SELECT COUNT(*) as count FROM "${tableName}";`).get.objs + return result.length > 0 ? result[0].count : 0 +} + +/** + * Query a table and return all rows as objects + * + * @param db - SQLite database connection object + * @param tableName - Name of the table to query + * @param columns - Optional array of column names to select (default: all columns) + * @param whereClause - Optional WHERE clause to filter results (without the WHERE keyword) + * @returns Promise[]> Array of row objects + */ +export async function queryTable( + db: SqliteDb, + tableName: string, + columns?: string[], + whereClause?: string +): Promise[]> { + const columnList = columns ? columns.map(c => `"${c}"`).join(', ') : '*' + const whereCondition = whereClause ? ` WHERE ${whereClause}` : '' + const query = `SELECT ${columnList} FROM "${tableName}"${whereCondition};` + const result = await db.exec(query).get.objs + return result +} + +/** + * Perform an in-memory join between main data and external data + * @param mainData - Array of records from the main table + * @param joinData - Array of records from the join table + * @param joinConfig - Configuration specifying join keys and type + * @returns Merged array with joined columns added to main records + */ +export function performJoin( + mainData: Record[], + joinData: Record[], + joinConfig: JoinConfig +): Record[] { + const joinLookup = new Map>() + for (const row of joinData) { + const key = row[joinConfig.rightKey] + if (key !== undefined && key !== null) { + joinLookup.set(key, row) + } + } + + return mainData + .map(mainRow => { + const joinRow = joinLookup.get(mainRow[joinConfig.leftKey]) + + if (!joinRow && joinConfig.type !== 'left') { + return null // Skip non-matching rows in inner join + } + + if (!joinRow) { + return mainRow // Left join: keep main row without joined data + } + + // Extract only the columns specified, or all columns + const joinedColumns = joinConfig.columns?.length + ? Object.fromEntries( + joinConfig.columns + .map(col => [col, joinRow[col]]) + .filter(([_, value]) => value !== undefined) + ) + : { ...joinRow } + + return { ...mainRow, ...joinedColumns } + }) + .filter((row): row is Record => row !== null) +} + +// Cache for joinData +const joinDataCache: Map>> = new Map() + +export function clearJoinCache(key?: string) { + if (key) { + joinDataCache.delete(key) + } else { + joinDataCache.clear() + } +} + +export function getJoinCacheStats() { + return { + keys: Array.from(joinDataCache.keys()), + size: joinDataCache.size, + } +} + +/** + * Get cached joinData or load it if not cached + * Accepts an optional neededColumn to reduce memory by querying fewer columns. + */ +export async function getCachedJoinData( + db: SqliteDb, + joinConfig: JoinConfig, + neededColumn?: string +): Promise>> { + const cacheKey = createJoinCacheKey( + joinConfig.database, + joinConfig.table, + neededColumn || undefined, + joinConfig.filter + ) + + if (joinDataCache.has(cacheKey)) { + return joinDataCache.get(cacheKey)! + } + + // Determine which columns to query - minimize memory usage + let columnsToQuery: string[] | undefined + if (neededColumn) { + columnsToQuery = [joinConfig.rightKey] + if (neededColumn !== joinConfig.rightKey) { + columnsToQuery.push(neededColumn) + } + } else if (joinConfig.columns && joinConfig.columns.length > 0) { + const colSet = new Set(joinConfig.columns) + colSet.add(joinConfig.rightKey) + columnsToQuery = Array.from(colSet) + } else { + columnsToQuery = undefined + } + + const joinRows = await queryTable(db, joinConfig.table, columnsToQuery, joinConfig.filter) + const joinData = new Map(joinRows.map(row => [row[joinConfig.rightKey], row])) + joinDataCache.set(cacheKey, joinData) + + return joinData +} + +// Helper: build properties object from a database row +function buildPropertiesFromRow( + selectedColumns: any[], + row: any, + layerName: string +): Record { + const properties: Record = { _layer: layerName } + for (const col of selectedColumns) { + const key = col.name + if (key !== 'geojson_geom' && key !== 'geom_type' && row[key] != null) { + properties[key] = row[key] + } + } + return properties +} + +// Helper: parse GeoJSON string/object and simplify coordinates if needed +function parseAndSimplifyGeometry(geojson: any, coordPrecision: number): any | null { + try { + const geometry = typeof geojson === 'string' ? JSON.parse(geojson) : geojson + if (geometry && geometry.coordinates && coordPrecision < 15) { + geometry.coordinates = simplifyCoordinates(geometry.coordinates, coordPrecision) + } + return geometry + } catch (e) { + return null + } +} + +/** + * Fetch GeoJSON features from a table, optionally merging joined data + * Memory-optimized: only stores essential properties and simplifies coordinates + * @param db - The main database connection + * @param table - Table metadata with name and columns + * @param layerName - Name of the layer for feature properties + * @param layerConfig - Layer configuration + * @param joinedData - Optional pre-joined data to merge into features (keyed by join column) + * @param joinConfig - Optional join configuration specifying the key column + * @param options - Optional settings for memory optimization + */ +export async function fetchGeoJSONFeatures( + db: SqliteDb, + table: { name: string; columns: any[] }, + layerName: string, + layerConfig: any, + // Backwards-compatible parameters: callers for POLARIS pass (db, table, layerName, layerConfig, options, filterConfigs) + // AequilibraE callers pass (db, table, layerName, layerConfig, joinedData?, joinConfig?, options?) + joinedDataOrOptions?: Map> | { limit?: number; coordinatePrecision?: number; minimalProperties?: boolean }, + joinConfigOrFilter?: JoinConfig | any, + optionsMaybe?: { + limit?: number + coordinatePrecision?: number + minimalProperties?: boolean + } +): Promise { + // Normalize parameters for both calling conventions + let joinedData: Map> | undefined + let joinConfig: JoinConfig | undefined + let options = optionsMaybe as { limit?: number; coordinatePrecision?: number; minimalProperties?: boolean } | undefined + + // If the 5th arg looks like an options object (has limit or coordinatePrecision), treat it as POLARIS-style call + if ( + joinedDataOrOptions && + typeof joinedDataOrOptions === 'object' && + !(joinedDataOrOptions instanceof Map) && + (('limit' in (joinedDataOrOptions as any)) || ('coordinatePrecision' in (joinedDataOrOptions as any)) || ('minimalProperties' in (joinedDataOrOptions as any))) + ) { + options = joinedDataOrOptions as any + // joinConfigOrFilter in this mode is polaris' filterConfigs; we ignore it here + joinedData = undefined + joinConfig = undefined + } else { + joinedData = joinedDataOrOptions as Map> | undefined + joinConfig = (joinConfigOrFilter as JoinConfig) ?? undefined + options = options ?? optionsMaybe + } + + const limit = options?.limit ?? 1000000 // Default limit + const coordPrecision = options?.coordinatePrecision ?? 5 + const minimalProps = options?.minimalProperties ?? true + + // Resolve joined data early so we can reuse the cached Map instead of rebuilding arrays per row + const cachedJoinedData = joinedData ?? (joinConfig ? await getCachedJoinData(db, joinConfig) : undefined) + + // Determine which columns we actually need + const usedColumns = getUsedColumns(layerConfig) + // Always include the join key if we have a join + if (joinConfig) { + usedColumns.add(joinConfig.leftKey) + } + + // Build column list - either all columns or just the ones we need + let columnNames: string + if (minimalProps && usedColumns.size > 0) { + // Only select columns that are used for styling + a few essential ones + const colsToSelect = table.columns + .filter((c: any) => { + const name = c.name.toLowerCase() + return ( + !isGeometryColumn(name) && + (usedColumns.has(c.name) || ESSENTIAL_SPATIAL_COLUMNS.has(name)) + ) + }) + .map((c: any) => `"${c.name}"`) + + // If no specific columns identified, fall back to all + columnNames = + colsToSelect.length > 0 + ? colsToSelect.join(', ') + : table.columns + .filter((c: any) => !isGeometryColumn(c.name)) + .map((c: any) => `"${c.name}"`) + .join(', ') + } else { + columnNames = table.columns + .filter((c: any) => !isGeometryColumn(c.name)) + .map((c: any) => `"${c.name}"`) + .join(', ') + } + + // Determine geometry column name for this table (support 'geometry', 'geo', or other names detected by isGeometryColumn) + function findGeometryColumn(cols: any[]): string { + for (const c of cols) { + const n = String(c.name).toLowerCase() + if (isGeometryColumn(n) || n === 'geo' || n === 'geometry') return c.name + } + // fallback to 'geometry' + return 'geometry' + } + + const geomCol = findGeometryColumn(table.columns) + + // Optionally allow a custom SQL filter from the YAML config for this layer + // e.g. layerConfig.sqlFilter = "link_type != 'centroid'" + let filterClause = `${geomCol} IS NOT NULL` + if ( + layerConfig && + typeof layerConfig.sqlFilter === 'string' && + layerConfig.sqlFilter.trim().length > 0 + ) { + const sqlFilter = layerConfig.sqlFilter.trim() + // Basic safety check: disallow obvious injection patterns and dangerous statements + const unsafePattern = + /;|--|\b(ALTER|DROP|INSERT|UPDATE|DELETE|REPLACE|ATTACH|DETACH|VACUUM|PRAGMA)\b/i + if (unsafePattern.test(sqlFilter)) { + throw new Error('Invalid sqlFilter in layer configuration') + } + filterClause += ` AND (${sqlFilter})` + } + + // Escape table name to prevent breaking out of the quoted identifier + const safeTableName = String(table.name).replace(/"/g, '""') + // Ensure LIMIT is a safe positive integer + const numericLimit = Number(limit) + const safeLimit = + Number.isFinite(numericLimit) && numericLimit > 0 ? Math.floor(numericLimit) : 1000 + + const safeGeomCol = String(geomCol).replace(/"/g, '""') + + const query = ` + SELECT ${columnNames}, + AsGeoJSON("${safeGeomCol}") as geojson_geom, + GeometryType("${safeGeomCol}") as geom_type + FROM "${safeTableName}" + WHERE ${filterClause} + LIMIT ${safeLimit}; + ` + + // Execute query and get rows + const queryResult = await db.exec(query) + let rows = await queryResult.get.objs + + // Pre-allocate features array - will trim at end + const features: GeoFeature[] = [] + + const joinType = joinConfig?.type || 'left' + + // Get the list of columns we actually selected + const selectedColumns = + minimalProps && usedColumns.size > 0 + ? table.columns.filter((c: any) => { + const name = c.name.toLowerCase() + return ( + !isGeometryColumn(name) && + (usedColumns.has(c.name) || ESSENTIAL_SPATIAL_COLUMNS.has(name)) + ) + }) + : table.columns.filter((c: any) => !isGeometryColumn(c.name)) + + // Projection lookup for this table (if spatial metadata exists) + const tableProjection = await getProjectionForTable(db, table.name) + + // Process rows in small batches to allow GC to run + const BATCH_SIZE = 5000 + + for (let batchStart = 0; batchStart < rows.length; batchStart += BATCH_SIZE) { + const batchEnd = Math.min(batchStart + BATCH_SIZE, rows.length) + + for (let r = batchStart; r < batchEnd; r++) { + const row = rows[r] + if (!row.geojson_geom) { + // Clear the row reference to help GC + rows[r] = null + continue + } + + // Build minimal properties object and merge joined data if present + const properties = buildPropertiesFromRow(selectedColumns, row, layerName) + + if (cachedJoinedData && joinConfig) { + const joinRow = cachedJoinedData.get(row[joinConfig.leftKey]) + if (joinRow) { + for (const [key, value] of Object.entries(joinRow)) { + if (!(key in properties)) properties[key] = value + else if (key !== joinConfig.rightKey) properties[`${joinConfig.table}_${key}`] = value + } + } else if (joinType === 'inner') { + continue + } + } + + // Parse geometry string/object, apply projection if needed, and simplify + let geometry = parseAndSimplifyGeometry(row.geojson_geom, coordPrecision) + if (!geometry) { + rows[r] = null + continue + } + + if (geometry.coordinates && tableProjection.transform) { + geometry.coordinates = transformCoordinates(geometry.coordinates, tableProjection.transform) + } + + features.push({ type: 'Feature', geometry, properties }) + + // Clear the row reference to allow GC to reclaim memory + rows[r] = null + } + + // Yield to allow GC between batches + if (batchEnd < rows.length) { + await new Promise(resolve => setTimeout(resolve, 0)) + } + } + + // Clear the rows array reference entirely to help GC + rows.length = 0 + rows = null as any // Remove reference so original array can be GC'd + + return features +} + +// --- Projection helpers (from polaris/db.ts) -------------------------------- +type ProjectionInfo = { + srid: number | null + transform: ((xy: [number, number]) => [number, number]) | null +} + +const tableProjectionCache = new Map() + +async function lookupTableSrid(db: any, tableName: string): Promise { + try { + const rows = await db.exec( + `SELECT srid FROM geometry_columns WHERE lower(f_table_name) = lower('${tableName}') LIMIT 1;` + ).get.objs + return rows?.[0]?.srid ?? null + } catch (err) { + return null + } +} + +async function lookupProjectionDefinition(db: any, srid: number): Promise { + try { + const rows = await db.exec( + `SELECT proj4text, srtext FROM spatial_ref_sys WHERE srid = ${srid} LIMIT 1;` + ).get.objs + if (!rows || rows.length === 0) return null + return rows[0].proj4text || rows[0].srtext || null + } catch (err) { + return null + } +} + +function transformCoordinates( + coords: any, + transform: (xy: [number, number]) => [number, number] +): any { + if (!coords || !Array.isArray(coords)) return coords + if (coords.length > 0 && typeof coords[0] === 'number') { + const [x, y, ...rest] = coords + const [tx, ty] = transform([x as number, y as number]) + return [tx, ty, ...rest] + } + return coords.map((c: any) => transformCoordinates(c, transform)) +} + +async function getProjectionForTable(db: any, tableName: string): Promise { + const cached = tableProjectionCache.get(tableName) + if (cached) return cached + + const srid = await lookupTableSrid(db, tableName) + if (!srid || srid === 4326) { + const info = { srid: srid ?? null, transform: null } + tableProjectionCache.set(tableName, info) + return info + } + + const definition = await lookupProjectionDefinition(db, srid) + if (!definition) { + const info = { srid, transform: null } + tableProjectionCache.set(tableName, info) + return info + } + + let transform: ProjectionInfo['transform'] = null + try { + const sourceProj = proj4(definition) + const destProj = proj4.WGS84 + transform = (xy: [number, number]) => proj4(sourceProj, destProj, xy as any) as [number, number] + } catch (err) { + transform = null + } + + const info = { srid, transform } + tableProjectionCache.set(tableName, info) + return info +} + +interface CachedDb { + db: any + refCount: number + path: string +} + +const dbCache = new Map() +const dbLoadPromises = new Map>() + +/** + * Open a database, using cache if available. + * Multiple maps using the same database file will share one instance. + */ +export async function openDb(spl: SPL, arrayBuffer: ArrayBuffer, path?: string): Promise { + // If no path provided, can't cache - just open directly + if (!path) { + return spl.db(arrayBuffer) + } + + // Check cache first + const cached = dbCache.get(path) + if (cached) { + cached.refCount++ + return cached.db + } + + // Check if already loading - wait for it and increment refCount + const loadingPromise = dbLoadPromises.get(path) + if (loadingPromise) { + const db = await loadingPromise + // Increment refCount for this caller + const nowCached = dbCache.get(path) + if (nowCached) { + nowCached.refCount++ + } + return db + } + + // Load and cache + const loadPromise = (async () => { + const db = await spl.db(arrayBuffer) + dbCache.set(path, { db, refCount: 1, path }) + dbLoadPromises.delete(path) + return db + })() + + dbLoadPromises.set(path, loadPromise) + return loadPromise +} + +/** + * Release a reference to a cached database. + * The database is closed when refCount reaches 0. + */ +export function releaseDb(path: string): void { + const cached = dbCache.get(path) + if (!cached) return + + cached.refCount-- + if (cached.refCount <= 0) { + try { + if (typeof cached.db.close === 'function') { + cached.db.close() + } + } catch (e) { + console.warn(`Failed to close database ${path}:`, e) + } + dbCache.delete(path) + } +} + +/** + * Force close all cached databases and clear file cache. Use with caution - only call when + * you're sure no maps are using any databases. + */ +export function clearAllDbCaches(): void { + for (const [path, cached] of dbCache) { + try { + if (cached.db && typeof cached.db.close === 'function') { + cached.db.close() + } + } catch (e) { + console.warn(`Failed to close database ${path}:`, e) + } + } + dbCache.clear() + dbLoadPromises.clear() + + // also clear the join data cache, if it exists + if (typeof joinDataCache !== 'undefined') { + joinDataCache.clear() + } +} + +// Backwards-compatible alias for clearing all caches +export const clearAllCaches = clearAllDbCaches + +export function getDbCacheStats() { + const entries = Array.from(dbCache.entries()).map(([path, v]) => ({ path, refCount: v.refCount })) + return { + size: dbCache.size, + entries, + } +} diff --git a/src/plugins/sqlite-map/feature-builder.ts b/src/plugins/sqlite-map/feature-builder.ts new file mode 100644 index 00000000..44cdbbb8 --- /dev/null +++ b/src/plugins/sqlite-map/feature-builder.ts @@ -0,0 +1,124 @@ +// Feature builder: buildTables and buildGeoFeatures moved into sqlite-map + +import type { SqliteDb } from './types' +import { + getTableNames, + getTableSchema, + getRowCount, + fetchGeoJSONFeatures, + getCachedJoinData, +} from './db' +import { hasGeometryColumn, getNeededJoinColumn } from './utils' +import type { LayerConfig } from './types' + +export async function buildTables( + db: SqliteDb, + layerConfigs: { [k: string]: LayerConfig }, + allNames?: string[] +) { + const names = allNames ?? (await getTableNames(db)) + const select = Object.keys(layerConfigs).length + ? [...new Set(Object.values(layerConfigs).map(c => c.table))] + : ['nodes', 'links', 'zones'] + + const tables: Array<{ name: string; type: string; rowCount: number; columns: any[] }> = [] + let hasGeometry = false + + for (const name of names) { + if (!select.includes(name)) continue + const schema = await getTableSchema(db, name) + const rowCount = await getRowCount(db, name) + const hasGeomCol = hasGeometryColumn(schema) + if (hasGeomCol) hasGeometry = true + tables.push({ name, type: 'table', rowCount, columns: schema }) + } + return { tables, hasGeometry } +} + +export interface GeoFeatureOptions { + limit?: number + coordinatePrecision?: number + minimalProperties?: boolean +} + +export type LazyDbLoader = (dbName: string) => Promise + +export async function buildGeoFeatures( + db: SqliteDb, + tables: Array<{ name: string; type: string; rowCount: number; columns: any[] }>, + layerConfigs: { [k: string]: LayerConfig }, + lazyDbLoader?: LazyDbLoader, + options?: GeoFeatureOptions +) { + // Use static imports for join helpers (avoid dynamic imports) + + const plain = Object.assign({}, layerConfigs) + const layersToProcess = Object.keys(plain).length + ? Object.entries(plain) + : tables + .filter(t => hasGeometryColumn(t.columns)) + .map(t => [t.name, { table: t.name, type: 'line' as const }]) + + const features: any[] = [] + const loadedExtraDbs = new Map() + + try { + for (const [layerName, cfg] of layersToProcess as any) { + const layerConfig = cfg as LayerConfig + const tableName = layerConfig.table || layerName + const table = tables.find(t => t.name === tableName) + if (!table) continue + if (!hasGeometryColumn(table.columns)) continue + + let joinedData: Map> | undefined + + if (layerConfig.join && lazyDbLoader) { + const neededColumn = getNeededJoinColumn(layerConfig) + + try { + let extraDb = loadedExtraDbs.get(layerConfig.join.database) + if (!extraDb) { + const maybeDb = await lazyDbLoader(layerConfig.join.database) + if (maybeDb) { + extraDb = maybeDb + loadedExtraDbs.set(layerConfig.join.database, maybeDb) + } + } + + if (extraDb) { + joinedData = await getCachedJoinData(extraDb, layerConfig.join, neededColumn) + } else { + console.warn( + `⚠️ Extra database '${layerConfig.join.database}' not found for layer '${layerName}'` + ) + } + } catch (e) { + console.warn( + `⚠️ Failed to load join data from ${layerConfig.join.database}.${layerConfig.join.table}:`, + e + ) + } + } + + const layerFeatures = await fetchGeoJSONFeatures( + db, + table, + layerName, + cfg, + joinedData, + layerConfig.join, + options + ) + + for (let i = 0; i < layerFeatures.length; i++) { + features.push(layerFeatures[i]) + } + + await new Promise(resolve => setTimeout(resolve, 50)) + } + } finally { + loadedExtraDbs.clear() + } + + return features +} diff --git a/src/plugins/sqlite-map/helpers.ts b/src/plugins/sqlite-map/helpers.ts new file mode 100644 index 00000000..9e3b0ebd --- /dev/null +++ b/src/plugins/sqlite-map/helpers.ts @@ -0,0 +1,134 @@ +/** + * Helper utilities for sqlite-map plugin + * + * Generic utilities for styling, database loading, and memory management + * that can be used by any visualization using sqlite-map. + */ + +import { markRaw } from 'vue' +import { buildStyleArrays } from './styling' +import type { SqliteDb, VizDetails } from './types' + +/** + * Apply pre-computed styles to a Vue component instance. + * Marks feature data as raw to prevent Vue reactivity overhead. + * + * @param vm - Vue component instance to update + * @param features - GeoJSON features array + * @param vizDetails - Visualization configuration + * @param layerConfigs - Layer style configurations + */ +export function applyStylesToVm( + vm: any, + features: any[], + vizDetails: VizDetails, + layerConfigs: any +) { + vm.geoJsonFeatures = markRaw(features) + const styles = buildStyleArrays({ + features: vm.geoJsonFeatures, + layers: layerConfigs, + defaults: vizDetails.defaults, + }) + Object.assign(vm, { + fillColors: styles.fillColors, + lineColors: styles.lineColors, + lineWidths: styles.lineWidths, + pointRadii: styles.pointRadii, + fillHeights: styles.fillHeights, + featureFilter: styles.featureFilter, + isRGBA: true, + redrawCounter: (vm.redrawCounter ?? 0) + 1, + }) +} + +/** + * Release the main database connection and clear related data. + * + * @param vm - Vue component instance to update + */ +export function releaseMainDbFromVm(vm: any) { + vm.db = null + vm.tables = [] +} + +/** + * Load a database from file. + * Abstracts away the file I/O details. + * + * @param spl - SPL engine instance + * @param fileApi - File system API with getFileBlob method + * @param openDb - Function to open database from buffer + * @param path - Path to database file + * @returns Opened database connection + */ +export async function loadDbWithCache( + spl: any, + fileApi: any, + openDb: (spl: any, b: ArrayBuffer, p?: string) => Promise, + path: string +) { + const blob = await fileApi.getFileBlob(path) + const arrayBuffer = await blob.arrayBuffer() + return await openDb(spl, arrayBuffer, path) +} + +/** + * Create a lazy database loader function for extra databases. + * Returns a function that can be passed to buildGeoFeatures. + * + * @param spl - SPL engine instance + * @param fileApi - File system API + * @param openDb - Database open function + * @param extraDbPaths - Map of database names to file paths + * @param onLoadingText - Optional callback to update loading message + * @returns Lazy loader function for use with buildGeoFeatures + */ +export function createLazyDbLoader( + spl: any, + fileApi: any, + openDb: (spl: any, b: ArrayBuffer, p?: string) => Promise, + extraDbPaths: Record, + onLoadingText?: (msg: string) => void +) { + return async (dbName: string) => { + const path = extraDbPaths[dbName] + if (!path) return null + + try { + if (onLoadingText) { + onLoadingText(`Loading ${dbName} database...`) + } + + const blob = await fileApi.getFileBlob(path) + const arrayBuffer = await blob.arrayBuffer() + return await openDb(spl, arrayBuffer, path) + } catch (error) { + console.warn(`Failed to load extra database '${dbName}':`, error) + return null + } + } +} + +/** + * Calculate memory limits based on number of concurrent maps loading. + * Used to auto-tune geometry extraction for better memory efficiency. + * + * @param totalMaps - Total number of maps currently loading + * @returns Object with autoLimit (max features) and autoPrecision (coordinate precision) + */ +export function getMemoryLimits(totalMaps: number): { autoLimit: number; autoPrecision: number } { + let autoLimit = 100000, + autoPrecision = 5 + if (totalMaps >= 8) { + autoLimit = 25000 + autoPrecision = 4 + } else if (totalMaps >= 5) { + autoLimit = 40000 + autoPrecision = 4 + } else if (totalMaps >= 3) { + autoLimit = 60000 + autoPrecision = 5 + } + return { autoLimit, autoPrecision } +} diff --git a/src/plugins/sqlite-map/loader.ts b/src/plugins/sqlite-map/loader.ts new file mode 100644 index 00000000..da3ef9ac --- /dev/null +++ b/src/plugins/sqlite-map/loader.ts @@ -0,0 +1,102 @@ +/** + * Global Loading Queue & SPL Engine Management + * + * Ensures only one map loads at a time to prevent memory exhaustion when + * loading many maps simultaneously. Also manages the shared SPL (SpatiaLite) + * engine instance, which is ~100MB+ in memory. + */ + +import SPL from 'spl.js' +import type { SPL as SPLType } from './types' + +// ============================================================================ +// GLOBAL LOADING QUEUE - Ensures only one map loads at a time +// ============================================================================ + +let loadingQueue: Promise = Promise.resolve() +let queueLength = 0 +let totalMapsLoading = 0 // Track total maps being loaded (not just waiting) + +/** + * Acquire a slot in the loading queue. Only one map can load at a time. + * Returns a release function that MUST be called when loading is complete. + */ +export function acquireLoadingSlot(): Promise<() => void> { + queueLength++ + totalMapsLoading++ + + let releaseSlot: () => void + + const myTurn = loadingQueue.then() + + // Create the next slot in the queue + loadingQueue = new Promise(resolve => { + releaseSlot = () => { + queueLength-- + resolve() + } + }) + + return myTurn.then(() => releaseSlot!) +} + +/** + * Call when a map has fully finished loading (after extractGeometries) + * to update the total count for memory tuning purposes. + */ +export function mapLoadingComplete(): void { + totalMapsLoading = Math.max(0, totalMapsLoading - 1) +} + +/** + * Get the current total number of maps being loaded. + * This can be used to adjust memory limits dynamically. + */ +export function getTotalMapsLoading(): number { + return totalMapsLoading +} + +// ============================================================================ +// SHARED SPL ENGINE - Critical for memory when loading multiple maps +// ============================================================================ + +let sharedSpl: SPLType | null = null +let splInitPromise: Promise | null = null +let splRefCount = 0 + +/** + * Get or create the shared SPL engine. + * Uses reference counting to know when it's safe to clean up. + */ +export async function initSql(): Promise { + splRefCount++ + + if (sharedSpl) { + return sharedSpl + } + + // If already initializing, wait for that to complete + if (splInitPromise) { + return splInitPromise + } + + // Initialize the shared SPL engine + splInitPromise = SPL().then((spl: SPLType) => { + sharedSpl = spl + splInitPromise = null + return spl + }) + + return splInitPromise! +} + +/** + * Release a reference to the shared SPL engine. + * Call this when a map component is unmounted. + */ +export function releaseSql(): void { + splRefCount = Math.max(0, splRefCount - 1) + // We keep the SPL engine alive even when refCount hits 0 + // because it's expensive to reinitialize. It will be GC'd + // when the page is unloaded. +} diff --git a/src/plugins/sqlite-map/reader.scss b/src/plugins/sqlite-map/reader.scss new file mode 100644 index 00000000..8bde224f --- /dev/null +++ b/src/plugins/sqlite-map/reader.scss @@ -0,0 +1,99 @@ +/** + * Styles shared for sqlite-backed readers (AequilibraE, Polaris, etc.) + * Centralized so all sqlite-map based readers have consistent appearance. + */ + +/* Main container styles */ +.c-aequilibrae-viewer { + position: absolute; + top: 0; + bottom: 0; + left: 0; + right: 0; + background-color: var(--bgCardFrame); + display: flex; + flex-direction: column; +} + +/* Map container */ +.map-viewer { + position: relative; + flex: 1; + width: 100%; + height: 100%; +} + +/* Loading state indicator */ +.loading { + padding: 2rem; + text-align: center; + font-size: 1.2rem; + color: var(--textFancy); + + /* Add subtle animation for loading state */ + &::after { + content: '...'; + display: inline-block; + animation: loading-dots 1.5s steps(4, end) infinite; + } +} + +@keyframes loading-dots { + 0%, + 20% { + content: ''; + } + 40% { + content: '.'; + } + 60% { + content: '..'; + } + 80%, + 100% { + content: '...'; + } +} + +/* Database content display styles */ +.database-content { + padding: 0.5rem 0; + + h3 { + margin-bottom: 1rem; + color: var(--textFancy); + font-size: 1.25rem; + font-weight: 600; + } + + h4 { + margin-top: 1.5rem; + margin-bottom: 0.5rem; + color: var(--link); + font-weight: bold; + font-size: 1.1rem; + } + + p { + margin: 0.25rem 0; + color: var(--text); + line-height: 1.4; + } + + .columns { + margin-left: 1rem; + font-size: 0.9rem; + color: var(--textFancy); + + div { + margin: 0.1rem 0; + padding: 0.1rem 0; + + &:hover { + background-color: var(--bgHover); + padding-left: 0.25rem; + border-left: 2px solid var(--link); + } + } + } +} diff --git a/src/plugins/sqlite-map/styling.ts b/src/plugins/sqlite-map/styling.ts new file mode 100644 index 00000000..a6311035 --- /dev/null +++ b/src/plugins/sqlite-map/styling.ts @@ -0,0 +1,340 @@ +import { RGBA, RGB, ColorStyle, LayerStyle, BuildArgs, BuildResult } from './types' +import { + hexToRgbA, + hexToRgb, + hexToRgba, + getPaletteColors, + buildColorEncoder, + buildCategoryEncoder, + toNumber, +} from './color-utils' + +// Local helper types +type ColumnProperties = Record +type PropertiesArray = ColumnProperties[] + +/** + * Safely converts a value to number, returning null for invalid values + * + * @param value - Value to convert to number + * @returns Number or null if conversion fails + */ + +const applyQuantitativeMapping = ( + values: (number | null)[], + dataRange: [number, number], + outputRange: [number, number], + target: Float32Array, + stride: number, + offset: number +) => { + const [dataMin, dataMax] = dataRange + const [outMin, outMax] = outputRange + const scale = dataMax === dataMin ? 0 : 1 / (dataMax - dataMin) + + for (let i = 0; i < values.length; i++) { + const num = values[i] + const normalized = num === null ? 0 : (num - dataMin) * scale + target[i * stride + offset] = Math.max( + outMin, + Math.min(outMax, normalized * (outMax - outMin) + outMin) + ) + } +} + +// Helper to write either a scalar or an array value into a typed array +function writeValue( + target: ArrayLike, + baseIndex: number, + stride: number, + offset: number, + value: number | number[] +) { + if (Array.isArray(value)) { + for (let k = 0; k < value.length; k++) { + ;(target as any)[baseIndex + stride * 0 + offset + k] = value[k] + } + } else { + ;(target as any)[baseIndex + offset] = value + } +} + +const applyStaticValue = ( + indices: number[], + value: T extends Uint8ClampedArray ? RGBA | RGB : number, + target: T, + stride: number = 1, + offset: number = 0 +) => { + for (let j = 0; j < indices.length; j++) { + const base = indices[j] * stride + writeValue(target, base, 1, offset, value as any) + } +} + +const applyEncodedValue = ( + indices: number[], + properties: any[], + column: string, + encoder: (value: any) => T extends Uint8ClampedArray ? RGBA | RGB : number, + target: T, + stride: number = 1, + offset: number = 0 +) => { + for (let j = 0; j < indices.length; j++) { + const base = indices[j] * stride + const encoded = encoder(properties[j]?.[column]) as any + writeValue(target, base, 1, offset, encoded) + } +} + +const applyStaticNumeric = (indices: number[], value: number, target: Float32Array) => { + applyStaticValue(indices, value as any, target as any, 1, 0) +} + +/** + * Apply numeric values from an array to features at given indices + */ +// Unified numeric writer: writes a scalar or array value into a Float32Array +const writeNumeric = ( + target: Float32Array, + indices: number[], + getValue: (i: number) => number | undefined, + defaultValue: number +) => { + for (let j = 0; j < indices.length; j++) { + const value = getValue(j) + target[indices[j]] = value ?? defaultValue + } +} + +const applyNumericArray = ( + indices: number[], + values: (number | undefined)[], + target: Float32Array, + defaultValue: number +) => { + writeNumeric(target, indices, i => values[i], defaultValue) +} + +/** + * Apply category-based numeric mapping + */ +const applyNumericCategory = ( + indices: number[], + properties: any[], + column: string, + mapping: Record, + target: Float32Array, + defaultValue: number +) => { + writeNumeric(target, indices, i => mapping[properties[i]?.[column]], defaultValue) +} + +/** + * Apply feature filter based on include/exclude lists + */ +const applyFeatureFilter = ( + indices: number[], + properties: any[], + column: string, + include: any[] | undefined, + exclude: any[] | undefined, + target: Float32Array +) => { + for (let j = 0; j < indices.length; j++) { + const i = indices[j] + const v = properties[j]?.[column] + let visible = true + if (include && include.length) visible = include.includes(v) + if (exclude && exclude.length) visible = visible && !exclude.includes(v) + target[i] = visible ? 1 : 0 + } +} + +/** + * Builds typed arrays for efficient WebGL rendering from feature data and styling rules + * + * This is the main function that converts GeoJSON features and layer styling + * configurations into the typed arrays needed for high-performance map rendering. + * It handles color encoding, size mapping, filtering, and optimization for GPU rendering. + * + * @param args - Build arguments containing features, layer configs, and defaults + * @returns BuildResult with typed arrays ready for WebGL rendering + */ +export function buildStyleArrays(args: BuildArgs): BuildResult { + const { features, layers, defaults = {} } = args + + const N = features.length + const fillColors = new Uint8ClampedArray(N * 4) + const lineColors = new Uint8ClampedArray(N * 3) // RGB only + const lineWidths = new Float32Array(N) + const pointRadii = new Float32Array(N) + const fillHeights = new Float32Array(N) + const featureFilter = new Float32Array(N) + + // Pre-index features by layer name in properties._layer to allow per-layer style + // Fallback to globalstyle if properties._layer isn’t present. + type LayerBucket = { idxs: number[]; props: any[]; style?: LayerStyle } + const buckets = new Map() + for (let i = 0; i < N; i++) { + const props = features[i]?.properties || {} + const layerName = props._layer || 'GLOBAL' + const bucket = buckets.get(layerName) || { + idxs: [], + props: [], + style: layers[layerName]?.style, + } + bucket.idxs.push(i) + bucket.props.push(props) + bucket.style = layers[layerName]?.style || bucket.style + buckets.set(layerName, bucket) + } + + // Global defaults used if no layer style present + const defaultFill = hexToRgba(defaults.fillColor || '#59a14f', 1) + const defaultLine = hexToRgb(defaults.lineColor || '#4e79a7') + const defaultWidth = defaults.lineWidth ?? 2 + const defaultRadius = defaults.pointRadius ?? 4 + const defaultHeight = defaults.fillHeight ?? 0 + + // Initialize everything to defaults first + for (let i = 0; i < N; i++) { + fillColors.set(defaultFill, i * 4) + lineColors.set(defaultLine, i * 3) // RGB offset + lineWidths[i] = defaultWidth + pointRadii[i] = defaultRadius + fillHeights[i] = defaultHeight + featureFilter[i] = 1 // visible by default + } + + // Apply per-layer styles + for (const bucket of Array.from(buckets.values())) { + const style = bucket.style + const idxs = bucket.idxs + const propsArr = bucket.props + + if (!style) continue + + // filter + if (style.filter && 'column' in style.filter) { + const col = (style.filter as any).column + const include: any[] | undefined = (style.filter as any).include + const exclude: any[] | undefined = (style.filter as any).exclude + applyFeatureFilter(idxs, propsArr, col, include, exclude, featureFilter) + } + + // Helper to apply color style to a target typed array + const applyColorStyle = ( + idxs: number[], + propsArr: any[], + styleVal: any, + target: Uint8ClampedArray, + stride: number, + offset: number, + defaultHex: string + ) => { + if (!styleVal) return + if (typeof styleVal === 'string') { + const color = stride === 3 ? hexToRgb(styleVal) : hexToRgba(styleVal, 1) + applyStaticValue(idxs, color as any, target as any, stride, offset) + } else if ('colors' in styleVal) { + const encoder = buildCategoryEncoder(styleVal.colors, '#808080') + applyEncodedValue( + idxs, + propsArr, + styleVal.column, + encoder as any, + target as any, + stride, + offset + ) + } else if ('column' in styleVal) { + const encoder = buildColorEncoder( + propsArr.map((p: any) => p?.[styleVal.column]), + styleVal, + styleVal.dataRange + ) + applyEncodedValue( + idxs, + propsArr, + styleVal.column, + encoder as any, + target as any, + stride, + offset + ) + } + } + + applyColorStyle(idxs, propsArr, style.fillColor, fillColors, 4, 0, '#808080') + applyColorStyle(idxs, propsArr, style.lineColor, lineColors, 3, 0, '#808080') + + // lineWidth - handle static, array, category, and column-based + if (style.lineWidth) { + const lw = style.lineWidth as any + if (Array.isArray(lw)) { + applyNumericArray(idxs, lw, lineWidths, defaultWidth) + } else if (typeof lw === 'number') { + applyStaticNumeric(idxs, lw, lineWidths) + } else if (typeof lw === 'object' && 'widths' in lw && 'column' in lw) { + applyNumericCategory(idxs, propsArr, lw.column, lw.widths || {}, lineWidths, defaultWidth) + } else if ('column' in lw) { + const values = propsArr.map((p: any) => toNumber(p?.[lw.column])) + applyQuantitativeMapping( + values, + lw.dataRange ?? [1, 6], + lw.widthRange ?? [1, 6], + lineWidths, + 1, + 0 + ) + } + } + + // pointRadius - handle both static and column-based + if (style.pointRadius) { + const pr = style.pointRadius as any + if (typeof pr === 'number') { + applyStaticNumeric(idxs, pr, pointRadii) + } else if ('column' in pr) { + const values = propsArr.map((p: any) => toNumber(p?.[pr.column])) + applyQuantitativeMapping( + values, + pr.dataRange ?? [2, 12], + pr.widthRange ?? [2, 12], + pointRadii, + 1, + 0 + ) + } + } + + // fillHeight - handle both static and column-based + if (style.fillHeight) { + const fh = style.fillHeight as any + if (typeof fh === 'number') { + applyStaticNumeric(idxs, fh, fillHeights) + } else if ('column' in fh) { + const values = propsArr.map((p: any) => toNumber(p?.[fh.column])) + applyQuantitativeMapping( + values, + fh.dataRange ?? [0, 100], + fh.widthRange ?? [0, 100], + fillHeights, + 1, + 0 + ) + } + } + } + + return { + fillColors, + lineColors, + lineWidths, + pointRadii, + fillHeights, + featureFilter, + } +} diff --git a/src/plugins/sqlite-map/types.ts b/src/plugins/sqlite-map/types.ts new file mode 100644 index 00000000..ae620fd6 --- /dev/null +++ b/src/plugins/sqlite-map/types.ts @@ -0,0 +1,179 @@ +// Type definitions for AequilibraE plugin (layers, styles, DB, runtime types) + +/** Supported geometry types for spatial data visualization */ + +export type GeometryType = 'polygon' | 'line' | 'point' + +/** + * Configuration for joining data from an external database table + * to a layer's features. This allows visualization of results data + * (e.g., simulation outputs) on top of network geometry. + */ +export interface JoinConfig { + /** Key referencing an entry in extraDatabases */ + database: string + /** Table name in the external database to join from */ + table: string + /** Column name in the main layer table to join on */ + leftKey: string + /** Column name in the external table to join on */ + rightKey: string + /** Join type: 'left' keeps all main records, 'inner' only keeps matches. Default: 'left' */ + type?: 'left' | 'inner' + /** Optional: specific columns to include from the joined table (default: all) */ + columns?: string[] + /** Optional: SQL WHERE clause to filter rows in the joined table (e.g., 'volume > 100') */ + filter?: string +} + +export interface LayerConfig { + table: string + type: GeometryType + join?: JoinConfig + fillColor?: string + strokeColor?: string + strokeWidth?: number + radius?: number + opacity?: number + zIndex?: number +} + +export interface VizDetails { + title: string + description: string + database: string + extraDatabases?: Record + view: 'table' | 'map' | '' + layers: { [key: string]: LayerConfig } + center?: [number, number] | string + zoom?: number + projection?: string + bearing?: number + pitch?: number + geometryLimit?: number + coordinatePrecision?: number + minimalProperties?: boolean + defaults?: { + fillColor?: string + lineColor?: string + lineWidth?: number + pointRadius?: number + fillHeight?: number + } + legend?: Array<{ + label?: string + color?: string + size?: number + shape?: string + subtitle?: string + }> +} + +// ============================================================================= +// Color and Styling Type Definitions +// ============================================================================= + +export type RGBA = [number, number, number, number] +export type RGB = [number, number, number] + +// Quantitative color style: numeric column mapped to a palette +export type QuantitativeColorStyle = { + column: string + type?: 'quantitative' + palette?: string + numColors?: number + // explicit range for color mapping (min, max) + range?: [number, number] + // optional data range override when computing encoders + dataRange?: [number, number] +} + +// Categorical color style: mapping of category -> hex color +export type CategoricalColorStyle = { + column: string + type?: 'categorical' + colors: Record +} + +// ColorStyle may be a simple hex string or one of the structured styles above +export type ColorStyle = string | QuantitativeColorStyle | CategoricalColorStyle + +// Numeric style may be a static number, an explicit array per-feature, or a column-driven mapping +export type NumericColumnStyle = { + column: string + dataRange?: [number, number] + widthRange?: [number, number] + // category -> width mapping + widths?: Record +} + +export type NumericStyle = number | number[] | NumericColumnStyle + +export type LayerStyle = { + fillColor?: ColorStyle + lineColor?: ColorStyle + // lineWidth can be static, per-feature array, or column/category-driven + lineWidth?: NumericStyle + // point radius and fill height share the same flexible numeric shape + pointRadius?: NumericStyle + fillHeight?: NumericStyle + filter?: { + column: string + include?: any[] + exclude?: any[] + } +} + +export type LayerConfigLite = { + table?: string + geometry?: string + style?: LayerStyle +} + +export type BuildArgs = { + features: Array<{ properties: any; geometry: any }> + layers: Record + defaults?: { + fillColor?: string + lineColor?: string + lineWidth?: number + pointRadius?: number + fillHeight?: number + } +} + +export type BuildResult = { + fillColors: Uint8ClampedArray + lineColors: Uint8ClampedArray + lineWidths: Float32Array + pointRadii: Float32Array + fillHeights: Float32Array + featureFilter: Float32Array +} + +// ============================================================================= +// Runtime / DB related types +// ============================================================================= + +/** Lightweight representation of a GeoJSON Feature used by the plugin */ +export interface GeoFeature { + type: 'Feature' + geometry: any + properties: Record +} + +/** Basic shape for the SQLite DB wrapper returned by the SPL runtime */ +export interface SqliteDb { + exec: (sql: string) => { get: { objs: any[] } } | any + close?: () => void + [k: string]: any +} + +/** SPL runtime instance interface (only the parts we use) */ +export interface SPL { + db: (arrayBuffer: ArrayBuffer) => Promise | SqliteDb + [k: string]: any +} + +/** Lazy loader for extra databases used by buildGeoFeatures */ +export type LazyDbLoader = (dbName: string) => Promise diff --git a/src/plugins/sqlite-map/utils.ts b/src/plugins/sqlite-map/utils.ts new file mode 100644 index 00000000..03decaff --- /dev/null +++ b/src/plugins/sqlite-map/utils.ts @@ -0,0 +1,258 @@ +// Shared utilities: path resolution, column analysis, and caching helpers + +import type { LayerConfig, JoinConfig } from './types' + +// Helper type for column metadata returned by PRAGMA table_info +export type ColumnInfo = { name: string; type?: string; nullable?: boolean } + +// ============================================================================ +// Constants +// ============================================================================ + +/** Standard column name for geometry in spatial databases */ +export const GEOMETRY_COLUMN = 'geometry' + +/** Essential spatial column names that should always be included */ +export const ESSENTIAL_SPATIAL_COLUMNS = new Set([ + 'id', + 'name', + 'link_id', + 'node_id', + 'zone_id', + 'a_node', + 'b_node', +]) + +/** Style properties that may reference data columns */ +export const STYLE_PROPERTIES = [ + 'fillColor', + 'lineColor', + 'lineWidth', + 'pointRadius', + 'fillHeight', + 'filter', +] + +// ============================================================================ +// Path Resolution +// ============================================================================ + +/** + * Resolve a file path, making it absolute if needed + * Handles both absolute paths (starting with /) and relative paths + * + * @param filePath - The file path to resolve + * @param subfolder - Optional subfolder to prepend for relative paths + * @returns Resolved absolute path + */ +export function resolvePath(filePath: string, subfolder?: string | null): string { + if (!filePath) throw new Error('File path is required') + + if (filePath.startsWith('/')) { + return filePath + } + + if (subfolder) { + return `${subfolder}/${filePath}` + } + + return filePath +} + +/** + * Resolve multiple paths at once + * + * @param paths - Record of path names to file paths + * @param subfolder - Optional subfolder to prepend for relative paths + * @returns Record with resolved paths + */ +export function resolvePaths( + paths: Record, + subfolder?: string | null +): Record { + const resolved: Record = {} + for (const [name, path] of Object.entries(paths)) { + resolved[name] = resolvePath(path, subfolder) + } + return resolved +} + +// ============================================================================ +// Column/Style Analysis +// ============================================================================ + +/** + * Extract all columns that are used for styling in a layer configuration. + * This identifies which columns from the database need to be loaded. + * + * @param layerConfig - Layer configuration with styling rules + * @returns Set of column names used for styling + */ +export function getUsedColumns(layerConfig: Partial | any): Set { + const used = new Set() + if (!layerConfig?.style) return used + + const style = layerConfig.style + + for (const prop of STYLE_PROPERTIES) { + const cfg = style[prop] + if (cfg && typeof cfg === 'object' && 'column' in cfg) { + used.add(cfg.column) + } + } + + return used +} + +/** + * Extract the specific column needed from a join for a layer's styling. + * This allows optimized loading - only fetching the columns that are actually needed. + * + * @param layerConfig - Layer configuration with styling rules + * @returns Column name needed from join data, or undefined if none + */ +export function getNeededJoinColumn(layerConfig: Partial | any): string | undefined { + const style = (layerConfig as any).style + if (!style) return undefined + + for (const prop of STYLE_PROPERTIES) { + const cfg = style[prop] + if (cfg && typeof cfg === 'object' && 'column' in cfg) { + return cfg.column + } + } + + return undefined +} + +/** + * Check if a column is the geometry column + * + * @param columnName - Name of the column to check + * @returns True if this is a geometry column + */ +export function isGeometryColumn(columnName: string): boolean { + return columnName.toLowerCase() === GEOMETRY_COLUMN +} + +/** + * Filter columns, excluding the geometry column + * + * @param columns - Array of column metadata objects with 'name' property + * @returns Filtered array without geometry columns + */ +export function filterOutGeometryColumns(columns: Array): Array { + return columns.filter(c => !isGeometryColumn(c.name)) +} + +/** + * Check if any column in the list is a geometry column + * + * @param columns - Array of column metadata objects with 'name' property + * @returns True if at least one geometry column is found + */ +export function hasGeometryColumn(columns: Array): boolean { + return columns.some(c => isGeometryColumn(c.name)) +} + +/** + * Build a set of columns to select from the database. + * Includes used columns, essential spatial columns, and optionally all columns. + * + * @param allColumns - All available columns in the table + * @param usedColumns - Set of columns needed for styling + * @param includeAll - If true, include all columns; if false, only include used + essential + * @returns Set of column names to select + */ +export function buildColumnSelection( + allColumns: Array, + usedColumns: Set, + includeAll: boolean = false +): Set { + const selection = new Set() + + if (includeAll) { + // Include all columns except geometry + for (const col of allColumns) { + if (!isGeometryColumn(col.name)) { + selection.add(col.name) + } + } + } else { + // Include used columns + essential spatial columns + for (const col of allColumns) { + const name = col.name + if ( + !isGeometryColumn(name) && + (usedColumns.has(name) || ESSENTIAL_SPATIAL_COLUMNS.has(name.toLowerCase())) + ) { + selection.add(name) + } + } + } + + return selection +} + +// ============================================================================ +// Caching Utilities +// ============================================================================ + +/** + * Create a cache key for join operations + * + * @param database - Database name + * @param table - Table name + * @param column - Column name (optional) + * @param filter - Filter clause (optional) + * @returns Cache key string + */ +export function createJoinCacheKey( + database: string, + table: string, + column?: string, + filter?: string +): string { + const parts = [database, table, column || '*', filter || ''] + return parts.join('::') +} + +/** + * Create a cache key for geometry data + * + * @param dbPath - Database file path + * @param tableName - Table name + * @returns Cache key string + */ +export function createGeometryCacheKey(dbPath: string, tableName: string): string { + return `${dbPath}::${tableName}` +} + +/** + * Simplify coordinates by reducing precision (removes ~30-50% memory for coordinates) + * Uses iterative approach to avoid stack overflow with large/nested geometries + * @param coords - Coordinate array to simplify + * @param precision - Number of decimal places to keep (default 6) + */ +export function simplifyCoordinates(coords: unknown, precision: number = 6): unknown { + if (!coords || !Array.isArray(coords)) return coords + + const factor = Math.pow(10, precision) + + const simplifyValue = (val: unknown): unknown => { + if (typeof val === 'number') { + return Math.round(val * factor) / factor + } + if (!Array.isArray(val)) { + return val + } + // Coordinate array of numbers + if (val.length > 0 && typeof val[0] === 'number') { + return val.map((n: number) => Math.round(n * factor) / factor) + } + // Nested arrays + return val.map((item: any) => simplifyValue(item)) + } + + return simplifyValue(coords) +} diff --git a/src/types/cartocolor.d.ts b/src/types/cartocolor.d.ts new file mode 100644 index 00000000..15cf7108 --- /dev/null +++ b/src/types/cartocolor.d.ts @@ -0,0 +1,47 @@ +declare module 'cartocolor' { + interface CartoColorPalette { + tags?: string[] + [key: number]: string[] + } + + const Burg: CartoColorPalette + const BurgYl: CartoColorPalette + const RedOr: CartoColorPalette + const OrYel: CartoColorPalette + const Peach: CartoColorPalette + const PinkYl: CartoColorPalette + const Mint: CartoColorPalette + const BluGrn: CartoColorPalette + const DarkMint: CartoColorPalette + const Emrld: CartoColorPalette + const BluYl: CartoColorPalette + const Teal: CartoColorPalette + const TealGrn: CartoColorPalette + const Purp: CartoColorPalette + const PurpOr: CartoColorPalette + const Sunset: CartoColorPalette + const Magenta: CartoColorPalette + const SunsetDark: CartoColorPalette + const BrwnYl: CartoColorPalette + + // Diverging + const ArmyRose: CartoColorPalette + const Fall: CartoColorPalette + const Geyser: CartoColorPalette + const Temps: CartoColorPalette + const TealRose: CartoColorPalette + const Tropic: CartoColorPalette + const Earth: CartoColorPalette + + // Qualitative + const Antique: CartoColorPalette + const Bold: CartoColorPalette + const Pastel: CartoColorPalette + const Prism: CartoColorPalette + const Safe: CartoColorPalette + const Vivid: CartoColorPalette + + const YlGn: CartoColorPalette + const Blues: CartoColorPalette + const Reds: CartoColorPalette +} diff --git a/src/types/spl.d.ts b/src/types/spl.d.ts new file mode 100644 index 00000000..b5914a1f --- /dev/null +++ b/src/types/spl.d.ts @@ -0,0 +1 @@ +declare module 'spl.js'; diff --git a/tsconfig.json b/tsconfig.json index 8223bb6c..0527ad56 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,7 +17,7 @@ "skipLibCheck": true, "strict": true, "target": "esnext", - "types": ["vite/client", "@types/jest", "vitest/globals"] + "types": ["vite/client", "@types/jest"] }, "include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.vue", "test/**/*.ts", "test/**/*.tsx"], "exclude": ["node_modules"]