From d0019a17476be222ab4bad6330a70299a32c22cd Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 28 Jan 2025 12:14:29 +0100 Subject: [PATCH 01/25] Use sealed classes for sync lines --- .../lib/src/bucket_storage.dart | 138 ------------- .../lib/src/streaming_sync.dart | 188 ++++++++--------- .../powersync_core/lib/src/sync_types.dart | 190 +++++++++++++++--- 3 files changed, 254 insertions(+), 262 deletions(-) diff --git a/packages/powersync_core/lib/src/bucket_storage.dart b/packages/powersync_core/lib/src/bucket_storage.dart index 804a278c..12137a02 100644 --- a/packages/powersync_core/lib/src/bucket_storage.dart +++ b/packages/powersync_core/lib/src/bucket_storage.dart @@ -347,105 +347,6 @@ class SyncDataBatch { SyncDataBatch(this.buckets); } -class SyncBucketData { - final String bucket; - final List data; - final bool hasMore; - final String? after; - final String? nextAfter; - - const SyncBucketData( - {required this.bucket, - required this.data, - this.hasMore = false, - this.after, - this.nextAfter}); - - SyncBucketData.fromJson(Map json) - : bucket = json['bucket'], - hasMore = json['has_more'] ?? false, - after = json['after'], - nextAfter = json['next_after'], - data = - (json['data'] as List).map((e) => OplogEntry.fromJson(e)).toList(); - - Map toJson() { - return { - 'bucket': bucket, - 'has_more': hasMore, - 'after': after, - 'next_after': nextAfter, - 'data': data - }; - } -} - -class OplogEntry { - final String opId; - - final OpType? op; - - /// rowType + rowId uniquely identifies an entry in the local database. - final String? rowType; - final String? rowId; - - /// Together with rowType and rowId, this uniquely identifies a source entry - /// per bucket in the oplog. There may be multiple source entries for a single - /// "rowType + rowId" combination. - final String? subkey; - - final String? data; - final int checksum; - - const OplogEntry( - {required this.opId, - required this.op, - this.subkey, - this.rowType, - this.rowId, - this.data, - required this.checksum}); - - OplogEntry.fromJson(Map json) - : opId = json['op_id'], - op = OpType.fromJson(json['op']), - rowType = json['object_type'], - rowId = json['object_id'], - checksum = json['checksum'], - data = json['data'] is String ? json['data'] : jsonEncode(json['data']), - subkey = json['subkey'] is String ? json['subkey'] : null; - - Map? get parsedData { - return data == null ? null : jsonDecode(data!); - } - - /// Key to uniquely represent a source entry in a bucket. - /// This is used to supersede old entries. - /// Relevant for put and remove ops. - String get key { - return "$rowType/$rowId/$subkey"; - } - - Map toJson() { - return { - 'op_id': opId, - 'op': op?.toJson(), - 'object_type': rowType, - 'object_id': rowId, - 'checksum': checksum, - 'subkey': subkey, - 'data': data - }; - } -} - -class SqliteOp { - String sql; - List args; - - SqliteOp(this.sql, this.args); -} - class SyncLocalDatabaseResult { final bool ready; final bool checkpointValid; @@ -483,42 +384,3 @@ class ChecksumCache { ChecksumCache(this.lastOpId, this.checksums); } - -enum OpType { - clear(1), - move(2), - put(3), - remove(4); - - final int value; - - const OpType(this.value); - - static OpType? fromJson(String json) { - switch (json) { - case 'CLEAR': - return clear; - case 'MOVE': - return move; - case 'PUT': - return put; - case 'REMOVE': - return remove; - default: - return null; - } - } - - String toJson() { - switch (this) { - case clear: - return 'CLEAR'; - case move: - return 'MOVE'; - case put: - return 'PUT'; - case remove: - return 'REMOVE'; - } - } -} diff --git a/packages/powersync_core/lib/src/streaming_sync.dart b/packages/powersync_core/lib/src/streaming_sync.dart index 6f4191ff..d2192377 100644 --- a/packages/powersync_core/lib/src/streaming_sync.dart +++ b/packages/powersync_core/lib/src/streaming_sync.dart @@ -53,7 +53,7 @@ class StreamingSyncImplementation implements StreamingSync { late final http.Client _client; - final StreamController _localPingController = + final StreamController _localPingController = StreamController.broadcast(); final Duration retryDelay; @@ -340,96 +340,19 @@ class StreamingSyncImplementation implements StreamingSync { } _updateStatus(connected: true, connecting: false); - if (line is Checkpoint) { - targetCheckpoint = line; - final Set bucketsToDelete = {...bucketSet}; - final Set newBuckets = {}; - for (final checksum in line.checksums) { - newBuckets.add(checksum.bucket); - bucketsToDelete.remove(checksum.bucket); - } - bucketSet = newBuckets; - await adapter.removeBuckets([...bucketsToDelete]); - _updateStatus(downloading: true); - } else if (line is StreamingSyncCheckpointComplete) { - final result = await adapter.syncLocalDatabase(targetCheckpoint!); - if (!result.checkpointValid) { - // This means checksums failed. Start again with a new checkpoint. - // TODO: better back-off - // await new Promise((resolve) => setTimeout(resolve, 50)); - return false; - } else if (!result.ready) { - // Checksums valid, but need more data for a consistent checkpoint. - // Continue waiting. - } else { - appliedCheckpoint = targetCheckpoint; - - _updateStatus( - downloading: false, - downloadError: _noError, - lastSyncedAt: DateTime.now()); - } - - validatedCheckpoint = targetCheckpoint; - } else if (line is StreamingSyncCheckpointDiff) { - // TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint - if (targetCheckpoint == null) { - throw PowerSyncProtocolException( - 'Checkpoint diff without previous checkpoint'); - } - _updateStatus(downloading: true); - final diff = line; - final Map newBuckets = {}; - for (var checksum in targetCheckpoint.checksums) { - newBuckets[checksum.bucket] = checksum; - } - for (var checksum in diff.updatedBuckets) { - newBuckets[checksum.bucket] = checksum; - } - for (var bucket in diff.removedBuckets) { - newBuckets.remove(bucket); - } - - final newCheckpoint = Checkpoint( - lastOpId: diff.lastOpId, - checksums: [...newBuckets.values], - writeCheckpoint: diff.writeCheckpoint); - targetCheckpoint = newCheckpoint; - - bucketSet = Set.from(newBuckets.keys); - await adapter.removeBuckets(diff.removedBuckets); - adapter.setTargetCheckpoint(targetCheckpoint); - } else if (line is SyncBucketData) { - _updateStatus(downloading: true); - await adapter.saveSyncData(SyncDataBatch([line])); - } else if (line is StreamingSyncKeepalive) { - if (line.tokenExpiresIn == 0) { - // Token expired already - stop the connection immediately - invalidCredentialsCallback?.call().ignore(); - break; - } else if (line.tokenExpiresIn <= 30) { - // Token expires soon - refresh it in the background - if (credentialsInvalidation == null && - invalidCredentialsCallback != null) { - credentialsInvalidation = invalidCredentialsCallback!().then((_) { - // Token has been refreshed - we should restart the connection. - haveInvalidated = true; - // trigger next loop iteration ASAP, don't wait for another - // message from the server. - _localPingController.add(null); - }, onError: (_) { - // Token refresh failed - retry on next keepalive. - credentialsInvalidation = null; - }); + switch (line) { + case Checkpoint(): + targetCheckpoint = line; + final Set bucketsToDelete = {...bucketSet}; + final Set newBuckets = {}; + for (final checksum in line.checksums) { + newBuckets.add(checksum.bucket); + bucketsToDelete.remove(checksum.bucket); } - } - } else { - if (targetCheckpoint == appliedCheckpoint) { - _updateStatus( - downloading: false, - downloadError: _noError, - lastSyncedAt: DateTime.now()); - } else if (validatedCheckpoint == targetCheckpoint) { + bucketSet = newBuckets; + await adapter.removeBuckets([...bucketsToDelete]); + _updateStatus(downloading: true); + case StreamingSyncCheckpointComplete(): final result = await adapter.syncLocalDatabase(targetCheckpoint!); if (!result.checkpointValid) { // This means checksums failed. Start again with a new checkpoint. @@ -447,7 +370,85 @@ class StreamingSyncImplementation implements StreamingSync { downloadError: _noError, lastSyncedAt: DateTime.now()); } - } + + validatedCheckpoint = targetCheckpoint; + case StreamingSyncCheckpointDiff(): + // TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint + if (targetCheckpoint == null) { + throw PowerSyncProtocolException( + 'Checkpoint diff without previous checkpoint'); + } + _updateStatus(downloading: true); + final diff = line; + final Map newBuckets = {}; + for (var checksum in targetCheckpoint.checksums) { + newBuckets[checksum.bucket] = checksum; + } + for (var checksum in diff.updatedBuckets) { + newBuckets[checksum.bucket] = checksum; + } + for (var bucket in diff.removedBuckets) { + newBuckets.remove(bucket); + } + + final newCheckpoint = Checkpoint( + lastOpId: diff.lastOpId, + checksums: [...newBuckets.values], + writeCheckpoint: diff.writeCheckpoint); + targetCheckpoint = newCheckpoint; + + bucketSet = Set.from(newBuckets.keys); + await adapter.removeBuckets(diff.removedBuckets); + adapter.setTargetCheckpoint(targetCheckpoint); + case SyncBucketData(): + _updateStatus(downloading: true); + await adapter.saveSyncData(SyncDataBatch([line])); + case StreamingSyncKeepalive(): + if (line.tokenExpiresIn == 0) { + // Token expired already - stop the connection immediately + invalidCredentialsCallback?.call().ignore(); + break; + } else if (line.tokenExpiresIn <= 30) { + // Token expires soon - refresh it in the background + if (credentialsInvalidation == null && + invalidCredentialsCallback != null) { + credentialsInvalidation = invalidCredentialsCallback!().then((_) { + // Token has been refreshed - we should restart the connection. + haveInvalidated = true; + // trigger next loop iteration ASAP, don't wait for another + // message from the server. + _localPingController.add(null); + }, onError: (_) { + // Token refresh failed - retry on next keepalive. + credentialsInvalidation = null; + }); + } + } + case null: // Local ping + if (targetCheckpoint == appliedCheckpoint) { + _updateStatus( + downloading: false, + downloadError: _noError, + lastSyncedAt: DateTime.now()); + } else if (validatedCheckpoint == targetCheckpoint) { + final result = await adapter.syncLocalDatabase(targetCheckpoint!); + if (!result.checkpointValid) { + // This means checksums failed. Start again with a new checkpoint. + // TODO: better back-off + // await new Promise((resolve) => setTimeout(resolve, 50)); + return false; + } else if (!result.ready) { + // Checksums valid, but need more data for a consistent checkpoint. + // Continue waiting. + } else { + appliedCheckpoint = targetCheckpoint; + + _updateStatus( + downloading: false, + downloadError: _noError, + lastSyncedAt: DateTime.now()); + } + } } if (haveInvalidated) { @@ -458,7 +459,8 @@ class StreamingSyncImplementation implements StreamingSync { return true; } - Stream streamingSyncRequest(StreamingSyncRequest data) async* { + Stream streamingSyncRequest( + StreamingSyncRequest data) async* { final credentials = await credentialsCallback(); if (credentials == null) { throw CredentialsException('Not logged in'); @@ -498,7 +500,7 @@ class StreamingSyncImplementation implements StreamingSync { if (aborted) { break; } - yield parseStreamingSyncLine(line as Map); + yield StreamingSyncLine.fromJson(line as Map); } } diff --git a/packages/powersync_core/lib/src/sync_types.dart b/packages/powersync_core/lib/src/sync_types.dart index 46eeb959..e1ac9303 100644 --- a/packages/powersync_core/lib/src/sync_types.dart +++ b/packages/powersync_core/lib/src/sync_types.dart @@ -1,6 +1,29 @@ -import 'bucket_storage.dart'; +import 'dart:convert'; -class Checkpoint { +sealed class StreamingSyncLine { + const StreamingSyncLine(); + + /// Parses a [StreamingSyncLine] from JSON, or return `null` if the line is + /// not understood by this client. + static StreamingSyncLine? fromJson(Map line) { + if (line.containsKey('checkpoint')) { + return Checkpoint.fromJson(line['checkpoint']); + } else if (line.containsKey('checkpoint_diff')) { + return StreamingSyncCheckpointDiff.fromJson(line['checkpoint_diff']); + } else if (line.containsKey('checkpoint_complete')) { + return StreamingSyncCheckpointComplete.fromJson( + line['checkpoint_complete']); + } else if (line.containsKey('data')) { + return SyncBucketData.fromJson(line['data']); + } else if (line.containsKey('token_expires_in')) { + return StreamingSyncKeepalive.fromJson(line); + } else { + return null; + } + } +} + +final class Checkpoint extends StreamingSyncLine { final String lastOpId; final String? writeCheckpoint; final List checksums; @@ -47,16 +70,7 @@ class BucketChecksum { lastOpId = json['last_op_id']; } -class StreamingSyncCheckpoint { - Checkpoint checkpoint; - - StreamingSyncCheckpoint(this.checkpoint); - - StreamingSyncCheckpoint.fromJson(Map json) - : checkpoint = Checkpoint.fromJson(json); -} - -class StreamingSyncCheckpointDiff { +final class StreamingSyncCheckpointDiff extends StreamingSyncLine { String lastOpId; List updatedBuckets; List removedBuckets; @@ -74,7 +88,7 @@ class StreamingSyncCheckpointDiff { removedBuckets = List.from(json['removed_buckets']); } -class StreamingSyncCheckpointComplete { +final class StreamingSyncCheckpointComplete extends StreamingSyncLine { String lastOpId; StreamingSyncCheckpointComplete(this.lastOpId); @@ -83,7 +97,7 @@ class StreamingSyncCheckpointComplete { : lastOpId = json['last_op_id']; } -class StreamingSyncKeepalive { +final class StreamingSyncKeepalive extends StreamingSyncLine { int tokenExpiresIn; StreamingSyncKeepalive(this.tokenExpiresIn); @@ -92,23 +106,6 @@ class StreamingSyncKeepalive { : tokenExpiresIn = json['token_expires_in']; } -Object? parseStreamingSyncLine(Map line) { - if (line.containsKey('checkpoint')) { - return Checkpoint.fromJson(line['checkpoint']); - } else if (line.containsKey('checkpoint_diff')) { - return StreamingSyncCheckpointDiff.fromJson(line['checkpoint_diff']); - } else if (line.containsKey('checkpoint_complete')) { - return StreamingSyncCheckpointComplete.fromJson( - line['checkpoint_complete']); - } else if (line.containsKey('data')) { - return SyncBucketData.fromJson(line['data']); - } else if (line.containsKey('token_expires_in')) { - return StreamingSyncKeepalive.fromJson(line); - } else { - return null; - } -} - class StreamingSyncRequest { List buckets; bool includeChecksum = true; @@ -144,3 +141,134 @@ class BucketRequest { 'after': after, }; } + +final class SyncBucketData extends StreamingSyncLine { + final String bucket; + final List data; + final bool hasMore; + final String? after; + final String? nextAfter; + + const SyncBucketData( + {required this.bucket, + required this.data, + this.hasMore = false, + this.after, + this.nextAfter}); + + SyncBucketData.fromJson(Map json) + : bucket = json['bucket'], + hasMore = json['has_more'] ?? false, + after = json['after'], + nextAfter = json['next_after'], + data = + (json['data'] as List).map((e) => OplogEntry.fromJson(e)).toList(); + + Map toJson() { + return { + 'bucket': bucket, + 'has_more': hasMore, + 'after': after, + 'next_after': nextAfter, + 'data': data + }; + } +} + +class OplogEntry { + final String opId; + + final OpType? op; + + /// rowType + rowId uniquely identifies an entry in the local database. + final String? rowType; + final String? rowId; + + /// Together with rowType and rowId, this uniquely identifies a source entry + /// per bucket in the oplog. There may be multiple source entries for a single + /// "rowType + rowId" combination. + final String? subkey; + + final String? data; + final int checksum; + + const OplogEntry( + {required this.opId, + required this.op, + this.subkey, + this.rowType, + this.rowId, + this.data, + required this.checksum}); + + OplogEntry.fromJson(Map json) + : opId = json['op_id'], + op = OpType.fromJson(json['op']), + rowType = json['object_type'], + rowId = json['object_id'], + checksum = json['checksum'], + data = json['data'] is String ? json['data'] : jsonEncode(json['data']), + subkey = json['subkey'] is String ? json['subkey'] : null; + + Map? get parsedData { + return data == null ? null : jsonDecode(data!); + } + + /// Key to uniquely represent a source entry in a bucket. + /// This is used to supersede old entries. + /// Relevant for put and remove ops. + String get key { + return "$rowType/$rowId/$subkey"; + } + + Map toJson() { + return { + 'op_id': opId, + 'op': op?.toJson(), + 'object_type': rowType, + 'object_id': rowId, + 'checksum': checksum, + 'subkey': subkey, + 'data': data + }; + } +} + +enum OpType { + clear(1), + move(2), + put(3), + remove(4); + + final int value; + + const OpType(this.value); + + static OpType? fromJson(String json) { + switch (json) { + case 'CLEAR': + return clear; + case 'MOVE': + return move; + case 'PUT': + return put; + case 'REMOVE': + return remove; + default: + return null; + } + } + + String toJson() { + switch (this) { + case clear: + return 'CLEAR'; + case move: + return 'MOVE'; + case put: + return 'PUT'; + case remove: + return 'REMOVE'; + } + } +} From 1fa7caa33881726fccebb44e85bc445a8f63f0e0 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 29 Jan 2025 10:30:49 +0100 Subject: [PATCH 02/25] Doc comments on sync lines --- .../powersync_core/lib/src/sync_types.dart | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/powersync_core/lib/src/sync_types.dart b/packages/powersync_core/lib/src/sync_types.dart index e1ac9303..ca4817d9 100644 --- a/packages/powersync_core/lib/src/sync_types.dart +++ b/packages/powersync_core/lib/src/sync_types.dart @@ -1,5 +1,6 @@ import 'dart:convert'; +/// Messages sent from the sync service. sealed class StreamingSyncLine { const StreamingSyncLine(); @@ -23,6 +24,10 @@ sealed class StreamingSyncLine { } } +/// Indicates that a checkpoint is available, along with checksums for each +/// bucket in the checkpoint. +/// +/// Note: Called `StreamingSyncCheckpoint` in sync-service. final class Checkpoint extends StreamingSyncLine { final String lastOpId; final String? writeCheckpoint; @@ -70,6 +75,11 @@ class BucketChecksum { lastOpId = json['last_op_id']; } +/// A variant of [Checkpoint] that may be sent when the server has already sent +/// a [Checkpoint] message before. +/// +/// It has the same conceptual meaning as a [Checkpoint] message, but only +/// contains details about changed buckets as an optimization. final class StreamingSyncCheckpointDiff extends StreamingSyncLine { String lastOpId; List updatedBuckets; @@ -88,6 +98,11 @@ final class StreamingSyncCheckpointDiff extends StreamingSyncLine { removedBuckets = List.from(json['removed_buckets']); } +/// Sent after the last [SyncBucketData] message for a checkpoint. +/// +/// Since this indicates that we may have a consistent view of the data, the +/// client may make previous [SyncBucketData] rows visible to the application +/// at this point. final class StreamingSyncCheckpointComplete extends StreamingSyncLine { String lastOpId; @@ -97,6 +112,11 @@ final class StreamingSyncCheckpointComplete extends StreamingSyncLine { : lastOpId = json['last_op_id']; } +/// Sent as a periodic ping to keep the connection alive and to notify the +/// client about the remaining lifetime of the JWT. +/// +/// When the token is nearing its expiry date, the client may ask for another +/// one and open a new sync session with that token. final class StreamingSyncKeepalive extends StreamingSyncLine { int tokenExpiresIn; From 6241a187991dc6b406e13c238d94bf31de23aaec Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 29 Jan 2025 17:35:02 +0100 Subject: [PATCH 03/25] Add priority field --- .../lib/src/bucket_storage.dart | 33 +++++---- .../lib/src/streaming_sync.dart | 71 ++++++++++++++----- .../powersync_core/lib/src/sync_types.dart | 32 +++++++-- 3 files changed, 102 insertions(+), 34 deletions(-) diff --git a/packages/powersync_core/lib/src/bucket_storage.dart b/packages/powersync_core/lib/src/bucket_storage.dart index 12137a02..56a7ec91 100644 --- a/packages/powersync_core/lib/src/bucket_storage.dart +++ b/packages/powersync_core/lib/src/bucket_storage.dart @@ -33,10 +33,14 @@ class BucketStorage { Future> getBucketStates() async { final rows = await select( - 'SELECT name as bucket, cast(last_op as TEXT) as op_id FROM ps_buckets WHERE pending_delete = 0 AND name != \'\$local\''); + 'SELECT name as bucket, priority, cast(last_op as TEXT) as op_id FROM ps_buckets WHERE pending_delete = 0 AND name != \'\$local\''); return [ for (var row in rows) - BucketState(bucket: row['bucket'], opId: row['op_id']) + BucketState( + bucket: row['bucket'], + priority: row['priority'], + opId: row['op_id'], + ) ]; } @@ -100,8 +104,8 @@ class BucketStorage { return false; } - Future syncLocalDatabase( - Checkpoint checkpoint) async { + Future syncLocalDatabase(Checkpoint checkpoint, + {int? forPriority}) async { final r = await validateChecksums(checkpoint); if (!r.checkpointValid) { @@ -124,7 +128,7 @@ class BucketStorage { // Not flushing here - the flush will happen in the next step }, flush: false); - final valid = await updateObjectsFromBuckets(checkpoint); + final valid = await updateObjectsFromBuckets(forPriority: forPriority); if (!valid) { return SyncLocalDatabaseResult(ready: false); } @@ -134,11 +138,11 @@ class BucketStorage { return SyncLocalDatabaseResult(ready: true); } - Future updateObjectsFromBuckets(Checkpoint checkpoint) async { + Future updateObjectsFromBuckets({int? forPriority}) async { return writeTransaction((tx) async { await tx.execute( "INSERT INTO powersync_operations(op, data) VALUES(?, ?)", - ['sync_local', '']); + ['sync_local', forPriority]); final rs = await tx.execute('SELECT last_insert_rowid() as result'); final result = rs[0]['result']; if (result == 1) { @@ -321,9 +325,11 @@ class BucketStorage { class BucketState { final String bucket; + final int priority; final String opId; - const BucketState({required this.bucket, required this.opId}); + const BucketState( + {required this.bucket, required this.priority, required this.opId}); @override String toString() { @@ -332,17 +338,20 @@ class BucketState { @override int get hashCode { - return Object.hash(bucket, opId); + return Object.hash(bucket, priority, opId); } @override bool operator ==(Object other) { - return other is BucketState && other.bucket == bucket && other.opId == opId; + return other is BucketState && + other.priority == priority && + other.bucket == bucket && + other.opId == opId; } } -class SyncDataBatch { - List buckets; +final class SyncDataBatch { + final List buckets; SyncDataBatch(this.buckets); } diff --git a/packages/powersync_core/lib/src/streaming_sync.dart b/packages/powersync_core/lib/src/streaming_sync.dart index d2192377..f6d1338a 100644 --- a/packages/powersync_core/lib/src/streaming_sync.dart +++ b/packages/powersync_core/lib/src/streaming_sync.dart @@ -302,29 +302,36 @@ class StreamingSyncImplementation implements StreamingSync { _statusStreamController.add(newStatus); } - Future streamingSyncIteration( - {AbortController? abortController}) async { - adapter.startSession(); + Future<(List, Map)> + _collectLocalBucketState() async { final bucketEntries = await adapter.getBucketStates(); - Map initialBucketStates = {}; + final initialRequests = [ + for (final entry in bucketEntries) BucketRequest(entry.bucket, entry.opId) + ]; + final localDescriptions = { + for (final entry in bucketEntries) + entry.bucket: ( + name: entry.bucket, + priority: entry.priority, + ) + }; - for (final entry in bucketEntries) { - initialBucketStates[entry.bucket] = entry.opId; - } + return (initialRequests, localDescriptions); + } - final List buckets = []; - for (var entry in initialBucketStates.entries) { - buckets.add(BucketRequest(entry.key, entry.value)); - } + Future streamingSyncIteration( + {AbortController? abortController}) async { + adapter.startSession(); + + var (bucketRequests, bucketMap) = await _collectLocalBucketState(); Checkpoint? targetCheckpoint; Checkpoint? validatedCheckpoint; Checkpoint? appliedCheckpoint; - var bucketSet = Set.from(initialBucketStates.keys); var requestStream = streamingSyncRequest( - StreamingSyncRequest(buckets, syncParameters, clientId!)); + StreamingSyncRequest(bucketRequests, syncParameters, clientId!)); var merged = addBroadcast(requestStream, _localPingController.stream); @@ -343,13 +350,16 @@ class StreamingSyncImplementation implements StreamingSync { switch (line) { case Checkpoint(): targetCheckpoint = line; - final Set bucketsToDelete = {...bucketSet}; - final Set newBuckets = {}; + final Set bucketsToDelete = {...bucketMap.keys}; + final Map newBuckets = {}; for (final checksum in line.checksums) { - newBuckets.add(checksum.bucket); + newBuckets[checksum.bucket] = ( + name: checksum.bucket, + priority: checksum.priority, + ); bucketsToDelete.remove(checksum.bucket); } - bucketSet = newBuckets; + bucketMap = newBuckets; await adapter.removeBuckets([...bucketsToDelete]); _updateStatus(downloading: true); case StreamingSyncCheckpointComplete(): @@ -371,6 +381,27 @@ class StreamingSyncImplementation implements StreamingSync { lastSyncedAt: DateTime.now()); } + validatedCheckpoint = targetCheckpoint; + case StreamingSyncCheckpointPartiallyComplete(:final bucketPriority): + final result = await adapter.syncLocalDatabase(targetCheckpoint!, + forPriority: bucketPriority); + if (!result.checkpointValid) { + // This means checksums failed. Start again with a new checkpoint. + // TODO: better back-off + // await new Promise((resolve) => setTimeout(resolve, 50)); + return false; + } else if (!result.ready) { + // Checksums valid, but need more data for a consistent checkpoint. + // Continue waiting. + } else { + appliedCheckpoint = targetCheckpoint; + + _updateStatus( + downloading: false, + downloadError: _noError, + lastSyncedAt: DateTime.now()); + } + validatedCheckpoint = targetCheckpoint; case StreamingSyncCheckpointDiff(): // TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint @@ -397,7 +428,8 @@ class StreamingSyncImplementation implements StreamingSync { writeCheckpoint: diff.writeCheckpoint); targetCheckpoint = newCheckpoint; - bucketSet = Set.from(newBuckets.keys); + bucketMap = newBuckets.map((name, checksum) => + MapEntry(name, (name: name, priority: checksum.priority))); await adapter.removeBuckets(diff.removedBuckets); adapter.setTargetCheckpoint(targetCheckpoint); case SyncBucketData(): @@ -424,6 +456,9 @@ class StreamingSyncImplementation implements StreamingSync { }); } } + case UnknownSyncLine(:final rawData): + isolateLogger.fine('Ignoring unknown sync line: $rawData'); + break; case null: // Local ping if (targetCheckpoint == appliedCheckpoint) { _updateStatus( diff --git a/packages/powersync_core/lib/src/sync_types.dart b/packages/powersync_core/lib/src/sync_types.dart index ca4817d9..e9f436b0 100644 --- a/packages/powersync_core/lib/src/sync_types.dart +++ b/packages/powersync_core/lib/src/sync_types.dart @@ -4,9 +4,8 @@ import 'dart:convert'; sealed class StreamingSyncLine { const StreamingSyncLine(); - /// Parses a [StreamingSyncLine] from JSON, or return `null` if the line is - /// not understood by this client. - static StreamingSyncLine? fromJson(Map line) { + /// Parses a [StreamingSyncLine] from JSON. + static StreamingSyncLine fromJson(Map line) { if (line.containsKey('checkpoint')) { return Checkpoint.fromJson(line['checkpoint']); } else if (line.containsKey('checkpoint_diff')) { @@ -19,11 +18,18 @@ sealed class StreamingSyncLine { } else if (line.containsKey('token_expires_in')) { return StreamingSyncKeepalive.fromJson(line); } else { - return null; + return UnknownSyncLine(line); } } } +/// A message from the sync service that this client doesn't support. +final class UnknownSyncLine implements StreamingSyncLine { + final Map rawData; + + const UnknownSyncLine(this.rawData); +} + /// Indicates that a checkpoint is available, along with checksums for each /// bucket in the checkpoint. /// @@ -54,8 +60,11 @@ final class Checkpoint extends StreamingSyncLine { } } +typedef BucketDescription = ({String name, int priority}); + class BucketChecksum { final String bucket; + final int priority; final int checksum; /// Count is informational only @@ -64,12 +73,14 @@ class BucketChecksum { const BucketChecksum( {required this.bucket, + required this.priority, required this.checksum, this.count, this.lastOpId}); BucketChecksum.fromJson(Map json) : bucket = json['bucket'], + priority = json['priority'], checksum = json['checksum'], count = json['count'], lastOpId = json['last_op_id']; @@ -112,6 +123,19 @@ final class StreamingSyncCheckpointComplete extends StreamingSyncLine { : lastOpId = json['last_op_id']; } +/// Sent after all the [SyncBucketData] messages for a given priority within a +/// checkpoint have been sent. +final class StreamingSyncCheckpointPartiallyComplete extends StreamingSyncLine { + String lastOpId; + int bucketPriority; + + StreamingSyncCheckpointPartiallyComplete(this.lastOpId, this.bucketPriority); + + StreamingSyncCheckpointPartiallyComplete.fromJson(Map json) + : lastOpId = json['last_op_id'], + bucketPriority = json['priority']; +} + /// Sent as a periodic ping to keep the connection alive and to notify the /// client about the remaining lifetime of the JWT. /// From 8a282047a34ac7f122c26e200f535f8ffdcb5dfd Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 29 Jan 2025 18:08:37 +0100 Subject: [PATCH 04/25] Track bucket priorities --- packages/powersync_core/lib/src/bucket_storage.dart | 13 +++++++++++-- packages/powersync_core/lib/src/streaming_sync.dart | 3 ++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/powersync_core/lib/src/bucket_storage.dart b/packages/powersync_core/lib/src/bucket_storage.dart index 56a7ec91..e77257e7 100644 --- a/packages/powersync_core/lib/src/bucket_storage.dart +++ b/packages/powersync_core/lib/src/bucket_storage.dart @@ -53,12 +53,20 @@ class BucketStorage { var count = 0; await writeTransaction((tx) async { + final descriptions = [ + for (final MapEntry(:key, :value) in batch.descriptions.entries) + { + key: {'priority': value.priority}, + } + ]; + for (var b in batch.buckets) { count += b.data.length; await _updateBucket2( tx, jsonEncode({ - 'buckets': [b] + 'buckets': [b], + 'descriptions': descriptions, })); } // No need to flush - the data is not directly visible to the user either way. @@ -352,8 +360,9 @@ class BucketState { final class SyncDataBatch { final List buckets; + final Map descriptions; - SyncDataBatch(this.buckets); + SyncDataBatch(this.buckets, this.descriptions); } class SyncLocalDatabaseResult { diff --git a/packages/powersync_core/lib/src/streaming_sync.dart b/packages/powersync_core/lib/src/streaming_sync.dart index f6d1338a..a8bdf586 100644 --- a/packages/powersync_core/lib/src/streaming_sync.dart +++ b/packages/powersync_core/lib/src/streaming_sync.dart @@ -433,8 +433,9 @@ class StreamingSyncImplementation implements StreamingSync { await adapter.removeBuckets(diff.removedBuckets); adapter.setTargetCheckpoint(targetCheckpoint); case SyncBucketData(): + // TODO: Merge multiple of these into a single one... _updateStatus(downloading: true); - await adapter.saveSyncData(SyncDataBatch([line])); + await adapter.saveSyncData(SyncDataBatch([line], bucketMap)); case StreamingSyncKeepalive(): if (line.tokenExpiresIn == 0) { // Token expired already - stop the connection immediately From ad25f05e29cc56891ed012341dc9007ec7246d7b Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 30 Jan 2025 17:04:19 +0100 Subject: [PATCH 05/25] Fix tests --- .../test/bucket_storage_test.dart | 196 +++++++++--------- 1 file changed, 103 insertions(+), 93 deletions(-) diff --git a/packages/powersync_core/test/bucket_storage_test.dart b/packages/powersync_core/test/bucket_storage_test.dart index beb05d16..f1a1f34a 100644 --- a/packages/powersync_core/test/bucket_storage_test.dart +++ b/packages/powersync_core/test/bucket_storage_test.dart @@ -39,6 +39,18 @@ const removeAsset1_4 = OplogEntry( const removeAsset1_5 = OplogEntry( opId: '5', op: OpType.remove, rowType: 'assets', rowId: 'O1', checksum: 5); +BucketChecksum checksum( + {required String bucket, required int checksum, int priority = 1}) { + return BucketChecksum(bucket: bucket, priority: priority, checksum: checksum); +} + +SyncDataBatch syncDataBatch(List data) { + return SyncDataBatch(data, { + for (final bucket in data.map((e) => e.bucket).toSet()) + bucket: (name: bucket, priority: 1), + }); +} + void main() { group('Bucket Storage Tests', () { late PowerSyncDatabase powersync; @@ -88,7 +100,7 @@ void main() { expect(await bucketStorage.getBucketStates(), equals([])); expect(await bucketStorage.hasCompletedSync(), equals(false)); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -96,12 +108,14 @@ void main() { ])); final bucketStates = await bucketStorage.getBucketStates(); - expect(bucketStates, - equals([const BucketState(bucket: 'bucket1', opId: '3')])); + expect( + bucketStates, + equals( + [const BucketState(bucket: 'bucket1', opId: '3', priority: 1)])); await syncLocalChecked(Checkpoint( lastOpId: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); await expectAsset1_3(); @@ -119,7 +133,7 @@ void main() { }); test('should get an object from multiple buckets', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_3], @@ -128,8 +142,8 @@ void main() { ])); await syncLocalChecked(Checkpoint(lastOpId: '3', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 3), - BucketChecksum(bucket: 'bucket2', checksum: 3) + checksum(bucket: 'bucket1', checksum: 3), + checksum(bucket: 'bucket2', checksum: 3) ])); await expectAsset1_3(); @@ -140,14 +154,14 @@ void main() { // In this case, there are two different versions in the different buckets. // While we should not get this with our server implementation, the client still specifies this behaviour: // The largest op_id wins. - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [putAsset1_3]), SyncBucketData(bucket: 'bucket2', data: [putAsset1_1]) ])); await syncLocalChecked(Checkpoint(lastOpId: '3', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 3), - BucketChecksum(bucket: 'bucket2', checksum: 1) + checksum(bucket: 'bucket1', checksum: 3), + checksum(bucket: 'bucket2', checksum: 1) ])); await expectAsset1_3(); @@ -155,14 +169,14 @@ void main() { test('should ignore a remove from one bucket', () async { // When we have 1 PUT and 1 REMOVE, the object must be kept. - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [putAsset1_3]), SyncBucketData(bucket: 'bucket2', data: [putAsset1_3, removeAsset1_4]) ])); await syncLocalChecked(Checkpoint(lastOpId: '4', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 3), - BucketChecksum(bucket: 'bucket2', checksum: 7) + checksum(bucket: 'bucket1', checksum: 3), + checksum(bucket: 'bucket2', checksum: 7) ])); await expectAsset1_3(); @@ -170,70 +184,70 @@ void main() { test('should remove when removed from all buckets', () async { // When we only have REMOVE left for an object, it must be deleted. - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [putAsset1_3, removeAsset1_5]), SyncBucketData(bucket: 'bucket2', data: [putAsset1_3, removeAsset1_4]) ])); await syncLocalChecked(Checkpoint(lastOpId: '5', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 8), - BucketChecksum(bucket: 'bucket2', checksum: 7) + checksum(bucket: 'bucket1', checksum: 8), + checksum(bucket: 'bucket2', checksum: 7) ])); await expectNoAssets(); }); test('put then remove', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [putAsset1_3]), ])); await syncLocalChecked(Checkpoint(lastOpId: '3', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 3), + checksum(bucket: 'bucket1', checksum: 3), ])); await expectAsset1_3(); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [removeAsset1_5]) ])); await syncLocalChecked(Checkpoint(lastOpId: '5', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 8), + checksum(bucket: 'bucket1', checksum: 8), ])); await expectNoAssets(); }); test('blank remove', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [putAsset1_3, removeAsset1_4]), ])); await syncLocalChecked(Checkpoint(lastOpId: '4', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 7), + checksum(bucket: 'bucket1', checksum: 7), ])); await expectNoAssets(); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [removeAsset1_5]) ])); await syncLocalChecked(Checkpoint(lastOpId: '5', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 12), + checksum(bucket: 'bucket1', checksum: 12), ])); await expectNoAssets(); }); test('put | put remove', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [putAsset1_1]), ])); await syncLocalChecked(Checkpoint(lastOpId: '1', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 1), + checksum(bucket: 'bucket1', checksum: 1), ])); expect( @@ -243,13 +257,13 @@ void main() { {'id': 'O1', 'description': 'bar', 'make': null} ])); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [putAsset1_3]), SyncBucketData(bucket: 'bucket1', data: [removeAsset1_5]) ])); await syncLocalChecked(Checkpoint(lastOpId: '5', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 9), + checksum(bucket: 'bucket1', checksum: 9), ])); await expectNoAssets(); @@ -275,13 +289,13 @@ void main() { rowId: 'O1', checksum: 5); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset1_3, put4]), ])); await syncLocalChecked(Checkpoint(lastOpId: '4', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 8), + checksum(bucket: 'bucket1', checksum: 8), ])); expect( @@ -291,12 +305,12 @@ void main() { {'id': 'O1', 'description': 'B', 'make': null} ])); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [remove5]), ])); await syncLocalChecked(Checkpoint(lastOpId: '5', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 13), + checksum(bucket: 'bucket1', checksum: 13), ])); await expectAsset1_3(); @@ -304,15 +318,15 @@ void main() { test('should fail checksum validation', () async { // Simple checksum validation - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3]), ])); var result = await bucketStorage .syncLocalDatabase(Checkpoint(lastOpId: '3', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 10), - BucketChecksum(bucket: 'bucket2', checksum: 1) + checksum(bucket: 'bucket1', checksum: 10), + checksum(bucket: 'bucket2', checksum: 1) ])); expect( result, @@ -325,7 +339,7 @@ void main() { }); test('should delete buckets', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_3], @@ -340,7 +354,7 @@ void main() { // The delete only takes effect after syncLocal. await syncLocalChecked(Checkpoint(lastOpId: '3', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 3), + checksum(bucket: 'bucket1', checksum: 3), ])); // Bucket is deleted, but object is still present in other buckets. @@ -354,7 +368,7 @@ void main() { test('should delete and re-create buckets', () async { // Save some data - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1], @@ -365,7 +379,7 @@ void main() { await bucketStorage.removeBuckets(['bucket1']); // Save some data again - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset1_3], @@ -375,7 +389,7 @@ void main() { await bucketStorage.removeBuckets(['bucket1']); // Final save of data - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset1_3], @@ -384,7 +398,7 @@ void main() { // Check that the data is there await syncLocalChecked(Checkpoint(lastOpId: '3', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 4), + checksum(bucket: 'bucket1', checksum: 4), ])); await expectAsset1_3(); @@ -395,14 +409,14 @@ void main() { }); test('should handle MOVE', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [OplogEntry(opId: '1', op: OpType.move, checksum: 1)], ), ])); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_3], @@ -411,14 +425,14 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 4)])); + checksums: [checksum(bucket: 'bucket1', checksum: 4)])); await expectAsset1_3(); }); test('should handle CLEAR', () async { // Save some data - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1], @@ -427,10 +441,10 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '1', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 1)])); + checksums: [checksum(bucket: 'bucket1', checksum: 1)])); // CLEAR, then save new data - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [ @@ -449,7 +463,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', // 2 + 3. 1 is replaced with 2. - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 5)])); + checksums: [checksum(bucket: 'bucket1', checksum: 5)])); await expectNoAsset1(); expect( @@ -472,7 +486,7 @@ void main() { await powersync.initialize(); bucketStorage = BucketStorage(powersync); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -481,7 +495,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); await expectLater(() async { await powersync.execute('SELECT * FROM assets'); @@ -499,7 +513,7 @@ void main() { }); test('should remove types', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -508,7 +522,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); await expectAsset1_3(); @@ -537,7 +551,7 @@ void main() { // Test compacting behaviour. // This test relies heavily on internals, and will have to be updated when the compact implementation is updated. - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, removeAsset1_4]) ])); @@ -545,14 +559,14 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '4', writeCheckpoint: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 7)])); + checksums: [checksum(bucket: 'bucket1', checksum: 7)])); await bucketStorage.forceCompact(); await syncLocalChecked(Checkpoint( lastOpId: '4', writeCheckpoint: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 7)])); + checksums: [checksum(bucket: 'bucket1', checksum: 7)])); final stats = await powersync.execute( 'SELECT row_type as type, row_id as id, count(*) as count FROM ps_oplog GROUP BY row_type, row_id ORDER BY row_type, row_id'); @@ -564,7 +578,7 @@ void main() { }); test('should compact with checksum wrapping', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [ OplogEntry( opId: '1', @@ -593,18 +607,14 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '4', writeCheckpoint: '4', - checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 2147483642) - ])); + checksums: [checksum(bucket: 'bucket1', checksum: 2147483642)])); await bucketStorage.forceCompact(); await syncLocalChecked(Checkpoint( lastOpId: '4', writeCheckpoint: '4', - checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 2147483642) - ])); + checksums: [checksum(bucket: 'bucket1', checksum: 2147483642)])); final stats = await powersync.execute( 'SELECT row_type as type, row_id as id, count(*) as count FROM ps_oplog GROUP BY row_type, row_id ORDER BY row_type, row_id'); @@ -616,7 +626,7 @@ void main() { }); test('should compact with checksum wrapping (2)', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [ OplogEntry( opId: '1', @@ -638,14 +648,14 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '4', writeCheckpoint: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: -3)])); + checksums: [checksum(bucket: 'bucket1', checksum: -3)])); await bucketStorage.forceCompact(); await syncLocalChecked(Checkpoint( lastOpId: '4', writeCheckpoint: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: -3)])); + checksums: [checksum(bucket: 'bucket1', checksum: -3)])); final stats = await powersync.execute( 'SELECT row_type as type, row_id as id, count(*) as count FROM ps_oplog GROUP BY row_type, row_id ORDER BY row_type, row_id'); @@ -658,7 +668,7 @@ void main() { test('should not sync local db with pending crud - server removed', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -667,7 +677,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Local save powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); @@ -681,7 +691,7 @@ void main() { final result = await bucketStorage.syncLocalDatabase(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect(result, equals(SyncLocalDatabaseResult(ready: false))); final batch = await bucketStorage.getCrudBatch(); @@ -694,7 +704,7 @@ void main() { final result3 = await bucketStorage.syncLocalDatabase(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect(result3, equals(SyncLocalDatabaseResult(ready: false))); // The data must still be present locally. @@ -705,14 +715,14 @@ void main() { ])); await bucketStorage.saveSyncData( - SyncDataBatch([SyncBucketData(bucket: 'bucket1', data: [])])); + syncDataBatch([SyncBucketData(bucket: 'bucket1', data: [])])); // Now we have synced the data back (or lack of data in this case), // so we can do a local sync. await syncLocalChecked(Checkpoint( lastOpId: '5', writeCheckpoint: '5', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Since the object was not in the sync response, it is deleted. expect(await powersync.execute('SELECT id FROM assets WHERE id = \'O3\''), @@ -722,7 +732,7 @@ void main() { test( 'should not sync local db with pending crud when more crud is added (1)', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -732,7 +742,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Local save powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); @@ -746,11 +756,11 @@ void main() { final result3 = await bucketStorage.syncLocalDatabase(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect(result3, equals(SyncLocalDatabaseResult(ready: false))); await bucketStorage.saveSyncData( - SyncDataBatch([SyncBucketData(bucket: 'bucket1', data: [])])); + syncDataBatch([SyncBucketData(bucket: 'bucket1', data: [])])); // Add more data before syncLocalDatabase. powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O4']); @@ -758,14 +768,14 @@ void main() { final result4 = await bucketStorage.syncLocalDatabase(Checkpoint( lastOpId: '5', writeCheckpoint: '5', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect(result4, equals(SyncLocalDatabaseResult(ready: false))); }); test( 'should not sync local db with pending crud when more crud is added (2)', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -775,7 +785,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Local save await powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); @@ -788,7 +798,7 @@ void main() { return '4'; }); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [], @@ -798,13 +808,13 @@ void main() { final result4 = await bucketStorage.syncLocalDatabase(Checkpoint( lastOpId: '5', writeCheckpoint: '5', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect(result4, equals(SyncLocalDatabaseResult(ready: false))); }); test('should not sync local db with pending crud - update on server', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -814,7 +824,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Local save powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); @@ -824,7 +834,7 @@ void main() { return '4'; }); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [ @@ -842,7 +852,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '5', writeCheckpoint: '5', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 11)])); + checksums: [checksum(bucket: 'bucket1', checksum: 11)])); expect( await powersync @@ -853,7 +863,7 @@ void main() { }); test('should revert a failing insert', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -863,7 +873,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Local insert, later rejected by server await powersync.execute( @@ -885,7 +895,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect( await powersync @@ -894,7 +904,7 @@ void main() { }); test('should revert a failing delete', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -904,7 +914,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Local delete, later rejected by server await powersync.execute('DELETE FROM assets WHERE id = ?', ['O2']); @@ -924,7 +934,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect( await powersync @@ -935,7 +945,7 @@ void main() { }); test('should revert a failing update', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -945,7 +955,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Local update, later rejected by server await powersync.execute( @@ -968,7 +978,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect( await powersync From d23876982c1c4ae9e92405ee60e6a384248a586d Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 3 Feb 2025 14:20:55 +0100 Subject: [PATCH 06/25] Validate partial checksums --- demos/django-todolist/.gitignore | 2 ++ demos/django-todolist/macos/Podfile.lock | 16 +++++----- .../macos/Runner/AppDelegate.swift | 6 +++- .../macos/Runner/Configs/AppInfo.xcconfig | 2 +- demos/django-todolist/pubspec.lock | 32 +++++++------------ .../lib/src/bucket_storage.dart | 15 ++++++--- .../powersync_core/lib/src/sync_types.dart | 3 ++ 7 files changed, 41 insertions(+), 35 deletions(-) diff --git a/demos/django-todolist/.gitignore b/demos/django-todolist/.gitignore index 0b04140a..777f9a68 100644 --- a/demos/django-todolist/.gitignore +++ b/demos/django-todolist/.gitignore @@ -5,9 +5,11 @@ *.swp .DS_Store .atom/ +.build/ .buildlog/ .history .svn/ +.swiftpm/ migrate_working_dir/ # IntelliJ related diff --git a/demos/django-todolist/macos/Podfile.lock b/demos/django-todolist/macos/Podfile.lock index 67953800..e129448a 100644 --- a/demos/django-todolist/macos/Podfile.lock +++ b/demos/django-todolist/macos/Podfile.lock @@ -3,10 +3,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.1.6) + - powersync-sqlite-core (0.3.9) - powersync_flutter_libs (0.0.1): - FlutterMacOS - - powersync-sqlite-core (~> 0.1.6) + - powersync-sqlite-core (~> 0.3.8) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -55,13 +55,13 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: FlutterMacOS: 8f6f14fa908a6fb3fba0cd85dbd81ec4b251fb24 - path_provider_foundation: 2b6b4c569c0fb62ec74538f866245ac84301af46 - powersync-sqlite-core: 4c38c8f470f6dca61346789fd5436a6826d1e3dd - powersync_flutter_libs: 1eb1c6790a72afe08e68d4cc489d71ab61da32ee - shared_preferences_foundation: fcdcbc04712aee1108ac7fda236f363274528f78 + path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 + powersync-sqlite-core: 7515d321eb8e3c08b5259cdadb9d19b1876fe13a + powersync_flutter_libs: 330d8309223a121ec15a7334d9edc105053e5f82 + shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 292c3e1bfe89f64e51ea7fc7dab9182a017c8630 - sqlite3_flutter_libs: 5ca46c1a04eddfbeeb5b16566164aa7ad1616e7b + sqlite3_flutter_libs: 03311aede9d32fb2d24e32bebb8cd01c3b2e6239 PODFILE CHECKSUM: 236401fc2c932af29a9fcf0e97baeeb2d750d367 -COCOAPODS: 1.15.2 +COCOAPODS: 1.16.2 diff --git a/demos/django-todolist/macos/Runner/AppDelegate.swift b/demos/django-todolist/macos/Runner/AppDelegate.swift index d53ef643..b3c17614 100644 --- a/demos/django-todolist/macos/Runner/AppDelegate.swift +++ b/demos/django-todolist/macos/Runner/AppDelegate.swift @@ -1,9 +1,13 @@ import Cocoa import FlutterMacOS -@NSApplicationMain +@main class AppDelegate: FlutterAppDelegate { override func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { return true } + + override func applicationSupportsSecureRestorableState(_ app: NSApplication) -> Bool { + return true + } } diff --git a/demos/django-todolist/macos/Runner/Configs/AppInfo.xcconfig b/demos/django-todolist/macos/Runner/Configs/AppInfo.xcconfig index 797d44b3..d00b6f29 100644 --- a/demos/django-todolist/macos/Runner/Configs/AppInfo.xcconfig +++ b/demos/django-todolist/macos/Runner/Configs/AppInfo.xcconfig @@ -8,7 +8,7 @@ PRODUCT_NAME = PowerSync Django Demo // The application's bundle identifier -PRODUCT_BUNDLE_IDENTIFIER = co.powersync.demotodolist +PRODUCT_BUNDLE_IDENTIFIER = co.powersync.demotodolist.django // The copyright displayed in application information PRODUCT_COPYRIGHT = Copyright © 2023 Journey Mobile, Inc. All rights reserved. diff --git a/demos/django-todolist/pubspec.lock b/demos/django-todolist/pubspec.lock index 5ae7bd20..444c7db5 100644 --- a/demos/django-todolist/pubspec.lock +++ b/demos/django-todolist/pubspec.lock @@ -5,10 +5,10 @@ packages: dependency: transitive description: name: args - sha256: "7cf60b9f0cc88203c5a190b4cd62a99feea42759a7fa695010eb5de1c0b2252a" + sha256: bf9f5caeea8d8fe6721a9c358dd8a5c1947b27f1cfaa18b39c301273594919e6 url: "https://pub.dev" source: hosted - version: "2.5.0" + version: "2.6.0" async: dependency: transitive description: @@ -152,14 +152,6 @@ packages: url: "https://pub.dev" source: hosted version: "4.0.2" - js: - dependency: transitive - description: - name: js - sha256: c1b2e9b5ea78c45e1a0788d29606ba27dc5f71f019f32ca5140f61ef071838cf - url: "https://pub.dev" - source: hosted - version: "0.7.1" json_annotation: dependency: transitive description: @@ -318,21 +310,21 @@ packages: path: "../../packages/powersync" relative: true source: path - version: "1.10.0" + version: "1.11.2" powersync_core: dependency: "direct overridden" description: path: "../../packages/powersync_core" relative: true source: path - version: "1.0.0" + version: "1.1.2" powersync_flutter_libs: dependency: "direct overridden" description: path: "../../packages/powersync_flutter_libs" relative: true source: path - version: "0.4.3" + version: "0.4.4" pub_semver: dependency: transitive description: @@ -430,10 +422,10 @@ packages: dependency: transitive description: name: sqlite3 - sha256: bb174b3ec2527f9c5f680f73a89af8149dd99782fbb56ea88ad0807c5638f2ed + sha256: "35d3726fe18ab1463403a5cc8d97dbc81f2a0b08082e8173851363fcc97b6627" url: "https://pub.dev" source: hosted - version: "2.4.7" + version: "2.7.2" sqlite3_flutter_libs: dependency: transitive description: @@ -446,18 +438,18 @@ packages: dependency: transitive description: name: sqlite3_web - sha256: f22d1dda7a40be0867984f55cdf5c2d599e5f05d3be4a642d78f38b38983f554 + sha256: "870f287c2375117af1f769893c5ea0941882ee820444af5c3dcceec3b217aab1" url: "https://pub.dev" source: hosted - version: "0.2.0" + version: "0.3.0" sqlite_async: dependency: "direct main" description: name: sqlite_async - sha256: d66fb6e6d07c1a834743326c033029f75becbb1fad6823d709f921872abc3d5b + sha256: "70836e6a5dae2d14ce585934d1d6ce22433c1be825e78908a4b2a18333f7073c" url: "https://pub.dev" source: hosted - version: "0.11.0" + version: "0.11.2" stack_trace: dependency: transitive description: @@ -563,5 +555,5 @@ packages: source: hosted version: "3.1.2" sdks: - dart: ">=3.4.0 <4.0.0" + dart: ">=3.5.0 <4.0.0" flutter: ">=3.22.0" diff --git a/packages/powersync_core/lib/src/bucket_storage.dart b/packages/powersync_core/lib/src/bucket_storage.dart index e77257e7..0d04724f 100644 --- a/packages/powersync_core/lib/src/bucket_storage.dart +++ b/packages/powersync_core/lib/src/bucket_storage.dart @@ -114,7 +114,7 @@ class BucketStorage { Future syncLocalDatabase(Checkpoint checkpoint, {int? forPriority}) async { - final r = await validateChecksums(checkpoint); + final r = await validateChecksums(checkpoint, priority: forPriority); if (!r.checkpointValid) { for (String b in r.checkpointFailures ?? []) { @@ -165,10 +165,15 @@ class BucketStorage { }, flush: true); } - Future validateChecksums( - Checkpoint checkpoint) async { - final rs = await select("SELECT powersync_validate_checkpoint(?) as result", - [jsonEncode(checkpoint)]); + Future validateChecksums(Checkpoint checkpoint, + {int? priority}) async { + final rs = + await select("SELECT powersync_validate_checkpoint(?) as result", [ + jsonEncode({ + ...checkpoint.toJson(), + if (priority != null) 'priority': priority, + }) + ]); final result = jsonDecode(rs[0]['result']); if (result['valid']) { return SyncLocalDatabaseResult(ready: true); diff --git a/packages/powersync_core/lib/src/sync_types.dart b/packages/powersync_core/lib/src/sync_types.dart index e9f436b0..7b08dcc1 100644 --- a/packages/powersync_core/lib/src/sync_types.dart +++ b/packages/powersync_core/lib/src/sync_types.dart @@ -13,6 +13,9 @@ sealed class StreamingSyncLine { } else if (line.containsKey('checkpoint_complete')) { return StreamingSyncCheckpointComplete.fromJson( line['checkpoint_complete']); + } else if (line.containsKey('partial_checkpoint_complete')) { + return StreamingSyncCheckpointPartiallyComplete.fromJson( + line['partial_checkpoint_complete']); } else if (line.containsKey('data')) { return SyncBucketData.fromJson(line['data']); } else if (line.containsKey('token_expires_in')) { From 1bd7447b46d82a93572897ef5d7f4c3e4f6ca2b0 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 3 Feb 2025 16:34:02 +0100 Subject: [PATCH 07/25] Use updated format not persisting priorities --- .../lib/src/bucket_storage.dart | 50 +++++++++---------- .../lib/src/streaming_sync.dart | 10 ++-- .../test/bucket_storage_test.dart | 11 ++-- 3 files changed, 31 insertions(+), 40 deletions(-) diff --git a/packages/powersync_core/lib/src/bucket_storage.dart b/packages/powersync_core/lib/src/bucket_storage.dart index 0d04724f..cdf2c58a 100644 --- a/packages/powersync_core/lib/src/bucket_storage.dart +++ b/packages/powersync_core/lib/src/bucket_storage.dart @@ -33,12 +33,11 @@ class BucketStorage { Future> getBucketStates() async { final rows = await select( - 'SELECT name as bucket, priority, cast(last_op as TEXT) as op_id FROM ps_buckets WHERE pending_delete = 0 AND name != \'\$local\''); + 'SELECT name as bucket, cast(last_op as TEXT) as op_id FROM ps_buckets WHERE pending_delete = 0 AND name != \'\$local\''); return [ for (var row in rows) BucketState( bucket: row['bucket'], - priority: row['priority'], opId: row['op_id'], ) ]; @@ -53,20 +52,12 @@ class BucketStorage { var count = 0; await writeTransaction((tx) async { - final descriptions = [ - for (final MapEntry(:key, :value) in batch.descriptions.entries) - { - key: {'priority': value.priority}, - } - ]; - for (var b in batch.buckets) { count += b.data.length; await _updateBucket2( tx, jsonEncode({ 'buckets': [b], - 'descriptions': descriptions, })); } // No need to flush - the data is not directly visible to the user either way. @@ -136,7 +127,8 @@ class BucketStorage { // Not flushing here - the flush will happen in the next step }, flush: false); - final valid = await updateObjectsFromBuckets(forPriority: forPriority); + final valid = await updateObjectsFromBuckets(checkpoint, + forPartialPriority: forPriority); if (!valid) { return SyncLocalDatabaseResult(ready: false); } @@ -146,11 +138,25 @@ class BucketStorage { return SyncLocalDatabaseResult(ready: true); } - Future updateObjectsFromBuckets({int? forPriority}) async { + Future updateObjectsFromBuckets(Checkpoint checkpoint, + {int? forPartialPriority}) async { return writeTransaction((tx) async { - await tx.execute( - "INSERT INTO powersync_operations(op, data) VALUES(?, ?)", - ['sync_local', forPriority]); + await tx + .execute("INSERT INTO powersync_operations(op, data) VALUES(?, ?)", [ + 'sync_local', + forPartialPriority != null + ? jsonEncode({ + 'priority': forPartialPriority, + // If we're at a partial checkpoint, we should only publish the + // buckets at the completed priority levels. + 'buckets': [ + for (final desc in checkpoint.checksums) + // Note that higher priorities are encoded as smaller values + if (desc.priority <= forPartialPriority) desc.bucket, + ], + }) + : null, + ]); final rs = await tx.execute('SELECT last_insert_rowid() as result'); final result = rs[0]['result']; if (result == 1) { @@ -338,11 +344,9 @@ class BucketStorage { class BucketState { final String bucket; - final int priority; final String opId; - const BucketState( - {required this.bucket, required this.priority, required this.opId}); + const BucketState({required this.bucket, required this.opId}); @override String toString() { @@ -351,23 +355,19 @@ class BucketState { @override int get hashCode { - return Object.hash(bucket, priority, opId); + return Object.hash(bucket, opId); } @override bool operator ==(Object other) { - return other is BucketState && - other.priority == priority && - other.bucket == bucket && - other.opId == opId; + return other is BucketState && other.bucket == bucket && other.opId == opId; } } final class SyncDataBatch { final List buckets; - final Map descriptions; - SyncDataBatch(this.buckets, this.descriptions); + SyncDataBatch(this.buckets); } class SyncLocalDatabaseResult { diff --git a/packages/powersync_core/lib/src/streaming_sync.dart b/packages/powersync_core/lib/src/streaming_sync.dart index a8bdf586..80aeb10a 100644 --- a/packages/powersync_core/lib/src/streaming_sync.dart +++ b/packages/powersync_core/lib/src/streaming_sync.dart @@ -302,7 +302,7 @@ class StreamingSyncImplementation implements StreamingSync { _statusStreamController.add(newStatus); } - Future<(List, Map)> + Future<(List, Map)> _collectLocalBucketState() async { final bucketEntries = await adapter.getBucketStates(); @@ -310,11 +310,7 @@ class StreamingSyncImplementation implements StreamingSync { for (final entry in bucketEntries) BucketRequest(entry.bucket, entry.opId) ]; final localDescriptions = { - for (final entry in bucketEntries) - entry.bucket: ( - name: entry.bucket, - priority: entry.priority, - ) + for (final entry in bucketEntries) entry.bucket: null }; return (initialRequests, localDescriptions); @@ -435,7 +431,7 @@ class StreamingSyncImplementation implements StreamingSync { case SyncBucketData(): // TODO: Merge multiple of these into a single one... _updateStatus(downloading: true); - await adapter.saveSyncData(SyncDataBatch([line], bucketMap)); + await adapter.saveSyncData(SyncDataBatch([line])); case StreamingSyncKeepalive(): if (line.tokenExpiresIn == 0) { // Token expired already - stop the connection immediately diff --git a/packages/powersync_core/test/bucket_storage_test.dart b/packages/powersync_core/test/bucket_storage_test.dart index f1a1f34a..0dee79e4 100644 --- a/packages/powersync_core/test/bucket_storage_test.dart +++ b/packages/powersync_core/test/bucket_storage_test.dart @@ -45,10 +45,7 @@ BucketChecksum checksum( } SyncDataBatch syncDataBatch(List data) { - return SyncDataBatch(data, { - for (final bucket in data.map((e) => e.bucket).toSet()) - bucket: (name: bucket, priority: 1), - }); + return SyncDataBatch(data); } void main() { @@ -108,10 +105,8 @@ void main() { ])); final bucketStates = await bucketStorage.getBucketStates(); - expect( - bucketStates, - equals( - [const BucketState(bucket: 'bucket1', opId: '3', priority: 1)])); + expect(bucketStates, + equals([const BucketState(bucket: 'bucket1', opId: '3')])); await syncLocalChecked(Checkpoint( lastOpId: '3', From f423ffdb3eab8af23bdcc4b2503491d7c7c0aea6 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 3 Feb 2025 16:36:02 +0100 Subject: [PATCH 08/25] Start with status API --- .../powersync_core/lib/src/sync_status.dart | 49 ++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/packages/powersync_core/lib/src/sync_status.dart b/packages/powersync_core/lib/src/sync_status.dart index f04e7300..d9546c4a 100644 --- a/packages/powersync_core/lib/src/sync_status.dart +++ b/packages/powersync_core/lib/src/sync_status.dart @@ -22,11 +22,11 @@ class SyncStatus { /// Time that a last sync has fully completed, if any. /// /// This is null while loading the database. - final DateTime? lastSyncedAt; + DateTime? get lastSyncedAt => statusInPriority.lastOrNull?.lastSyncedAt; /// Indicates whether there has been at least one full sync, if any. /// Is null when unknown, for example when state is still being loaded from the database. - final bool? hasSynced; + bool? get hasSynced => statusInPriority.lastOrNull?.hasSynced; /// Error during uploading. /// @@ -38,15 +38,19 @@ class SyncStatus { /// Cleared on the next successful data download. final Object? downloadError; - const SyncStatus( - {this.connected = false, - this.connecting = false, - this.lastSyncedAt, - this.hasSynced, - this.downloading = false, - this.uploading = false, - this.downloadError, - this.uploadError}); + final List statusInPriority; + + const SyncStatus({ + this.connected = false, + this.connecting = false, + this.lastSyncedAt, + this.hasSynced, + this.downloading = false, + this.uploading = false, + this.downloadError, + this.uploadError, + this.statusInPriority = const [], + }); @override bool operator ==(Object other) { @@ -100,6 +104,29 @@ class SyncStatus { } } +/// The priority of a PowerSync bucket. +extension type const BucketPriority._(int priorityNumber) { + static const _highest = 0; + static const _lowests = 3; + + factory BucketPriority(int i) { + assert(i >= _highest && i <= _lowests); + return BucketPriority._(i); + } + + /// A [Comparator] instance suitable for comparing [BucketPriority] values. + static Comparator comparator = + (a, b) => -a.priorityNumber.compareTo(b.priorityNumber); +} + +/// Partial information about the synchronization status for buckets within a +/// priority. +typedef SyncPriorityStatus = ({ + BucketPriority priority, + DateTime lastSyncedAt, + bool hasSynced, +}); + /// Stats of the local upload queue. class UploadQueueStats { /// Number of records in the upload queue. From c99c0431dd4b700ec9dd24bee9a95e7fba11ae3e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 4 Feb 2025 11:45:22 +0100 Subject: [PATCH 09/25] Add status for each priority --- .../lib/src/database/powersync_db_mixin.dart | 22 +++- .../lib/src/streaming_sync.dart | 106 +++++++++++++----- .../powersync_core/lib/src/sync_status.dart | 51 +++++++-- .../lib/src/web/sync_worker_protocol.dart | 22 ++++ 4 files changed, 157 insertions(+), 44 deletions(-) diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index a5fb93a5..8869c162 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -135,13 +135,27 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { } } - /// Returns a [Future] which will resolve once the first full sync has completed. - Future waitForFirstSync() async { - if (currentStatus.hasSynced ?? false) { + /// Returns a [Future] which will resolve once a synchronization operation has + /// completed. + /// + /// When [priority] is null (the default), this method waits for a full sync + /// operation to complete. When set to a [BucketPriority] however, it also + /// completes once a partial sync operation containing that priority has + /// completed. + Future waitForFirstSync({BucketPriority? priority}) async { + bool matches(SyncStatus status) { + if (priority == null) { + return status.hasSynced == true; + } else { + return status.statusForPriority(priority)?.hasSynced == true; + } + } + + if (matches(currentStatus)) { return; } await for (final result in statusStream) { - if (result.hasSynced ?? false) { + if (matches(result)) { break; } } diff --git a/packages/powersync_core/lib/src/streaming_sync.dart b/packages/powersync_core/lib/src/streaming_sync.dart index 80aeb10a..b4b6f214 100644 --- a/packages/powersync_core/lib/src/streaming_sync.dart +++ b/packages/powersync_core/lib/src/streaming_sync.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'dart:convert' as convert; +import 'package:collection/collection.dart'; import 'package:http/http.dart' as http; import 'package:powersync_core/src/abort_controller.dart'; import 'package:powersync_core/src/exceptions.dart'; @@ -273,31 +274,62 @@ class StreamingSyncImplementation implements StreamingSync { return body['data']['write_checkpoint'] as String; } + void _updateStatusForPriority(SyncPriorityStatus completed) { + // Note: statusInPriority is sorted by priorities (ascending) + final existingPriorityState = lastStatus.statusInPriority; + + for (final (i, priority) in existingPriorityState.indexed) { + switch ( + BucketPriority.comparator(priority.priority, completed.priority)) { + case < 0: + // Entries from here on have a higher priority than the one that was + // just completed + final copy = existingPriorityState.toList(); + copy.insert(i, completed); + _updateStatus(statusInPriority: copy); + return; + case 0: + final copy = existingPriorityState.toList(); + copy[i] = completed; + _updateStatus(statusInPriority: copy); + return; + case > 0: + continue; + } + } + + _updateStatus(statusInPriority: [...existingPriorityState, completed]); + } + /// Update sync status based on any non-null parameters. /// To clear errors, use [_noError] instead of null. - void _updateStatus( - {DateTime? lastSyncedAt, - bool? hasSynced, - bool? connected, - bool? connecting, - bool? downloading, - bool? uploading, - Object? uploadError, - Object? downloadError}) { + void _updateStatus({ + DateTime? lastSyncedAt, + bool? hasSynced, + bool? connected, + bool? connecting, + bool? downloading, + bool? uploading, + Object? uploadError, + Object? downloadError, + List? statusInPriority, + }) { final c = connected ?? lastStatus.connected; var newStatus = SyncStatus( - connected: c, - connecting: !c && (connecting ?? lastStatus.connecting), - lastSyncedAt: lastSyncedAt ?? lastStatus.lastSyncedAt, - hasSynced: hasSynced ?? lastStatus.hasSynced, - downloading: downloading ?? lastStatus.downloading, - uploading: uploading ?? lastStatus.uploading, - uploadError: uploadError == _noError - ? null - : (uploadError ?? lastStatus.uploadError), - downloadError: downloadError == _noError - ? null - : (downloadError ?? lastStatus.downloadError)); + connected: c, + connecting: !c && (connecting ?? lastStatus.connecting), + lastSyncedAt: lastSyncedAt ?? lastStatus.lastSyncedAt, + hasSynced: hasSynced ?? lastStatus.hasSynced, + downloading: downloading ?? lastStatus.downloading, + uploading: uploading ?? lastStatus.uploading, + uploadError: uploadError == _noError + ? null + : (uploadError ?? lastStatus.uploadError), + downloadError: downloadError == _noError + ? null + : (downloadError ?? lastStatus.downloadError), + statusInPriority: statusInPriority ?? lastStatus.statusInPriority, + ); lastStatus = newStatus; _statusStreamController.add(newStatus); } @@ -371,10 +403,25 @@ class StreamingSyncImplementation implements StreamingSync { } else { appliedCheckpoint = targetCheckpoint; + final now = DateTime.now(); _updateStatus( - downloading: false, - downloadError: _noError, - lastSyncedAt: DateTime.now()); + downloading: false, + downloadError: _noError, + lastSyncedAt: now, + statusInPriority: [ + if (appliedCheckpoint.checksums.isNotEmpty) + ( + hasSynced: true, + lastSyncedAt: now, + priority: maxBy( + appliedCheckpoint.checksums + .map((cs) => BucketPriority(cs.priority)), + (priority) => priority, + compare: BucketPriority.comparator, + )!, + ) + ], + ); } validatedCheckpoint = targetCheckpoint; @@ -390,12 +437,11 @@ class StreamingSyncImplementation implements StreamingSync { // Checksums valid, but need more data for a consistent checkpoint. // Continue waiting. } else { - appliedCheckpoint = targetCheckpoint; - - _updateStatus( - downloading: false, - downloadError: _noError, - lastSyncedAt: DateTime.now()); + _updateStatusForPriority(( + priority: BucketPriority(bucketPriority), + lastSyncedAt: DateTime.now(), + hasSynced: true, + )); } validatedCheckpoint = targetCheckpoint; diff --git a/packages/powersync_core/lib/src/sync_status.dart b/packages/powersync_core/lib/src/sync_status.dart index d9546c4a..799e69bb 100644 --- a/packages/powersync_core/lib/src/sync_status.dart +++ b/packages/powersync_core/lib/src/sync_status.dart @@ -1,4 +1,6 @@ -class SyncStatus { +import 'package:collection/collection.dart'; + +final class SyncStatus { /// true if currently connected. /// /// This means the PowerSync connection is ready to download, and @@ -22,11 +24,11 @@ class SyncStatus { /// Time that a last sync has fully completed, if any. /// /// This is null while loading the database. - DateTime? get lastSyncedAt => statusInPriority.lastOrNull?.lastSyncedAt; + final DateTime? lastSyncedAt; /// Indicates whether there has been at least one full sync, if any. /// Is null when unknown, for example when state is still being loaded from the database. - bool? get hasSynced => statusInPriority.lastOrNull?.hasSynced; + final bool? hasSynced; /// Error during uploading. /// @@ -62,7 +64,8 @@ class SyncStatus { other.downloadError == downloadError && other.uploadError == uploadError && other.lastSyncedAt == lastSyncedAt && - other.hasSynced == hasSynced); + other.hasSynced == hasSynced && + _statusEquality.equals(other.statusInPriority, statusInPriority)); } SyncStatus copyWith({ @@ -74,6 +77,7 @@ class SyncStatus { Object? downloadError, DateTime? lastSyncedAt, bool? hasSynced, + List? statusInPriority, }) { return SyncStatus( connected: connected ?? this.connected, @@ -84,6 +88,7 @@ class SyncStatus { downloadError: downloadError ?? this.downloadError, lastSyncedAt: lastSyncedAt ?? this.lastSyncedAt, hasSynced: hasSynced ?? this.hasSynced, + statusInPriority: statusInPriority ?? this.statusInPriority, ); } @@ -92,31 +97,57 @@ class SyncStatus { return downloadError ?? uploadError; } + /// Returns [lastSyncedAt] and [hasSynced] information for a partial sync + /// operation, or `null` if the status for that priority is unknown. + SyncPriorityStatus? statusForPriority(BucketPriority priority) { + assert(statusInPriority.isSortedByCompare( + (e) => e.priority, BucketPriority.comparator)); + + for (final known in statusInPriority) { + // Lower-priority buckets are synchronized after higher-priority buckets, + // and since statusInPriority is sorted we look for the first entry that + // doesn't have a higher priority. + if (BucketPriority.comparator(known.priority, priority) <= 0) { + return known; + } + } + + return null; + } + @override int get hashCode { - return Object.hash(connected, downloading, uploading, connecting, - uploadError, downloadError, lastSyncedAt); + return Object.hash( + connected, + downloading, + uploading, + connecting, + uploadError, + downloadError, + lastSyncedAt, + _statusEquality.hash(statusInPriority)); } @override String toString() { return "SyncStatus"; } + + static const _statusEquality = ListEquality(); } /// The priority of a PowerSync bucket. extension type const BucketPriority._(int priorityNumber) { static const _highest = 0; - static const _lowests = 3; factory BucketPriority(int i) { - assert(i >= _highest && i <= _lowests); + assert(i >= _highest); return BucketPriority._(i); } /// A [Comparator] instance suitable for comparing [BucketPriority] values. - static Comparator comparator = - (a, b) => -a.priorityNumber.compareTo(b.priorityNumber); + static int comparator(BucketPriority a, BucketPriority b) => + -a.priorityNumber.compareTo(b.priorityNumber); } /// Partial information about the synchronization status for buckets within a diff --git a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart index 9d8b8484..4713b392 100644 --- a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart +++ b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart @@ -157,6 +157,7 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { required bool? hasSyned, required String? uploadError, required String? downloadError, + required JSArray? statusInPriority, }); factory SerializedSyncStatus.from(SyncStatus status) { @@ -169,6 +170,14 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { hasSyned: status.hasSynced, uploadError: status.uploadError?.toString(), downloadError: status.downloadError?.toString(), + statusInPriority: [ + for (final entry in status.statusInPriority) + [ + entry.priority.priorityNumber.toJS, + entry.lastSyncedAt.microsecondsSinceEpoch.toJS, + entry.hasSynced.toJS, + ].toJS + ].toJS, ); } @@ -180,6 +189,7 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { external bool? hasSynced; external String? uploadError; external String? downloadError; + external JSArray? statusInPriority; SyncStatus asSyncStatus() { return SyncStatus( @@ -193,6 +203,18 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { hasSynced: hasSynced, uploadError: uploadError, downloadError: downloadError, + statusInPriority: statusInPriority?.toDart.map((e) { + final [rawPriority, rawSynced, rawHasSynced, ...] = + (e as JSArray).toDart; + + return ( + priority: BucketPriority((rawPriority as JSNumber).toDartInt), + lastSyncedAt: DateTime.fromMicrosecondsSinceEpoch( + (rawSynced as JSNumber).toDartInt), + hasSynced: (rawHasSynced as JSBoolean).toDart, + ); + }).toList() ?? + const [], ); } } From 7ed6945b2a55285e7aa1436ed386ea2cbc471455 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 4 Feb 2025 15:54:49 +0100 Subject: [PATCH 10/25] Only verify relevant buckets --- packages/powersync_core/lib/src/bucket_storage.dart | 2 +- packages/powersync_core/lib/src/sync_types.dart | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/powersync_core/lib/src/bucket_storage.dart b/packages/powersync_core/lib/src/bucket_storage.dart index cdf2c58a..f36c8ff5 100644 --- a/packages/powersync_core/lib/src/bucket_storage.dart +++ b/packages/powersync_core/lib/src/bucket_storage.dart @@ -176,7 +176,7 @@ class BucketStorage { final rs = await select("SELECT powersync_validate_checkpoint(?) as result", [ jsonEncode({ - ...checkpoint.toJson(), + ...checkpoint.toJson(priority: priority), if (priority != null) 'priority': priority, }) ]); diff --git a/packages/powersync_core/lib/src/sync_types.dart b/packages/powersync_core/lib/src/sync_types.dart index 7b08dcc1..842f44ab 100644 --- a/packages/powersync_core/lib/src/sync_types.dart +++ b/packages/powersync_core/lib/src/sync_types.dart @@ -52,11 +52,12 @@ final class Checkpoint extends StreamingSyncLine { .map((b) => BucketChecksum.fromJson(b)) .toList(); - Map toJson() { + Map toJson({int? priority}) { return { 'last_op_id': lastOpId, 'write_checkpoint': writeCheckpoint, 'buckets': checksums + .where((c) => priority == null || c.priority <= priority) .map((c) => {'bucket': c.bucket, 'checksum': c.checksum}) .toList(growable: false) }; From 793e01e06037b5cf0d6aa9e99da2cce57c7b1df0 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 5 Feb 2025 14:39:03 +0100 Subject: [PATCH 11/25] Tests for partial sync operations --- .../lib/src/database/powersync_db_mixin.dart | 67 +++++- .../lib/src/streaming_sync.dart | 87 ++++---- .../powersync_core/lib/src/sync_status.dart | 21 +- .../powersync_core/lib/src/sync_types.dart | 6 +- .../powersync_core/test/connected_test.dart | 1 - .../powersync_core/test/disconnect_test.dart | 2 +- .../test/in_memory_sync_test.dart | 209 ++++++++++++++++++ .../sync_server/in_memory_sync_server.dart | 64 ++++++ .../server/sync_server/mock_sync_server.dart | 37 +--- .../test/streaming_sync_test.dart | 20 +- .../test/utils/abstract_test_utils.dart | 64 +++++- .../test/utils/in_memory_http.dart | 56 +++++ .../test/utils/native_test_utils.dart | 34 ++- .../test/utils/stub_test_utils.dart | 10 + .../test/utils/web_test_utils.dart | 31 ++- 15 files changed, 591 insertions(+), 118 deletions(-) create mode 100644 packages/powersync_core/test/in_memory_sync_test.dart create mode 100644 packages/powersync_core/test/server/sync_server/in_memory_sync_server.dart create mode 100644 packages/powersync_core/test/utils/in_memory_http.dart diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index 8869c162..dd857080 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:convert'; import 'package:logging/logging.dart'; import 'package:meta/meta.dart'; @@ -121,17 +122,56 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { Future _updateHasSynced() async { // Query the database to see if any data has been synced. - final result = - await database.get('SELECT powersync_last_synced_at() as synced_at'); - final timestamp = result['synced_at'] as String?; - final hasSynced = timestamp != null; - - if (hasSynced != currentStatus.hasSynced) { - final lastSyncedAt = - timestamp == null ? null : DateTime.parse('${timestamp}Z').toLocal(); - final status = - SyncStatus(hasSynced: hasSynced, lastSyncedAt: lastSyncedAt); - setStatus(status); + final result = await database.get(''' + SELECT CASE + WHEN EXISTS (SELECT 1 FROM sqlite_master WHERE name = 'ps_sync_state') + THEN (SELECT json_group_array( + json_object('prio', priority, 'last_sync', last_synced_at) + ) FROM ps_sync_state ORDER BY priority) + ELSE powersync_last_synced_at() + END AS r; + '''); + final value = result['r'] as String?; + final hasData = value != null; + + DateTime parseDateTime(String sql) { + return DateTime.parse('${sql}Z').toLocal(); + } + + if (hasData) { + DateTime? lastCompleteSync; + final priorityStatus = []; + var hasSynced = false; + + if (value.startsWith('[')) { + for (final entry in jsonDecode(value) as List) { + final priority = entry['prio'] as int; + final lastSyncedAt = parseDateTime(entry['last_sync'] as String); + + if (priority == -1) { + hasSynced = true; + lastCompleteSync = lastSyncedAt; + } else { + priorityStatus.add(( + hasSynced: true, + lastSyncedAt: lastSyncedAt, + priority: BucketPriority(priority) + )); + } + } + } else { + hasSynced = true; + lastCompleteSync = parseDateTime(value); + } + + if (hasSynced != currentStatus.hasSynced) { + final status = SyncStatus( + hasSynced: hasSynced, + lastSyncedAt: lastCompleteSync, + statusInPriority: priorityStatus, + ); + setStatus(status); + } } } @@ -201,7 +241,10 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { await disconnect(); // Now we can close the database await database.close(); - await statusStreamController.close(); + + // If there are paused subscriptionso n the status stream, don't delay + // closing the database because of that. + unawaited(statusStreamController.close()); } /// Connect to the PowerSync service, and keep the databases in sync. diff --git a/packages/powersync_core/lib/src/streaming_sync.dart b/packages/powersync_core/lib/src/streaming_sync.dart index b4b6f214..63787c90 100644 --- a/packages/powersync_core/lib/src/streaming_sync.dart +++ b/packages/powersync_core/lib/src/streaming_sync.dart @@ -119,6 +119,7 @@ class StreamingSyncImplementation implements StreamingSync { // Now close the client in all cases not covered above _client.close(); + _statusStreamController.close(); } bool get aborted { @@ -281,7 +282,7 @@ class StreamingSyncImplementation implements StreamingSync { for (final (i, priority) in existingPriorityState.indexed) { switch ( BucketPriority.comparator(priority.priority, completed.priority)) { - case < 0: + case > 0: // Entries from here on have a higher priority than the one that was // just completed final copy = existingPriorityState.toList(); @@ -293,7 +294,7 @@ class StreamingSyncImplementation implements StreamingSync { copy[i] = completed; _updateStatus(statusInPriority: copy); return; - case > 0: + case < 0: continue; } } @@ -537,49 +538,55 @@ class StreamingSyncImplementation implements StreamingSync { return true; } - Stream streamingSyncRequest( - StreamingSyncRequest data) async* { - final credentials = await credentialsCallback(); - if (credentials == null) { - throw CredentialsException('Not logged in'); - } - final uri = credentials.endpointUri('sync/stream'); - - final request = http.Request('POST', uri); - request.headers['Content-Type'] = 'application/json'; - request.headers['Authorization'] = "Token ${credentials.token}"; - request.headers.addAll(_userAgentHeaders); - - request.body = convert.jsonEncode(data); - - http.StreamedResponse res; - try { - // Do not close the client during the request phase - this causes uncaught errors. - _safeToClose = false; - res = await _client.send(request); - } finally { - _safeToClose = true; - } - if (aborted) { - return; - } + Stream streamingSyncRequest(StreamingSyncRequest data) { + Future setup() async { + final credentials = await credentialsCallback(); + if (credentials == null) { + throw CredentialsException('Not logged in'); + } + final uri = credentials.endpointUri('sync/stream'); + + final request = http.Request('POST', uri); + request.headers['Content-Type'] = 'application/json'; + request.headers['Authorization'] = "Token ${credentials.token}"; + request.headers.addAll(_userAgentHeaders); + + request.body = convert.jsonEncode(data); + + http.StreamedResponse res; + try { + // Do not close the client during the request phase - this causes uncaught errors. + _safeToClose = false; + res = await _client.send(request); + } finally { + _safeToClose = true; + } + if (aborted) { + return null; + } - if (res.statusCode == 401) { - if (invalidCredentialsCallback != null) { - await invalidCredentialsCallback!(); + if (res.statusCode == 401) { + if (invalidCredentialsCallback != null) { + await invalidCredentialsCallback!(); + } } - } - if (res.statusCode != 200) { - throw await SyncResponseException.fromStreamedResponse(res); + if (res.statusCode != 200) { + throw await SyncResponseException.fromStreamedResponse(res); + } + + return res.stream; } - // Note: The response stream is automatically closed when this loop errors - await for (var line in ndjson(res.stream)) { - if (aborted) { - break; + return Stream.fromFuture(setup()).asyncExpand((stream) { + if (stream == null || aborted) { + return const Stream.empty(); + } else { + return ndjson(stream) + .map((line) => + StreamingSyncLine.fromJson(line as Map)) + .takeWhile((_) => !aborted); } - yield StreamingSyncLine.fromJson(line as Map); - } + }); } /// Delays the standard `retryDelay` Duration, but exits early if diff --git a/packages/powersync_core/lib/src/sync_status.dart b/packages/powersync_core/lib/src/sync_status.dart index 799e69bb..ff658921 100644 --- a/packages/powersync_core/lib/src/sync_status.dart +++ b/packages/powersync_core/lib/src/sync_status.dart @@ -99,7 +99,15 @@ final class SyncStatus { /// Returns [lastSyncedAt] and [hasSynced] information for a partial sync /// operation, or `null` if the status for that priority is unknown. - SyncPriorityStatus? statusForPriority(BucketPriority priority) { + /// + /// The information returned may be more generic than requested. For instance, + /// a completed sync operation (as expressed by [lastSyncedAt]) also + /// guarantees that every bucket priority was synchronized before that. + /// Similarly, requesting the sync status for priority `1` may return + /// information extracted from the lower priority `2` since each partial sync + /// in priority `2` necessarily includes a consistent view over data in + /// priority `1`. + SyncPriorityStatus statusForPriority(BucketPriority priority) { assert(statusInPriority.isSortedByCompare( (e) => e.priority, BucketPriority.comparator)); @@ -112,7 +120,12 @@ final class SyncStatus { } } - return null; + // If we have a complete sync, that necessarily includes all priorities. + return ( + priority: priority, + hasSynced: hasSynced, + lastSyncedAt: lastSyncedAt + ); } @override @@ -154,8 +167,8 @@ extension type const BucketPriority._(int priorityNumber) { /// priority. typedef SyncPriorityStatus = ({ BucketPriority priority, - DateTime lastSyncedAt, - bool hasSynced, + DateTime? lastSyncedAt, + bool? hasSynced, }); /// Stats of the local upload queue. diff --git a/packages/powersync_core/lib/src/sync_types.dart b/packages/powersync_core/lib/src/sync_types.dart index 842f44ab..6d02766f 100644 --- a/packages/powersync_core/lib/src/sync_types.dart +++ b/packages/powersync_core/lib/src/sync_types.dart @@ -58,7 +58,11 @@ final class Checkpoint extends StreamingSyncLine { 'write_checkpoint': writeCheckpoint, 'buckets': checksums .where((c) => priority == null || c.priority <= priority) - .map((c) => {'bucket': c.bucket, 'checksum': c.checksum}) + .map((c) => { + 'bucket': c.bucket, + 'checksum': c.checksum, + 'priority': c.priority, + }) .toList(growable: false) }; } diff --git a/packages/powersync_core/test/connected_test.dart b/packages/powersync_core/test/connected_test.dart index ff6a9b0c..ace3ae90 100644 --- a/packages/powersync_core/test/connected_test.dart +++ b/packages/powersync_core/test/connected_test.dart @@ -9,7 +9,6 @@ import 'package:powersync_core/powersync_core.dart'; import 'package:test/test.dart'; import 'server/sync_server/mock_sync_server.dart'; -import 'streaming_sync_test.dart'; import 'utils/abstract_test_utils.dart'; import 'utils/test_utils_impl.dart'; diff --git a/packages/powersync_core/test/disconnect_test.dart b/packages/powersync_core/test/disconnect_test.dart index 489928ef..a3706537 100644 --- a/packages/powersync_core/test/disconnect_test.dart +++ b/packages/powersync_core/test/disconnect_test.dart @@ -1,7 +1,7 @@ import 'package:powersync_core/powersync_core.dart'; import 'package:powersync_core/sqlite_async.dart'; import 'package:test/test.dart'; -import 'streaming_sync_test.dart'; +import 'utils/abstract_test_utils.dart'; import 'utils/test_utils_impl.dart'; import 'watch_test.dart'; diff --git a/packages/powersync_core/test/in_memory_sync_test.dart b/packages/powersync_core/test/in_memory_sync_test.dart new file mode 100644 index 00000000..a830a55a --- /dev/null +++ b/packages/powersync_core/test/in_memory_sync_test.dart @@ -0,0 +1,209 @@ +import 'package:async/async.dart'; +import 'package:logging/logging.dart'; +import 'package:powersync_core/powersync_core.dart'; +import 'package:powersync_core/sqlite3_common.dart'; +import 'package:powersync_core/src/log_internal.dart'; +import 'package:powersync_core/src/streaming_sync.dart'; +import 'package:powersync_core/src/sync_types.dart'; +import 'package:test/test.dart'; + +import 'server/sync_server/in_memory_sync_server.dart'; +import 'utils/abstract_test_utils.dart'; +import 'utils/in_memory_http.dart'; +import 'utils/test_utils_impl.dart'; + +void main() { + final testUtils = TestUtils(); + + group('in-memory sync tests', () { + late TestPowerSyncFactory factory; + late CommonDatabase raw; + late PowerSyncDatabase database; + late MockSyncService syncService; + late StreamingSyncImplementation syncClient; + + setUp(() async { + final (client, server) = inMemoryServer(); + syncService = MockSyncService(); + server.mount(syncService.router.call); + + factory = await testUtils.testFactory(); + (raw, database) = await factory.openInMemoryDatabase(); + await database.initialize(); + syncClient = database.connectWithMockService( + client, + TestConnector(() async { + return PowerSyncCredentials( + endpoint: server.url.toString(), + token: 'token not used here', + expiresAt: DateTime.now(), + ); + }), + ); + }); + + tearDown(() async { + await syncClient.abort(); + await database.close(); + await syncService.stop(); + }); + + Future> waitForConnection( + {bool expectNoWarnings = true}) async { + if (expectNoWarnings) { + isolateLogger.onRecord.listen((e) { + if (e.level >= Level.WARNING) { + fail('Unexpected log: $e'); + } + }); + } + syncClient.streamingSync(); + await syncService.waitForListener; + + expect(database.currentStatus.lastSyncedAt, isNull); + expect(database.currentStatus.downloading, isFalse); + final status = StreamQueue(database.statusStream); + addTearDown(status.cancel); + + syncService.addKeepAlive(); + await expectLater( + status, emits(isSyncStatus(connected: true, hasSynced: false))); + return status; + } + + test('persists completed sync information', () async { + final status = await waitForConnection(); + + syncService.addLine({ + 'checkpoint': Checkpoint( + lastOpId: '0', + writeCheckpoint: null, + checksums: [BucketChecksum(bucket: 'bkt', priority: 1, checksum: 0)], + ) + }); + await expectLater(status, emits(isSyncStatus(downloading: true))); + + syncService.addLine({ + 'checkpoint_complete': {'last_op_id': '0'} + }); + await expectLater( + status, emits(isSyncStatus(downloading: false, hasSynced: true))); + + final independentDb = factory.wrapRaw(raw); + // Even though this database doesn't have a sync client attached to it, + // is should reconstruct hasSynced from the database. + await independentDb.initialize(); + expect(independentDb.currentStatus.hasSynced, isTrue); + // A complete sync also means that all partial syncs have completed + expect( + independentDb.currentStatus + .statusForPriority(BucketPriority(3)) + ?.hasSynced, + isTrue); + }); + + group('partial sync', () { + test('updates sync state incrementally', () async { + final status = await waitForConnection(); + + final checksums = [ + for (var prio = 0; prio <= 3; prio++) + BucketChecksum(bucket: 'prio$prio', priority: prio, checksum: 0) + ]; + syncService.addLine({ + 'checkpoint': Checkpoint( + lastOpId: '0', + writeCheckpoint: null, + checksums: checksums, + ) + }); + + // Receiving the checkpoint sets the state to downloading + await expectLater( + status, emits(isSyncStatus(downloading: true, hasSynced: false))); + + // Emit partial sync complete for each priority but the last. + for (var prio = 0; prio < 3; prio++) { + syncService.addLine({ + 'partial_checkpoint_complete': { + 'last_op_id': '0', + 'priority': prio, + } + }); + + await expectLater( + status, + emits(isSyncStatus(downloading: true, hasSynced: false).having( + (e) => e.statusForPriority(BucketPriority(0))?.hasSynced, + 'status for $prio', + isTrue, + )), + ); + } + + // Complete the sync + syncService.addLine({ + 'checkpoint_complete': {'last_op_id': '0'} + }); + + await expectLater( + status, emits(isSyncStatus(downloading: false, hasSynced: true))); + }); + + test('remembers last partial sync state', () async { + final status = await waitForConnection(); + + syncService.addLine({ + 'checkpoint': Checkpoint( + lastOpId: '0', + writeCheckpoint: null, + checksums: [ + BucketChecksum(bucket: 'bkt', priority: 1, checksum: 0) + ], + ) + }); + await expectLater(status, emits(isSyncStatus(downloading: true))); + + syncService.addLine({ + 'partial_checkpoint_complete': { + 'last_op_id': '0', + 'priority': 1, + } + }); + await database.waitForFirstSync(priority: BucketPriority(1)); + expect(database.currentStatus.hasSynced, isFalse); + + final independentDb = factory.wrapRaw(raw); + await independentDb.initialize(); + expect(independentDb.currentStatus.hasSynced, isFalse); + // Completing a sync for prio 1 implies a completed sync for prio 0 + expect( + independentDb.currentStatus + .statusForPriority(BucketPriority(0)) + ?.hasSynced, + isTrue); + expect( + independentDb.currentStatus + .statusForPriority(BucketPriority(3)) + ?.hasSynced, + isFalse); + }); + }); + }); +} + +TypeMatcher isSyncStatus( + {Object? downloading, Object? connected, Object? hasSynced}) { + var matcher = isA(); + if (downloading != null) { + matcher = matcher.having((e) => e.downloading, 'downloading', downloading); + } + if (connected != null) { + matcher = matcher.having((e) => e.connected, 'connected', connected); + } + if (hasSynced != null) { + matcher = matcher.having((e) => e.hasSynced, 'hasSynced', hasSynced); + } + + return matcher; +} diff --git a/packages/powersync_core/test/server/sync_server/in_memory_sync_server.dart b/packages/powersync_core/test/server/sync_server/in_memory_sync_server.dart new file mode 100644 index 00000000..b31dffc2 --- /dev/null +++ b/packages/powersync_core/test/server/sync_server/in_memory_sync_server.dart @@ -0,0 +1,64 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:shelf/shelf.dart'; +import 'package:shelf_router/shelf_router.dart'; + +final class MockSyncService { + // Use a queued stream to make tests easier. + StreamController _controller = StreamController(); + Completer _listener = Completer(); + + final router = Router(); + + MockSyncService() { + router + ..post('/sync/stream', (Request request) async { + _listener.complete(); + // Respond immediately with a stream + return Response.ok(_controller.stream.transform(utf8.encoder), + headers: { + 'Content-Type': 'application/x-ndjson', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + }, + context: { + "shelf.io.buffer_output": false + }); + }) + ..get('/write-checkpoint2.json', (request) { + return Response.ok('{"data": {"write_checkpoint": "10"}}', headers: { + 'Content-Type': 'application/json', + }); + }); + } + + Future get waitForListener => _listener.future; + + // Queue events which will be sent to connected clients. + void addRawEvent(String data) { + _controller.add(data); + } + + void addLine(Object? message) { + addRawEvent('${json.encode(message)}\n'); + } + + void addKeepAlive([int tokenExpiresIn = 3600]) { + addLine({'token_expires_in': tokenExpiresIn}); + } + + // Clear events. We rely on a buffered controller here. Create a new controller + // in order to clear the buffer. + Future clearEvents() async { + await _controller.close(); + _listener = Completer(); + _controller = StreamController(); + } + + Future stop() async { + if (_controller.hasListener) { + await _controller.close(); + } + } +} diff --git a/packages/powersync_core/test/server/sync_server/mock_sync_server.dart b/packages/powersync_core/test/server/sync_server/mock_sync_server.dart index 9844692f..e4710f3d 100644 --- a/packages/powersync_core/test/server/sync_server/mock_sync_server.dart +++ b/packages/powersync_core/test/server/sync_server/mock_sync_server.dart @@ -1,58 +1,37 @@ import 'dart:async'; -import 'dart:convert'; import 'dart:io'; -import 'package:shelf/shelf.dart'; import 'package:shelf/shelf_io.dart' as io; -import 'package:shelf_router/shelf_router.dart'; + +import 'in_memory_sync_server.dart'; // A basic Mock PowerSync service server which queues commands // which clients can receive via connecting to the `/sync/stream` route. // This assumes only one client will ever be connected at a time. class TestHttpServerHelper { - // Use a queued stream to make tests easier. - StreamController _controller = StreamController(); + final MockSyncService service = MockSyncService(); late HttpServer _server; + Uri get uri => Uri.parse('http://localhost:${_server.port}'); Future start() async { - final router = Router() - ..post('/sync/stream', (Request request) async { - // Respond immediately with a stream - return Response.ok(_controller.stream.transform(utf8.encoder), - headers: { - 'Content-Type': 'application/x-ndjson', - 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive', - }, - context: { - "shelf.io.buffer_output": false - }); - }) - ..get('/write-checkpoint2.json', (request) { - return Response.ok('{"data": {"write_checkpoint": "10"}}', headers: { - 'Content-Type': 'application/json', - }); - }); - - _server = await io.serve(router.call, 'localhost', 0); + _server = await io.serve(service.router.call, 'localhost', 0); print('Test server running at ${_server.address}:${_server.port}'); } // Queue events which will be sent to connected clients. void addEvent(String data) { - _controller.add(data); + service.addRawEvent(data); } // Clear events. We rely on a buffered controller here. Create a new controller // in order to clear the buffer. Future clearEvents() async { - await _controller.close(); - _controller = StreamController(); + await service.clearEvents(); } Future stop() async { - await _controller.close(); + await service.stop(); await _server.close(); } } diff --git a/packages/powersync_core/test/streaming_sync_test.dart b/packages/powersync_core/test/streaming_sync_test.dart index 5238dd05..b9bbed07 100644 --- a/packages/powersync_core/test/streaming_sync_test.dart +++ b/packages/powersync_core/test/streaming_sync_test.dart @@ -9,29 +9,11 @@ import 'package:powersync_core/powersync_core.dart'; import 'package:test/test.dart'; import 'test_server.dart'; +import 'utils/abstract_test_utils.dart'; import 'utils/test_utils_impl.dart'; final testUtils = TestUtils(); -class TestConnector extends PowerSyncBackendConnector { - final Function _fetchCredentials; - final Future Function(PowerSyncDatabase)? _uploadData; - - TestConnector(this._fetchCredentials, - {Future Function(PowerSyncDatabase)? uploadData}) - : _uploadData = uploadData; - - @override - Future fetchCredentials() { - return _fetchCredentials(); - } - - @override - Future uploadData(PowerSyncDatabase database) async { - await _uploadData?.call(database); - } -} - void main() { group('Streaming Sync Test', () { late String path; diff --git a/packages/powersync_core/test/utils/abstract_test_utils.dart b/packages/powersync_core/test/utils/abstract_test_utils.dart index 561e5c3b..d94b92f2 100644 --- a/packages/powersync_core/test/utils/abstract_test_utils.dart +++ b/packages/powersync_core/test/utils/abstract_test_utils.dart @@ -1,5 +1,8 @@ +import 'package:http/http.dart'; import 'package:logging/logging.dart'; import 'package:powersync_core/powersync_core.dart'; +import 'package:powersync_core/src/bucket_storage.dart'; +import 'package:powersync_core/src/streaming_sync.dart'; import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test_api/src/backend/invoker.dart'; @@ -52,6 +55,23 @@ Logger _makeTestLogger({Level level = Level.ALL, String? name}) { return logger; } +abstract mixin class TestPowerSyncFactory implements PowerSyncOpenFactory { + Future openRawInMemoryDatabase(); + + Future<(CommonDatabase, PowerSyncDatabase)> openInMemoryDatabase() async { + final raw = await openRawInMemoryDatabase(); + return (raw, wrapRaw(raw)); + } + + PowerSyncDatabase wrapRaw(CommonDatabase raw) { + return PowerSyncDatabase.withDatabase( + schema: schema, + database: SqliteDatabase.singleConnection( + SqliteConnection.synchronousWrapper(raw)), + ); + } +} + abstract class AbstractTestUtils { String get _testName => Invoker.current!.liveTest.test.name; @@ -63,12 +83,10 @@ abstract class AbstractTestUtils { } /// Generates a test open factory - Future testFactory( + Future testFactory( {String? path, String sqlitePath = '', - SqliteOptions options = const SqliteOptions.defaults()}) async { - return PowerSyncOpenFactory(path: path ?? dbPath(), sqliteOptions: options); - } + SqliteOptions options = const SqliteOptions.defaults()}); /// Creates a SqliteDatabaseConnection Future setupPowerSync( @@ -93,3 +111,41 @@ abstract class AbstractTestUtils { /// Deletes any DB data Future cleanDb({required String path}); } + +class TestConnector extends PowerSyncBackendConnector { + final Function _fetchCredentials; + final Future Function(PowerSyncDatabase)? _uploadData; + + TestConnector(this._fetchCredentials, + {Future Function(PowerSyncDatabase)? uploadData}) + : _uploadData = uploadData; + + @override + Future fetchCredentials() { + return _fetchCredentials(); + } + + @override + Future uploadData(PowerSyncDatabase database) async { + await _uploadData?.call(database); + } +} + +extension MockSync on PowerSyncDatabase { + StreamingSyncImplementation connectWithMockService( + Client client, PowerSyncBackendConnector connector) { + final impl = StreamingSyncImplementation( + adapter: BucketStorage(this), + client: client, + retryDelay: const Duration(seconds: 5), + credentialsCallback: connector.getCredentialsCached, + invalidCredentialsCallback: connector.prefetchCredentials, + uploadCrud: () => connector.uploadData(this), + crudUpdateTriggerStream: database + .onChange(['ps_crud'], throttle: const Duration(milliseconds: 10)), + ); + impl.statusStream.listen(setStatus); + + return impl; + } +} diff --git a/packages/powersync_core/test/utils/in_memory_http.dart b/packages/powersync_core/test/utils/in_memory_http.dart new file mode 100644 index 00000000..61550e3c --- /dev/null +++ b/packages/powersync_core/test/utils/in_memory_http.dart @@ -0,0 +1,56 @@ +import 'package:http/http.dart'; +import 'package:http/testing.dart'; +import 'package:shelf/shelf.dart' as shelf; + +final Uri mockHttpUri = Uri.parse('https://testing.powersync.com/'); + +/// Returns a [Client] that can send HTTP requests to the returned +/// [shelf.Server]. +/// +/// The server can be used to serve shelf routes via [shelf.Server.mount]. +(Client, shelf.Server) inMemoryServer() { + final server = _MockServer(); + final client = MockClient.streaming(server.handleRequest); + + return (client, server); +} + +final class _MockServer implements shelf.Server { + shelf.Handler? _handler; + + @override + void mount(shelf.Handler handler) { + if (_handler != null) { + throw StateError('already has a handler'); + } + + _handler = handler; + } + + @override + Future close() async {} + + @override + Uri get url => mockHttpUri; + + Future handleRequest( + BaseRequest request, ByteStream body) async { + if (_handler case final endpoint?) { + final shelfRequest = shelf.Request( + request.method, + request.url, + headers: request.headers, + body: body, + ); + final shelfResponse = await endpoint(shelfRequest); + + return StreamedResponse( + shelfResponse.read(), + shelfResponse.statusCode, + headers: shelfResponse.headers, + ); + } else { + throw StateError('Request before handler was set on mock server'); + } + } +} diff --git a/packages/powersync_core/test/utils/native_test_utils.dart b/packages/powersync_core/test/utils/native_test_utils.dart index ab8cfd6a..95e55b3a 100644 --- a/packages/powersync_core/test/utils/native_test_utils.dart +++ b/packages/powersync_core/test/utils/native_test_utils.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:ffi'; import 'dart:io'; import 'package:powersync_core/powersync_core.dart'; +import 'package:powersync_core/sqlite3.dart'; import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:sqlite3/open.dart' as sqlite_open; @@ -10,17 +11,28 @@ import 'abstract_test_utils.dart'; const defaultSqlitePath = 'libsqlite3.so.0'; -class TestOpenFactory extends PowerSyncOpenFactory { +class TestOpenFactory extends PowerSyncOpenFactory with TestPowerSyncFactory { TestOpenFactory({required super.path}); - @override - CommonDatabase open(SqliteOpenOptions options) { + void applyOpenOverride() { sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { return DynamicLibrary.open('libsqlite3.so.0'); }); sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.macOS, () { return DynamicLibrary.open('libsqlite3.dylib'); }); + } + + @override + void enableExtension() { + var powersyncLib = getLibraryForPlatform(); + sqlite3.ensureExtensionLoaded(SqliteExtension.inLibrary( + DynamicLibrary.open(powersyncLib), 'sqlite3_powersync_init')); + } + + @override + CommonDatabase open(SqliteOpenOptions options) { + applyOpenOverride(); return super.open(options); } @@ -52,6 +64,22 @@ class TestOpenFactory extends PowerSyncOpenFactory { ); } } + + @override + Future openRawInMemoryDatabase() async { + applyOpenOverride(); + + try { + enableExtension(); + } on PowersyncNotReadyException catch (e) { + autoLogger.severe(e.message); + rethrow; + } + + final db = sqlite3.openInMemory(); + setupFunctions(db); + return db; + } } class TestUtils extends AbstractTestUtils { diff --git a/packages/powersync_core/test/utils/stub_test_utils.dart b/packages/powersync_core/test/utils/stub_test_utils.dart index 3f86512c..d0cdb428 100644 --- a/packages/powersync_core/test/utils/stub_test_utils.dart +++ b/packages/powersync_core/test/utils/stub_test_utils.dart @@ -1,3 +1,5 @@ +import 'package:sqlite_async/src/sqlite_options.dart'; + import 'abstract_test_utils.dart'; class TestUtils extends AbstractTestUtils { @@ -5,4 +7,12 @@ class TestUtils extends AbstractTestUtils { Future cleanDb({required String path}) { throw UnimplementedError(); } + + @override + Future testFactory( + {String? path, + String sqlitePath = '', + SqliteOptions options = const SqliteOptions.defaults()}) { + throw UnimplementedError(); + } } diff --git a/packages/powersync_core/test/utils/web_test_utils.dart b/packages/powersync_core/test/utils/web_test_utils.dart index 71a462a2..3a74a448 100644 --- a/packages/powersync_core/test/utils/web_test_utils.dart +++ b/packages/powersync_core/test/utils/web_test_utils.dart @@ -3,7 +3,7 @@ import 'dart:js_interop'; import 'package:logging/logging.dart'; import 'package:powersync_core/powersync_core.dart'; -import 'package:sqlite_async/sqlite3_common.dart'; +import 'package:sqlite_async/sqlite3_wasm.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; import 'package:web/web.dart' show Blob, BlobPropertyBag; @@ -12,6 +12,29 @@ import 'abstract_test_utils.dart'; @JS('URL.createObjectURL') external String _createObjectURL(Blob blob); +class TestOpenFactory extends PowerSyncOpenFactory with TestPowerSyncFactory { + TestOpenFactory({required super.path, super.sqliteOptions}); + + @override + Future openRawInMemoryDatabase() async { + final sqlite = await WasmSqlite3.loadFromUrl( + Uri.parse(sqliteOptions.webSqliteOptions.wasmUri)); + sqlite.registerVirtualFileSystem(InMemoryFileSystem(), makeDefault: true); + + final db = sqlite.openInMemory(); + + try { + enableExtension(); + } on PowersyncNotReadyException catch (e) { + autoLogger.severe(e.message); + rethrow; + } + + setupFunctions(db); + return db; + } +} + class TestUtils extends AbstractTestUtils { late Future _isInitialized; late final String sqlite3WASMUri; @@ -39,16 +62,16 @@ class TestUtils extends AbstractTestUtils { Future cleanDb({required String path}) async {} @override - Future testFactory( + Future testFactory( {String? path, - String? sqlitePath, + String sqlitePath = '', SqliteOptions options = const SqliteOptions.defaults()}) async { await _isInitialized; final webOptions = SqliteOptions( webSqliteOptions: WebSqliteOptions(wasmUri: sqlite3WASMUri, workerUri: workerUri)); - return super.testFactory(path: path, options: webOptions); + return TestOpenFactory(path: path ?? '', sqliteOptions: webOptions); } @override From 9f6fec59aebaad24851d616a663441fcf4b8800f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 10 Feb 2025 12:17:15 +0100 Subject: [PATCH 12/25] Fix unecessary null checks --- .../powersync_core/lib/src/database/powersync_db_mixin.dart | 2 +- packages/powersync_core/test/in_memory_sync_test.dart | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index fdec550e..5c7b8231 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -187,7 +187,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { if (priority == null) { return status.hasSynced == true; } else { - return status.statusForPriority(priority)?.hasSynced == true; + return status.statusForPriority(priority).hasSynced == true; } } diff --git a/packages/powersync_core/test/in_memory_sync_test.dart b/packages/powersync_core/test/in_memory_sync_test.dart index 24b73594..bc7869c3 100644 --- a/packages/powersync_core/test/in_memory_sync_test.dart +++ b/packages/powersync_core/test/in_memory_sync_test.dart @@ -134,7 +134,7 @@ void main() { await expectLater( status, emits(isSyncStatus(downloading: true, hasSynced: false).having( - (e) => e.statusForPriority(BucketPriority(0))?.hasSynced, + (e) => e.statusForPriority(BucketPriority(0)).hasSynced, 'status for $prio', isTrue, )), @@ -180,12 +180,12 @@ void main() { expect( independentDb.currentStatus .statusForPriority(BucketPriority(0)) - ?.hasSynced, + .hasSynced, isTrue); expect( independentDb.currentStatus .statusForPriority(BucketPriority(3)) - ?.hasSynced, + .hasSynced, isFalse); }); }); From 0da750c4fd3ce3cbfda16cb54ba3d2534d2abd79 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 10 Feb 2025 16:40:02 +0100 Subject: [PATCH 13/25] Fix bad merges --- .../lib/src/database/powersync_db_mixin.dart | 10 +--- .../lib/src/streaming_sync.dart | 11 ++-- .../test/in_memory_sync_test.dart | 60 +++++++++++++++++++ .../test/utils/native_test_utils.dart | 12 ++-- 4 files changed, 73 insertions(+), 20 deletions(-) diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index 5c7b8231..9aefbc9c 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -123,13 +123,9 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { Future _updateHasSynced() async { // Query the database to see if any data has been synced. final result = await database.get(''' - SELECT CASE - WHEN EXISTS (SELECT 1 FROM sqlite_master WHERE name = 'ps_sync_state') - THEN (SELECT json_group_array( - json_object('prio', priority, 'last_sync', last_synced_at) - ) FROM ps_sync_state ORDER BY priority) - ELSE powersync_last_synced_at() - END AS r; + SELECT json_group_array( + json_object('prio', priority, 'last_sync', last_synced_at) + ) AS r FROM ps_sync_state ORDER BY priority '''); final value = result['r'] as String?; final hasData = value != null; diff --git a/packages/powersync_core/lib/src/streaming_sync.dart b/packages/powersync_core/lib/src/streaming_sync.dart index d6cb5815..c34867e9 100644 --- a/packages/powersync_core/lib/src/streaming_sync.dart +++ b/packages/powersync_core/lib/src/streaming_sync.dart @@ -444,8 +444,6 @@ class StreamingSyncImplementation implements StreamingSync { hasSynced: true, )); } - - validatedCheckpoint = targetCheckpoint; case StreamingSyncCheckpointDiff(): // TODO: It may be faster to just keep track of the diff, instead of // the entire checkpoint @@ -479,12 +477,12 @@ class StreamingSyncImplementation implements StreamingSync { case SyncDataBatch(): _updateStatus(downloading: true); await adapter.saveSyncData(line); - case StreamingSyncKeepalive(): - if (line.tokenExpiresIn == 0) { + case StreamingSyncKeepalive(:final tokenExpiresIn): + if (tokenExpiresIn == 0) { // Token expired already - stop the connection immediately invalidCredentialsCallback?.call().ignore(); break; - } else if (line.tokenExpiresIn <= 30) { + } else if (tokenExpiresIn <= 30) { // Token expires soon - refresh it in the background if (credentialsInvalidation == null && invalidCredentialsCallback != null) { @@ -501,8 +499,7 @@ class StreamingSyncImplementation implements StreamingSync { } } case UnknownSyncLine(:final rawData): - isolateLogger.fine('Ignoring unknown sync line: $rawData'); - break; + isolateLogger.fine('Unknown sync line: $rawData'); case null: // Local ping if (targetCheckpoint == appliedCheckpoint) { _updateStatus( diff --git a/packages/powersync_core/test/in_memory_sync_test.dart b/packages/powersync_core/test/in_memory_sync_test.dart index bc7869c3..c7636de3 100644 --- a/packages/powersync_core/test/in_memory_sync_test.dart +++ b/packages/powersync_core/test/in_memory_sync_test.dart @@ -102,6 +102,66 @@ void main() { isTrue); }); + test('can save independent buckets in same transaction', () async { + final status = await waitForConnection(); + + syncService.addLine({ + 'checkpoint': Checkpoint( + lastOpId: '0', + writeCheckpoint: null, + checksums: [ + BucketChecksum(bucket: 'a', checksum: 0, priority: 3), + BucketChecksum(bucket: 'b', checksum: 0, priority: 3), + ], + ) + }); + await expectLater(status, emits(isSyncStatus(downloading: true))); + + var commits = 0; + raw.commits.listen((_) => commits++); + + syncService + ..addLine({ + 'data': { + 'bucket': 'a', + 'data': >[ + { + 'op_id': '1', + 'op': 'PUT', + 'object_type': 'a', + 'object_id': '1', + 'checksum': 0, + 'data': {}, + } + ], + } + }) + ..addLine({ + 'data': { + 'bucket': 'b', + 'data': >[ + { + 'op_id': '2', + 'op': 'PUT', + 'object_type': 'b', + 'object_id': '1', + 'checksum': 0, + 'data': {}, + } + ], + } + }); + + // Wait for the operations to be inserted. + while (raw.select('SELECT * FROM ps_oplog;').length < 2) { + await pumpEventQueue(); + } + + // The two buckets should have been inserted in a single transaction + // because the messages were received in quick succession. + expect(commits, 1); + }); + group('partial sync', () { test('updates sync state incrementally', () async { final status = await waitForConnection(); diff --git a/packages/powersync_core/test/utils/native_test_utils.dart b/packages/powersync_core/test/utils/native_test_utils.dart index 95e55b3a..65916a5f 100644 --- a/packages/powersync_core/test/utils/native_test_utils.dart +++ b/packages/powersync_core/test/utils/native_test_utils.dart @@ -23,6 +23,12 @@ class TestOpenFactory extends PowerSyncOpenFactory with TestPowerSyncFactory { }); } + @override + CommonDatabase open(SqliteOpenOptions options) { + applyOpenOverride(); + return super.open(options); + } + @override void enableExtension() { var powersyncLib = getLibraryForPlatform(); @@ -30,12 +36,6 @@ class TestOpenFactory extends PowerSyncOpenFactory with TestPowerSyncFactory { DynamicLibrary.open(powersyncLib), 'sqlite3_powersync_init')); } - @override - CommonDatabase open(SqliteOpenOptions options) { - applyOpenOverride(); - return super.open(options); - } - @override String getLibraryForPlatform({String? path = "."}) { switch (Abi.current()) { From 97db1fca2f61d14e059f2765462749132702c774 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 10 Feb 2025 16:51:32 +0100 Subject: [PATCH 14/25] Fix initial hasSynced state --- .../lib/src/database/powersync_db_mixin.dart | 65 ++++++++----------- 1 file changed, 26 insertions(+), 39 deletions(-) diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index 9aefbc9c..92a37e07 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -1,5 +1,4 @@ import 'dart:async'; -import 'dart:convert'; import 'package:logging/logging.dart'; import 'package:meta/meta.dart'; @@ -122,52 +121,40 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { Future _updateHasSynced() async { // Query the database to see if any data has been synced. - final result = await database.get(''' - SELECT json_group_array( - json_object('prio', priority, 'last_sync', last_synced_at) - ) AS r FROM ps_sync_state ORDER BY priority - '''); - final value = result['r'] as String?; - final hasData = value != null; + final result = await database.getAll( + 'SELECT priority, last_synced_at FROM ps_sync_state ORDER BY priority;', + ); + var hasSynced = false; + DateTime? lastCompleteSync; + final priorityStatus = []; DateTime parseDateTime(String sql) { return DateTime.parse('${sql}Z').toLocal(); } - if (hasData) { - DateTime? lastCompleteSync; - final priorityStatus = []; - var hasSynced = false; - - if (value.startsWith('[')) { - for (final entry in jsonDecode(value) as List) { - final priority = entry['prio'] as int; - final lastSyncedAt = parseDateTime(entry['last_sync'] as String); - - if (priority == -1) { - hasSynced = true; - lastCompleteSync = lastSyncedAt; - } else { - priorityStatus.add(( - hasSynced: true, - lastSyncedAt: lastSyncedAt, - priority: BucketPriority(priority) - )); - } - } - } else { + for (final row in result) { + final priority = row.columnAt(0) as int; + final lastSyncedAt = parseDateTime(row.columnAt(1) as String); + + if (priority == -1) { hasSynced = true; - lastCompleteSync = parseDateTime(value); + lastCompleteSync = lastSyncedAt; + } else { + priorityStatus.add(( + hasSynced: true, + lastSyncedAt: lastSyncedAt, + priority: BucketPriority(priority) + )); } + } - if (hasSynced != currentStatus.hasSynced) { - final status = SyncStatus( - hasSynced: hasSynced, - lastSyncedAt: lastCompleteSync, - statusInPriority: priorityStatus, - ); - setStatus(status); - } + if (hasSynced != currentStatus.hasSynced) { + final status = SyncStatus( + hasSynced: hasSynced, + lastSyncedAt: lastCompleteSync, + statusInPriority: priorityStatus, + ); + setStatus(status); } } From 6feb3e075d7214fe4d86c4dbf991305dd42b987e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 17 Feb 2025 16:44:24 +0100 Subject: [PATCH 15/25] Fix checking for completed syncs --- .../powersync_core/lib/src/database/powersync_db_mixin.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index 92a37e07..996691bd 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -124,6 +124,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { final result = await database.getAll( 'SELECT priority, last_synced_at FROM ps_sync_state ORDER BY priority;', ); + const prioritySentinel = 2147483647; var hasSynced = false; DateTime? lastCompleteSync; final priorityStatus = []; @@ -136,7 +137,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { final priority = row.columnAt(0) as int; final lastSyncedAt = parseDateTime(row.columnAt(1) as String); - if (priority == -1) { + if (priority == prioritySentinel) { hasSynced = true; lastCompleteSync = lastSyncedAt; } else { From fc650397223122fa53cee128eb423c5f0a1a6146 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 18 Feb 2025 09:55:19 +0100 Subject: [PATCH 16/25] Raise minimum core version --- packages/powersync_core/lib/src/setup_web.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/powersync_core/lib/src/setup_web.dart b/packages/powersync_core/lib/src/setup_web.dart index e4d63d2b..b3ad99c9 100644 --- a/packages/powersync_core/lib/src/setup_web.dart +++ b/packages/powersync_core/lib/src/setup_web.dart @@ -135,7 +135,7 @@ bool coreVersionIsInRange(String tag) { // Sets the range of powersync core version that is compatible with the sqlite3 version // We're a little more selective in the versions chosen here than the range // we're compatible with. - VersionConstraint constraint = VersionConstraint.parse('>=0.3.0 <0.4.0'); + VersionConstraint constraint = VersionConstraint.parse('>=0.3.10 <0.4.0'); List parts = tag.split('-'); String powersyncPart = parts[1]; From fe8805795bb99450354ebd97d4e8b4fb87f3d00a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 18 Feb 2025 10:14:57 +0100 Subject: [PATCH 17/25] Fix web disconnect tests --- packages/powersync_core/lib/src/streaming_sync.dart | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/powersync_core/lib/src/streaming_sync.dart b/packages/powersync_core/lib/src/streaming_sync.dart index c34867e9..e6d2dc1f 100644 --- a/packages/powersync_core/lib/src/streaming_sync.dart +++ b/packages/powersync_core/lib/src/streaming_sync.dart @@ -331,8 +331,11 @@ class StreamingSyncImplementation implements StreamingSync { : (downloadError ?? lastStatus.downloadError), statusInPriority: statusInPriority ?? lastStatus.statusInPriority, ); - lastStatus = newStatus; - _statusStreamController.add(newStatus); + + if (!_statusStreamController.isClosed) { + lastStatus = newStatus; + _statusStreamController.add(newStatus); + } } Future<(List, Map)> From 9df1e92415d5990e9e964f81879963150b5c44a2 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 18 Feb 2025 14:43:25 +0100 Subject: [PATCH 18/25] Don't pass unused priority to validate checkpoint impl --- packages/powersync_core/lib/src/bucket_storage.dart | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/powersync_core/lib/src/bucket_storage.dart b/packages/powersync_core/lib/src/bucket_storage.dart index 08b8d925..344c92ee 100644 --- a/packages/powersync_core/lib/src/bucket_storage.dart +++ b/packages/powersync_core/lib/src/bucket_storage.dart @@ -173,10 +173,7 @@ class BucketStorage { {int? priority}) async { final rs = await select("SELECT powersync_validate_checkpoint(?) as result", [ - jsonEncode({ - ...checkpoint.toJson(priority: priority), - if (priority != null) 'priority': priority, - }) + jsonEncode({...checkpoint.toJson(priority: priority)}) ]); final result = jsonDecode(rs[0]['result'] as String) as Map; From 8ac9f45c9fcf351b3fca1f142bcee14f1e97418b Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 19 Feb 2025 12:31:41 +0100 Subject: [PATCH 19/25] Test partial sync with data --- .../lib/src/bucket_storage.dart | 7 ++-- .../test/in_memory_sync_test.dart | 36 ++++++++++++++++--- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/packages/powersync_core/lib/src/bucket_storage.dart b/packages/powersync_core/lib/src/bucket_storage.dart index 344c92ee..d79e4c05 100644 --- a/packages/powersync_core/lib/src/bucket_storage.dart +++ b/packages/powersync_core/lib/src/bucket_storage.dart @@ -111,13 +111,16 @@ class BucketStorage { } return r; } - final bucketNames = [for (final c in checkpoint.checksums) c.bucket]; + final bucketNames = [ + for (final c in checkpoint.checksums) + if (forPriority == null || c.priority <= forPriority) c.bucket + ]; await writeTransaction((tx) async { await tx.execute( "UPDATE ps_buckets SET last_op = ? WHERE name IN (SELECT json_each.value FROM json_each(?))", [checkpoint.lastOpId, jsonEncode(bucketNames)]); - if (checkpoint.writeCheckpoint != null) { + if (forPriority == null && checkpoint.writeCheckpoint != null) { await tx.execute( "UPDATE ps_buckets SET last_op = ? WHERE name = '\$local'", [checkpoint.writeCheckpoint]); diff --git a/packages/powersync_core/test/in_memory_sync_test.dart b/packages/powersync_core/test/in_memory_sync_test.dart index c7636de3..3cb0e232 100644 --- a/packages/powersync_core/test/in_memory_sync_test.dart +++ b/packages/powersync_core/test/in_memory_sync_test.dart @@ -168,15 +168,35 @@ void main() { final checksums = [ for (var prio = 0; prio <= 3; prio++) - BucketChecksum(bucket: 'prio$prio', priority: prio, checksum: 0) + BucketChecksum( + bucket: 'prio$prio', priority: prio, checksum: 10 + prio) ]; syncService.addLine({ 'checkpoint': Checkpoint( - lastOpId: '0', + lastOpId: '4', writeCheckpoint: null, checksums: checksums, ) }); + var operationId = 1; + + void addRow(int priority) { + syncService.addLine({ + 'data': { + 'bucket': 'prio$priority', + 'data': [ + { + 'checksum': priority + 10, + 'data': {'name': 'test', 'email': 'email'}, + 'op': 'PUT', + 'op_id': '${operationId++}', + 'object_id': 'prio$priority', + 'object_type': 'customers' + } + ] + } + }); + } // Receiving the checkpoint sets the state to downloading await expectLater( @@ -184,9 +204,10 @@ void main() { // Emit partial sync complete for each priority but the last. for (var prio = 0; prio < 3; prio++) { + addRow(prio); syncService.addLine({ 'partial_checkpoint_complete': { - 'last_op_id': '0', + 'last_op_id': operationId.toString(), 'priority': prio, } }); @@ -199,15 +220,22 @@ void main() { isTrue, )), ); + + await database.waitForFirstSync(priority: BucketPriority(prio)); + expect(await database.getAll('SELECT * FROM customers'), + hasLength(prio + 1)); } // Complete the sync + addRow(3); syncService.addLine({ - 'checkpoint_complete': {'last_op_id': '0'} + 'checkpoint_complete': {'last_op_id': operationId.toString()} }); await expectLater( status, emits(isSyncStatus(downloading: false, hasSynced: true))); + await database.waitForFirstSync(); + expect(await database.getAll('SELECT * FROM customers'), hasLength(4)); }); test('remembers last partial sync state', () async { From 4ebad7383f4ebd37076dda33df9f23f640798331 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 21 Feb 2025 16:48:23 +0100 Subject: [PATCH 20/25] Avoid sync "operation" when talking about checkpoints --- .../lib/src/database/powersync_db_mixin.dart | 13 +++++++------ packages/powersync_core/lib/src/sync_status.dart | 11 +++++++---- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index 996691bd..73da4d6e 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -159,13 +159,14 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { } } - /// Returns a [Future] which will resolve once a synchronization operation has - /// completed. + /// Returns a [Future] which will resolve once at least one full sync cycle + /// has completed (meaninng that the first consistent checkpoint has been + /// reached across all buckets). /// - /// When [priority] is null (the default), this method waits for a full sync - /// operation to complete. When set to a [BucketPriority] however, it also - /// completes once a partial sync operation containing that priority has - /// completed. + /// When [priority] is null (the default), this method waits for the first + /// full sync checkpoint to complete. When set to a [BucketPriority] however, + /// it completes once all buckets within that priority (as well as those in + /// higher priorities) have been synchronized at least once. Future waitForFirstSync({BucketPriority? priority}) async { bool matches(SyncStatus status) { if (priority == null) { diff --git a/packages/powersync_core/lib/src/sync_status.dart b/packages/powersync_core/lib/src/sync_status.dart index ff658921..e100871f 100644 --- a/packages/powersync_core/lib/src/sync_status.dart +++ b/packages/powersync_core/lib/src/sync_status.dart @@ -97,12 +97,15 @@ final class SyncStatus { return downloadError ?? uploadError; } - /// Returns [lastSyncedAt] and [hasSynced] information for a partial sync - /// operation, or `null` if the status for that priority is unknown. + /// Returns information for [lastSyncedAt] and [hasSynced] information at a + /// partial sync priority, or `null` if the status for that priority is + /// unknown. /// /// The information returned may be more generic than requested. For instance, - /// a completed sync operation (as expressed by [lastSyncedAt]) also - /// guarantees that every bucket priority was synchronized before that. + /// a fully-completed sync cycle (as expressed by [lastSyncedAt]) necessarily + /// includes all buckets across all priorities. So, if no further partial + /// checkpoints have been received since that complete sync, + /// [statusForPriority] may return information for that complete sync. /// Similarly, requesting the sync status for priority `1` may return /// information extracted from the lower priority `2` since each partial sync /// in priority `2` necessarily includes a consistent view over data in From b060725ffebdf97f3379fa603da8d614a7a4e0d5 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 21 Feb 2025 17:31:25 +0100 Subject: [PATCH 21/25] Adopt priorities in example app --- demos/supabase-todolist/README.md | 19 +++++++++++++++++++ .../lib/widgets/lists_page.dart | 5 ++++- .../lib/widgets/todo_list_page.dart | 19 ++++++++++++++----- .../xcshareddata/xcschemes/Runner.xcscheme | 1 + demos/supabase-todolist/pubspec.lock | 8 ++++---- 5 files changed, 42 insertions(+), 10 deletions(-) diff --git a/demos/supabase-todolist/README.md b/demos/supabase-todolist/README.md index 542aa14d..4845b3e1 100644 --- a/demos/supabase-todolist/README.md +++ b/demos/supabase-todolist/README.md @@ -29,6 +29,25 @@ Create a new PowerSync instance, connecting to the database of the Supabase proj Then deploy the following sync rules: +```yaml +bucket_definitions: + user_lists: + priority: 1 + parameters: select id as list_id from lists where owner_id = request.user_id() + data: + - select * from lists where id = bucket.list_id + + user_todos: + parameters: select id as list_id from lists where owner_id = request.user_id() + data: + - select * from todos where list_id = bucket.list_id +``` + +The rules synchronize list with a higher priority the items within the list. This can be +useful to keep the list overview page reactive during a large sync cycle affecting many +rows in the `user_todos` bucket. The two buckets can also be unified into a single one if +priorities are not important (the app will work without changes): + ```yaml bucket_definitions: user_lists: diff --git a/demos/supabase-todolist/lib/widgets/lists_page.dart b/demos/supabase-todolist/lib/widgets/lists_page.dart index 142d9e9f..5b237b4e 100644 --- a/demos/supabase-todolist/lib/widgets/lists_page.dart +++ b/demos/supabase-todolist/lib/widgets/lists_page.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:flutter/material.dart'; +import 'package:powersync/powersync.dart'; import './list_item.dart'; import './list_item_dialog.dart'; @@ -51,6 +52,8 @@ class ListsWidget extends StatefulWidget { } class _ListsWidgetState extends State { + static final _listsPriority = BucketPriority(1); + List _data = []; bool hasSynced = false; StreamSubscription? _subscription; @@ -75,7 +78,7 @@ class _ListsWidgetState extends State { return; } setState(() { - hasSynced = status.hasSynced ?? false; + hasSynced = status.statusForPriority(_listsPriority).hasSynced ?? false; }); }); } diff --git a/demos/supabase-todolist/lib/widgets/todo_list_page.dart b/demos/supabase-todolist/lib/widgets/todo_list_page.dart index e36e1867..70dde161 100644 --- a/demos/supabase-todolist/lib/widgets/todo_list_page.dart +++ b/demos/supabase-todolist/lib/widgets/todo_list_page.dart @@ -79,11 +79,20 @@ class TodoListWidgetState extends State { @override Widget build(BuildContext context) { - return ListView( - padding: const EdgeInsets.symmetric(vertical: 8.0), - children: _data.map((todo) { - return TodoItemWidget(todo: todo); - }).toList(), + return StreamBuilder( + stream: TodoList.watchSyncStatus().map((e) => e.hasSynced), + builder: (context, snapshot) { + if (snapshot.data ?? false) { + return const Text('Busy with sync'); + } + + return ListView( + padding: const EdgeInsets.symmetric(vertical: 8.0), + children: _data.map((todo) { + return TodoItemWidget(todo: todo); + }).toList(), + ); + }, ); } } diff --git a/demos/supabase-todolist/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/demos/supabase-todolist/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme index 992a778b..943aed19 100644 --- a/demos/supabase-todolist/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +++ b/demos/supabase-todolist/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme @@ -59,6 +59,7 @@ ignoresPersistentStateOnLaunch = "NO" debugDocumentVersioning = "YES" debugServiceExtension = "internal" + enableGPUValidationMode = "1" allowLocationSimulation = "YES"> diff --git a/demos/supabase-todolist/pubspec.lock b/demos/supabase-todolist/pubspec.lock index c6d85781..969bb7bd 100644 --- a/demos/supabase-todolist/pubspec.lock +++ b/demos/supabase-todolist/pubspec.lock @@ -478,28 +478,28 @@ packages: path: "../../packages/powersync" relative: true source: path - version: "1.11.2" + version: "1.11.3" powersync_attachments_helper: dependency: "direct main" description: path: "../../packages/powersync_attachments_helper" relative: true source: path - version: "0.6.18" + version: "0.6.18+1" powersync_core: dependency: "direct overridden" description: path: "../../packages/powersync_core" relative: true source: path - version: "1.1.2" + version: "1.1.3" powersync_flutter_libs: dependency: "direct overridden" description: path: "../../packages/powersync_flutter_libs" relative: true source: path - version: "0.4.4" + version: "0.4.5" pub_semver: dependency: transitive description: From ebc265236fbc143098b4e9b1b8b6fa467d9346d5 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 24 Feb 2025 10:41:32 +0100 Subject: [PATCH 22/25] Update core version --- demos/benchmarks/ios/Podfile.lock | 8 ++++---- demos/django-todolist/ios/Podfile.lock | 8 ++++---- demos/firebase-nodejs-todolist/ios/Podfile.lock | 8 ++++---- demos/supabase-anonymous-auth/ios/Podfile.lock | 8 ++++---- demos/supabase-edge-function-auth/ios/Podfile.lock | 8 ++++---- demos/supabase-simple-chat/ios/Podfile.lock | 8 ++++---- demos/supabase-todolist-drift/ios/Podfile.lock | 8 ++++---- demos/supabase-todolist-optional-sync/ios/Podfile.lock | 8 ++++---- demos/supabase-todolist/ios/Podfile.lock | 8 ++++---- .../xcshareddata/xcschemes/Runner.xcscheme | 1 + demos/supabase-trello/ios/Podfile.lock | 8 ++++---- packages/powersync_flutter_libs/android/build.gradle | 2 +- .../ios/powersync_flutter_libs.podspec | 2 +- .../macos/powersync_flutter_libs.podspec | 2 +- scripts/download_core_binary_demos.dart | 2 +- 15 files changed, 45 insertions(+), 44 deletions(-) diff --git a/demos/benchmarks/ios/Podfile.lock b/demos/benchmarks/ios/Podfile.lock index e9e1beec..c285523d 100644 --- a/demos/benchmarks/ios/Podfile.lock +++ b/demos/benchmarks/ios/Podfile.lock @@ -3,10 +3,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - "sqlite3 (3.46.1+1)": - "sqlite3/common (= 3.46.1+1)" - "sqlite3/common (3.46.1+1)" @@ -50,8 +50,8 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 sqlite3: 0bb0e6389d824e40296f531b858a2a0b71c0d2fb sqlite3_flutter_libs: 9379996d65aa23dcda7585a5b58766cebe0aa042 diff --git a/demos/django-todolist/ios/Podfile.lock b/demos/django-todolist/ios/Podfile.lock index 84722073..257da83e 100644 --- a/demos/django-todolist/ios/Podfile.lock +++ b/demos/django-todolist/ios/Podfile.lock @@ -3,10 +3,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -57,8 +57,8 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 sqlite3_flutter_libs: 3c323550ef3b928bc0aa9513c841e45a7d242832 diff --git a/demos/firebase-nodejs-todolist/ios/Podfile.lock b/demos/firebase-nodejs-todolist/ios/Podfile.lock index 236f21aa..2cc24cac 100644 --- a/demos/firebase-nodejs-todolist/ios/Podfile.lock +++ b/demos/firebase-nodejs-todolist/ios/Podfile.lock @@ -58,10 +58,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - RecaptchaInterop (100.0.0) - shared_preferences_foundation (0.0.1): - Flutter @@ -149,8 +149,8 @@ SPEC CHECKSUMS: GoogleUtilities: 26a3abef001b6533cf678d3eb38fd3f614b7872d GTMSessionFetcher: 257ead9ba8e15a2d389d79496e02b9cc5dd0c62c path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 RecaptchaInterop: 7d1a4a01a6b2cb1610a47ef3f85f0c411434cb21 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 diff --git a/demos/supabase-anonymous-auth/ios/Podfile.lock b/demos/supabase-anonymous-auth/ios/Podfile.lock index ac661d6b..633f5e81 100644 --- a/demos/supabase-anonymous-auth/ios/Podfile.lock +++ b/demos/supabase-anonymous-auth/ios/Podfile.lock @@ -5,10 +5,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -68,8 +68,8 @@ SPEC CHECKSUMS: app_links: 3da4c36b46cac3bf24eb897f1a6ce80bda109874 Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 sqlite3_flutter_libs: 3c323550ef3b928bc0aa9513c841e45a7d242832 diff --git a/demos/supabase-edge-function-auth/ios/Podfile.lock b/demos/supabase-edge-function-auth/ios/Podfile.lock index ac661d6b..633f5e81 100644 --- a/demos/supabase-edge-function-auth/ios/Podfile.lock +++ b/demos/supabase-edge-function-auth/ios/Podfile.lock @@ -5,10 +5,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -68,8 +68,8 @@ SPEC CHECKSUMS: app_links: 3da4c36b46cac3bf24eb897f1a6ce80bda109874 Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 sqlite3_flutter_libs: 3c323550ef3b928bc0aa9513c841e45a7d242832 diff --git a/demos/supabase-simple-chat/ios/Podfile.lock b/demos/supabase-simple-chat/ios/Podfile.lock index a25eeb9d..0534655c 100644 --- a/demos/supabase-simple-chat/ios/Podfile.lock +++ b/demos/supabase-simple-chat/ios/Podfile.lock @@ -5,10 +5,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -68,8 +68,8 @@ SPEC CHECKSUMS: app_links: 3da4c36b46cac3bf24eb897f1a6ce80bda109874 Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 sqlite3_flutter_libs: 3c323550ef3b928bc0aa9513c841e45a7d242832 diff --git a/demos/supabase-todolist-drift/ios/Podfile.lock b/demos/supabase-todolist-drift/ios/Podfile.lock index 933d99fc..2548039b 100644 --- a/demos/supabase-todolist-drift/ios/Podfile.lock +++ b/demos/supabase-todolist-drift/ios/Podfile.lock @@ -7,10 +7,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -74,8 +74,8 @@ SPEC CHECKSUMS: camera_avfoundation: 04b44aeb14070126c6529e5ab82cc7c9fca107cf Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 sqlite3_flutter_libs: 3c323550ef3b928bc0aa9513c841e45a7d242832 diff --git a/demos/supabase-todolist-optional-sync/ios/Podfile.lock b/demos/supabase-todolist-optional-sync/ios/Podfile.lock index 756d4114..e68081bc 100644 --- a/demos/supabase-todolist-optional-sync/ios/Podfile.lock +++ b/demos/supabase-todolist-optional-sync/ios/Podfile.lock @@ -7,10 +7,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -73,8 +73,8 @@ SPEC CHECKSUMS: camera_avfoundation: 7262a4e34c2e028f6aa5fb523ae74c9b74d3bd76 Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 0bb0e6389d824e40296f531b858a2a0b71c0d2fb sqlite3_flutter_libs: 9379996d65aa23dcda7585a5b58766cebe0aa042 diff --git a/demos/supabase-todolist/ios/Podfile.lock b/demos/supabase-todolist/ios/Podfile.lock index ddc83040..a3ff04f2 100644 --- a/demos/supabase-todolist/ios/Podfile.lock +++ b/demos/supabase-todolist/ios/Podfile.lock @@ -7,10 +7,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -74,8 +74,8 @@ SPEC CHECKSUMS: camera_avfoundation: 04b44aeb14070126c6529e5ab82cc7c9fca107cf Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 sqlite3_flutter_libs: 3c323550ef3b928bc0aa9513c841e45a7d242832 diff --git a/demos/supabase-todolist/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/demos/supabase-todolist/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme index 5e31d3d3..c53e2b31 100644 --- a/demos/supabase-todolist/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +++ b/demos/supabase-todolist/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme @@ -48,6 +48,7 @@ ignoresPersistentStateOnLaunch = "NO" debugDocumentVersioning = "YES" debugServiceExtension = "internal" + enableGPUValidationMode = "1" allowLocationSimulation = "YES"> diff --git a/demos/supabase-trello/ios/Podfile.lock b/demos/supabase-trello/ios/Podfile.lock index 51d75873..7c22cdc7 100644 --- a/demos/supabase-trello/ios/Podfile.lock +++ b/demos/supabase-trello/ios/Podfile.lock @@ -41,10 +41,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - SDWebImage (5.20.0): - SDWebImage/Core (= 5.20.0) - SDWebImage/Core (5.20.0) @@ -122,8 +122,8 @@ SPEC CHECKSUMS: Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 image_picker_ios: 7fe1ff8e34c1790d6fff70a32484959f563a928a path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 SDWebImage: 73c6079366fea25fa4bb9640d5fb58f0893facd8 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 diff --git a/packages/powersync_flutter_libs/android/build.gradle b/packages/powersync_flutter_libs/android/build.gradle index 613d3b52..3a1a85a9 100644 --- a/packages/powersync_flutter_libs/android/build.gradle +++ b/packages/powersync_flutter_libs/android/build.gradle @@ -50,5 +50,5 @@ android { } dependencies { - implementation 'co.powersync:powersync-sqlite-core:0.3.10' + implementation 'co.powersync:powersync-sqlite-core:0.3.11' } diff --git a/packages/powersync_flutter_libs/ios/powersync_flutter_libs.podspec b/packages/powersync_flutter_libs/ios/powersync_flutter_libs.podspec index 2e5ab535..4f32ee2c 100644 --- a/packages/powersync_flutter_libs/ios/powersync_flutter_libs.podspec +++ b/packages/powersync_flutter_libs/ios/powersync_flutter_libs.podspec @@ -22,7 +22,7 @@ A new Flutter FFI plugin project. s.dependency 'Flutter' s.platform = :ios, '11.0' - s.dependency "powersync-sqlite-core", "~> 0.3.10" + s.dependency "powersync-sqlite-core", "~> 0.3.11" # Flutter.framework does not contain a i386 slice. s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386' } diff --git a/packages/powersync_flutter_libs/macos/powersync_flutter_libs.podspec b/packages/powersync_flutter_libs/macos/powersync_flutter_libs.podspec index 54e251b4..24632626 100644 --- a/packages/powersync_flutter_libs/macos/powersync_flutter_libs.podspec +++ b/packages/powersync_flutter_libs/macos/powersync_flutter_libs.podspec @@ -21,7 +21,7 @@ A new Flutter FFI plugin project. s.source_files = 'Classes/**/*' s.dependency 'FlutterMacOS' - s.dependency "powersync-sqlite-core", "~> 0.3.10" + s.dependency "powersync-sqlite-core", "~> 0.3.11" s.platform = :osx, '10.11' s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES' } diff --git a/scripts/download_core_binary_demos.dart b/scripts/download_core_binary_demos.dart index e5c53990..77d8f47d 100644 --- a/scripts/download_core_binary_demos.dart +++ b/scripts/download_core_binary_demos.dart @@ -3,7 +3,7 @@ import 'dart:io'; final coreUrl = - 'https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v0.3.10'; + 'https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v0.3.11'; void main() async { final powersyncLibsLinuxPath = "packages/powersync_flutter_libs/linux"; From cfdc111c87abb15cdeab5285e87bc2669d0674c5 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 24 Feb 2025 10:52:14 +0100 Subject: [PATCH 23/25] Don't require priority on checksum --- packages/powersync_core/lib/src/sync_types.dart | 5 ++++- .../powersync_core/test/in_memory_sync_test.dart | 14 +++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/powersync_core/lib/src/sync_types.dart b/packages/powersync_core/lib/src/sync_types.dart index 380ee8eb..968b53d4 100644 --- a/packages/powersync_core/lib/src/sync_types.dart +++ b/packages/powersync_core/lib/src/sync_types.dart @@ -166,7 +166,10 @@ class BucketChecksum { BucketChecksum.fromJson(Map json) : bucket = json['bucket'] as String, - priority = json['priority'] as int, + // Use the default priority (3) as a fallback if the server doesn't send + // priorities. This value is arbitrary though, it won't get used since + // servers not sending priorities also won't send partial checkpoints. + priority = json['priority'] as int? ?? 3, checksum = json['checksum'] as int, count = json['count'] as int?, lastOpId = json['last_op_id'] as String?; diff --git a/packages/powersync_core/test/in_memory_sync_test.dart b/packages/powersync_core/test/in_memory_sync_test.dart index 3cb0e232..258d80fe 100644 --- a/packages/powersync_core/test/in_memory_sync_test.dart +++ b/packages/powersync_core/test/in_memory_sync_test.dart @@ -75,11 +75,15 @@ void main() { final status = await waitForConnection(); syncService.addLine({ - 'checkpoint': Checkpoint( - lastOpId: '0', - writeCheckpoint: null, - checksums: [BucketChecksum(bucket: 'bkt', priority: 1, checksum: 0)], - ) + 'checkpoint': { + 'last_op_id': '0', + 'buckets': [ + { + 'bucket': 'bkt', + 'checksum': 0, + } + ], + }, }); await expectLater(status, emits(isSyncStatus(downloading: true))); From 505acb76d551cd4cf3c36f713bfcf7ace9cb484c Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 24 Feb 2025 14:51:38 +0100 Subject: [PATCH 24/25] Clean up wait for first sync in lists widget --- demos/supabase-todolist/README.md | 3 +- .../lib/widgets/lists_page.dart | 82 ++++++------------- 2 files changed, 29 insertions(+), 56 deletions(-) diff --git a/demos/supabase-todolist/README.md b/demos/supabase-todolist/README.md index 4845b3e1..55241dde 100644 --- a/demos/supabase-todolist/README.md +++ b/demos/supabase-todolist/README.md @@ -43,7 +43,8 @@ bucket_definitions: - select * from todos where list_id = bucket.list_id ``` -The rules synchronize list with a higher priority the items within the list. This can be +**Note**: These rules showcase [prioritized sync](https://docs.powersync.com/usage/use-case-examples/prioritized-sync), +by syncing a user's lists with a higher priority than the items within a list (todos). This can be useful to keep the list overview page reactive during a large sync cycle affecting many rows in the `user_todos` bucket. The two buckets can also be unified into a single one if priorities are not important (the app will work without changes): diff --git a/demos/supabase-todolist/lib/widgets/lists_page.dart b/demos/supabase-todolist/lib/widgets/lists_page.dart index 5b237b4e..d60cb9f5 100644 --- a/demos/supabase-todolist/lib/widgets/lists_page.dart +++ b/demos/supabase-todolist/lib/widgets/lists_page.dart @@ -1,7 +1,6 @@ -import 'dart:async'; - import 'package:flutter/material.dart'; import 'package:powersync/powersync.dart'; +import 'package:powersync_flutter_demo/powersync.dart'; import './list_item.dart'; import './list_item_dialog.dart'; @@ -42,63 +41,36 @@ class ListsPage extends StatelessWidget { } } -class ListsWidget extends StatefulWidget { +final class ListsWidget extends StatelessWidget { const ListsWidget({super.key}); - @override - State createState() { - return _ListsWidgetState(); - } -} - -class _ListsWidgetState extends State { - static final _listsPriority = BucketPriority(1); - - List _data = []; - bool hasSynced = false; - StreamSubscription? _subscription; - StreamSubscription? _syncStatusSubscription; - - _ListsWidgetState(); - - @override - void initState() { - super.initState(); - final stream = TodoList.watchListsWithStats(); - _subscription = stream.listen((data) { - if (!context.mounted) { - return; - } - setState(() { - _data = data; - }); - }); - _syncStatusSubscription = TodoList.watchSyncStatus().listen((status) { - if (!context.mounted) { - return; - } - setState(() { - hasSynced = status.statusForPriority(_listsPriority).hasSynced ?? false; - }); - }); - } - - @override - void dispose() { - super.dispose(); - _subscription?.cancel(); - _syncStatusSubscription?.cancel(); - } - @override Widget build(BuildContext context) { - return !hasSynced - ? const Text("Busy with sync...") - : ListView( - padding: const EdgeInsets.symmetric(vertical: 8.0), - children: _data.map((list) { - return ListItemWidget(list: list); - }).toList(), + return FutureBuilder( + future: db.waitForFirstSync(priority: _listsPriority), + builder: (context, snapshot) { + if (snapshot.connectionState == ConnectionState.done) { + return StreamBuilder( + stream: TodoList.watchListsWithStats(), + builder: (context, snapshot) { + if (snapshot.data case final todoLists?) { + return ListView( + padding: const EdgeInsets.symmetric(vertical: 8.0), + children: todoLists.map((list) { + return ListItemWidget(list: list); + }).toList(), + ); + } else { + return const CircularProgressIndicator(); + } + }, ); + } else { + return const Text('Busy with sync...'); + } + }, + ); } + + static final _listsPriority = BucketPriority(1); } From 12c4124d869baca82c5ecf5d68447b2bc717de76 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 25 Feb 2025 15:15:11 +0100 Subject: [PATCH 25/25] Rename statusInPriority to priorityStatusEntries --- .../lib/src/database/powersync_db_mixin.dart | 6 +-- .../lib/src/streaming_sync.dart | 38 ++++++------------- .../powersync_core/lib/src/sync_status.dart | 29 ++++++++------ .../lib/src/web/sync_worker_protocol.dart | 10 ++--- .../powersync_core/test/sync_types_test.dart | 8 ++++ 5 files changed, 45 insertions(+), 46 deletions(-) diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index 73da4d6e..564e8f5a 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -127,7 +127,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { const prioritySentinel = 2147483647; var hasSynced = false; DateTime? lastCompleteSync; - final priorityStatus = []; + final priorityStatusEntries = []; DateTime parseDateTime(String sql) { return DateTime.parse('${sql}Z').toLocal(); @@ -141,7 +141,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { hasSynced = true; lastCompleteSync = lastSyncedAt; } else { - priorityStatus.add(( + priorityStatusEntries.add(( hasSynced: true, lastSyncedAt: lastSyncedAt, priority: BucketPriority(priority) @@ -153,7 +153,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { final status = SyncStatus( hasSynced: hasSynced, lastSyncedAt: lastCompleteSync, - statusInPriority: priorityStatus, + priorityStatusEntries: priorityStatusEntries, ); setStatus(status); } diff --git a/packages/powersync_core/lib/src/streaming_sync.dart b/packages/powersync_core/lib/src/streaming_sync.dart index e6d2dc1f..716b38bf 100644 --- a/packages/powersync_core/lib/src/streaming_sync.dart +++ b/packages/powersync_core/lib/src/streaming_sync.dart @@ -276,30 +276,13 @@ class StreamingSyncImplementation implements StreamingSync { } void _updateStatusForPriority(SyncPriorityStatus completed) { - // Note: statusInPriority is sorted by priorities (ascending) - final existingPriorityState = lastStatus.statusInPriority; - - for (final (i, priority) in existingPriorityState.indexed) { - switch ( - BucketPriority.comparator(priority.priority, completed.priority)) { - case > 0: - // Entries from here on have a higher priority than the one that was - // just completed - final copy = existingPriorityState.toList(); - copy.insert(i, completed); - _updateStatus(statusInPriority: copy); - return; - case 0: - final copy = existingPriorityState.toList(); - copy[i] = completed; - _updateStatus(statusInPriority: copy); - return; - case < 0: - continue; - } - } - - _updateStatus(statusInPriority: [...existingPriorityState, completed]); + // All status entries with a higher priority can be deleted since this + // partial sync includes them. + _updateStatus(priorityStatusEntries: [ + for (final entry in lastStatus.priorityStatusEntries) + if (entry.priority < completed.priority) entry, + completed + ]); } /// Update sync status based on any non-null parameters. @@ -313,7 +296,7 @@ class StreamingSyncImplementation implements StreamingSync { bool? uploading, Object? uploadError, Object? downloadError, - List? statusInPriority, + List? priorityStatusEntries, }) { final c = connected ?? lastStatus.connected; var newStatus = SyncStatus( @@ -329,7 +312,8 @@ class StreamingSyncImplementation implements StreamingSync { downloadError: downloadError == _noError ? null : (downloadError ?? lastStatus.downloadError), - statusInPriority: statusInPriority ?? lastStatus.statusInPriority, + priorityStatusEntries: + priorityStatusEntries ?? lastStatus.priorityStatusEntries, ); if (!_statusStreamController.isClosed) { @@ -412,7 +396,7 @@ class StreamingSyncImplementation implements StreamingSync { downloading: false, downloadError: _noError, lastSyncedAt: now, - statusInPriority: [ + priorityStatusEntries: [ if (appliedCheckpoint.checksums.isNotEmpty) ( hasSynced: true, diff --git a/packages/powersync_core/lib/src/sync_status.dart b/packages/powersync_core/lib/src/sync_status.dart index e100871f..3d883757 100644 --- a/packages/powersync_core/lib/src/sync_status.dart +++ b/packages/powersync_core/lib/src/sync_status.dart @@ -40,7 +40,7 @@ final class SyncStatus { /// Cleared on the next successful data download. final Object? downloadError; - final List statusInPriority; + final List priorityStatusEntries; const SyncStatus({ this.connected = false, @@ -51,7 +51,7 @@ final class SyncStatus { this.uploading = false, this.downloadError, this.uploadError, - this.statusInPriority = const [], + this.priorityStatusEntries = const [], }); @override @@ -65,7 +65,8 @@ final class SyncStatus { other.uploadError == uploadError && other.lastSyncedAt == lastSyncedAt && other.hasSynced == hasSynced && - _statusEquality.equals(other.statusInPriority, statusInPriority)); + _statusEquality.equals( + other.priorityStatusEntries, priorityStatusEntries)); } SyncStatus copyWith({ @@ -77,7 +78,7 @@ final class SyncStatus { Object? downloadError, DateTime? lastSyncedAt, bool? hasSynced, - List? statusInPriority, + List? priorityStatusEntries, }) { return SyncStatus( connected: connected ?? this.connected, @@ -88,7 +89,8 @@ final class SyncStatus { downloadError: downloadError ?? this.downloadError, lastSyncedAt: lastSyncedAt ?? this.lastSyncedAt, hasSynced: hasSynced ?? this.hasSynced, - statusInPriority: statusInPriority ?? this.statusInPriority, + priorityStatusEntries: + priorityStatusEntries ?? this.priorityStatusEntries, ); } @@ -111,14 +113,14 @@ final class SyncStatus { /// in priority `2` necessarily includes a consistent view over data in /// priority `1`. SyncPriorityStatus statusForPriority(BucketPriority priority) { - assert(statusInPriority.isSortedByCompare( + assert(priorityStatusEntries.isSortedByCompare( (e) => e.priority, BucketPriority.comparator)); - for (final known in statusInPriority) { + for (final known in priorityStatusEntries) { // Lower-priority buckets are synchronized after higher-priority buckets, - // and since statusInPriority is sorted we look for the first entry that - // doesn't have a higher priority. - if (BucketPriority.comparator(known.priority, priority) <= 0) { + // and since priorityStatusEntries is sorted we look for the first entry + // that doesn't have a higher priority. + if (known.priority <= priority) { return known; } } @@ -141,7 +143,7 @@ final class SyncStatus { uploadError, downloadError, lastSyncedAt, - _statusEquality.hash(statusInPriority)); + _statusEquality.hash(priorityStatusEntries)); } @override @@ -161,6 +163,11 @@ extension type const BucketPriority._(int priorityNumber) { return BucketPriority._(i); } + bool operator >(BucketPriority other) => comparator(this, other) > 0; + bool operator >=(BucketPriority other) => comparator(this, other) >= 0; + bool operator <(BucketPriority other) => comparator(this, other) < 0; + bool operator <=(BucketPriority other) => comparator(this, other) <= 0; + /// A [Comparator] instance suitable for comparing [BucketPriority] values. static int comparator(BucketPriority a, BucketPriority b) => -a.priorityNumber.compareTo(b.priorityNumber); diff --git a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart index 252e0220..8a064e1e 100644 --- a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart +++ b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart @@ -157,7 +157,7 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { required bool? hasSyned, required String? uploadError, required String? downloadError, - required JSArray? statusInPriority, + required JSArray? priorityStatusEntries, }); factory SerializedSyncStatus.from(SyncStatus status) { @@ -170,8 +170,8 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { hasSyned: status.hasSynced, uploadError: status.uploadError?.toString(), downloadError: status.downloadError?.toString(), - statusInPriority: [ - for (final entry in status.statusInPriority) + priorityStatusEntries: [ + for (final entry in status.priorityStatusEntries) [ entry.priority.priorityNumber.toJS, entry.lastSyncedAt?.microsecondsSinceEpoch.toJS, @@ -189,7 +189,7 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { external bool? hasSynced; external String? uploadError; external String? downloadError; - external JSArray? statusInPriority; + external JSArray? priorityStatusEntries; SyncStatus asSyncStatus() { return SyncStatus( @@ -203,7 +203,7 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { hasSynced: hasSynced, uploadError: uploadError, downloadError: downloadError, - statusInPriority: statusInPriority?.toDart.map((e) { + priorityStatusEntries: priorityStatusEntries?.toDart.map((e) { final [rawPriority, rawSynced, rawHasSynced, ...] = (e as JSArray).toDart; final syncedMillis = (rawSynced as JSNumber?)?.toDartInt; diff --git a/packages/powersync_core/test/sync_types_test.dart b/packages/powersync_core/test/sync_types_test.dart index fa4e71fa..18e06931 100644 --- a/packages/powersync_core/test/sync_types_test.dart +++ b/packages/powersync_core/test/sync_types_test.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:powersync_core/src/sync_status.dart'; import 'package:powersync_core/src/sync_types.dart'; import 'package:test/test.dart'; @@ -214,5 +215,12 @@ void main() { }); } }); + + test('bucket priority comparisons', () { + expect(BucketPriority(0) < BucketPriority(3), isFalse); + expect(BucketPriority(0) > BucketPriority(3), isTrue); + expect(BucketPriority(0) >= BucketPriority(3), isTrue); + expect(BucketPriority(0) >= BucketPriority(0), isTrue); + }); }); }