From 5c25fcdf070189223f370c2b942114621690aa67 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 29 Feb 2024 16:52:04 +0200 Subject: [PATCH 01/21] Fix issue with re-using a shared Mutex. --- lib/src/isolate_connection_factory.dart | 8 ++++---- lib/src/mutex.dart | 18 ++---------------- test/mutex_test.dart | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/lib/src/isolate_connection_factory.dart b/lib/src/isolate_connection_factory.dart index d6ab74d..adcd37c 100644 --- a/lib/src/isolate_connection_factory.dart +++ b/lib/src/isolate_connection_factory.dart @@ -37,8 +37,8 @@ class IsolateConnectionFactory { readOnly: readOnly, debugName: debugName, updates: updates.stream, - closeFunction: () { - openMutex.close(); + closeFunction: () async { + await openMutex.close(); updates.close(); }); } @@ -89,7 +89,7 @@ class _IsolateUpdateListener { } class _IsolateSqliteConnection extends SqliteConnectionImpl { - final void Function() closeFunction; + final Future Function() closeFunction; _IsolateSqliteConnection( {required super.openFactory, @@ -103,6 +103,6 @@ class _IsolateSqliteConnection extends SqliteConnectionImpl { @override Future close() async { await super.close(); - closeFunction(); + await closeFunction(); } } diff --git a/lib/src/mutex.dart b/lib/src/mutex.dart index 5725554..54179f7 100644 --- a/lib/src/mutex.dart +++ b/lib/src/mutex.dart @@ -188,7 +188,8 @@ class SharedMutex implements Mutex { closed = true; // Wait for any existing locks to complete, then prevent any further locks from being taken out. await _acquire(); - client.fire(const _CloseMessage()); + // Release the lock + _unlock(); // Close client immediately after _unlock(), // so that we're sure no further locks are acquired. // This also cancels any lock request in process. @@ -201,7 +202,6 @@ class _SharedMutexServer { Completer? unlock; late final SerializedMutex serialized; final Mutex mutex; - bool closed = false; late final PortServer server; @@ -216,11 +216,6 @@ class _SharedMutexServer { if (arg is _AcquireMessage) { var lock = Completer.sync(); mutex.lock(() async { - if (closed) { - // The client will error already - we just need to ensure - // we don't take out another lock. - return; - } assert(unlock == null); unlock = Completer.sync(); lock.complete(); @@ -231,10 +226,6 @@ class _SharedMutexServer { } else if (arg is _UnlockMessage) { assert(unlock != null); unlock!.complete(); - } else if (arg is _CloseMessage) { - // Unlock and close (from client side) - closed = true; - unlock?.complete(); } } @@ -251,11 +242,6 @@ class _UnlockMessage { const _UnlockMessage(); } -/// Unlock and close -class _CloseMessage { - const _CloseMessage(); -} - class LockError extends Error { final String message; diff --git a/test/mutex_test.dart b/test/mutex_test.dart index 0eb90e1..297bf53 100644 --- a/test/mutex_test.dart +++ b/test/mutex_test.dart @@ -22,6 +22,25 @@ void main() { expect(result, equals(5)); } }); + + test('Re-use after closing', () async { + // Test that shared locks can be opened and closed multiple times. + final mutex = SimpleMutex(); + final serialized = mutex.shared; + + final result = await Isolate.run(() async { + return _lockInIsolate(serialized); + }); + + final result2 = await Isolate.run(() async { + return _lockInIsolate(serialized); + }); + + await mutex.lock(() async {}); + + expect(result, equals(5)); + expect(result2, equals(5)); + }); }, timeout: const Timeout(Duration(milliseconds: 5000))); } From 5dbcb64ebb625bfe5c4c4f794a08157cf2cc5be5 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 2 Apr 2024 12:05:41 +0200 Subject: [PATCH 02/21] Use busy-timeout for database-level locks. --- lib/src/sqlite_connection_impl.dart | 25 +++++----------------- lib/src/sqlite_open_factory.dart | 22 +++++++++++++++++-- lib/src/sqlite_options.dart | 11 ++++++++-- test/basic_test.dart | 33 +++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 24 deletions(-) diff --git a/lib/src/sqlite_connection_impl.dart b/lib/src/sqlite_connection_impl.dart index 643152e..9fffa08 100644 --- a/lib/src/sqlite_connection_impl.dart +++ b/lib/src/sqlite_connection_impl.dart @@ -123,30 +123,15 @@ class SqliteConnectionImpl with SqliteQueries implements SqliteConnection { @override Future writeLock(Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { - final stopWatch = lockTimeout == null ? null : (Stopwatch()..start()); // Private lock to synchronize this with other statements on the same connection, // to ensure that transactions aren't interleaved. return await _connectionMutex.lock(() async { - Duration? innerTimeout; - if (lockTimeout != null && stopWatch != null) { - innerTimeout = lockTimeout - stopWatch.elapsed; - stopWatch.stop(); + final ctx = _TransactionContext(_isolateClient); + try { + return await callback(ctx); + } finally { + await ctx.close(); } - // DB lock so that only one write happens at a time - return await _writeMutex.lock(() async { - final ctx = _TransactionContext(_isolateClient); - try { - return await callback(ctx); - } finally { - await ctx.close(); - } - }, timeout: innerTimeout).catchError((error, stackTrace) { - if (error is TimeoutException) { - return Future.error(TimeoutException( - 'Failed to acquire global write lock', lockTimeout)); - } - return Future.error(error, stackTrace); - }); }, timeout: lockTimeout); } } diff --git a/lib/src/sqlite_open_factory.dart b/lib/src/sqlite_open_factory.dart index 4f1845a..97f1647 100644 --- a/lib/src/sqlite_open_factory.dart +++ b/lib/src/sqlite_open_factory.dart @@ -1,6 +1,6 @@ import 'dart:async'; -import 'package:sqlite3/sqlite3.dart' as sqlite; +import 'package:sqlite_async/sqlite3.dart' as sqlite; import 'sqlite_options.dart'; @@ -29,6 +29,11 @@ class DefaultSqliteOpenFactory implements SqliteOpenFactory { List pragmaStatements(SqliteOpenOptions options) { List statements = []; + if (sqliteOptions.busyTimeout != null) { + statements.add( + 'PRAGMA busy_timeout = ${sqliteOptions.busyTimeout!.inMilliseconds}'); + } + if (options.primaryConnection && sqliteOptions.journalMode != null) { // Persisted - only needed on the primary connection statements @@ -51,8 +56,21 @@ class DefaultSqliteOpenFactory implements SqliteOpenFactory { final mode = options.openMode; var db = sqlite.sqlite3.open(path, mode: mode, mutex: false); + // Pragma statements don't have the same BUSY_TIMEOUT behavior as normal statements. + // We add a manual retry loop for those. for (var statement in pragmaStatements(options)) { - db.execute(statement); + for (var tries = 0; tries < 30; tries++) { + try { + db.execute(statement); + break; + } on sqlite.SqliteException catch (e) { + if (e.resultCode == sqlite.SqlError.SQLITE_BUSY && tries < 29) { + continue; + } else { + rethrow; + } + } + } } return db; } diff --git a/lib/src/sqlite_options.dart b/lib/src/sqlite_options.dart index 36beb7c..87d7ee8 100644 --- a/lib/src/sqlite_options.dart +++ b/lib/src/sqlite_options.dart @@ -11,15 +11,22 @@ class SqliteOptions { /// attempt to truncate the file afterwards. final int? journalSizeLimit; + /// Timeout waiting for locks to be released by other write connections. + /// Defaults to 30 seconds. + /// Set to 0 to fail immediately when the database is locked. + final Duration? busyTimeout; + const SqliteOptions.defaults() : journalMode = SqliteJournalMode.wal, journalSizeLimit = 6 * 1024 * 1024, // 1.5x the default checkpoint size - synchronous = SqliteSynchronous.normal; + synchronous = SqliteSynchronous.normal, + busyTimeout = const Duration(seconds: 30); const SqliteOptions( {this.journalMode = SqliteJournalMode.wal, this.journalSizeLimit = 6 * 1024 * 1024, - this.synchronous = SqliteSynchronous.normal}); + this.synchronous = SqliteSynchronous.normal, + this.busyTimeout = const Duration(seconds: 30)}); } /// SQLite journal mode. Set on the primary connection. diff --git a/test/basic_test.dart b/test/basic_test.dart index 9b563c8..55a7d12 100644 --- a/test/basic_test.dart +++ b/test/basic_test.dart @@ -64,6 +64,39 @@ void main() { } }); + test('Concurrency 2', () async { + final db1 = + SqliteDatabase.withFactory(testFactory(path: path), maxReaders: 3); + + final db2 = + SqliteDatabase.withFactory(testFactory(path: path), maxReaders: 3); + await db1.initialize(); + await createTables(db1); + await db2.initialize(); + print("${DateTime.now()} start"); + + var futures1 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11].map((i) { + return db1.execute( + "INSERT OR REPLACE INTO test_data(id, description) SELECT ? as i, test_sleep(?) || ' ' || test_connection_name() || ' 1 ' || datetime() as connection RETURNING *", + [ + i, + 5 + Random().nextInt(20) + ]).then((value) => print("${DateTime.now()} $value")); + }).toList(); + + var futures2 = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11].map((i) { + return db2.execute( + "INSERT OR REPLACE INTO test_data(id, description) SELECT ? as i, test_sleep(?) || ' ' || test_connection_name() || ' 2 ' || datetime() as connection RETURNING *", + [ + i, + 5 + Random().nextInt(20) + ]).then((value) => print("${DateTime.now()} $value")); + }).toList(); + await Future.wait(futures1); + await Future.wait(futures2); + print("${DateTime.now()} done"); + }); + test('read-only transactions', () async { final db = await setupDatabase(path: path); await createTables(db); From b8a68f9d1f5db4e01d5b8679af8c094b484db540 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 2 Apr 2024 14:38:37 +0200 Subject: [PATCH 03/21] Revert mutex change. --- lib/src/sqlite_connection_impl.dart | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/lib/src/sqlite_connection_impl.dart b/lib/src/sqlite_connection_impl.dart index 9fffa08..643152e 100644 --- a/lib/src/sqlite_connection_impl.dart +++ b/lib/src/sqlite_connection_impl.dart @@ -123,15 +123,30 @@ class SqliteConnectionImpl with SqliteQueries implements SqliteConnection { @override Future writeLock(Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { + final stopWatch = lockTimeout == null ? null : (Stopwatch()..start()); // Private lock to synchronize this with other statements on the same connection, // to ensure that transactions aren't interleaved. return await _connectionMutex.lock(() async { - final ctx = _TransactionContext(_isolateClient); - try { - return await callback(ctx); - } finally { - await ctx.close(); + Duration? innerTimeout; + if (lockTimeout != null && stopWatch != null) { + innerTimeout = lockTimeout - stopWatch.elapsed; + stopWatch.stop(); } + // DB lock so that only one write happens at a time + return await _writeMutex.lock(() async { + final ctx = _TransactionContext(_isolateClient); + try { + return await callback(ctx); + } finally { + await ctx.close(); + } + }, timeout: innerTimeout).catchError((error, stackTrace) { + if (error is TimeoutException) { + return Future.error(TimeoutException( + 'Failed to acquire global write lock', lockTimeout)); + } + return Future.error(error, stackTrace); + }); }, timeout: lockTimeout); } } From 713cb128c2232e6fcfe02c3d942040a12cd0c6b6 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 2 Apr 2024 14:41:00 +0200 Subject: [PATCH 04/21] Rename to lockTimeout. --- lib/src/sqlite_open_factory.dart | 5 +++-- lib/src/sqlite_options.dart | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/src/sqlite_open_factory.dart b/lib/src/sqlite_open_factory.dart index 97f1647..b4779ef 100644 --- a/lib/src/sqlite_open_factory.dart +++ b/lib/src/sqlite_open_factory.dart @@ -29,9 +29,10 @@ class DefaultSqliteOpenFactory implements SqliteOpenFactory { List pragmaStatements(SqliteOpenOptions options) { List statements = []; - if (sqliteOptions.busyTimeout != null) { + if (sqliteOptions.lockTimeout != null) { + // May be replaced by a Dart-level retry mechanism in the future statements.add( - 'PRAGMA busy_timeout = ${sqliteOptions.busyTimeout!.inMilliseconds}'); + 'PRAGMA busy_timeout = ${sqliteOptions.lockTimeout!.inMilliseconds}'); } if (options.primaryConnection && sqliteOptions.journalMode != null) { diff --git a/lib/src/sqlite_options.dart b/lib/src/sqlite_options.dart index 87d7ee8..9602fdd 100644 --- a/lib/src/sqlite_options.dart +++ b/lib/src/sqlite_options.dart @@ -11,22 +11,22 @@ class SqliteOptions { /// attempt to truncate the file afterwards. final int? journalSizeLimit; - /// Timeout waiting for locks to be released by other write connections. + /// Timeout waiting for locks to be released by other connections. /// Defaults to 30 seconds. - /// Set to 0 to fail immediately when the database is locked. - final Duration? busyTimeout; + /// Set to null or [Duration.zero] to fail immediately when the database is locked. + final Duration? lockTimeout; const SqliteOptions.defaults() : journalMode = SqliteJournalMode.wal, journalSizeLimit = 6 * 1024 * 1024, // 1.5x the default checkpoint size synchronous = SqliteSynchronous.normal, - busyTimeout = const Duration(seconds: 30); + lockTimeout = const Duration(seconds: 30); const SqliteOptions( {this.journalMode = SqliteJournalMode.wal, this.journalSizeLimit = 6 * 1024 * 1024, this.synchronous = SqliteSynchronous.normal, - this.busyTimeout = const Duration(seconds: 30)}); + this.lockTimeout = const Duration(seconds: 30)}); } /// SQLite journal mode. Set on the primary connection. From b8121c2ccd4981e6559f27021957c9a46e3f5c11 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 2 Apr 2024 16:43:39 +0200 Subject: [PATCH 05/21] Bump dart SDK. --- .github/workflows/test.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c9fa2df..f3882c0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,19 +32,19 @@ jobs: include: - sqlite_version: "3440200" sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3440200.tar.gz" - dart_sdk: 3.2.4 + dart_sdk: 3.3.3 - sqlite_version: "3430200" sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3430200.tar.gz" - dart_sdk: 3.2.4 + dart_sdk: 3.3.3 - sqlite_version: "3420000" sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3420000.tar.gz" - dart_sdk: 3.2.4 + dart_sdk: 3.3.3 - sqlite_version: "3410100" sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3410100.tar.gz" - dart_sdk: 3.2.4 + dart_sdk: 3.3.3 - sqlite_version: "3380000" sqlite_url: "https://www.sqlite.org/2022/sqlite-autoconf-3380000.tar.gz" - dart_sdk: 3.2.0 + dart_sdk: 3.3.3 steps: - uses: actions/checkout@v3 - uses: dart-lang/setup-dart@v1 From 28cf2667427cf5da8b4faa12f11c9fc6e3767054 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 3 Apr 2024 09:19:07 +0200 Subject: [PATCH 06/21] Clarify docs on multiple instances. --- lib/src/sqlite_database.dart | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/src/sqlite_database.dart b/lib/src/sqlite_database.dart index c937743..d792092 100644 --- a/lib/src/sqlite_database.dart +++ b/lib/src/sqlite_database.dart @@ -15,8 +15,10 @@ import 'update_notification.dart'; /// A SQLite database instance. /// -/// Use one instance per database file. If multiple instances are used, update -/// notifications may not trigger, and calls may fail with "SQLITE_BUSY" errors. +/// Use one instance per database file where feasible. +/// +/// If multiple instances are used, update notifications will not be propagated between them. +/// For update notifications across isolates, use [isolateConnectionFactory]. class SqliteDatabase with SqliteQueries implements SqliteConnection { /// The maximum number of concurrent read transactions if not explicitly specified. static const int defaultMaxReaders = 5; From eef16ade3d7ea7fb02dc34ffc90694bdf710ff8e Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 3 Apr 2024 12:04:17 +0200 Subject: [PATCH 07/21] Add failing test. --- test/basic_test.dart | 24 ++++++++++++++++++++++++ test/util.dart | 25 ++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/test/basic_test.dart b/test/basic_test.dart index 9b563c8..2e4804a 100644 --- a/test/basic_test.dart +++ b/test/basic_test.dart @@ -368,6 +368,30 @@ void main() { expect(await savedTx!.getAutoCommit(), equals(true)); expect(savedTx!.closed, equals(true)); }); + + test('closing', () async { + // Test race condition in SqliteConnectionPool: + // 1. Open two concurrent queries, which opens two connection. + // 2. Second connection takes longer to open than first. + // 3. Call db.close(). + // 4. Now second connection is ready. Second query has two connections to choose from. + // 5. However, first connection is closed, so it's removed from the pool. + // 6. Triggers `Concurrent modification during iteration: Instance(length:1) of '_GrowableList'` + + final db = + SqliteDatabase.withFactory(testFactory(path: path, initStatements: [ + // Second connection to sleep more than first connection + 'SELECT test_sleep(test_connection_number() * 10)' + ])); + await createTables(db); + + final future1 = db.get('SELECT test_sleep(10) as sleep'); + final future2 = db.get('SELECT test_sleep(10) as sleep'); + final closeFuture = db.close(); + await closeFuture; + await future1; + await future2; + }); }); } diff --git a/test/util.dart b/test/util.dart index 12458fc..5bcbb50 100644 --- a/test/util.dart +++ b/test/util.dart @@ -14,11 +14,13 @@ const defaultSqlitePath = 'libsqlite3.so.0'; class TestSqliteOpenFactory extends DefaultSqliteOpenFactory { String sqlitePath; + List initStatements; TestSqliteOpenFactory( {required super.path, super.sqliteOptions, - this.sqlitePath = defaultSqlitePath}); + this.sqlitePath = defaultSqlitePath, + this.initStatements = const []}); @override sqlite.Database open(SqliteOpenOptions options) { @@ -45,12 +47,29 @@ class TestSqliteOpenFactory extends DefaultSqliteOpenFactory { }, ); + db.createFunction( + functionName: 'test_connection_number', + argumentCount: const sqlite.AllowedArgumentCount(0), + function: (args) { + // write: 0, read: 1 - 5 + final name = Isolate.current.debugName ?? '-0'; + var nr = name.split('-').last; + return int.tryParse(nr) ?? 0; + }, + ); + + for (var s in initStatements) { + db.execute(s); + } + return db; } } -SqliteOpenFactory testFactory({String? path}) { - return TestSqliteOpenFactory(path: path ?? dbPath()); +SqliteOpenFactory testFactory( + {String? path, List initStatements = const []}) { + return TestSqliteOpenFactory( + path: path ?? dbPath(), initStatements: initStatements); } Future setupDatabase({String? path}) async { From 84173344b0e2b3fed0be4553cb2a6c5e8cd778b9 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 3 Apr 2024 12:04:48 +0200 Subject: [PATCH 08/21] Use ClosedException. --- lib/src/connection_pool.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/src/connection_pool.dart b/lib/src/connection_pool.dart index 23b1e0f..5b5316b 100644 --- a/lib/src/connection_pool.dart +++ b/lib/src/connection_pool.dart @@ -53,7 +53,7 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { @override Future getAutoCommit() async { if (_writeConnection == null) { - throw AssertionError('Closed'); + throw ClosedException(); } return await _writeConnection!.getAutoCommit(); } @@ -118,7 +118,7 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { Future writeLock(Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, String? debugContext}) { if (closed) { - throw AssertionError('Closed'); + throw ClosedException(); } if (_writeConnection?.closed == true) { _writeConnection = null; From a7221567a4427913609c12ffd69528748fba2ce7 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 3 Apr 2024 12:07:10 +0200 Subject: [PATCH 09/21] Fix "Concurrent modification during iteration" errors. --- lib/src/connection_pool.dart | 7 ++++++- test/basic_test.dart | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/src/connection_pool.dart b/lib/src/connection_pool.dart index 5b5316b..6a99a59 100644 --- a/lib/src/connection_pool.dart +++ b/lib/src/connection_pool.dart @@ -183,7 +183,12 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { @override Future close() async { closed = true; - for (var connection in _readConnections) { + + // It is possible that `readLock()` removes connections from the pool while we're + // closing connections, but not possible for new connections to be added. + // Create a copy of the list, to avoid this triggering "Concurrent modification during iteration" + final toClose = _readConnections.sublist(0); + for (var connection in toClose) { await connection.close(); } // Closing the write connection cleans up the journal files (-shm and -wal files). diff --git a/test/basic_test.dart b/test/basic_test.dart index 2e4804a..4bc9a42 100644 --- a/test/basic_test.dart +++ b/test/basic_test.dart @@ -383,7 +383,6 @@ void main() { // Second connection to sleep more than first connection 'SELECT test_sleep(test_connection_number() * 10)' ])); - await createTables(db); final future1 = db.get('SELECT test_sleep(10) as sleep'); final future2 = db.get('SELECT test_sleep(10) as sleep'); From dea85050b910e2c7629151370e8845f648095b38 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 3 Apr 2024 12:39:29 +0200 Subject: [PATCH 10/21] Handle ClosedException in SqliteConnectionPool. --- lib/src/connection_pool.dart | 2 ++ test/basic_test.dart | 1 + 2 files changed, 3 insertions(+) diff --git a/lib/src/connection_pool.dart b/lib/src/connection_pool.dart index 6a99a59..f6ae300 100644 --- a/lib/src/connection_pool.dart +++ b/lib/src/connection_pool.dart @@ -94,6 +94,8 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { }, lockTimeout: lockTimeout, debugContext: debugContext); } on TimeoutException { return false; + } on ClosedException { + return false; } }); diff --git a/test/basic_test.dart b/test/basic_test.dart index 4bc9a42..8ed43dc 100644 --- a/test/basic_test.dart +++ b/test/basic_test.dart @@ -383,6 +383,7 @@ void main() { // Second connection to sleep more than first connection 'SELECT test_sleep(test_connection_number() * 10)' ])); + await db.initialize(); final future1 = db.get('SELECT test_sleep(10) as sleep'); final future2 = db.get('SELECT test_sleep(10) as sleep'); From dd2bce05cca0951ae53f55b9a33e75a01417ffa5 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 3 Apr 2024 12:41:56 +0200 Subject: [PATCH 11/21] Fix race condition in closing. --- lib/src/connection_pool.dart | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/src/connection_pool.dart b/lib/src/connection_pool.dart index f6ae300..7852b8e 100644 --- a/lib/src/connection_pool.dart +++ b/lib/src/connection_pool.dart @@ -191,6 +191,9 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { // Create a copy of the list, to avoid this triggering "Concurrent modification during iteration" final toClose = _readConnections.sublist(0); for (var connection in toClose) { + // Wait for connection initialization, so that any existing readLock() + // requests go through before closing. + await connection.ready; await connection.close(); } // Closing the write connection cleans up the journal files (-shm and -wal files). From 5cf85db6ebd9fafa0c3c2780f99c37cc0cd27b0c Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 3 Apr 2024 13:10:01 +0200 Subject: [PATCH 12/21] Slightly better handling of connection initialization errors. --- lib/src/port_channel.dart | 24 +++++++++++++++++++----- lib/src/sqlite_connection_impl.dart | 4 ++++ test/basic_test.dart | 6 +++--- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/lib/src/port_channel.dart b/lib/src/port_channel.dart index c32b7b9..9701023 100644 --- a/lib/src/port_channel.dart +++ b/lib/src/port_channel.dart @@ -18,7 +18,8 @@ abstract class PortClient { class ParentPortClient implements PortClient { late Future sendPortFuture; SendPort? sendPort; - ReceivePort receivePort = ReceivePort(); + final ReceivePort _receivePort = ReceivePort(); + final ReceivePort _errorPort = ReceivePort(); bool closed = false; int _nextId = 1; @@ -30,7 +31,7 @@ class ParentPortClient implements PortClient { sendPortFuture.then((value) { sendPort = value; }); - receivePort.listen((message) { + _receivePort.listen((message) { if (message is _InitMessage) { assert(!initCompleter.isCompleted); initCompleter.complete(message.port); @@ -57,6 +58,17 @@ class ParentPortClient implements PortClient { } close(); }); + _errorPort.listen((message) { + var [error, stackTrace] = message; + print('got an error ${initCompleter.isCompleted} $error'); + if (!initCompleter.isCompleted) { + if (stackTrace == null) { + initCompleter.completeError(error); + } else { + initCompleter.completeError(error, StackTrace.fromString(stackTrace)); + } + } + }); } Future get ready async { @@ -94,20 +106,22 @@ class ParentPortClient implements PortClient { } RequestPortServer server() { - return RequestPortServer(receivePort.sendPort); + return RequestPortServer(_receivePort.sendPort); } void close() async { if (!closed) { closed = true; - receivePort.close(); + _receivePort.close(); + _errorPort.close(); _cancelAll(const ClosedException()); } } tieToIsolate(Isolate isolate) { - isolate.addOnExitListener(receivePort.sendPort, response: _closeMessage); + isolate.addErrorListener(_errorPort.sendPort); + isolate.addOnExitListener(_receivePort.sendPort, response: _closeMessage); } } diff --git a/lib/src/sqlite_connection_impl.dart b/lib/src/sqlite_connection_impl.dart index 643152e..7057a73 100644 --- a/lib/src/sqlite_connection_impl.dart +++ b/lib/src/sqlite_connection_impl.dart @@ -88,6 +88,9 @@ class SqliteConnectionImpl with SqliteQueries implements SqliteConnection { @override Future close() async { await _connectionMutex.lock(() async { + if (closed) { + return; + } if (readOnly) { await _isolateClient.post(const _SqliteIsolateConnectionClose()); } else { @@ -97,6 +100,7 @@ class SqliteConnectionImpl with SqliteQueries implements SqliteConnection { await _isolateClient.post(const _SqliteIsolateConnectionClose()); }); } + _isolateClient.close(); _isolate.kill(); }); } diff --git a/test/basic_test.dart b/test/basic_test.dart index 8ed43dc..113ccb2 100644 --- a/test/basic_test.dart +++ b/test/basic_test.dart @@ -377,7 +377,6 @@ void main() { // 4. Now second connection is ready. Second query has two connections to choose from. // 5. However, first connection is closed, so it's removed from the pool. // 6. Triggers `Concurrent modification during iteration: Instance(length:1) of '_GrowableList'` - final db = SqliteDatabase.withFactory(testFactory(path: path, initStatements: [ // Second connection to sleep more than first connection @@ -387,8 +386,9 @@ void main() { final future1 = db.get('SELECT test_sleep(10) as sleep'); final future2 = db.get('SELECT test_sleep(10) as sleep'); - final closeFuture = db.close(); - await closeFuture; + + await db.close(); + await future1; await future2; }); From 3b79bce346cc9dfe800f2f9b6cd5eee0d92c7073 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 3 Apr 2024 13:14:15 +0200 Subject: [PATCH 13/21] Cleanup. --- lib/src/port_channel.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/src/port_channel.dart b/lib/src/port_channel.dart index 9701023..4b952bf 100644 --- a/lib/src/port_channel.dart +++ b/lib/src/port_channel.dart @@ -60,7 +60,6 @@ class ParentPortClient implements PortClient { }); _errorPort.listen((message) { var [error, stackTrace] = message; - print('got an error ${initCompleter.isCompleted} $error'); if (!initCompleter.isCompleted) { if (stackTrace == null) { initCompleter.completeError(error); From 1af625b2d6c6ad6e9cce811644253eccb6b312c5 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 3 Apr 2024 13:40:12 +0200 Subject: [PATCH 14/21] Improve handling of uncaught errors in Isolates. --- lib/src/connection_pool.dart | 3 +- lib/src/port_channel.dart | 58 ++++++++++++++++++++++++++++-------- test/basic_test.dart | 15 +++++++--- 3 files changed, 59 insertions(+), 17 deletions(-) diff --git a/lib/src/connection_pool.dart b/lib/src/connection_pool.dart index 7852b8e..d0c8912 100644 --- a/lib/src/connection_pool.dart +++ b/lib/src/connection_pool.dart @@ -158,7 +158,8 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { if (closed || _readConnections.length >= maxReaders) { return; } - bool hasCapacity = _readConnections.any((connection) => !connection.locked); + bool hasCapacity = _readConnections + .any((connection) => !connection.locked && !connection.closed); if (!hasCapacity) { var name = debugName == null ? null diff --git a/lib/src/port_channel.dart b/lib/src/port_channel.dart index 4b952bf..8b05feb 100644 --- a/lib/src/port_channel.dart +++ b/lib/src/port_channel.dart @@ -21,6 +21,8 @@ class ParentPortClient implements PortClient { final ReceivePort _receivePort = ReceivePort(); final ReceivePort _errorPort = ReceivePort(); bool closed = false; + Object? _closeError; + String? _isolateDebugName; int _nextId = 1; Map> handlers = HashMap(); @@ -59,14 +61,15 @@ class ParentPortClient implements PortClient { close(); }); _errorPort.listen((message) { - var [error, stackTrace] = message; + final [error, stackTraceString] = message; + final stackTrace = stackTraceString == null + ? null + : StackTrace.fromString(stackTraceString); if (!initCompleter.isCompleted) { - if (stackTrace == null) { - initCompleter.completeError(error); - } else { - initCompleter.completeError(error, StackTrace.fromString(stackTrace)); - } + initCompleter.completeError(error, stackTrace); } + _close(IsolateError(cause: error, isolateDebugName: _isolateDebugName), + stackTrace); }); } @@ -74,18 +77,18 @@ class ParentPortClient implements PortClient { await sendPortFuture; } - void _cancelAll(Object error) { + void _cancelAll(Object error, [StackTrace? stackTrace]) { var handlers = this.handlers; this.handlers = {}; for (var message in handlers.values) { - message.completeError(error); + message.completeError(error, stackTrace); } } @override Future post(Object message) async { if (closed) { - throw ClosedException(); + throw _closeError ?? const ClosedException(); } var completer = Completer.sync(); var id = _nextId++; @@ -98,7 +101,7 @@ class ParentPortClient implements PortClient { @override void fire(Object message) async { if (closed) { - throw ClosedException(); + throw _closeError ?? ClosedException(); } final port = sendPort ?? await sendPortFuture; port.send(_FireMessage(message)); @@ -108,17 +111,27 @@ class ParentPortClient implements PortClient { return RequestPortServer(_receivePort.sendPort); } - void close() async { + void _close([Object? error, StackTrace? stackTrace]) { if (!closed) { closed = true; _receivePort.close(); _errorPort.close(); - _cancelAll(const ClosedException()); + if (error == null) { + _cancelAll(const ClosedException()); + } else { + _closeError = error; + _cancelAll(error, stackTrace); + } } } + void close() { + _close(); + } + tieToIsolate(Isolate isolate) { + _isolateDebugName = isolate.debugName; isolate.addErrorListener(_errorPort.sendPort); isolate.addOnExitListener(_receivePort.sendPort, response: _closeMessage); } @@ -274,6 +287,27 @@ class _RequestMessage { class ClosedException implements Exception { const ClosedException(); + + @override + String toString() { + return 'ClosedException'; + } +} + +class IsolateError extends Error { + final Object cause; + final String? isolateDebugName; + + IsolateError({required this.cause, this.isolateDebugName}); + + @override + String toString() { + if (isolateDebugName != null) { + return 'IsolateError in $isolateDebugName: $cause'; + } else { + return 'IsolateError: $cause'; + } + } } class _PortChannelResult { diff --git a/test/basic_test.dart b/test/basic_test.dart index 113ccb2..c3ad150 100644 --- a/test/basic_test.dart +++ b/test/basic_test.dart @@ -4,6 +4,7 @@ import 'dart:math'; import 'package:sqlite3/sqlite3.dart' as sqlite; import 'package:sqlite_async/mutex.dart'; import 'package:sqlite_async/sqlite_async.dart'; +import 'package:test/expect.dart'; import 'package:test/test.dart'; import 'util.dart'; @@ -301,8 +302,11 @@ void main() { }).catchError((error) { caughtError = error; }); - // This may change into a better error in the future - expect(caughtError.toString(), equals("Instance of 'ClosedException'")); + // The specific error message may change + expect( + caughtError.toString(), + equals( + "IsolateError in sqlite-writer: Invalid argument(s): uncaught async error")); // Check that we can still continue afterwards final computed = await db.computeWithDatabase((db) async { @@ -328,8 +332,11 @@ void main() { }).catchError((error) { caughtError = error; }); - // This may change into a better error in the future - expect(caughtError.toString(), equals("Instance of 'ClosedException'")); + // The specific message may change + expect( + caughtError.toString(), + matches(RegExp( + r'IsolateError in sqlite-\d+: Invalid argument\(s\): uncaught async error'))); } // Check that we can still continue afterwards From c01e637ba3a274c8a9031b76114de1a3c3ddf728 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 3 Apr 2024 15:59:47 +0200 Subject: [PATCH 15/21] Rewrite connection pooling queue. --- lib/src/connection_pool.dart | 173 ++++++++++++++++++----------------- 1 file changed, 90 insertions(+), 83 deletions(-) diff --git a/lib/src/connection_pool.dart b/lib/src/connection_pool.dart index d0c8912..57da980 100644 --- a/lib/src/connection_pool.dart +++ b/lib/src/connection_pool.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:collection'; import 'mutex.dart'; import 'port_channel.dart'; @@ -12,7 +13,9 @@ import 'update_notification.dart'; class SqliteConnectionPool with SqliteQueries implements SqliteConnection { SqliteConnection? _writeConnection; - final List _readConnections = []; + final Set _allReadConnections = {}; + final Queue _availableReadConnections = Queue(); + final Queue<_PendingItem> _queue = Queue(); final SqliteOpenFactory _factory; final SerializedPortClient _upstreamPort; @@ -58,62 +61,60 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { return await _writeConnection!.getAutoCommit(); } - @override - Future readLock(Future Function(SqliteReadContext tx) callback, - {Duration? lockTimeout, String? debugContext}) async { - await _expandPool(); - - return _runZoned(() async { - bool haveLock = false; - var completer = Completer(); - - var futures = _readConnections.sublist(0).map((connection) async { - if (connection.closed) { - _readConnections.remove(connection); - } - try { - return await connection.readLock((ctx) async { - if (haveLock) { - // Already have a different lock - release this one. - return false; - } - haveLock = true; - - var future = callback(ctx); - completer.complete(future); - - // We have to wait for the future to complete before we can release the - // lock. - try { - await future; - } catch (_) { - // Ignore - } - - return true; - }, lockTimeout: lockTimeout, debugContext: debugContext); - } on TimeoutException { - return false; - } on ClosedException { - return false; - } - }); - - final stream = Stream.fromFutures(futures); - var gotAny = await stream.any((element) => element); - - if (!gotAny) { - // All TimeoutExceptions - throw TimeoutException('Failed to get a read connection', lockTimeout); + void _nextRead() { + if (_queue.isEmpty) { + // Wait for queue item + return; + } else if (closed) { + while (_queue.isNotEmpty) { + final nextItem = _queue.removeFirst(); + nextItem.completer.completeError(const ClosedException()); } + return; + } + + while (_availableReadConnections.isNotEmpty && + _availableReadConnections.last.closed) { + // Remove connections that may have errored + final connection = _availableReadConnections.removeLast(); + _allReadConnections.remove(connection); + } + + if (_availableReadConnections.isEmpty && + _allReadConnections.length == maxReaders) { + // Wait for available connection + return; + } + final nextItem = _queue.removeFirst(); + nextItem.completer.complete(Future.sync(() async { + final nextConnection = _availableReadConnections.isEmpty + ? await _expandPool() + : _availableReadConnections.removeLast(); try { - return await completer.future; - } catch (e) { - // throw e; - rethrow; + final result = await nextConnection.readLock(nextItem.callback); + return result; + } finally { + _availableReadConnections.add(nextConnection); + _nextRead(); } - }, debugContext: debugContext ?? 'get*()'); + })); + } + + @override + Future readLock(ReadCallback callback, + {Duration? lockTimeout, String? debugContext}) async { + if (closed) { + throw ClosedException(); + } + final zone = _getZone(debugContext: debugContext ?? 'get*()'); + final item = _PendingItem((ctx) { + return zone.runUnary(callback, ctx); + }); + _queue.add(item); + _nextRead(); + + return await item.completer.future; } @override @@ -146,41 +147,38 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { /// connection (with a different lock). /// 2. Give a more specific error message when it happens. T _runZoned(T Function() callback, {required String debugContext}) { + return _getZone(debugContext: debugContext).run(callback); + } + + Zone _getZone({required String debugContext}) { if (Zone.current[this] != null) { throw LockError( 'Recursive lock is not allowed. Use `tx.$debugContext` instead of `db.$debugContext`.'); } - var zone = Zone.current.fork(zoneValues: {this: true}); - return zone.run(callback); + return Zone.current.fork(zoneValues: {this: true}); } - Future _expandPool() async { - if (closed || _readConnections.length >= maxReaders) { - return; - } - bool hasCapacity = _readConnections - .any((connection) => !connection.locked && !connection.closed); - if (!hasCapacity) { - var name = debugName == null - ? null - : '$debugName-${_readConnections.length + 1}'; - var connection = SqliteConnectionImpl( - upstreamPort: _upstreamPort, - primary: false, - updates: updates, - debugName: name, - mutex: mutex, - readOnly: true, - openFactory: _factory); - _readConnections.add(connection); - - // Edge case: - // If we don't await here, there is a chance that a different connection - // is used for the transaction, and that it finishes and deletes the database - // while this one is still opening. This is specifically triggered in tests. - // To avoid that, we wait for the connection to be ready. - await connection.ready; - } + Future _expandPool() async { + var name = debugName == null + ? null + : '$debugName-${_allReadConnections.length + 1}'; + var connection = SqliteConnectionImpl( + upstreamPort: _upstreamPort, + primary: false, + updates: updates, + debugName: name, + mutex: mutex, + readOnly: true, + openFactory: _factory); + _allReadConnections.add(connection); + + // Edge case: + // If we don't await here, there is a chance that a different connection + // is used for the transaction, and that it finishes and deletes the database + // while this one is still opening. This is specifically triggered in tests. + // To avoid that, we wait for the connection to be ready. + await connection.ready; + return connection; } @override @@ -190,7 +188,7 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { // It is possible that `readLock()` removes connections from the pool while we're // closing connections, but not possible for new connections to be added. // Create a copy of the list, to avoid this triggering "Concurrent modification during iteration" - final toClose = _readConnections.sublist(0); + final toClose = _allReadConnections.toList(); for (var connection in toClose) { // Wait for connection initialization, so that any existing readLock() // requests go through before closing. @@ -203,3 +201,12 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { await _writeConnection?.close(); } } + +typedef ReadCallback = Future Function(SqliteReadContext tx); + +class _PendingItem { + ReadCallback callback; + Completer completer = Completer.sync(); + + _PendingItem(this.callback); +} From 726197ffab4be8d280627ef3452427cbb3f392a1 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 3 Apr 2024 17:17:06 +0200 Subject: [PATCH 16/21] Re-implement lockTimeout. --- lib/src/connection_pool.dart | 46 +++++++++++++++++++++++++++++++----- test/basic_test.dart | 24 +++++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/lib/src/connection_pool.dart b/lib/src/connection_pool.dart index 57da980..3a08dbb 100644 --- a/lib/src/connection_pool.dart +++ b/lib/src/connection_pool.dart @@ -86,17 +86,29 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { return; } - final nextItem = _queue.removeFirst(); + var nextItem = _queue.removeFirst(); + while (nextItem.completer.isCompleted) { + // This item already timed out - try the next one if available + if (_queue.isEmpty) { + return; + } + nextItem = _queue.removeFirst(); + } + + nextItem.lockTimer?.cancel(); + nextItem.completer.complete(Future.sync(() async { final nextConnection = _availableReadConnections.isEmpty ? await _expandPool() : _availableReadConnections.removeLast(); try { + // At this point the connection is expected to be available immediately. + // No need to calculate a new lockTimeout here. final result = await nextConnection.readLock(nextItem.callback); return result; } finally { _availableReadConnections.add(nextConnection); - _nextRead(); + Timer.run(_nextRead); } })); } @@ -110,11 +122,11 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { final zone = _getZone(debugContext: debugContext ?? 'get*()'); final item = _PendingItem((ctx) { return zone.runUnary(callback, ctx); - }); + }, lockTimeout: lockTimeout); _queue.add(item); _nextRead(); - return await item.completer.future; + return (await item.future) as T; } @override @@ -207,6 +219,28 @@ typedef ReadCallback = Future Function(SqliteReadContext tx); class _PendingItem { ReadCallback callback; Completer completer = Completer.sync(); - - _PendingItem(this.callback); + late Future future = completer.future; + DateTime? deadline; + final Duration? lockTimeout; + late final Timer? lockTimer; + + _PendingItem(this.callback, {this.lockTimeout}) { + if (lockTimeout != null) { + deadline = DateTime.now().add(lockTimeout!); + lockTimer = Timer(lockTimeout!, () { + // Note: isCompleted is true when `nextItem.completer.complete` is called, not when the result is available. + // This matches the behavior we need for a timeout on the lock, but not the entire operation. + if (!completer.isCompleted) { + // completer.completeError( + // TimeoutException('Failed to get a read connection', lockTimeout)); + completer.complete(Future.sync(() async { + throw TimeoutException( + 'Failed to get a read connection', lockTimeout); + })); + } + }); + } else { + lockTimer = null; + } + } } diff --git a/test/basic_test.dart b/test/basic_test.dart index c3ad150..3d561a3 100644 --- a/test/basic_test.dart +++ b/test/basic_test.dart @@ -1,5 +1,6 @@ import 'dart:async'; import 'dart:math'; +import 'dart:typed_data'; import 'package:sqlite3/sqlite3.dart' as sqlite; import 'package:sqlite_async/mutex.dart'; @@ -399,6 +400,29 @@ void main() { await future1; await future2; }); + + test('lockTimeout', () async { + final db = + SqliteDatabase.withFactory(testFactory(path: path), maxReaders: 2); + await db.initialize(); + + final f1 = db.readTransaction((tx) async { + await tx.get('select test_sleep(100)'); + }, lockTimeout: const Duration(milliseconds: 200)); + + final f2 = db.readTransaction((tx) async { + await tx.get('select test_sleep(100)'); + }, lockTimeout: const Duration(milliseconds: 200)); + + // At this point, both read connections are in use + await expectLater(() async { + await db.readLock((tx) async { + await tx.get('select test_sleep(10)'); + }, lockTimeout: const Duration(milliseconds: 2)); + }, throwsA((e) => e is TimeoutException)); + + await Future.wait([f1, f2]); + }); }); } From 275b38dc81d870c89e238e06d1ac662edc169b0a Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 3 Apr 2024 17:19:50 +0200 Subject: [PATCH 17/21] Fix imports. --- test/basic_test.dart | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/basic_test.dart b/test/basic_test.dart index 3d561a3..f0713e4 100644 --- a/test/basic_test.dart +++ b/test/basic_test.dart @@ -1,11 +1,9 @@ import 'dart:async'; import 'dart:math'; -import 'dart:typed_data'; import 'package:sqlite3/sqlite3.dart' as sqlite; import 'package:sqlite_async/mutex.dart'; import 'package:sqlite_async/sqlite_async.dart'; -import 'package:test/expect.dart'; import 'package:test/test.dart'; import 'util.dart'; From ea27d31e76a18ab86ae4143e6b860d898c8bcf9a Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 4 Apr 2024 14:23:47 +0200 Subject: [PATCH 18/21] Add notes explaining the different locks. --- lib/src/sqlite_connection.dart | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/src/sqlite_connection.dart b/lib/src/sqlite_connection.dart index 156f967..1fc5525 100644 --- a/lib/src/sqlite_connection.dart +++ b/lib/src/sqlite_connection.dart @@ -84,7 +84,8 @@ abstract class SqliteConnection extends SqliteWriteContext { /// Open a read-write transaction. /// /// This takes a global lock - only one write transaction can execute against - /// the database at a time. + /// the database at a time. This applies even when constructing separate + /// [SqliteDatabase] instances for the same database file. /// /// Statements within the transaction must be done on the provided /// [SqliteWriteContext] - attempting statements on the [SqliteConnection] @@ -104,6 +105,9 @@ abstract class SqliteConnection extends SqliteWriteContext { /// Takes a read lock, without starting a transaction. /// + /// The lock only applies to a single [SqliteConnection], and multiple + /// connections may hold read locks at the same time. + /// /// In most cases, [readTransaction] should be used instead. Future readLock(Future Function(SqliteReadContext tx) callback, {Duration? lockTimeout, String? debugContext}); @@ -111,6 +115,10 @@ abstract class SqliteConnection extends SqliteWriteContext { /// Takes a global lock, without starting a transaction. /// /// In most cases, [writeTransaction] should be used instead. + /// + /// The lock applies to all [SqliteConnection] instances for a [SqliteDatabase]. + /// Locks for separate [SqliteDatabase] instances on the same database file + /// may be held concurrently. Future writeLock(Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, String? debugContext}); From 308076c372859ffa4b44c8ab38f563cadc5b83fa Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 4 Apr 2024 16:10:01 +0200 Subject: [PATCH 19/21] v0.6.1 --- CHANGELOG.md | 10 ++++++++++ pubspec.yaml | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb3af42..76b2b11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## 0.6.1 + +- Fix errors when closing a `SqliteDatabase`. +- Configure SQLite `busy_timeout` (30s default). This fixes "database is locked (code 5)" error when using multiple `SqliteDatabase` instances for the same database. +- Fix errors when opening multiple connections at the same time, e.g. when running multiple read queries concurrently + right after opening the dtaabase. +- Improved error handling when an Isolate crashes with an uncaught error. +- Rewrite connection pool logic to fix performance issues when multiple read connections are open. +- Fix using `SqliteDatabase.isolateConnectionFactory()` in multiple isolates. + ## 0.6.0 - Allow catching errors and continuing the transaction. This is technically a breaking change, although it should not be an issue in most cases. diff --git a/pubspec.yaml b/pubspec.yaml index 8d4fba5..8290330 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,12 +1,12 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.6.0 +version: 0.6.1 repository: https://github.com/powersync-ja/sqlite_async.dart environment: - sdk: '>=3.2.0 <4.0.0' + sdk: ">=3.2.0 <4.0.0" dependencies: - sqlite3: '^2.3.0' + sqlite3: "^2.3.0" async: ^2.10.0 collection: ^1.17.0 From 34e3161cd7909bccdab788bf02fc9055e143abb7 Mon Sep 17 00:00:00 2001 From: Dominic Gunther Bauer <46312751+DominicGBauer@users.noreply.github.com> Date: Tue, 16 Apr 2024 14:48:11 +0200 Subject: [PATCH 20/21] chore: folder structure changes (#40) * chore: folder structure changes * chore: add example changes --------- Co-authored-by: DominicGBauer --- .gitignore | 2 + example/custom_functions_example.dart | 11 +- example/linux_cli_example.dart | 5 +- lib/sqlite3_common.dart | 2 + lib/sqlite_async.dart | 6 + lib/src/common/abstract_open_factory.dart | 113 ++++++ .../connection/sync_sqlite_connection.dart | 116 ++++++ .../common/isolate_connection_factory.dart | 46 +++ lib/src/common/mutex.dart | 32 ++ lib/src/{ => common}/port_channel.dart | 0 lib/src/common/sqlite_database.dart | 88 +++++ .../impl/isolate_connection_factory_impl.dart | 3 + lib/src/impl/mutex_impl.dart | 3 + lib/src/impl/open_factory_impl.dart | 3 + lib/src/impl/sqlite_database_impl.dart | 3 + .../impl/stub_isolate_connection_factory.dart | 47 +++ lib/src/impl/stub_mutex.dart | 18 + lib/src/impl/stub_sqlite_database.dart | 67 ++++ lib/src/impl/stub_sqlite_open_factory.dart | 27 ++ lib/src/isolate_connection_factory.dart | 112 +----- lib/src/mutex.dart | 256 +------------- .../database}/connection_pool.dart | 71 ++-- .../native_sqlite_connection_impl.dart} | 51 +-- .../database/native_sqlite_database.dart | 166 +++++++++ lib/src/native/database/upstream_updates.dart | 67 ++++ .../native_isolate_connection_factory.dart | 102 ++++++ lib/src/native/native_isolate_mutex.dart | 257 ++++++++++++++ .../native/native_sqlite_open_factory.dart | 60 ++++ lib/src/sqlite_connection.dart | 10 +- lib/src/sqlite_database.dart | 238 +------------ lib/src/sqlite_open_factory.dart | 100 +----- lib/src/sqlite_queries.dart | 9 +- lib/src/utils/database_utils.dart | 2 + lib/src/utils/native_database_utils.dart | 13 + .../shared_utils.dart} | 15 +- lib/utils.dart | 1 + pubspec.yaml | 5 + scripts/benchmark.dart | 7 +- test/close_test.dart | 13 +- test/isolate_test.dart | 13 +- test/json1_test.dart | 23 +- test/migration_test.dart | 18 +- test/mutex_test.dart | 3 +- test/{ => native}/basic_test.dart | 162 ++------- test/native/watch_test.dart | 147 ++++++++ test/util.dart | 116 ------ test/utils/abstract_test_utils.dart | 48 +++ test/utils/native_test_utils.dart | 100 ++++++ test/utils/stub_test_utils.dart | 13 + test/utils/test_utils_impl.dart | 3 + test/watch_test.dart | 334 ------------------ 51 files changed, 1728 insertions(+), 1399 deletions(-) create mode 100644 lib/sqlite3_common.dart create mode 100644 lib/src/common/abstract_open_factory.dart create mode 100644 lib/src/common/connection/sync_sqlite_connection.dart create mode 100644 lib/src/common/isolate_connection_factory.dart create mode 100644 lib/src/common/mutex.dart rename lib/src/{ => common}/port_channel.dart (100%) create mode 100644 lib/src/common/sqlite_database.dart create mode 100644 lib/src/impl/isolate_connection_factory_impl.dart create mode 100644 lib/src/impl/mutex_impl.dart create mode 100644 lib/src/impl/open_factory_impl.dart create mode 100644 lib/src/impl/sqlite_database_impl.dart create mode 100644 lib/src/impl/stub_isolate_connection_factory.dart create mode 100644 lib/src/impl/stub_mutex.dart create mode 100644 lib/src/impl/stub_sqlite_database.dart create mode 100644 lib/src/impl/stub_sqlite_open_factory.dart rename lib/src/{ => native/database}/connection_pool.dart (82%) rename lib/src/{sqlite_connection_impl.dart => native/database/native_sqlite_connection_impl.dart} (89%) create mode 100644 lib/src/native/database/native_sqlite_database.dart create mode 100644 lib/src/native/database/upstream_updates.dart create mode 100644 lib/src/native/native_isolate_connection_factory.dart create mode 100644 lib/src/native/native_isolate_mutex.dart create mode 100644 lib/src/native/native_sqlite_open_factory.dart create mode 100644 lib/src/utils/database_utils.dart create mode 100644 lib/src/utils/native_database_utils.dart rename lib/src/{database_utils.dart => utils/shared_utils.dart} (91%) create mode 100644 lib/utils.dart rename test/{ => native}/basic_test.dart (68%) create mode 100644 test/native/watch_test.dart delete mode 100644 test/util.dart create mode 100644 test/utils/abstract_test_utils.dart create mode 100644 test/utils/native_test_utils.dart create mode 100644 test/utils/stub_test_utils.dart create mode 100644 test/utils/test_utils_impl.dart delete mode 100644 test/watch_test.dart diff --git a/.gitignore b/.gitignore index 295944b..102bf3d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ pubspec.lock test-db sqlite-autoconf-* doc + +build diff --git a/example/custom_functions_example.dart b/example/custom_functions_example.dart index bede8d6..e451927 100644 --- a/example/custom_functions_example.dart +++ b/example/custom_functions_example.dart @@ -1,8 +1,9 @@ +import 'dart:async'; import 'dart:io'; import 'dart:isolate'; +import 'package:sqlite3/common.dart'; import 'package:sqlite_async/sqlite_async.dart'; -import 'package:sqlite3/sqlite3.dart' as sqlite; /// Since the functions need to be created on every SQLite connection, /// we do this in a SqliteOpenFactory. @@ -10,12 +11,12 @@ class TestOpenFactory extends DefaultSqliteOpenFactory { TestOpenFactory({required super.path, super.sqliteOptions}); @override - sqlite.Database open(SqliteOpenOptions options) { - final db = super.open(options); + FutureOr open(SqliteOpenOptions options) async { + final db = await super.open(options); db.createFunction( functionName: 'sleep', - argumentCount: const sqlite.AllowedArgumentCount(1), + argumentCount: const AllowedArgumentCount(1), function: (args) { final millis = args[0] as int; sleep(Duration(milliseconds: millis)); @@ -25,7 +26,7 @@ class TestOpenFactory extends DefaultSqliteOpenFactory { db.createFunction( functionName: 'isolate_name', - argumentCount: const sqlite.AllowedArgumentCount(0), + argumentCount: const AllowedArgumentCount(0), function: (args) { return Isolate.current.debugName; }, diff --git a/example/linux_cli_example.dart b/example/linux_cli_example.dart index 82cfee9..8d5aa43 100644 --- a/example/linux_cli_example.dart +++ b/example/linux_cli_example.dart @@ -1,8 +1,9 @@ +import 'dart:async'; import 'dart:ffi'; +import 'package:sqlite3/common.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:sqlite3/open.dart' as sqlite_open; -import 'package:sqlite3/sqlite3.dart' as sqlite; const defaultSqlitePath = 'libsqlite3.so.0'; @@ -16,7 +17,7 @@ class TestOpenFactory extends DefaultSqliteOpenFactory { this.sqlitePath = defaultSqlitePath}); @override - sqlite.Database open(SqliteOpenOptions options) { + FutureOr open(SqliteOpenOptions options) async { // For details, see: // https://pub.dev/packages/sqlite3#manually-providing-sqlite3-libraries sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { diff --git a/lib/sqlite3_common.dart b/lib/sqlite3_common.dart new file mode 100644 index 0000000..eae3c6f --- /dev/null +++ b/lib/sqlite3_common.dart @@ -0,0 +1,2 @@ +// Exports common Sqlite3 exports which are available in different environments. +export 'package:sqlite3/common.dart'; diff --git a/lib/sqlite_async.dart b/lib/sqlite_async.dart index 1d6e5dd..f930e5b 100644 --- a/lib/sqlite_async.dart +++ b/lib/sqlite_async.dart @@ -3,6 +3,12 @@ /// See [SqliteDatabase] as a starting point. library; +export 'src/common/abstract_open_factory.dart'; +export 'src/common/connection/sync_sqlite_connection.dart'; +export 'src/common/isolate_connection_factory.dart'; +export 'src/common/mutex.dart'; +export 'src/common/port_channel.dart'; +export 'src/common/sqlite_database.dart'; export 'src/isolate_connection_factory.dart'; export 'src/sqlite_connection.dart'; export 'src/sqlite_database.dart'; diff --git a/lib/src/common/abstract_open_factory.dart b/lib/src/common/abstract_open_factory.dart new file mode 100644 index 0000000..e421e20 --- /dev/null +++ b/lib/src/common/abstract_open_factory.dart @@ -0,0 +1,113 @@ +import 'dart:async'; +import 'package:meta/meta.dart'; + +import 'package:sqlite_async/sqlite3_common.dart' as sqlite; +import 'package:sqlite_async/src/common/mutex.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/sqlite_options.dart'; +import 'package:sqlite_async/src/update_notification.dart'; + +/// Factory to create new SQLite database connections. +/// +/// Since connections are opened in dedicated background isolates, this class +/// must be safe to pass to different isolates. +abstract class SqliteOpenFactory { + String get path; + + /// Opens a direct connection to the SQLite database + FutureOr open(SqliteOpenOptions options); + + /// Opens an asynchronous [SqliteConnection] + FutureOr openConnection(SqliteOpenOptions options); +} + +class SqliteOpenOptions { + /// Whether this is the primary write connection for the database. + final bool primaryConnection; + + /// Whether this connection is read-only. + final bool readOnly; + + /// Mutex to use in [SqliteConnection]s + final Mutex? mutex; + + /// Name used in debug logs + final String? debugName; + + /// Stream of external update notifications + final Stream? updates; + + const SqliteOpenOptions( + {required this.primaryConnection, + required this.readOnly, + this.mutex, + this.debugName, + this.updates}); + + sqlite.OpenMode get openMode { + if (primaryConnection) { + return sqlite.OpenMode.readWriteCreate; + } else if (readOnly) { + return sqlite.OpenMode.readOnly; + } else { + return sqlite.OpenMode.readWrite; + } + } +} + +/// The default database factory. +/// +/// This takes care of opening the database, and running PRAGMA statements +/// to configure the connection. +/// +/// Override the [open] method to customize the process. +abstract class AbstractDefaultSqliteOpenFactory< + Database extends sqlite.CommonDatabase> + implements SqliteOpenFactory { + @override + final String path; + final SqliteOptions sqliteOptions; + + const AbstractDefaultSqliteOpenFactory( + {required this.path, + this.sqliteOptions = const SqliteOptions.defaults()}); + + List pragmaStatements(SqliteOpenOptions options); + + @protected + + /// Opens a direct connection to a SQLite database connection + FutureOr openDB(SqliteOpenOptions options); + + @override + + /// Opens a direct connection to a SQLite database connection + /// and executes setup pragma statements to initialize the DB + FutureOr open(SqliteOpenOptions options) async { + var db = await openDB(options); + + // Pragma statements don't have the same BUSY_TIMEOUT behavior as normal statements. + // We add a manual retry loop for those. + for (var statement in pragmaStatements(options)) { + for (var tries = 0; tries < 30; tries++) { + try { + db.execute(statement); + break; + } on sqlite.SqliteException catch (e) { + if (e.resultCode == sqlite.SqlError.SQLITE_BUSY && tries < 29) { + continue; + } else { + rethrow; + } + } + } + } + return db; + } + + @override + + /// Opens an asynchronous [SqliteConnection] to a SQLite database + /// and executes setup pragma statements to initialize the DB + FutureOr openConnection(SqliteOpenOptions options); +} diff --git a/lib/src/common/connection/sync_sqlite_connection.dart b/lib/src/common/connection/sync_sqlite_connection.dart new file mode 100644 index 0000000..f29520f --- /dev/null +++ b/lib/src/common/connection/sync_sqlite_connection.dart @@ -0,0 +1,116 @@ +import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/src/common/mutex.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/sqlite_queries.dart'; +import 'package:sqlite_async/src/update_notification.dart'; + +/// A simple "synchronous" connection which provides the async SqliteConnection +/// implementation using a synchronous SQLite connection +class SyncSqliteConnection extends SqliteConnection with SqliteQueries { + final CommonDatabase db; + late Mutex mutex; + @override + late final Stream updates; + + bool _closed = false; + + SyncSqliteConnection(this.db, Mutex m) { + mutex = m.open(); + updates = db.updates.map( + (event) { + return UpdateNotification({event.tableName}); + }, + ); + } + + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + return mutex.lock(() => callback(SyncReadContext(db)), + timeout: lockTimeout); + } + + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + return mutex.lock(() => callback(SyncWriteContext(db)), + timeout: lockTimeout); + } + + @override + Future close() async { + _closed = true; + return db.dispose(); + } + + @override + bool get closed => _closed; + + @override + Future getAutoCommit() async { + return db.autocommit; + } +} + +class SyncReadContext implements SqliteReadContext { + CommonDatabase db; + + SyncReadContext(this.db); + + @override + Future computeWithDatabase( + Future Function(CommonDatabase db) compute) { + return compute(db); + } + + @override + Future get(String sql, [List parameters = const []]) async { + return db.select(sql, parameters).first; + } + + @override + Future getAll(String sql, + [List parameters = const []]) async { + return db.select(sql, parameters); + } + + @override + Future getOptional(String sql, + [List parameters = const []]) async { + final rows = await getAll(sql, parameters); + return rows.isEmpty ? null : rows.first; + } + + @override + bool get closed => false; + + @override + Future getAutoCommit() async { + return db.autocommit; + } +} + +class SyncWriteContext extends SyncReadContext implements SqliteWriteContext { + SyncWriteContext(super.db); + + @override + Future execute(String sql, + [List parameters = const []]) async { + return db.select(sql, parameters); + } + + @override + Future executeBatch( + String sql, List> parameterSets) async { + return computeWithDatabase((db) async { + final statement = db.prepare(sql, checkNoTail: true); + try { + for (var parameters in parameterSets) { + statement.execute(parameters); + } + } finally { + statement.dispose(); + } + }); + } +} diff --git a/lib/src/common/isolate_connection_factory.dart b/lib/src/common/isolate_connection_factory.dart new file mode 100644 index 0000000..90dc9b1 --- /dev/null +++ b/lib/src/common/isolate_connection_factory.dart @@ -0,0 +1,46 @@ +import 'dart:async'; +import 'package:sqlite_async/sqlite3_common.dart' as sqlite; +import 'package:sqlite_async/src/common/mutex.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/impl/isolate_connection_factory_impl.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'port_channel.dart'; + +mixin IsolateOpenFactoryMixin { + AbstractDefaultSqliteOpenFactory get openFactory; + + /// Opens a synchronous sqlite.Database directly in the current isolate. + /// + /// This gives direct access to the database, but: + /// 1. No app-level locking is performed automatically. Transactions may fail + /// with SQLITE_BUSY if another isolate is using the database at the same time. + /// 2. Other connections are not notified of any updates to tables made within + /// this connection. + FutureOr openRawDatabase({bool readOnly = false}) async { + return openFactory + .open(SqliteOpenOptions(primaryConnection: false, readOnly: readOnly)); + } +} + +/// A connection factory that can be passed to different isolates. +abstract class IsolateConnectionFactory + with IsolateOpenFactoryMixin { + Mutex get mutex; + + SerializedPortClient get upstreamPort; + + factory IsolateConnectionFactory( + {required openFactory, + required mutex, + required SerializedPortClient upstreamPort}) { + return IsolateConnectionFactoryImpl( + openFactory: openFactory, + mutex: mutex, + upstreamPort: upstreamPort) as IsolateConnectionFactory; + } + + /// Open a new SqliteConnection. + /// + /// This opens a single connection in a background execution isolate. + SqliteConnection open({String? debugName, bool readOnly = false}); +} diff --git a/lib/src/common/mutex.dart b/lib/src/common/mutex.dart new file mode 100644 index 0000000..ccdcc49 --- /dev/null +++ b/lib/src/common/mutex.dart @@ -0,0 +1,32 @@ +import 'package:sqlite_async/src/impl/mutex_impl.dart'; + +abstract class Mutex { + factory Mutex() { + return MutexImpl(); + } + + /// timeout is a timeout for acquiring the lock, not for the callback + Future lock(Future Function() callback, {Duration? timeout}); + + /// Use [open] to get a [AbstractMutex] instance. + /// This is mainly used for shared mutexes + Mutex open() { + return this; + } + + /// Release resources used by the Mutex. + /// + /// Subsequent calls to [lock] may fail, or may never call the callback. + Future close(); +} + +class LockError extends Error { + final String message; + + LockError(this.message); + + @override + String toString() { + return 'LockError: $message'; + } +} diff --git a/lib/src/port_channel.dart b/lib/src/common/port_channel.dart similarity index 100% rename from lib/src/port_channel.dart rename to lib/src/common/port_channel.dart diff --git a/lib/src/common/sqlite_database.dart b/lib/src/common/sqlite_database.dart new file mode 100644 index 0000000..8df4174 --- /dev/null +++ b/lib/src/common/sqlite_database.dart @@ -0,0 +1,88 @@ +import 'dart:async'; + +import 'package:meta/meta.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/common/isolate_connection_factory.dart'; +import 'package:sqlite_async/src/impl/sqlite_database_impl.dart'; +import 'package:sqlite_async/src/sqlite_options.dart'; +import 'package:sqlite_async/src/sqlite_queries.dart'; +import 'package:sqlite_async/src/update_notification.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; + +mixin SqliteDatabaseMixin implements SqliteConnection, SqliteQueries { + /// Maximum number of concurrent read transactions. + int get maxReaders; + + /// Factory that opens a raw database connection in each isolate. + /// + /// This must be safe to pass to different isolates. + /// + /// Use a custom class for this to customize the open process. + AbstractDefaultSqliteOpenFactory get openFactory; + + /// Use this stream to subscribe to notifications of updates to tables. + @override + Stream get updates; + + final StreamController updatesController = + StreamController.broadcast(); + + @protected + Future get isInitialized; + + /// Wait for initialization to complete. + /// + /// While initializing is automatic, this helps to catch and report initialization errors. + Future initialize() async { + await isInitialized; + } + + /// A connection factory that can be passed to different isolates. + /// + /// Use this to access the database in background isolates. + IsolateConnectionFactory isolateConnectionFactory(); +} + +/// A SQLite database instance. +/// +/// Use one instance per database file. If multiple instances are used, update +/// notifications may not trigger, and calls may fail with "SQLITE_BUSY" errors. +abstract class SqliteDatabase + with SqliteQueries, SqliteDatabaseMixin + implements SqliteConnection { + /// The maximum number of concurrent read transactions if not explicitly specified. + static const int defaultMaxReaders = 5; + + /// Open a SqliteDatabase. + /// + /// Only a single SqliteDatabase per [path] should be opened at a time. + /// + /// A connection pool is used by default, allowing multiple concurrent read + /// transactions, and a single concurrent write transaction. Write transactions + /// do not block read transactions, and read transactions will see the state + /// from the last committed write transaction. + /// + /// A maximum of [maxReaders] concurrent read transactions are allowed. + factory SqliteDatabase( + {required path, + int maxReaders = SqliteDatabase.defaultMaxReaders, + SqliteOptions options = const SqliteOptions.defaults()}) { + return SqliteDatabaseImpl( + path: path, maxReaders: maxReaders, options: options); + } + + /// Advanced: Open a database with a specified factory. + /// + /// The factory is used to open each database connection in background isolates. + /// + /// Use when control is required over the opening process. Examples include: + /// 1. Specifying the path to `libsqlite.so` on Linux. + /// 2. Running additional per-connection PRAGMA statements on each connection. + /// 3. Creating custom SQLite functions. + /// 4. Creating temporary views or triggers. + factory SqliteDatabase.withFactory( + AbstractDefaultSqliteOpenFactory openFactory, + {int maxReaders = SqliteDatabase.defaultMaxReaders}) { + return SqliteDatabaseImpl.withFactory(openFactory, maxReaders: maxReaders); + } +} diff --git a/lib/src/impl/isolate_connection_factory_impl.dart b/lib/src/impl/isolate_connection_factory_impl.dart new file mode 100644 index 0000000..d8e8860 --- /dev/null +++ b/lib/src/impl/isolate_connection_factory_impl.dart @@ -0,0 +1,3 @@ +export 'stub_isolate_connection_factory.dart' + // ignore: uri_does_not_exist + if (dart.library.io) '../native/native_isolate_connection_factory.dart'; diff --git a/lib/src/impl/mutex_impl.dart b/lib/src/impl/mutex_impl.dart new file mode 100644 index 0000000..8e03805 --- /dev/null +++ b/lib/src/impl/mutex_impl.dart @@ -0,0 +1,3 @@ +export 'stub_mutex.dart' + // ignore: uri_does_not_exist + if (dart.library.io) '../native/native_isolate_mutex.dart'; diff --git a/lib/src/impl/open_factory_impl.dart b/lib/src/impl/open_factory_impl.dart new file mode 100644 index 0000000..1e22487 --- /dev/null +++ b/lib/src/impl/open_factory_impl.dart @@ -0,0 +1,3 @@ +export 'stub_sqlite_open_factory.dart' + // ignore: uri_does_not_exist + if (dart.library.io) '../native/native_sqlite_open_factory.dart'; diff --git a/lib/src/impl/sqlite_database_impl.dart b/lib/src/impl/sqlite_database_impl.dart new file mode 100644 index 0000000..c88837a --- /dev/null +++ b/lib/src/impl/sqlite_database_impl.dart @@ -0,0 +1,3 @@ +export 'stub_sqlite_database.dart' + // ignore: uri_does_not_exist + if (dart.library.io) '../native/database/native_sqlite_database.dart'; diff --git a/lib/src/impl/stub_isolate_connection_factory.dart b/lib/src/impl/stub_isolate_connection_factory.dart new file mode 100644 index 0000000..208b7b8 --- /dev/null +++ b/lib/src/impl/stub_isolate_connection_factory.dart @@ -0,0 +1,47 @@ +import 'dart:async'; + +import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/src/common/isolate_connection_factory.dart'; +import 'package:sqlite_async/src/common/mutex.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/common/port_channel.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; + +/// A connection factory that can be passed to different isolates. +class IsolateConnectionFactoryImpl + implements IsolateConnectionFactory { + @override + AbstractDefaultSqliteOpenFactory openFactory; + + IsolateConnectionFactoryImpl( + {required this.openFactory, + required Mutex mutex, + SerializedPortClient? upstreamPort}); + + @override + + /// Open a new SqliteConnection. + /// + /// This opens a single connection in a background execution isolate. + SqliteConnection open({String? debugName, bool readOnly = false}) { + throw UnimplementedError(); + } + + /// Opens a synchronous sqlite.Database directly in the current isolate. + /// + /// This gives direct access to the database, but: + /// 1. No app-level locking is performed automatically. Transactions may fail + /// with SQLITE_BUSY if another isolate is using the database at the same time. + /// 2. Other connections are not notified of any updates to tables made within + /// this connection. + @override + Future openRawDatabase({bool readOnly = false}) async { + throw UnimplementedError(); + } + + @override + Mutex get mutex => throw UnimplementedError(); + + @override + SerializedPortClient get upstreamPort => throw UnimplementedError(); +} diff --git a/lib/src/impl/stub_mutex.dart b/lib/src/impl/stub_mutex.dart new file mode 100644 index 0000000..1e700fa --- /dev/null +++ b/lib/src/impl/stub_mutex.dart @@ -0,0 +1,18 @@ +import 'package:sqlite_async/src/common/mutex.dart'; + +class MutexImpl implements Mutex { + @override + Future close() { + throw UnimplementedError(); + } + + @override + Future lock(Future Function() callback, {Duration? timeout}) { + throw UnimplementedError(); + } + + @override + Mutex open() { + throw UnimplementedError(); + } +} diff --git a/lib/src/impl/stub_sqlite_database.dart b/lib/src/impl/stub_sqlite_database.dart new file mode 100644 index 0000000..29db641 --- /dev/null +++ b/lib/src/impl/stub_sqlite_database.dart @@ -0,0 +1,67 @@ +import 'package:meta/meta.dart'; +import 'package:sqlite_async/src/common/isolate_connection_factory.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/common/sqlite_database.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/sqlite_options.dart'; +import 'package:sqlite_async/src/sqlite_queries.dart'; +import 'package:sqlite_async/src/update_notification.dart'; + +class SqliteDatabaseImpl + with SqliteQueries, SqliteDatabaseMixin + implements SqliteDatabase { + @override + bool get closed => throw UnimplementedError(); + + @override + AbstractDefaultSqliteOpenFactory openFactory; + + @override + int maxReaders; + + factory SqliteDatabaseImpl( + {required path, + int maxReaders = SqliteDatabase.defaultMaxReaders, + SqliteOptions options = const SqliteOptions.defaults()}) { + throw UnimplementedError(); + } + + SqliteDatabaseImpl.withFactory(this.openFactory, + {this.maxReaders = SqliteDatabase.defaultMaxReaders}) { + throw UnimplementedError(); + } + + @override + @protected + Future get isInitialized => throw UnimplementedError(); + + @override + Stream get updates => throw UnimplementedError(); + + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + throw UnimplementedError(); + } + + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + throw UnimplementedError(); + } + + @override + Future close() { + throw UnimplementedError(); + } + + @override + IsolateConnectionFactory isolateConnectionFactory() { + throw UnimplementedError(); + } + + @override + Future getAutoCommit() { + throw UnimplementedError(); + } +} diff --git a/lib/src/impl/stub_sqlite_open_factory.dart b/lib/src/impl/stub_sqlite_open_factory.dart new file mode 100644 index 0000000..1108752 --- /dev/null +++ b/lib/src/impl/stub_sqlite_open_factory.dart @@ -0,0 +1,27 @@ +import 'dart:async'; + +import 'package:sqlite_async/sqlite3_common.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/sqlite_options.dart'; + +class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { + const DefaultSqliteOpenFactory( + {required super.path, + super.sqliteOptions = const SqliteOptions.defaults()}); + + @override + CommonDatabase openDB(SqliteOpenOptions options) { + throw UnimplementedError(); + } + + @override + List pragmaStatements(SqliteOpenOptions options) { + throw UnimplementedError(); + } + + @override + FutureOr openConnection(SqliteOpenOptions options) { + throw UnimplementedError(); + } +} diff --git a/lib/src/isolate_connection_factory.dart b/lib/src/isolate_connection_factory.dart index adcd37c..6958495 100644 --- a/lib/src/isolate_connection_factory.dart +++ b/lib/src/isolate_connection_factory.dart @@ -1,108 +1,4 @@ -import 'dart:async'; -import 'dart:isolate'; - -import 'package:sqlite3/sqlite3.dart' as sqlite; - -import 'database_utils.dart'; -import 'mutex.dart'; -import 'port_channel.dart'; -import 'sqlite_connection.dart'; -import 'sqlite_connection_impl.dart'; -import 'sqlite_open_factory.dart'; -import 'update_notification.dart'; - -/// A connection factory that can be passed to different isolates. -class IsolateConnectionFactory { - SqliteOpenFactory openFactory; - SerializedMutex mutex; - SerializedPortClient upstreamPort; - - IsolateConnectionFactory( - {required this.openFactory, - required this.mutex, - required this.upstreamPort}); - - /// Open a new SqliteConnection. - /// - /// This opens a single connection in a background execution isolate. - SqliteConnection open({String? debugName, bool readOnly = false}) { - final updates = _IsolateUpdateListener(upstreamPort); - - var openMutex = mutex.open(); - - return _IsolateSqliteConnection( - openFactory: openFactory, - mutex: openMutex, - upstreamPort: upstreamPort, - readOnly: readOnly, - debugName: debugName, - updates: updates.stream, - closeFunction: () async { - await openMutex.close(); - updates.close(); - }); - } - - /// Opens a synchronous sqlite.Database directly in the current isolate. - /// - /// This gives direct access to the database, but: - /// 1. No app-level locking is performed automatically. Transactions may fail - /// with SQLITE_BUSY if another isolate is using the database at the same time. - /// 2. Other connections are not notified of any updates to tables made within - /// this connection. - Future openRawDatabase({bool readOnly = false}) async { - final db = await openFactory - .open(SqliteOpenOptions(primaryConnection: false, readOnly: readOnly)); - return db; - } -} - -class _IsolateUpdateListener { - final ChildPortClient client; - final ReceivePort port = ReceivePort(); - late final StreamController controller; - - _IsolateUpdateListener(SerializedPortClient upstreamPort) - : client = upstreamPort.open() { - controller = StreamController.broadcast(onListen: () { - client.fire(SubscribeToUpdates(port.sendPort)); - }, onCancel: () { - client.fire(UnsubscribeToUpdates(port.sendPort)); - }); - - port.listen((message) { - if (message is UpdateNotification) { - controller.add(message); - } - }); - } - - Stream get stream { - return controller.stream; - } - - close() { - client.fire(UnsubscribeToUpdates(port.sendPort)); - controller.close(); - port.close(); - } -} - -class _IsolateSqliteConnection extends SqliteConnectionImpl { - final Future Function() closeFunction; - - _IsolateSqliteConnection( - {required super.openFactory, - required super.mutex, - required super.upstreamPort, - super.updates, - super.debugName, - super.readOnly = false, - required this.closeFunction}); - - @override - Future close() async { - await super.close(); - await closeFunction(); - } -} +// This follows the pattern from here: https://stackoverflow.com/questions/58710226/how-to-import-platform-specific-dependency-in-flutter-dart-combine-web-with-an +// To conditionally export an implementation for either web or "native" platforms +// The sqlite library uses dart:ffi which is not supported on web +export 'impl/isolate_connection_factory_impl.dart'; diff --git a/lib/src/mutex.dart b/lib/src/mutex.dart index 54179f7..0c973c7 100644 --- a/lib/src/mutex.dart +++ b/lib/src/mutex.dart @@ -1,254 +1,2 @@ -// Adapted from: -// https://github.com/tekartik/synchronized.dart -// (MIT) -import 'dart:async'; - -import 'port_channel.dart'; - -abstract class Mutex { - factory Mutex() { - return SimpleMutex(); - } - - /// timeout is a timeout for acquiring the lock, not for the callback - Future lock(Future Function() callback, {Duration? timeout}); - - /// Release resources used by the Mutex. - /// - /// Subsequent calls to [lock] may fail, or may never call the callback. - Future close(); -} - -/// Mutex maintains a queue of Future-returning functions that -/// are executed sequentially. -/// The internal lock is not shared across Isolates by default. -class SimpleMutex implements Mutex { - // Adapted from https://github.com/tekartik/synchronized.dart/blob/master/synchronized/lib/src/basic_lock.dart - - Future? last; - - // Hack to make sure the Mutex is not copied to another isolate. - // ignore: unused_field - final Finalizer _f = Finalizer((_) {}); - - SimpleMutex(); - - bool get locked => last != null; - - _SharedMutexServer? _shared; - - @override - Future lock(Future Function() callback, {Duration? timeout}) async { - if (Zone.current[this] != null) { - throw LockError('Recursive lock is not allowed'); - } - var zone = Zone.current.fork(zoneValues: {this: true}); - - return zone.run(() async { - final prev = last; - final completer = Completer.sync(); - last = completer.future; - try { - // If there is a previous running block, wait for it - if (prev != null) { - if (timeout != null) { - // This could throw a timeout error - try { - await prev.timeout(timeout); - } catch (error) { - if (error is TimeoutException) { - throw TimeoutException('Failed to acquire lock', timeout); - } else { - rethrow; - } - } - } else { - await prev; - } - } - - // Run the function and return the result - return await callback(); - } finally { - // Cleanup - // waiting for the previous task to be done in case of timeout - void complete() { - // Only mark it unlocked when the last one complete - if (identical(last, completer.future)) { - last = null; - } - completer.complete(); - } - - // In case of timeout, wait for the previous one to complete too - // before marking this task as complete - if (prev != null && timeout != null) { - // But we still returns immediately - prev.then((_) { - complete(); - }).ignore(); - } else { - complete(); - } - } - }); - } - - @override - Future close() async { - _shared?.close(); - await lock(() async {}); - } - - /// Get a serialized instance that can be passed over to a different isolate. - SerializedMutex get shared { - _shared ??= _SharedMutexServer._withMutex(this); - return _shared!.serialized; - } -} - -/// Serialized version of a Mutex, can be passed over to different isolates. -/// Use [open] to get a [SharedMutex] instance. -/// -/// Uses a [SendPort] to communicate with the source mutex. -class SerializedMutex { - final SerializedPortClient client; - - const SerializedMutex(this.client); - - SharedMutex open() { - return SharedMutex._(client.open()); - } -} - -/// Mutex instantiated from a source mutex, potentially in a different isolate. -/// -/// Uses a [SendPort] to communicate with the source mutex. -class SharedMutex implements Mutex { - final ChildPortClient client; - bool closed = false; - - SharedMutex._(this.client); - - @override - Future lock(Future Function() callback, {Duration? timeout}) async { - if (Zone.current[this] != null) { - throw LockError('Recursive lock is not allowed'); - } - return runZoned(() async { - if (closed) { - throw const ClosedException(); - } - await _acquire(timeout: timeout); - try { - final T result = await callback(); - return result; - } finally { - _unlock(); - } - }, zoneValues: {this: true}); - } - - _unlock() { - client.fire(const _UnlockMessage()); - } - - Future _acquire({Duration? timeout}) async { - final lockFuture = client.post(const _AcquireMessage()); - bool timedout = false; - - var handledLockFuture = lockFuture.then((_) { - if (timedout) { - _unlock(); - throw TimeoutException('Failed to acquire lock', timeout); - } - }); - - if (timeout != null) { - handledLockFuture = - handledLockFuture.timeout(timeout).catchError((error, stacktrace) { - timedout = true; - if (error is TimeoutException) { - throw TimeoutException('Failed to acquire SharedMutex lock', timeout); - } - throw error; - }); - } - return await handledLockFuture; - } - - @override - - /// Wait for existing locks to be released, then close this SharedMutex - /// and prevent further locks from being taken out. - Future close() async { - if (closed) { - return; - } - closed = true; - // Wait for any existing locks to complete, then prevent any further locks from being taken out. - await _acquire(); - // Release the lock - _unlock(); - // Close client immediately after _unlock(), - // so that we're sure no further locks are acquired. - // This also cancels any lock request in process. - client.close(); - } -} - -/// Manages a [SerializedMutex], allowing a [Mutex] to be shared across isolates. -class _SharedMutexServer { - Completer? unlock; - late final SerializedMutex serialized; - final Mutex mutex; - - late final PortServer server; - - _SharedMutexServer._withMutex(this.mutex) { - server = PortServer((Object? arg) async { - return await _handle(arg); - }); - serialized = SerializedMutex(server.client()); - } - - Future _handle(Object? arg) async { - if (arg is _AcquireMessage) { - var lock = Completer.sync(); - mutex.lock(() async { - assert(unlock == null); - unlock = Completer.sync(); - lock.complete(); - await unlock!.future; - unlock = null; - }); - await lock.future; - } else if (arg is _UnlockMessage) { - assert(unlock != null); - unlock!.complete(); - } - } - - void close() async { - server.close(); - } -} - -class _AcquireMessage { - const _AcquireMessage(); -} - -class _UnlockMessage { - const _UnlockMessage(); -} - -class LockError extends Error { - final String message; - - LockError(this.message); - - @override - String toString() { - return 'LockError: $message'; - } -} +export 'impl/mutex_impl.dart'; +export 'common/mutex.dart'; diff --git a/lib/src/connection_pool.dart b/lib/src/native/database/connection_pool.dart similarity index 82% rename from lib/src/connection_pool.dart rename to lib/src/native/database/connection_pool.dart index 3a08dbb..d1b4281 100644 --- a/lib/src/connection_pool.dart +++ b/lib/src/native/database/connection_pool.dart @@ -1,33 +1,35 @@ import 'dart:async'; import 'dart:collection'; -import 'mutex.dart'; -import 'port_channel.dart'; -import 'sqlite_connection.dart'; -import 'sqlite_connection_impl.dart'; -import 'sqlite_open_factory.dart'; -import 'sqlite_queries.dart'; -import 'update_notification.dart'; +import 'package:sqlite_async/sqlite_async.dart'; +import '../database/native_sqlite_connection_impl.dart'; +import '../native_isolate_mutex.dart'; /// A connection pool with a single write connection and multiple read connections. class SqliteConnectionPool with SqliteQueries implements SqliteConnection { - SqliteConnection? _writeConnection; + final StreamController updatesController = + StreamController.broadcast(); + + @override + + /// The write connection might be recreated if it's closed + /// This will allow the update stream remain constant even + /// after using a new write connection. + late final Stream updates = updatesController.stream; + + SqliteConnectionImpl? _writeConnection; final Set _allReadConnections = {}; final Queue _availableReadConnections = Queue(); final Queue<_PendingItem> _queue = Queue(); - final SqliteOpenFactory _factory; - final SerializedPortClient _upstreamPort; - - @override - final Stream? updates; + final AbstractDefaultSqliteOpenFactory _factory; final int maxReaders; final String? debugName; - final Mutex mutex; + final MutexImpl mutex; @override bool closed = false; @@ -43,14 +45,14 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { /// Read connections are opened in read-only mode, and will reject any statements /// that modify the database. SqliteConnectionPool(this._factory, - {this.updates, - this.maxReaders = 5, - SqliteConnection? writeConnection, + {this.maxReaders = 5, + SqliteConnectionImpl? writeConnection, this.debugName, - required this.mutex, - required SerializedPortClient upstreamPort}) - : _writeConnection = writeConnection, - _upstreamPort = upstreamPort; + required this.mutex}) + : _writeConnection = writeConnection { + // Use the write connection's updates + _writeConnection?.updates?.forEach(updatesController.add); + } /// Returns true if the _write_ connection is currently in autocommit mode. @override @@ -131,21 +133,24 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { @override Future writeLock(Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout, String? debugContext}) { + {Duration? lockTimeout, String? debugContext}) async { if (closed) { throw ClosedException(); } if (_writeConnection?.closed == true) { _writeConnection = null; } - _writeConnection ??= SqliteConnectionImpl( - upstreamPort: _upstreamPort, - primary: false, - updates: updates, - debugName: debugName != null ? '$debugName-writer' : null, - mutex: mutex, - readOnly: false, - openFactory: _factory); + + if (_writeConnection == null) { + _writeConnection = (await _factory.openConnection(SqliteOpenOptions( + primaryConnection: true, + debugName: debugName != null ? '$debugName-writer' : null, + mutex: mutex, + readOnly: false))) as SqliteConnectionImpl; + // Expose the new updates on the connection pool + _writeConnection!.updates?.forEach(updatesController.add); + } + return _runZoned(() { return _writeConnection!.writeLock(callback, lockTimeout: lockTimeout, debugContext: debugContext); @@ -175,7 +180,7 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { ? null : '$debugName-${_allReadConnections.length + 1}'; var connection = SqliteConnectionImpl( - upstreamPort: _upstreamPort, + upstreamPort: upstreamPort, primary: false, updates: updates, debugName: name, @@ -193,6 +198,10 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { return connection; } + SerializedPortClient? get upstreamPort { + return _writeConnection?.upstreamPort; + } + @override Future close() async { closed = true; diff --git a/lib/src/sqlite_connection_impl.dart b/lib/src/native/database/native_sqlite_connection_impl.dart similarity index 89% rename from lib/src/sqlite_connection_impl.dart rename to lib/src/native/database/native_sqlite_connection_impl.dart index 7057a73..5056299 100644 --- a/lib/src/sqlite_connection_impl.dart +++ b/lib/src/native/database/native_sqlite_connection_impl.dart @@ -2,42 +2,50 @@ import 'dart:async'; import 'dart:isolate'; import 'package:sqlite3/sqlite3.dart' as sqlite; +import 'package:sqlite_async/sqlite3_common.dart'; +import '../../common/abstract_open_factory.dart'; +import '../../common/mutex.dart'; +import '../../common/port_channel.dart'; +import '../../native/native_isolate_mutex.dart'; +import '../../sqlite_connection.dart'; +import '../../sqlite_queries.dart'; +import '../../update_notification.dart'; +import '../../utils/shared_utils.dart'; -import 'database_utils.dart'; -import 'mutex.dart'; -import 'port_channel.dart'; -import 'sqlite_connection.dart'; -import 'sqlite_open_factory.dart'; -import 'sqlite_queries.dart'; -import 'update_notification.dart'; +import 'upstream_updates.dart'; -typedef TxCallback = Future Function(sqlite.Database db); +typedef TxCallback = Future Function(CommonDatabase db); /// Implements a SqliteConnection using a separate isolate for the database /// operations. -class SqliteConnectionImpl with SqliteQueries implements SqliteConnection { +class SqliteConnectionImpl + with SqliteQueries, UpStreamTableUpdates + implements SqliteConnection { /// Private to this connection final SimpleMutex _connectionMutex = SimpleMutex(); final Mutex _writeMutex; /// Must be a broadcast stream @override - final Stream? updates; + late final Stream? updates; final ParentPortClient _isolateClient = ParentPortClient(); late final Isolate _isolate; final String? debugName; final bool readOnly; SqliteConnectionImpl( - {required SqliteOpenFactory openFactory, + {required openFactory, required Mutex mutex, - required SerializedPortClient upstreamPort, - this.updates, + SerializedPortClient? upstreamPort, + Stream? updates, this.debugName, this.readOnly = false, bool primary = false}) : _writeMutex = mutex { - _open(openFactory, primary: primary, upstreamPort: upstreamPort); + this.upstreamPort = upstreamPort ?? listenForEvents(); + // Accept an incoming stream of updates, or expose one if not given. + this.updates = updates ?? updatesController.stream; + _open(openFactory, primary: primary, upstreamPort: this.upstreamPort); } Future get ready async { @@ -64,7 +72,7 @@ class SqliteConnectionImpl with SqliteQueries implements SqliteConnection { } } - Future _open(SqliteOpenFactory openFactory, + Future _open(AbstractDefaultSqliteOpenFactory openFactory, {required bool primary, required SerializedPortClient upstreamPort}) async { await _connectionMutex.lock(() async { @@ -80,17 +88,15 @@ class SqliteConnectionImpl with SqliteQueries implements SqliteConnection { paused: true); _isolateClient.tieToIsolate(_isolate); _isolate.resume(_isolate.pauseCapability!); - + isInitialized = _isolateClient.ready; await _isolateClient.ready; }); } @override Future close() async { + eventsPort?.close(); await _connectionMutex.lock(() async { - if (closed) { - return; - } if (readOnly) { await _isolateClient.post(const _SqliteIsolateConnectionClose()); } else { @@ -100,7 +106,6 @@ class SqliteConnectionImpl with SqliteQueries implements SqliteConnection { await _isolateClient.post(const _SqliteIsolateConnectionClose()); }); } - _isolateClient.close(); _isolate.kill(); }); } @@ -211,7 +216,7 @@ class _TransactionContext implements SqliteWriteContext { @override Future computeWithDatabase( - Future Function(sqlite.Database db) compute) async { + Future Function(CommonDatabase db) compute) async { return _sendPort.post(_SqliteIsolateClosure(compute)); } @@ -271,7 +276,7 @@ void _sqliteConnectionIsolate(_SqliteConnectionParams params) async { } Future _sqliteConnectionIsolateInner(_SqliteConnectionParams params, - ChildPortClient client, sqlite.Database db) async { + ChildPortClient client, CommonDatabase db) async { final server = params.portServer; final commandPort = ReceivePort(); @@ -372,7 +377,7 @@ class _SqliteConnectionParams { final SerializedPortClient port; final bool primary; - final SqliteOpenFactory openFactory; + final AbstractDefaultSqliteOpenFactory openFactory; _SqliteConnectionParams( {required this.openFactory, diff --git a/lib/src/native/database/native_sqlite_database.dart b/lib/src/native/database/native_sqlite_database.dart new file mode 100644 index 0000000..5e2030c --- /dev/null +++ b/lib/src/native/database/native_sqlite_database.dart @@ -0,0 +1,166 @@ +import 'dart:async'; + +import 'package:meta/meta.dart'; +import '../../common/abstract_open_factory.dart'; +import '../../common/sqlite_database.dart'; +import '../../native/database/connection_pool.dart'; +import '../../native/database/native_sqlite_connection_impl.dart'; +import '../../native/native_isolate_connection_factory.dart'; +import '../../native/native_isolate_mutex.dart'; +import '../../native/native_sqlite_open_factory.dart'; +import '../../sqlite_connection.dart'; +import '../../sqlite_options.dart'; +import '../../sqlite_queries.dart'; +import '../../update_notification.dart'; + +/// A SQLite database instance. +/// +/// Use one instance per database file. If multiple instances are used, update +/// notifications may not trigger, and calls may fail with "SQLITE_BUSY" errors. +class SqliteDatabaseImpl + with SqliteQueries, SqliteDatabaseMixin + implements SqliteDatabase { + @override + final DefaultSqliteOpenFactory openFactory; + + @override + late Stream updates; + + @override + int maxReaders; + + /// Global lock to serialize write transactions. + final SimpleMutex mutex = SimpleMutex(); + + @override + @protected + // Native doesn't require any asynchronous initialization + late Future isInitialized = Future.value(); + + late final SqliteConnectionImpl _internalConnection; + late final SqliteConnectionPool _pool; + + /// Open a SqliteDatabase. + /// + /// Only a single SqliteDatabase per [path] should be opened at a time. + /// + /// A connection pool is used by default, allowing multiple concurrent read + /// transactions, and a single concurrent write transaction. Write transactions + /// do not block read transactions, and read transactions will see the state + /// from the last committed write transaction. + /// + /// A maximum of [maxReaders] concurrent read transactions are allowed. + factory SqliteDatabaseImpl( + {required path, + int maxReaders = SqliteDatabase.defaultMaxReaders, + SqliteOptions options = const SqliteOptions.defaults()}) { + final factory = + DefaultSqliteOpenFactory(path: path, sqliteOptions: options); + return SqliteDatabaseImpl.withFactory(factory, maxReaders: maxReaders); + } + + /// Advanced: Open a database with a specified factory. + /// + /// The factory is used to open each database connection in background isolates. + /// + /// Use when control is required over the opening process. Examples include: + /// 1. Specifying the path to `libsqlite.so` on Linux. + /// 2. Running additional per-connection PRAGMA statements on each connection. + /// 3. Creating custom SQLite functions. + /// 4. Creating temporary views or triggers. + SqliteDatabaseImpl.withFactory(AbstractDefaultSqliteOpenFactory factory, + {this.maxReaders = SqliteDatabase.defaultMaxReaders}) + : openFactory = factory as DefaultSqliteOpenFactory { + _internalConnection = _openPrimaryConnection(debugName: 'sqlite-writer'); + _pool = SqliteConnectionPool(openFactory, + writeConnection: _internalConnection, + debugName: 'sqlite', + maxReaders: maxReaders, + mutex: mutex); + // Updates get updates from the pool + updates = _pool.updates; + } + + @override + bool get closed { + return _pool.closed; + } + + /// Returns true if the _write_ connection is in auto-commit mode + /// (no active transaction). + @override + Future getAutoCommit() { + return _pool.getAutoCommit(); + } + + /// A connection factory that can be passed to different isolates. + /// + /// Use this to access the database in background isolates. + @override + IsolateConnectionFactoryImpl isolateConnectionFactory() { + return IsolateConnectionFactoryImpl( + openFactory: openFactory, + mutex: mutex.shared, + upstreamPort: _pool.upstreamPort!); + } + + @override + Future close() async { + await _pool.close(); + updatesController.close(); + await mutex.close(); + } + + /// Open a read-only transaction. + /// + /// Up to [maxReaders] read transactions can run concurrently. + /// After that, read transactions are queued. + /// + /// Read transactions can run concurrently to a write transaction. + /// + /// Changes from any write transaction are not visible to read transactions + /// started before it. + @override + Future readTransaction( + Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout}) { + return _pool.readTransaction(callback, lockTimeout: lockTimeout); + } + + /// Open a read-write transaction. + /// + /// Only a single write transaction can run at a time - any concurrent + /// transactions are queued. + /// + /// The write transaction is automatically committed when the callback finishes, + /// or rolled back on any error. + @override + Future writeTransaction( + Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout}) { + return _pool.writeTransaction(callback, lockTimeout: lockTimeout); + } + + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + return _pool.readLock(callback, + lockTimeout: lockTimeout, debugContext: debugContext); + } + + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + return _pool.writeLock(callback, + lockTimeout: lockTimeout, debugContext: debugContext); + } + + SqliteConnectionImpl _openPrimaryConnection({String? debugName}) { + return SqliteConnectionImpl( + primary: true, + debugName: debugName, + mutex: mutex, + readOnly: false, + openFactory: openFactory); + } +} diff --git a/lib/src/native/database/upstream_updates.dart b/lib/src/native/database/upstream_updates.dart new file mode 100644 index 0000000..28ea768 --- /dev/null +++ b/lib/src/native/database/upstream_updates.dart @@ -0,0 +1,67 @@ +import 'dart:async'; +import 'dart:isolate'; + +import 'package:meta/meta.dart'; +import '../../common/port_channel.dart'; +import '../../update_notification.dart'; +import '../../utils/native_database_utils.dart'; +import '../../utils/shared_utils.dart'; + +mixin UpStreamTableUpdates { + final StreamController updatesController = + StreamController.broadcast(); + + late SerializedPortClient upstreamPort; + + @protected + + /// Resolves once the primary connection is initialized + late Future isInitialized; + + @protected + PortServer? eventsPort; + + @protected + SerializedPortClient listenForEvents() { + UpdateNotification? updates; + + Map subscriptions = {}; + + eventsPort = PortServer((message) async { + if (message is UpdateNotification) { + if (updates == null) { + updates = message; + // Use the mutex to only send updates after the current transaction. + // Do take care to avoid getting a lock for each individual update - + // that could add massive performance overhead. + if (updates != null) { + updatesController.add(updates!); + updates = null; + } + } else { + updates!.tables.addAll(message.tables); + } + return null; + } else if (message is InitDb) { + await isInitialized; + return null; + } else if (message is SubscribeToUpdates) { + if (subscriptions.containsKey(message.port)) { + return; + } + final subscription = updatesController.stream.listen((event) { + message.port.send(event); + }); + subscriptions[message.port] = subscription; + return null; + } else if (message is UnsubscribeToUpdates) { + final subscription = subscriptions.remove(message.port); + subscription?.cancel(); + return null; + } else { + throw ArgumentError('Unknown message type: $message'); + } + }); + return upstreamPort = eventsPort!.client(); + } +} diff --git a/lib/src/native/native_isolate_connection_factory.dart b/lib/src/native/native_isolate_connection_factory.dart new file mode 100644 index 0000000..915d2ad --- /dev/null +++ b/lib/src/native/native_isolate_connection_factory.dart @@ -0,0 +1,102 @@ +import 'dart:async'; +import 'dart:isolate'; + +import 'package:sqlite_async/src/common/isolate_connection_factory.dart'; +import 'package:sqlite_async/src/common/port_channel.dart'; +import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; +import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/update_notification.dart'; +import 'package:sqlite_async/src/utils/database_utils.dart'; +import 'database/native_sqlite_connection_impl.dart'; + +/// A connection factory that can be passed to different isolates. +class IsolateConnectionFactoryImpl + with IsolateOpenFactoryMixin + implements IsolateConnectionFactory { + @override + DefaultSqliteOpenFactory openFactory; + + @override + SerializedMutex mutex; + + @override + final SerializedPortClient upstreamPort; + + IsolateConnectionFactoryImpl( + {required this.openFactory, + required this.mutex, + required this.upstreamPort}); + + /// Open a new SqliteConnection. + /// + /// This opens a single connection in a background execution isolate. + @override + SqliteConnection open({String? debugName, bool readOnly = false}) { + final updates = _IsolateUpdateListener(upstreamPort); + + var openMutex = mutex.open(); + + return _IsolateSqliteConnection( + openFactory: openFactory, + mutex: openMutex, + upstreamPort: upstreamPort, + readOnly: readOnly, + debugName: debugName, + updates: updates.stream, + closeFunction: () async { + await openMutex.close(); + updates.close(); + }); + } +} + +class _IsolateUpdateListener { + final ChildPortClient client; + final ReceivePort port = ReceivePort(); + late final StreamController controller; + + _IsolateUpdateListener(SerializedPortClient upstreamPort) + : client = upstreamPort.open() { + controller = StreamController.broadcast(onListen: () { + client.fire(SubscribeToUpdates(port.sendPort)); + }, onCancel: () { + client.fire(UnsubscribeToUpdates(port.sendPort)); + }); + + port.listen((message) { + if (message is UpdateNotification) { + controller.add(message); + } + }); + } + + Stream get stream { + return controller.stream; + } + + close() { + client.fire(UnsubscribeToUpdates(port.sendPort)); + controller.close(); + port.close(); + } +} + +class _IsolateSqliteConnection extends SqliteConnectionImpl { + final Future Function() closeFunction; + + _IsolateSqliteConnection( + {required super.openFactory, + required super.mutex, + super.upstreamPort, + super.updates, + super.debugName, + super.readOnly = false, + required this.closeFunction}); + + @override + Future close() async { + await super.close(); + await closeFunction(); + } +} diff --git a/lib/src/native/native_isolate_mutex.dart b/lib/src/native/native_isolate_mutex.dart new file mode 100644 index 0000000..23e5443 --- /dev/null +++ b/lib/src/native/native_isolate_mutex.dart @@ -0,0 +1,257 @@ +// Adapted from: +// https://github.com/tekartik/synchronized.dart +// (MIT) +import 'dart:async'; + +import 'package:sqlite_async/src/common/mutex.dart'; +import 'package:sqlite_async/src/common/port_channel.dart'; + +abstract class MutexImpl implements Mutex { + factory MutexImpl() { + return SimpleMutex(); + } +} + +/// Mutex maintains a queue of Future-returning functions that +/// are executed sequentially. +/// The internal lock is not shared across Isolates by default. +class SimpleMutex implements MutexImpl { + // Adapted from https://github.com/tekartik/synchronized.dart/blob/master/synchronized/lib/src/basic_lock.dart + + Future? last; + + // Hack to make sure the Mutex is not copied to another isolate. + // ignore: unused_field + final Finalizer _f = Finalizer((_) {}); + + SimpleMutex(); + + bool get locked => last != null; + + _SharedMutexServer? _shared; + + @override + Future lock(Future Function() callback, {Duration? timeout}) async { + if (Zone.current[this] != null) { + throw LockError('Recursive lock is not allowed'); + } + var zone = Zone.current.fork(zoneValues: {this: true}); + + return zone.run(() async { + final prev = last; + final completer = Completer.sync(); + last = completer.future; + try { + // If there is a previous running block, wait for it + if (prev != null) { + if (timeout != null) { + // This could throw a timeout error + try { + await prev.timeout(timeout); + } catch (error) { + if (error is TimeoutException) { + throw TimeoutException('Failed to acquire lock', timeout); + } else { + rethrow; + } + } + } else { + await prev; + } + } + + // Run the function and return the result + return await callback(); + } finally { + // Cleanup + // waiting for the previous task to be done in case of timeout + void complete() { + // Only mark it unlocked when the last one complete + if (identical(last, completer.future)) { + last = null; + } + completer.complete(); + } + + // In case of timeout, wait for the previous one to complete too + // before marking this task as complete + if (prev != null && timeout != null) { + // But we still returns immediately + prev.then((_) { + complete(); + }).ignore(); + } else { + complete(); + } + } + }); + } + + @override + open() { + return this; + } + + @override + Future close() async { + _shared?.close(); + await lock(() async {}); + } + + /// Get a serialized instance that can be passed over to a different isolate. + SerializedMutex get shared { + _shared ??= _SharedMutexServer._withMutex(this); + return _shared!.serialized; + } +} + +/// Serialized version of a Mutex, can be passed over to different isolates. +/// Use [open] to get a [SharedMutex] instance. +/// +/// Uses a [SendPort] to communicate with the source mutex. +class SerializedMutex implements Mutex { + final SerializedPortClient client; + + const SerializedMutex(this.client); + + @override + SharedMutex open() { + return SharedMutex._(client.open()); + } + + @override + Future close() { + throw UnimplementedError(); + } + + @override + Future lock(Future Function() callback, {Duration? timeout}) { + throw UnimplementedError(); + } +} + +/// Mutex instantiated from a source mutex, potentially in a different isolate. +/// +/// Uses a [SendPort] to communicate with the source mutex. +class SharedMutex implements Mutex { + final ChildPortClient client; + bool closed = false; + + SharedMutex._(this.client); + + @override + Future lock(Future Function() callback, {Duration? timeout}) async { + if (Zone.current[this] != null) { + throw LockError('Recursive lock is not allowed'); + } + return runZoned(() async { + if (closed) { + throw const ClosedException(); + } + await _acquire(timeout: timeout); + try { + final T result = await callback(); + return result; + } finally { + _unlock(); + } + }, zoneValues: {this: true}); + } + + _unlock() { + client.fire(const _UnlockMessage()); + } + + @override + open() { + return this; + } + + Future _acquire({Duration? timeout}) async { + final lockFuture = client.post(const _AcquireMessage()); + bool timedout = false; + + var handledLockFuture = lockFuture.then((_) { + if (timedout) { + _unlock(); + throw TimeoutException('Failed to acquire lock', timeout); + } + }); + + if (timeout != null) { + handledLockFuture = + handledLockFuture.timeout(timeout).catchError((error, stacktrace) { + timedout = true; + if (error is TimeoutException) { + throw TimeoutException('Failed to acquire SharedMutex lock', timeout); + } + throw error; + }); + } + return await handledLockFuture; + } + + @override + + /// Wait for existing locks to be released, then close this SharedMutex + /// and prevent further locks from being taken out. + Future close() async { + if (closed) { + return; + } + closed = true; + // Wait for any existing locks to complete, then prevent any further locks from being taken out. + await _acquire(); + // Release the lock + _unlock(); + // Close client immediately after _unlock(), + // so that we're sure no further locks are acquired. + // This also cancels any lock request in process. + client.close(); + } +} + +/// Manages a [SerializedMutex], allowing a [Mutex] to be shared across isolates. +class _SharedMutexServer { + Completer? unlock; + late final SerializedMutex serialized; + final Mutex mutex; + + late final PortServer server; + + _SharedMutexServer._withMutex(this.mutex) { + server = PortServer((Object? arg) async { + return await _handle(arg); + }); + serialized = SerializedMutex(server.client()); + } + + Future _handle(Object? arg) async { + if (arg is _AcquireMessage) { + var lock = Completer.sync(); + mutex.lock(() async { + assert(unlock == null); + unlock = Completer.sync(); + lock.complete(); + await unlock!.future; + unlock = null; + }); + await lock.future; + } else if (arg is _UnlockMessage) { + assert(unlock != null); + unlock!.complete(); + } + } + + void close() async { + server.close(); + } +} + +class _AcquireMessage { + const _AcquireMessage(); +} + +class _UnlockMessage { + const _UnlockMessage(); +} diff --git a/lib/src/native/native_sqlite_open_factory.dart b/lib/src/native/native_sqlite_open_factory.dart new file mode 100644 index 0000000..5773be9 --- /dev/null +++ b/lib/src/native/native_sqlite_open_factory.dart @@ -0,0 +1,60 @@ +import 'package:sqlite_async/sqlite3.dart' as sqlite; +import 'package:sqlite_async/sqlite3_common.dart'; + +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/native/database/native_sqlite_connection_impl.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/sqlite_options.dart'; + +/// Native implementation of [AbstractDefaultSqliteOpenFactory] +class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { + const DefaultSqliteOpenFactory( + {required super.path, + super.sqliteOptions = const SqliteOptions.defaults()}); + + @override + CommonDatabase openDB(SqliteOpenOptions options) { + final mode = options.openMode; + var db = sqlite.sqlite3.open(path, mode: mode, mutex: false); + return db; + } + + @override + List pragmaStatements(SqliteOpenOptions options) { + List statements = []; + + if (sqliteOptions.lockTimeout != null) { + // May be replaced by a Dart-level retry mechanism in the future + statements.add( + 'PRAGMA busy_timeout = ${sqliteOptions.lockTimeout!.inMilliseconds}'); + } + + if (options.primaryConnection && sqliteOptions.journalMode != null) { + // Persisted - only needed on the primary connection + statements + .add('PRAGMA journal_mode = ${sqliteOptions.journalMode!.name}'); + } + if (!options.readOnly && sqliteOptions.journalSizeLimit != null) { + // Needed on every writable connection + statements.add( + 'PRAGMA journal_size_limit = ${sqliteOptions.journalSizeLimit!}'); + } + if (sqliteOptions.synchronous != null) { + // Needed on every connection + statements.add('PRAGMA synchronous = ${sqliteOptions.synchronous!.name}'); + } + return statements; + } + + @override + SqliteConnection openConnection(SqliteOpenOptions options) { + return SqliteConnectionImpl( + primary: options.primaryConnection, + readOnly: options.readOnly, + mutex: options.mutex!, + debugName: options.debugName, + updates: options.updates, + openFactory: this, + ); + } +} diff --git a/lib/src/sqlite_connection.dart b/lib/src/sqlite_connection.dart index 1fc5525..f92d318 100644 --- a/lib/src/sqlite_connection.dart +++ b/lib/src/sqlite_connection.dart @@ -1,4 +1,7 @@ -import 'package:sqlite3/sqlite3.dart' as sqlite; +import 'dart:async'; + +import 'package:sqlite3/common.dart' as sqlite; +import 'package:sqlite_async/src/update_notification.dart'; /// Abstract class representing calls available in a read-only or read-write context. abstract class SqliteReadContext { @@ -55,7 +58,7 @@ abstract class SqliteReadContext { /// } /// ``` Future computeWithDatabase( - Future Function(sqlite.Database db) compute); + Future Function(sqlite.CommonDatabase db) compute); } /// Abstract class representing calls available in a read-write context. @@ -72,6 +75,9 @@ abstract class SqliteWriteContext extends SqliteReadContext { /// Abstract class representing a connection to the SQLite database. abstract class SqliteConnection extends SqliteWriteContext { + /// Reports table change update notifications + Stream? get updates; + /// Open a read-only transaction. /// /// Statements within the transaction must be done on the provided diff --git a/lib/src/sqlite_database.dart b/lib/src/sqlite_database.dart index d792092..080cb10 100644 --- a/lib/src/sqlite_database.dart +++ b/lib/src/sqlite_database.dart @@ -1,237 +1 @@ -import 'dart:async'; -import 'dart:isolate'; - -import 'connection_pool.dart'; -import 'database_utils.dart'; -import 'isolate_connection_factory.dart'; -import 'mutex.dart'; -import 'port_channel.dart'; -import 'sqlite_connection.dart'; -import 'sqlite_connection_impl.dart'; -import 'sqlite_open_factory.dart'; -import 'sqlite_options.dart'; -import 'sqlite_queries.dart'; -import 'update_notification.dart'; - -/// A SQLite database instance. -/// -/// Use one instance per database file where feasible. -/// -/// If multiple instances are used, update notifications will not be propagated between them. -/// For update notifications across isolates, use [isolateConnectionFactory]. -class SqliteDatabase with SqliteQueries implements SqliteConnection { - /// The maximum number of concurrent read transactions if not explicitly specified. - static const int defaultMaxReaders = 5; - - /// Maximum number of concurrent read transactions. - final int maxReaders; - - /// Global lock to serialize write transactions. - final SimpleMutex mutex = SimpleMutex(); - - /// Factory that opens a raw database connection in each isolate. - /// - /// This must be safe to pass to different isolates. - /// - /// Use a custom class for this to customize the open process. - final SqliteOpenFactory openFactory; - - /// Use this stream to subscribe to notifications of updates to tables. - @override - late final Stream updates; - - final StreamController _updatesController = - StreamController.broadcast(); - - late final PortServer _eventsPort; - - late final SqliteConnectionImpl _internalConnection; - late final SqliteConnectionPool _pool; - late final Future _initialized; - - /// Open a SqliteDatabase. - /// - /// Only a single SqliteDatabase per [path] should be opened at a time. - /// - /// A connection pool is used by default, allowing multiple concurrent read - /// transactions, and a single concurrent write transaction. Write transactions - /// do not block read transactions, and read transactions will see the state - /// from the last committed write transaction. - /// - /// A maximum of [maxReaders] concurrent read transactions are allowed. - factory SqliteDatabase( - {required path, - int maxReaders = defaultMaxReaders, - SqliteOptions options = const SqliteOptions.defaults()}) { - final factory = - DefaultSqliteOpenFactory(path: path, sqliteOptions: options); - return SqliteDatabase.withFactory(factory, maxReaders: maxReaders); - } - - /// Advanced: Open a database with a specified factory. - /// - /// The factory is used to open each database connection in background isolates. - /// - /// Use when control is required over the opening process. Examples include: - /// 1. Specifying the path to `libsqlite.so` on Linux. - /// 2. Running additional per-connection PRAGMA statements on each connection. - /// 3. Creating custom SQLite functions. - /// 4. Creating temporary views or triggers. - SqliteDatabase.withFactory(this.openFactory, - {this.maxReaders = defaultMaxReaders}) { - updates = _updatesController.stream; - - _listenForEvents(); - - _internalConnection = _openPrimaryConnection(debugName: 'sqlite-writer'); - _pool = SqliteConnectionPool(openFactory, - upstreamPort: _eventsPort.client(), - updates: updates, - writeConnection: _internalConnection, - debugName: 'sqlite', - maxReaders: maxReaders, - mutex: mutex); - - _initialized = _init(); - } - - Future _init() async { - await _internalConnection.ready; - } - - /// Wait for initialization to complete. - /// - /// While initializing is automatic, this helps to catch and report initialization errors. - Future initialize() async { - await _initialized; - } - - @override - bool get closed { - return _pool.closed; - } - - /// Returns true if the _write_ connection is in auto-commit mode - /// (no active transaction). - @override - Future getAutoCommit() { - return _pool.getAutoCommit(); - } - - void _listenForEvents() { - UpdateNotification? updates; - - Map subscriptions = {}; - - _eventsPort = PortServer((message) async { - if (message is UpdateNotification) { - if (updates == null) { - updates = message; - // Use the mutex to only send updates after the current transaction. - // Do take care to avoid getting a lock for each individual update - - // that could add massive performance overhead. - mutex.lock(() async { - if (updates != null) { - _updatesController.add(updates!); - updates = null; - } - }); - } else { - updates!.tables.addAll(message.tables); - } - return null; - } else if (message is InitDb) { - await _initialized; - return null; - } else if (message is SubscribeToUpdates) { - if (subscriptions.containsKey(message.port)) { - return; - } - final subscription = _updatesController.stream.listen((event) { - message.port.send(event); - }); - subscriptions[message.port] = subscription; - return null; - } else if (message is UnsubscribeToUpdates) { - final subscription = subscriptions.remove(message.port); - subscription?.cancel(); - return null; - } else { - throw ArgumentError('Unknown message type: $message'); - } - }); - } - - /// A connection factory that can be passed to different isolates. - /// - /// Use this to access the database in background isolates. - IsolateConnectionFactory isolateConnectionFactory() { - return IsolateConnectionFactory( - openFactory: openFactory, - mutex: mutex.shared, - upstreamPort: _eventsPort.client()); - } - - SqliteConnectionImpl _openPrimaryConnection({String? debugName}) { - return SqliteConnectionImpl( - upstreamPort: _eventsPort.client(), - primary: true, - updates: updates, - debugName: debugName, - mutex: mutex, - readOnly: false, - openFactory: openFactory); - } - - @override - Future close() async { - await _pool.close(); - _updatesController.close(); - _eventsPort.close(); - await mutex.close(); - } - - /// Open a read-only transaction. - /// - /// Up to [maxReaders] read transactions can run concurrently. - /// After that, read transactions are queued. - /// - /// Read transactions can run concurrently to a write transaction. - /// - /// Changes from any write transaction are not visible to read transactions - /// started before it. - @override - Future readTransaction( - Future Function(SqliteReadContext tx) callback, - {Duration? lockTimeout}) { - return _pool.readTransaction(callback, lockTimeout: lockTimeout); - } - - /// Open a read-write transaction. - /// - /// Only a single write transaction can run at a time - any concurrent - /// transactions are queued. - /// - /// The write transaction is automatically committed when the callback finishes, - /// or rolled back on any error. - @override - Future writeTransaction( - Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout}) { - return _pool.writeTransaction(callback, lockTimeout: lockTimeout); - } - - @override - Future readLock(Future Function(SqliteReadContext tx) callback, - {Duration? lockTimeout, String? debugContext}) { - return _pool.readLock(callback, - lockTimeout: lockTimeout, debugContext: debugContext); - } - - @override - Future writeLock(Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout, String? debugContext}) { - return _pool.writeLock(callback, - lockTimeout: lockTimeout, debugContext: debugContext); - } -} +export 'package:sqlite_async/src/impl/sqlite_database_impl.dart'; diff --git a/lib/src/sqlite_open_factory.dart b/lib/src/sqlite_open_factory.dart index b4779ef..0fd42cb 100644 --- a/lib/src/sqlite_open_factory.dart +++ b/lib/src/sqlite_open_factory.dart @@ -1,99 +1 @@ -import 'dart:async'; - -import 'package:sqlite_async/sqlite3.dart' as sqlite; - -import 'sqlite_options.dart'; - -/// Factory to create new SQLite database connections. -/// -/// Since connections are opened in dedicated background isolates, this class -/// must be safe to pass to different isolates. -abstract class SqliteOpenFactory { - FutureOr open(SqliteOpenOptions options); -} - -/// The default database factory. -/// -/// This takes care of opening the database, and running PRAGMA statements -/// to configure the connection. -/// -/// Override the [open] method to customize the process. -class DefaultSqliteOpenFactory implements SqliteOpenFactory { - final String path; - final SqliteOptions sqliteOptions; - - const DefaultSqliteOpenFactory( - {required this.path, - this.sqliteOptions = const SqliteOptions.defaults()}); - - List pragmaStatements(SqliteOpenOptions options) { - List statements = []; - - if (sqliteOptions.lockTimeout != null) { - // May be replaced by a Dart-level retry mechanism in the future - statements.add( - 'PRAGMA busy_timeout = ${sqliteOptions.lockTimeout!.inMilliseconds}'); - } - - if (options.primaryConnection && sqliteOptions.journalMode != null) { - // Persisted - only needed on the primary connection - statements - .add('PRAGMA journal_mode = ${sqliteOptions.journalMode!.name}'); - } - if (!options.readOnly && sqliteOptions.journalSizeLimit != null) { - // Needed on every writable connection - statements.add( - 'PRAGMA journal_size_limit = ${sqliteOptions.journalSizeLimit!}'); - } - if (sqliteOptions.synchronous != null) { - // Needed on every connection - statements.add('PRAGMA synchronous = ${sqliteOptions.synchronous!.name}'); - } - return statements; - } - - @override - sqlite.Database open(SqliteOpenOptions options) { - final mode = options.openMode; - var db = sqlite.sqlite3.open(path, mode: mode, mutex: false); - - // Pragma statements don't have the same BUSY_TIMEOUT behavior as normal statements. - // We add a manual retry loop for those. - for (var statement in pragmaStatements(options)) { - for (var tries = 0; tries < 30; tries++) { - try { - db.execute(statement); - break; - } on sqlite.SqliteException catch (e) { - if (e.resultCode == sqlite.SqlError.SQLITE_BUSY && tries < 29) { - continue; - } else { - rethrow; - } - } - } - } - return db; - } -} - -class SqliteOpenOptions { - /// Whether this is the primary write connection for the database. - final bool primaryConnection; - - /// Whether this connection is read-only. - final bool readOnly; - - const SqliteOpenOptions( - {required this.primaryConnection, required this.readOnly}); - - sqlite.OpenMode get openMode { - if (primaryConnection) { - return sqlite.OpenMode.readWriteCreate; - } else if (readOnly) { - return sqlite.OpenMode.readOnly; - } else { - return sqlite.OpenMode.readWrite; - } - } -} +export 'impl/open_factory_impl.dart'; diff --git a/lib/src/sqlite_queries.dart b/lib/src/sqlite_queries.dart index 055c11c..d0eab7a 100644 --- a/lib/src/sqlite_queries.dart +++ b/lib/src/sqlite_queries.dart @@ -1,6 +1,6 @@ -import 'package:sqlite3/sqlite3.dart' as sqlite; +import 'package:sqlite3/common.dart' as sqlite; -import 'database_utils.dart'; +import 'utils/shared_utils.dart'; import 'sqlite_connection.dart'; import 'update_notification.dart'; @@ -9,9 +9,6 @@ import 'update_notification.dart'; /// Classes using this need to implement [SqliteConnection.readLock] /// and [SqliteConnection.writeLock]. mixin SqliteQueries implements SqliteWriteContext, SqliteConnection { - /// Broadcast stream that is notified of any table updates - Stream? get updates; - @override Future execute(String sql, [List parameters = const []]) async { @@ -122,7 +119,7 @@ mixin SqliteQueries implements SqliteWriteContext, SqliteConnection { /// write transaction. @override Future computeWithDatabase( - Future Function(sqlite.Database db) compute) { + Future Function(sqlite.CommonDatabase db) compute) { return writeTransaction((tx) async { return tx.computeWithDatabase(compute); }); diff --git a/lib/src/utils/database_utils.dart b/lib/src/utils/database_utils.dart new file mode 100644 index 0000000..9041e2a --- /dev/null +++ b/lib/src/utils/database_utils.dart @@ -0,0 +1,2 @@ +export 'native_database_utils.dart'; +export 'shared_utils.dart'; diff --git a/lib/src/utils/native_database_utils.dart b/lib/src/utils/native_database_utils.dart new file mode 100644 index 0000000..6b03913 --- /dev/null +++ b/lib/src/utils/native_database_utils.dart @@ -0,0 +1,13 @@ +import 'dart:isolate'; + +class SubscribeToUpdates { + final SendPort port; + + SubscribeToUpdates(this.port); +} + +class UnsubscribeToUpdates { + final SendPort port; + + UnsubscribeToUpdates(this.port); +} diff --git a/lib/src/database_utils.dart b/lib/src/utils/shared_utils.dart similarity index 91% rename from lib/src/database_utils.dart rename to lib/src/utils/shared_utils.dart index b19abda..c911bbc 100644 --- a/lib/src/database_utils.dart +++ b/lib/src/utils/shared_utils.dart @@ -1,8 +1,7 @@ import 'dart:async'; import 'dart:convert'; -import 'dart:isolate'; -import 'sqlite_connection.dart'; +import '../sqlite_connection.dart'; Future internalReadTransaction(SqliteReadContext ctx, Future Function(SqliteReadContext tx) callback) async { @@ -78,18 +77,6 @@ class InitDb { const InitDb(); } -class SubscribeToUpdates { - final SendPort port; - - SubscribeToUpdates(this.port); -} - -class UnsubscribeToUpdates { - final SendPort port; - - UnsubscribeToUpdates(this.port); -} - Object? mapParameter(Object? parameter) { if (parameter == null || parameter is int || diff --git a/lib/utils.dart b/lib/utils.dart new file mode 100644 index 0000000..9883819 --- /dev/null +++ b/lib/utils.dart @@ -0,0 +1 @@ +export 'src/utils.dart'; diff --git a/pubspec.yaml b/pubspec.yaml index 8290330..cc5b647 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -9,6 +9,7 @@ dependencies: sqlite3: "^2.3.0" async: ^2.10.0 collection: ^1.17.0 + meta: ^1.10.0 dev_dependencies: lints: ^3.0.0 @@ -16,3 +17,7 @@ dev_dependencies: test_api: ^0.7.0 glob: ^2.1.1 benchmarking: ^0.6.1 + shelf: ^1.4.1 + shelf_static: ^1.1.2 + stream_channel: ^2.1.2 + path: ^1.9.0 diff --git a/scripts/benchmark.dart b/scripts/benchmark.dart index 10a133e..ac919cf 100644 --- a/scripts/benchmark.dart +++ b/scripts/benchmark.dart @@ -4,10 +4,11 @@ import 'dart:math'; import 'package:benchmarking/benchmarking.dart'; import 'package:collection/collection.dart'; - import 'package:sqlite_async/sqlite_async.dart'; -import '../test/util.dart'; +import '../test/utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); typedef BenchmarkFunction = Future Function( SqliteDatabase, List>); @@ -177,7 +178,7 @@ void main() async { await db.execute('PRAGMA wal_checkpoint(TRUNCATE)'); } - final db = await setupDatabase(path: 'test-db/benchmark.db'); + final db = await testUtils.setupDatabase(path: 'test-db/benchmark.db'); await db.execute('PRAGMA wal_autocheckpoint = 0'); await createTables(db); diff --git a/test/close_test.dart b/test/close_test.dart index be3e933..eed427c 100644 --- a/test/close_test.dart +++ b/test/close_test.dart @@ -1,21 +1,24 @@ +@TestOn('!browser') import 'dart:io'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; -import 'util.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); void main() { group('Close Tests', () { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); }); tearDown(() async { - await cleanDb(path: path); + await testUtils.cleanDb(path: path); }); createTables(SqliteDatabase db) async { @@ -30,7 +33,7 @@ void main() { // If the write connection is closed before the read connections, that is // not the case. - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); await db.execute( diff --git a/test/isolate_test.dart b/test/isolate_test.dart index 69b5035..60bea87 100644 --- a/test/isolate_test.dart +++ b/test/isolate_test.dart @@ -1,24 +1,27 @@ +@TestOn('!browser') import 'dart:isolate'; import 'package:test/test.dart'; -import 'util.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); void main() { group('Isolate Tests', () { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); }); tearDown(() async { - await cleanDb(path: path); + await testUtils.cleanDb(path: path); }); test('Basic Isolate usage', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); final factory = db.isolateConnectionFactory(); final result = await Isolate.run(() async { diff --git a/test/json1_test.dart b/test/json1_test.dart index 62bd6c3..93e44cc 100644 --- a/test/json1_test.dart +++ b/test/json1_test.dart @@ -1,7 +1,11 @@ +import 'dart:convert'; + import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; -import 'util.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); class TestUser { int? id; @@ -24,12 +28,12 @@ void main() { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); }); tearDown(() async { - await cleanDb(path: path); + await testUtils.cleanDb(path: path); }); createTables(SqliteDatabase db) async { @@ -40,9 +44,8 @@ void main() { } test('Inserts', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); - var users1 = [ TestUser(name: 'Bob', email: 'bob@example.org'), TestUser(name: 'Alice', email: 'alice@example.org') @@ -51,27 +54,29 @@ void main() { TestUser(name: 'Charlie', email: 'charlie@example.org'), TestUser(name: 'Dan', email: 'dan@example.org') ]; + var ids1 = await db.execute( "INSERT INTO users(name, email) SELECT e.value ->> 'name', e.value ->> 'email' FROM json_each(?) e RETURNING id", - [users1]); + [jsonEncode(users1)]); var ids2 = await db.execute( "INSERT INTO users(name, email) ${selectJsonColumns([ 'name', 'email' ])} RETURNING id", - [users2]); + [jsonEncode(users2)]); var ids = [ for (var row in ids1) row, for (var row in ids2) row, ]; + var results = [ for (var row in await db.getAll( "SELECT id, name, email FROM users WHERE id IN (${selectJsonColumns([ 'id' ])}) ORDER BY name", - [ids])) + [jsonEncode(ids)])) TestUser.fromMap(row) ]; diff --git a/test/migration_test.dart b/test/migration_test.dart index b0c6145..37d4c79 100644 --- a/test/migration_test.dart +++ b/test/migration_test.dart @@ -1,23 +1,25 @@ import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; -import 'util.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); void main() { group('Basic Tests', () { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); }); tearDown(() async { - await cleanDb(path: path); + await testUtils.cleanDb(path: path); }); test('Basic Migrations', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); final migrations = SqliteMigrations(); migrations.add(SqliteMigration(1, (tx) async { await tx.execute( @@ -51,7 +53,7 @@ void main() { }); test('Migration with createDatabase', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); final migrations = SqliteMigrations(); migrations.add(SqliteMigration(1, (tx) async { await tx.execute( @@ -83,7 +85,7 @@ void main() { }); test('Migration with down migrations', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); final migrations = SqliteMigrations(); migrations.add(SqliteMigration(1, (tx) async { await tx.execute( @@ -135,7 +137,7 @@ void main() { }); test('Migration with double down migrations', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); final migrations = SqliteMigrations(); migrations.add(SqliteMigration(1, (tx) async { await tx.execute( diff --git a/test/mutex_test.dart b/test/mutex_test.dart index 297bf53..699a877 100644 --- a/test/mutex_test.dart +++ b/test/mutex_test.dart @@ -1,6 +1,7 @@ +@TestOn('!browser') import 'dart:isolate'; -import 'package:sqlite_async/mutex.dart'; +import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; import 'package:test/test.dart'; void main() { diff --git a/test/basic_test.dart b/test/native/basic_test.dart similarity index 68% rename from test/basic_test.dart rename to test/native/basic_test.dart index a3676cf..263ab39 100644 --- a/test/basic_test.dart +++ b/test/native/basic_test.dart @@ -1,24 +1,26 @@ +@TestOn('!browser') import 'dart:async'; import 'dart:math'; -import 'package:sqlite3/sqlite3.dart' as sqlite; -import 'package:sqlite_async/mutex.dart'; +import 'package:sqlite3/common.dart' as sqlite; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; -import 'util.dart'; +import '../utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); void main() { group('Basic Tests', () { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); }); tearDown(() async { - await cleanDb(path: path); + await testUtils.cleanDb(path: path); }); createTables(SqliteDatabase db) async { @@ -29,7 +31,7 @@ void main() { } test('Basic Setup', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); await db.execute( @@ -50,8 +52,9 @@ void main() { // Manually verified test('Concurrency', () async { - final db = - SqliteDatabase.withFactory(testFactory(path: path), maxReaders: 3); + final db = SqliteDatabase.withFactory( + await testUtils.testFactory(path: path), + maxReaders: 3); await db.initialize(); await createTables(db); @@ -65,11 +68,9 @@ void main() { }); test('Concurrency 2', () async { - final db1 = - SqliteDatabase.withFactory(testFactory(path: path), maxReaders: 3); + final db1 = await testUtils.setupDatabase(path: path, maxReaders: 3); + final db2 = await testUtils.setupDatabase(path: path, maxReaders: 3); - final db2 = - SqliteDatabase.withFactory(testFactory(path: path), maxReaders: 3); await db1.initialize(); await createTables(db1); await db2.initialize(); @@ -98,7 +99,7 @@ void main() { }); test('read-only transactions', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); // Can read @@ -135,22 +136,9 @@ void main() { }); }); - test('should not allow direct db calls within a transaction callback', - () async { - final db = await setupDatabase(path: path); - await createTables(db); - - await db.writeTransaction((tx) async { - await expectLater(() async { - await db.execute( - 'INSERT INTO test_data(description) VALUES(?)', ['test']); - }, throwsA((e) => e is LockError && e.message.contains('tx.execute'))); - }); - }); - test('should not allow read-only db calls within transaction callback', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); await db.writeTransaction((tx) async { @@ -174,7 +162,7 @@ void main() { }); test('should not allow read-only db calls within lock callback', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); // Locks - should behave the same as transactions above @@ -198,7 +186,7 @@ void main() { test( 'should allow read-only db calls within transaction callback in separate zone', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); // Get a reference to the parent zone (outside the transaction). @@ -234,95 +222,24 @@ void main() { // }); }); - test('should allow PRAMGAs', () async { - final db = await setupDatabase(path: path); - await createTables(db); - // Not allowed in transactions, but does work as a direct statement. - await db.execute('PRAGMA wal_checkpoint(TRUNCATE)'); - await db.execute('VACUUM'); - }); - test('should allow ignoring errors', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); ignore(db.execute( 'INSERT INTO test_data(description) VALUES(json(?))', ['test3'])); }); - test('should properly report errors in transactions', () async { - final db = await setupDatabase(path: path); - await createTables(db); - - var tp = db.writeTransaction((tx) async { - await tx.execute( - 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', - [1, 'test1']); - await tx.execute( - 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', - [2, 'test2']); - expect(await tx.getAutoCommit(), equals(false)); - try { - await tx.execute( - 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', - [2, 'test3']); - } catch (e) { - // Ignore - } - expect(await tx.getAutoCommit(), equals(true)); - expect(tx.closed, equals(false)); - - // Will not be executed because of the above rollback - ignore(tx.execute( - 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', - [4, 'test4'])); - }); - - // The error propagates up to the transaction - await expectLater( - tp, - throwsA((e) => - e is sqlite.SqliteException && - e.message - .contains('Transaction rolled back by earlier statement') && - e.message.contains('UNIQUE constraint failed'))); - - expect(await db.get('SELECT count() count FROM test_data'), - equals({'count': 0})); - - // Check that we can open another transaction afterwards - await db.writeTransaction((tx) async {}); - }); - test('should error on dangling transactions', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); await expectLater(() async { await db.execute('BEGIN'); }, throwsA((e) => e is sqlite.SqliteException)); }); - test('should handle normal errors', () async { - final db = await setupDatabase(path: path); - await createTables(db); - Error? caughtError; - final syntheticError = ArgumentError('foobar'); - await db.computeWithDatabase((db) async { - throw syntheticError; - }).catchError((error) { - caughtError = error; - }); - expect(caughtError.toString(), equals(syntheticError.toString())); - - // Check that we can still continue afterwards - final computed = await db.computeWithDatabase((db) async { - return 5; - }); - expect(computed, equals(5)); - }); - test('should handle uncaught errors', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); Object? caughtError; await db.computeWithDatabase((db) async { @@ -348,7 +265,7 @@ void main() { }); test('should handle uncaught errors in read connections', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); for (var i = 0; i < 10; i++) { Object? caughtError; @@ -380,34 +297,6 @@ void main() { expect(computed, equals(5)); }); - test('should allow resuming transaction after errors', () async { - final db = await setupDatabase(path: path); - await createTables(db); - SqliteWriteContext? savedTx; - await db.writeTransaction((tx) async { - savedTx = tx; - var caught = false; - try { - // This error does not rollback the transaction - await tx.execute('NOT A VALID STATEMENT'); - } catch (e) { - // Ignore - caught = true; - } - expect(caught, equals(true)); - - expect(await tx.getAutoCommit(), equals(false)); - expect(tx.closed, equals(false)); - - final rs = await tx.execute( - 'INSERT INTO test_data(description) VALUES(?) RETURNING description', - ['Test Data']); - expect(rs.rows[0], equals(['Test Data'])); - }); - expect(await savedTx!.getAutoCommit(), equals(true)); - expect(savedTx!.closed, equals(true)); - }); - test('closing', () async { // Test race condition in SqliteConnectionPool: // 1. Open two concurrent queries, which opens two connection. @@ -416,8 +305,8 @@ void main() { // 4. Now second connection is ready. Second query has two connections to choose from. // 5. However, first connection is closed, so it's removed from the pool. // 6. Triggers `Concurrent modification during iteration: Instance(length:1) of '_GrowableList'` - final db = - SqliteDatabase.withFactory(testFactory(path: path, initStatements: [ + final db = SqliteDatabase.withFactory( + await testUtils.testFactory(path: path, initStatements: [ // Second connection to sleep more than first connection 'SELECT test_sleep(test_connection_number() * 10)' ])); @@ -433,8 +322,7 @@ void main() { }); test('lockTimeout', () async { - final db = - SqliteDatabase.withFactory(testFactory(path: path), maxReaders: 2); + final db = await testUtils.setupDatabase(path: path, maxReaders: 2); await db.initialize(); final f1 = db.readTransaction((tx) async { diff --git a/test/native/watch_test.dart b/test/native/watch_test.dart new file mode 100644 index 0000000..e6a5953 --- /dev/null +++ b/test/native/watch_test.dart @@ -0,0 +1,147 @@ +@TestOn('!browser') +import 'dart:async'; +import 'dart:isolate'; +import 'dart:math'; + +import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite_async/src/utils/database_utils.dart'; +import 'package:test/test.dart'; + +import '../utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); + +void main() { + createTables(SqliteDatabase db) async { + await db.writeTransaction((tx) async { + await tx.execute( + 'CREATE TABLE assets(id INTEGER PRIMARY KEY AUTOINCREMENT, make TEXT, customer_id INTEGER)'); + await tx.execute('CREATE INDEX assets_customer ON assets(customer_id)'); + await tx.execute( + 'CREATE TABLE customers(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)'); + await tx.execute( + 'CREATE TABLE other_customers(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)'); + await tx.execute('CREATE VIEW assets_alias AS SELECT * FROM assets'); + }); + } + + group('Query Watch Tests', () { + late String path; + + setUp(() async { + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); + }); + + for (var sqlite in testUtils.findSqliteLibraries()) { + test('getSourceTables - $sqlite', () async { + final db = SqliteDatabase.withFactory( + await testUtils.testFactory(path: path, sqlitePath: sqlite)); + await db.initialize(); + await createTables(db); + + var versionRow = await db.get('SELECT sqlite_version() as version'); + print('Testing SQLite ${versionRow['version']} - $sqlite'); + + final tables = await getSourceTables(db, + 'SELECT * FROM assets INNER JOIN customers ON assets.customer_id = customers.id'); + expect(tables, equals({'assets', 'customers'})); + + final tables2 = await getSourceTables(db, + 'SELECT count() FROM assets INNER JOIN "other_customers" AS oc ON assets.customer_id = oc.id AND assets.make = oc.name'); + expect(tables2, equals({'assets', 'other_customers'})); + + final tables3 = await getSourceTables(db, 'SELECT count() FROM assets'); + expect(tables3, equals({'assets'})); + + final tables4 = + await getSourceTables(db, 'SELECT count() FROM assets_alias'); + expect(tables4, equals({'assets'})); + + final tables5 = + await getSourceTables(db, 'SELECT sqlite_version() as version'); + expect(tables5, equals({})); + }); + } + + test('watch in isolate', () async { + final db = await testUtils.setupDatabase(path: path); + await createTables(db); + + const baseTime = 20; + + const throttleDuration = Duration(milliseconds: baseTime); + + final rows = await db.execute( + 'INSERT INTO customers(name) VALUES (?) RETURNING id', + ['a customer']); + final id = rows[0]['id']; + + var done = false; + inserts() async { + while (!done) { + await db.execute( + 'INSERT INTO assets(make, customer_id) VALUES (?, ?)', + ['test', id]); + await Future.delayed( + Duration(milliseconds: Random().nextInt(baseTime * 2))); + } + } + + const numberOfQueries = 10; + + inserts(); + + final factory = db.isolateConnectionFactory(); + + var l = await inIsolateWatch(factory, numberOfQueries, throttleDuration); + + var results = l[0] as List; + var times = l[1] as List; + done = true; + + var lastCount = 0; + for (var r in results) { + final count = r.first['count']; + // This is not strictly incrementing, since we can't guarantee the + // exact order between reads and writes. + // We can guarantee that there will always be a read after the last write, + // but the previous read may have been after the same write in some cases. + expect(count, greaterThanOrEqualTo(lastCount)); + lastCount = count; + } + + // The number of read queries must not be greater than the number of writes overall. + expect(numberOfQueries, lessThanOrEqualTo(results.last.first['count'])); + + DateTime? lastTime; + for (var r in times) { + if (lastTime != null) { + var diff = r.difference(lastTime); + expect(diff, greaterThanOrEqualTo(throttleDuration)); + } + lastTime = r; + } + }); + }); +} + +Future> inIsolateWatch(IsolateConnectionFactory factory, + int numberOfQueries, Duration throttleDuration) async { + return await Isolate.run(() async { + final db = factory.open(); + + final stream = db.watch( + 'SELECT count() AS count FROM assets INNER JOIN customers ON customers.id = assets.customer_id', + throttle: throttleDuration); + List times = []; + final results = await stream.take(numberOfQueries).map((e) { + times.add(DateTime.now()); + return e; + }).toList(); + + db.close(); + return [results, times]; + }); +} diff --git a/test/util.dart b/test/util.dart deleted file mode 100644 index 5bcbb50..0000000 --- a/test/util.dart +++ /dev/null @@ -1,116 +0,0 @@ -import 'dart:ffi'; -import 'dart:io'; -import 'dart:isolate'; - -import 'package:glob/glob.dart'; -import 'package:glob/list_local_fs.dart'; -import 'package:sqlite3/open.dart' as sqlite_open; -import 'package:sqlite3/sqlite3.dart' as sqlite; -import 'package:sqlite_async/sqlite_async.dart'; -import 'package:test_api/src/backend/invoker.dart'; - -const defaultSqlitePath = 'libsqlite3.so.0'; -// const defaultSqlitePath = './sqlite-autoconf-3410100/.libs/libsqlite3.so.0'; - -class TestSqliteOpenFactory extends DefaultSqliteOpenFactory { - String sqlitePath; - List initStatements; - - TestSqliteOpenFactory( - {required super.path, - super.sqliteOptions, - this.sqlitePath = defaultSqlitePath, - this.initStatements = const []}); - - @override - sqlite.Database open(SqliteOpenOptions options) { - sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { - return DynamicLibrary.open(sqlitePath); - }); - final db = super.open(options); - - db.createFunction( - functionName: 'test_sleep', - argumentCount: const sqlite.AllowedArgumentCount(1), - function: (args) { - final millis = args[0] as int; - sleep(Duration(milliseconds: millis)); - return millis; - }, - ); - - db.createFunction( - functionName: 'test_connection_name', - argumentCount: const sqlite.AllowedArgumentCount(0), - function: (args) { - return Isolate.current.debugName; - }, - ); - - db.createFunction( - functionName: 'test_connection_number', - argumentCount: const sqlite.AllowedArgumentCount(0), - function: (args) { - // write: 0, read: 1 - 5 - final name = Isolate.current.debugName ?? '-0'; - var nr = name.split('-').last; - return int.tryParse(nr) ?? 0; - }, - ); - - for (var s in initStatements) { - db.execute(s); - } - - return db; - } -} - -SqliteOpenFactory testFactory( - {String? path, List initStatements = const []}) { - return TestSqliteOpenFactory( - path: path ?? dbPath(), initStatements: initStatements); -} - -Future setupDatabase({String? path}) async { - final db = SqliteDatabase.withFactory(testFactory(path: path)); - await db.initialize(); - return db; -} - -Future cleanDb({required String path}) async { - try { - await File(path).delete(); - } on PathNotFoundException { - // Not an issue - } - try { - await File("$path-shm").delete(); - } on PathNotFoundException { - // Not an issue - } - try { - await File("$path-wal").delete(); - } on PathNotFoundException { - // Not an issue - } -} - -List findSqliteLibraries() { - var glob = Glob('sqlite-*/.libs/libsqlite3.so'); - List sqlites = [ - 'libsqlite3.so.0', - for (var sqlite in glob.listSync()) sqlite.path - ]; - return sqlites; -} - -String dbPath() { - final test = Invoker.current!.liveTest; - var testName = test.test.name; - var testShortName = - testName.replaceAll(RegExp(r'[\s\./]'), '_').toLowerCase(); - var dbName = "test-db/$testShortName.db"; - Directory("test-db").createSync(recursive: false); - return dbName; -} diff --git a/test/utils/abstract_test_utils.dart b/test/utils/abstract_test_utils.dart new file mode 100644 index 0000000..f1ec6ea --- /dev/null +++ b/test/utils/abstract_test_utils.dart @@ -0,0 +1,48 @@ +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:test_api/src/backend/invoker.dart'; + +class TestDefaultSqliteOpenFactory extends DefaultSqliteOpenFactory { + final String sqlitePath; + + TestDefaultSqliteOpenFactory( + {required super.path, super.sqliteOptions, this.sqlitePath = ''}); +} + +abstract class AbstractTestUtils { + String dbPath() { + final test = Invoker.current!.liveTest; + var testName = test.test.name; + var testShortName = + testName.replaceAll(RegExp(r'[\s\./]'), '_').toLowerCase(); + var dbName = "test-db/$testShortName.db"; + return dbName; + } + + /// Generates a test open factory + Future testFactory( + {String? path, + String sqlitePath = '', + List initStatements = const [], + SqliteOptions options = const SqliteOptions.defaults()}) async { + return TestDefaultSqliteOpenFactory( + path: path ?? dbPath(), + sqliteOptions: options, + ); + } + + /// Creates a SqliteDatabaseConnection + Future setupDatabase( + {String? path, + List initStatements = const [], + int maxReaders = SqliteDatabase.defaultMaxReaders}) async { + final db = SqliteDatabase.withFactory(await testFactory(path: path), + maxReaders: maxReaders); + await db.initialize(); + return db; + } + + /// Deletes any DB data + Future cleanDb({required String path}); + + List findSqliteLibraries(); +} diff --git a/test/utils/native_test_utils.dart b/test/utils/native_test_utils.dart new file mode 100644 index 0000000..2d7a924 --- /dev/null +++ b/test/utils/native_test_utils.dart @@ -0,0 +1,100 @@ +import 'dart:async'; +import 'dart:ffi'; +import 'dart:io'; +import 'dart:isolate'; + +import 'package:glob/glob.dart'; +import 'package:glob/list_local_fs.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite3/open.dart' as sqlite_open; + +import 'abstract_test_utils.dart'; + +const defaultSqlitePath = 'libsqlite3.so.0'; + +class TestSqliteOpenFactory extends TestDefaultSqliteOpenFactory { + TestSqliteOpenFactory( + {required super.path, + super.sqliteOptions, + super.sqlitePath = defaultSqlitePath, + initStatements}); + + @override + FutureOr open(SqliteOpenOptions options) async { + sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { + return DynamicLibrary.open(sqlitePath); + }); + final db = await super.open(options); + + db.createFunction( + functionName: 'test_sleep', + argumentCount: const AllowedArgumentCount(1), + function: (args) { + final millis = args[0] as int; + sleep(Duration(milliseconds: millis)); + return millis; + }, + ); + + db.createFunction( + functionName: 'test_connection_name', + argumentCount: const AllowedArgumentCount(0), + function: (args) { + return Isolate.current.debugName; + }, + ); + + return db; + } +} + +class TestUtils extends AbstractTestUtils { + @override + String dbPath() { + Directory("test-db").createSync(recursive: false); + return super.dbPath(); + } + + @override + Future cleanDb({required String path}) async { + try { + await File(path).delete(); + } on PathNotFoundException { + // Not an issue + } + try { + await File("$path-shm").delete(); + } on PathNotFoundException { + // Not an issue + } + try { + await File("$path-wal").delete(); + } on PathNotFoundException { + // Not an issue + } + } + + @override + List findSqliteLibraries() { + var glob = Glob('sqlite-*/.libs/libsqlite3.so'); + List sqlites = [ + 'libsqlite3.so.0', + for (var sqlite in glob.listSync()) sqlite.path + ]; + return sqlites; + } + + @override + Future testFactory( + {String? path, + String sqlitePath = defaultSqlitePath, + List initStatements = const [], + SqliteOptions options = const SqliteOptions.defaults()}) async { + return TestSqliteOpenFactory( + path: path ?? dbPath(), + sqlitePath: sqlitePath, + sqliteOptions: options, + initStatements: initStatements); + } +} diff --git a/test/utils/stub_test_utils.dart b/test/utils/stub_test_utils.dart new file mode 100644 index 0000000..5e3a953 --- /dev/null +++ b/test/utils/stub_test_utils.dart @@ -0,0 +1,13 @@ +import 'abstract_test_utils.dart'; + +class TestUtils extends AbstractTestUtils { + @override + Future cleanDb({required String path}) { + throw UnimplementedError(); + } + + @override + List findSqliteLibraries() { + throw UnimplementedError(); + } +} diff --git a/test/utils/test_utils_impl.dart b/test/utils/test_utils_impl.dart new file mode 100644 index 0000000..523c2fd --- /dev/null +++ b/test/utils/test_utils_impl.dart @@ -0,0 +1,3 @@ +export 'stub_test_utils.dart' + // ignore: uri_does_not_exist + if (dart.library.io) 'native_test_utils.dart'; diff --git a/test/watch_test.dart b/test/watch_test.dart deleted file mode 100644 index f102001..0000000 --- a/test/watch_test.dart +++ /dev/null @@ -1,334 +0,0 @@ -import 'dart:async'; -import 'dart:isolate'; -import 'dart:math'; - -import 'package:sqlite3/sqlite3.dart'; -import 'package:sqlite_async/sqlite_async.dart'; -import 'package:sqlite_async/src/database_utils.dart'; -import 'package:test/test.dart'; - -import 'util.dart'; - -void main() { - createTables(SqliteDatabase db) async { - await db.writeTransaction((tx) async { - await tx.execute( - 'CREATE TABLE assets(id INTEGER PRIMARY KEY AUTOINCREMENT, make TEXT, customer_id INTEGER)'); - await tx.execute('CREATE INDEX assets_customer ON assets(customer_id)'); - await tx.execute( - 'CREATE TABLE customers(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)'); - await tx.execute( - 'CREATE TABLE other_customers(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)'); - await tx.execute('CREATE VIEW assets_alias AS SELECT * FROM assets'); - }); - } - - group('Query Watch Tests', () { - late String path; - - setUp(() async { - path = dbPath(); - await cleanDb(path: path); - }); - - for (var sqlite in findSqliteLibraries()) { - test('getSourceTables - $sqlite', () async { - final db = SqliteDatabase.withFactory( - TestSqliteOpenFactory(path: path, sqlitePath: sqlite)); - await db.initialize(); - await createTables(db); - - var versionRow = await db.get('SELECT sqlite_version() as version'); - print('Testing SQLite ${versionRow['version']} - $sqlite'); - - final tables = await getSourceTables(db, - 'SELECT * FROM assets INNER JOIN customers ON assets.customer_id = customers.id'); - expect(tables, equals({'assets', 'customers'})); - - final tables2 = await getSourceTables(db, - 'SELECT count() FROM assets INNER JOIN "other_customers" AS oc ON assets.customer_id = oc.id AND assets.make = oc.name'); - expect(tables2, equals({'assets', 'other_customers'})); - - final tables3 = await getSourceTables(db, 'SELECT count() FROM assets'); - expect(tables3, equals({'assets'})); - - final tables4 = - await getSourceTables(db, 'SELECT count() FROM assets_alias'); - expect(tables4, equals({'assets'})); - - final tables5 = - await getSourceTables(db, 'SELECT sqlite_version() as version'); - expect(tables5, equals({})); - }); - } - - test('watch', () async { - final db = await setupDatabase(path: path); - await createTables(db); - - const baseTime = 20; - - const throttleDuration = Duration(milliseconds: baseTime); - - final stream = db.watch( - 'SELECT count() AS count FROM assets INNER JOIN customers ON customers.id = assets.customer_id', - throttle: throttleDuration); - - final rows = await db.execute( - 'INSERT INTO customers(name) VALUES (?) RETURNING id', - ['a customer']); - final id = rows[0]['id']; - - var done = false; - inserts() async { - while (!done) { - await db.execute( - 'INSERT INTO assets(make, customer_id) VALUES (?, ?)', - ['test', id]); - await Future.delayed( - Duration(milliseconds: Random().nextInt(baseTime * 2))); - } - } - - const numberOfQueries = 10; - - inserts(); - try { - List times = []; - final results = await stream.take(numberOfQueries).map((e) { - times.add(DateTime.now()); - return e; - }).toList(); - - var lastCount = 0; - for (var r in results) { - final count = r.first['count']; - // This is not strictly incrementing, since we can't guarantee the - // exact order between reads and writes. - // We can guarantee that there will always be a read after the last write, - // but the previous read may have been after the same write in some cases. - expect(count, greaterThanOrEqualTo(lastCount)); - lastCount = count; - } - - // The number of read queries must not be greater than the number of writes overall. - expect(numberOfQueries, lessThanOrEqualTo(results.last.first['count'])); - - DateTime? lastTime; - for (var r in times) { - if (lastTime != null) { - var diff = r.difference(lastTime); - expect(diff, greaterThanOrEqualTo(throttleDuration)); - } - lastTime = r; - } - } finally { - done = true; - } - }); - - test('onChange', () async { - final db = await setupDatabase(path: path); - await createTables(db); - - const baseTime = 20; - - const throttleDuration = Duration(milliseconds: baseTime); - - var done = false; - inserts() async { - while (!done) { - await db.execute('INSERT INTO assets(make) VALUES (?)', ['test']); - await Future.delayed( - Duration(milliseconds: Random().nextInt(baseTime))); - } - } - - inserts(); - - final stream = db.onChange({'assets', 'customers'}, - throttle: throttleDuration).asyncMap((event) async { - // This is where queries would typically be executed - return event; - }); - - var events = await stream.take(3).toList(); - done = true; - - expect( - events, - equals([ - UpdateNotification.empty(), - UpdateNotification.single('assets'), - UpdateNotification.single('assets') - ])); - }); - - test('single onChange', () async { - final db = await setupDatabase(path: path); - await createTables(db); - - const baseTime = 20; - - const throttleDuration = Duration(milliseconds: baseTime); - - final stream = db.onChange({'assets', 'customers'}, - throttle: throttleDuration, - triggerImmediately: false).asyncMap((event) async { - // This is where queries would typically be executed - return event; - }); - - var eventsFuture = stream.take(1).toList(); - await db.execute('INSERT INTO assets(make) VALUES (?)', ['test']); - var events = await eventsFuture; - - expect(events, equals([UpdateNotification.single('assets')])); - }); - - test('watch in isolate', () async { - final db = await setupDatabase(path: path); - await createTables(db); - - const baseTime = 20; - - const throttleDuration = Duration(milliseconds: baseTime); - - final rows = await db.execute( - 'INSERT INTO customers(name) VALUES (?) RETURNING id', - ['a customer']); - final id = rows[0]['id']; - - var done = false; - inserts() async { - while (!done) { - await db.execute( - 'INSERT INTO assets(make, customer_id) VALUES (?, ?)', - ['test', id]); - await Future.delayed( - Duration(milliseconds: Random().nextInt(baseTime * 2))); - } - } - - const numberOfQueries = 10; - - inserts(); - - final factory = db.isolateConnectionFactory(); - - var l = await inIsolateWatch(factory, numberOfQueries, throttleDuration); - - var results = l[0] as List; - var times = l[1] as List; - done = true; - - var lastCount = 0; - for (var r in results) { - final count = r.first['count']; - // This is not strictly incrementing, since we can't guarantee the - // exact order between reads and writes. - // We can guarantee that there will always be a read after the last write, - // but the previous read may have been after the same write in some cases. - expect(count, greaterThanOrEqualTo(lastCount)); - lastCount = count; - } - - // The number of read queries must not be greater than the number of writes overall. - expect(numberOfQueries, lessThanOrEqualTo(results.last.first['count'])); - - DateTime? lastTime; - for (var r in times) { - if (lastTime != null) { - var diff = r.difference(lastTime); - expect(diff, greaterThanOrEqualTo(throttleDuration)); - } - lastTime = r; - } - }); - - test('watch with parameters', () async { - final db = await setupDatabase(path: path); - await createTables(db); - - const baseTime = 20; - - const throttleDuration = Duration(milliseconds: baseTime); - - final rows = await db.execute( - 'INSERT INTO customers(name) VALUES (?) RETURNING id', - ['a customer']); - final id = rows[0]['id']; - - final stream = db.watch( - 'SELECT count() AS count FROM assets WHERE customer_id = ?', - parameters: [id], - throttle: throttleDuration); - - var done = false; - inserts() async { - while (!done) { - await db.execute( - 'INSERT INTO assets(make, customer_id) VALUES (?, ?)', - ['test', id]); - await Future.delayed( - Duration(milliseconds: Random().nextInt(baseTime * 2))); - } - } - - const numberOfQueries = 10; - - inserts(); - try { - List times = []; - final results = await stream.take(numberOfQueries).map((e) { - times.add(DateTime.now()); - return e; - }).toList(); - - var lastCount = 0; - for (var r in results) { - final count = r.first['count']; - // This is not strictly incrementing, since we can't guarantee the - // exact order between reads and writes. - // We can guarantee that there will always be a read after the last write, - // but the previous read may have been after the same write in some cases. - expect(count, greaterThanOrEqualTo(lastCount)); - lastCount = count; - } - - // The number of read queries must not be greater than the number of writes overall. - expect(numberOfQueries, lessThanOrEqualTo(results.last.first['count'])); - - DateTime? lastTime; - for (var r in times) { - if (lastTime != null) { - var diff = r.difference(lastTime); - expect(diff, greaterThanOrEqualTo(throttleDuration)); - } - lastTime = r; - } - } finally { - done = true; - } - }); - }); -} - -Future> inIsolateWatch(IsolateConnectionFactory factory, - int numberOfQueries, Duration throttleDuration) async { - return await Isolate.run(() async { - final db = factory.open(); - - final stream = db.watch( - 'SELECT count() AS count FROM assets INNER JOIN customers ON customers.id = assets.customer_id', - throttle: throttleDuration); - List times = []; - final results = await stream.take(numberOfQueries).map((e) { - times.add(DateTime.now()); - return e; - }).toList(); - - db.close(); - return [results, times]; - }); -} From b5fc3c9dd1e4cb76a61ddb12c5f9abcdd16f39e3 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 28 May 2024 13:45:41 +0200 Subject: [PATCH 21/21] qoutes --- pubspec.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pubspec.yaml b/pubspec.yaml index de5a3a3..aacb854 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -3,7 +3,7 @@ description: High-performance asynchronous interface for SQLite on Dart and Flut version: 0.7.0-alpha.3 repository: https://github.com/powersync-ja/sqlite_async.dart environment: - sdk: '>=3.2.0 <4.0.0' + sdk: ">=3.2.0 <4.0.0" dependencies: drift: ^2.15.0 @@ -13,7 +13,7 @@ dependencies: # url: https://github.com/powersync-ja/drift.git # ref: test # branch name # path: drift - sqlite3: '^2.3.0' + sqlite3: "^2.3.0" js: ^0.6.3 async: ^2.10.0 collection: ^1.17.0