diff --git a/.github/workflows/demos.yml b/.github/workflows/demos.yml index 829e0908..78023fc3 100644 --- a/.github/workflows/demos.yml +++ b/.github/workflows/demos.yml @@ -1,13 +1,13 @@ name: Demos checks concurrency: - group: demos-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + group: demos-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true on: push: branches: - - "**" + - '**' jobs: build: @@ -23,7 +23,7 @@ jobs: - name: Install Melos run: flutter pub global activate melos - name: Install dependencies - run: melos bootstrap + run: melos prepare - name: Check formatting run: melos format:check:demos - name: Lint @@ -43,6 +43,6 @@ jobs: - name: Install melos run: flutter pub global activate melos - name: Install dependencies - run: melos bootstrap + run: melos prepare - name: Run tests run: melos test diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index b7cd7fc7..d1d01717 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -1,13 +1,13 @@ name: Packages check concurrency: - group: packages-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + group: packages-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true on: push: branches: - - "**" + - '**' jobs: build: @@ -23,7 +23,7 @@ jobs: - name: Install Melos run: flutter pub global activate melos - name: Install dependencies - run: melos bootstrap + run: melos prepare - name: Check formatting run: melos format:check:packages - name: Lint @@ -46,7 +46,8 @@ jobs: channel: 'stable' - name: Install melos run: flutter pub global activate melos - - name: Install dependencies - run: melos bootstrap + - name: Install dependencies and prepare project + run: melos prepare + - name: Run tests run: melos test diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bfc8fa67..70d6cead 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,7 +7,7 @@ jobs: permissions: contents: write id-token: write # Required for authentication using OIDC - runs-on: [ ubuntu-latest ] + runs-on: [ubuntu-latest] steps: - uses: actions/checkout@v3 - uses: subosito/flutter-action@v2 diff --git a/.github/workflows/scripts/run-pana.sh b/.github/workflows/scripts/run-pana.sh index c117cc98..85905771 100755 --- a/.github/workflows/scripts/run-pana.sh +++ b/.github/workflows/scripts/run-pana.sh @@ -17,7 +17,7 @@ for PACKAGE in "$PACKAGES_DIR"/*; do cd "$PACKAGE" || exit # Run the pana command - flutter pub global run pana --no-warning --exit-code-threshold 0 + flutter pub global run pana --no-warning --exit-code-threshold 10 # Return to the root directory cd "$ROOT_DIR" || exit diff --git a/.gitignore b/.gitignore index f3d06acc..1f9cd70b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,15 @@ .DS_Store pubspec_overrides.yaml .idea +.vscode *.iml .flutter-plugins-dependencies .flutter-plugins +build + +# Shared assets +assets + +# Web assets +powersync_db.worker.js +sqlite3.wasm diff --git a/README.md b/README.md index 5a961254..fc2254d7 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ This monorepo uses [melos](https://melos.invertase.dev/) to handle command and p For detailed usage, check out the inner [powersync](https://github.com/powersync-ja/powersync.dart/tree/master/packages/powersync) and [attachments helper](https://github.com/powersync-ja/powersync.dart/tree/master/packages/powersync_attachments_helper) packages. +To configure the monorepo for development run `melos prepare` after cloning + #### Blog posts - [Flutter Tutorial: building an offline-first chat app with Supabase and PowerSync](https://www.powersync.com/blog/flutter-tutorial-building-an-offline-first-chat-app-with-supabase-and-powersync) diff --git a/demos/supabase-anonymous-auth/lib/powersync.dart b/demos/supabase-anonymous-auth/lib/powersync.dart index 0e9e00d3..8bf284cc 100644 --- a/demos/supabase-anonymous-auth/lib/powersync.dart +++ b/demos/supabase-anonymous-auth/lib/powersync.dart @@ -1,6 +1,7 @@ // This file performs setup of the PowerSync database import 'dart:io'; +import 'package:flutter/foundation.dart'; import 'package:logging/logging.dart'; import 'package:path/path.dart'; import 'package:path_provider/path_provider.dart'; @@ -136,7 +137,7 @@ String? getUserId() { } Future getDatabasePath() async { - final dir = await getApplicationSupportDirectory(); + final dir = kIsWeb ? Directory('/') : await getApplicationSupportDirectory(); return join(dir.path, 'powersync-demo.db'); } diff --git a/demos/supabase-anonymous-auth/pubspec.lock b/demos/supabase-anonymous-auth/pubspec.lock index 28a50339..beff5f40 100644 --- a/demos/supabase-anonymous-auth/pubspec.lock +++ b/demos/supabase-anonymous-auth/pubspec.lock @@ -49,6 +49,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.18.0" + convert: + dependency: transitive + description: + name: convert + sha256: "0f08b14755d163f6e2134cb58222dd25ea2a2ee8a195e53983d57c075324d592" + url: "https://pub.dev" + source: hosted + version: "3.1.1" crypto: dependency: transitive description: @@ -57,6 +65,14 @@ packages: url: "https://pub.dev" source: hosted version: "3.0.3" + drift: + dependency: transitive + description: + name: drift + sha256: b50a8342c6ddf05be53bda1d246404cbad101b64dc73e8d6d1ac1090d119b4e2 + url: "https://pub.dev" + source: hosted + version: "2.15.0" fake_async: dependency: transitive description: @@ -65,6 +81,22 @@ packages: url: "https://pub.dev" source: hosted version: "1.3.1" + fetch_api: + dependency: transitive + description: + name: fetch_api + sha256: "74a1e426d41ed9c89353703b2d80400c5d0ecfa144b2d8a7bd8882fbc9e48787" + url: "https://pub.dev" + source: hosted + version: "1.0.3" + fetch_client: + dependency: transitive + description: + name: fetch_client + sha256: "83c07b07a63526a43630572c72715707ca113a8aa3459efbc7b2d366b79402af" + url: "https://pub.dev" + source: hosted + version: "1.0.2" ffi: dependency: transitive description: @@ -81,6 +113,14 @@ packages: url: "https://pub.dev" source: hosted version: "7.0.0" + fixnum: + dependency: transitive + description: + name: fixnum + sha256: "25517a4deb0c03aa0f32fd12db525856438902d9c16536311e76cdc57b31d7d1" + url: "https://pub.dev" + source: hosted + version: "1.1.0" flutter: dependency: "direct main" description: flutter @@ -116,10 +156,10 @@ packages: dependency: transitive description: name: gotrue - sha256: f6040d446881658b4b2115a17c649e61c3ff3f82ff47d6c785aaa45f9608ae0d + sha256: f40610bacf1074723354b0856a4f586508ffb075b799f72466f34e843133deb9 url: "https://pub.dev" source: hosted - version: "2.2.0" + version: "2.5.0" gtk: dependency: transitive description: @@ -128,30 +168,14 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.0" - hive: - dependency: transitive - description: - name: hive - sha256: "8dcf6db979d7933da8217edcec84e9df1bdb4e4edc7fc77dbd5aa74356d6d941" - url: "https://pub.dev" - source: hosted - version: "2.2.3" - hive_flutter: - dependency: transitive - description: - name: hive_flutter - sha256: dca1da446b1d808a51689fb5d0c6c9510c0a2ba01e22805d492c73b68e33eecc - url: "https://pub.dev" - source: hosted - version: "1.1.0" http: dependency: transitive description: name: http - sha256: d4872660c46d929f6b8a9ef4e7a7eff7e49bbf0c4ec3f385ee32df5119175139 + sha256: a2bbf9d017fcced29139daa8ed2bba4ece450ab222871df93ca9eec6f80c34ba url: "https://pub.dev" source: hosted - version: "1.1.2" + version: "1.2.0" http_parser: dependency: transitive description: @@ -220,10 +244,18 @@ packages: dependency: transitive description: name: mime - sha256: e4ff8e8564c03f255408decd16e7899da1733852a9110a58fe6d1b817684a63e + sha256: "2e123074287cc9fd6c09de8336dae606d1ddb88d9ac47358826db698c176a1f2" url: "https://pub.dev" source: hosted - version: "1.0.4" + version: "1.0.5" + mutex: + dependency: transitive + description: + name: mutex + sha256: "8827da25de792088eb33e572115a5eb0d61d61a3c01acbc8bcbe76ed78f1a1f2" + url: "https://pub.dev" + source: hosted + version: "3.1.0" path: dependency: "direct main" description: @@ -236,10 +268,10 @@ packages: dependency: "direct main" description: name: path_provider - sha256: a1aa8aaa2542a6bc57e381f132af822420216c80d4781f7aa085ca3229208aaa + sha256: b27217933eeeba8ff24845c34003b003b2b22151de3c908d0e679e8fe1aa078b url: "https://pub.dev" source: hosted - version: "2.1.1" + version: "2.1.2" path_provider_android: dependency: transitive description: @@ -252,10 +284,10 @@ packages: dependency: transitive description: name: path_provider_foundation - sha256: "19314d595120f82aca0ba62787d58dde2cc6b5df7d2f0daf72489e38d1b57f2d" + sha256: "5a7999be66e000916500be4f15a3633ebceb8302719b47b9cc49ce924125350f" url: "https://pub.dev" source: hosted - version: "2.3.1" + version: "2.3.2" path_provider_linux: dependency: transitive description: @@ -268,10 +300,10 @@ packages: dependency: transitive description: name: path_provider_platform_interface - sha256: "94b1e0dd80970c1ce43d5d4e050a9918fce4f4a775e6142424c30a29a363265c" + sha256: "88f5779f72ba699763fa3a3b06aa4bf6de76c8e5de842cf6f29e2e06476c2334" url: "https://pub.dev" source: hosted - version: "2.1.1" + version: "2.1.2" path_provider_windows: dependency: transitive description: @@ -310,7 +342,7 @@ packages: path: "../../packages/powersync" relative: true source: path - version: "1.2.0" + version: "1.3.0-alpha.1" realtime_client: dependency: transitive description: @@ -355,10 +387,10 @@ packages: dependency: transitive description: name: shared_preferences_foundation - sha256: "7bf53a9f2d007329ee6f3df7268fd498f8373602f943c975598bbb34649b62a7" + sha256: "7708d83064f38060c7b39db12aefe449cb8cdc031d6062280087bc4cdb988f5c" url: "https://pub.dev" source: hosted - version: "2.3.4" + version: "2.3.5" shared_preferences_linux: dependency: transitive description: @@ -371,10 +403,10 @@ packages: dependency: transitive description: name: shared_preferences_platform_interface - sha256: d4ec5fc9ebb2f2e056c617112aa75dcf92fc2e4faaf2ae999caa297473f75d8a + sha256: "22e2ecac9419b4246d7c22bfbbda589e3acf5c0351137d87dd2939d984d37c3b" url: "https://pub.dev" source: hosted - version: "2.3.1" + version: "2.3.2" shared_preferences_web: dependency: transitive description: @@ -416,26 +448,26 @@ packages: dependency: transitive description: name: sqlite3 - sha256: c4a4c5a4b2a32e2d0f6837b33d7c91a67903891a5b7dbe706cf4b1f6b0c798c5 + sha256: "072128763f1547e3e9b4735ce846bfd226d68019ccda54db4cd427b12dfdedc9" url: "https://pub.dev" source: hosted - version: "2.3.0" + version: "2.4.0" sqlite3_flutter_libs: dependency: transitive description: name: sqlite3_flutter_libs - sha256: "3e3583b77cf888a68eae2e49ee4f025f66b86623ef0d83c297c8d903daa14871" + sha256: d6c31c8511c441d1f12f20b607343df1afe4eddf24a1cf85021677c8eea26060 url: "https://pub.dev" source: hosted - version: "0.5.18" + version: "0.5.20" sqlite_async: dependency: "direct main" description: name: sqlite_async - sha256: "609a8405b8b608ac396dd7f478ed42e230c496eb38fe53dd97e9c592e1cd5cda" + sha256: "91f454cddc85617bea2c7c1544ff386887d0d2cf0ecdb3599015c05cc141ff4d" url: "https://pub.dev" source: hosted - version: "0.6.0" + version: "0.7.0-alpha.1" stack_trace: dependency: transitive description: @@ -448,10 +480,10 @@ packages: dependency: transitive description: name: storage_client - sha256: b49ff2e1e6738c0ef445546d6ec77040829947f0c7ef0b115acb125656127c83 + sha256: bf5589d5de61a2451edb1b8960a0e673d4bb5c42ecc4dddf7c051a93789ced34 url: "https://pub.dev" source: hosted - version: "2.0.0" + version: "2.0.1" stream_channel: dependency: transitive description: @@ -472,18 +504,18 @@ packages: dependency: transitive description: name: supabase - sha256: "13d24f1c7c489219f37bbb53842eafca008f2855fdea5cf731de835aebfc50bc" + sha256: "4bce9c49f264f4cd44b4ffc895647af2dca0c40125c169045be9f708fd2a2a40" url: "https://pub.dev" source: hosted - version: "2.0.3" + version: "2.0.7" supabase_flutter: dependency: "direct main" description: name: supabase_flutter - sha256: "57ca6c970042e6cef29056b0dc71bb17fc9d6848188df5ddc69e819cf28c7923" + sha256: "5ef71289c380b6429216e941c69971c75eaab50d67fd7b540f6c1f6ebfc00ed7" url: "https://pub.dev" source: hosted - version: "2.1.0" + version: "2.3.3" term_glyph: dependency: transitive description: @@ -512,26 +544,26 @@ packages: dependency: transitive description: name: url_launcher - sha256: e9aa5ea75c84cf46b3db4eea212523591211c3cf2e13099ee4ec147f54201c86 + sha256: c512655380d241a337521703af62d2c122bf7b77a46ff7dd750092aa9433499c url: "https://pub.dev" source: hosted - version: "6.2.2" + version: "6.2.4" url_launcher_android: dependency: transitive description: name: url_launcher_android - sha256: c0766a55ab42cefaa728cabc951e82919ab41a3a4fee0aaa96176ca82da8cc51 + sha256: d4ed0711849dd8e33eb2dd69c25db0d0d3fdc37e0a62e629fe32f57a22db2745 url: "https://pub.dev" source: hosted - version: "6.2.1" + version: "6.3.0" url_launcher_ios: dependency: transitive description: name: url_launcher_ios - sha256: "46b81e3109cbb2d6b81702ad3077540789a3e74e22795eb9f0b7d494dbaa72ea" + sha256: "75bb6fe3f60070407704282a2d295630cab232991eb52542b18347a8a941df03" url: "https://pub.dev" source: hosted - version: "6.2.2" + version: "6.2.4" url_launcher_linux: dependency: transitive description: @@ -552,10 +584,10 @@ packages: dependency: transitive description: name: url_launcher_platform_interface - sha256: f099b552bd331eacd69affed7ff2f23bfa6b0cb825b629edf3d844375a7501ad + sha256: a932c3a8082e118f80a475ce692fde89dc20fddb24c57360b96bc56f7035de1f url: "https://pub.dev" source: hosted - version: "2.2.1" + version: "2.3.1" url_launcher_web: dependency: transitive description: @@ -576,10 +608,10 @@ packages: dependency: transitive description: name: uuid - sha256: "22c94e5ad1e75f9934b766b53c742572ee2677c56bc871d850a57dad0f82127f" + sha256: cd210a09f7c18cbe5a02511718e0334de6559871052c90a90c0cca46a4aa81c8 url: "https://pub.dev" source: hosted - version: "4.2.2" + version: "4.3.3" vector_math: dependency: transitive description: @@ -608,10 +640,10 @@ packages: dependency: transitive description: name: win32 - sha256: b0f37db61ba2f2e9b7a78a1caece0052564d1bc70668156cf3a29d676fe4e574 + sha256: "464f5674532865248444b4c3daca12bd9bf2d7c47f759ce2617986e7229494a8" url: "https://pub.dev" source: hosted - version: "5.1.1" + version: "5.2.0" xdg_directories: dependency: transitive description: @@ -629,5 +661,5 @@ packages: source: hosted version: "2.0.0" sdks: - dart: ">=3.2.0 <4.0.0" + dart: ">=3.2.3 <4.0.0" flutter: ">=3.16.0" diff --git a/demos/supabase-anonymous-auth/pubspec.yaml b/demos/supabase-anonymous-auth/pubspec.yaml index 21117368..ae250d3b 100644 --- a/demos/supabase-anonymous-auth/pubspec.yaml +++ b/demos/supabase-anonymous-auth/pubspec.yaml @@ -1,22 +1,22 @@ name: supabase_anonymous_auth description: PowerSync Supabase Anonymous Auth Demo -publish_to: "none" +publish_to: 'none' version: 1.0.1 environment: - sdk: ^3.2.0 + sdk: ^3.2.3 dependencies: flutter: sdk: flutter - powersync: ^1.2.0 + powersync: ^1.3.0-alpha.1 path_provider: ^2.1.1 supabase_flutter: ^2.0.2 path: ^1.8.3 logging: ^1.2.0 - sqlite_async: ^0.6.0 + sqlite_async: ^0.7.0-alpha.1 dev_dependencies: flutter_test: diff --git a/demos/supabase-edge-function-auth/lib/powersync.dart b/demos/supabase-edge-function-auth/lib/powersync.dart index 4d8e83ce..39f5ca83 100644 --- a/demos/supabase-edge-function-auth/lib/powersync.dart +++ b/demos/supabase-edge-function-auth/lib/powersync.dart @@ -1,6 +1,7 @@ // This file performs setup of the PowerSync database import 'dart:io'; +import 'package:flutter/foundation.dart'; import 'package:logging/logging.dart'; import 'package:path/path.dart'; import 'package:path_provider/path_provider.dart'; @@ -139,7 +140,7 @@ String? getUserId() { } Future getDatabasePath() async { - final dir = await getApplicationSupportDirectory(); + final dir = kIsWeb ? Directory('/') : await getApplicationSupportDirectory(); return join(dir.path, 'powersync-demo.db'); } diff --git a/demos/supabase-edge-function-auth/pubspec.lock b/demos/supabase-edge-function-auth/pubspec.lock index 28a50339..beff5f40 100644 --- a/demos/supabase-edge-function-auth/pubspec.lock +++ b/demos/supabase-edge-function-auth/pubspec.lock @@ -49,6 +49,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.18.0" + convert: + dependency: transitive + description: + name: convert + sha256: "0f08b14755d163f6e2134cb58222dd25ea2a2ee8a195e53983d57c075324d592" + url: "https://pub.dev" + source: hosted + version: "3.1.1" crypto: dependency: transitive description: @@ -57,6 +65,14 @@ packages: url: "https://pub.dev" source: hosted version: "3.0.3" + drift: + dependency: transitive + description: + name: drift + sha256: b50a8342c6ddf05be53bda1d246404cbad101b64dc73e8d6d1ac1090d119b4e2 + url: "https://pub.dev" + source: hosted + version: "2.15.0" fake_async: dependency: transitive description: @@ -65,6 +81,22 @@ packages: url: "https://pub.dev" source: hosted version: "1.3.1" + fetch_api: + dependency: transitive + description: + name: fetch_api + sha256: "74a1e426d41ed9c89353703b2d80400c5d0ecfa144b2d8a7bd8882fbc9e48787" + url: "https://pub.dev" + source: hosted + version: "1.0.3" + fetch_client: + dependency: transitive + description: + name: fetch_client + sha256: "83c07b07a63526a43630572c72715707ca113a8aa3459efbc7b2d366b79402af" + url: "https://pub.dev" + source: hosted + version: "1.0.2" ffi: dependency: transitive description: @@ -81,6 +113,14 @@ packages: url: "https://pub.dev" source: hosted version: "7.0.0" + fixnum: + dependency: transitive + description: + name: fixnum + sha256: "25517a4deb0c03aa0f32fd12db525856438902d9c16536311e76cdc57b31d7d1" + url: "https://pub.dev" + source: hosted + version: "1.1.0" flutter: dependency: "direct main" description: flutter @@ -116,10 +156,10 @@ packages: dependency: transitive description: name: gotrue - sha256: f6040d446881658b4b2115a17c649e61c3ff3f82ff47d6c785aaa45f9608ae0d + sha256: f40610bacf1074723354b0856a4f586508ffb075b799f72466f34e843133deb9 url: "https://pub.dev" source: hosted - version: "2.2.0" + version: "2.5.0" gtk: dependency: transitive description: @@ -128,30 +168,14 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.0" - hive: - dependency: transitive - description: - name: hive - sha256: "8dcf6db979d7933da8217edcec84e9df1bdb4e4edc7fc77dbd5aa74356d6d941" - url: "https://pub.dev" - source: hosted - version: "2.2.3" - hive_flutter: - dependency: transitive - description: - name: hive_flutter - sha256: dca1da446b1d808a51689fb5d0c6c9510c0a2ba01e22805d492c73b68e33eecc - url: "https://pub.dev" - source: hosted - version: "1.1.0" http: dependency: transitive description: name: http - sha256: d4872660c46d929f6b8a9ef4e7a7eff7e49bbf0c4ec3f385ee32df5119175139 + sha256: a2bbf9d017fcced29139daa8ed2bba4ece450ab222871df93ca9eec6f80c34ba url: "https://pub.dev" source: hosted - version: "1.1.2" + version: "1.2.0" http_parser: dependency: transitive description: @@ -220,10 +244,18 @@ packages: dependency: transitive description: name: mime - sha256: e4ff8e8564c03f255408decd16e7899da1733852a9110a58fe6d1b817684a63e + sha256: "2e123074287cc9fd6c09de8336dae606d1ddb88d9ac47358826db698c176a1f2" url: "https://pub.dev" source: hosted - version: "1.0.4" + version: "1.0.5" + mutex: + dependency: transitive + description: + name: mutex + sha256: "8827da25de792088eb33e572115a5eb0d61d61a3c01acbc8bcbe76ed78f1a1f2" + url: "https://pub.dev" + source: hosted + version: "3.1.0" path: dependency: "direct main" description: @@ -236,10 +268,10 @@ packages: dependency: "direct main" description: name: path_provider - sha256: a1aa8aaa2542a6bc57e381f132af822420216c80d4781f7aa085ca3229208aaa + sha256: b27217933eeeba8ff24845c34003b003b2b22151de3c908d0e679e8fe1aa078b url: "https://pub.dev" source: hosted - version: "2.1.1" + version: "2.1.2" path_provider_android: dependency: transitive description: @@ -252,10 +284,10 @@ packages: dependency: transitive description: name: path_provider_foundation - sha256: "19314d595120f82aca0ba62787d58dde2cc6b5df7d2f0daf72489e38d1b57f2d" + sha256: "5a7999be66e000916500be4f15a3633ebceb8302719b47b9cc49ce924125350f" url: "https://pub.dev" source: hosted - version: "2.3.1" + version: "2.3.2" path_provider_linux: dependency: transitive description: @@ -268,10 +300,10 @@ packages: dependency: transitive description: name: path_provider_platform_interface - sha256: "94b1e0dd80970c1ce43d5d4e050a9918fce4f4a775e6142424c30a29a363265c" + sha256: "88f5779f72ba699763fa3a3b06aa4bf6de76c8e5de842cf6f29e2e06476c2334" url: "https://pub.dev" source: hosted - version: "2.1.1" + version: "2.1.2" path_provider_windows: dependency: transitive description: @@ -310,7 +342,7 @@ packages: path: "../../packages/powersync" relative: true source: path - version: "1.2.0" + version: "1.3.0-alpha.1" realtime_client: dependency: transitive description: @@ -355,10 +387,10 @@ packages: dependency: transitive description: name: shared_preferences_foundation - sha256: "7bf53a9f2d007329ee6f3df7268fd498f8373602f943c975598bbb34649b62a7" + sha256: "7708d83064f38060c7b39db12aefe449cb8cdc031d6062280087bc4cdb988f5c" url: "https://pub.dev" source: hosted - version: "2.3.4" + version: "2.3.5" shared_preferences_linux: dependency: transitive description: @@ -371,10 +403,10 @@ packages: dependency: transitive description: name: shared_preferences_platform_interface - sha256: d4ec5fc9ebb2f2e056c617112aa75dcf92fc2e4faaf2ae999caa297473f75d8a + sha256: "22e2ecac9419b4246d7c22bfbbda589e3acf5c0351137d87dd2939d984d37c3b" url: "https://pub.dev" source: hosted - version: "2.3.1" + version: "2.3.2" shared_preferences_web: dependency: transitive description: @@ -416,26 +448,26 @@ packages: dependency: transitive description: name: sqlite3 - sha256: c4a4c5a4b2a32e2d0f6837b33d7c91a67903891a5b7dbe706cf4b1f6b0c798c5 + sha256: "072128763f1547e3e9b4735ce846bfd226d68019ccda54db4cd427b12dfdedc9" url: "https://pub.dev" source: hosted - version: "2.3.0" + version: "2.4.0" sqlite3_flutter_libs: dependency: transitive description: name: sqlite3_flutter_libs - sha256: "3e3583b77cf888a68eae2e49ee4f025f66b86623ef0d83c297c8d903daa14871" + sha256: d6c31c8511c441d1f12f20b607343df1afe4eddf24a1cf85021677c8eea26060 url: "https://pub.dev" source: hosted - version: "0.5.18" + version: "0.5.20" sqlite_async: dependency: "direct main" description: name: sqlite_async - sha256: "609a8405b8b608ac396dd7f478ed42e230c496eb38fe53dd97e9c592e1cd5cda" + sha256: "91f454cddc85617bea2c7c1544ff386887d0d2cf0ecdb3599015c05cc141ff4d" url: "https://pub.dev" source: hosted - version: "0.6.0" + version: "0.7.0-alpha.1" stack_trace: dependency: transitive description: @@ -448,10 +480,10 @@ packages: dependency: transitive description: name: storage_client - sha256: b49ff2e1e6738c0ef445546d6ec77040829947f0c7ef0b115acb125656127c83 + sha256: bf5589d5de61a2451edb1b8960a0e673d4bb5c42ecc4dddf7c051a93789ced34 url: "https://pub.dev" source: hosted - version: "2.0.0" + version: "2.0.1" stream_channel: dependency: transitive description: @@ -472,18 +504,18 @@ packages: dependency: transitive description: name: supabase - sha256: "13d24f1c7c489219f37bbb53842eafca008f2855fdea5cf731de835aebfc50bc" + sha256: "4bce9c49f264f4cd44b4ffc895647af2dca0c40125c169045be9f708fd2a2a40" url: "https://pub.dev" source: hosted - version: "2.0.3" + version: "2.0.7" supabase_flutter: dependency: "direct main" description: name: supabase_flutter - sha256: "57ca6c970042e6cef29056b0dc71bb17fc9d6848188df5ddc69e819cf28c7923" + sha256: "5ef71289c380b6429216e941c69971c75eaab50d67fd7b540f6c1f6ebfc00ed7" url: "https://pub.dev" source: hosted - version: "2.1.0" + version: "2.3.3" term_glyph: dependency: transitive description: @@ -512,26 +544,26 @@ packages: dependency: transitive description: name: url_launcher - sha256: e9aa5ea75c84cf46b3db4eea212523591211c3cf2e13099ee4ec147f54201c86 + sha256: c512655380d241a337521703af62d2c122bf7b77a46ff7dd750092aa9433499c url: "https://pub.dev" source: hosted - version: "6.2.2" + version: "6.2.4" url_launcher_android: dependency: transitive description: name: url_launcher_android - sha256: c0766a55ab42cefaa728cabc951e82919ab41a3a4fee0aaa96176ca82da8cc51 + sha256: d4ed0711849dd8e33eb2dd69c25db0d0d3fdc37e0a62e629fe32f57a22db2745 url: "https://pub.dev" source: hosted - version: "6.2.1" + version: "6.3.0" url_launcher_ios: dependency: transitive description: name: url_launcher_ios - sha256: "46b81e3109cbb2d6b81702ad3077540789a3e74e22795eb9f0b7d494dbaa72ea" + sha256: "75bb6fe3f60070407704282a2d295630cab232991eb52542b18347a8a941df03" url: "https://pub.dev" source: hosted - version: "6.2.2" + version: "6.2.4" url_launcher_linux: dependency: transitive description: @@ -552,10 +584,10 @@ packages: dependency: transitive description: name: url_launcher_platform_interface - sha256: f099b552bd331eacd69affed7ff2f23bfa6b0cb825b629edf3d844375a7501ad + sha256: a932c3a8082e118f80a475ce692fde89dc20fddb24c57360b96bc56f7035de1f url: "https://pub.dev" source: hosted - version: "2.2.1" + version: "2.3.1" url_launcher_web: dependency: transitive description: @@ -576,10 +608,10 @@ packages: dependency: transitive description: name: uuid - sha256: "22c94e5ad1e75f9934b766b53c742572ee2677c56bc871d850a57dad0f82127f" + sha256: cd210a09f7c18cbe5a02511718e0334de6559871052c90a90c0cca46a4aa81c8 url: "https://pub.dev" source: hosted - version: "4.2.2" + version: "4.3.3" vector_math: dependency: transitive description: @@ -608,10 +640,10 @@ packages: dependency: transitive description: name: win32 - sha256: b0f37db61ba2f2e9b7a78a1caece0052564d1bc70668156cf3a29d676fe4e574 + sha256: "464f5674532865248444b4c3daca12bd9bf2d7c47f759ce2617986e7229494a8" url: "https://pub.dev" source: hosted - version: "5.1.1" + version: "5.2.0" xdg_directories: dependency: transitive description: @@ -629,5 +661,5 @@ packages: source: hosted version: "2.0.0" sdks: - dart: ">=3.2.0 <4.0.0" + dart: ">=3.2.3 <4.0.0" flutter: ">=3.16.0" diff --git a/demos/supabase-edge-function-auth/pubspec.yaml b/demos/supabase-edge-function-auth/pubspec.yaml index 717f269f..27025645 100644 --- a/demos/supabase-edge-function-auth/pubspec.yaml +++ b/demos/supabase-edge-function-auth/pubspec.yaml @@ -1,22 +1,22 @@ name: supabase_jwt_auth description: PowerSync Supabase JWT Auth Demo -publish_to: "none" +publish_to: 'none' version: 1.0.1 environment: - sdk: ^3.2.0 + sdk: ^3.2.3 dependencies: flutter: sdk: flutter - powersync: ^1.2.0 + powersync: ^1.3.0-alpha.1 path_provider: ^2.1.1 supabase_flutter: ^2.0.2 path: ^1.8.3 logging: ^1.2.0 - sqlite_async: ^0.6.0 + sqlite_async: ^0.7.0-alpha.1 dev_dependencies: flutter_test: diff --git a/demos/supabase-simple-chat/lib/powersync.dart b/demos/supabase-simple-chat/lib/powersync.dart index edbd3fb5..6c4770ae 100644 --- a/demos/supabase-simple-chat/lib/powersync.dart +++ b/demos/supabase-simple-chat/lib/powersync.dart @@ -1,3 +1,6 @@ +import 'dart:io'; + +import 'package:flutter/foundation.dart'; import 'package:powersync/powersync.dart'; import 'package:supabase_flutter/supabase_flutter.dart'; import 'package:path/path.dart'; @@ -24,7 +27,7 @@ final List fatalResponseCodes = [ late final PowerSyncDatabase db; Future getDatabasePath() async { - final dir = await getApplicationSupportDirectory(); + final dir = kIsWeb ? Directory('/') : await getApplicationSupportDirectory(); return join(dir.path, 'powersync-demo.db'); } diff --git a/demos/supabase-simple-chat/pubspec.lock b/demos/supabase-simple-chat/pubspec.lock index b3a8064c..e35048f5 100644 --- a/demos/supabase-simple-chat/pubspec.lock +++ b/demos/supabase-simple-chat/pubspec.lock @@ -49,6 +49,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.18.0" + convert: + dependency: transitive + description: + name: convert + sha256: "0f08b14755d163f6e2134cb58222dd25ea2a2ee8a195e53983d57c075324d592" + url: "https://pub.dev" + source: hosted + version: "3.1.1" crypto: dependency: transitive description: @@ -65,6 +73,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.0.6" + drift: + dependency: transitive + description: + name: drift + sha256: b50a8342c6ddf05be53bda1d246404cbad101b64dc73e8d6d1ac1090d119b4e2 + url: "https://pub.dev" + source: hosted + version: "2.15.0" fake_async: dependency: transitive description: @@ -73,6 +89,22 @@ packages: url: "https://pub.dev" source: hosted version: "1.3.1" + fetch_api: + dependency: transitive + description: + name: fetch_api + sha256: "74a1e426d41ed9c89353703b2d80400c5d0ecfa144b2d8a7bd8882fbc9e48787" + url: "https://pub.dev" + source: hosted + version: "1.0.3" + fetch_client: + dependency: transitive + description: + name: fetch_client + sha256: "83c07b07a63526a43630572c72715707ca113a8aa3459efbc7b2d366b79402af" + url: "https://pub.dev" + source: hosted + version: "1.0.2" ffi: dependency: transitive description: @@ -89,6 +121,14 @@ packages: url: "https://pub.dev" source: hosted version: "7.0.0" + fixnum: + dependency: transitive + description: + name: fixnum + sha256: "25517a4deb0c03aa0f32fd12db525856438902d9c16536311e76cdc57b31d7d1" + url: "https://pub.dev" + source: hosted + version: "1.1.0" flutter: dependency: "direct main" description: flutter @@ -156,10 +196,10 @@ packages: dependency: transitive description: name: http - sha256: d4872660c46d929f6b8a9ef4e7a7eff7e49bbf0c4ec3f385ee32df5119175139 + sha256: a2bbf9d017fcced29139daa8ed2bba4ece450ab222871df93ca9eec6f80c34ba url: "https://pub.dev" source: hosted - version: "1.1.2" + version: "1.2.0" http_parser: dependency: transitive description: @@ -172,10 +212,10 @@ packages: dependency: transitive description: name: intl - sha256: "3bc132a9dbce73a7e4a21a17d06e1878839ffbf975568bc875c60537824b0c4d" + sha256: d6f56758b7d3014a48af9701c085700aac781a92a87a62b1333b46d8879661cf url: "https://pub.dev" source: hosted - version: "0.18.1" + version: "0.19.0" js: dependency: transitive description: @@ -236,10 +276,18 @@ packages: dependency: transitive description: name: mime - sha256: e4ff8e8564c03f255408decd16e7899da1733852a9110a58fe6d1b817684a63e + sha256: "2e123074287cc9fd6c09de8336dae606d1ddb88d9ac47358826db698c176a1f2" url: "https://pub.dev" source: hosted - version: "1.0.4" + version: "1.0.5" + mutex: + dependency: transitive + description: + name: mutex + sha256: "8827da25de792088eb33e572115a5eb0d61d61a3c01acbc8bcbe76ed78f1a1f2" + url: "https://pub.dev" + source: hosted + version: "3.1.0" path: dependency: "direct main" description: @@ -252,10 +300,10 @@ packages: dependency: "direct main" description: name: path_provider - sha256: a1aa8aaa2542a6bc57e381f132af822420216c80d4781f7aa085ca3229208aaa + sha256: b27217933eeeba8ff24845c34003b003b2b22151de3c908d0e679e8fe1aa078b url: "https://pub.dev" source: hosted - version: "2.1.1" + version: "2.1.2" path_provider_android: dependency: transitive description: @@ -268,10 +316,10 @@ packages: dependency: transitive description: name: path_provider_foundation - sha256: "19314d595120f82aca0ba62787d58dde2cc6b5df7d2f0daf72489e38d1b57f2d" + sha256: "5a7999be66e000916500be4f15a3633ebceb8302719b47b9cc49ce924125350f" url: "https://pub.dev" source: hosted - version: "2.3.1" + version: "2.3.2" path_provider_linux: dependency: transitive description: @@ -284,10 +332,10 @@ packages: dependency: transitive description: name: path_provider_platform_interface - sha256: "94b1e0dd80970c1ce43d5d4e050a9918fce4f4a775e6142424c30a29a363265c" + sha256: "88f5779f72ba699763fa3a3b06aa4bf6de76c8e5de842cf6f29e2e06476c2334" url: "https://pub.dev" source: hosted - version: "2.1.1" + version: "2.1.2" path_provider_windows: dependency: transitive description: @@ -326,7 +374,7 @@ packages: path: "../../packages/powersync" relative: true source: path - version: "1.2.0" + version: "1.3.0-alpha.1" realtime_client: dependency: transitive description: @@ -371,10 +419,10 @@ packages: dependency: transitive description: name: shared_preferences_foundation - sha256: "7bf53a9f2d007329ee6f3df7268fd498f8373602f943c975598bbb34649b62a7" + sha256: "7708d83064f38060c7b39db12aefe449cb8cdc031d6062280087bc4cdb988f5c" url: "https://pub.dev" source: hosted - version: "2.3.4" + version: "2.3.5" shared_preferences_linux: dependency: transitive description: @@ -387,10 +435,10 @@ packages: dependency: transitive description: name: shared_preferences_platform_interface - sha256: d4ec5fc9ebb2f2e056c617112aa75dcf92fc2e4faaf2ae999caa297473f75d8a + sha256: "22e2ecac9419b4246d7c22bfbbda589e3acf5c0351137d87dd2939d984d37c3b" url: "https://pub.dev" source: hosted - version: "2.3.1" + version: "2.3.2" shared_preferences_web: dependency: transitive description: @@ -456,26 +504,26 @@ packages: dependency: transitive description: name: sqlite3 - sha256: c4a4c5a4b2a32e2d0f6837b33d7c91a67903891a5b7dbe706cf4b1f6b0c798c5 + sha256: "072128763f1547e3e9b4735ce846bfd226d68019ccda54db4cd427b12dfdedc9" url: "https://pub.dev" source: hosted - version: "2.3.0" + version: "2.4.0" sqlite3_flutter_libs: dependency: transitive description: name: sqlite3_flutter_libs - sha256: "3e3583b77cf888a68eae2e49ee4f025f66b86623ef0d83c297c8d903daa14871" + sha256: d6c31c8511c441d1f12f20b607343df1afe4eddf24a1cf85021677c8eea26060 url: "https://pub.dev" source: hosted - version: "0.5.18" + version: "0.5.20" sqlite_async: dependency: transitive description: name: sqlite_async - sha256: "609a8405b8b608ac396dd7f478ed42e230c496eb38fe53dd97e9c592e1cd5cda" + sha256: "91f454cddc85617bea2c7c1544ff386887d0d2cf0ecdb3599015c05cc141ff4d" url: "https://pub.dev" source: hosted - version: "0.6.0" + version: "0.7.0-alpha.1" stack_trace: dependency: transitive description: @@ -544,10 +592,10 @@ packages: dependency: "direct main" description: name: timeago - sha256: c44b80cbc6b44627c00d76960f2af571f6f50e5dbedef4d9215d455e4335165b + sha256: d3204eb4c788214883380253da7f23485320a58c11d145babc82ad16bf4e7764 url: "https://pub.dev" source: hosted - version: "3.6.0" + version: "3.6.1" typed_data: dependency: transitive description: @@ -560,26 +608,26 @@ packages: dependency: transitive description: name: url_launcher - sha256: e9aa5ea75c84cf46b3db4eea212523591211c3cf2e13099ee4ec147f54201c86 + sha256: c512655380d241a337521703af62d2c122bf7b77a46ff7dd750092aa9433499c url: "https://pub.dev" source: hosted - version: "6.2.2" + version: "6.2.4" url_launcher_android: dependency: transitive description: name: url_launcher_android - sha256: c0766a55ab42cefaa728cabc951e82919ab41a3a4fee0aaa96176ca82da8cc51 + sha256: d4ed0711849dd8e33eb2dd69c25db0d0d3fdc37e0a62e629fe32f57a22db2745 url: "https://pub.dev" source: hosted - version: "6.2.1" + version: "6.3.0" url_launcher_ios: dependency: transitive description: name: url_launcher_ios - sha256: "46b81e3109cbb2d6b81702ad3077540789a3e74e22795eb9f0b7d494dbaa72ea" + sha256: "75bb6fe3f60070407704282a2d295630cab232991eb52542b18347a8a941df03" url: "https://pub.dev" source: hosted - version: "6.2.2" + version: "6.2.4" url_launcher_linux: dependency: transitive description: @@ -600,10 +648,10 @@ packages: dependency: transitive description: name: url_launcher_platform_interface - sha256: f099b552bd331eacd69affed7ff2f23bfa6b0cb825b629edf3d844375a7501ad + sha256: a932c3a8082e118f80a475ce692fde89dc20fddb24c57360b96bc56f7035de1f url: "https://pub.dev" source: hosted - version: "2.2.1" + version: "2.3.1" url_launcher_web: dependency: transitive description: @@ -624,10 +672,10 @@ packages: dependency: transitive description: name: uuid - sha256: "22c94e5ad1e75f9934b766b53c742572ee2677c56bc871d850a57dad0f82127f" + sha256: cd210a09f7c18cbe5a02511718e0334de6559871052c90a90c0cca46a4aa81c8 url: "https://pub.dev" source: hosted - version: "4.2.2" + version: "4.3.3" vector_math: dependency: transitive description: @@ -656,42 +704,42 @@ packages: dependency: transitive description: name: webview_flutter - sha256: "60e23976834e995c404c0b21d3b9db37ecd77d3303ef74f8b8d7a7b19947fc04" + sha256: "25e1b6e839e8cbfbd708abc6f85ed09d1727e24e08e08c6b8590d7c65c9a8932" url: "https://pub.dev" source: hosted - version: "4.4.3" + version: "4.7.0" webview_flutter_android: dependency: transitive description: name: webview_flutter_android - sha256: "161af93c2abaf94ef2192bffb53a3658b2d721a3bf99b69aa1e47814ee18cc96" + sha256: "3e5f4e9d818086b0d01a66fb1ff9cc72ab0cc58c71980e3d3661c5685ea0efb0" url: "https://pub.dev" source: hosted - version: "3.13.2" + version: "3.15.0" webview_flutter_platform_interface: dependency: transitive description: name: webview_flutter_platform_interface - sha256: dbe745ee459a16b6fec296f7565a8ef430d0d681001d8ae521898b9361854943 + sha256: d937581d6e558908d7ae3dc1989c4f87b786891ab47bb9df7de548a151779d8d url: "https://pub.dev" source: hosted - version: "2.9.0" + version: "2.10.0" webview_flutter_wkwebview: dependency: transitive description: name: webview_flutter_wkwebview - sha256: "02d8f3ebbc842704b2b662377b3ee11c0f8f1bbaa8eab6398262f40049819160" + sha256: "9bf168bccdf179ce90450b5f37e36fe263f591c9338828d6bf09b6f8d0f57f86" url: "https://pub.dev" source: hosted - version: "3.10.1" + version: "3.12.0" win32: dependency: transitive description: name: win32 - sha256: b0f37db61ba2f2e9b7a78a1caece0052564d1bc70668156cf3a29d676fe4e574 + sha256: "464f5674532865248444b4c3daca12bd9bf2d7c47f759ce2617986e7229494a8" url: "https://pub.dev" source: hosted - version: "5.1.1" + version: "5.2.0" xdg_directories: dependency: transitive description: @@ -709,5 +757,5 @@ packages: source: hosted version: "1.1.1" sdks: - dart: ">=3.2.0 <4.0.0" - flutter: ">=3.16.0" + dart: ">=3.2.3 <4.0.0" + flutter: ">=3.16.6" diff --git a/demos/supabase-simple-chat/pubspec.yaml b/demos/supabase-simple-chat/pubspec.yaml index 75757f37..1ec83e1d 100644 --- a/demos/supabase-simple-chat/pubspec.yaml +++ b/demos/supabase-simple-chat/pubspec.yaml @@ -2,7 +2,7 @@ name: supabase_tutorial_chat_app description: A new Flutter project. # The following line prevents the package from being accidentally published to # pub.dev using `flutter pub publish`. This is preferred for private packages. -publish_to: "none" # Remove this line if you wish to publish to pub.dev +publish_to: 'none' # Remove this line if you wish to publish to pub.dev # The following defines the version and build number for your application. # A version number is three numbers separated by dots, like 1.2.43 @@ -19,7 +19,7 @@ publish_to: "none" # Remove this line if you wish to publish to pub.dev version: 1.0.0+1 environment: - sdk: ^3.2.0 + sdk: ^3.2.3 # Dependencies specify other packages that your package needs in order to work. # To automatically upgrade your package dependencies to the latest versions @@ -37,7 +37,7 @@ dependencies: supabase_flutter: ^1.10.25 timeago: ^3.6.0 - powersync: ^1.2.0 + powersync: ^1.3.0-alpha.1 path_provider: ^2.1.1 path: ^1.8.3 logging: ^1.2.0 diff --git a/demos/supabase-todolist/.gitignore b/demos/supabase-todolist/.gitignore index 0b04140a..1a825b5b 100644 --- a/demos/supabase-todolist/.gitignore +++ b/demos/supabase-todolist/.gitignore @@ -47,4 +47,4 @@ app.*.map.json .tool-versions # secrets -app_config.dart +app_config.dart \ No newline at end of file diff --git a/demos/supabase-todolist/ios/Podfile.lock b/demos/supabase-todolist/ios/Podfile.lock index 67398eaf..8acfd0ee 100644 --- a/demos/supabase-todolist/ios/Podfile.lock +++ b/demos/supabase-todolist/ios/Podfile.lock @@ -10,18 +10,18 @@ PODS: - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS - - sqlite3 (3.44.0): - - sqlite3/common (= 3.44.0) - - sqlite3/common (3.44.0) - - sqlite3/fts5 (3.44.0): + - sqlite3 (3.45.1): + - sqlite3/common (= 3.45.1) + - sqlite3/common (3.45.1) + - sqlite3/fts5 (3.45.1): - sqlite3/common - - sqlite3/perf-threadsafe (3.44.0): + - sqlite3/perf-threadsafe (3.45.1): - sqlite3/common - - sqlite3/rtree (3.44.0): + - sqlite3/rtree (3.45.1): - sqlite3/common - sqlite3_flutter_libs (0.0.1): - Flutter - - sqlite3 (~> 3.44.0) + - sqlite3 (~> 3.45.1) - sqlite3/fts5 - sqlite3/perf-threadsafe - sqlite3/rtree @@ -59,14 +59,14 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: app_links: 5ef33d0d295a89d9d16bb81b0e3b0d5f70d6c875 - camera_avfoundation: 8b8d780bcfb6a4a02b0fbe2b4bd17b5b71946e68 + camera_avfoundation: 759172d1a77ae7be0de08fc104cfb79738b8a59e Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 3784922295ac71e43754bd15e0653ccfd36a147c shared_preferences_foundation: b4c3b4cddf1c21f02770737f147a3f5da9d39695 - sqlite3: 6e2d4a4879854d0ec86b476bf3c3e30870bac273 - sqlite3_flutter_libs: eb769059df0356dc52ddda040f09cacc9391a7cf + sqlite3: 73b7fc691fdc43277614250e04d183740cb15078 + sqlite3_flutter_libs: af0e8fe9bce48abddd1ffdbbf839db0302d72d80 url_launcher_ios: bbd758c6e7f9fd7b5b1d4cde34d2b95fcce5e812 PODFILE CHECKSUM: 13e359f40c4925bcdf0c1bfa13aeba35011fde30 -COCOAPODS: 1.14.3 +COCOAPODS: 1.13.0 diff --git a/demos/supabase-todolist/lib/models/todo_item.dart b/demos/supabase-todolist/lib/models/todo_item.dart index 5e15f3f8..1cd43de0 100644 --- a/demos/supabase-todolist/lib/models/todo_item.dart +++ b/demos/supabase-todolist/lib/models/todo_item.dart @@ -1,7 +1,7 @@ import 'package:powersync_flutter_demo/models/schema.dart'; import '../powersync.dart'; -import 'package:powersync/sqlite3.dart' as sqlite; +import 'package:powersync/sqlite3_common.dart' as sqlite; /// TodoItem represents a result row of a query on "todos". /// diff --git a/demos/supabase-todolist/lib/models/todo_list.dart b/demos/supabase-todolist/lib/models/todo_list.dart index 489d1631..e871b6b6 100644 --- a/demos/supabase-todolist/lib/models/todo_list.dart +++ b/demos/supabase-todolist/lib/models/todo_list.dart @@ -1,4 +1,4 @@ -import 'package:powersync/sqlite3.dart' as sqlite; +import 'package:powersync/sqlite3_common.dart' as sqlite; import './todo_item.dart'; import '../powersync.dart'; diff --git a/demos/supabase-todolist/lib/powersync.dart b/demos/supabase-todolist/lib/powersync.dart index aaf952b4..0c5f79dd 100644 --- a/demos/supabase-todolist/lib/powersync.dart +++ b/demos/supabase-todolist/lib/powersync.dart @@ -1,4 +1,7 @@ // This file performs setup of the PowerSync database +import 'dart:io'; + +import 'package:flutter/foundation.dart'; import 'package:logging/logging.dart'; import 'package:path/path.dart'; import 'package:path_provider/path_provider.dart'; @@ -145,7 +148,7 @@ String? getUserId() { } Future getDatabasePath() async { - final dir = await getApplicationSupportDirectory(); + final dir = kIsWeb ? Directory('/') : await getApplicationSupportDirectory(); return join(dir.path, 'powersync-demo.db'); } diff --git a/demos/supabase-todolist/lib/widgets/query_widget.dart b/demos/supabase-todolist/lib/widgets/query_widget.dart index b9a285dd..67786520 100644 --- a/demos/supabase-todolist/lib/widgets/query_widget.dart +++ b/demos/supabase-todolist/lib/widgets/query_widget.dart @@ -1,7 +1,7 @@ import 'dart:async'; import 'package:flutter/material.dart'; -import 'package:powersync/sqlite3.dart' as sqlite; +import 'package:powersync/sqlite3_common.dart' as sqlite; import './resultset_table.dart'; import '../powersync.dart'; diff --git a/demos/supabase-todolist/lib/widgets/resultset_table.dart b/demos/supabase-todolist/lib/widgets/resultset_table.dart index b1606adf..f348e4ff 100644 --- a/demos/supabase-todolist/lib/widgets/resultset_table.dart +++ b/demos/supabase-todolist/lib/widgets/resultset_table.dart @@ -1,5 +1,5 @@ import 'package:flutter/material.dart'; -import 'package:powersync/sqlite3.dart' as sqlite; +import 'package:powersync/sqlite3_common.dart' as sqlite; /// Stateless DataTable rendering results from a SQLite query class ResultSetTable extends StatelessWidget { diff --git a/demos/supabase-todolist/macos/Podfile.lock b/demos/supabase-todolist/macos/Podfile.lock index f54327c0..f3ca846a 100644 --- a/demos/supabase-todolist/macos/Podfile.lock +++ b/demos/supabase-todolist/macos/Podfile.lock @@ -8,20 +8,18 @@ PODS: - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS - - sign_in_with_apple (0.0.1): - - FlutterMacOS - - sqlite3 (3.43.1): - - sqlite3/common (= 3.43.1) - - sqlite3/common (3.43.1) - - sqlite3/fts5 (3.43.1): + - sqlite3 (3.45.1): + - sqlite3/common (= 3.45.1) + - sqlite3/common (3.45.1) + - sqlite3/fts5 (3.45.1): - sqlite3/common - - sqlite3/perf-threadsafe (3.43.1): + - sqlite3/perf-threadsafe (3.45.1): - sqlite3/common - - sqlite3/rtree (3.43.1): + - sqlite3/rtree (3.45.1): - sqlite3/common - sqlite3_flutter_libs (0.0.1): - FlutterMacOS - - sqlite3 (~> 3.43.1) + - sqlite3 (~> 3.45.1) - sqlite3/fts5 - sqlite3/perf-threadsafe - sqlite3/rtree @@ -33,7 +31,6 @@ DEPENDENCIES: - FlutterMacOS (from `Flutter/ephemeral`) - path_provider_foundation (from `Flutter/ephemeral/.symlinks/plugins/path_provider_foundation/darwin`) - shared_preferences_foundation (from `Flutter/ephemeral/.symlinks/plugins/shared_preferences_foundation/darwin`) - - sign_in_with_apple (from `Flutter/ephemeral/.symlinks/plugins/sign_in_with_apple/macos`) - sqlite3_flutter_libs (from `Flutter/ephemeral/.symlinks/plugins/sqlite3_flutter_libs/macos`) - url_launcher_macos (from `Flutter/ephemeral/.symlinks/plugins/url_launcher_macos/macos`) @@ -50,8 +47,6 @@ EXTERNAL SOURCES: :path: Flutter/ephemeral/.symlinks/plugins/path_provider_foundation/darwin shared_preferences_foundation: :path: Flutter/ephemeral/.symlinks/plugins/shared_preferences_foundation/darwin - sign_in_with_apple: - :path: Flutter/ephemeral/.symlinks/plugins/sign_in_with_apple/macos sqlite3_flutter_libs: :path: Flutter/ephemeral/.symlinks/plugins/sqlite3_flutter_libs/macos url_launcher_macos: @@ -60,13 +55,12 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: app_links: 4481ed4d71f384b0c3ae5016f4633aa73d32ff67 FlutterMacOS: 8f6f14fa908a6fb3fba0cd85dbd81ec4b251fb24 - path_provider_foundation: 29f094ae23ebbca9d3d0cec13889cd9060c0e943 - shared_preferences_foundation: 5b919d13b803cadd15ed2dc053125c68730e5126 - sign_in_with_apple: a9e97e744e8edc36aefc2723111f652102a7a727 - sqlite3: e0a0623a33a20a47cb5921552aebc6e9e437dc91 - sqlite3_flutter_libs: a91655e4a75a499364f693041aa1c6d1b36b66d0 + path_provider_foundation: 3784922295ac71e43754bd15e0653ccfd36a147c + shared_preferences_foundation: b4c3b4cddf1c21f02770737f147a3f5da9d39695 + sqlite3: 73b7fc691fdc43277614250e04d183740cb15078 + sqlite3_flutter_libs: 06a05802529659a272beac4ee1350bfec294f386 url_launcher_macos: d2691c7dd33ed713bf3544850a623080ec693d95 PODFILE CHECKSUM: 236401fc2c932af29a9fcf0e97baeeb2d750d367 -COCOAPODS: 1.12.1 +COCOAPODS: 1.13.0 diff --git a/demos/supabase-todolist/pubspec.lock b/demos/supabase-todolist/pubspec.lock index cd570c1b..9f83e15f 100644 --- a/demos/supabase-todolist/pubspec.lock +++ b/demos/supabase-todolist/pubspec.lock @@ -53,18 +53,18 @@ packages: dependency: transitive description: name: camera_avfoundation - sha256: "608b56b0880722f703871329c4d7d4c2f379c8e2936940851df7fc041abc6f51" + sha256: "7d0763dfcbf060f56aa254a68c103210280bee9e97bbe4fdef23e257a4f70ab9" url: "https://pub.dev" source: hosted - version: "0.9.13+10" + version: "0.9.14" camera_platform_interface: dependency: transitive description: name: camera_platform_interface - sha256: e971ebca970f7cfee396f76ef02070b5e441b4aa04942da9c108d725f57bbd32 + sha256: fceb2c36038b6392317b1d5790c6ba9e6ca9f1da3031181b8bea03882bf9387a url: "https://pub.dev" source: hosted - version: "2.7.2" + version: "2.7.3" camera_web: dependency: transitive description: @@ -121,6 +121,14 @@ packages: url: "https://pub.dev" source: hosted version: "3.0.3" + drift: + dependency: transitive + description: + name: drift + sha256: b50a8342c6ddf05be53bda1d246404cbad101b64dc73e8d6d1ac1090d119b4e2 + url: "https://pub.dev" + source: hosted + version: "2.15.0" fake_async: dependency: transitive description: @@ -129,6 +137,22 @@ packages: url: "https://pub.dev" source: hosted version: "1.3.1" + fetch_api: + dependency: transitive + description: + name: fetch_api + sha256: "74a1e426d41ed9c89353703b2d80400c5d0ecfa144b2d8a7bd8882fbc9e48787" + url: "https://pub.dev" + source: hosted + version: "1.0.3" + fetch_client: + dependency: transitive + description: + name: fetch_client + sha256: "83c07b07a63526a43630572c72715707ca113a8aa3459efbc7b2d366b79402af" + url: "https://pub.dev" + source: hosted + version: "1.0.2" ffi: dependency: transitive description: @@ -196,10 +220,10 @@ packages: dependency: transitive description: name: gotrue - sha256: "0af635a935d4ec78e8885f71d71d1994460b1a855faee9b5b520e9b5417ceb02" + sha256: f40610bacf1074723354b0856a4f586508ffb075b799f72466f34e843133deb9 url: "https://pub.dev" source: hosted - version: "2.3.0" + version: "2.5.0" gtk: dependency: transitive description: @@ -208,30 +232,14 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.0" - hive: - dependency: transitive - description: - name: hive - sha256: "8dcf6db979d7933da8217edcec84e9df1bdb4e4edc7fc77dbd5aa74356d6d941" - url: "https://pub.dev" - source: hosted - version: "2.2.3" - hive_flutter: - dependency: transitive - description: - name: hive_flutter - sha256: dca1da446b1d808a51689fb5d0c6c9510c0a2ba01e22805d492c73b68e33eecc - url: "https://pub.dev" - source: hosted - version: "1.1.0" http: dependency: transitive description: name: http - sha256: d4872660c46d929f6b8a9ef4e7a7eff7e49bbf0c4ec3f385ee32df5119175139 + sha256: a2bbf9d017fcced29139daa8ed2bba4ece450ab222871df93ca9eec6f80c34ba url: "https://pub.dev" source: hosted - version: "1.1.2" + version: "1.2.0" http_parser: dependency: transitive description: @@ -244,10 +252,10 @@ packages: dependency: "direct main" description: name: image - sha256: "004a2e90ce080f8627b5a04aecb4cdfac87d2c3f3b520aa291260be5a32c033d" + sha256: "4c68bfd5ae83e700b5204c1e74451e7bf3cf750e6843c6e158289cf56bda018e" url: "https://pub.dev" source: hosted - version: "4.1.4" + version: "4.1.7" js: dependency: transitive description: @@ -308,10 +316,18 @@ packages: dependency: transitive description: name: mime - sha256: e4ff8e8564c03f255408decd16e7899da1733852a9110a58fe6d1b817684a63e + sha256: "2e123074287cc9fd6c09de8336dae606d1ddb88d9ac47358826db698c176a1f2" url: "https://pub.dev" source: hosted - version: "1.0.4" + version: "1.0.5" + mutex: + dependency: transitive + description: + name: mutex + sha256: "8827da25de792088eb33e572115a5eb0d61d61a3c01acbc8bcbe76ed78f1a1f2" + url: "https://pub.dev" + source: hosted + version: "3.1.0" path: dependency: "direct main" description: @@ -396,10 +412,10 @@ packages: dependency: transitive description: name: pointycastle - sha256: "7c1e5f0d23c9016c5bbd8b1473d0d3fb3fc851b876046039509e18e0c7485f2c" + sha256: "43ac87de6e10afabc85c445745a7b799e04de84cebaa4fd7bf55a5e1e9604d29" url: "https://pub.dev" source: hosted - version: "3.7.3" + version: "3.7.4" postgrest: dependency: transitive description: @@ -414,14 +430,14 @@ packages: path: "../../packages/powersync" relative: true source: path - version: "1.2.0" + version: "1.3.0-alpha.1" powersync_attachments_helper: dependency: "direct main" description: path: "../../packages/powersync_attachments_helper" relative: true source: path - version: "0.2.0" + version: "0.3.0-alpha.1" realtime_client: dependency: transitive description: @@ -527,26 +543,26 @@ packages: dependency: transitive description: name: sqlite3 - sha256: c4a4c5a4b2a32e2d0f6837b33d7c91a67903891a5b7dbe706cf4b1f6b0c798c5 + sha256: "072128763f1547e3e9b4735ce846bfd226d68019ccda54db4cd427b12dfdedc9" url: "https://pub.dev" source: hosted - version: "2.3.0" + version: "2.4.0" sqlite3_flutter_libs: dependency: transitive description: name: sqlite3_flutter_libs - sha256: "3e3583b77cf888a68eae2e49ee4f025f66b86623ef0d83c297c8d903daa14871" + sha256: d6c31c8511c441d1f12f20b607343df1afe4eddf24a1cf85021677c8eea26060 url: "https://pub.dev" source: hosted - version: "0.5.18" + version: "0.5.20" sqlite_async: dependency: "direct main" description: name: sqlite_async - sha256: "609a8405b8b608ac396dd7f478ed42e230c496eb38fe53dd97e9c592e1cd5cda" + sha256: "91f454cddc85617bea2c7c1544ff386887d0d2cf0ecdb3599015c05cc141ff4d" url: "https://pub.dev" source: hosted - version: "0.6.0" + version: "0.7.0-alpha.1" stack_trace: dependency: transitive description: @@ -559,10 +575,10 @@ packages: dependency: transitive description: name: storage_client - sha256: b49ff2e1e6738c0ef445546d6ec77040829947f0c7ef0b115acb125656127c83 + sha256: bf5589d5de61a2451edb1b8960a0e673d4bb5c42ecc4dddf7c051a93789ced34 url: "https://pub.dev" source: hosted - version: "2.0.0" + version: "2.0.1" stream_channel: dependency: transitive description: @@ -591,18 +607,18 @@ packages: dependency: transitive description: name: supabase - sha256: cee9fd3fe6465a81a2536e2e2e0d9ec0b48d5f195015e060f70e2bd1b619bb26 + sha256: "4bce9c49f264f4cd44b4ffc895647af2dca0c40125c169045be9f708fd2a2a40" url: "https://pub.dev" source: hosted - version: "2.0.4" + version: "2.0.7" supabase_flutter: dependency: "direct main" description: name: supabase_flutter - sha256: "118070e5d827e4dfd7feaf5a1679e0ecdca548b16a1b6ec15c2a507cf2520bb2" + sha256: "5ef71289c380b6429216e941c69971c75eaab50d67fd7b540f6c1f6ebfc00ed7" url: "https://pub.dev" source: hosted - version: "2.2.0" + version: "2.3.3" term_glyph: dependency: transitive description: @@ -631,18 +647,18 @@ packages: dependency: transitive description: name: url_launcher - sha256: d25bb0ca00432a5e1ee40e69c36c85863addf7cc45e433769d61bed3fe81fd96 + sha256: c512655380d241a337521703af62d2c122bf7b77a46ff7dd750092aa9433499c url: "https://pub.dev" source: hosted - version: "6.2.3" + version: "6.2.4" url_launcher_android: dependency: transitive description: name: url_launcher_android - sha256: "507dc655b1d9cb5ebc756032eb785f114e415f91557b73bf60b7e201dfedeb2f" + sha256: d4ed0711849dd8e33eb2dd69c25db0d0d3fdc37e0a62e629fe32f57a22db2745 url: "https://pub.dev" source: hosted - version: "6.2.2" + version: "6.3.0" url_launcher_ios: dependency: transitive description: @@ -756,5 +772,5 @@ packages: source: hosted version: "2.0.0" sdks: - dart: ">=3.2.0 <4.0.0" - flutter: ">=3.16.0" + dart: ">=3.2.3 <4.0.0" + flutter: ">=3.16.6" diff --git a/demos/supabase-todolist/pubspec.yaml b/demos/supabase-todolist/pubspec.yaml index f263e09f..cfda6024 100644 --- a/demos/supabase-todolist/pubspec.yaml +++ b/demos/supabase-todolist/pubspec.yaml @@ -1,23 +1,23 @@ name: powersync_flutter_demo description: PowerSync Flutter Demo -publish_to: "none" +publish_to: 'none' version: 1.0.1 environment: - sdk: ^3.2.0 + sdk: ^3.2.3 dependencies: flutter: sdk: flutter - powersync_attachments_helper: ^0.1.4 + powersync_attachments_helper: ^0.3.0-alpha.1 - powersync: ^1.2.0 + powersync: ^1.3.0-alpha.1 path_provider: ^2.1.1 supabase_flutter: ^2.0.1 path: ^1.8.3 logging: ^1.2.0 - sqlite_async: ^0.6.0 + sqlite_async: ^0.7.0-alpha.1 camera: ^0.10.5+7 image: ^4.1.3 diff --git a/demos/supabase-todolist/web/favicon.png b/demos/supabase-todolist/web/favicon.png new file mode 100644 index 00000000..8aaa46ac Binary files /dev/null and b/demos/supabase-todolist/web/favicon.png differ diff --git a/demos/supabase-todolist/web/icons/Icon-192.png b/demos/supabase-todolist/web/icons/Icon-192.png new file mode 100644 index 00000000..b749bfef Binary files /dev/null and b/demos/supabase-todolist/web/icons/Icon-192.png differ diff --git a/demos/supabase-todolist/web/icons/Icon-512.png b/demos/supabase-todolist/web/icons/Icon-512.png new file mode 100644 index 00000000..88cfd48d Binary files /dev/null and b/demos/supabase-todolist/web/icons/Icon-512.png differ diff --git a/demos/supabase-todolist/web/icons/Icon-maskable-192.png b/demos/supabase-todolist/web/icons/Icon-maskable-192.png new file mode 100644 index 00000000..eb9b4d76 Binary files /dev/null and b/demos/supabase-todolist/web/icons/Icon-maskable-192.png differ diff --git a/demos/supabase-todolist/web/icons/Icon-maskable-512.png b/demos/supabase-todolist/web/icons/Icon-maskable-512.png new file mode 100644 index 00000000..d69c5669 Binary files /dev/null and b/demos/supabase-todolist/web/icons/Icon-maskable-512.png differ diff --git a/demos/supabase-todolist/web/index.html b/demos/supabase-todolist/web/index.html new file mode 100644 index 00000000..b3b0c490 --- /dev/null +++ b/demos/supabase-todolist/web/index.html @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + supabase_todolist + + + + + + + + + + diff --git a/demos/supabase-todolist/web/manifest.json b/demos/supabase-todolist/web/manifest.json new file mode 100644 index 00000000..9dcf6fe4 --- /dev/null +++ b/demos/supabase-todolist/web/manifest.json @@ -0,0 +1,35 @@ +{ + "name": "supabase_todolist", + "short_name": "supabase_todolist", + "start_url": ".", + "display": "standalone", + "background_color": "#0175C2", + "theme_color": "#0175C2", + "description": "A new Flutter project.", + "orientation": "portrait-primary", + "prefer_related_applications": false, + "icons": [ + { + "src": "icons/Icon-192.png", + "sizes": "192x192", + "type": "image/png" + }, + { + "src": "icons/Icon-512.png", + "sizes": "512x512", + "type": "image/png" + }, + { + "src": "icons/Icon-maskable-192.png", + "sizes": "192x192", + "type": "image/png", + "purpose": "maskable" + }, + { + "src": "icons/Icon-maskable-512.png", + "sizes": "512x512", + "type": "image/png", + "purpose": "maskable" + } + ] +} diff --git a/melos.yaml b/melos.yaml index 576d92cd..0ecc1bfa 100644 --- a/melos.yaml +++ b/melos.yaml @@ -9,6 +9,8 @@ ide: intellij: false scripts: + prepare: melos bootstrap && melos compile:webworker && melos update:wasm + format: description: Format Dart code. run: dart format . @@ -29,9 +31,18 @@ scripts: description: Analyze Dart code in demos. run: dart analyze demos --fatal-infos + compile:webworker: + description: Compile Javascript web worker distributable + exec: dart run powersync_web_worker:compile_webworker + packageFilters: + scope: + - powersync_web_worker + + update:wasm: sh scripts/init_sqlite_wasm.sh + test: description: Run tests in a specific package. - run: dart test + run: dart test -p vm,chrome exec: concurrency: 1 packageFilters: diff --git a/packages/powersync/CHANGELOG.md b/packages/powersync/CHANGELOG.md index d54482bb..32ea38a2 100644 --- a/packages/powersync/CHANGELOG.md +++ b/packages/powersync/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.3.0-alpha.1 + +- Added initial support for Web platform. + ## 1.2.1 - Fix indexes incorrectly dropped after the first run. diff --git a/packages/powersync/README.md b/packages/powersync/README.md index 611a54a8..3683b823 100644 --- a/packages/powersync/README.md +++ b/packages/powersync/README.md @@ -4,12 +4,12 @@ ## SDK Features -* Real-time streaming of changes. -* Direct access to the SQLite database - use SQL on the client and server. -* Operations are asynchronous by default - does not block the UI. -* Supports one write and many reads concurrently. -* No need for client-side database migrations - these are handled automatically. -* Subscribe to queries for live updates. +- Real-time streaming of changes. +- Direct access to the SQLite database - use SQL on the client and server. +- Operations are asynchronous by default - does not block the UI. +- Supports one write and many reads concurrently. +- No need for client-side database migrations - these are handled automatically. +- Subscribe to queries for live updates. ## Examples @@ -149,3 +149,52 @@ Logger.root.onRecord.listen((record) { }); ``` +## Web support + +Web support is currently in an alpha release. + +### Setup + +Web support requires `sqlite3.wasm` and `powersync_db.worker.js` assets to be served from the web application. This is typically achieved by placing the files in the project `web` directory. + +These assets are automatically configured in this monorepo when running `melos prepare`. + +- `sqlite3.wasm` can be found [here](https://github.com/simolus3/sqlite3.dart/releases) +- `powersync_db.worker.js` will eventually be released in the repo's releases. + - In the interim the asset can be retrieved from the `./assets` folder after executing `melos prepare` + +Currently the Drift SQLite library is used under the hood for DB connections. See [here](https://drift.simonbinder.eu/web/#getting-started) for detailed compatibility +and setup notes. + +The same code is used for initializing native and web `PowerSyncDatabase` clients. + +### Limitations + +The API for web is essentially the same as for native platforms. Some features within `PowerSyncDatabase` clients are not available. + +#### Imports + +Flutter Web does not support importing directly from `sqlite3.dart` as it uses `dart:ffi`. + +Change imports from + +```Dart +import 'package/powersync/sqlite3.dart` +``` + +to + +```Dart +import 'package/powersync/sqlite3_common.dart' +``` + +In code which needs to run on the Web platform. Isolated native specific code can still import from `sqlite3.dart`. + +#### Database connections + +Web DB connections do not support concurrency. A single DB connection is used. `readLock` and `writeLock` contexts do not +implement checks for preventing writable queries in read connections and vice-versa. + +Direct access to the synchronous `CommonDatabase` (`sqlite.Database` equivalent for web) connection is not available. `computeWithDatabase` is not available on web. + +Multiple tab support is not yet available. Using multiple tabs will break. diff --git a/packages/powersync/example/getting_started.dart b/packages/powersync/example/getting_started.dart index cf44cbc4..94b6ca34 100644 --- a/packages/powersync/example/getting_started.dart +++ b/packages/powersync/example/getting_started.dart @@ -1,3 +1,6 @@ +import 'dart:io'; + +import 'package:flutter/foundation.dart'; import 'package:powersync/powersync.dart'; import 'package:path_provider/path_provider.dart'; import 'package:path/path.dart'; @@ -25,7 +28,7 @@ class BackendConnector extends PowerSyncBackendConnector { } openDatabase() async { - final dir = await getApplicationSupportDirectory(); + final dir = kIsWeb ? Directory('/') : await getApplicationSupportDirectory(); final path = join(dir.path, 'powersync-dart.db'); // Setup the database. db = PowerSyncDatabase(schema: schema, path: path); diff --git a/packages/powersync/lib/powersync.dart b/packages/powersync/lib/powersync.dart index 05963fb9..d931d4f4 100644 --- a/packages/powersync/lib/powersync.dart +++ b/packages/powersync/lib/powersync.dart @@ -5,6 +5,7 @@ library; export 'src/connector.dart'; export 'src/crud.dart'; +export 'src/database/powersync_database.dart'; export 'src/exceptions.dart'; export 'src/log.dart'; export 'src/open_factory.dart'; diff --git a/packages/powersync/lib/sqlite3_common.dart b/packages/powersync/lib/sqlite3_common.dart new file mode 100644 index 00000000..0d73acf4 --- /dev/null +++ b/packages/powersync/lib/sqlite3_common.dart @@ -0,0 +1,5 @@ +/// Re-exports [sqlite3](https://pub.dev/packages/sqlite3) to expose sqlite3 without +/// adding it as a direct dependency. +library; + +export 'package:sqlite_async/sqlite3_common.dart'; diff --git a/packages/powersync/lib/src/bucket_storage.dart b/packages/powersync/lib/src/bucket_storage.dart index adede76c..73a78491 100644 --- a/packages/powersync/lib/src/bucket_storage.dart +++ b/packages/powersync/lib/src/bucket_storage.dart @@ -2,50 +2,56 @@ import 'dart:async'; import 'dart:convert'; import 'package:collection/collection.dart'; -import 'package:powersync/src/log_internal.dart'; -import 'package:sqlite_async/mutex.dart'; -import 'package:sqlite_async/sqlite3.dart' as sqlite; +import 'package:meta/meta.dart'; +import 'package:powersync/sqlite_async.dart'; +import 'package:sqlite_async/sqlite3_common.dart' as sqlite; +import 'package:sqlite_async/sqlite_async.dart'; import 'crud.dart'; -import 'database_utils.dart'; -import 'schema_logic.dart'; +import 'schema_helpers.dart'; import 'sync_types.dart'; import 'uuid.dart'; +import 'log_internal.dart'; const compactOperationInterval = 1000; class BucketStorage { - final sqlite.Database _internalDb; - final Mutex mutex; + final SqliteConnection _internalDb; bool _hasCompletedSync = false; bool _pendingBucketDeletes = false; Set tableNames = {}; int _compactCounter = compactOperationInterval; ChecksumCache? _checksumCache; + late Future _isInitialized; - BucketStorage(sqlite.Database db, {required this.mutex}) : _internalDb = db { - _init(); + BucketStorage(SqliteConnection db) : _internalDb = db { + _isInitialized = _init(); } - _init() { - final existingTableRows = select( + _init() async { + final existingTableRows = await select( "SELECT name FROM sqlite_master WHERE type='table' AND name GLOB 'ps_data_*'"); for (final row in existingTableRows) { tableNames.add(row['name'] as String); } } + initialized() { + return _isInitialized; + } + // Use only for read statements - sqlite.ResultSet select(String query, [List parameters = const []]) { - return _internalDb.select(query, parameters); + Future select(String query, + [List parameters = const []]) async { + return _internalDb.execute(query, parameters); } void startSession() { _checksumCache = null; } - List getBucketStates() { - final rows = select( + Future> getBucketStates() async { + final rows = await select( 'SELECT name as bucket, cast(last_op as TEXT) as op_id FROM ps_buckets WHERE pending_delete = 0'); return [ for (var row in rows) @@ -56,21 +62,21 @@ class BucketStorage { Future saveSyncData(SyncDataBatch batch) async { var count = 0; - await writeTransaction((db) { + await writeTransaction((tx) async { for (var b in batch.buckets) { var bucket = b.bucket; var data = b.data; count += data.length; final isFinal = !b.hasMore; - _updateBucket(db, bucket, data, isFinal); + await _updateBucket(tx, bucket, data, isFinal); } }); _compactCounter += count; } - void _updateBucket(sqlite.Database db, String bucket, List data, - bool finalBucketUpdate) { + Future _updateBucket(SqliteWriteContext tx, String bucket, + List data, bool finalBucketUpdate) async { if (data.isEmpty) { return; } @@ -141,7 +147,7 @@ class BucketStorage { } // Mark old ops as superseded - db.execute(""" + await tx.execute(""" UPDATE ps_oplog AS oplog SET superseded = 1, op = ${OpType.move.value}, @@ -151,45 +157,42 @@ class BucketStorage { AND oplog.key IN (SELECT json_each.value FROM json_each(?)) """, [bucket, jsonEncode(allEntries)]); - var stmt = db.prepare( - 'INSERT INTO ps_oplog(op_id, op, bucket, key, row_type, row_id, data, hash, superseded) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)'); - try { - for (var insert in inserts) { - stmt.execute([ - insert['op_id'], - insert['op'], - insert['bucket'], - insert['key'], - insert['row_type'], - insert['row_id'], - insert['data'], - insert['checksum'], - insert['superseded'] - ]); - } - } finally { - stmt.dispose(); - } - - db.execute("INSERT OR IGNORE INTO ps_buckets(name) VALUES(?)", [bucket]); + await tx.executeBatch( + 'INSERT INTO ps_oplog(op_id, op, bucket, key, row_type, row_id, data, hash, superseded) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', + inserts + .map((insert) => [ + insert['op_id'], + insert['op'], + insert['bucket'], + insert['key'], + insert['row_type'], + insert['row_id'], + insert['data'], + insert['checksum'], + insert['superseded'] + ]) + .toList()); + + await tx + .execute("INSERT OR IGNORE INTO ps_buckets(name) VALUES(?)", [bucket]); if (lastOp != null) { - db.execute( + await tx.execute( "UPDATE ps_buckets SET last_op = ? WHERE name = ?", [lastOp, bucket]); } if (targetOp != null) { - db.execute( + await tx.execute( "UPDATE ps_buckets AS buckets SET target_op = MAX(?, buckets.target_op) WHERE name = ?", [targetOp.toString(), bucket]); } for (final op in clearOps) { - db.execute(op.sql, op.args); + await tx.execute(op.sql, op.args); } // Compact superseded ops immediately, but only _after_ clearing if (firstOp != null && lastOp != null) { - db.execute("""UPDATE ps_buckets AS buckets + await tx.execute("""UPDATE ps_buckets AS buckets SET add_checksum = add_checksum + (SELECT IFNULL(SUM(hash), 0) FROM ps_oplog AS oplog WHERE superseded = 1 @@ -198,7 +201,7 @@ class BucketStorage { AND oplog.op_id <= ?) WHERE buckets.name = ?""", [bucket, firstOp, lastOp, bucket]); - db.execute("""DELETE + await tx.execute("""DELETE FROM ps_oplog WHERE superseded = 1 AND bucket = ? @@ -216,15 +219,15 @@ class BucketStorage { Future deleteBucket(String bucket) async { final newName = "\$delete_${bucket}_${uuid.v4()}"; - await writeTransaction((db) { - db.execute( + await writeTransaction((tx) async { + await tx.execute( "UPDATE ps_oplog SET op=${OpType.remove.value}, data=NULL WHERE op=${OpType.put.value} AND superseded=0 AND bucket=?", [bucket]); // Rename bucket - db.execute( + await tx.execute( "UPDATE ps_oplog SET bucket=? WHERE bucket=?", [newName, bucket]); - db.execute("DELETE FROM ps_buckets WHERE name = ?", [bucket]); - db.execute( + await tx.execute("DELETE FROM ps_buckets WHERE name = ?", [bucket]); + await tx.execute( "INSERT INTO ps_buckets(name, pending_delete, last_op) SELECT ?, 1, IFNULL(MAX(op_id), 0) FROM ps_oplog WHERE bucket = ?", [newName, newName]); }); @@ -232,11 +235,11 @@ class BucketStorage { _pendingBucketDeletes = true; } - bool hasCompletedSync() { + Future hasCompletedSync() async { if (_hasCompletedSync) { return true; } - final rs = select( + final rs = await select( "SELECT name, last_applied_op FROM ps_buckets WHERE last_applied_op > 0 LIMIT 1"); if (rs.isNotEmpty) { _hasCompletedSync = true; @@ -247,22 +250,23 @@ class BucketStorage { Future syncLocalDatabase( Checkpoint checkpoint) async { - final r = validateChecksums(checkpoint); + final r = await validateChecksums(checkpoint); if (!r.checkpointValid) { for (String b in r.checkpointFailures ?? []) { - deleteBucket(b); + await deleteBucket(b); } return r; } final bucketNames = [for (final c in checkpoint.checksums) c.bucket]; - await writeTransaction((db) { - db.execute( + await writeTransaction((tx) async { + await tx.execute( "UPDATE ps_buckets SET last_op = ? WHERE name IN (SELECT json_each.value FROM json_each(?))", [checkpoint.lastOpId, jsonEncode(bucketNames)]); if (checkpoint.writeCheckpoint != null) { - db.execute("UPDATE ps_buckets SET last_op = ? WHERE name = '\$local'", + await tx.execute( + "UPDATE ps_buckets SET last_op = ? WHERE name = '\$local'", [checkpoint.writeCheckpoint]); } }); @@ -278,8 +282,8 @@ class BucketStorage { } Future updateObjectsFromBuckets(Checkpoint checkpoint) async { - return writeTransaction((db) { - if (!_canUpdateLocal(db)) { + return writeTransaction((tx) async { + if (!(await canUpdateLocal(tx))) { return false; } @@ -300,7 +304,7 @@ class BucketStorage { // |--SEARCH r USING INDEX ps_oplog_by_row (row_type=? AND row_id=?) // `--USE TEMP B-TREE FOR GROUP BY // language=DbSqlite - var stmt = db.prepare( + var opRows = await tx.execute( """-- 3. Group the objects from different buckets together into a single one (ops). SELECT r.row_type as type, r.row_id as id, @@ -321,27 +325,10 @@ class BucketStorage { -- Group for (3) GROUP BY r.row_type, r.row_id """); - try { - // TODO: Perhaps we don't need batching for this? - var cursor = stmt.selectCursor([]); - List rows = []; - while (cursor.moveNext()) { - var row = cursor.current; - rows.add(row); - - if (rows.length >= 10000) { - saveOps(db, rows); - rows = []; - } - } - if (rows.isNotEmpty) { - saveOps(db, rows); - } - } finally { - stmt.dispose(); - } - db.execute("""UPDATE ps_buckets + await saveOps(tx, opRows); + + await tx.execute("""UPDATE ps_buckets SET last_applied_op = last_op WHERE last_applied_op != last_op"""); @@ -351,7 +338,7 @@ class BucketStorage { } // { type: string; id: string; data: string; buckets: string; op_id: string }[] - void saveOps(sqlite.Database db, List rows) { + Future saveOps(SqliteWriteContext tx, List rows) async { Map> byType = {}; for (final row in rows) { byType.putIfAbsent(row['type'], () => []).add(row); @@ -360,7 +347,7 @@ class BucketStorage { for (final entry in byType.entries) { final type = entry.key; final typeRows = entry.value; - final table = _getTypeTableName(type); + final table = getTypeTableName(type); // Note that "PUT" and "DELETE" are split, and not applied in row order. // So we only do either PUT or DELETE for each individual object, not both. @@ -378,24 +365,24 @@ class BucketStorage { puts = puts.where((update) => !removeIds.contains(update['id'])).toList(); if (tableNames.contains(table)) { - db.execute("""REPLACE INTO "$table"(id, data) + await tx.execute("""REPLACE INTO "$table"(id, data) SELECT json_extract(json_each.value, '\$.id'), json_extract(json_each.value, '\$.data') FROM json_each(?)""", [jsonEncode(puts)]); - db.execute("""DELETE + await tx.execute("""DELETE FROM "$table" WHERE id IN (SELECT json_each.value FROM json_each(?))""", [ jsonEncode([...removeIds]) ]); } else { - db.execute(r"""REPLACE INTO ps_untyped(type, id, data) + await tx.execute(r"""REPLACE INTO ps_untyped(type, id, data) SELECT ?, json_extract(json_each.value, '$.id'), json_extract(json_each.value, '$.data') FROM json_each(?)""", [type, jsonEncode(puts)]); - db.execute("""DELETE FROM ps_untyped + await tx.execute("""DELETE FROM ps_untyped WHERE type = ? AND id IN (SELECT json_each.value FROM json_each(?))""", [type, jsonEncode(removeIds.toList())]); @@ -403,9 +390,10 @@ class BucketStorage { } } - bool _canUpdateLocal(sqlite.Database db) { - final invalidBuckets = db.select( - "SELECT name, target_op, last_op, last_applied_op FROM ps_buckets WHERE target_op > last_op AND (name = '\$local' OR pending_delete = 0)"); + @protected + Future canUpdateLocal(SqliteWriteContext tx) async { + final invalidBuckets = await tx.execute( + "SELECT name, CAST(target_op AS TEXT), last_op, last_applied_op FROM ps_buckets WHERE target_op > last_op AND (name = '\$local' OR pending_delete = 0)"); if (invalidBuckets.isNotEmpty) { if (invalidBuckets.first['name'] == '\$local') { isolateLogger.fine('Waiting for local changes to be acknowledged'); @@ -415,15 +403,16 @@ class BucketStorage { return false; } // This is specifically relevant for when data is added to crud before another batch is completed. - final rows = db.select('SELECT 1 FROM ps_crud LIMIT 1'); + final rows = await tx.execute('SELECT 1 FROM ps_crud LIMIT 1'); if (rows.isNotEmpty) { return false; } return true; } - SyncLocalDatabaseResult validateChecksums(Checkpoint checkpoint) { - final rows = select("""WITH + Future validateChecksums( + Checkpoint checkpoint) async { + final rows = await select("""WITH bucket_list(bucket, lower_op_id) AS ( SELECT json_extract(json_each.value, '\$.bucket') as bucket, @@ -547,8 +536,8 @@ class BucketStorage { // ignore: unused_element Future _compactWal() async { try { - await writeTransaction((db) { - db.select('PRAGMA wal_checkpoint(TRUNCATE)'); + await writeTransaction((tx) async { + await tx.execute('PRAGMA wal_checkpoint(TRUNCATE)'); }); } on sqlite.SqliteException catch (e) { // Ignore SQLITE_BUSY @@ -563,10 +552,10 @@ class BucketStorage { Future _deletePendingBuckets() async { if (_pendingBucketDeletes) { // Executed once after start-up, and again when there are pending deletes. - await writeTransaction((db) { - db.execute( + await writeTransaction((tx) async { + await tx.execute( 'DELETE FROM ps_oplog WHERE bucket IN (SELECT name FROM ps_buckets WHERE pending_delete = 1 AND last_applied_op = last_op AND last_op >= target_op)'); - db.execute( + await tx.execute( 'DELETE FROM ps_buckets WHERE pending_delete = 1 AND last_applied_op = last_op AND last_op >= target_op'); }); _pendingBucketDeletes = false; @@ -578,13 +567,13 @@ class BucketStorage { return; } - final rows = select( + final rows = await select( 'SELECT name, cast(last_applied_op as TEXT) as last_applied_op, cast(last_op as TEXT) as last_op FROM ps_buckets WHERE pending_delete = 0'); for (var row in rows) { - await writeTransaction((db) { + await writeTransaction((tx) async { // Note: The row values here may be different from when queried. That should not be an issue. - db.execute("""UPDATE ps_buckets AS buckets + await tx.execute("""UPDATE ps_buckets AS buckets SET add_checksum = add_checksum + (SELECT IFNULL(SUM(hash), 0) FROM ps_oplog AS oplog WHERE (superseded = 1 OR op != ${OpType.put.value}) @@ -592,7 +581,7 @@ class BucketStorage { AND oplog.op_id <= ?) WHERE buckets.name = ?""", [row['name'], row['last_applied_op'], row['name']]); - db.execute( + await tx.execute( """DELETE FROM ps_oplog WHERE (superseded = 1 OR op != ${OpType.put.value}) @@ -611,14 +600,14 @@ class BucketStorage { Future updateLocalTarget( Future Function() checkpointCallback) async { - final rs1 = select( - 'SELECT target_op FROM ps_buckets WHERE name = \'\$local\' AND target_op = $maxOpId'); + final rs1 = await select( + 'SELECT CAST(target_op AS TEXT) FROM ps_buckets WHERE name = \'\$local\' AND target_op = $maxOpId'); if (rs1.isEmpty) { // Nothing to update return false; } - final rs = - select('SELECT seq FROM sqlite_sequence WHERE name = \'ps_crud\''); + final rs = await select( + 'SELECT seq FROM sqlite_sequence WHERE name = \'ps_crud\''); if (rs.isEmpty) { // Nothing to update return false; @@ -626,13 +615,13 @@ class BucketStorage { int seqBefore = rs.first['seq']; var opId = await checkpointCallback(); - return await writeTransaction((tx) { - final anyData = tx.select('SELECT 1 FROM ps_crud LIMIT 1'); + return await writeTransaction((tx) async { + final anyData = await tx.execute('SELECT 1 FROM ps_crud LIMIT 1'); if (anyData.isNotEmpty) { return false; } - final rs = - tx.select('SELECT seq FROM sqlite_sequence WHERE name = \'ps_crud\''); + final rs = await tx + .execute('SELECT seq FROM sqlite_sequence WHERE name = \'ps_crud\''); assert(rs.isNotEmpty); int seqAfter = rs.first['seq']; @@ -641,26 +630,26 @@ class BucketStorage { return false; } - tx.select( + await tx.execute( "UPDATE ps_buckets SET target_op = ? WHERE name='\$local'", [opId]); return true; }); } - bool hasCrud() { - final anyData = select('SELECT 1 FROM ps_crud LIMIT 1'); + Future hasCrud() async { + final anyData = await select('SELECT 1 FROM ps_crud LIMIT 1'); return anyData.isNotEmpty; } /// For tests only. Others should use the version on PowerSyncDatabase. - CrudBatch? getCrudBatch({int limit = 100}) { - if (!hasCrud()) { + Future getCrudBatch({int limit = 100}) async { + if (!(await hasCrud())) { return null; } final rows = - select('SELECT * FROM ps_crud ORDER BY id ASC LIMIT ?', [limit]); + await select('SELECT * FROM ps_crud ORDER BY id ASC LIMIT ?', [limit]); List all = []; for (var row in rows) { all.add(CrudEntry.fromRow(row)); @@ -673,14 +662,15 @@ class BucketStorage { crud: all, haveMore: true, complete: ({String? writeCheckpoint}) async { - await writeTransaction((db) { - db.execute('DELETE FROM ps_crud WHERE id <= ?', [last.clientId]); + await writeTransaction((tx) async { + await tx + .execute('DELETE FROM ps_crud WHERE id <= ?', [last.clientId]); if (writeCheckpoint != null && - db.select('SELECT 1 FROM ps_crud LIMIT 1').isEmpty) { - db.execute( + (await tx.execute('SELECT 1 FROM ps_crud LIMIT 1')).isEmpty) { + await tx.execute( 'UPDATE ps_buckets SET target_op = $writeCheckpoint WHERE name=\'\$local\''); } else { - db.execute( + await tx.execute( 'UPDATE ps_buckets SET target_op = $maxOpId WHERE name=\'\$local\''); } }); @@ -692,12 +682,9 @@ class BucketStorage { /// is assumed that multiple functions on this instance won't be called /// concurrently. Future writeTransaction( - FutureOr Function(sqlite.Database tx) callback, + Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout}) async { - return mutex.lock(() async { - final r = await asyncDirectTransaction(_internalDb, callback); - return r; - }); + return _internalDb.writeTransaction(callback, lockTimeout: lockTimeout); } } @@ -875,7 +862,8 @@ enum OpType { /// The table name must always be enclosed in "quotes" when using inside a SQL query. /// /// @param type -String _getTypeTableName(String type) { +@protected +String getTypeTableName(String type) { // Test for invalid characters rather than escaping. if (invalidSqliteCharacters.hasMatch(type)) { throw AssertionError("Invalid characters in type name: $type"); diff --git a/packages/powersync/lib/src/connector.dart b/packages/powersync/lib/src/connector.dart index 2d4ec38b..07617861 100644 --- a/packages/powersync/lib/src/connector.dart +++ b/packages/powersync/lib/src/connector.dart @@ -1,9 +1,6 @@ import 'dart:convert'; -import 'dart:io'; - import 'package:http/http.dart' as http; - -import 'powersync_database.dart'; +import 'database/powersync_database.dart'; /// Implement this to connect an app backend. /// @@ -264,7 +261,7 @@ class DevConnector extends PowerSyncBackendConnector { token: parsed['data']['token'], userId: parsed['data']['user_id'])); } else { - throw HttpException(res.reasonPhrase ?? 'Request failed', uri: uri); + throw http.ClientException(res.reasonPhrase ?? 'Request failed', uri); } } @@ -282,7 +279,7 @@ class DevConnector extends PowerSyncBackendConnector { clearDevToken(); } if (res.statusCode != 200) { - throw HttpException(res.reasonPhrase ?? 'Request failed', uri: uri); + throw http.ClientException(res.reasonPhrase ?? 'Request failed', uri); } return PowerSyncCredentials.fromJson(jsonDecode(res.body)['data']); @@ -316,9 +313,8 @@ class DevConnector extends PowerSyncBackendConnector { } if (response.statusCode != 200) { - throw HttpException( - response.reasonPhrase ?? "Failed due to server error.", - uri: uri); + throw http.ClientException( + response.reasonPhrase ?? "Failed due to server error.", uri); } final body = jsonDecode(response.body); diff --git a/packages/powersync/lib/src/crud.dart b/packages/powersync/lib/src/crud.dart index 10785280..66ad699d 100644 --- a/packages/powersync/lib/src/crud.dart +++ b/packages/powersync/lib/src/crud.dart @@ -1,7 +1,7 @@ import 'dart:convert'; import 'package:collection/collection.dart'; -import 'package:sqlite_async/sqlite3.dart' as sqlite; +import 'package:sqlite_async/sqlite3_common.dart' as sqlite; /// A batch of client-side changes. class CrudBatch { diff --git a/packages/powersync/lib/src/database/native/native_bucket_storage.dart b/packages/powersync/lib/src/database/native/native_bucket_storage.dart new file mode 100644 index 00000000..d2ecd10a --- /dev/null +++ b/packages/powersync/lib/src/database/native/native_bucket_storage.dart @@ -0,0 +1,150 @@ +import 'dart:convert'; + +import 'package:powersync/sqlite3_common.dart'; +import 'package:powersync/src/log_internal.dart'; +import 'package:powersync/src/sync_types.dart'; +import 'package:sqlite_async/sqlite3.dart' as sqlite; + +import 'package:powersync/src/bucket_storage.dart'; + +/// Native implementation for [BucketStorage] +/// Uses direct SQLite3 connection for memory and performance +/// optimizations. +class NativeBucketStorage extends BucketStorage { + NativeBucketStorage(super.db); + + @override + + /// Native specific version for updating objects from buckets + /// this uses the SQLite3 database directly for better memory usage. + Future updateObjectsFromBuckets(Checkpoint checkpoint) async { + // Internal connection is private, but can be accessed via a transaction + return writeTransaction((tx) { + return tx.computeWithDatabase((db) async { + if (!(await canUpdateLocal(tx))) { + return false; + } + + // Updated objects + // TODO: Reduce memory usage + // Some options here: + // 1. Create a VIEW objects_updates, which contains triggers to update individual tables. + // This works well for individual tables, but difficult to have a catch all for untyped data, + // and performance degrades when we have hundreds of object types. + // 2. Similar, but use a TEMP TABLE instead. We can then query those from JS, and populate the tables from JS. + // 3. Skip the temp table, and query the data directly. Sorting and limiting becomes tricky. + // 3a. LIMIT on the first oplog step. This prevents us from using JOIN after this. + // 3b. LIMIT after the second oplog query + + // QUERY PLAN + // |--SCAN buckets + // |--SEARCH b USING INDEX ps_oplog_by_opid (bucket=? AND op_id>?) + // |--SEARCH r USING INDEX ps_oplog_by_row (row_type=? AND row_id=?) + // `--USE TEMP B-TREE FOR GROUP BY + // language=DbSqlite + var stmt = db.prepare( + """-- 3. Group the objects from different buckets together into a single one (ops). + SELECT r.row_type as type, + r.row_id as id, + r.data as data, + json_group_array(r.bucket) FILTER (WHERE r.op=${OpType.put.value}) as buckets, + /* max() affects which row is used for 'data' */ + max(r.op_id) FILTER (WHERE r.op=${OpType.put.value}) as op_id + -- 1. Filter oplog by the ops added but not applied yet (oplog b). + FROM ps_buckets AS buckets + CROSS JOIN ps_oplog AS b ON b.bucket = buckets.name + AND (b.op_id > buckets.last_applied_op) + -- 2. Find *all* current ops over different buckets for those objects (oplog r). + INNER JOIN ps_oplog AS r + ON r.row_type = b.row_type + AND r.row_id = b.row_id + WHERE r.superseded = 0 + AND b.superseded = 0 + -- Group for (3) + GROUP BY r.row_type, r.row_id + """); + try { + // TODO: Perhaps we don't need batching for this? + var cursor = stmt.selectCursor([]); + List rows = []; + while (cursor.moveNext()) { + var row = cursor.current; + rows.add(row); + + if (rows.length >= 10000) { + _saveOps(db, rows); + rows = []; + } + } + if (rows.isNotEmpty) { + _saveOps(db, rows); + } + } finally { + stmt.dispose(); + } + + db.execute("""UPDATE ps_buckets + SET last_applied_op = last_op + WHERE last_applied_op != last_op"""); + + isolateLogger.fine('Applied checkpoint ${checkpoint.lastOpId}'); + return true; + }); + }); + } + + /// Native specific version of saveOps which operates directly + /// on the SQLite3 connection + /// { type: string; id: string; data: string; buckets: string; op_id: string }[] + void _saveOps(CommonDatabase db, List rows) { + Map> byType = {}; + for (final row in rows) { + byType.putIfAbsent(row['type'], () => []).add(row); + } + + for (final entry in byType.entries) { + final type = entry.key; + final typeRows = entry.value; + final table = getTypeTableName(type); + + // Note that "PUT" and "DELETE" are split, and not applied in row order. + // So we only do either PUT or DELETE for each individual object, not both. + final Set removeIds = {}; + List puts = []; + for (final row in typeRows) { + if (row['buckets'] == '[]') { + removeIds.add(row['id']); + } else { + puts.add(row); + removeIds.remove(row['id']); + } + } + + puts = puts.where((update) => !removeIds.contains(update['id'])).toList(); + + if (tableNames.contains(table)) { + db.execute("""REPLACE INTO "$table"(id, data) + SELECT json_extract(json_each.value, '\$.id'), + json_extract(json_each.value, '\$.data') + FROM json_each(?)""", [jsonEncode(puts)]); + + db.execute("""DELETE + FROM "$table" + WHERE id IN (SELECT json_each.value FROM json_each(?))""", [ + jsonEncode([...removeIds]) + ]); + } else { + db.execute(r"""REPLACE INTO ps_untyped(type, id, data) + SELECT ?, + json_extract(json_each.value, '$.id'), + json_extract(json_each.value, '$.data') + FROM json_each(?)""", [type, jsonEncode(puts)]); + + db.execute("""DELETE FROM ps_untyped + WHERE type = ? + AND id IN (SELECT json_each.value FROM json_each(?))""", + [type, jsonEncode(removeIds.toList())]); + } + } + } +} diff --git a/packages/powersync/lib/src/database/native/native_powersync_database.dart b/packages/powersync/lib/src/database/native/native_powersync_database.dart new file mode 100644 index 00000000..531e2dcc --- /dev/null +++ b/packages/powersync/lib/src/database/native/native_powersync_database.dart @@ -0,0 +1,366 @@ +import 'dart:async'; +import 'dart:isolate'; +import 'package:meta/meta.dart'; + +import 'package:http/http.dart' as http; +import 'package:logging/logging.dart'; +import 'package:powersync/src/abort_controller.dart'; +import 'package:powersync/src/connector.dart'; +import 'package:powersync/src/database/native/native_bucket_storage.dart'; +import 'package:powersync/src/database/powersync_database.dart'; +import 'package:powersync/src/database/powersync_db_mixin.dart'; +import 'package:powersync/src/isolate_completer.dart'; +import 'package:powersync/src/log.dart'; +import 'package:powersync/src/log_internal.dart'; +import 'package:powersync/src/open_factory/abstract_powersync_open_factory.dart'; +import 'package:powersync/src/open_factory/native/native_open_factory.dart'; +import 'package:powersync/src/schema.dart'; +import 'package:powersync/src/schema_logic.dart'; +import 'package:powersync/src/streaming_sync.dart'; +import 'package:powersync/src/sync_status.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; +import 'package:sqlite_async/sqlite_async.dart'; + +/// A PowerSync managed database. +/// +///Native implementation for [PowerSyncDatabase] +/// +/// Use one instance per database file. +/// +/// Use [PowerSyncDatabase.connect] to connect to the PowerSync service, +/// to keep the local database in sync with the remote database. +/// +/// All changes to local tables are automatically recorded, whether connected +/// or not. Once connected, the changes are uploaded. +class PowerSyncDatabaseImpl + with SqliteQueries, PowerSyncDatabaseMixin + implements PowerSyncDatabase { + @override + Schema schema; + + @override + SqliteDatabase database; + + @override + @protected + late Future isInitialized; + + @override + + /// The Logger used by this [PowerSyncDatabase]. + /// + /// The default is [autoLogger], which logs to the console in debug builds. + /// Use [debugLogger] to always log to the console. + /// Use [attachedLogger] to propagate logs to [Logger.root] for custom logging. + late final Logger logger; + + /// Open a [PowerSyncDatabase]. + /// + /// Only a single [PowerSyncDatabase] per [path] should be opened at a time. + /// + /// The specified [schema] is used for the database. + /// + /// A connection pool is used by default, allowing multiple concurrent read + /// transactions, and a single concurrent write transaction. Write transactions + /// do not block read transactions, and read transactions will see the state + /// from the last committed write transaction. + /// + /// A maximum of [maxReaders] concurrent read transactions are allowed. + /// + /// [logger] defaults to [autoLogger], which logs to the console in debug builds. + factory PowerSyncDatabaseImpl( + {required Schema schema, + required String path, + int maxReaders = SqliteDatabase.defaultMaxReaders, + Logger? logger, + @Deprecated("Use [PowerSyncDatabase.withFactory] instead") + // ignore: deprecated_member_use_from_same_package + SqliteConnectionSetup? sqliteSetup}) { + // ignore: deprecated_member_use_from_same_package + DefaultSqliteOpenFactory factory = + // ignore: deprecated_member_use_from_same_package + PowerSyncOpenFactory(path: path, sqliteSetup: sqliteSetup); + return PowerSyncDatabaseImpl.withFactory(factory, + schema: schema, logger: logger); + } + + /// Open a [PowerSyncDatabase] with a [PowerSyncOpenFactory]. + /// + /// The factory determines which database file is opened, as well as any + /// additional logic to run inside the database isolate before or after opening. + /// + /// Subclass [PowerSyncOpenFactory] to add custom logic to this process. + /// + /// [logger] defaults to [autoLogger], which logs to the console in debug builds. + factory PowerSyncDatabaseImpl.withFactory( + DefaultSqliteOpenFactory openFactory, + {required Schema schema, + int maxReaders = SqliteDatabase.defaultMaxReaders, + Logger? logger}) { + final db = SqliteDatabase.withFactory(openFactory, maxReaders: maxReaders); + return PowerSyncDatabaseImpl.withDatabase( + schema: schema, database: db, logger: logger); + } + + /// Open a PowerSyncDatabase on an existing [SqliteDatabase]. + /// + /// Migrations are run on the database when this constructor is called. + /// + /// [logger] defaults to [autoLogger], which logs to the console in debug builds.s + PowerSyncDatabaseImpl.withDatabase( + {required this.schema, required this.database, Logger? logger}) { + if (logger != null) { + this.logger = logger; + } else { + this.logger = autoLogger; + } + isInitialized = baseInit(); + } + + @override + + /// Connect to the PowerSync service, and keep the databases in sync. + /// + /// The connection is automatically re-opened if it fails for any reason. + /// + /// Status changes are reported on [statusStream]. + connect({required PowerSyncBackendConnector connector}) async { + await initialize(); + + // Disconnect if connected + await disconnect(); + disconnecter = AbortController(); + + await isInitialized; + final dbref = database.isolateConnectionFactory(); + ReceivePort rPort = ReceivePort(); + StreamSubscription? updateSubscription; + rPort.listen((data) async { + if (data is List) { + String action = data[0]; + if (action == "getCredentials") { + await (data[1] as PortCompleter).handle(() async { + final token = await connector.getCredentialsCached(); + logger.fine('Credentials: $token'); + return token; + }); + } else if (action == "invalidateCredentials") { + logger.fine('Refreshing credentials'); + await (data[1] as PortCompleter).handle(() async { + await connector.prefetchCredentials(); + }); + } else if (action == 'init') { + SendPort port = data[1]; + var throttled = UpdateNotification.throttleStream( + updates, const Duration(milliseconds: 10)); + updateSubscription = throttled.listen((event) { + port.send(['update']); + }); + disconnecter?.onAbort.then((_) { + port.send(['close']); + }).ignore(); + } else if (action == 'uploadCrud') { + await (data[1] as PortCompleter).handle(() async { + await connector.uploadData(this); + }); + } else if (action == 'status') { + final SyncStatus status = data[1]; + setStatus(status); + } else if (action == 'close') { + setStatus(SyncStatus( + connected: false, lastSyncedAt: currentStatus.lastSyncedAt)); + rPort.close(); + updateSubscription?.cancel(); + } else if (action == 'log') { + LogRecord record = data[1]; + logger.log( + record.level, record.message, record.error, record.stackTrace); + } + } + }); + + var errorPort = ReceivePort(); + errorPort.listen((message) async { + // Sample error: + // flutter: [PowerSync] WARNING: 2023-06-28 16:34:11.566122: Sync Isolate error + // flutter: [Connection closed while receiving data, #0 IOClient.send. (package:http/src/io_client.dart:76:13) + // #1 Stream.handleError. (dart:async/stream.dart:929:16) + // #2 _HandleErrorStream._handleError (dart:async/stream_pipe.dart:269:17) + // #3 _ForwardingStreamSubscription._handleError (dart:async/stream_pipe.dart:157:13) + // #4 _HttpClientResponse.listen. (dart:_http/http_impl.dart:707:16) + // ... + logger.severe('Sync Isolate error', message); + + // Reconnect + connect(connector: connector); + }); + + disconnected() { + disconnecter?.completeAbort(); + disconnecter = null; + rPort.close(); + // Clear status apart from lastSyncedAt + setStatus(SyncStatus(lastSyncedAt: currentStatus.lastSyncedAt)); + } + + var exitPort = ReceivePort(); + exitPort.listen((message) { + logger.fine('Sync Isolate exit'); + disconnected(); + }); + + if (disconnecter?.aborted == true) { + disconnected(); + return; + } + + Isolate.spawn(_powerSyncDatabaseIsolate, + _PowerSyncDatabaseIsolateArgs(rPort.sendPort, dbref, retryDelay), + debugName: 'PowerSyncDatabase', + onError: errorPort.sendPort, + onExit: exitPort.sendPort); + } + + /// Takes a read lock, without starting a transaction. + /// + /// In most cases, [readTransaction] should be used instead. + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {String? debugContext, Duration? lockTimeout}) async { + await isInitialized; + return database.readLock(callback, + debugContext: debugContext, lockTimeout: lockTimeout); + } + + /// Takes a global lock, without starting a transaction. + /// + /// In most cases, [writeTransaction] should be used instead. + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {String? debugContext, Duration? lockTimeout}) async { + await isInitialized; + return database.writeLock(callback, + debugContext: debugContext, lockTimeout: lockTimeout); + } + + @override + Future updateSchema(Schema schema) { + if (disconnecter != null) { + throw AssertionError('Cannot update schema while connected'); + } + this.schema = schema; + return updateSchemaInIsolate(database, schema); + } +} + +class _PowerSyncDatabaseIsolateArgs { + final SendPort sPort; + final IsolateConnectionFactory dbRef; + final Duration retryDelay; + + _PowerSyncDatabaseIsolateArgs(this.sPort, this.dbRef, this.retryDelay); +} + +Future _powerSyncDatabaseIsolate( + _PowerSyncDatabaseIsolateArgs args) async { + final sPort = args.sPort; + ReceivePort rPort = ReceivePort(); + StreamController updateController = StreamController.broadcast(); + final upstreamDbClient = args.dbRef.upstreamPort.open(); + + CommonDatabase? db; + final mutex = args.dbRef.mutex.open(); + + rPort.listen((message) async { + if (message is List) { + String action = message[0]; + if (action == 'update') { + updateController.add('update'); + } else if (action == 'close') { + // This prevents any further transactions being opened, which would + // eventually terminate the sync loop. + await mutex.close(); + db?.dispose(); + db = null; + updateController.close(); + // upstreamDbClient.close(); + Isolate.current.kill(); + } + } + }); + Isolate.current.addOnExitListener(sPort, response: const ['close']); + sPort.send(["init", rPort.sendPort]); + + // Is there a way to avoid the overhead if logging is not enabled? + // This only takes effect in this isolate. + isolateLogger.level = Level.ALL; + isolateLogger.onRecord.listen((record) { + var copy = LogRecord(record.level, record.message, record.loggerName, + record.error, record.stackTrace); + sPort.send(["log", copy]); + }); + + Future loadCredentials() async { + final r = IsolateResult(); + sPort.send(["getCredentials", r.completer]); + return r.future; + } + + Future invalidateCredentials() async { + final r = IsolateResult(); + sPort.send(["invalidateCredentials", r.completer]); + return r.future; + } + + Future uploadCrud() async { + final r = IsolateResult(); + sPort.send(["uploadCrud", r.completer]); + return r.future; + } + + runZonedGuarded(() async { + db = await args.dbRef.openFactory + .open(SqliteOpenOptions(primaryConnection: false, readOnly: false)); + final connection = SyncSqliteConnection(db!, mutex); + + final storage = NativeBucketStorage(connection); + final sync = StreamingSyncImplementation( + adapter: storage, + credentialsCallback: loadCredentials, + invalidCredentialsCallback: invalidateCredentials, + uploadCrud: uploadCrud, + updateStream: updateController.stream, + retryDelay: args.retryDelay, + client: http.Client()); + sync.streamingSync(); + sync.statusStream.listen((event) { + sPort.send(['status', event]); + }); + + Timer? updateDebouncer; + Set updatedTables = {}; + + void maybeFireUpdates() { + if (updatedTables.isNotEmpty) { + upstreamDbClient.fire(UpdateNotification(updatedTables)); + updatedTables.clear(); + updateDebouncer?.cancel(); + updateDebouncer = null; + } + } + + db!.updates.listen((event) { + updatedTables.add(event.tableName); + + updateDebouncer ??= + Timer(const Duration(milliseconds: 10), maybeFireUpdates); + }); + }, (error, stack) { + // Properly dispose the database if an uncaught error occurs. + // Unfortunately, this does not handle disposing while the database is opening. + // This should be rare - any uncaught error is a bug. And in most cases, + // it should occur after the database is already open. + db?.dispose(); + throw error; + }); +} diff --git a/packages/powersync/lib/src/database/powersync_database.dart b/packages/powersync/lib/src/database/powersync_database.dart new file mode 100644 index 00000000..52b4976c --- /dev/null +++ b/packages/powersync/lib/src/database/powersync_database.dart @@ -0,0 +1,67 @@ +import 'package:logging/logging.dart'; +import 'package:powersync/src/database/powersync_database_impl.dart'; +import 'package:powersync/src/database/powersync_db_mixin.dart'; +import 'package:sqlite_async/sqlite_async.dart'; + +import '../schema.dart'; + +/// A PowerSync managed database. +/// +/// Use one instance per database file. +/// +/// Use [PowerSyncDatabase.connect] to connect to the PowerSync service, +/// to keep the local database in sync with the remote database. +/// +/// All changes to local tables are automatically recorded, whether connected +/// or not. Once connected, the changes are uploaded. +abstract class PowerSyncDatabase + with SqliteQueries, PowerSyncDatabaseMixin + implements SqliteConnection { + /// Open a [PowerSyncDatabase]. + /// + /// Only a single [PowerSyncDatabase] per [path] should be opened at a time. + /// + /// The specified [schema] is used for the database. + /// + /// A connection pool is used by default, allowing multiple concurrent read + /// transactions, and a single concurrent write transaction. Write transactions + /// do not block read transactions, and read transactions will see the state + /// from the last committed write transaction. + /// + /// A maximum of [maxReaders] concurrent read transactions are allowed. + /// + /// [logger] defaults to [autoLogger], which logs to the console in debug builds. + factory PowerSyncDatabase( + {required Schema schema, required String path, Logger? logger}) { + return PowerSyncDatabaseImpl(schema: schema, path: path, logger: logger); + } + + /// Open a [PowerSyncDatabase] with a [PowerSyncOpenFactory]. + /// + /// The factory determines which database file is opened, as well as any + /// additional logic to run inside the database isolate before or after opening. + /// + /// Subclass [PowerSyncOpenFactory] to add custom logic to this process. + /// + /// [logger] defaults to [autoLogger], which logs to the console in debug builds. + factory PowerSyncDatabase.withFactory(DefaultSqliteOpenFactory openFactory, + {required Schema schema, + int maxReaders = SqliteDatabase.defaultMaxReaders, + Logger? logger}) { + return PowerSyncDatabaseImpl.withFactory(openFactory, + schema: schema, logger: logger); + } + + /// Open a PowerSyncDatabase on an existing [SqliteDatabase]. + /// + /// Migrations are run on the database when this constructor is called. + /// + /// [logger] defaults to [autoLogger], which logs to the console in debug builds. + factory PowerSyncDatabase.withDatabase( + {required Schema schema, + required SqliteDatabase database, + Logger? loggers}) { + return PowerSyncDatabaseImpl.withDatabase( + schema: schema, database: database); + } +} diff --git a/packages/powersync/lib/src/database/powersync_database_impl.dart b/packages/powersync/lib/src/database/powersync_database_impl.dart new file mode 100644 index 00000000..7c98975f --- /dev/null +++ b/packages/powersync/lib/src/database/powersync_database_impl.dart @@ -0,0 +1,9 @@ +// This follows the pattern from here: https://stackoverflow.com/questions/58710226/how-to-import-platform-specific-dependency-in-flutter-dart-combine-web-with-an +// To conditionally export an implementation for either web or "native" platforms +// The sqlite library uses dart:ffi which is not supported on web + +export 'powersync_database_impl_stub.dart' + // ignore: uri_does_not_exist + if (dart.library.io) './native/native_powersync_database.dart' + // ignore: uri_does_not_exist + if (dart.library.html) './web/web_powersync_database.dart'; diff --git a/packages/powersync/lib/src/database/powersync_database_impl_stub.dart b/packages/powersync/lib/src/database/powersync_database_impl_stub.dart new file mode 100644 index 00000000..13906171 --- /dev/null +++ b/packages/powersync/lib/src/database/powersync_database_impl_stub.dart @@ -0,0 +1,115 @@ +import 'dart:async'; + +import 'package:logging/logging.dart'; +import 'package:powersync/sqlite_async.dart'; +import 'package:powersync/src/database/powersync_db_mixin.dart'; +import 'package:powersync/src/open_factory/abstract_powersync_open_factory.dart'; +import 'powersync_database.dart'; + +import '../connector.dart'; +import '../schema.dart'; + +class PowerSyncDatabaseImpl + with SqliteQueries, PowerSyncDatabaseMixin + implements PowerSyncDatabase { + @override + Future close() { + throw UnimplementedError(); + } + + @override + bool get closed => throw UnimplementedError(); + + @override + Schema get schema => throw UnimplementedError(); + + @override + SqliteDatabase get database => throw UnimplementedError(); + + @override + Future get isInitialized => throw UnimplementedError(); + + /// Open a [PowerSyncDatabase]. + /// + /// Only a single [PowerSyncDatabase] per [path] should be opened at a time. + /// + /// The specified [schema] is used for the database. + /// + /// A connection pool is used by default, allowing multiple concurrent read + /// transactions, and a single concurrent write transaction. Write transactions + /// do not block read transactions, and read transactions will see the state + /// from the last committed write transaction. + /// + /// A maximum of [maxReaders] concurrent read transactions are allowed. + /// + /// [logger] defaults to [autoLogger], which logs to the console in debug builds. + factory PowerSyncDatabaseImpl( + {required Schema schema, + required String path, + int maxReaders = SqliteDatabase.defaultMaxReaders, + Logger? logger, + @Deprecated("Use [PowerSyncDatabase.withFactory] instead") + // ignore: deprecated_member_use_from_same_package + SqliteConnectionSetup? sqliteSetup}) { + throw UnimplementedError(); + } + + /// Open a [PowerSyncDatabase] with a [PowerSyncOpenFactory]. + /// + /// The factory determines which database file is opened, as well as any + /// additional logic to run inside the database isolate before or after opening. + /// + /// Subclass [PowerSyncOpenFactory] to add custom logic to this process. + /// + /// [logger] defaults to [autoLogger], which logs to the console in debug builds.s + factory PowerSyncDatabaseImpl.withFactory( + DefaultSqliteOpenFactory openFactory, { + required Schema schema, + int maxReaders = SqliteDatabase.defaultMaxReaders, + Logger? logger, + }) { + throw UnimplementedError(); + } + + /// Open a PowerSyncDatabase on an existing [SqliteDatabase]. + /// + /// Migrations are run on the database when this constructor is called. + /// + /// [logger] defaults to [autoLogger], which logs to the console in debug builds.s + factory PowerSyncDatabaseImpl.withDatabase( + {required Schema schema, + required SqliteDatabase database, + Logger? loggers}) { + throw UnimplementedError(); + } + + @override + connect({required PowerSyncBackendConnector connector}) { + throw UnimplementedError(); + } + + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {String? debugContext, Duration? lockTimeout}) { + throw UnimplementedError(); + } + + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {String? debugContext, Duration? lockTimeout}) { + throw UnimplementedError(); + } + + @override + Future getAutoCommit() { + throw UnimplementedError(); + } + + @override + Future updateSchema(Schema schema) { + throw UnimplementedError(); + } + + @override + Logger get logger => throw UnimplementedError(); +} diff --git a/packages/powersync/lib/src/database/powersync_db_mixin.dart b/packages/powersync/lib/src/database/powersync_db_mixin.dart new file mode 100644 index 00000000..c178fe67 --- /dev/null +++ b/packages/powersync/lib/src/database/powersync_db_mixin.dart @@ -0,0 +1,345 @@ +import 'dart:async'; + +import 'package:logging/logging.dart'; +import 'package:meta/meta.dart'; +import 'package:powersync/sqlite_async.dart'; +import 'package:powersync/src/abort_controller.dart'; +import 'package:powersync/src/connector.dart'; +import 'package:powersync/src/crud.dart'; +import 'package:powersync/src/database_utils.dart'; +import 'package:powersync/src/migrations.dart'; +import 'package:powersync/src/powersync_update_notification.dart'; +import 'package:powersync/src/schema.dart'; +import 'package:powersync/src/schema_helpers.dart'; +import 'package:powersync/src/sync_status.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; + +mixin PowerSyncDatabaseMixin implements SqliteConnection { + /// Schema used for the local database. + Schema get schema; + + /// The underlying database. + /// + /// For the most part, behavior is the same whether querying on the underlying + /// database, or on [PowerSyncDatabase]. The main difference is in update notifications: + /// the underlying database reports updates to the underlying tables, while + /// [PowerSyncDatabase] reports updates to the higher-level views. + SqliteDatabase get database; + + /// The Logger used by this [PowerSyncDatabase]. + /// + /// The default is [autoLogger], which logs to the console in debug builds. + /// Use [debugLogger] to always log to the console. + /// Use [attachedLogger] to propagate logs to [Logger.root] for custom logging. + Logger get logger; + + /// Current connection status. + SyncStatus currentStatus = + const SyncStatus(connected: false, lastSyncedAt: null); + + /// Use this stream to subscribe to connection status updates. + late final Stream statusStream; + + @protected + StreamController statusStreamController = + StreamController.broadcast(); + + /// Broadcast stream that is notified of any table updates. + /// + /// Unlike in [SqliteDatabase.updates], the tables reported here are the + /// higher-level views as defined in the [Schema], and exclude the low-level + /// PowerSync tables. + late final Stream updates; + + /// Delay between retrying failed requests. + /// Defaults to 5 seconds. + /// Only has an effect if changed before calling [connect]. + Duration retryDelay = const Duration(seconds: 5); + + @protected + Future get isInitialized; + + /// null when disconnected, present when connecting or connected + @protected + AbortController? disconnecter; + + @protected + Future baseInit() async { + statusStream = statusStreamController.stream; + updates = database.updates + .map((update) => + PowerSyncUpdateNotification.fromUpdateNotification(update)) + .where((update) => update.isNotEmpty) + .cast(); + + await database.initialize(); + await migrations.migrate(database); + await updateSchema(schema); + } + + /// Wait for initialization to complete. + /// + /// While initializing is automatic, this helps to catch and report initialization errors. + Future initialize() { + return isInitialized; + } + + @protected + void setStatus(SyncStatus status) { + if (status != currentStatus) { + currentStatus = status; + statusStreamController.add(status); + } + } + + @override + bool get closed { + return database.closed; + } + + /// Close the database, releasing resources. + /// + /// Also [disconnect]s any active connection. + /// + /// Once close is called, this connection cannot be used again - a new one + /// must be constructed. + @override + Future close() async { + // Don't close in the middle of the initialization process. + await isInitialized; + // Disconnect any active sync connection. + await disconnect(); + // Now we can close the database + await database.close(); + } + + /// Connect to the PowerSync service, and keep the databases in sync. + /// + /// The connection is automatically re-opened if it fails for any reason. + /// + /// Status changes are reported on [statusStream]. + Future connect({required PowerSyncBackendConnector connector}); + + /// Close the sync connection. + /// + /// Use [connect] to connect again. + Future disconnect() async { + if (disconnecter != null) { + await disconnecter!.abort(); + } + setStatus( + SyncStatus(connected: false, lastSyncedAt: currentStatus.lastSyncedAt)); + } + + /// Disconnect and clear the database. + /// + /// Use this when logging out. + /// + /// The database can still be queried after this is called, but the tables + /// would be empty. + /// + /// To preserve data in local-only tables, set [clearLocal] to false. + Future disconnectAndClear({bool clearLocal = true}) async { + await disconnect(); + + await writeTransaction((tx) async { + await tx.execute('DELETE FROM ps_oplog'); + await tx.execute('DELETE FROM ps_crud'); + await tx.execute('DELETE FROM ps_buckets'); + + final tableGlob = clearLocal ? 'ps_data_*' : 'ps_data__*'; + final existingTableRows = await tx.getAll( + "SELECT name FROM sqlite_master WHERE type='table' AND name GLOB ?", + [tableGlob]); + + for (var row in existingTableRows) { + await tx.execute('DELETE FROM ${quoteIdentifier(row['name'])}'); + } + }); + } + + @Deprecated('Use [disconnectAndClear] instead.') + Future disconnectedAndClear() async { + await disconnectAndClear(); + } + + /// Whether a connection to the PowerSync service is currently open. + bool get connected { + return currentStatus.connected; + } + + /// Replace the schema with a new version. + /// This is for advanced use cases - typically the schema should just be + /// specified once in the constructor. + /// + /// Cannot be used while connected - this should only be called before [connect]. + Future updateSchema(Schema schema); + + /// A connection factory that can be passed to different isolates. + /// + /// Use this to access the database in background isolates. + isolateConnectionFactory() { + return database.isolateConnectionFactory(); + } + + /// Get upload queue size estimate and count. + Future getUploadQueueStats( + {bool includeSize = false}) async { + if (includeSize) { + final row = await getOptional( + 'SELECT SUM(cast(data as blob) + 20) as size, count(*) as count FROM ps_crud'); + return UploadQueueStats( + count: row?['count'] ?? 0, size: row?['size'] ?? 0); + } else { + final row = await getOptional('SELECT count(*) as count FROM ps_crud'); + return UploadQueueStats(count: row?['count'] ?? 0); + } + } + + /// Get a batch of crud data to upload. + /// + /// Returns null if there is no data to upload. + /// + /// Use this from the [PowerSyncBackendConnector.uploadData]` callback. + /// + /// Once the data have been successfully uploaded, call [CrudBatch.complete] before + /// requesting the next batch. + /// + /// Use [limit] to specify the maximum number of updates to return in a single + /// batch. + /// + /// This method does include transaction ids in the result, but does not group + /// data by transaction. One batch may contain data from multiple transactions, + /// and a single transaction may be split over multiple batches. + Future getCrudBatch({limit = 100}) async { + final rows = await getAll( + 'SELECT id, tx_id, data FROM ps_crud ORDER BY id ASC LIMIT ?', + [limit + 1]); + List all = [for (var row in rows) CrudEntry.fromRow(row)]; + + var haveMore = false; + if (all.length > limit) { + all.removeLast(); + haveMore = true; + } + if (all.isEmpty) { + return null; + } + final last = all[all.length - 1]; + return CrudBatch( + crud: all, + haveMore: haveMore, + complete: ({String? writeCheckpoint}) async { + await writeTransaction((db) async { + await db + .execute('DELETE FROM ps_crud WHERE id <= ?', [last.clientId]); + if (writeCheckpoint != null && + await db.getOptional('SELECT 1 FROM ps_crud LIMIT 1') == null) { + await db.execute( + 'UPDATE ps_buckets SET target_op = $writeCheckpoint WHERE name=\'\$local\''); + } else { + await db.execute( + 'UPDATE ps_buckets SET target_op = $maxOpId WHERE name=\'\$local\''); + } + }); + }); + } + + /// Get the next recorded transaction to upload. + /// + /// Returns null if there is no data to upload. + /// + /// Use this from the [PowerSyncBackendConnector.uploadData]` callback. + /// + /// Once the data have been successfully uploaded, call [CrudTransaction.complete] before + /// requesting the next transaction. + /// + /// Unlike [getCrudBatch], this only returns data from a single transaction at a time. + /// All data for the transaction is loaded into memory. + Future getNextCrudTransaction() async { + return await readTransaction((tx) async { + final first = await tx.getOptional( + 'SELECT id, tx_id, data FROM ps_crud ORDER BY id ASC LIMIT 1'); + if (first == null) { + return null; + } + + final int? txId = first['tx_id']; + List all; + if (txId == null) { + all = [CrudEntry.fromRow(first)]; + } else { + final rows = await tx.getAll( + 'SELECT id, tx_id, data FROM ps_crud WHERE tx_id = ? ORDER BY id ASC', + [txId]); + all = [for (var row in rows) CrudEntry.fromRow(row)]; + } + + final last = all[all.length - 1]; + + return CrudTransaction( + transactionId: txId, + crud: all, + complete: ({String? writeCheckpoint}) async { + await writeTransaction((db) async { + await db.execute( + 'DELETE FROM ps_crud WHERE id <= ?', [last.clientId]); + if (writeCheckpoint != null && + await db.getOptional('SELECT 1 FROM ps_crud LIMIT 1') == + null) { + await db.execute( + 'UPDATE ps_buckets SET target_op = $writeCheckpoint WHERE name=\'\$local\''); + } else { + await db.execute( + 'UPDATE ps_buckets SET target_op = $maxOpId WHERE name=\'\$local\''); + } + }); + }); + }); + } + + /// Takes a read lock, without starting a transaction. + /// + /// In most cases, [readTransaction] should be used instead. + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {String? debugContext, Duration? lockTimeout}); + + /// Takes a global lock, without starting a transaction. + /// + /// In most cases, [writeTransaction] should be used instead. + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {String? debugContext, Duration? lockTimeout}); + + @override + Future writeTransaction( + Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, + String? debugContext}) async { + return writeLock((ctx) async { + return await internalTrackedWriteTransaction(ctx, callback); + }, + lockTimeout: lockTimeout, + debugContext: debugContext ?? 'writeTransaction()'); + } + + @override + Future execute(String sql, + [List parameters = const []]) async { + return writeLock((ctx) async { + try { + await ctx.execute( + 'UPDATE ps_tx SET current_tx = next_tx, next_tx = next_tx + 1 WHERE id = 1'); + return await ctx.execute(sql, parameters); + } finally { + await ctx.execute('UPDATE ps_tx SET current_tx = NULL WHERE id = 1'); + } + }, debugContext: 'execute()'); + } + + @override + Future getAutoCommit() { + return database.getAutoCommit(); + } +} diff --git a/packages/powersync/lib/src/database/web/web_powersync_database.dart b/packages/powersync/lib/src/database/web/web_powersync_database.dart new file mode 100644 index 00000000..4e3844a2 --- /dev/null +++ b/packages/powersync/lib/src/database/web/web_powersync_database.dart @@ -0,0 +1,205 @@ +import 'dart:async'; +import 'package:meta/meta.dart'; +import 'package:fetch_client/fetch_client.dart'; +import 'package:logging/logging.dart'; +import 'package:powersync/src/abort_controller.dart'; +import 'package:powersync/src/bucket_storage.dart'; +import 'package:powersync/src/connector.dart'; +import 'package:powersync/src/database/powersync_database.dart'; +import 'package:powersync/src/database/powersync_db_mixin.dart'; +import 'package:powersync/src/database_utils.dart'; +import 'package:powersync/src/log.dart'; +import 'package:powersync/src/open_factory/abstract_powersync_open_factory.dart'; +import 'package:powersync/src/open_factory/web/web_open_factory.dart'; +import 'package:powersync/src/schema.dart'; +import 'package:powersync/src/streaming_sync.dart'; +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:powersync/src/schema_helpers.dart' as schema_helpers; + +/// A PowerSync managed database. +/// +/// Web implementation for [PowerSyncDatabase] +/// +/// Use one instance per database file. +/// +/// Use [PowerSyncDatabase.connect] to connect to the PowerSync service, +/// to keep the local database in sync with the remote database. +/// +/// All changes to local tables are automatically recorded, whether connected +/// or not. Once connected, the changes are uploaded. +class PowerSyncDatabaseImpl + with SqliteQueries, PowerSyncDatabaseMixin + implements PowerSyncDatabase { + @override + Schema schema; + + @override + SqliteDatabase database; + + late final DefaultSqliteOpenFactory openFactory; + + @override + @protected + late Future isInitialized; + + @override + + /// The Logger used by this [PowerSyncDatabase]. + /// + /// The default is [autoLogger], which logs to the console in debug builds. + /// Use [debugLogger] to always log to the console. + /// Use [attachedLogger] to propagate logs to [Logger.root] for custom logging. + late final Logger logger; + + /// Open a [PowerSyncDatabase]. + /// + /// Only a single [PowerSyncDatabase] per [path] should be opened at a time. + /// + /// The specified [schema] is used for the database. + /// + /// A connection pool is used by default, allowing multiple concurrent read + /// transactions, and a single concurrent write transaction. Write transactions + /// do not block read transactions, and read transactions will see the state + /// from the last committed write transaction. + /// + /// A maximum of [maxReaders] concurrent read transactions are allowed. + /// + /// [logger] defaults to [autoLogger], which logs to the console in debug builds. + factory PowerSyncDatabaseImpl( + {required Schema schema, + required String path, + int maxReaders = SqliteDatabase.defaultMaxReaders, + Logger? logger, + @Deprecated("Use [PowerSyncDatabase.withFactory] instead") + // ignore: deprecated_member_use_from_same_package + SqliteConnectionSetup? sqliteSetup}) { + // ignore: deprecated_member_use_from_same_package + DefaultSqliteOpenFactory factory = PowerSyncOpenFactory(path: path); + return PowerSyncDatabaseImpl.withFactory(factory, + maxReaders: maxReaders, logger: logger, schema: schema); + } + + /// Open a [PowerSyncDatabase] with a [PowerSyncOpenFactory]. + /// + /// The factory determines which database file is opened, as well as any + /// additional logic to run inside the database isolate before or after opening. + /// + /// Subclass [PowerSyncOpenFactory] to add custom logic to this process. + /// + /// [logger] defaults to [autoLogger], which logs to the console in debug builds. + factory PowerSyncDatabaseImpl.withFactory( + DefaultSqliteOpenFactory openFactory, + {required Schema schema, + int maxReaders = SqliteDatabase.defaultMaxReaders, + Logger? logger}) { + final db = SqliteDatabase.withFactory(openFactory, maxReaders: 1); + return PowerSyncDatabaseImpl.withDatabase( + schema: schema, logger: logger, database: db); + } + + /// Open a PowerSyncDatabase on an existing [SqliteDatabase]. + /// + /// Migrations are run on the database when this constructor is called. + /// + /// [logger] defaults to [autoLogger], which logs to the console in debug builds. + PowerSyncDatabaseImpl.withDatabase( + {required this.schema, required this.database, Logger? logger}) { + if (logger != null) { + this.logger = logger; + } else { + this.logger = autoLogger; + } + isInitialized = baseInit(); + } + + @override + + /// Connect to the PowerSync service, and keep the databases in sync. + /// + /// The connection is automatically re-opened if it fails for any reason. + /// + /// Status changes are reported on [statusStream]. + connect({required PowerSyncBackendConnector connector}) async { + await initialize(); + + // Disconnect if connected + await disconnect(); + final disconnector = AbortController(); + disconnecter = disconnector; + + await isInitialized; + + // TODO multitab support + final storage = BucketStorage(database); + + final sync = StreamingSyncImplementation( + adapter: storage, + credentialsCallback: connector.getCredentialsCached, + invalidCredentialsCallback: connector.fetchCredentials, + uploadCrud: () => connector.uploadData(this), + updateStream: updates, + retryDelay: Duration(seconds: 3), + // HTTP streaming is not supported on web with the standard http package + // https://github.com/dart-lang/http/issues/595 + client: FetchClient(mode: RequestMode.cors, streamRequests: true)); + sync.statusStream.listen((event) { + setStatus(event); + }); + sync.streamingSync(); + } + + /// Takes a read lock, without starting a transaction. + /// + /// In most cases, [readTransaction] should be used instead. + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {String? debugContext, Duration? lockTimeout}) async { + await isInitialized; + return database.readLock(callback, + debugContext: debugContext, lockTimeout: lockTimeout); + } + + @override + Future readTransaction( + Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout, + String? debugContext}) async { + await isInitialized; + return database.readTransaction(callback, lockTimeout: lockTimeout); + } + + /// Takes a global lock, without starting a transaction. + /// + /// In most cases, [writeTransaction] should be used instead. + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {String? debugContext, Duration? lockTimeout}) async { + await isInitialized; + return database.writeLock(callback, + debugContext: debugContext, lockTimeout: lockTimeout); + } + + @override + + /// Uses the database writeTransaction instead of the locally + /// scoped writeLock. This is to allow the Database transaction + /// tracking to be correctly configured. + Future writeTransaction( + Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, + String? debugContext}) async { + await isInitialized; + return database.writeTransaction( + (context) => internalTrackedWrite(context, callback), + lockTimeout: lockTimeout); + } + + @override + Future updateSchema(Schema schema) { + if (disconnecter != null) { + throw AssertionError('Cannot update schema while connected'); + } + this.schema = schema; + return database.writeLock((tx) => schema_helpers.updateSchema(tx, schema)); + } +} diff --git a/packages/powersync/lib/src/database_utils.dart b/packages/powersync/lib/src/database_utils.dart index 95d9098b..26369cc1 100644 --- a/packages/powersync/lib/src/database_utils.dart +++ b/packages/powersync/lib/src/database_utils.dart @@ -1,10 +1,10 @@ import 'dart:async'; -import 'package:sqlite_async/sqlite3.dart' as sqlite; import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite_async/sqlite3_common.dart' as sqlite; -Future asyncDirectTransaction(sqlite.Database db, - FutureOr Function(sqlite.Database db) callback) async { +Future asyncDirectTransaction(sqlite.CommonDatabase db, + FutureOr Function(sqlite.CommonDatabase db) callback) async { for (var i = 50; i >= 0; i--) { try { db.execute('BEGIN IMMEDIATE'); @@ -40,10 +40,7 @@ Future internalTrackedWriteTransaction(SqliteWriteContext ctx, Future Function(SqliteWriteContext tx) callback) async { try { await ctx.execute('BEGIN IMMEDIATE'); - await ctx.execute( - 'UPDATE ps_tx SET current_tx = next_tx, next_tx = next_tx + 1 WHERE id = 1'); - final result = await callback(ctx); - await ctx.execute('UPDATE ps_tx SET current_tx = NULL WHERE id = 1'); + final result = await internalTrackedWrite(ctx, callback); await ctx.execute('COMMIT'); return result; } catch (e) { @@ -56,3 +53,14 @@ Future internalTrackedWriteTransaction(SqliteWriteContext ctx, rethrow; } } + +/// Internally tracks a write +/// The transaction is assumed to be started externally +Future internalTrackedWrite(SqliteWriteContext ctx, + Future Function(SqliteWriteContext tx) callback) async { + await ctx.execute( + 'UPDATE ps_tx SET current_tx = next_tx, next_tx = next_tx + 1 WHERE id = 1'); + final result = await callback(ctx); + await ctx.execute('UPDATE ps_tx SET current_tx = NULL WHERE id = 1'); + return result; +} diff --git a/packages/powersync/lib/src/log_internal.dart b/packages/powersync/lib/src/log_internal.dart index fe9d46d2..22608a25 100644 --- a/packages/powersync/lib/src/log_internal.dart +++ b/packages/powersync/lib/src/log_internal.dart @@ -3,6 +3,7 @@ import 'package:logging/logging.dart'; // Duplicate from package:flutter/foundation.dart, so we don't need to depend on Flutter const bool kReleaseMode = bool.fromEnvironment('dart.vm.product'); const bool kProfileMode = bool.fromEnvironment('dart.vm.profile'); +const bool kIsWeb = bool.fromEnvironment('dart.library.js_util'); const bool kDebugMode = !kReleaseMode && !kProfileMode; // Implementation note: The loggers here are only initialized if used - it adds diff --git a/packages/powersync/lib/src/migrations.dart b/packages/powersync/lib/src/migrations.dart index a6f7e7a6..47320852 100644 --- a/packages/powersync/lib/src/migrations.dart +++ b/packages/powersync/lib/src/migrations.dart @@ -2,27 +2,30 @@ import 'package:sqlite_async/sqlite_async.dart'; final migrations = SqliteMigrations(migrationTable: 'ps_migration') ..add(SqliteMigration(1, (tx) async { - await tx.computeWithDatabase((db) async { - db.execute(''' - DROP TABLE IF EXISTS crud; - DROP TABLE IF EXISTS oplog; - DROP TABLE IF EXISTS buckets; - DROP TABLE IF EXISTS objects_untyped; - DROP TABLE IF EXISTS ps_oplog; - DROP TABLE IF EXISTS ps_buckets; - DROP TABLE IF EXISTS ps_untyped; - DROP TABLE IF EXISTS ps_migrations; - '''); + final List dropCommands = [ + 'DROP TABLE IF EXISTS crud;', + 'DROP TABLE IF EXISTS oplog;', + 'DROP TABLE IF EXISTS buckets;', + 'DROP TABLE IF EXISTS objects_untyped;', + 'DROP TABLE IF EXISTS ps_oplog;', + 'DROP TABLE IF EXISTS ps_buckets;', + 'DROP TABLE IF EXISTS ps_untyped;', + 'DROP TABLE IF EXISTS ps_migrations;' + ]; - final existingTableRows = db.select( - "SELECT name FROM sqlite_master WHERE type='table' AND name GLOB 'objects__*'"); + for (var row in dropCommands) { + await tx.execute(row); + } - for (var row in existingTableRows) { - db.execute('DROP TABLE ${row['name']}'); - } + final existingTableRows = await tx.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name GLOB 'objects__*'"); - db.execute(''' - CREATE TABLE ps_oplog( + for (var row in existingTableRows) { + await tx.execute('DROP TABLE ${row['name']}'); + } + + final List opLogCommands = [ + '''CREATE TABLE ps_oplog( bucket TEXT NOT NULL, op_id INTEGER NOT NULL, op INTEGER NOT NULL, @@ -31,36 +34,35 @@ final migrations = SqliteMigrations(migrationTable: 'ps_migration') key TEXT, data TEXT, hash INTEGER NOT NULL, - superseded INTEGER NOT NULL); - - CREATE INDEX ps_oplog_by_row ON ps_oplog (row_type, row_id) WHERE superseded = 0; - CREATE INDEX ps_oplog_by_opid ON ps_oplog (bucket, op_id); - CREATE INDEX ps_oplog_by_key ON ps_oplog (bucket, key) WHERE superseded = 0; - - CREATE TABLE ps_buckets( + superseded INTEGER NOT NULL);''', + '''CREATE INDEX ps_oplog_by_row ON ps_oplog (row_type, row_id) WHERE superseded = 0;''', + '''CREATE INDEX ps_oplog_by_opid ON ps_oplog (bucket, op_id);''', + '''CREATE INDEX ps_oplog_by_key ON ps_oplog (bucket, key) WHERE superseded = 0;''', + '''CREATE TABLE ps_buckets( name TEXT PRIMARY KEY, last_applied_op INTEGER NOT NULL DEFAULT 0, last_op INTEGER NOT NULL DEFAULT 0, target_op INTEGER NOT NULL DEFAULT 0, add_checksum INTEGER NOT NULL DEFAULT 0, pending_delete INTEGER NOT NULL DEFAULT 0 - ); - - CREATE TABLE ps_untyped(type TEXT NOT NULL, id TEXT NOT NULL, data TEXT, PRIMARY KEY (type, id)); - - CREATE TABLE IF NOT EXISTS ps_crud (id INTEGER PRIMARY KEY AUTOINCREMENT, data TEXT); - '''); - }); + );''', + '''CREATE TABLE ps_untyped(type TEXT NOT NULL, id TEXT NOT NULL, data TEXT, PRIMARY KEY (type, id));''', + '''CREATE TABLE IF NOT EXISTS ps_crud (id INTEGER PRIMARY KEY AUTOINCREMENT, data TEXT);''' + ]; + + for (var row in opLogCommands) { + await tx.execute(row); + } })) ..add(SqliteMigration(2, (tx) async { - await tx.computeWithDatabase((db) async { - db.execute(''' -CREATE TABLE ps_tx(id INTEGER PRIMARY KEY NOT NULL, current_tx INTEGER, next_tx INTEGER); -INSERT INTO ps_tx(id, current_tx, next_tx) VALUES(1, NULL, 1); - -ALTER TABLE ps_crud ADD COLUMN tx_id INTEGER; - '''); - }); + final List opLogCommands = [ + 'CREATE TABLE ps_tx(id INTEGER PRIMARY KEY NOT NULL, current_tx INTEGER, next_tx INTEGER);', + 'INSERT INTO ps_tx(id, current_tx, next_tx) VALUES(1, NULL, 1);', + 'ALTER TABLE ps_crud ADD COLUMN tx_id INTEGER;' + ]; + for (var row in opLogCommands) { + await tx.execute(row); + } }, downMigration: SqliteDownMigration(toVersion: 1) ..add('DROP TABLE ps_tx') diff --git a/packages/powersync/lib/src/open_factory.dart b/packages/powersync/lib/src/open_factory.dart index a9ea79fd..c32daf83 100644 --- a/packages/powersync/lib/src/open_factory.dart +++ b/packages/powersync/lib/src/open_factory.dart @@ -1,138 +1,9 @@ -import 'dart:async'; -import 'dart:convert'; -import 'dart:io'; -import 'dart:isolate'; -import 'dart:math'; - -import 'package:sqlite_async/sqlite3.dart' as sqlite; -import 'package:sqlite_async/sqlite_async.dart'; - -import 'uuid.dart'; - -/// Advanced: Define custom setup for each SQLite connection. -@Deprecated('Use SqliteOpenFactory instead') -class SqliteConnectionSetup { - final FutureOr Function() _setup; - - /// The setup parameter is called every time a database connection is opened. - /// This can be used to configure dynamic library loading if required. - const SqliteConnectionSetup(FutureOr Function() setup) : _setup = setup; - - Future setup() async { - await _setup(); - } -} - -class PowerSyncOpenFactory extends DefaultSqliteOpenFactory { - @Deprecated('Override PowerSyncOpenFactory instead') - final SqliteConnectionSetup? _sqliteSetup; - - PowerSyncOpenFactory( - {required super.path, - super.sqliteOptions, - @Deprecated('Override PowerSyncOpenFactory instead') - // ignore: deprecated_member_use_from_same_package - SqliteConnectionSetup? sqliteSetup}) - // ignore: deprecated_member_use_from_same_package - : _sqliteSetup = sqliteSetup; - - @override - sqlite.Database open(SqliteOpenOptions options) { - // ignore: deprecated_member_use_from_same_package - _sqliteSetup?.setup(); - final db = _retriedOpen(options); - db.execute('PRAGMA recursive_triggers = TRUE'); - setupFunctions(db); - return db; - } - - /// When opening the powersync connection and the standard write connection - /// at the same time, one could fail with this error: - /// - /// SqliteException(5): while opening the database, automatic extension loading failed: , database is locked (code 5) - /// - /// It happens before we have a chance to set the busy timeout, so we just - /// retry opening the database. - /// - /// Usually a delay of 1-2ms is sufficient for the next try to succeed, but - /// we increase the retry delay up to 16ms per retry, and a maximum of 500ms - /// in total. - sqlite.Database _retriedOpen(SqliteOpenOptions options) { - final stopwatch = Stopwatch()..start(); - var retryDelay = 2; - while (stopwatch.elapsedMilliseconds < 500) { - try { - return super.open(options); - } catch (e) { - if (e is sqlite.SqliteException && e.resultCode == 5) { - sleep(Duration(milliseconds: retryDelay)); - retryDelay = min(retryDelay * 2, 16); - continue; - } - rethrow; - } - } - throw AssertionError('Cannot reach this point'); - } - - void setupFunctions(sqlite.Database db) { - db.createFunction( - functionName: 'uuid', - argumentCount: const sqlite.AllowedArgumentCount(0), - function: (args) => uuid.v4(), - ); - db.createFunction( - // Postgres compatibility - functionName: 'gen_random_uuid', - argumentCount: const sqlite.AllowedArgumentCount(0), - function: (args) => uuid.v4(), - ); - - db.createFunction( - functionName: 'powersync_diff', - argumentCount: const sqlite.AllowedArgumentCount(2), - deterministic: true, - directOnly: false, - function: (args) { - final oldData = jsonDecode(args[0] as String) as Map; - final newData = jsonDecode(args[1] as String) as Map; - - Map result = {}; - - for (final newEntry in newData.entries) { - final oldValue = oldData[newEntry.key]; - final newValue = newEntry.value; - - if (newValue != oldValue) { - result[newEntry.key] = newValue; - } - } - - for (final key in oldData.keys) { - if (!newData.containsKey(key)) { - result[key] = null; - } - } - - return jsonEncode(result); - }); - - db.createFunction( - functionName: 'powersync_sleep', - argumentCount: const sqlite.AllowedArgumentCount(1), - function: (args) { - final millis = args[0] as int; - sleep(Duration(milliseconds: millis)); - return millis; - }, - ); - - db.createFunction( - functionName: 'powersync_connection_name', - argumentCount: const sqlite.AllowedArgumentCount(0), - function: (args) { - return Isolate.current.debugName; - }, - ); - } -} +// This follows the pattern from here: https://stackoverflow.com/questions/58710226/how-to-import-platform-specific-dependency-in-flutter-dart-combine-web-with-an +// To conditionally export an implementation for either web or "native" platforms +// The sqlite library uses dart:ffi which is not supported on web + +export './open_factory/open_factory_stub.dart' + // ignore: uri_does_not_exist + if (dart.library.io) './open_factory/native/native_open_factory.dart' + // ignore: uri_does_not_exist + if (dart.library.html) './open_factory/web/web_open_factory.dart'; diff --git a/packages/powersync/lib/src/open_factory/abstract_powersync_open_factory.dart b/packages/powersync/lib/src/open_factory/abstract_powersync_open_factory.dart new file mode 100644 index 00000000..3c93c749 --- /dev/null +++ b/packages/powersync/lib/src/open_factory/abstract_powersync_open_factory.dart @@ -0,0 +1,82 @@ +import 'dart:async'; +import 'dart:math'; + +import 'package:powersync/sqlite_async.dart'; +import 'package:powersync/src/open_factory/common_db_functions.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; + +const powerSyncDefaultSqliteOptions = SqliteOptions( + webSqliteOptions: WebSqliteOptions( + wasmUri: 'sqlite3.wasm', workerUri: 'powersync_db.worker.js')); + +abstract class AbstractPowerSyncOpenFactory extends DefaultSqliteOpenFactory { + AbstractPowerSyncOpenFactory( + {required super.path, + super.sqliteOptions = powerSyncDefaultSqliteOptions}); + + void enableExtension(); + + void setupFunctions(CommonDatabase db) { + return setupCommonDBFunctions(db); + } + + @override + List pragmaStatements(SqliteOpenOptions options) { + final basePragmaStatements = super.pragmaStatements(options); + basePragmaStatements.add('PRAGMA recursive_triggers = TRUE'); + return basePragmaStatements; + } + + @override + FutureOr open(SqliteOpenOptions options) async { + var db = await _retriedOpen(options); + for (final statement in pragmaStatements(options)) { + db.select(statement); + } + setupFunctions(db); + return db; + } + + /// When opening the powersync connection and the standard write connection + /// at the same time, one could fail with this error: + /// + /// SqliteException(5): while opening the database, automatic extension loading failed: , database is locked (code 5) + /// + /// It happens before we have a chance to set the busy timeout, so we just + /// retry opening the database. + /// + /// Usually a delay of 1-2ms is sufficient for the next try to succeed, but + /// we increase the retry delay up to 16ms per retry, and a maximum of 500ms + /// in total. + FutureOr _retriedOpen(SqliteOpenOptions options) async { + final stopwatch = Stopwatch()..start(); + var retryDelay = 2; + while (stopwatch.elapsedMilliseconds < 500) { + try { + return super.open(options); + } catch (e) { + if (e is SqliteException && e.resultCode == 5) { + await Future.delayed(Duration(milliseconds: retryDelay)); + retryDelay = min(retryDelay * 2, 16); + continue; + } + rethrow; + } + } + throw AssertionError('Cannot reach this point'); + } +} + +/// Advanced: Define custom setup for each SQLite connection. +@Deprecated('Use SqliteOpenFactory instead') +class SqliteConnectionSetup { + final FutureOr Function() _setup; + + /// The setup parameter is called every time a database connection is opened. + /// This can be used to configure dynamic library loading if required. + const SqliteConnectionSetup(FutureOr Function() setup) : _setup = setup; + + Future setup() async { + await _setup(); + } +} diff --git a/packages/powersync/lib/src/open_factory/common_db_functions.dart b/packages/powersync/lib/src/open_factory/common_db_functions.dart new file mode 100644 index 00000000..f36906f1 --- /dev/null +++ b/packages/powersync/lib/src/open_factory/common_db_functions.dart @@ -0,0 +1,35 @@ +import 'dart:convert'; + +import 'package:sqlite_async/sqlite3_common.dart' as sqlite; +import 'package:sqlite_async/sqlite3_common.dart'; + +void setupCommonDBFunctions(CommonDatabase db) { + db.createFunction( + functionName: 'powersync_diff', + argumentCount: const sqlite.AllowedArgumentCount(2), + deterministic: true, + directOnly: false, + function: (args) { + final oldData = jsonDecode(args[0] as String) as Map; + final newData = jsonDecode(args[1] as String) as Map; + + Map result = {}; + + for (final newEntry in newData.entries) { + final oldValue = oldData[newEntry.key]; + final newValue = newEntry.value; + + if (newValue != oldValue) { + result[newEntry.key] = newValue; + } + } + + for (final key in oldData.keys) { + if (!newData.containsKey(key)) { + result[key] = null; + } + } + + return jsonEncode(result); + }); +} diff --git a/packages/powersync/lib/src/open_factory/native/native_open_factory.dart b/packages/powersync/lib/src/open_factory/native/native_open_factory.dart new file mode 100644 index 00000000..29d156c5 --- /dev/null +++ b/packages/powersync/lib/src/open_factory/native/native_open_factory.dart @@ -0,0 +1,69 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:isolate'; +import 'package:powersync/src/open_factory/abstract_powersync_open_factory.dart'; +import 'package:sqlite_async/sqlite3.dart' as sqlite; +import 'package:sqlite_async/sqlite3_common.dart'; +import 'package:sqlite_async/sqlite_async.dart'; +import '../../uuid.dart'; + +/// Native implementation for [AbstractPowerSyncOpenFactory] +class PowerSyncOpenFactory extends AbstractPowerSyncOpenFactory { + @Deprecated('Override PowerSyncOpenFactory instead') + final SqliteConnectionSetup? _sqliteSetup; + + PowerSyncOpenFactory( + {required super.path, + super.sqliteOptions, + @Deprecated('Override PowerSyncOpenFactory instead') + SqliteConnectionSetup? sqliteSetup}) + // ignore: deprecated_member_use_from_same_package + : _sqliteSetup = sqliteSetup; + + @override + void enableExtension() {} + + @override + setupFunctions(CommonDatabase db) { + super.setupFunctions(db); + db.createFunction( + functionName: 'uuid', + argumentCount: const AllowedArgumentCount(0), + function: (args) { + return uuid.v4(); + }, + ); + db.createFunction( + // Postgres compatibility + functionName: 'gen_random_uuid', + argumentCount: const AllowedArgumentCount(0), + function: (args) => uuid.v4(), + ); + db.createFunction( + functionName: 'powersync_sleep', + argumentCount: const sqlite.AllowedArgumentCount(1), + function: (args) { + final millis = args[0] as int; + sleep(Duration(milliseconds: millis)); + return millis; + }, + ); + + db.createFunction( + functionName: 'powersync_connection_name', + argumentCount: const sqlite.AllowedArgumentCount(0), + function: (args) { + return Isolate.current.debugName; + }, + ); + } + + @override + FutureOr open(SqliteOpenOptions options) async { + // ignore: deprecated_member_use_from_same_package + _sqliteSetup?.setup(); + var db = await super.open(options); + db.execute('PRAGMA recursive_triggers = TRUE'); + return db; + } +} diff --git a/packages/powersync/lib/src/open_factory/open_factory_stub.dart b/packages/powersync/lib/src/open_factory/open_factory_stub.dart new file mode 100644 index 00000000..3c588925 --- /dev/null +++ b/packages/powersync/lib/src/open_factory/open_factory_stub.dart @@ -0,0 +1,21 @@ +import 'package:sqlite_async/sqlite3_common.dart'; +import 'abstract_powersync_open_factory.dart' as open_factory; + +class PowerSyncOpenFactory extends open_factory.AbstractPowerSyncOpenFactory { + PowerSyncOpenFactory( + {required super.path, + super.sqliteOptions, + @Deprecated('Override PowerSyncOpenFactory instead') + // ignore: deprecated_member_use_from_same_package + open_factory.SqliteConnectionSetup? sqliteSetup}); + + @override + void enableExtension() { + throw UnimplementedError(); + } + + @override + void setupFunctions(CommonDatabase db) { + throw UnimplementedError(); + } +} diff --git a/packages/powersync/lib/src/open_factory/web/web_open_factory.dart b/packages/powersync/lib/src/open_factory/web/web_open_factory.dart new file mode 100644 index 00000000..8fbe9987 --- /dev/null +++ b/packages/powersync/lib/src/open_factory/web/web_open_factory.dart @@ -0,0 +1,38 @@ +import 'package:powersync/src/open_factory/abstract_powersync_open_factory.dart'; +import 'package:powersync/src/uuid.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; + +/// Web implementation for [AbstractPowerSyncOpenFactory] +class PowerSyncOpenFactory extends AbstractPowerSyncOpenFactory { + PowerSyncOpenFactory({ + required super.path, + super.sqliteOptions, + }); + + @override + void enableExtension() { + // No op for web + } + + @override + + /// This is only called when synchronous connections are created in the same + /// Dart/JS context. Worker runners need to setupFunctions manually + setupFunctions(CommonDatabase db) { + super.setupFunctions(db); + + db.createFunction( + functionName: 'uuid', + argumentCount: const AllowedArgumentCount(0), + function: (args) { + return uuid.v4(); + }, + ); + db.createFunction( + // Postgres compatibility + functionName: 'gen_random_uuid', + argumentCount: const AllowedArgumentCount(0), + function: (args) => uuid.v4(), + ); + } +} diff --git a/packages/powersync/lib/src/powersync_database.dart b/packages/powersync/lib/src/powersync_database.dart index 9677afd9..c051b90d 100644 --- a/packages/powersync/lib/src/powersync_database.dart +++ b/packages/powersync/lib/src/powersync_database.dart @@ -1,633 +1 @@ -import 'dart:async'; -import 'dart:isolate'; - -import 'package:logging/logging.dart'; -import 'package:powersync/src/log_internal.dart'; -import 'package:sqlite_async/sqlite3.dart' as sqlite; -import 'package:sqlite_async/sqlite_async.dart'; - -import 'abort_controller.dart'; -import 'bucket_storage.dart'; -import 'connector.dart'; -import 'crud.dart'; -import 'database_utils.dart'; -import 'isolate_completer.dart'; -import 'log.dart'; -import 'migrations.dart'; -import 'open_factory.dart'; -import 'powersync_update_notification.dart'; -import 'schema.dart'; -import 'schema_logic.dart'; -import 'streaming_sync.dart'; -import 'sync_status.dart'; - -/// A PowerSync managed database. -/// -/// Use one instance per database file. -/// -/// Use [PowerSyncDatabase.connect] to connect to the PowerSync service, -/// to keep the local database in sync with the remote database. -/// -/// All changes to local tables are automatically recorded, whether connected -/// or not. Once connected, the changes are uploaded. -class PowerSyncDatabase with SqliteQueries implements SqliteConnection { - /// Schema used for the local database. - Schema schema; - - /// The underlying database. - /// - /// For the most part, behavior is the same whether querying on the underlying - /// database, or on [PowerSyncDatabase]. The main difference is in update notifications: - /// the underlying database reports updates to the underlying tables, while - /// [PowerSyncDatabase] reports updates to the higher-level views. - final SqliteDatabase database; - - /// Current connection status. - SyncStatus currentStatus = const SyncStatus(); - - /// Use this stream to subscribe to connection status updates. - late final Stream statusStream; - - final StreamController _statusStreamController = - StreamController.broadcast(); - - /// Broadcast stream that is notified of any table updates. - /// - /// Unlike in [SqliteDatabase.updates], the tables reported here are the - /// higher-level views as defined in the [Schema], and exclude the low-level - /// PowerSync tables. - @override - late final Stream updates; - - /// Delay between retrying failed requests. - /// Defaults to 5 seconds. - /// Only has an effect if changed before calling [connect]. - Duration retryDelay = const Duration(seconds: 5); - - late Future _initialized; - - /// null when disconnected, present when connecting or connected - AbortController? _disconnecter; - - /// The Logger used by this [PowerSyncDatabase]. - /// - /// The default is [autoLogger], which logs to the console in debug builds. - /// Use [debugLogger] to always log to the console. - /// Use [attachedLogger] to propagate logs to [Logger.root] for custom logging. - late final Logger logger; - - /// Open a [PowerSyncDatabase]. - /// - /// Only a single [PowerSyncDatabase] per [path] should be opened at a time. - /// - /// The specified [schema] is used for the database. - /// - /// A connection pool is used by default, allowing multiple concurrent read - /// transactions, and a single concurrent write transaction. Write transactions - /// do not block read transactions, and read transactions will see the state - /// from the last committed write transaction. - /// - /// A maximum of [maxReaders] concurrent read transactions are allowed. - /// - /// [logger] defaults to [autoLogger], which logs to the console in debug builds. - factory PowerSyncDatabase( - {required Schema schema, - required String path, - int maxReaders = SqliteDatabase.defaultMaxReaders, - Logger? logger, - @Deprecated("Use [PowerSyncDatabase.withFactory] instead") - // ignore: deprecated_member_use_from_same_package - SqliteConnectionSetup? sqliteSetup}) { - // ignore: deprecated_member_use_from_same_package - var factory = PowerSyncOpenFactory(path: path, sqliteSetup: sqliteSetup); - return PowerSyncDatabase.withFactory(factory, - schema: schema, logger: logger); - } - - /// Open a [PowerSyncDatabase] with a [PowerSyncOpenFactory]. - /// - /// The factory determines which database file is opened, as well as any - /// additional logic to run inside the database isolate before or after opening. - /// - /// Subclass [PowerSyncOpenFactory] to add custom logic to this process. - /// - /// [logger] defaults to [autoLogger], which logs to the console in debug builds. - factory PowerSyncDatabase.withFactory( - PowerSyncOpenFactory openFactory, { - required Schema schema, - int maxReaders = SqliteDatabase.defaultMaxReaders, - Logger? logger, - }) { - final db = SqliteDatabase.withFactory(openFactory, maxReaders: maxReaders); - return PowerSyncDatabase.withDatabase( - schema: schema, database: db, logger: logger); - } - - /// Open a PowerSyncDatabase on an existing [SqliteDatabase]. - /// - /// Migrations are run on the database when this constructor is called. - /// - /// [logger] defaults to [autoLogger], which logs to the console in debug builds. - PowerSyncDatabase.withDatabase({ - required this.schema, - required this.database, - Logger? logger, - }) { - if (logger != null) { - this.logger = logger; - } else { - this.logger = autoLogger; - } - - updates = database.updates - .map((update) => - PowerSyncUpdateNotification.fromUpdateNotification(update)) - .where((update) => update.isNotEmpty) - .cast(); - _initialized = _init(); - } - - Future _init() async { - statusStream = _statusStreamController.stream; - await database.initialize(); - await migrations.migrate(database); - await updateSchema(schema); - } - - /// Replace the schema with a new version. - /// This is for advanced use cases - typically the schema should just be - /// specified once in the constructor. - /// - /// Cannot be used while connected - this should only be called before [connect]. - Future updateSchema(Schema schema) async { - if (_disconnecter != null) { - throw AssertionError('Cannot update schema while connected'); - } - this.schema = schema; - await updateSchemaInIsolate(database, schema); - } - - /// Wait for initialization to complete. - /// - /// While initializing is automatic, this helps to catch and report initialization errors. - Future initialize() { - return _initialized; - } - - @override - bool get closed { - return database.closed; - } - - /// Connect to the PowerSync service, and keep the databases in sync. - /// - /// The connection is automatically re-opened if it fails for any reason. - /// - /// Status changes are reported on [statusStream]. - Future connect({required PowerSyncBackendConnector connector}) async { - await initialize(); - - // Disconnect if connected - await disconnect(); - final disconnector = AbortController(); - _disconnecter = disconnector; - - await _initialized; - final dbref = database.isolateConnectionFactory(); - ReceivePort rPort = ReceivePort(); - StreamSubscription? updateSubscription; - rPort.listen((data) async { - if (data is List) { - String action = data[0]; - if (action == "getCredentials") { - await (data[1] as PortCompleter).handle(() async { - final token = await connector.getCredentialsCached(); - logger.fine('Credentials: $token'); - return token; - }); - } else if (action == "invalidateCredentials") { - logger.fine('Refreshing credentials'); - await (data[1] as PortCompleter).handle(() async { - await connector.prefetchCredentials(); - }); - } else if (action == 'init') { - SendPort port = data[1]; - var throttled = UpdateNotification.throttleStream( - updates, const Duration(milliseconds: 10)); - updateSubscription = throttled.listen((event) { - port.send(['update']); - }); - disconnector.onAbort.then((_) { - port.send(['close']); - }).ignore(); - } else if (action == 'uploadCrud') { - await (data[1] as PortCompleter).handle(() async { - await connector.uploadData(this); - }); - } else if (action == 'status') { - final SyncStatus status = data[1]; - _setStatus(status); - } else if (action == 'close') { - // Clear status apart from lastSyncedAt - _setStatus(SyncStatus(lastSyncedAt: currentStatus.lastSyncedAt)); - rPort.close(); - updateSubscription?.cancel(); - } else if (action == 'log') { - LogRecord record = data[1]; - logger.log( - record.level, record.message, record.error, record.stackTrace); - } - } - }); - - var errorPort = ReceivePort(); - errorPort.listen((message) async { - // Sample error: - // flutter: [PowerSync] WARNING: 2023-06-28 16:34:11.566122: Sync Isolate error - // flutter: [Connection closed while receiving data, #0 IOClient.send. (package:http/src/io_client.dart:76:13) - // #1 Stream.handleError. (dart:async/stream.dart:929:16) - // #2 _HandleErrorStream._handleError (dart:async/stream_pipe.dart:269:17) - // #3 _ForwardingStreamSubscription._handleError (dart:async/stream_pipe.dart:157:13) - // #4 _HttpClientResponse.listen. (dart:_http/http_impl.dart:707:16) - // ... - logger.severe('Sync Isolate error', message); - - // Reconnect - connect(connector: connector); - }); - - disconnected() { - _disconnecter?.completeAbort(); - _disconnecter = null; - rPort.close(); - // Clear status apart from lastSyncedAt - _setStatus(SyncStatus(lastSyncedAt: currentStatus.lastSyncedAt)); - } - - var exitPort = ReceivePort(); - exitPort.listen((message) { - logger.fine('Sync Isolate exit'); - disconnected(); - }); - - if (_disconnecter?.aborted == true) { - disconnected(); - return; - } - - Isolate.spawn(_powerSyncDatabaseIsolate, - _PowerSyncDatabaseIsolateArgs(rPort.sendPort, dbref, retryDelay), - debugName: 'PowerSyncDatabase', - onError: errorPort.sendPort, - onExit: exitPort.sendPort); - } - - void _setStatus(SyncStatus status) { - if (status != currentStatus) { - currentStatus = status; - _statusStreamController.add(status); - } - } - - /// Close the sync connection. - /// - /// Use [connect] to connect again. - Future disconnect() async { - if (_disconnecter != null) { - await _disconnecter!.abort(); - } - } - - /// Disconnect and clear the database. - /// - /// Use this when logging out. - /// - /// The database can still be queried after this is called, but the tables - /// would be empty. - /// - /// To preserve data in local-only tables, set [clearLocal] to false. - Future disconnectAndClear({bool clearLocal = true}) async { - await disconnect(); - - await writeTransaction((tx) async { - await tx.execute('DELETE FROM ps_oplog'); - await tx.execute('DELETE FROM ps_crud'); - await tx.execute('DELETE FROM ps_buckets'); - - final tableGlob = clearLocal ? 'ps_data_*' : 'ps_data__*'; - final existingTableRows = await tx.getAll( - "SELECT name FROM sqlite_master WHERE type='table' AND name GLOB ?", - [tableGlob]); - - for (var row in existingTableRows) { - await tx.execute('DELETE FROM ${quoteIdentifier(row['name'])}'); - } - }); - } - - @Deprecated('Use [disconnectAndClear] instead.') - Future disconnectedAndClear() async { - await disconnectAndClear(); - } - - /// Whether a connection to the PowerSync service is currently open. - bool get connected { - return currentStatus.connected; - } - - /// A connection factory that can be passed to different isolates. - /// - /// Use this to access the database in background isolates. - IsolateConnectionFactory isolateConnectionFactory() { - return database.isolateConnectionFactory(); - } - - /// Get upload queue size estimate and count. - Future getUploadQueueStats( - {bool includeSize = false}) async { - if (includeSize) { - final row = await getOptional( - 'SELECT SUM(cast(data as blob) + 20) as size, count(*) as count FROM ps_crud'); - return UploadQueueStats( - count: row?['count'] ?? 0, size: row?['size'] ?? 0); - } else { - final row = await getOptional('SELECT count(*) as count FROM ps_crud'); - return UploadQueueStats(count: row?['count'] ?? 0); - } - } - - /// Get a batch of crud data to upload. - /// - /// Returns null if there is no data to upload. - /// - /// Use this from the [PowerSyncBackendConnector.uploadData]` callback. - /// - /// Once the data have been successfully uploaded, call [CrudBatch.complete] before - /// requesting the next batch. - /// - /// Use [limit] to specify the maximum number of updates to return in a single - /// batch. - /// - /// This method does include transaction ids in the result, but does not group - /// data by transaction. One batch may contain data from multiple transactions, - /// and a single transaction may be split over multiple batches. - Future getCrudBatch({limit = 100}) async { - final rows = await getAll( - 'SELECT id, tx_id, data FROM ps_crud ORDER BY id ASC LIMIT ?', - [limit + 1]); - List all = [for (var row in rows) CrudEntry.fromRow(row)]; - - var haveMore = false; - if (all.length > limit) { - all.removeLast(); - haveMore = true; - } - if (all.isEmpty) { - return null; - } - final last = all[all.length - 1]; - return CrudBatch( - crud: all, - haveMore: haveMore, - complete: ({String? writeCheckpoint}) async { - await writeTransaction((db) async { - await db - .execute('DELETE FROM ps_crud WHERE id <= ?', [last.clientId]); - if (writeCheckpoint != null && - await db.getOptional('SELECT 1 FROM ps_crud LIMIT 1') == null) { - await db.execute( - 'UPDATE ps_buckets SET target_op = $writeCheckpoint WHERE name=\'\$local\''); - } else { - await db.execute( - 'UPDATE ps_buckets SET target_op = $maxOpId WHERE name=\'\$local\''); - } - }); - }); - } - - /// Get the next recorded transaction to upload. - /// - /// Returns null if there is no data to upload. - /// - /// Use this from the [PowerSyncBackendConnector.uploadData]` callback. - /// - /// Once the data have been successfully uploaded, call [CrudTransaction.complete] before - /// requesting the next transaction. - /// - /// Unlike [getCrudBatch], this only returns data from a single transaction at a time. - /// All data for the transaction is loaded into memory. - Future getNextCrudTransaction() async { - return await readTransaction((tx) async { - final first = await tx.getOptional( - 'SELECT id, tx_id, data FROM ps_crud ORDER BY id ASC LIMIT 1'); - if (first == null) { - return null; - } - final int? txId = first['tx_id']; - List all; - if (txId == null) { - all = [CrudEntry.fromRow(first)]; - } else { - final rows = await tx.getAll( - 'SELECT id, tx_id, data FROM ps_crud WHERE tx_id = ? ORDER BY id ASC', - [txId]); - all = [for (var row in rows) CrudEntry.fromRow(row)]; - } - - final last = all[all.length - 1]; - - return CrudTransaction( - transactionId: txId, - crud: all, - complete: ({String? writeCheckpoint}) async { - await writeTransaction((db) async { - await db.execute( - 'DELETE FROM ps_crud WHERE id <= ?', [last.clientId]); - if (writeCheckpoint != null && - await db.getOptional('SELECT 1 FROM ps_crud LIMIT 1') == - null) { - await db.execute( - 'UPDATE ps_buckets SET target_op = $writeCheckpoint WHERE name=\'\$local\''); - } else { - await db.execute( - 'UPDATE ps_buckets SET target_op = $maxOpId WHERE name=\'\$local\''); - } - }); - }); - }); - } - - /// Close the database, releasing resources. - /// - /// Also [disconnect]s any active connection. - /// - /// Once close is called, this connection cannot be used again - a new one - /// must be constructed. - @override - Future close() async { - // Don't close in the middle of the initialization process. - await _initialized; - // Disconnect any active sync connection. - await disconnect(); - // Now we can close the database - await database.close(); - } - - /// Takes a read lock, without starting a transaction. - /// - /// In most cases, [readTransaction] should be used instead. - @override - Future readLock(Future Function(SqliteReadContext tx) callback, - {String? debugContext, Duration? lockTimeout}) async { - await _initialized; - return database.readLock(callback, - debugContext: debugContext, lockTimeout: lockTimeout); - } - - /// Takes a global lock, without starting a transaction. - /// - /// In most cases, [writeTransaction] should be used instead. - @override - Future writeLock(Future Function(SqliteWriteContext tx) callback, - {String? debugContext, Duration? lockTimeout}) async { - await _initialized; - return database.writeLock(callback, - debugContext: debugContext, lockTimeout: lockTimeout); - } - - @override - Future writeTransaction( - Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout, - String? debugContext}) async { - return writeLock((ctx) async { - return await internalTrackedWriteTransaction(ctx, callback); - }, - lockTimeout: lockTimeout, - debugContext: debugContext ?? 'writeTransaction()'); - } - - @override - Future execute(String sql, - [List parameters = const []]) async { - return writeLock((ctx) async { - try { - await ctx.execute( - 'UPDATE ps_tx SET current_tx = next_tx, next_tx = next_tx + 1 WHERE id = 1'); - return await ctx.execute(sql, parameters); - } finally { - await ctx.execute('UPDATE ps_tx SET current_tx = NULL WHERE id = 1'); - } - }, debugContext: 'execute()'); - } - - @override - Future getAutoCommit() { - return database.getAutoCommit(); - } -} - -class _PowerSyncDatabaseIsolateArgs { - final SendPort sPort; - final IsolateConnectionFactory dbRef; - final Duration retryDelay; - - _PowerSyncDatabaseIsolateArgs(this.sPort, this.dbRef, this.retryDelay); -} - -Future _powerSyncDatabaseIsolate( - _PowerSyncDatabaseIsolateArgs args) async { - final sPort = args.sPort; - ReceivePort rPort = ReceivePort(); - StreamController updateController = StreamController.broadcast(); - final upstreamDbClient = args.dbRef.upstreamPort.open(); - - sqlite.Database? db; - rPort.listen((message) { - if (message is List) { - String action = message[0]; - if (action == 'update') { - updateController.add('update'); - } else if (action == 'close') { - db?.dispose(); - updateController.close(); - upstreamDbClient.close(); - Isolate.current.kill(); - } - } - }); - Isolate.current.addOnExitListener(sPort, response: const ['close']); - sPort.send(["init", rPort.sendPort]); - - // Is there a way to avoid the overhead if logging is not enabled? - // This only takes effect in this isolate. - isolateLogger.level = Level.ALL; - isolateLogger.onRecord.listen((record) { - var copy = LogRecord(record.level, record.message, record.loggerName, - record.error, record.stackTrace); - sPort.send(["log", copy]); - }); - - Future loadCredentials() async { - final r = IsolateResult(); - sPort.send(["getCredentials", r.completer]); - return r.future; - } - - Future invalidateCredentials() async { - final r = IsolateResult(); - sPort.send(["invalidateCredentials", r.completer]); - return r.future; - } - - Future uploadCrud() async { - final r = IsolateResult(); - sPort.send(["uploadCrud", r.completer]); - return r.future; - } - - runZonedGuarded(() async { - final mutex = args.dbRef.mutex.open(); - db = await args.dbRef.openFactory - .open(SqliteOpenOptions(primaryConnection: false, readOnly: false)); - - final storage = BucketStorage(db!, mutex: mutex); - final sync = StreamingSyncImplementation( - adapter: storage, - credentialsCallback: loadCredentials, - invalidCredentialsCallback: invalidateCredentials, - uploadCrud: uploadCrud, - updateStream: updateController.stream, - retryDelay: args.retryDelay); - sync.streamingSync(); - sync.statusStream.listen((event) { - sPort.send(['status', event]); - }); - - Timer? updateDebouncer; - Set updatedTables = {}; - - void maybeFireUpdates() { - if (updatedTables.isNotEmpty) { - upstreamDbClient.fire(UpdateNotification(updatedTables)); - updatedTables.clear(); - updateDebouncer?.cancel(); - updateDebouncer = null; - } - } - - db!.updates.listen((event) { - updatedTables.add(event.tableName); - - updateDebouncer ??= - Timer(const Duration(milliseconds: 10), maybeFireUpdates); - }); - }, (error, stack) { - // Properly dispose the database if an uncaught error occurs. - // Unfortunately, this does not handle disposing while the database is opening. - // This should be rare - any uncaught error is a bug. And in most cases, - // it should occur after the database is already open. - db?.dispose(); - throw error; - }); -} +export 'package:powersync/src/database/powersync_database.dart'; diff --git a/packages/powersync/lib/src/powersync_update_notification.dart b/packages/powersync/lib/src/powersync_update_notification.dart index 701ba3d9..0f09e374 100644 --- a/packages/powersync/lib/src/powersync_update_notification.dart +++ b/packages/powersync/lib/src/powersync_update_notification.dart @@ -1,6 +1,5 @@ import 'package:sqlite_async/sqlite_async.dart'; - -import 'schema_logic.dart'; +import 'schema_helpers.dart'; class PowerSyncUpdateNotification extends UpdateNotification { PowerSyncUpdateNotification(super.tables); diff --git a/packages/powersync/lib/src/schema.dart b/packages/powersync/lib/src/schema.dart index 1be0b22f..8993b30d 100644 --- a/packages/powersync/lib/src/schema.dart +++ b/packages/powersync/lib/src/schema.dart @@ -1,4 +1,4 @@ -import 'schema_logic.dart'; +import './schema_helpers.dart'; /// The schema used by the database. /// diff --git a/packages/powersync/lib/src/schema_helpers.dart b/packages/powersync/lib/src/schema_helpers.dart new file mode 100644 index 00000000..72d95d9e --- /dev/null +++ b/packages/powersync/lib/src/schema_helpers.dart @@ -0,0 +1,318 @@ +import 'package:powersync/sqlite_async.dart'; + +import 'schema.dart'; + +const String maxOpId = '9223372036854775807'; + +final invalidSqliteCharacters = RegExp(r'''["'%,\.#\s\[\]]'''); + +/// Since view names don't have a static prefix, mark views as auto-generated by adding a comment. +final _autoGenerated = '-- powersync-auto-generated'; + +String createViewStatement(Table table) { + final columnNames = + table.columns.map((column) => quoteIdentifier(column.name)).join(', '); + + if (table.insertOnly) { + final nulls = table.columns.map((column) => 'NULL').join(', '); + return 'CREATE VIEW ${quoteIdentifier(table.viewName)}("id", $columnNames) AS SELECT NULL, $nulls WHERE 0 $_autoGenerated'; + } + final select = table.columns.map(mapColumn).join(', '); + return 'CREATE VIEW ${quoteIdentifier(table.viewName)}("id", $columnNames) AS SELECT "id", $select FROM ${quoteIdentifier(table.internalName)} $_autoGenerated'; +} + +String mapColumn(Column column) { + return "CAST(json_extract(data, ${quoteJsonPath(column.name)}) as ${column.type.sqlite})"; +} + +List createViewTriggerStatements(Table table) { + if (table.localOnly) { + return createViewTriggerStatementsLocal(table); + } else if (table.insertOnly) { + return createViewTriggerStatementsInsert(table); + } + final viewName = table.viewName; + final type = table.name; + final internalNameE = quoteIdentifier(table.internalName); + + final jsonFragment = table.columns + .map((column) => + "${quoteString(column.name)}, NEW.${quoteIdentifier(column.name)}") + .join(', '); + final jsonFragmentOld = table.columns + .map((column) => + "${quoteString(column.name)}, OLD.${quoteIdentifier(column.name)}") + .join(', '); + // Names in alphabetical order + return [ + """ +CREATE TRIGGER ${quoteIdentifier('ps_view_delete_$viewName')} +INSTEAD OF DELETE ON ${quoteIdentifier(viewName)} +FOR EACH ROW +BEGIN + DELETE FROM $internalNameE WHERE id = OLD.id; + INSERT INTO ps_crud(tx_id, data) SELECT current_tx, json_object('op', 'DELETE', 'type', ${quoteString(type)}, 'id', OLD.id) FROM ps_tx WHERE id = 1; + INSERT INTO ps_oplog(bucket, op_id, op, row_type, row_id, hash, superseded) + SELECT '\$local', + 1, + 'REMOVE', + ${quoteString(type)}, + OLD.id, + 0, + 0; + INSERT OR REPLACE INTO ps_buckets(name, pending_delete, last_op, target_op) VALUES('\$local', 1, 0, $maxOpId); +END""", + """ +CREATE TRIGGER ${quoteIdentifier('ps_view_insert_$viewName')} +INSTEAD OF INSERT ON ${quoteIdentifier(viewName)} +FOR EACH ROW +BEGIN + SELECT CASE + WHEN (NEW.id IS NULL) + THEN RAISE (FAIL, 'id is required') + END; + INSERT INTO $internalNameE(id, data) + SELECT NEW.id, json_object($jsonFragment); + INSERT INTO ps_crud(tx_id, data) SELECT current_tx, json_object('op', 'PUT', 'type', ${quoteString(type)}, 'id', NEW.id, 'data', json(powersync_diff('{}', json_object($jsonFragment)))) FROM ps_tx WHERE id = 1; + INSERT INTO ps_oplog(bucket, op_id, op, row_type, row_id, hash, superseded) + SELECT '\$local', + 1, + 'REMOVE', + ${quoteString(type)}, + NEW.id, + 0, + 0; + INSERT OR REPLACE INTO ps_buckets(name, pending_delete, last_op, target_op) VALUES('\$local', 1, 0, $maxOpId); +END""", + """ +CREATE TRIGGER ${quoteIdentifier('ps_view_update_$viewName')} +INSTEAD OF UPDATE ON ${quoteIdentifier(viewName)} +FOR EACH ROW +BEGIN + SELECT CASE + WHEN (OLD.id != NEW.id) + THEN RAISE (FAIL, 'Cannot update id') + END; + UPDATE $internalNameE + SET data = json_object($jsonFragment) + WHERE id = NEW.id; + INSERT INTO ps_crud(tx_id, data) SELECT current_tx, json_object('op', 'PATCH', 'type', ${quoteString(type)}, 'id', NEW.id, 'data', json(powersync_diff(json_object($jsonFragmentOld), json_object($jsonFragment)))) FROM ps_tx WHERE id = 1; + INSERT INTO ps_oplog(bucket, op_id, op, row_type, row_id, hash, superseded) + SELECT '\$local', + 1, + 'REMOVE', + ${quoteString(type)}, + NEW.id, + 0, + 0; + INSERT OR REPLACE INTO ps_buckets(name, pending_delete, last_op, target_op) VALUES('\$local', 1, 0, $maxOpId); +END""" + ]; +} + +List createViewTriggerStatementsLocal(Table table) { + final viewName = table.viewName; + final internalNameE = quoteIdentifier(table.internalName); + + final jsonFragment = table.columns + .map((column) => + "${quoteString(column.name)}, NEW.${quoteIdentifier(column.name)}") + .join(', '); + // Names in alphabetical order + return [ + """ +CREATE TRIGGER ${quoteIdentifier('ps_view_delete_$viewName')} +INSTEAD OF DELETE ON ${quoteIdentifier(viewName)} +FOR EACH ROW +BEGIN + DELETE FROM $internalNameE WHERE id = OLD.id; +END""", + """ +CREATE TRIGGER ${quoteIdentifier('ps_view_insert_$viewName')} +INSTEAD OF INSERT ON ${quoteIdentifier(viewName)} +FOR EACH ROW +BEGIN + INSERT INTO $internalNameE(id, data) + SELECT NEW.id, json_object($jsonFragment); +END""", + """ +CREATE TRIGGER ${quoteIdentifier('ps_view_update_$viewName')} +INSTEAD OF UPDATE ON ${quoteIdentifier(viewName)} +FOR EACH ROW +BEGIN + SELECT CASE + WHEN (OLD.id != NEW.id) + THEN RAISE (FAIL, 'Cannot update id') + END; + UPDATE $internalNameE + SET data = json_object($jsonFragment) + WHERE id = NEW.id; +END""" + ]; +} + +List createViewTriggerStatementsInsert(Table table) { + final type = table.name; + final viewName = table.viewName; + + final jsonFragment = table.columns + .map((column) => + "${quoteString(column.name)}, NEW.${quoteIdentifier(column.name)}") + .join(', '); + return [ + """ +CREATE TRIGGER ${quoteIdentifier('ps_view_insert_$viewName')} +INSTEAD OF INSERT ON ${quoteIdentifier(viewName)} +FOR EACH ROW +BEGIN + INSERT INTO ps_crud(tx_id, data) SELECT current_tx, json_object('op', 'PUT', 'type', ${quoteString(type)}, 'id', NEW.id, 'data', json(powersync_diff('{}', json_object($jsonFragment)))) FROM ps_tx WHERE id = 1; +END""" + ]; +} + +/// Sync the schema to the local database. +/// Must be wrapped in a transaction. +Future updateSchema(SqliteWriteContext tx, Schema schema) async { + for (var table in schema.tables) { + table.validate(); + } + + await _createTablesAndIndexes(tx, schema); + + final existingViewRows = await tx.execute( + "SELECT name FROM sqlite_master WHERE type='view' AND sql GLOB '*$_autoGenerated'"); + + Set toRemove = {for (var row in existingViewRows) row['name']}; + + for (var table in schema.tables) { + toRemove.remove(table.viewName); + + var createViewOp = createViewStatement(table); + var triggers = createViewTriggerStatements(table); + var existingRows = await tx.execute( + "SELECT sql FROM sqlite_master WHERE (type = 'view' AND name = ?) OR (type = 'trigger' AND tbl_name = ?) ORDER BY type DESC, name ASC", + [table.viewName, table.viewName]); + if (existingRows.isNotEmpty) { + final dbSql = existingRows.map((row) => row['sql']).join('\n\n'); + final generatedSql = + [createViewOp, for (var trigger in triggers) trigger].join('\n\n'); + if (dbSql == generatedSql) { + // No change - keep it. + continue; + } else { + // View and/or triggers changed - delete and re-create. + await tx.execute('DROP VIEW ${quoteIdentifier(table.viewName)}'); + } + } else { + // New - create + } + await tx.execute(createViewOp); + for (final op in triggers) { + await tx.execute(op); + } + } + + for (var name in toRemove) { + await tx.execute('DROP VIEW ${quoteIdentifier(name)}'); + } +} + +/// Sync the schema to the local database. +/// +/// Does not create triggers or temporary views. +/// +/// Must be wrapped in a transaction. +Future _createTablesAndIndexes( + SqliteWriteContext tx, Schema schema) async { + // Make sure to refresh tables in the same transaction as updating them + final existingTableRows = await tx.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name GLOB 'ps_data_*'"); + final existingIndexRows = await tx.execute( + "SELECT name, sql FROM sqlite_master WHERE type='index' AND name GLOB 'ps_data_*'"); + + final Set remainingTables = {}; + final Map indexesToDrop = {}; + final List createIndexes = []; + for (final row in existingTableRows) { + remainingTables.add(row['name'] as String); + } + for (final row in existingIndexRows) { + indexesToDrop[row['name'] as String] = row['sql'] as String; + } + + for (final table in schema.tables) { + for (final index in table.indexes) { + final fullName = index.fullName(table); + final sql = index.toSqlDefinition(table); + if (indexesToDrop.containsKey(fullName)) { + final existingSql = indexesToDrop[fullName]; + if (existingSql == sql) { + // No change (don't drop) + indexesToDrop.remove(fullName); + } else { + // Drop and create + createIndexes.add(sql); + } + } else { + // New index - create + createIndexes.add(sql); + } + } + } + + for (final table in schema.tables) { + if (table.insertOnly) { + // Does not have a physical table + continue; + } + final tableName = table.internalName; + final exists = remainingTables.contains(tableName); + remainingTables.remove(tableName); + if (exists) { + continue; + } + + await tx.execute("""CREATE TABLE ${quoteIdentifier(tableName)} + ( + id TEXT PRIMARY KEY NOT NULL, + data TEXT + )"""); + + if (!table.localOnly) { + await tx.execute("""INSERT INTO ${quoteIdentifier(tableName)}(id, data) + SELECT id, data + FROM ps_untyped + WHERE type = ?""", [table.name]); + await tx.execute("""DELETE + FROM ps_untyped + WHERE type = ?""", [table.name]); + } + } + + for (final indexName in indexesToDrop.keys) { + await tx.execute('DROP INDEX ${quoteIdentifier(indexName)}'); + } + + for (final sql in createIndexes) { + await tx.execute(sql); + } + + for (final tableName in remainingTables) { + final typeMatch = RegExp("^ps_data__(.+)\$").firstMatch(tableName); + if (typeMatch != null) { + // Not local-only + final type = typeMatch[1]; + await tx.execute( + 'INSERT INTO ps_untyped(type, id, data) SELECT ?, id, data FROM ${quoteIdentifier(tableName)}', + [type]); + } + await tx.execute('DROP TABLE ${quoteIdentifier(tableName)}'); + } +} + +String? friendlyTableName(String table) { + final re = RegExp(r"^ps_data__(.+)$"); + final re2 = RegExp(r"^ps_data_local__(.+)$"); + final match = re.firstMatch(table) ?? re2.firstMatch(table); + return match?.group(1); +} diff --git a/packages/powersync/lib/src/schema_logic.dart b/packages/powersync/lib/src/schema_logic.dart index 0641540c..3066d98c 100644 --- a/packages/powersync/lib/src/schema_logic.dart +++ b/packages/powersync/lib/src/schema_logic.dart @@ -1,326 +1,11 @@ -import 'package:sqlite_async/sqlite3.dart' as sqlite; -import 'package:sqlite_async/sqlite_async.dart'; +import 'package:powersync/sqlite_async.dart'; import 'schema.dart'; - -const String maxOpId = '9223372036854775807'; - -final invalidSqliteCharacters = RegExp(r'''["'%,\.#\s\[\]]'''); - -/// Since view names don't have a static prefix, mark views as auto-generated by adding a comment. -final _autoGenerated = '-- powersync-auto-generated'; - -String createViewStatement(Table table) { - final columnNames = - table.columns.map((column) => quoteIdentifier(column.name)).join(', '); - - if (table.insertOnly) { - final nulls = table.columns.map((column) => 'NULL').join(', '); - return 'CREATE VIEW ${quoteIdentifier(table.viewName)}("id", $columnNames) AS SELECT NULL, $nulls WHERE 0 $_autoGenerated'; - } - final select = table.columns.map(mapColumn).join(', '); - return 'CREATE VIEW ${quoteIdentifier(table.viewName)}("id", $columnNames) AS SELECT "id", $select FROM ${quoteIdentifier(table.internalName)} $_autoGenerated'; -} - -String mapColumn(Column column) { - return "CAST(json_extract(data, ${quoteJsonPath(column.name)}) as ${column.type.sqlite})"; -} - -List createViewTriggerStatements(Table table) { - if (table.localOnly) { - return createViewTriggerStatementsLocal(table); - } else if (table.insertOnly) { - return createViewTriggerStatementsInsert(table); - } - final viewName = table.viewName; - final type = table.name; - final internalNameE = quoteIdentifier(table.internalName); - - final jsonFragment = table.columns - .map((column) => - "${quoteString(column.name)}, NEW.${quoteIdentifier(column.name)}") - .join(', '); - final jsonFragmentOld = table.columns - .map((column) => - "${quoteString(column.name)}, OLD.${quoteIdentifier(column.name)}") - .join(', '); - // Names in alphabetical order - return [ - """ -CREATE TRIGGER ${quoteIdentifier('ps_view_delete_$viewName')} -INSTEAD OF DELETE ON ${quoteIdentifier(viewName)} -FOR EACH ROW -BEGIN - DELETE FROM $internalNameE WHERE id = OLD.id; - INSERT INTO ps_crud(tx_id, data) SELECT current_tx, json_object('op', 'DELETE', 'type', ${quoteString(type)}, 'id', OLD.id) FROM ps_tx WHERE id = 1; - INSERT INTO ps_oplog(bucket, op_id, op, row_type, row_id, hash, superseded) - SELECT '\$local', - 1, - 'REMOVE', - ${quoteString(type)}, - OLD.id, - 0, - 0; - INSERT OR REPLACE INTO ps_buckets(name, pending_delete, last_op, target_op) VALUES('\$local', 1, 0, $maxOpId); -END""", - """ -CREATE TRIGGER ${quoteIdentifier('ps_view_insert_$viewName')} -INSTEAD OF INSERT ON ${quoteIdentifier(viewName)} -FOR EACH ROW -BEGIN - SELECT CASE - WHEN (NEW.id IS NULL) - THEN RAISE (FAIL, 'id is required') - END; - INSERT INTO $internalNameE(id, data) - SELECT NEW.id, json_object($jsonFragment); - INSERT INTO ps_crud(tx_id, data) SELECT current_tx, json_object('op', 'PUT', 'type', ${quoteString(type)}, 'id', NEW.id, 'data', json(powersync_diff('{}', json_object($jsonFragment)))) FROM ps_tx WHERE id = 1; - INSERT INTO ps_oplog(bucket, op_id, op, row_type, row_id, hash, superseded) - SELECT '\$local', - 1, - 'REMOVE', - ${quoteString(type)}, - NEW.id, - 0, - 0; - INSERT OR REPLACE INTO ps_buckets(name, pending_delete, last_op, target_op) VALUES('\$local', 1, 0, $maxOpId); -END""", - """ -CREATE TRIGGER ${quoteIdentifier('ps_view_update_$viewName')} -INSTEAD OF UPDATE ON ${quoteIdentifier(viewName)} -FOR EACH ROW -BEGIN - SELECT CASE - WHEN (OLD.id != NEW.id) - THEN RAISE (FAIL, 'Cannot update id') - END; - UPDATE $internalNameE - SET data = json_object($jsonFragment) - WHERE id = NEW.id; - INSERT INTO ps_crud(tx_id, data) SELECT current_tx, json_object('op', 'PATCH', 'type', ${quoteString(type)}, 'id', NEW.id, 'data', json(powersync_diff(json_object($jsonFragmentOld), json_object($jsonFragment)))) FROM ps_tx WHERE id = 1; - INSERT INTO ps_oplog(bucket, op_id, op, row_type, row_id, hash, superseded) - SELECT '\$local', - 1, - 'REMOVE', - ${quoteString(type)}, - NEW.id, - 0, - 0; - INSERT OR REPLACE INTO ps_buckets(name, pending_delete, last_op, target_op) VALUES('\$local', 1, 0, $maxOpId); -END""" - ]; -} - -List createViewTriggerStatementsLocal(Table table) { - final viewName = table.viewName; - final internalNameE = quoteIdentifier(table.internalName); - - final jsonFragment = table.columns - .map((column) => - "${quoteString(column.name)}, NEW.${quoteIdentifier(column.name)}") - .join(', '); - // Names in alphabetical order - return [ - """ -CREATE TRIGGER ${quoteIdentifier('ps_view_delete_$viewName')} -INSTEAD OF DELETE ON ${quoteIdentifier(viewName)} -FOR EACH ROW -BEGIN - DELETE FROM $internalNameE WHERE id = OLD.id; -END""", - """ -CREATE TRIGGER ${quoteIdentifier('ps_view_insert_$viewName')} -INSTEAD OF INSERT ON ${quoteIdentifier(viewName)} -FOR EACH ROW -BEGIN - INSERT INTO $internalNameE(id, data) - SELECT NEW.id, json_object($jsonFragment); -END""", - """ -CREATE TRIGGER ${quoteIdentifier('ps_view_update_$viewName')} -INSTEAD OF UPDATE ON ${quoteIdentifier(viewName)} -FOR EACH ROW -BEGIN - SELECT CASE - WHEN (OLD.id != NEW.id) - THEN RAISE (FAIL, 'Cannot update id') - END; - UPDATE $internalNameE - SET data = json_object($jsonFragment) - WHERE id = NEW.id; -END""" - ]; -} - -List createViewTriggerStatementsInsert(Table table) { - final type = table.name; - final viewName = table.viewName; - - final jsonFragment = table.columns - .map((column) => - "${quoteString(column.name)}, NEW.${quoteIdentifier(column.name)}") - .join(', '); - return [ - """ -CREATE TRIGGER ${quoteIdentifier('ps_view_insert_$viewName')} -INSTEAD OF INSERT ON ${quoteIdentifier(viewName)} -FOR EACH ROW -BEGIN - INSERT INTO ps_crud(tx_id, data) SELECT current_tx, json_object('op', 'PUT', 'type', ${quoteString(type)}, 'id', NEW.id, 'data', json(powersync_diff('{}', json_object($jsonFragment)))) FROM ps_tx WHERE id = 1; -END""" - ]; -} - -/// Sync the schema to the local database. -/// -/// Must be wrapped in a transaction. -void updateSchema(sqlite.Database db, Schema schema) { - for (var table in schema.tables) { - table.validate(); - } - - _createTablesAndIndexes(db, schema); - - final existingViewRows = db.select( - "SELECT name FROM sqlite_master WHERE type='view' AND sql GLOB '*$_autoGenerated'"); - - Set toRemove = {for (var row in existingViewRows) row['name']}; - - for (var table in schema.tables) { - toRemove.remove(table.viewName); - - var createViewOp = createViewStatement(table); - var triggers = createViewTriggerStatements(table); - var existingRows = db.select( - "SELECT sql FROM sqlite_master WHERE (type = 'view' AND name = ?) OR (type = 'trigger' AND tbl_name = ?) ORDER BY type DESC, name ASC", - [table.viewName, table.viewName]); - if (existingRows.isNotEmpty) { - final dbSql = existingRows.map((row) => row['sql']).join('\n\n'); - final generatedSql = - [createViewOp, for (var trigger in triggers) trigger].join('\n\n'); - if (dbSql == generatedSql) { - // No change - keep it. - continue; - } else { - // View and/or triggers changed - delete and re-create. - db.execute('DROP VIEW ${quoteIdentifier(table.viewName)}'); - } - } else { - // New - create - } - db.execute(createViewOp); - for (final op in triggers) { - db.execute(op); - } - } - - for (var name in toRemove) { - db.execute('DROP VIEW ${quoteIdentifier(name)}'); - } -} +import 'schema_helpers.dart'; Future updateSchemaInIsolate( SqliteConnection database, Schema schema) async { - await database.computeWithDatabase((db) async { - updateSchema(db, schema); + await database.writeTransaction((tx) async { + await updateSchema(tx, schema); }); } - -/// Sync the schema to the local database. -/// -/// Does not create triggers or temporary views. -/// -/// Must be wrapped in a transaction. -void _createTablesAndIndexes(sqlite.Database db, Schema schema) { - // Make sure to refresh tables in the same transaction as updating them - final existingTableRows = db.select( - "SELECT name FROM sqlite_master WHERE type='table' AND name GLOB 'ps_data_*'"); - final existingIndexRows = db.select( - "SELECT name, sql FROM sqlite_master WHERE type='index' AND name GLOB 'ps_data_*'"); - - final Set remainingTables = {}; - final Map indexesToDrop = {}; - final List createIndexes = []; - for (final row in existingTableRows) { - remainingTables.add(row['name'] as String); - } - for (final row in existingIndexRows) { - indexesToDrop[row['name'] as String] = row['sql'] as String; - } - - for (final table in schema.tables) { - for (final index in table.indexes) { - final fullName = index.fullName(table); - final sql = index.toSqlDefinition(table); - if (indexesToDrop.containsKey(fullName)) { - final existingSql = indexesToDrop[fullName]; - if (existingSql == sql) { - // No change (don't drop) - indexesToDrop.remove(fullName); - } else { - // Drop and create - createIndexes.add(sql); - } - } else { - // New index - create - createIndexes.add(sql); - } - } - } - - for (final table in schema.tables) { - if (table.insertOnly) { - // Does not have a physical table - continue; - } - final tableName = table.internalName; - final exists = remainingTables.contains(tableName); - remainingTables.remove(tableName); - if (exists) { - continue; - } - - db.execute("""CREATE TABLE ${quoteIdentifier(tableName)} - ( - id TEXT PRIMARY KEY NOT NULL, - data TEXT - )"""); - - if (!table.localOnly) { - db.execute("""INSERT INTO ${quoteIdentifier(tableName)}(id, data) - SELECT id, data - FROM ps_untyped - WHERE type = ?""", [table.name]); - db.execute("""DELETE - FROM ps_untyped - WHERE type = ?""", [table.name]); - } - } - - for (final indexName in indexesToDrop.keys) { - db.execute('DROP INDEX ${quoteIdentifier(indexName)}'); - } - - for (final sql in createIndexes) { - db.execute(sql); - } - - for (final tableName in remainingTables) { - final typeMatch = RegExp("^ps_data__(.+)\$").firstMatch(tableName); - if (typeMatch != null) { - // Not local-only - final type = typeMatch[1]; - db.execute( - 'INSERT INTO ps_untyped(type, id, data) SELECT ?, id, data FROM ${quoteIdentifier(tableName)}', - [type]); - } - db.execute('DROP TABLE ${quoteIdentifier(tableName)}'); - } -} - -String? friendlyTableName(String table) { - final re = RegExp(r"^ps_data__(.+)$"); - final re2 = RegExp(r"^ps_data_local__(.+)$"); - final match = re.firstMatch(table) ?? re2.firstMatch(table); - return match?.group(1); -} diff --git a/packages/powersync/lib/src/stream_utils.dart b/packages/powersync/lib/src/stream_utils.dart index 3ead1be9..5cf7297d 100644 --- a/packages/powersync/lib/src/stream_utils.dart +++ b/packages/powersync/lib/src/stream_utils.dart @@ -57,8 +57,12 @@ Stream mergeStreams(List> streams) { Stream ndjson(ByteStream input) { final textInput = input.transform(convert.utf8.decoder); final lineInput = textInput.transform(const convert.LineSplitter()); - final jsonInput = lineInput.transform(StreamTransformer.fromHandlers( - handleData: (String data, EventSink sink) { + final jsonInput = lineInput.transform( + StreamTransformer.fromHandlers(handleError: (error, stackTrace, sink) { + /// On Web if the connection is closed, this error will throw, but + /// the stream is never closed. This closes the stream on error. + sink.close(); + }, handleData: (String data, EventSink sink) { sink.add(convert.jsonDecode(data)); })); return jsonInput; diff --git a/packages/powersync/lib/src/streaming_sync.dart b/packages/powersync/lib/src/streaming_sync.dart index 0ac44c87..b470cb7a 100644 --- a/packages/powersync/lib/src/streaming_sync.dart +++ b/packages/powersync/lib/src/streaming_sync.dart @@ -1,7 +1,5 @@ import 'dart:async'; import 'dart:convert' as convert; -import 'dart:io'; - import 'package:http/http.dart' as http; import 'package:powersync/src/exceptions.dart'; import 'package:powersync/src/log_internal.dart'; @@ -24,14 +22,14 @@ class StreamingSyncImplementation { final Future Function() uploadCrud; - late http.Client _client; - final Stream updateStream; final StreamController _statusStreamController = StreamController.broadcast(); late final Stream statusStream; + late final http.Client _client; + final StreamController _localPingController = StreamController.broadcast(); final Duration retryDelay; @@ -44,8 +42,9 @@ class StreamingSyncImplementation { this.invalidCredentialsCallback, required this.uploadCrud, required this.updateStream, - required this.retryDelay}) { - _client = http.Client(); + required this.retryDelay, + required http.Client client}) { + _client = client; statusStream = _statusStreamController.stream; } @@ -106,7 +105,7 @@ class StreamingSyncImplementation { } Future uploadCrudBatch() async { - if (adapter.hasCrud()) { + if ((await adapter.hasCrud())) { _updateStatus(uploading: true); await uploadCrud(); return false; @@ -132,7 +131,6 @@ class StreamingSyncImplementation { final response = await _client.get(uri, headers: { 'Content-Type': 'application/json', - 'User-Id': credentials.userId ?? '', 'Authorization': "Token ${credentials.token}" }); if (response.statusCode == 401) { @@ -177,7 +175,7 @@ class StreamingSyncImplementation { Future streamingSyncIteration() async { adapter.startSession(); - final bucketEntries = adapter.getBucketStates(); + final bucketEntries = await adapter.getBucketStates(); Map initialBucketStates = {}; @@ -331,11 +329,11 @@ class StreamingSyncImplementation { final request = http.Request('POST', uri); request.headers['Content-Type'] = 'application/json'; - request.headers['User-Id'] = credentials.userId ?? ''; request.headers['Authorization'] = "Token ${credentials.token}"; request.body = convert.jsonEncode(data); final res = await _client.send(request); + if (res.statusCode == 401) { if (invalidCredentialsCallback != null) { await invalidCredentialsCallback!(); @@ -357,7 +355,7 @@ class StreamingSyncImplementation { String _syncErrorMessage(Object? error) { if (error == null) { return 'Unknown'; - } else if (error is HttpException) { + } else if (error is http.ClientException) { return 'Sync service error'; } else if (error is SyncResponseException) { if (error.statusCode == 401) { @@ -365,8 +363,6 @@ String _syncErrorMessage(Object? error) { } else { return 'Sync service error'; } - } else if (error is SocketException) { - return 'Connection error'; } else if (error is ArgumentError || error is FormatException) { return 'Configuration error'; } else if (error is CredentialsException) { diff --git a/packages/powersync/lib/src/web/powersync_db.worker.dart b/packages/powersync/lib/src/web/powersync_db.worker.dart new file mode 100644 index 00000000..9432bb31 --- /dev/null +++ b/packages/powersync/lib/src/web/powersync_db.worker.dart @@ -0,0 +1,53 @@ +library; + +/// This file needs to be compiled to JavaScript with the command +/// dart compile js -O4 packages/powersync/lib/src/web/powersync_db.worker.dart -o assets/db_worker.js +/// The output should then be included in each project's `web` directory + +import 'package:powersync/src/open_factory/common_db_functions.dart'; +import 'package:sqlite_async/drift.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; +import 'package:uuid/uuid.dart'; + +void setupPowerSyncDatabase(CommonDatabase database) { + setupCommonDBFunctions(database); + setupCommonWorkerDB(database); + final uuid = Uuid(); + + database.createFunction( + functionName: 'uuid', + argumentCount: const AllowedArgumentCount(0), + function: (args) { + return uuid.v4(); + }, + ); + database.createFunction( + // Postgres compatibility + functionName: 'gen_random_uuid', + argumentCount: const AllowedArgumentCount(0), + function: (args) => uuid.v4(), + ); + database.createFunction( + functionName: 'powersync_sleep', + argumentCount: const AllowedArgumentCount(1), + function: (args) { + // Can't perform synchronous sleep on web + final millis = args[0] as int; + return millis; + }, + ); + + database.createFunction( + functionName: 'powersync_connection_name', + argumentCount: const AllowedArgumentCount(0), + function: (args) { + return 'N/A'; + }, + ); +} + +void main() { + WasmDatabase.workerMainForOpen( + setupAllDatabases: setupPowerSyncDatabase, + ); +} diff --git a/packages/powersync/lib/web_worker.dart b/packages/powersync/lib/web_worker.dart new file mode 100644 index 00000000..19983751 --- /dev/null +++ b/packages/powersync/lib/web_worker.dart @@ -0,0 +1 @@ +export 'src/web/powersync_db.worker.dart'; diff --git a/packages/powersync/pubspec.yaml b/packages/powersync/pubspec.yaml index dc4cf58f..6ac5498f 100644 --- a/packages/powersync/pubspec.yaml +++ b/packages/powersync/pubspec.yaml @@ -1,5 +1,5 @@ name: powersync -version: 1.2.1 +version: 1.3.0-alpha.1 homepage: https://powersync.com repository: https://github.com/powersync-ja/powersync.dart description: PowerSync Flutter SDK - keep PostgreSQL databases in sync with on-device SQLite databases. @@ -10,15 +10,19 @@ dependencies: flutter: sdk: flutter - sqlite_async: ^0.6.0 + sqlite_async: ^0.7.0-alpha.1 sqlite3_flutter_libs: ^0.5.15 + meta: ^1.0.0 http: ^1.1.0 uuid: ^4.2.0 async: ^2.10.0 logging: ^1.1.1 collection: ^1.17.0 + fetch_client: ^1.0.2 + js: ^0.6.7 dev_dependencies: + dcli: ^3.3.5 lints: ^3.0.0 test: ^1.25.0 test_api: ^0.7.0 @@ -26,7 +30,10 @@ dev_dependencies: sqlite3: ^2.3.0 shelf: ^1.4.1 shelf_router: ^1.1.4 + shelf_static: ^1.1.2 + stream_channel: ^2.1.2 path: ^1.8.3 + drift: 2.15.0 platforms: android: @@ -34,3 +41,4 @@ platforms: linux: macos: windows: + web: diff --git a/packages/powersync/test/bucket_storage_test.dart b/packages/powersync/test/bucket_storage_test.dart index 05548784..777bc558 100644 --- a/packages/powersync/test/bucket_storage_test.dart +++ b/packages/powersync/test/bucket_storage_test.dart @@ -1,11 +1,13 @@ import 'package:powersync/powersync.dart'; import 'package:powersync/src/bucket_storage.dart'; import 'package:powersync/src/sync_types.dart'; -import 'package:sqlite_async/sqlite3.dart' as sqlite; -import 'package:sqlite_async/mutex.dart'; +import 'package:sqlite3/common.dart'; import 'package:test/test.dart'; -import 'util.dart'; +import 'utils/abstract_test_utils.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); const putAsset1_1 = OplogEntry( opId: '1', @@ -40,17 +42,20 @@ const removeAsset1_5 = OplogEntry( void main() { group('Bucket Storage Tests', () { late PowerSyncDatabase powersync; - late sqlite.Database db; late BucketStorage bucketStorage; late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); + + powersync = await testUtils.setupPowerSync(path: path); + bucketStorage = BucketStorage(powersync); + await bucketStorage.initialized(); + }); - powersync = await setupPowerSync(path: path); - db = await setupSqlite(powersync: powersync); - bucketStorage = BucketStorage(db, mutex: Mutex()); + tearDown(() async { + await powersync.close(); }); Future syncLocalChecked(Checkpoint checkpoint) async { @@ -58,26 +63,30 @@ void main() { expect(result, equals(SyncLocalDatabaseResult(ready: true))); } - void expectAsset1_3() { + Future expectAsset1_3() async { expect( - db.select("SELECT id, description, make FROM assets WHERE id = 'O1'"), + await powersync.execute( + "SELECT id, description, make FROM assets WHERE id = 'O1'"), equals([ {'id': 'O1', 'description': 'bard', 'make': null} ])); } - void expectNoAsset1() { + Future expectNoAsset1() async { expect( - db.select("SELECT id, description, make FROM assets WHERE id = 'O1'"), + await powersync.execute( + "SELECT id, description, make FROM assets WHERE id = 'O1'"), equals([])); } - void expectNoAssets() { - expect(db.select("SELECT id, description, make FROM assets"), equals([])); + Future expectNoAssets() async { + expect( + await powersync.execute("SELECT id, description, make FROM assets"), + equals([])); } test('Basic Setup', () async { - expect(bucketStorage.getBucketStates(), equals([])); + expect(await bucketStorage.getBucketStates(), equals([])); await bucketStorage.saveSyncData(SyncDataBatch([ SyncBucketData( @@ -86,14 +95,15 @@ void main() { ) ])); - expect(bucketStorage.getBucketStates(), + final bucketStates = await bucketStorage.getBucketStates(); + expect(bucketStates, equals([const BucketState(bucket: 'bucket1', opId: '3')])); await syncLocalChecked(Checkpoint( lastOpId: '3', checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); - expectAsset1_3(); + await expectAsset1_3(); }); test('should get an object from multiple buckets', () async { @@ -110,7 +120,7 @@ void main() { BucketChecksum(bucket: 'bucket2', checksum: 3) ])); - expectAsset1_3(); + await expectAsset1_3(); }); test('should prioritize later updates', () async { @@ -128,7 +138,7 @@ void main() { BucketChecksum(bucket: 'bucket2', checksum: 1) ])); - expectAsset1_3(); + await expectAsset1_3(); }); test('should ignore a remove from one bucket', () async { @@ -143,7 +153,7 @@ void main() { BucketChecksum(bucket: 'bucket2', checksum: 7) ])); - expectAsset1_3(); + await expectAsset1_3(); }); test('should remove when removed from all buckets', () async { @@ -158,7 +168,7 @@ void main() { BucketChecksum(bucket: 'bucket2', checksum: 7) ])); - expectNoAssets(); + await expectNoAssets(); }); test('should use subkeys', () async { @@ -191,7 +201,8 @@ void main() { ])); expect( - db.select("SELECT id, description, make FROM assets WHERE id = 'O1'"), + await powersync.execute( + "SELECT id, description, make FROM assets WHERE id = 'O1'"), equals([ {'id': 'O1', 'description': 'B', 'make': null} ])); @@ -204,7 +215,7 @@ void main() { BucketChecksum(bucket: 'bucket1', checksum: 13), ])); - expectAsset1_3(); + await expectAsset1_3(); }); test('should fail checksum validation', () async { @@ -226,7 +237,7 @@ void main() { checkpointValid: false, checkpointFailures: ['bucket1', 'bucket2']))); - expectNoAssets(); + await expectNoAssets(); }); test('should delete buckets', () async { @@ -249,12 +260,12 @@ void main() { ])); // Bucket is deleted, but object is still present in other buckets. - expectAsset1_3(); + await expectAsset1_3(); await bucketStorage.removeBuckets(['bucket1']); await syncLocalChecked(Checkpoint(lastOpId: '3', checksums: [])); // Both buckets deleted - object removed. - expectNoAssets(); + await expectNoAssets(); }); test('should delete and re-create buckets', () async { @@ -291,12 +302,12 @@ void main() { await syncLocalChecked(Checkpoint(lastOpId: '3', checksums: [ BucketChecksum(bucket: 'bucket1', checksum: 4), ])); - expectAsset1_3(); + await expectAsset1_3(); // Now final delete await bucketStorage.removeBuckets(['bucket1']); await syncLocalChecked(Checkpoint(lastOpId: '3', checksums: [])); - expectNoAssets(); + await expectNoAssets(); }); test('should handle MOVE', () async { @@ -331,7 +342,7 @@ void main() { lastOpId: '3', checksums: [BucketChecksum(bucket: 'bucket1', checksum: 4)])); - expectAsset1_3(); + await expectAsset1_3(); }); test('should handle CLEAR', () async { @@ -369,9 +380,10 @@ void main() { // 2 + 3. 1 is replaced with 2. checksums: [BucketChecksum(bucket: 'bucket1', checksum: 5)])); - expectNoAsset1(); + await expectNoAsset1(); expect( - db.select("SELECT id, description FROM assets WHERE id = 'O2'"), + await powersync + .execute("SELECT id, description FROM assets WHERE id = 'O2'"), equals([ {'id': 'O2', 'description': 'bar'} ])); @@ -381,13 +393,13 @@ void main() { // Test case where a type is added to the schema after we already have the data. // Re-initialize with empty database - await cleanDb(path: path); + await testUtils.cleanDb(path: path); - powersync = PowerSyncDatabase.withFactory(TestOpenFactory(path: path), + powersync = PowerSyncDatabase.withFactory( + await testUtils.testFactory(path: path), schema: const Schema([])); await powersync.initialize(); - db = await setupSqlite(powersync: powersync); - bucketStorage = BucketStorage(db, mutex: Mutex()); + bucketStorage = BucketStorage(powersync); await bucketStorage.saveSyncData(SyncDataBatch([ SyncBucketData( @@ -399,20 +411,20 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '4', checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); - expect( - () => db.select('SELECT * FROM assets'), + + await expectLater(() async { + await powersync.execute('SELECT * FROM assets'); + }, throwsA((e) => - e is sqlite.SqliteException && - e.message.contains('no such table'))); + e is SqliteException && e.message.contains('no such table'))); await powersync.close(); // Now open another instance with new schema - powersync = PowerSyncDatabase.withFactory(TestOpenFactory(path: path), - schema: schema); - db = await setupSqlite(powersync: powersync); - - expectAsset1_3(); + powersync = PowerSyncDatabase.withFactory( + await testUtils.testFactory(path: path), + schema: defaultSchema); + await expectAsset1_3(); }); test('should remove types', () async { @@ -427,26 +439,25 @@ void main() { lastOpId: '3', checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); - expectAsset1_3(); + await expectAsset1_3(); await powersync.close(); // Now open another instance with new schema - powersync = PowerSyncDatabase.withFactory(TestOpenFactory(path: path), + powersync = PowerSyncDatabase.withFactory( + await testUtils.testFactory(path: path), schema: const Schema([])); - db = await setupSqlite(powersync: powersync); - expect( - () => db.select('SELECT * FROM assets'), + + await expectLater( + () async => await powersync.execute('SELECT * FROM assets'), throwsA((e) => - e is sqlite.SqliteException && - e.message.contains('no such table'))); + e is SqliteException && e.message.contains('no such table'))); // Add schema again - powersync = PowerSyncDatabase.withFactory(TestOpenFactory(path: path), + powersync = PowerSyncDatabase.withFactory( + await testUtils.testFactory(path: path), schema: schema); - db = await setupSqlite(powersync: powersync); - - expectAsset1_3(); + await expectAsset1_3(); }); test('should compact', () async { @@ -470,7 +481,7 @@ void main() { writeCheckpoint: '4', checksums: [BucketChecksum(bucket: 'bucket1', checksum: 7)])); - final stats = db.select( + final stats = await powersync.execute( 'SELECT row_type as type, row_id as id, count(*) as count FROM ps_oplog GROUP BY row_type, row_id ORDER BY row_type, row_id'); expect( stats, @@ -493,9 +504,9 @@ void main() { checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); // Local save - db.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); + powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); expect( - db.select('SELECT id FROM assets WHERE id = \'O3\''), + await powersync.execute('SELECT id FROM assets WHERE id = \'O3\''), equals([ {'id': 'O3'} ])); @@ -507,7 +518,7 @@ void main() { checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); expect(result, equals(SyncLocalDatabaseResult(ready: false))); - final batch = bucketStorage.getCrudBatch(); + final batch = await bucketStorage.getCrudBatch(); await batch!.complete(); await bucketStorage.updateLocalTarget(() async { return '4'; @@ -522,7 +533,7 @@ void main() { // The data must still be present locally. expect( - db.select('SELECT id FROM assets WHERE id = \'O3\''), + await powersync.execute('SELECT id FROM assets WHERE id = \'O3\''), equals([ {'id': 'O3'} ])); @@ -538,7 +549,8 @@ void main() { checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); // Since the object was not in the sync response, it is deleted. - expect(db.select('SELECT id FROM assets WHERE id = \'O3\''), equals([])); + expect(await powersync.execute('SELECT id FROM assets WHERE id = \'O3\''), + equals([])); }); test( @@ -557,9 +569,9 @@ void main() { checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); // Local save - db.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); + powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); - final batch = bucketStorage.getCrudBatch(); + final batch = await bucketStorage.getCrudBatch(); await batch!.complete(); await bucketStorage.updateLocalTarget(() async { return '4'; @@ -575,7 +587,7 @@ void main() { SyncDataBatch([SyncBucketData(bucket: 'bucket1', data: [])])); // Add more data before syncLocalDatabase. - db.execute('INSERT INTO assets(id) VALUES(?)', ['O4']); + powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O4']); final result4 = await bucketStorage.syncLocalDatabase(Checkpoint( lastOpId: '5', @@ -600,11 +612,11 @@ void main() { checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); // Local save - db.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); - final batch = bucketStorage.getCrudBatch(); + await powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); + final batch = await bucketStorage.getCrudBatch(); // Add more data before the complete() call - db.execute('INSERT INTO assets(id) VALUES(?)', ['O4']); + await powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O4']); await batch!.complete(); await bucketStorage.updateLocalTarget(() async { return '4'; @@ -639,8 +651,8 @@ void main() { checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); // Local save - db.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); - final batch = bucketStorage.getCrudBatch(); + powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); + final batch = await bucketStorage.getCrudBatch(); await batch!.complete(); await bucketStorage.updateLocalTarget(() async { return '4'; @@ -667,7 +679,8 @@ void main() { checksums: [BucketChecksum(bucket: 'bucket1', checksum: 11)])); expect( - db.select('SELECT description FROM assets WHERE id = \'O3\''), + await powersync + .execute('SELECT description FROM assets WHERE id = \'O3\''), equals([ {'description': 'server updated'} ])); @@ -687,16 +700,18 @@ void main() { checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); // Local insert, later rejected by server - db.execute('INSERT INTO assets(id, description) VALUES(?, ?)', + await powersync.execute( + 'INSERT INTO assets(id, description) VALUES(?, ?)', ['O3', 'inserted']); - final batch = bucketStorage.getCrudBatch(); + final batch = await bucketStorage.getCrudBatch(); await batch!.complete(); await bucketStorage.updateLocalTarget(() async { return '4'; }); expect( - db.select('SELECT description FROM assets WHERE id = \'O3\''), + await powersync + .execute('SELECT description FROM assets WHERE id = \'O3\''), equals([ {'description': 'inserted'} ])); @@ -706,7 +721,9 @@ void main() { writeCheckpoint: '4', checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); - expect(db.select('SELECT description FROM assets WHERE id = \'O3\''), + expect( + await powersync + .execute('SELECT description FROM assets WHERE id = \'O3\''), equals([])); }); @@ -724,12 +741,14 @@ void main() { checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); // Local delete, later rejected by server - db.execute('DELETE FROM assets WHERE id = ?', ['O2']); + await powersync.execute('DELETE FROM assets WHERE id = ?', ['O2']); - expect(db.select('SELECT description FROM assets WHERE id = \'O2\''), + expect( + await powersync + .execute('SELECT description FROM assets WHERE id = \'O2\''), equals([])); // Simulate a permissions error when uploading - data should be preserved. - final batch = bucketStorage.getCrudBatch(); + final batch = await bucketStorage.getCrudBatch(); await batch!.complete(); await bucketStorage.updateLocalTarget(() async { @@ -742,7 +761,8 @@ void main() { checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); expect( - db.select('SELECT description FROM assets WHERE id = \'O2\''), + await powersync + .execute('SELECT description FROM assets WHERE id = \'O2\''), equals([ {'description': 'bar'} ])); @@ -762,16 +782,17 @@ void main() { checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); // Local update, later rejected by server - db.execute( + await powersync.execute( 'UPDATE assets SET description = ? WHERE id = ?', ['updated', 'O2']); expect( - db.select('SELECT description FROM assets WHERE id = \'O2\''), + await powersync + .execute('SELECT description FROM assets WHERE id = \'O2\''), equals([ {'description': 'updated'} ])); // Simulate a permissions error when uploading - data should be preserved. - final batch = bucketStorage.getCrudBatch(); + final batch = await bucketStorage.getCrudBatch(); await batch!.complete(); await bucketStorage.updateLocalTarget(() async { @@ -784,7 +805,8 @@ void main() { checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); expect( - db.select('SELECT description FROM assets WHERE id = \'O2\''), + await powersync + .execute('SELECT description FROM assets WHERE id = \'O2\''), equals([ {'description': 'bar'} ])); diff --git a/packages/powersync/test/crud_test.dart b/packages/powersync/test/crud_test.dart index 4abe6e79..ebe036db 100644 --- a/packages/powersync/test/crud_test.dart +++ b/packages/powersync/test/crud_test.dart @@ -1,9 +1,10 @@ import 'package:powersync/powersync.dart'; -import 'package:sqlite_async/sqlite3.dart' as sqlite; +import 'package:sqlite_async/sqlite3_common.dart'; import 'package:test/test.dart'; -import 'util.dart'; +import 'utils/test_utils_impl.dart'; +final testUtils = TestUtils(); const testId = "2290de4f-0488-4e50-abed-f8e8eb1d0b42"; void main() { @@ -12,10 +13,10 @@ void main() { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); - powersync = await setupPowerSync(path: path); + powersync = await testUtils.setupPowerSync(path: path); }); test('INSERT', () async { @@ -72,7 +73,7 @@ void main() { [testId, 'test3']); }, throwsA((e) => - e is sqlite.SqliteException && + e is SqliteException && e.message.contains('UNIQUE constraint failed'))); }); @@ -132,13 +133,13 @@ void main() { [testId, 'test2', 'test3']); }, throwsA((e) => - e is sqlite.SqliteException && + e is SqliteException && e.message.contains('cannot UPSERT a view'))); }); test('INSERT-only tables', () async { await powersync.disconnectAndClear(); - powersync = await setupPowerSync( + powersync = await testUtils.setupPowerSync( path: path, schema: const Schema([ Table.insertOnly( @@ -161,6 +162,7 @@ void main() { expect(await powersync.getAll('SELECT * FROM logs'), equals([])); var tx = (await powersync.getNextCrudTransaction())!; + expect(tx.transactionId, equals(2)); expect( tx.crud, @@ -250,6 +252,7 @@ void main() { }); var tx1 = (await powersync.getNextCrudTransaction())!; + expect(tx1.transactionId, equals(1)); expect( tx1.crud, diff --git a/packages/powersync/test/offline_online_test.dart b/packages/powersync/test/offline_online_test.dart index ba27d0eb..d270f48a 100644 --- a/packages/powersync/test/offline_online_test.dart +++ b/packages/powersync/test/offline_online_test.dart @@ -3,7 +3,9 @@ import 'dart:convert'; import 'package:powersync/powersync.dart'; import 'package:test/test.dart'; -import 'util.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); const assetId = "2290de4f-0488-4e50-abed-f8e8eb1d0b42"; const userId = "3390de4f-0488-4e50-abed-f8e8eb1d0b42"; @@ -72,14 +74,15 @@ void main() { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); }); test('Switch from offline-only to online', () async { // Start with "offline-only" schema. // This does not record any operations to the crud queue. - final db = await setupPowerSync(path: path, schema: makeSchema(false)); + final db = + await testUtils.setupPowerSync(path: path, schema: makeSchema(false)); await db.execute('INSERT INTO customers(id, name, email) VALUES(?, ?, ?)', [customerId, 'test customer', 'test@example.org']); diff --git a/packages/powersync/test/performance_native_test.dart b/packages/powersync/test/performance_native_test.dart new file mode 100644 index 00000000..9def5945 --- /dev/null +++ b/packages/powersync/test/performance_native_test.dart @@ -0,0 +1,119 @@ +@TestOn('!browser') +import 'package:powersync/powersync.dart'; +import 'package:test/test.dart'; + +import 'performance_shared_test.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); + +void main() { + group('Performance Tests', () { + late String path; + + setUp(() async { + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); + }); + + tearDown(() async { + // await cleanDb(path: path); + }); + + // Manual tests + test('Insert Performance 3a - computeWithDatabase', () async { + final db = PowerSyncDatabase.withFactory( + await testUtils.testFactory(path: path), + schema: pschema); + await db.initialize(); + final timer = Stopwatch()..start(); + + await db.computeWithDatabase((db) async { + for (var i = 0; i < 1000; i++) { + db.execute( + 'INSERT INTO customers(id, name, email) VALUES(uuid(), ?, ?)', + ['Test User', 'user@example.org']); + } + }); + + print("Completed synchronous inserts in ${timer.elapsed}"); + expect(await db.get('SELECT count(*) as count FROM customers'), + equals({'count': 1000})); + }); + + test('Insert Performance 3b - prepared statement', () async { + final db = PowerSyncDatabase.withFactory( + await testUtils.testFactory(path: path), + schema: pschema); + await db.initialize(); + final timer = Stopwatch()..start(); + + await db.computeWithDatabase((db) async { + var stmt = db.prepare( + 'INSERT INTO customers(id, name, email) VALUES(uuid(), ?, ?)'); + try { + for (var i = 0; i < 1000; i++) { + stmt.execute(['Test User', 'user@example.org']); + } + } finally { + stmt.dispose(); + } + }); + + print("Completed synchronous inserts prepared in ${timer.elapsed}"); + expect(await db.get('SELECT count(*) as count FROM customers'), + equals({'count': 1000})); + }); + + test('Insert Performance 3c - prepared statement, dart-generated ids', + () async { + final db = PowerSyncDatabase.withFactory( + await testUtils.testFactory(path: path), + schema: pschema); + await db.initialize(); + // Test to exclude the function overhead time of generating uuids + final timer = Stopwatch()..start(); + + await db.computeWithDatabase((db) async { + var ids = List.generate(1000, (index) => uuid.v4()); + var stmt = db + .prepare('INSERT INTO customers(id, name, email) VALUES(?, ?, ?)'); + try { + for (var id in ids) { + stmt.execute([id, 'Test User', 'user@example.org']); + } + } finally { + stmt.dispose(); + } + }); + + print("Completed synchronous inserts prepared in ${timer.elapsed}"); + expect(await db.get('SELECT count(*) as count FROM customers'), + equals({'count': 1000})); + }); + test('Insert Performance 3d - prepared statement, pre-generated ids', + () async { + final db = PowerSyncDatabase.withFactory( + await testUtils.testFactory(path: path), + schema: pschema); + await db.initialize(); + // Test to completely exclude time taken to generate uuids + var ids = List.generate(1000, (index) => uuid.v4()); + + final timer = Stopwatch()..start(); + + await db.computeWithDatabase((db) async { + var stmt = db + .prepare('INSERT INTO customers(id, name, email) VALUES(?, ?, ?)'); + for (var id in ids) { + stmt.execute([id, 'Test User', 'user@example.org']); + } + stmt.dispose(); + }); + + print("Completed synchronous inserts prepared in ${timer.elapsed}"); + expect(await db.get('SELECT count(*) as count FROM customers'), + equals({'count': 1000})); + }); + }); +} diff --git a/packages/powersync/test/performance_shared_test.dart b/packages/powersync/test/performance_shared_test.dart new file mode 100644 index 00000000..5044c099 --- /dev/null +++ b/packages/powersync/test/performance_shared_test.dart @@ -0,0 +1,110 @@ +import 'package:powersync/powersync.dart'; +import 'package:test/test.dart'; + +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); + +const pschema = Schema([ + Table.localOnly('assets', [ + Column.text('created_at'), + Column.text('make'), + Column.text('model'), + Column.text('serial_number'), + Column.integer('quantity'), + Column.text('user_id'), + Column.text('customer_id'), + Column.text('description'), + ]), + Table.localOnly('customers', [Column.text('name'), Column.text('email')]) +]); + +void main() { + group('Performance Tests', () { + late String path; + + setUp(() async { + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); + }); + + tearDown(() async { + // await cleanDb(path: path); + }); + + // Manual tests + test('Insert Performance 1 - direct', () async { + final db = PowerSyncDatabase.withFactory( + await testUtils.testFactory(path: path), + schema: pschema); + await db.initialize(); + final timer = Stopwatch()..start(); + + for (var i = 0; i < 1000; i++) { + await db.execute( + 'INSERT INTO customers(id, name, email) VALUES(uuid(), ?, ?)', + ['Test User', 'user@example.org']); + } + print("Completed sequential inserts in ${timer.elapsed}"); + expect(await db.get('SELECT count(*) as count FROM customers'), + equals({'count': 1000})); + }); + + test('Insert Performance 2 - writeTransaction', () async { + final db = PowerSyncDatabase.withFactory( + await testUtils.testFactory(path: path), + schema: pschema); + await db.initialize(); + final timer = Stopwatch()..start(); + + await db.writeTransaction((tx) async { + for (var i = 0; i < 1000; i++) { + await tx.execute( + 'INSERT INTO customers(id, name, email) VALUES(uuid(), ?, ?)', + ['Test User', 'user@example.org']); + } + }); + print("Completed transaction inserts in ${timer.elapsed}"); + expect(await db.get('SELECT count(*) as count FROM customers'), + equals({'count': 1000})); + }); + + test('Insert Performance 4 - pipelined', () async { + final db = PowerSyncDatabase.withFactory( + await testUtils.testFactory(path: path), + schema: pschema); + await db.initialize(); + final timer = Stopwatch()..start(); + + await db.writeTransaction((tx) async { + List futures = []; + for (var i = 0; i < 1000; i++) { + var future = tx.execute( + 'INSERT INTO customers(id, name, email) VALUES(uuid(), ?, ?)', + ['Test User', 'user@example.org']); + futures.add(future); + } + await Future.wait(futures); + }); + print("Completed pipelined inserts in ${timer.elapsed}"); + expect(await db.get('SELECT count(*) as count FROM customers'), + equals({'count': 1000})); + }); + + test('Insert Performance 5 - executeBatch', () async { + final db = PowerSyncDatabase.withFactory( + await testUtils.testFactory(path: path), + schema: pschema); + await db.initialize(); + final timer = Stopwatch()..start(); + + var parameters = List.generate( + 1000, (index) => [uuid.v4(), 'Test user', 'user@example.org']); + await db.executeBatch( + 'INSERT INTO customers(id, name, email) VALUES(?, ?, ?)', parameters); + print("Completed executeBatch in ${timer.elapsed}"); + expect(await db.get('SELECT count(*) as count FROM customers'), + equals({'count': 1000})); + }); + }); +} diff --git a/packages/powersync/test/performance_test.dart b/packages/powersync/test/performance_test.dart deleted file mode 100644 index 61a5bb1e..00000000 --- a/packages/powersync/test/performance_test.dart +++ /dev/null @@ -1,195 +0,0 @@ -import 'package:powersync/powersync.dart'; -import 'package:test/test.dart'; - -import 'util.dart'; - -const pschema = Schema([ - Table.localOnly('assets', [ - Column.text('created_at'), - Column.text('make'), - Column.text('model'), - Column.text('serial_number'), - Column.integer('quantity'), - Column.text('user_id'), - Column.text('customer_id'), - Column.text('description'), - ]), - Table.localOnly('customers', [Column.text('name'), Column.text('email')]) -]); - -void main() { - group('Performance Tests', () { - late String path; - - setUp(() async { - path = dbPath(); - await cleanDb(path: path); - }); - - tearDown(() async { - // await cleanDb(path: path); - }); - - // Manual tests - test('Insert Performance 1 - direct', () async { - final db = PowerSyncDatabase.withFactory(TestOpenFactory(path: path), - schema: pschema); - await db.initialize(); - final timer = Stopwatch()..start(); - - for (var i = 0; i < 1000; i++) { - await db.execute( - 'INSERT INTO customers(id, name, email) VALUES(uuid(), ?, ?)', - ['Test User', 'user@example.org']); - } - print("Completed sequential inserts in ${timer.elapsed}"); - expect(await db.get('SELECT count(*) as count FROM customers'), - equals({'count': 1000})); - }); - - test('Insert Performance 2 - writeTransaction', () async { - final db = PowerSyncDatabase.withFactory(TestOpenFactory(path: path), - schema: pschema); - await db.initialize(); - final timer = Stopwatch()..start(); - - await db.writeTransaction((tx) async { - for (var i = 0; i < 1000; i++) { - await tx.execute( - 'INSERT INTO customers(id, name, email) VALUES(uuid(), ?, ?)', - ['Test User', 'user@example.org']); - } - }); - print("Completed transaction inserts in ${timer.elapsed}"); - expect(await db.get('SELECT count(*) as count FROM customers'), - equals({'count': 1000})); - }); - - test('Insert Performance 3a - computeWithDatabase', () async { - final db = PowerSyncDatabase.withFactory(TestOpenFactory(path: path), - schema: pschema); - await db.initialize(); - final timer = Stopwatch()..start(); - - await db.computeWithDatabase((db) async { - for (var i = 0; i < 1000; i++) { - db.execute( - 'INSERT INTO customers(id, name, email) VALUES(uuid(), ?, ?)', - ['Test User', 'user@example.org']); - } - }); - - print("Completed synchronous inserts in ${timer.elapsed}"); - expect(await db.get('SELECT count(*) as count FROM customers'), - equals({'count': 1000})); - }); - - test('Insert Performance 3b - prepared statement', () async { - final db = PowerSyncDatabase.withFactory(TestOpenFactory(path: path), - schema: pschema); - await db.initialize(); - final timer = Stopwatch()..start(); - - await db.computeWithDatabase((db) async { - var stmt = db.prepare( - 'INSERT INTO customers(id, name, email) VALUES(uuid(), ?, ?)'); - try { - for (var i = 0; i < 1000; i++) { - stmt.execute(['Test User', 'user@example.org']); - } - } finally { - stmt.dispose(); - } - }); - - print("Completed synchronous inserts prepared in ${timer.elapsed}"); - expect(await db.get('SELECT count(*) as count FROM customers'), - equals({'count': 1000})); - }); - - test('Insert Performance 3c - prepared statement, dart-generated ids', - () async { - final db = PowerSyncDatabase.withFactory(TestOpenFactory(path: path), - schema: pschema); - await db.initialize(); - // Test to exclude the function overhead time of generating uuids - final timer = Stopwatch()..start(); - - await db.computeWithDatabase((db) async { - var ids = List.generate(1000, (index) => uuid.v4()); - var stmt = db - .prepare('INSERT INTO customers(id, name, email) VALUES(?, ?, ?)'); - try { - for (var id in ids) { - stmt.execute([id, 'Test User', 'user@example.org']); - } - } finally { - stmt.dispose(); - } - }); - - print("Completed synchronous inserts prepared in ${timer.elapsed}"); - expect(await db.get('SELECT count(*) as count FROM customers'), - equals({'count': 1000})); - }); - test('Insert Performance 3d - prepared statement, pre-generated ids', - () async { - final db = PowerSyncDatabase.withFactory(TestOpenFactory(path: path), - schema: pschema); - await db.initialize(); - // Test to completely exclude time taken to generate uuids - var ids = List.generate(1000, (index) => uuid.v4()); - - final timer = Stopwatch()..start(); - - await db.computeWithDatabase((db) async { - var stmt = db - .prepare('INSERT INTO customers(id, name, email) VALUES(?, ?, ?)'); - for (var id in ids) { - stmt.execute([id, 'Test User', 'user@example.org']); - } - stmt.dispose(); - }); - - print("Completed synchronous inserts prepared in ${timer.elapsed}"); - expect(await db.get('SELECT count(*) as count FROM customers'), - equals({'count': 1000})); - }); - - test('Insert Performance 4 - pipelined', () async { - final db = PowerSyncDatabase.withFactory(TestOpenFactory(path: path), - schema: pschema); - await db.initialize(); - final timer = Stopwatch()..start(); - - await db.writeTransaction((tx) async { - List futures = []; - for (var i = 0; i < 1000; i++) { - var future = tx.execute( - 'INSERT INTO customers(id, name, email) VALUES(uuid(), ?, ?)', - ['Test User', 'user@example.org']); - futures.add(future); - } - await Future.wait(futures); - }); - print("Completed pipelined inserts in ${timer.elapsed}"); - expect(await db.get('SELECT count(*) as count FROM customers'), - equals({'count': 1000})); - }); - - test('Insert Performance 5 - executeBatch', () async { - final db = PowerSyncDatabase.withFactory(TestOpenFactory(path: path), - schema: pschema); - await db.initialize(); - final timer = Stopwatch()..start(); - - var parameters = List.generate( - 1000, (index) => [uuid.v4(), 'Test user', 'user@example.org']); - await db.executeBatch( - 'INSERT INTO customers(id, name, email) VALUES(?, ?, ?)', parameters); - print("Completed executeBatch in ${timer.elapsed}"); - expect(await db.get('SELECT count(*) as count FROM customers'), - equals({'count': 1000})); - }); - }); -} diff --git a/packages/powersync/test/powersync_test.dart b/packages/powersync/test/powersync_native_test.dart similarity index 54% rename from packages/powersync/test/powersync_test.dart rename to packages/powersync/test/powersync_native_test.dart index 4bac8121..aeda23d8 100644 --- a/packages/powersync/test/powersync_test.dart +++ b/packages/powersync/test/powersync_native_test.dart @@ -1,27 +1,30 @@ +@TestOn('!browser') import 'dart:async'; import 'dart:math'; import 'package:powersync/powersync.dart'; -import 'package:sqlite_async/mutex.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; import 'package:test/test.dart'; -import 'package:sqlite_async/sqlite3.dart' as sqlite; -import 'util.dart'; +import 'utils/abstract_test_utils.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); void main() { group('Basic Tests', () { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); }); tearDown(() async { - await cleanDb(path: path); + await testUtils.cleanDb(path: path); }); test('Basic Setup', () async { - final db = await setupPowerSync(path: path); + final db = await testUtils.setupPowerSync(path: path); await db.execute( 'INSERT INTO assets(id, make) VALUES(uuid(), ?)', ['Test Make']); final result = await db.get('SELECT make FROM assets'); @@ -40,8 +43,10 @@ void main() { // Manual test test('Concurrency', () async { - final db = PowerSyncDatabase.withFactory(TestOpenFactory(path: path), - schema: schema, maxReaders: 3); + final db = PowerSyncDatabase.withFactory( + await testUtils.testFactory(path: path), + schema: defaultSchema, + maxReaders: 3); await db.initialize(); print("${DateTime.now()} start"); @@ -54,7 +59,7 @@ void main() { }); test('read-only transactions', () async { - final db = await setupPowerSync(path: path); + final db = await testUtils.setupPowerSync(path: path); // Can read await db.getAll("WITH test AS (SELECT 1 AS one) SELECT * FROM test"); @@ -64,7 +69,7 @@ void main() { await db.getAll('INSERT INTO assets(id) VALUES(?)', ['test']); }, throwsA((e) => - e is sqlite.SqliteException && + e is SqliteException && e.message .contains('attempt to write in a read-only transaction'))); @@ -77,7 +82,7 @@ void main() { "WITH test AS (SELECT 1 AS one) INSERT INTO assets(id) SELECT one FROM test"); }, throwsA((e) => - e is sqlite.SqliteException && + e is SqliteException && e.message .contains('attempt to write in a read-only transaction'))); @@ -88,66 +93,10 @@ void main() { }); }); - test('should not allow direct db calls within a transaction callback', - () async { - final db = await setupPowerSync(path: path); - - await db.writeTransaction((tx) async { - await expectLater(() async { - await db.execute('INSERT INTO assets(id) VALUES(?)', ['test']); - }, throwsA((e) => e is LockError && e.message.contains('tx.execute'))); - }); - }); - - test('should not allow read-only db calls within transaction callback', - () async { - final db = await setupPowerSync(path: path); - - await db.writeTransaction((tx) async { - // This uses a different connection, so it _could_ work. - // But it's likely unintentional and could cause weird bugs, so we don't - // allow it by default. - await expectLater(() async { - await db.getAll('SELECT * FROM assets'); - }, throwsA((e) => e is LockError && e.message.contains('tx.getAll'))); - }); - - await db.readTransaction((tx) async { - // This does actually attempt a lock on the same connection, so it - // errors. - // This also exposes an interesting test case where the read transaction - // opens another connection, but doesn't use it. - await expectLater(() async { - await db.getAll('SELECT * FROM assets'); - }, throwsA((e) => e is LockError && e.message.contains('tx.getAll'))); - }); - }); - - test('should not allow read-only db calls within lock callback', () async { - final db = await setupPowerSync(path: path); - // Locks - should behave the same as transactions above - - await db.writeLock((tx) async { - await expectLater(() async { - await db.getOptional('SELECT * FROM assets'); - }, - throwsA( - (e) => e is LockError && e.message.contains('tx.getOptional'))); - }); - - await db.readLock((tx) async { - await expectLater(() async { - await db.getOptional('SELECT * FROM assets'); - }, - throwsA( - (e) => e is LockError && e.message.contains('tx.getOptional'))); - }); - }); - test( 'should allow read-only db calls within transaction callback in separate zone', () async { - final db = await setupPowerSync(path: path); + final db = await testUtils.setupPowerSync(path: path); // Get a reference to the parent zone (outside the transaction). final zone = Zone.current; @@ -181,12 +130,5 @@ void main() { // }); // }); }); - - test('should allow PRAMGAs', () async { - final db = await setupPowerSync(path: path); - // Not allowed in transactions, but does work as a direct statement. - await db.execute('PRAGMA wal_checkpoint(TRUNCATE)'); - await db.execute('VACUUM'); - }); }); } diff --git a/packages/powersync/test/powersync_shared_test.dart b/packages/powersync/test/powersync_shared_test.dart new file mode 100644 index 00000000..3cb3a9c3 --- /dev/null +++ b/packages/powersync/test/powersync_shared_test.dart @@ -0,0 +1,83 @@ +import 'package:sqlite_async/mutex.dart'; +import 'package:test/test.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); + +void main() { + group('Basic Tests', () { + late String path; + + setUp(() async { + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); + }); + + tearDown(() async { + await testUtils.cleanDb(path: path); + }); + + test('should not allow direct db calls within a transaction callback', + () async { + final db = await testUtils.setupPowerSync(path: path); + + await db.writeTransaction((tx) async { + await expectLater(() async { + await db.execute('INSERT INTO assets(id) VALUES(?)', ['test']); + }, throwsA((e) => e is LockError && e.message.contains('tx.execute'))); + }); + }); + + test('should not allow read-only db calls within transaction callback', + () async { + final db = await testUtils.setupPowerSync(path: path); + + await db.writeTransaction((tx) async { + // This uses a different connection, so it _could_ work. + // But it's likely unintentional and could cause weird bugs, so we don't + // allow it by default. + await expectLater(() async { + await db.getAll('SELECT * FROM assets'); + }, throwsA((e) => e is LockError && e.message.contains('tx.getAll'))); + }); + + await db.readTransaction((tx) async { + // This does actually attempt a lock on the same connection, so it + // errors. + // This also exposes an interesting test case where the read transaction + // opens another connection, but doesn't use it. + await expectLater(() async { + await db.getAll('SELECT * FROM assets'); + }, throwsA((e) => e is LockError && e.message.contains('tx.getAll'))); + }); + }); + + test('should not allow read-only db calls within lock callback', () async { + final db = await testUtils.setupPowerSync(path: path); + // Locks - should behave the same as transactions above + + await db.writeLock((tx) async { + await expectLater(() async { + await db.getOptional('SELECT * FROM assets'); + }, + throwsA( + (e) => e is LockError && e.message.contains('tx.getOptional'))); + }); + + await db.readLock((tx) async { + await expectLater(() async { + await db.getOptional('SELECT * FROM assets'); + }, + throwsA( + (e) => e is LockError && e.message.contains('tx.getOptional'))); + }); + }); + + test('should allow PRAMGAs', () async { + final db = await testUtils.setupPowerSync(path: path); + // Not allowed in transactions, but does work as a direct statement. + await db.execute('PRAGMA wal_checkpoint(TRUNCATE)'); + await db.execute('VACUUM'); + }); + }); +} diff --git a/packages/powersync/test/schema_test.dart b/packages/powersync/test/schema_test.dart index 613808ce..f6a1bf55 100644 --- a/packages/powersync/test/schema_test.dart +++ b/packages/powersync/test/schema_test.dart @@ -1,7 +1,9 @@ import 'package:powersync/powersync.dart'; import 'package:test/test.dart'; -import 'util.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); const testId = "2290de4f-0488-4e50-abed-f8e8eb1d0b42"; final schema = Schema([ @@ -28,15 +30,16 @@ void main() { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); }); test('Schema versioning', () async { // Test that powersync_replace_schema() is a no-op when the schema is not // modified. - final powersync = await setupPowerSync(path: path, schema: schema); + final powersync = + await testUtils.setupPowerSync(path: path, schema: schema); final versionBefore = await powersync.get('PRAGMA schema_version'); await powersync.updateSchema(schema); @@ -106,8 +109,10 @@ void main() { greaterThan(versionAfter2['schema_version'])); }); + /// The assets table is locked after performing the EXPLAIN QUERY test('Indexing', () async { - final powersync = await setupPowerSync(path: path, schema: schema); + final powersync = + await testUtils.setupPowerSync(path: path, schema: schema); final results = await powersync.execute( 'EXPLAIN QUERY PLAN SELECT * FROM assets WHERE make = ?', ['test']); diff --git a/packages/powersync/test/server/asset_server.dart b/packages/powersync/test/server/asset_server.dart new file mode 100644 index 00000000..272e0bfa --- /dev/null +++ b/packages/powersync/test/server/asset_server.dart @@ -0,0 +1,41 @@ +import 'dart:io'; + +import 'package:shelf/shelf.dart'; +import 'package:shelf/shelf_io.dart' as io; +import 'package:shelf_static/shelf_static.dart'; +import 'package:stream_channel/stream_channel.dart'; + +const _corsHeaders = {'Access-Control-Allow-Origin': '*'}; + +Middleware cors() { + Response? handleOptionsRequest(Request request) { + if (request.method == 'OPTIONS') { + return Response.ok(null, headers: _corsHeaders); + } else { + // Returning null will run the regular request handler + return null; + } + } + + Response addCorsHeaders(Response response) { + return response.change(headers: _corsHeaders); + } + + return createMiddleware( + requestHandler: handleOptionsRequest, responseHandler: addCorsHeaders); +} + +Future hybridMain(StreamChannel channel) async { + final server = await HttpServer.bind('localhost', 0); + + final handler = const Pipeline() + .addMiddleware(cors()) + .addHandler(createStaticHandler('./assets')); + io.serveRequests(server, handler); + + channel.sink.add(server.port); + await channel.stream + .listen(null) + .asFuture() + .then((_) => server.close()); +} diff --git a/packages/powersync/test/server/worker_server.dart b/packages/powersync/test/server/worker_server.dart new file mode 100644 index 00000000..c0b12572 --- /dev/null +++ b/packages/powersync/test/server/worker_server.dart @@ -0,0 +1,43 @@ +import 'dart:io'; + +import 'package:dcli/dcli.dart'; +import 'package:path/path.dart' as p; +import 'package:shelf/shelf.dart'; +import 'package:shelf/shelf_io.dart' as io; +import 'package:shelf_static/shelf_static.dart'; +import 'package:stream_channel/stream_channel.dart'; +import 'asset_server.dart'; + +Future hybridMain(StreamChannel channel) async { + final assetsDirectory = p.normalize( + p.join(DartScript.self.pathToScriptDirectory, '../../../../assets')); + + // Copy sqlite3.wasm file expected by the worker + final sqliteOutputPath = p.join(assetsDirectory, 'sqlite3.wasm'); + + if (!(await File(sqliteOutputPath).exists())) { + throw AssertionError( + 'sqlite3.wasm file should be present in the root assets folder'); + } + + final driftWorkerOutputPath = + p.join(assetsDirectory, 'powersync_db.worker.js'); + + if (!(await File(driftWorkerOutputPath).exists())) { + throw AssertionError( + 'powersync_db.worker.js file should be present in the ./assets folder'); + } + + final server = await HttpServer.bind('localhost', 0); + + final handler = const Pipeline() + .addMiddleware(cors()) + .addHandler(createStaticHandler(assetsDirectory)); + io.serveRequests(server, handler); + + channel.sink.add(server.port); + await channel.stream.listen(null).asFuture().then((_) async { + print('closing server'); + await server.close(); + }); +} diff --git a/packages/powersync/test/stream_test.dart b/packages/powersync/test/stream_test.dart index 050d199c..412711f4 100644 --- a/packages/powersync/test/stream_test.dart +++ b/packages/powersync/test/stream_test.dart @@ -1,3 +1,4 @@ +@TestOn('!browser') import 'dart:async'; import 'dart:convert'; import 'dart:io'; diff --git a/packages/powersync/test/streaming_sync_test.dart b/packages/powersync/test/streaming_sync_test.dart index aff9c71c..c80ff016 100644 --- a/packages/powersync/test/streaming_sync_test.dart +++ b/packages/powersync/test/streaming_sync_test.dart @@ -1,3 +1,5 @@ +@TestOn('!browser') +// TODO setup hybrid server import 'dart:async'; import 'dart:io'; import 'dart:math'; @@ -6,7 +8,9 @@ import 'package:powersync/powersync.dart'; import 'package:test/test.dart'; import 'test_server.dart'; -import 'util.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); class TestConnector extends PowerSyncBackendConnector { final Function _fetchCredentials; @@ -27,12 +31,12 @@ void main() { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); }); tearDown(() async { - await cleanDb(path: path); + await testUtils.cleanDb(path: path); }); test('full powersync reconnect', () async { @@ -52,7 +56,7 @@ void main() { expiresAt: DateTime.now()); } - final pdb = await setupPowerSync(path: path); + final pdb = await testUtils.setupPowerSync(path: path); pdb.retryDelay = Duration(milliseconds: 5000); var connector = TestConnector(credentialsCallback); pdb.connect(connector: connector); @@ -95,7 +99,7 @@ void main() { expiresAt: DateTime.now()); } - final pdb = await setupPowerSync(path: path); + final pdb = await testUtils.setupPowerSync(path: path); pdb.retryDelay = const Duration(milliseconds: 5); var connector = TestConnector(credentialsCallback); pdb.connect(connector: connector); diff --git a/packages/powersync/test/util.dart b/packages/powersync/test/util.dart deleted file mode 100644 index 2b96f563..00000000 --- a/packages/powersync/test/util.dart +++ /dev/null @@ -1,111 +0,0 @@ -import 'dart:async'; -import 'dart:ffi'; -import 'dart:io'; - -import 'package:logging/logging.dart'; -import 'package:powersync/powersync.dart'; -import 'package:powersync/sqlite3.dart' as sqlite; -import 'package:powersync/sqlite_async.dart'; -import 'package:sqlite3/open.dart' as sqlite_open; -import 'package:test_api/src/backend/invoker.dart'; - -const schema = Schema([ - Table('assets', [ - Column.text('created_at'), - Column.text('make'), - Column.text('model'), - Column.text('serial_number'), - Column.integer('quantity'), - Column.text('user_id'), - Column.text('customer_id'), - Column.text('description'), - ], indexes: [ - Index('makemodel', [IndexedColumn('make'), IndexedColumn('model')]) - ]), - Table('customers', [Column.text('name'), Column.text('email')]) -]); - -const defaultSchema = schema; - -class TestOpenFactory extends PowerSyncOpenFactory { - TestOpenFactory({required super.path}); - - @override - sqlite.Database open(SqliteOpenOptions options) { - sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { - return DynamicLibrary.open('libsqlite3.so.0'); - }); - return super.open(options); - } -} - -Future setupPowerSync( - {required String path, Schema? schema}) async { - final db = PowerSyncDatabase.withFactory(TestOpenFactory(path: path), - schema: schema ?? defaultSchema, logger: testLogger); - return db; -} - -Future setupSqlite( - {required PowerSyncDatabase powersync}) async { - await powersync.initialize(); - - final sqliteDb = await powersync.isolateConnectionFactory().openRawDatabase(); - - return sqliteDb; -} - -Future cleanDb({required String path}) async { - try { - await File(path).delete(); - } on PathNotFoundException { - // Not an issue - } - try { - await File("$path-shm").delete(); - } on PathNotFoundException { - // Not an issue - } - try { - await File("$path-wal").delete(); - } on PathNotFoundException { - // Not an issue - } -} - -String dbPath() { - final test = Invoker.current!.liveTest; - var testName = test.test.name; - var testShortName = testName.replaceAll(RegExp(r'\s'), '_').toLowerCase(); - var dbName = "test-db/$testShortName.db"; - Directory("test-db").createSync(recursive: false); - return dbName; -} - -final testLogger = _makeTestLogger(); - -Logger _makeTestLogger() { - final logger = Logger.detached('PowerSync Tests'); - logger.level = Level.ALL; - logger.onRecord.listen((record) { - print( - '[${record.loggerName}] ${record.level.name}: ${record.time}: ${record.message}'); - if (record.error != null) { - print(record.error); - } - if (record.stackTrace != null) { - print(record.stackTrace); - } - - if (record.error != null && record.level >= Level.SEVERE) { - // Hack to fail the test if a SEVERE error is logged. - // Not ideal, but works to catch "Sync Isolate error". - uncaughtError() async { - throw record.error!; - } - - uncaughtError(); - } - }); - return logger; -} diff --git a/packages/powersync/test/utils/abstract_test_utils.dart b/packages/powersync/test/utils/abstract_test_utils.dart new file mode 100644 index 00000000..59587573 --- /dev/null +++ b/packages/powersync/test/utils/abstract_test_utils.dart @@ -0,0 +1,92 @@ +import 'package:logging/logging.dart'; +import 'package:powersync/powersync.dart'; +import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:test_api/src/backend/invoker.dart'; + +const schema = Schema([ + Table('assets', [ + Column.text('created_at'), + Column.text('make'), + Column.text('model'), + Column.text('serial_number'), + Column.integer('quantity'), + Column.text('user_id'), + Column.text('customer_id'), + Column.text('description'), + ], indexes: [ + Index('makemodel', [IndexedColumn('make'), IndexedColumn('model')]) + ]), + Table('customers', [Column.text('name'), Column.text('email')]) +]); + +const defaultSchema = schema; + +final testLogger = _makeTestLogger(); + +Logger _makeTestLogger() { + final logger = Logger.detached('PowerSync Tests'); + logger.level = Level.ALL; + logger.onRecord.listen((record) { + print( + '[${record.loggerName}] ${record.level.name}: ${record.time}: ${record.message}'); + if (record.error != null) { + print(record.error); + } + if (record.stackTrace != null) { + print(record.stackTrace); + } + + if (record.error != null && record.level >= Level.SEVERE) { + // Hack to fail the test if a SEVERE error is logged. + // Not ideal, but works to catch "Sync Isolate error". + uncaughtError() async { + throw record.error!; + } + + uncaughtError(); + } + }); + return logger; +} + +abstract class AbstractTestUtils { + String dbPath() { + final test = Invoker.current!.liveTest; + var testName = test.test.name; + var testShortName = + testName.replaceAll(RegExp(r'[\s\./]'), '_').toLowerCase(); + var dbName = "test-db/$testShortName.db"; + return dbName; + } + + /// Generates a test open factory + Future testFactory( + {String? path, + String sqlitePath = '', + SqliteOptions options = const SqliteOptions.defaults()}) async { + return PowerSyncOpenFactory(path: path ?? dbPath(), sqliteOptions: options); + } + + /// Creates a SqliteDatabaseConnection + Future setupPowerSync( + {String? path, Schema? schema}) async { + final db = PowerSyncDatabase.withFactory(await testFactory(path: path), + schema: schema ?? defaultSchema, logger: testLogger); + await db.initialize(); + return db; + } + + Future setupSqlite( + {required PowerSyncDatabase powersync}) async { + await powersync.initialize(); + + final sqliteDb = + await powersync.isolateConnectionFactory().openRawDatabase(); + + return sqliteDb; + } + + /// Deletes any DB data + Future cleanDb({required String path}); +} diff --git a/packages/powersync/test/utils/native_test_utils.dart b/packages/powersync/test/utils/native_test_utils.dart new file mode 100644 index 00000000..c1dcaa73 --- /dev/null +++ b/packages/powersync/test/utils/native_test_utils.dart @@ -0,0 +1,58 @@ +import 'dart:async'; +import 'dart:ffi'; +import 'dart:io'; +import 'package:powersync/powersync.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite3/open.dart' as sqlite_open; + +import 'abstract_test_utils.dart'; + +const defaultSqlitePath = 'libsqlite3.so.0'; + +class TestOpenFactory extends PowerSyncOpenFactory { + TestOpenFactory({required super.path}); + + @override + FutureOr open(SqliteOpenOptions options) { + sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { + return DynamicLibrary.open('libsqlite3.so.0'); + }); + return super.open(options); + } +} + +class TestUtils extends AbstractTestUtils { + @override + String dbPath() { + Directory("test-db").createSync(recursive: false); + return super.dbPath(); + } + + @override + Future cleanDb({required String path}) async { + try { + await File(path).delete(); + } on PathNotFoundException { + // Not an issue + } + try { + await File("$path-shm").delete(); + } on PathNotFoundException { + // Not an issue + } + try { + await File("$path-wal").delete(); + } on PathNotFoundException { + // Not an issue + } + } + + @override + Future testFactory( + {String? path, + String sqlitePath = defaultSqlitePath, + SqliteOptions options = const SqliteOptions.defaults()}) async { + return TestOpenFactory(path: path ?? dbPath()); + } +} diff --git a/packages/powersync/test/utils/stub_test_utils.dart b/packages/powersync/test/utils/stub_test_utils.dart new file mode 100644 index 00000000..3f86512c --- /dev/null +++ b/packages/powersync/test/utils/stub_test_utils.dart @@ -0,0 +1,8 @@ +import 'abstract_test_utils.dart'; + +class TestUtils extends AbstractTestUtils { + @override + Future cleanDb({required String path}) { + throw UnimplementedError(); + } +} diff --git a/packages/powersync/test/utils/test_utils_impl.dart b/packages/powersync/test/utils/test_utils_impl.dart new file mode 100644 index 00000000..99a34d39 --- /dev/null +++ b/packages/powersync/test/utils/test_utils_impl.dart @@ -0,0 +1,5 @@ +export 'stub_test_utils.dart' + // ignore: uri_does_not_exist + if (dart.library.io) 'native_test_utils.dart' + // ignore: uri_does_not_exist + if (dart.library.html) 'web_test_utils.dart'; diff --git a/packages/powersync/test/utils/web_test_utils.dart b/packages/powersync/test/utils/web_test_utils.dart new file mode 100644 index 00000000..2bb28c1c --- /dev/null +++ b/packages/powersync/test/utils/web_test_utils.dart @@ -0,0 +1,65 @@ +import 'dart:async'; +import 'dart:html'; + +import 'package:js/js.dart'; +import 'package:powersync/powersync.dart'; +import 'package:sqlite3/src/database.dart'; +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:test/test.dart'; +import 'abstract_test_utils.dart'; + +@JS('URL.createObjectURL') +external String _createObjectURL(Blob blob); + +class TestUtils extends AbstractTestUtils { + late Future _isInitialized; + late final String sqlite3WASMUri; + late final String driftUri; + + TestUtils() { + _isInitialized = _init(); + } + + Future _init() async { + final channel = + spawnHybridUri('/test/server/worker_server.dart', stayAlive: true); + final port = await channel.stream.first as int; + sqlite3WASMUri = 'http://localhost:$port/sqlite3.wasm'; + // Cross origin workers are not supported, but we can supply a Blob + final driftUriSource = 'http://localhost:$port/powersync_db.worker.js'; + + final blob = Blob(['importScripts("$driftUriSource");'], + 'application/javascript'); + driftUri = _createObjectURL(blob); + } + + @override + Future cleanDb({required String path}) async {} + + @override + Future testFactory( + {String? path, + String? sqlitePath, + SqliteOptions options = const SqliteOptions.defaults()}) async { + await _isInitialized; + + final webOptions = SqliteOptions( + webSqliteOptions: + WebSqliteOptions(wasmUri: sqlite3WASMUri, workerUri: driftUri)); + return super.testFactory(path: path, options: webOptions); + } + + @override + Future setupPowerSync( + {String? path, Schema? schema}) async { + await _isInitialized; + return super.setupPowerSync(path: path, schema: schema); + } + + @override + Future setupSqlite( + {required PowerSyncDatabase powersync}) async { + await _isInitialized; + return super.setupSqlite(powersync: powersync); + } +} diff --git a/packages/powersync/test/watch_test.dart b/packages/powersync/test/watch_test.dart index 7ccf53da..e10ef5f4 100644 --- a/packages/powersync/test/watch_test.dart +++ b/packages/powersync/test/watch_test.dart @@ -1,3 +1,9 @@ +@TestOn('!browser') + +/// TODO, this requires a custom Drift worker script on web. +/// Verified manually for now +library; + import 'dart:async'; import 'dart:math'; @@ -5,7 +11,9 @@ import 'package:powersync/powersync.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; -import 'util.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); const testSchema = Schema([ Table('assets', [ @@ -29,12 +37,13 @@ void main() { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); }); test('watch', () async { - final powersync = await setupPowerSync(path: path, schema: testSchema); + final powersync = + await testUtils.setupPowerSync(path: path, schema: testSchema); const baseTime = 20; @@ -97,7 +106,8 @@ void main() { }); test('onChange', () async { - final powersync = await setupPowerSync(path: path, schema: testSchema); + final powersync = + await testUtils.setupPowerSync(path: path, schema: testSchema); const baseTime = 20; diff --git a/packages/powersync_attachments_helper/CHANGELOG.md b/packages/powersync_attachments_helper/CHANGELOG.md index c59f0d09..95c3ac31 100644 --- a/packages/powersync_attachments_helper/CHANGELOG.md +++ b/packages/powersync_attachments_helper/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.3.0-alpha.1 + +- Added initial support for Web platform. + ## 0.2.0 - Potentially BREAKING CHANGE for users who rely on multiple attachment queues. diff --git a/packages/powersync_attachments_helper/lib/src/attachments_queue_table.dart b/packages/powersync_attachments_helper/lib/src/attachments_queue_table.dart index f1bf5a0d..966d115a 100644 --- a/packages/powersync_attachments_helper/lib/src/attachments_queue_table.dart +++ b/packages/powersync_attachments_helper/lib/src/attachments_queue_table.dart @@ -1,5 +1,5 @@ import 'package:powersync/powersync.dart'; -import 'package:powersync/sqlite3.dart' as sqlite; +import 'package:powersync/sqlite3_common.dart' as sqlite; const defaultAttachmentsQueueTableName = 'attachments_queue'; diff --git a/packages/powersync_attachments_helper/lib/src/attachments_service.dart b/packages/powersync_attachments_helper/lib/src/attachments_service.dart index 3c9a3e34..da67244a 100644 --- a/packages/powersync_attachments_helper/lib/src/attachments_service.dart +++ b/packages/powersync_attachments_helper/lib/src/attachments_service.dart @@ -2,7 +2,7 @@ import './attachments_queue.dart'; import './attachments_queue_table.dart'; import './local_storage_adapter.dart'; import 'package:powersync/powersync.dart'; -import 'package:sqlite_async/sqlite3.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; /// Service for interacting with the attachment queue. class AttachmentsService { diff --git a/packages/powersync_attachments_helper/pubspec.yaml b/packages/powersync_attachments_helper/pubspec.yaml index 9656994d..27ec8d8d 100644 --- a/packages/powersync_attachments_helper/pubspec.yaml +++ b/packages/powersync_attachments_helper/pubspec.yaml @@ -1,18 +1,19 @@ name: powersync_attachments_helper description: A helper library for handling attachments when using PowerSync. -version: 0.2.0 +version: 0.3.0-alpha.1 repository: https://github.com/powersync-ja/powersync.dart homepage: https://www.powersync.com/ environment: - sdk: ^3.2.0 + sdk: ^3.2.3 dependencies: flutter: sdk: flutter - powersync: ^1.1.0 + powersync: ^1.3.0-alpha.1 logging: ^1.2.0 - sqlite_async: ^0.6.0 + sqlite3: ^2.3.0 + sqlite_async: ^0.7.0-alpha.1 path_provider: ^2.1.1 dev_dependencies: diff --git a/packages/powersync_web_worker/.gitignore b/packages/powersync_web_worker/.gitignore new file mode 100644 index 00000000..920d19bd --- /dev/null +++ b/packages/powersync_web_worker/.gitignore @@ -0,0 +1,15 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +**/doc/api/ +**/ios/Flutter/.last_build_id +.dart_tool/ +.flutter-plugins +.flutter-plugins-dependencies +.packages +.pub-cache/ +.pub/ +/build/ + +# Avoid committing pubspec.lock for library packages; see +# https://dart.dev/guides/libraries/private-files#pubspeclock. +pubspec.lock diff --git a/packages/powersync_web_worker/README.md b/packages/powersync_web_worker/README.md new file mode 100644 index 00000000..7deab737 --- /dev/null +++ b/packages/powersync_web_worker/README.md @@ -0,0 +1,3 @@ +# PowerSync Dart Web Worker Builder + +Temporary package which builds the PowerSync Javascript worker code. \ No newline at end of file diff --git a/packages/powersync_web_worker/bin/compile_webworker.dart b/packages/powersync_web_worker/bin/compile_webworker.dart new file mode 100644 index 00000000..217b1898 --- /dev/null +++ b/packages/powersync_web_worker/bin/compile_webworker.dart @@ -0,0 +1,47 @@ +import 'dart:io'; + +import 'package:path/path.dart' as path; + +Future main() async { + // This should be the package root + final cwd = Directory.current.absolute.path; + final repoRoot = path.normalize(path.join(cwd, '../../')); + + /// The monorepo root assets directory + final workerFilename = 'powersync_db.worker.js'; + final outputPath = path.join(repoRoot, 'assets/$workerFilename'); + + final workerSourcePath = './lib/src/powersync_db.worker.dart'; + + // And compile worker code + final process = await Process.run( + Platform.executable, + [ + 'compile', + 'js', + '-o', + outputPath, + '-O0', + workerSourcePath, + ], + workingDirectory: cwd); + + if (process.exitCode != 0) { + throw Exception('Could not compile worker: ${process.stdout.toString()}'); + } + + // Copy this to all demo apps web folders + final demosRoot = path.join(repoRoot, 'demos'); + final demoDirectories = + Directory(demosRoot).listSync().whereType().toList(); + + for (final demoDir in demoDirectories) { + // only if the demo is web enabled + final demoWebDir = path.join(demoDir.absolute.path, 'web'); + if (!Directory(demoWebDir).existsSync()) { + continue; + } + final demoOutputPath = path.join(demoWebDir, workerFilename); + File(outputPath).copySync(demoOutputPath); + } +} diff --git a/packages/powersync_web_worker/lib/src/powersync_db.worker.dart b/packages/powersync_web_worker/lib/src/powersync_db.worker.dart new file mode 100644 index 00000000..2c4020b5 --- /dev/null +++ b/packages/powersync_web_worker/lib/src/powersync_db.worker.dart @@ -0,0 +1,120 @@ +library; + +/// This file needs to be compiled to JavaScript with the command +/// dart compile js -O4 packages/powersync_web_worker/lib/src/powersync_db.worker.dart -o assets/powersync_db.worker.js +/// The output should then be included in each project's `web` directory +/// +/// NOTE: This package contains some code duplicated from [sqlite_async.dart] +/// and [powersync.dart]. +/// This is only necessary while we are using a +/// [forked](https://github.com/powersync-ja/drift/tree/test) version of Drift +/// which is not published as a package, but imported from a Git Repository. +/// +/// [sqlite_async.dart] is a published package which cannot depend on Git +/// Repository dependencies, it instead depends on `drift: 2.15.0`. +/// This is possible since the forked changes are only on the compiled +/// worker side. SQLite Async can use the standard Drift client. +/// +/// [powersync.dart] depends on [sqlite_async.dart], but it needs to use +/// the forked [drift.dart] library in order to compile its web worker. +/// Since both packages are published, they cannot depend on the forked +/// Drift Git repository. +/// +/// This intermediate package exists only to compile the Javascript +/// web worker for [powersync.dart]. It cannot depend on [powersync.dart] +/// or [sqlite_async.dart] since those require the hosted `drift: 2.15.0` +/// dependency. Dart's package manager cannot resolve using both. +/// +/// Code duplication is required since this package cannot depend on the +/// other libraries. This will be solved once a published Drift package +/// is available. + +import 'dart:convert'; + +import 'package:drift/wasm.dart'; +import 'package:sqlite3/common.dart'; +import 'package:uuid/uuid.dart'; +import 'package:uuid/data.dart'; +import 'package:uuid/rng.dart'; + +/// Custom function which exposes CommonDatabase.autocommit +const sqliteAsyncAutoCommitCommand = 'sqlite_async_autocommit'; + +void setupDB(CommonDatabase database) { + /// Duplicate from [sqlite_async.dart] + database.createFunction( + functionName: sqliteAsyncAutoCommitCommand, + argumentCount: const AllowedArgumentCount(0), + function: (args) { + return database.autocommit; + }); + + /// Functions below are duplicates from [powersync.dart] + database.createFunction( + functionName: 'powersync_diff', + argumentCount: const AllowedArgumentCount(2), + deterministic: true, + directOnly: false, + function: (args) { + final oldData = jsonDecode(args[0] as String) as Map; + final newData = jsonDecode(args[1] as String) as Map; + + Map result = {}; + + for (final newEntry in newData.entries) { + final oldValue = oldData[newEntry.key]; + final newValue = newEntry.value; + + if (newValue != oldValue) { + result[newEntry.key] = newValue; + } + } + + for (final key in oldData.keys) { + if (!newData.containsKey(key)) { + result[key] = null; + } + } + + return jsonEncode(result); + }); + + final uuid = Uuid(goptions: GlobalOptions(CryptoRNG())); + + database.createFunction( + functionName: 'uuid', + argumentCount: const AllowedArgumentCount(0), + function: (args) { + return uuid.v4(); + }, + ); + database.createFunction( + // Postgres compatibility + functionName: 'gen_random_uuid', + argumentCount: const AllowedArgumentCount(0), + function: (args) => uuid.v4(), + ); + database.createFunction( + functionName: 'powersync_sleep', + argumentCount: const AllowedArgumentCount(1), + function: (args) { + // Can't perform synchronous sleep on web + final millis = args[0] as int; + return millis; + }, + ); + + database.createFunction( + functionName: 'powersync_connection_name', + argumentCount: const AllowedArgumentCount(0), + function: (args) { + return 'N/A'; + }, + ); +} + +void main() { + WasmDatabase.workerMainForOpen( + setupAllDatabases: setupDB, + ); +} diff --git a/packages/powersync_web_worker/lib/worker.dart b/packages/powersync_web_worker/lib/worker.dart new file mode 100644 index 00000000..e495f5b8 --- /dev/null +++ b/packages/powersync_web_worker/lib/worker.dart @@ -0,0 +1 @@ +export 'src/powersync_db.worker.dart'; diff --git a/packages/powersync_web_worker/pubspec.yaml b/packages/powersync_web_worker/pubspec.yaml new file mode 100644 index 00000000..977c489e --- /dev/null +++ b/packages/powersync_web_worker/pubspec.yaml @@ -0,0 +1,27 @@ +name: powersync_web_worker +version: 0.0.1 +homepage: https://powersync.com +repository: https://github.com/powersync-ja/powersync.dart +description: PowerSync Dart Web Worker Builder +environment: + sdk: ^3.2.3 + +publish_to: none + +executables: + compile_worker: bin/compile_worker.dart + +dev_dependencies: + dcli: ^3.3.5 + uuid: ^4.2.0 + async: ^2.10.0 + path: ^1.8.3 + sqlite3: ^2.3.0 + drift: + git: + url: https://github.com/powersync-ja/drift.git + ref: test # branch name + path: drift + +platforms: + web: diff --git a/pubspec.lock b/pubspec.lock index 609e995e..b7baa1d7 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -5,18 +5,18 @@ packages: dependency: transitive description: name: _fe_analyzer_shared - sha256: "36a321c3d2cbe01cbcb3540a87b8843846e0206df3e691fa7b23e19e78de6d49" + sha256: "0b2f2bd91ba804e53a61d757b986f89f1f9eaed5b11e4b2f5a2468d86d6c9fc7" url: "https://pub.dev" source: hosted - version: "65.0.0" + version: "67.0.0" analyzer: dependency: transitive description: name: analyzer - sha256: dfe03b90ec022450e22513b5e5ca1f01c0c01de9c3fba2f7fd233cb57a6b9a07 + sha256: "37577842a27e4338429a1cbc32679d508836510b056f1eedf0c8d20e39c1383d" url: "https://pub.dev" source: hosted - version: "6.3.0" + version: "6.4.1" ansi_styles: dependency: transitive description: @@ -113,6 +113,22 @@ packages: url: "https://pub.dev" source: hosted version: "3.0.3" + drift: + dependency: transitive + description: + name: drift + sha256: b50a8342c6ddf05be53bda1d246404cbad101b64dc73e8d6d1ac1090d119b4e2 + url: "https://pub.dev" + source: hosted + version: "2.15.0" + ffi: + dependency: transitive + description: + name: ffi + sha256: "7bf0adc28a23d395f19f3f1eb21dd7cfd1dd9f8e1c50051c069122e6853bc878" + url: "https://pub.dev" + source: hosted + version: "2.1.0" file: dependency: transitive description: @@ -149,10 +165,10 @@ packages: dependency: transitive description: name: http - sha256: d4872660c46d929f6b8a9ef4e7a7eff7e49bbf0c4ec3f385ee32df5119175139 + sha256: a2bbf9d017fcced29139daa8ed2bba4ece450ab222871df93ca9eec6f80c34ba url: "https://pub.dev" source: hosted - version: "1.1.2" + version: "1.2.0" http_multi_server: dependency: transitive description: @@ -237,10 +253,10 @@ packages: dependency: transitive description: name: mime - sha256: e4ff8e8564c03f255408decd16e7899da1733852a9110a58fe6d1b817684a63e + sha256: "2e123074287cc9fd6c09de8336dae606d1ddb88d9ac47358826db698c176a1f2" url: "https://pub.dev" source: hosted - version: "1.0.4" + version: "1.0.5" mustache_template: dependency: transitive description: @@ -249,6 +265,14 @@ packages: url: "https://pub.dev" source: hosted version: "2.0.0" + mutex: + dependency: transitive + description: + name: mutex + sha256: "8827da25de792088eb33e572115a5eb0d61d61a3c01acbc8bcbe76ed78f1a1f2" + url: "https://pub.dev" + source: hosted + version: "3.1.0" node_preamble: dependency: transitive description: @@ -277,10 +301,10 @@ packages: dependency: transitive description: name: platform - sha256: "0a279f0707af40c890e80b1e9df8bb761694c074ba7e1d4ab1bc4b728e200b59" + sha256: "12220bb4b65720483f8fa9450b4332347737cf8213dd2840d8b2c823e47243ec" url: "https://pub.dev" source: hosted - version: "3.1.3" + version: "3.1.4" pool: dependency: transitive description: @@ -393,6 +417,23 @@ packages: url: "https://pub.dev" source: hosted version: "1.10.0" + sqlite3: + dependency: transitive + description: + name: sqlite3 + sha256: c4a4c5a4b2a32e2d0f6837b33d7c91a67903891a5b7dbe706cf4b1f6b0c798c5 + url: "https://pub.dev" + source: hosted + version: "2.3.0" + sqlite_async: + dependency: "direct overridden" + description: + path: "." + ref: web + resolved-ref: f946e1ad70d61ae0eee930c59e567e8a01f1ca77 + url: "https://github.com/powersync-ja/sqlite_async.dart.git" + source: git + version: "0.6.0" stack_trace: dependency: transitive description: @@ -429,10 +470,10 @@ packages: dependency: "direct dev" description: name: test - sha256: "3d028996109ad5c253674c7f347822fb994a087614d6f353e6039704b4661ff2" + sha256: "7ee446762c2c50b3bd4ea96fe13ffac69919352bd3b4b17bac3f3465edc58073" url: "https://pub.dev" source: hosted - version: "1.25.0" + version: "1.25.2" test_api: dependency: transitive description: @@ -485,18 +526,18 @@ packages: dependency: transitive description: name: web - sha256: edc8a9573dd8c5a83a183dae1af2b6fd4131377404706ca4e5420474784906fa + sha256: "4188706108906f002b3a293509234588823c8c979dc83304e229ff400c996b05" url: "https://pub.dev" source: hosted - version: "0.4.0" + version: "0.4.2" web_socket_channel: dependency: transitive description: name: web_socket_channel - sha256: d88238e5eac9a42bb43ca4e721edba3c08c6354d4a53063afaa568516217621b + sha256: "939ab60734a4f8fa95feacb55804fa278de28bdeef38e616dc08e44a84adea23" url: "https://pub.dev" source: hosted - version: "2.4.0" + version: "2.4.3" webkit_inspection_protocol: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 8d3b181f..20fa5f20 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -7,8 +7,12 @@ environment: sdk: ^3.2.3 # Add regular dependencies here. -dependencies: - # path: ^1.8.0 + +dependency_overrides: + sqlite_async: + git: + url: https://github.com/powersync-ja/sqlite_async.dart.git + ref: web # branch name dev_dependencies: lints: ^2.1.1 diff --git a/scripts/init_sqlite_wasm.sh b/scripts/init_sqlite_wasm.sh new file mode 100644 index 00000000..e1c972bd --- /dev/null +++ b/scripts/init_sqlite_wasm.sh @@ -0,0 +1,24 @@ +# sqlite3.wasm needs to be in the root assets folder +# and inside each Flutter app's web folder + +mkdir -p assets + +sqlite_filename="sqlite3.wasm" +sqlite_path="assets/$sqlite_filename" + +curl -LJ https://github.com/simolus3/sqlite3.dart/releases/download/sqlite3-2.3.0/sqlite3.wasm \ +-o $sqlite_path + +# Copy to each demo's web dir + +# Destination directory pattern +destination_pattern="demos/*/web/" + +# Iterate over directories matching the pattern +for dir in $destination_pattern; do + if [ -d "$dir" ]; then + # If the directory exists, copy the file + cp "$sqlite_path" "$dir/$sqlite_filename" + echo "Copied $sqlite_path to $dir" + fi +done \ No newline at end of file