From 84b9708db214f1f1451864c3fbd0f8b9499beadf Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 5 Nov 2024 14:43:59 +0200 Subject: [PATCH 1/5] Properly log output if sync worker compilation fails. --- scripts/compile_webworker.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/compile_webworker.dart b/scripts/compile_webworker.dart index 7520fe2b..3e85f8a9 100644 --- a/scripts/compile_webworker.dart +++ b/scripts/compile_webworker.dart @@ -30,7 +30,7 @@ Future main() async { if (dbWorkerProcess.exitCode != 0) { throw Exception( - 'Could not compile db worker: ${dbWorkerProcess.stdout.toString()}'); + 'Could not compile db worker.\nstdout: ${dbWorkerProcess.stdout.toString()}\nstderr: ${dbWorkerProcess.stderr.toString()}'); } final syncWorkerFilename = 'powersync_sync.worker.js'; @@ -54,7 +54,7 @@ Future main() async { if (syncWorkerProcess.exitCode != 0) { throw Exception( - 'Could not compile sync worker: ${dbWorkerProcess.stdout.toString()}'); + 'Could not compile sync worker:\nstdout: ${syncWorkerProcess.stdout.toString()}\nstderr: ${syncWorkerProcess.stderr.toString()}'); } // Copy this to all demo apps web folders From d5b48d21ffa9086cedff0c3ff7f3ab1bb9925e14 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 5 Nov 2024 14:44:29 +0200 Subject: [PATCH 2/5] Selectively configure flushing in initial sync. --- .../powersync/lib/src/bucket_storage.dart | 61 ++++++++----------- .../powersync/lib/src/web/sync_worker.dart | 4 +- .../lib/src/web/web_bucket_storage.dart | 20 ++++++ 3 files changed, 48 insertions(+), 37 deletions(-) create mode 100644 packages/powersync/lib/src/web/web_bucket_storage.dart diff --git a/packages/powersync/lib/src/bucket_storage.dart b/packages/powersync/lib/src/bucket_storage.dart index e21ee5db..ab26f654 100644 --- a/packages/powersync/lib/src/bucket_storage.dart +++ b/packages/powersync/lib/src/bucket_storage.dart @@ -45,14 +45,6 @@ class BucketStorage { return rows.first['client_id'] as String; } - Future streamOp(String op) async { - await writeTransaction((tx) async { - await tx.execute( - 'INSERT INTO powersync_operations(op, data) VALUES(?, ?)', - ['stream', op]); - }); - } - Future saveSyncData(SyncDataBatch batch) async { var count = 0; @@ -65,7 +57,10 @@ class BucketStorage { 'buckets': [b] })); } - }); + // No need to flush - the data is not directly visible to the user either way. + // We get major initial sync performance improvements with IndexedDB by + // not flushing here. + }, flush: false); _compactCounter += count; } @@ -85,7 +80,8 @@ class BucketStorage { await tx.execute( 'INSERT INTO powersync_operations(op, data) VALUES(?, ?)', ['delete_bucket', bucket]); - }); + // No need to flush - not directly visible to the user + }, flush: false); _pendingBucketDeletes = true; } @@ -125,7 +121,8 @@ class BucketStorage { "UPDATE ps_buckets SET last_op = ? WHERE name = '\$local'", [checkpoint.writeCheckpoint]); } - }); + // Not flushing here - the flush will happen in the next step + }, flush: false); final valid = await updateObjectsFromBuckets(checkpoint); if (!valid) { @@ -150,7 +147,10 @@ class BucketStorage { // can_update_local(db) == false return false; } - }); + // Important to flush here. + // After this step, the synced data will be visible to the user, + // and we don't want that to be reverted. + }, flush: true); } Future validateChecksums( @@ -176,39 +176,25 @@ class BucketStorage { } Future autoCompact() async { + // This is a no-op since powersync-sqlite-core v0.3.0 + // 1. Delete buckets await _deletePendingBuckets(); // 2. Clear REMOVE operations, only keeping PUT ones await _clearRemoveOps(); - - // await _compactWal(); - } - - // ignore: unused_element - Future _compactWal() async { - try { - await writeTransaction((tx) async { - await tx.execute('PRAGMA wal_checkpoint(TRUNCATE)'); - }); - } on SqliteException catch (e) { - // Ignore SQLITE_BUSY - if (e.resultCode == 5) { - // Ignore - } else if (e.resultCode == 6) { - // Ignore - } - } } Future _deletePendingBuckets() async { + // This is a no-op since powersync-sqlite-core v0.3.0 if (_pendingBucketDeletes) { // Executed once after start-up, and again when there are pending deletes. await writeTransaction((tx) async { await tx.execute( 'INSERT INTO powersync_operations(op, data) VALUES (?, ?)', ['delete_pending_buckets', '']); - }); + // No need to flush - not directly visible to the user + }, flush: false); _pendingBucketDeletes = false; } } @@ -218,11 +204,13 @@ class BucketStorage { return; } + // This is a no-op since powersync-sqlite-core v0.3.0 await writeTransaction((tx) async { await tx.execute( 'INSERT INTO powersync_operations(op, data) VALUES (?, ?)', ['clear_remove_ops', '']); - }); + // No need to flush - not directly visible to the user + }, flush: false); _compactCounter = 0; } @@ -267,7 +255,8 @@ class BucketStorage { [opId]); return true; - }); + // Flush here - don't want to lose the write checkpoint updates. + }, flush: true); } Future nextCrudItem() async { @@ -313,7 +302,8 @@ class BucketStorage { await tx.execute( 'UPDATE ps_buckets SET target_op = $maxOpId WHERE name=\'\$local\''); } - }); + // Flush here - don't want to lose the write checkpoint updates. + }, flush: true); }); } @@ -323,7 +313,8 @@ class BucketStorage { /// concurrently. Future writeTransaction( Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout}) async { + {Duration? lockTimeout, + required bool flush}) async { return _internalDb.writeTransaction(callback, lockTimeout: lockTimeout); } } diff --git a/packages/powersync/lib/src/web/sync_worker.dart b/packages/powersync/lib/src/web/sync_worker.dart index a5013e8a..0b405bb1 100644 --- a/packages/powersync/lib/src/web/sync_worker.dart +++ b/packages/powersync/lib/src/web/sync_worker.dart @@ -16,8 +16,8 @@ import 'package:powersync/src/streaming_sync.dart'; import 'package:sqlite_async/web.dart'; import 'package:web/web.dart' hide RequestMode; -import '../bucket_storage.dart'; import 'sync_worker_protocol.dart'; +import 'web_bucket_storage.dart'; final _logger = autoLogger; @@ -258,7 +258,7 @@ class _SyncRunner { : jsonDecode(syncParamsEncoded!) as Map; sync = StreamingSyncImplementation( - adapter: BucketStorage(database), + adapter: WebBucketStorage(database), credentialsCallback: client.channel.credentialsCallback, invalidCredentialsCallback: client.channel.invalidCredentialsCallback, uploadCrud: client.channel.uploadCrud, diff --git a/packages/powersync/lib/src/web/web_bucket_storage.dart b/packages/powersync/lib/src/web/web_bucket_storage.dart new file mode 100644 index 00000000..ba106b26 --- /dev/null +++ b/packages/powersync/lib/src/web/web_bucket_storage.dart @@ -0,0 +1,20 @@ +import 'package:powersync/sqlite_async.dart'; +import 'package:powersync/src/bucket_storage.dart'; +import 'package:sqlite_async/web.dart'; + +class WebBucketStorage extends BucketStorage { + final WebSqliteConnection _webDb; + + WebBucketStorage(this._webDb) : super(_webDb); + + @override + + /// Override to implement the flush parameter for web. + Future writeTransaction( + Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, + required bool flush}) async { + return _webDb.writeTransaction(callback, + lockTimeout: lockTimeout, flush: flush); + } +} From e65328a9b6e95714f39ead3f40df0d6330bae50d Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 6 Nov 2024 09:42:44 +0200 Subject: [PATCH 3/5] Bump sqlite_async and drift_sqlite_async versions. --- demos/django-todolist/pubspec.yaml | 2 +- demos/supabase-anonymous-auth/pubspec.yaml | 2 +- demos/supabase-edge-function-auth/pubspec.yaml | 2 +- demos/supabase-todolist-drift/pubspec.yaml | 4 ++-- demos/supabase-todolist-optional-sync/pubspec.yaml | 2 +- demos/supabase-todolist/pubspec.yaml | 2 +- packages/powersync/pubspec.yaml | 2 +- packages/powersync_attachments_helper/pubspec.yaml | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/demos/django-todolist/pubspec.yaml b/demos/django-todolist/pubspec.yaml index 165abc38..31dc731b 100644 --- a/demos/django-todolist/pubspec.yaml +++ b/demos/django-todolist/pubspec.yaml @@ -14,7 +14,7 @@ dependencies: path_provider: ^2.1.1 path: ^1.8.3 logging: ^1.2.0 - sqlite_async: ^0.10.1 + sqlite_async: ^0.11.0 http: ^1.2.1 shared_preferences: ^2.2.3 diff --git a/demos/supabase-anonymous-auth/pubspec.yaml b/demos/supabase-anonymous-auth/pubspec.yaml index 4d9e0324..862c8b96 100644 --- a/demos/supabase-anonymous-auth/pubspec.yaml +++ b/demos/supabase-anonymous-auth/pubspec.yaml @@ -16,7 +16,7 @@ dependencies: supabase_flutter: ^2.0.2 path: ^1.8.3 logging: ^1.2.0 - sqlite_async: ^0.10.1 + sqlite_async: ^0.11.0 universal_io: ^2.2.2 dev_dependencies: diff --git a/demos/supabase-edge-function-auth/pubspec.yaml b/demos/supabase-edge-function-auth/pubspec.yaml index 2ddb588a..af7161ee 100644 --- a/demos/supabase-edge-function-auth/pubspec.yaml +++ b/demos/supabase-edge-function-auth/pubspec.yaml @@ -16,7 +16,7 @@ dependencies: supabase_flutter: ^2.0.2 path: ^1.8.3 logging: ^1.2.0 - sqlite_async: ^0.10.1 + sqlite_async: ^0.11.0 universal_io: ^2.2.2 dev_dependencies: diff --git a/demos/supabase-todolist-drift/pubspec.yaml b/demos/supabase-todolist-drift/pubspec.yaml index 371af697..6813e8bd 100644 --- a/demos/supabase-todolist-drift/pubspec.yaml +++ b/demos/supabase-todolist-drift/pubspec.yaml @@ -18,9 +18,9 @@ dependencies: camera: ^0.10.5+7 image: ^4.1.3 universal_io: ^2.2.2 - sqlite_async: ^0.10.1 + sqlite_async: ^0.11.0 drift: ^2.20.2 - drift_sqlite_async: ^0.2.0-alpha.3 + drift_sqlite_async: ^0.2.0 dev_dependencies: flutter_test: diff --git a/demos/supabase-todolist-optional-sync/pubspec.yaml b/demos/supabase-todolist-optional-sync/pubspec.yaml index 41de5db8..4595be39 100644 --- a/demos/supabase-todolist-optional-sync/pubspec.yaml +++ b/demos/supabase-todolist-optional-sync/pubspec.yaml @@ -18,7 +18,7 @@ dependencies: camera: ^0.10.5+7 image: ^4.1.3 universal_io: ^2.2.2 - sqlite_async: ^0.10.1 + sqlite_async: ^0.11.0 dev_dependencies: flutter_test: diff --git a/demos/supabase-todolist/pubspec.yaml b/demos/supabase-todolist/pubspec.yaml index fc121306..13ba7e71 100644 --- a/demos/supabase-todolist/pubspec.yaml +++ b/demos/supabase-todolist/pubspec.yaml @@ -19,7 +19,7 @@ dependencies: camera: ^0.10.5+7 image: ^4.1.3 universal_io: ^2.2.2 - sqlite_async: ^0.10.1 + sqlite_async: ^0.11.0 dev_dependencies: flutter_test: diff --git a/packages/powersync/pubspec.yaml b/packages/powersync/pubspec.yaml index 4f75e2a6..d7eddf3f 100644 --- a/packages/powersync/pubspec.yaml +++ b/packages/powersync/pubspec.yaml @@ -10,7 +10,7 @@ dependencies: flutter: sdk: flutter - sqlite_async: ^0.10.1 + sqlite_async: ^0.11.0 # We only use sqlite3 as a transitive dependency, # but right now we need a minimum of v2.4.6. sqlite3: ^2.4.6 diff --git a/packages/powersync_attachments_helper/pubspec.yaml b/packages/powersync_attachments_helper/pubspec.yaml index 3216b96f..d1206609 100644 --- a/packages/powersync_attachments_helper/pubspec.yaml +++ b/packages/powersync_attachments_helper/pubspec.yaml @@ -12,7 +12,7 @@ dependencies: powersync: ^1.9.1 logging: ^1.2.0 - sqlite_async: ^0.10.1 + sqlite_async: ^0.11.0 path_provider: ^2.0.13 dev_dependencies: From 21f9d00e30e40c1efce91ffb84c32c0e9bf91aad Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 6 Nov 2024 09:54:47 +0200 Subject: [PATCH 4/5] Split out pana checks; remove "demo tests". --- .github/workflows/demos.yml | 18 ------------------ .github/workflows/packages.yml | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/.github/workflows/demos.yml b/.github/workflows/demos.yml index 55f6f9ce..1a83c277 100644 --- a/.github/workflows/demos.yml +++ b/.github/workflows/demos.yml @@ -29,21 +29,3 @@ jobs: run: | ./.github/workflows/scripts/copy-config.sh melos analyze:demos - - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Install Flutter - uses: subosito/flutter-action@v2 - with: - flutter-version: "3.x" - channel: "stable" - - name: Install melos - run: flutter pub global activate melos - - name: Install dependencies - run: melos prepare - - name: Run flutter tests - run: melos test - - name: Run dart tests - run: melos test:web diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 34b33195..4b8537c3 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -29,7 +29,22 @@ jobs: run: melos analyze:packages - name: Publish dry-run run: melos publish --dry-run --yes - - name: Check publish score + + pana: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Flutter + uses: subosito/flutter-action@v2 + with: + flutter-version: "3.x" + channel: "stable" + + - name: Install Melos + run: flutter pub global activate melos + - name: Install dependencies + run: melos prepare + - name: Check pana score run: | flutter pub global activate pana melos analyze:packages:pana --no-select From f940b30475f6f74ba37e3ada0fa57c6687d0ab01 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 6 Nov 2024 09:59:28 +0200 Subject: [PATCH 5/5] chore(release): publish packages - powersync@1.9.2 - powersync_attachments_helper@0.6.15+1 --- CHANGELOG.md | 28 +++++++++++++++++++ demos/django-todolist/pubspec.yaml | 2 +- demos/supabase-anonymous-auth/pubspec.yaml | 2 +- .../supabase-edge-function-auth/pubspec.yaml | 2 +- demos/supabase-simple-chat/pubspec.yaml | 2 +- demos/supabase-todolist-drift/pubspec.yaml | 4 +-- .../pubspec.yaml | 2 +- demos/supabase-todolist/pubspec.yaml | 4 +-- packages/powersync/CHANGELOG.md | 4 +++ packages/powersync/lib/src/version.dart | 2 +- packages/powersync/pubspec.yaml | 2 +- .../powersync_attachments_helper/CHANGELOG.md | 4 +++ .../powersync_attachments_helper/pubspec.yaml | 4 +-- 13 files changed, 49 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d9e6c48..f005e678 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,34 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## 2024-11-06 + +### Changes + +--- + +Packages with breaking changes: + + - There are no breaking changes in this release. + +Packages with other changes: + + - [`powersync` - `v1.9.2`](#powersync---v192) + - [`powersync_attachments_helper` - `v0.6.15+1`](#powersync_attachments_helper---v06151) + +Packages with dependency updates only: + +> Packages listed below depend on other packages in this workspace that have had changes. Their versions have been incremented to bump the minimum dependency versions of the packages they depend upon in this project. + + - `powersync_attachments_helper` - `v0.6.15+1` + +--- + +#### `powersync` - `v1.9.2` + + - [Web] Automatically flush IndexedDB storage to fix durability issues + + ## 2024-11-04 ### Changes diff --git a/demos/django-todolist/pubspec.yaml b/demos/django-todolist/pubspec.yaml index 31dc731b..ed4705c4 100644 --- a/demos/django-todolist/pubspec.yaml +++ b/demos/django-todolist/pubspec.yaml @@ -10,7 +10,7 @@ environment: dependencies: flutter: sdk: flutter - powersync: ^1.9.1 + powersync: ^1.9.2 path_provider: ^2.1.1 path: ^1.8.3 logging: ^1.2.0 diff --git a/demos/supabase-anonymous-auth/pubspec.yaml b/demos/supabase-anonymous-auth/pubspec.yaml index 862c8b96..347b187d 100644 --- a/demos/supabase-anonymous-auth/pubspec.yaml +++ b/demos/supabase-anonymous-auth/pubspec.yaml @@ -11,7 +11,7 @@ dependencies: flutter: sdk: flutter - powersync: ^1.9.1 + powersync: ^1.9.2 path_provider: ^2.1.1 supabase_flutter: ^2.0.2 path: ^1.8.3 diff --git a/demos/supabase-edge-function-auth/pubspec.yaml b/demos/supabase-edge-function-auth/pubspec.yaml index af7161ee..16b84696 100644 --- a/demos/supabase-edge-function-auth/pubspec.yaml +++ b/demos/supabase-edge-function-auth/pubspec.yaml @@ -11,7 +11,7 @@ dependencies: flutter: sdk: flutter - powersync: ^1.9.1 + powersync: ^1.9.2 path_provider: ^2.1.1 supabase_flutter: ^2.0.2 path: ^1.8.3 diff --git a/demos/supabase-simple-chat/pubspec.yaml b/demos/supabase-simple-chat/pubspec.yaml index c8018cd6..a67fc121 100644 --- a/demos/supabase-simple-chat/pubspec.yaml +++ b/demos/supabase-simple-chat/pubspec.yaml @@ -37,7 +37,7 @@ dependencies: supabase_flutter: ^2.0.2 timeago: ^3.6.0 - powersync: ^1.9.1 + powersync: ^1.9.2 path_provider: ^2.1.1 path: ^1.8.3 logging: ^1.2.0 diff --git a/demos/supabase-todolist-drift/pubspec.yaml b/demos/supabase-todolist-drift/pubspec.yaml index 6813e8bd..a75c0958 100644 --- a/demos/supabase-todolist-drift/pubspec.yaml +++ b/demos/supabase-todolist-drift/pubspec.yaml @@ -9,8 +9,8 @@ environment: dependencies: flutter: sdk: flutter - powersync_attachments_helper: ^0.6.15 - powersync: ^1.9.1 + powersync_attachments_helper: ^0.6.15+1 + powersync: ^1.9.2 path_provider: ^2.1.1 supabase_flutter: ^2.0.1 path: ^1.8.3 diff --git a/demos/supabase-todolist-optional-sync/pubspec.yaml b/demos/supabase-todolist-optional-sync/pubspec.yaml index 4595be39..2b92d08c 100644 --- a/demos/supabase-todolist-optional-sync/pubspec.yaml +++ b/demos/supabase-todolist-optional-sync/pubspec.yaml @@ -10,7 +10,7 @@ environment: dependencies: flutter: sdk: flutter - powersync: ^1.9.1 + powersync: ^1.9.2 path_provider: ^2.1.1 supabase_flutter: ^2.0.1 path: ^1.8.3 diff --git a/demos/supabase-todolist/pubspec.yaml b/demos/supabase-todolist/pubspec.yaml index 13ba7e71..d421c2b7 100644 --- a/demos/supabase-todolist/pubspec.yaml +++ b/demos/supabase-todolist/pubspec.yaml @@ -10,8 +10,8 @@ environment: dependencies: flutter: sdk: flutter - powersync_attachments_helper: ^0.6.15 - powersync: ^1.9.1 + powersync_attachments_helper: ^0.6.15+1 + powersync: ^1.9.2 path_provider: ^2.1.1 supabase_flutter: ^2.0.1 path: ^1.8.3 diff --git a/packages/powersync/CHANGELOG.md b/packages/powersync/CHANGELOG.md index 1e38eb56..991d86de 100644 --- a/packages/powersync/CHANGELOG.md +++ b/packages/powersync/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.9.2 + + - [Web] Automatically flush IndexedDB storage to fix durability issues + ## 1.9.1 - Flutter Web Beta release diff --git a/packages/powersync/lib/src/version.dart b/packages/powersync/lib/src/version.dart index c66eb9de..30da80f3 100644 --- a/packages/powersync/lib/src/version.dart +++ b/packages/powersync/lib/src/version.dart @@ -1 +1 @@ -const String libraryVersion = '1.9.1'; +const String libraryVersion = '1.9.2'; diff --git a/packages/powersync/pubspec.yaml b/packages/powersync/pubspec.yaml index d7eddf3f..1d736c54 100644 --- a/packages/powersync/pubspec.yaml +++ b/packages/powersync/pubspec.yaml @@ -1,5 +1,5 @@ name: powersync -version: 1.9.1 +version: 1.9.2 homepage: https://powersync.com repository: https://github.com/powersync-ja/powersync.dart description: PowerSync Flutter SDK - sync engine for building local-first apps. diff --git a/packages/powersync_attachments_helper/CHANGELOG.md b/packages/powersync_attachments_helper/CHANGELOG.md index ebda9775..a1c57c77 100644 --- a/packages/powersync_attachments_helper/CHANGELOG.md +++ b/packages/powersync_attachments_helper/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.6.15+1 + + - Update a dependency to the latest release. + ## 0.6.15 - Update a dependency to the latest release. diff --git a/packages/powersync_attachments_helper/pubspec.yaml b/packages/powersync_attachments_helper/pubspec.yaml index d1206609..54bf1eb7 100644 --- a/packages/powersync_attachments_helper/pubspec.yaml +++ b/packages/powersync_attachments_helper/pubspec.yaml @@ -1,6 +1,6 @@ name: powersync_attachments_helper description: A helper library for handling attachments when using PowerSync. -version: 0.6.15 +version: 0.6.15+1 repository: https://github.com/powersync-ja/powersync.dart homepage: https://www.powersync.com/ environment: @@ -10,7 +10,7 @@ dependencies: flutter: sdk: flutter - powersync: ^1.9.1 + powersync: ^1.9.2 logging: ^1.2.0 sqlite_async: ^0.11.0 path_provider: ^2.0.13