Merge pull request #82 from powersync-ja/encryption-support

Package maintenance, prepare encryption and dart2wasm support

Author: simolus3

.github/workflows/test.yaml

@@ -35,19 +35,19 @@ jobs:
         include:
           - sqlite_version: "3440200"
             sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3440200.tar.gz"
-            dart_sdk: 3.5.0
+            dart_sdk: stable
           - sqlite_version: "3430200"
             sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3430200.tar.gz"
-            dart_sdk: 3.5.0
+            dart_sdk: stable
           - sqlite_version: "3420000"
             sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3420000.tar.gz"
-            dart_sdk: 3.5.0
+            dart_sdk: stable
           - sqlite_version: "3410100"
             sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3410100.tar.gz"
-            dart_sdk: 3.5.0
+            dart_sdk: stable
           - sqlite_version: "3380000"
             sqlite_url: "https://www.sqlite.org/2022/sqlite-autoconf-3380000.tar.gz"
-            dart_sdk: 3.5.0
+            dart_sdk: stable
     steps:
       - uses: actions/checkout@v3
       - uses: dart-lang/setup-dart@v1

melos.yaml

@@ -41,7 +41,7 @@ scripts:
 
   test:
     description: Run tests in a specific package.
-    run: dart test -p chrome,vm
+    run: dart test -p chrome,vm --compiler dart2js,dart2wasm
     exec:
       concurrency: 1
     packageFilters:

packages/drift_sqlite_async/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.1
+
+- Fix lints.
+
 ## 0.2.0
 
  - Automatically run Drift migrations

analysis_options.yaml packages/drift_sqlite_async/analysis_options.yaml

@@ -1,4 +1,2 @@
-library drift_sqlite_async;
-
 export './src/connection.dart';
 export './src/executor.dart';

packages/drift_sqlite_async/lib/drift_sqlite_async.dart

@@ -1,7 +1,7 @@
 import 'dart:async';
 
 import 'package:drift/backends.dart';
-import 'package:drift/src/runtime/query_builder/query_builder.dart';
+import 'package:drift/drift.dart';
 import 'package:sqlite_async/sqlite3_common.dart';
 import 'package:sqlite_async/sqlite_async.dart';
 
@@ -19,6 +19,7 @@ class _SqliteAsyncDelegate extends _SqliteAsyncQueryDelegate
 
   _SqliteAsyncDelegate(this.db) : super(db, db.writeLock);
 
+  @override
   bool isInTransaction = false; // unused
 
   @override

packages/drift_sqlite_async/lib/src/executor.dart

@@ -1,5 +1,5 @@
 name: drift_sqlite_async
-version: 0.2.0
+version: 0.2.1
 homepage: https://github.com/powersync-ja/sqlite_async.dart
 repository: https://github.com/powersync-ja/sqlite_async.dart
 description: Use Drift with a sqlite_async database, allowing both to be used in the same application.
@@ -16,10 +16,12 @@ environment:
 dependencies:
   drift: ">=2.19.0 <3.0.0"
   sqlite_async: ^0.11.0
+
 dev_dependencies:
   build_runner: ^2.4.8
   drift_dev: ">=2.19.0 <3.0.0"
   glob: ^2.1.2
+  lints: ^5.0.0
   sqlite3: ^2.4.0
   test: ^1.25.2
   test_api: ^0.7.0

packages/drift_sqlite_async/pubspec.yaml

@@ -1,5 +1,7 @@
 // TODO
 @TestOn('!browser')
+library;
+
 import 'dart:async';
 
 import 'package:drift/drift.dart';

packages/drift_sqlite_async/test/basic_test.dart

@@ -1,5 +1,7 @@
 // TODO
 @TestOn('!browser')
+library;
+
 import 'package:drift/drift.dart';
 import 'package:sqlite_async/sqlite_async.dart';
 import 'package:test/test.dart';

packages/drift_sqlite_async/test/db_test.dart

@@ -21,7 +21,7 @@ class TodoDatabase extends _$TodoDatabase {
 }
 
 class TodosMigrationDatabase extends TodoDatabase {
-  TodosMigrationDatabase(SqliteConnection db) : super(db);
+  TodosMigrationDatabase(super.db);
 
   @override
   MigrationStrategy get migration {

packages/drift_sqlite_async/test/generated/database.dart

@@ -1,4 +1,6 @@
 @TestOn('!browser')
+library;
+
 import 'package:sqlite_async/sqlite_async.dart';
 import 'package:test/test.dart';
 

packages/drift_sqlite_async/test/migration_test.dart

@@ -1,3 +1,8 @@
+## 0.11.2
+
+- Support latest version of `package:sqlite3_web`.
+- Support `dart2wasm`.
+
 ## 0.11.1
 
 - Remove remaining `dart:js_util` imports in favor of new interop APIs.

packages/sqlite_async/CHANGELOG.md

@@ -0,0 +1 @@
+include: package:lints/recommended.yaml

packages/sqlite_async/analysis_options.yaml

@@ -1,8 +1,10 @@
 import 'dart:async';
 import 'dart:js_interop';
+import 'dart:js_interop_unsafe';
 
 import 'package:sqlite3/common.dart';
 import 'package:sqlite3_web/sqlite3_web.dart';
+import 'package:sqlite3_web/protocol_utils.dart' as proto;
 import 'package:sqlite_async/sqlite_async.dart';
 import 'package:sqlite_async/src/utils/shared_utils.dart';
 import 'package:sqlite_async/src/web/database/broadcast_updates.dart';
@@ -256,9 +258,15 @@ class _ExclusiveTransactionContext extends _ExclusiveContext {
     // JavaScript object. This is the converted into a Dart ResultSet.
     return await wrapSqliteException(() async {
       var res = await _database._database.customRequest(CustomDatabaseMessage(
-          CustomDatabaseMessageKind.executeInTransaction, sql, parameters));
-      var result =
-          Map<String, dynamic>.from((res as JSObject).dartify() as Map);
+              CustomDatabaseMessageKind.executeInTransaction, sql, parameters))
+          as JSObject;
+
+      if (res.has('format') && (res['format'] as JSNumber).toDartInt == 2) {
+        // Newer workers use a serialization format more efficient than dartify().
+        return proto.deserializeResultSet(res['r'] as JSObject);
+      }
+
+      var result = Map<String, dynamic>.from(res.dartify() as Map);
       final columnNames = [
         for (final entry in result['columnNames']) entry as String
       ];
@@ -303,9 +311,14 @@ Future<T> wrapSqliteException<T>(Future<T> Function() callback) async {
   try {
     return await callback();
   } on RemoteException catch (ex) {
+    if (ex.exception case final serializedCause?) {
+      throw serializedCause;
+    }
+
+    // Older versions of package:sqlite_web reported SqliteExceptions as strings
+    // only.
     if (ex.toString().contains('SqliteException')) {
       RegExp regExp = RegExp(r'SqliteException\((\d+)\)');
-      // The SQLite Web package wraps these in remote errors
       throw SqliteException(
           int.parse(regExp.firstMatch(ex.message)?.group(1) ?? '0'),
           ex.message);

packages/sqlite_async/lib/src/web/database.dart

@@ -3,6 +3,7 @@
 library;
 
 import 'dart:js_interop';
+import 'package:sqlite3_web/protocol_utils.dart' as proto;
 
 enum CustomDatabaseMessageKind {
   requestSharedLock,
@@ -19,15 +20,24 @@ extension type CustomDatabaseMessage._raw(JSObject _) implements JSObject {
     required JSString rawKind,
     JSString rawSql,
     JSArray rawParameters,
+    JSArrayBuffer typeInfo,
   });
 
   factory CustomDatabaseMessage(CustomDatabaseMessageKind kind,
       [String? sql, List<Object?> parameters = const []]) {
-    final rawSql = sql?.toJS ?? ''.toJS;
-    final rawParameters =
-        <JSAny?>[for (final parameter in parameters) parameter.jsify()].toJS;
+    final rawSql = (sql ?? '').toJS;
+    // Serializing parameters this way is backwards-compatible with dartify()
+    // on the other end, but a bit more efficient while also suppporting sound
+    // communcation between dart2js workers and dart2wasm clients.
+    // Older workers ignore the typeInfo, but that's not a problem.
+    final (rawParameters, typeInfo) = proto.serializeParameters(parameters);
+
     return CustomDatabaseMessage._(
-        rawKind: kind.name.toJS, rawSql: rawSql, rawParameters: rawParameters);
+      rawKind: kind.name.toJS,
+      rawSql: rawSql,
+      rawParameters: rawParameters,
+      typeInfo: typeInfo,
+    );
   }
 
   external JSString get rawKind;
@@ -36,6 +46,9 @@ extension type CustomDatabaseMessage._raw(JSObject _) implements JSObject {
 
   external JSArray get rawParameters;
 
+  /// Not set in earlier versions of this package.
+  external JSArrayBuffer? get typeInfo;
+
   CustomDatabaseMessageKind get kind {
     return CustomDatabaseMessageKind.values.byName(rawKind.toDart);
   }

packages/sqlite_async/lib/src/web/protocol.dart

@@ -15,7 +15,7 @@ Map<String, FutureOr<WebSqlite>> webSQLiteImplementations = {};
 /// Web implementation of [AbstractDefaultSqliteOpenFactory]
 class DefaultSqliteOpenFactory
     extends AbstractDefaultSqliteOpenFactory<CommonDatabase>
-    implements WebSqliteOpenFactory {
+    with WebSqliteOpenFactory {
   late final Future<WebSqlite> _initialized = Future.sync(() {
     final cacheKey = sqliteOptions.webSqliteOptions.wasmUri +
         sqliteOptions.webSqliteOptions.workerUri;
@@ -45,9 +45,8 @@ class DefaultSqliteOpenFactory
     );
   }
 
-  @override
-
   /// This is currently not supported on web
+  @override
   CommonDatabase openDB(SqliteOpenOptions options) {
     throw UnimplementedError(
         'Direct access to CommonDatabase is not available on web.');
@@ -61,7 +60,7 @@ class DefaultSqliteOpenFactory
   /// Due to being asynchronous, the under laying CommonDatabase is not accessible
   Future<SqliteConnection> openConnection(SqliteOpenOptions options) async {
     final workers = await _initialized;
-    final connection = await workers.connectToRecommended(path);
+    final connection = await connectToWorker(workers, path);
 
     // When the database is accessed through a shared worker, we implement
     // mutexes over custom messages sent through the shared worker. In other

packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart

@@ -107,6 +107,7 @@ class ThrottledCommonDatabase extends CommonDatabase {
   @override
   VoidPredicate? get commitFilter => _db.commitFilter;
 
+  @override
   set commitFilter(VoidPredicate? filter) => _db.commitFilter = filter;
 
   @override

packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart

@@ -1,8 +1,12 @@
 import 'dart:js_interop';
+import 'dart:js_interop_unsafe';
 
+import 'package:meta/meta.dart';
 import 'package:mutex/mutex.dart';
 import 'package:sqlite3/wasm.dart';
 import 'package:sqlite3_web/sqlite3_web.dart';
+import 'package:sqlite3_web/protocol_utils.dart' as proto;
+
 import 'throttled_common_database.dart';
 
 import '../protocol.dart';
@@ -12,9 +16,9 @@ import '../protocol.dart';
 /// can be extended to perform custom requests.
 base class AsyncSqliteController extends DatabaseController {
   @override
-  Future<WorkerDatabase> openDatabase(
-      WasmSqlite3 sqlite3, String path, String vfs) async {
-    final db = sqlite3.open(path, vfs: vfs);
+  Future<WorkerDatabase> openDatabase(WasmSqlite3 sqlite3, String path,
+      String vfs, JSAny? additionalData) async {
+    final db = openUnderlying(sqlite3, path, vfs, additionalData);
 
     // Register any custom functions here if needed
 
@@ -23,6 +27,18 @@ base class AsyncSqliteController extends DatabaseController {
     return AsyncSqliteDatabase(database: throttled);
   }
 
+  /// Opens a database with the `sqlite3` package that will be wrapped in a
+  /// [ThrottledCommonDatabase] for [openDatabase].
+  @visibleForOverriding
+  CommonDatabase openUnderlying(
+    WasmSqlite3 sqlite3,
+    String path,
+    String vfs,
+    JSAny? additionalData,
+  ) {
+    return sqlite3.open(path, vfs: vfs);
+  }
+
   @override
   Future<JSAny?> handleCustomRequest(
       ClientConnection connection, JSAny? request) {
@@ -61,25 +77,32 @@ class AsyncSqliteDatabase extends WorkerDatabase {
         return database.autocommit.toJS;
       case CustomDatabaseMessageKind.executeInTransaction:
         final sql = message.rawSql.toDart;
-        final parameters = [
-          for (final raw in (message.rawParameters).toDart) raw.dartify()
-        ];
+        final hasTypeInfo = message.typeInfo.isDefinedAndNotNull;
+        final parameters = proto.deserializeParameters(
+            message.rawParameters, message.typeInfo);
         if (database.autocommit) {
           throw SqliteException(0,
               "Transaction rolled back by earlier statement. Cannot execute: $sql");
         }
-        var res = database.select(sql, parameters);
 
-        var dartMap = resultSetToMap(res);
-
-        var jsObject = dartMap.jsify();
+        var res = database.select(sql, parameters);
+        if (hasTypeInfo) {
+          // If the client is sending a request that has parameters with type
+          // information, it will also support a newer serialization format for
+          // result sets.
+          return JSObject()
+            ..['format'] = 2.toJS
+            ..['r'] = proto.serializeResultSet(res);
+        } else {
+          var dartMap = resultSetToMap(res);
+          var jsObject = dartMap.jsify();
+          return jsObject;
+        }
 
-        return jsObject;
       case CustomDatabaseMessageKind.executeBatchInTransaction:
         final sql = message.rawSql.toDart;
-        final parameters = [
-          for (final raw in (message.rawParameters).toDart) raw.dartify()
-        ];
+        final parameters = proto.deserializeParameters(
+            message.rawParameters, message.typeInfo);
         if (database.autocommit) {
           throw SqliteException(0,
               "Transaction rolled back by earlier statement. Cannot execute: $sql");