Merge pull request #492 from powersync-ja/cache-temp-storage

Cache size & temp storage

Author: rkistner

.changeset/lovely-impalas-do.md

@@ -0,0 +1,5 @@
+---
+'@powersync/web': minor
+---
+
+Add cacheSizeKb option, defaulting to 50MB.

.changeset/slow-spiders-smash.md

@@ -0,0 +1,5 @@
+---
+'@powersync/diagnostics-app': minor
+---
+
+Switch diagnostics app to OPFS.

.changeset/spotty-students-serve.md

@@ -0,0 +1,5 @@
+---
+'@powersync/op-sqlite': minor
+---
+
+Default to using memory for temp store, and 50MB cache size.

packages/powersync-op-sqlite/README.md

@@ -74,9 +74,7 @@ To load additional SQLite extensions include the `extensions` option in `sqliteO
 
 ```js
 sqliteOptions: {
-  extensions: [
-      { path: libPath, entryPoint: 'sqlite3_powersync_init' }
-  ]
+  extensions: [{ path: libPath, entryPoint: 'sqlite3_powersync_init' }];
 }
 ```
 
@@ -87,38 +85,21 @@ Example usage:
 ```ts
 import { getDylibPath } from '@op-engineering/op-sqlite';
 
-let libPath: string
+let libPath: string;
 if (Platform.OS === 'ios') {
-  libPath = getDylibPath('co.powersync.sqlitecore', 'powersync-sqlite-core')
+  libPath = getDylibPath('co.powersync.sqlitecore', 'powersync-sqlite-core');
 } else {
   libPath = 'libpowersync';
 }
 
 const factory = new OPSqliteOpenFactory({
   dbFilename: 'sqlite.db',
   sqliteOptions: {
-    extensions: [
-        { path: libPath, entryPoint: 'sqlite3_powersync_init' }
-    ]
+    extensions: [{ path: libPath, entryPoint: 'sqlite3_powersync_init' }]
   }
 });
 ```
 
-## Using the Memory Temporary Store
-
-For some targets like Android 12/API 31, syncing of large datasets may cause disk IO errors due to the default temporary store option (file) used.
-To resolve this you can use the `memory` option, by adding the following configuration option to your application's `package.json`
-
-```json
-{
-  // your normal package.json
-  // ...
-  "op-sqlite": {
-    "sqliteFlags": "-DSQLITE_TEMP_STORE=2"
-  }
-}
-```
-
 ## Native Projects
 
 This package uses native libraries. Create native Android and iOS projects (if not created already) by running:

packages/powersync-op-sqlite/src/db/OPSqliteAdapter.ts

@@ -44,19 +44,28 @@ export class OPSQLiteDBAdapter extends BaseObserver<DBAdapterListener> implement
   }
 
   protected async init() {
-    const { lockTimeoutMs, journalMode, journalSizeLimit, synchronous } = this.options.sqliteOptions!;
+    const { lockTimeoutMs, journalMode, journalSizeLimit, synchronous, cacheSizeKb, temporaryStorage } =
+      this.options.sqliteOptions!;
     const dbFilename = this.options.name;
 
     this.writeConnection = await this.openConnection(dbFilename);
 
-    const statements: string[] = [
+    const baseStatements = [
       `PRAGMA busy_timeout = ${lockTimeoutMs}`,
+      `PRAGMA cache_size = -${cacheSizeKb}`,
+      `PRAGMA temp_store = ${temporaryStorage}`
+    ];
+
+    const writeConnectionStatements = [
+      ...baseStatements,
       `PRAGMA journal_mode = ${journalMode}`,
       `PRAGMA journal_size_limit = ${journalSizeLimit}`,
       `PRAGMA synchronous = ${synchronous}`
     ];
 
-    for (const statement of statements) {
+    const readConnectionStatements = [...baseStatements, 'PRAGMA query_only = true'];
+
+    for (const statement of writeConnectionStatements) {
       for (let tries = 0; tries < 30; tries++) {
         try {
           await this.writeConnection!.execute(statement);
@@ -79,7 +88,9 @@ export class OPSQLiteDBAdapter extends BaseObserver<DBAdapterListener> implement
     this.readConnections = [];
     for (let i = 0; i < READ_CONNECTIONS; i++) {
       const conn = await this.openConnection(dbFilename);
-      await conn.execute('PRAGMA query_only = true');
+      for (let statement of readConnectionStatements) {
+        await conn.execute(statement);
+      }
       this.readConnections.push({ busy: false, connection: conn });
     }
   }

packages/powersync-op-sqlite/src/db/SqliteOptions.ts

@@ -30,6 +30,21 @@ export interface SqliteOptions {
    */
   encryptionKey?: string | null;
 
+  /**
+   * Where to store SQLite temporary files. Defaults to 'MEMORY'.
+   * Setting this to `FILESYSTEM` can cause issues with larger queries or datasets.
+   *
+   * For details, see: https://www.sqlite.org/pragma.html#pragma_temp_store
+   */
+  temporaryStorage?: TemporaryStorageOption;
+
+  /**
+   * Maximum SQLite cache size. Defaults to 50MB.
+   *
+   * For details, see: https://www.sqlite.org/pragma.html#pragma_cache_size
+   */
+  cacheSizeKb?: number;
+
   /**
    * Load extensions using the path and entryPoint.
    * More info can be found here https://op-engineering.github.io/op-sqlite/docs/api#loading-extensions.
@@ -40,6 +55,11 @@ export interface SqliteOptions {
   }>;
 }
 
+export enum TemporaryStorageOption {
+  MEMORY = 'memory',
+  FILESYSTEM = 'file'
+}
+
 // SQLite journal mode. Set on the primary connection.
 // This library is written with WAL mode in mind - other modes may cause
 // unexpected locking behavior.
@@ -65,6 +85,8 @@ export const DEFAULT_SQLITE_OPTIONS: Required<SqliteOptions> = {
   journalMode: SqliteJournalMode.wal,
   synchronous: SqliteSynchronous.normal,
   journalSizeLimit: 6 * 1024 * 1024,
+  cacheSizeKb: 50 * 1024,
+  temporaryStorage: TemporaryStorageOption.MEMORY,
   lockTimeoutMs: 30000,
   encryptionKey: null,
   extensions: []

packages/web/src/db/adapters/wa-sqlite/WASQLiteConnection.ts

@@ -235,6 +235,7 @@ export class WASqliteConnection
     await this.openDB();
     this.registerBroadcastListeners();
     await this.executeSingleStatement(`PRAGMA temp_store = ${this.options.temporaryStorage};`);
+    await this.executeSingleStatement(`PRAGMA cache_size = -${this.options.cacheSizeKb};`);
     await this.executeEncryptionPragma();
 
     this.sqliteAPI.update_hook(this.dbP, (updateType: number, dbName: string | null, tableName: string | null) => {

packages/web/src/db/adapters/wa-sqlite/WASQLiteDBAdapter.ts

@@ -3,7 +3,12 @@ import * as Comlink from 'comlink';
 import { resolveWebPowerSyncFlags } from '../../PowerSyncDatabase';
 import { OpenAsyncDatabaseConnection } from '../AsyncDatabaseConnection';
 import { LockedAsyncDatabaseAdapter } from '../LockedAsyncDatabaseAdapter';
-import { ResolvedWebSQLOpenOptions, TemporaryStorageOption, WebSQLFlags } from '../web-sql-flags';
+import {
+  DEFAULT_CACHE_SIZE_KB,
+  ResolvedWebSQLOpenOptions,
+  TemporaryStorageOption,
+  WebSQLFlags
+} from '../web-sql-flags';
 import { WorkerWrappedAsyncDatabaseConnection } from '../WorkerWrappedAsyncDatabaseConnection';
 import { WASQLiteVFS } from './WASQLiteConnection';
 import { WASQLiteOpenFactory } from './WASQLiteOpenFactory';
@@ -27,6 +32,7 @@ export interface WASQLiteDBAdapterOptions extends Omit<PowerSyncOpenFactoryOptio
 
   vfs?: WASQLiteVFS;
   temporaryStorage?: TemporaryStorageOption;
+  cacheSizeKb?: number;
 
   /**
    * Encryption key for the database.
@@ -43,7 +49,7 @@ export class WASQLiteDBAdapter extends LockedAsyncDatabaseAdapter {
     super({
       name: options.dbFilename,
       openConnection: async () => {
-        const { workerPort, temporaryStorage } = options;
+        const { workerPort, temporaryStorage, cacheSizeKb } = options;
         if (workerPort) {
           const remote = Comlink.wrap<OpenAsyncDatabaseConnection>(workerPort);
           return new WorkerWrappedAsyncDatabaseConnection({
@@ -52,6 +58,7 @@ export class WASQLiteDBAdapter extends LockedAsyncDatabaseAdapter {
             baseConnection: await remote({
               ...options,
               temporaryStorage: temporaryStorage ?? TemporaryStorageOption.MEMORY,
+              cacheSizeKb: cacheSizeKb ?? DEFAULT_CACHE_SIZE_KB,
               flags: resolveWebPowerSyncFlags(options.flags),
               encryptionKey: options.encryptionKey
             })
@@ -63,6 +70,7 @@ export class WASQLiteDBAdapter extends LockedAsyncDatabaseAdapter {
           debugMode: options.debugMode,
           flags: options.flags,
           temporaryStorage,
+          cacheSizeKb,
           logger: options.logger,
           vfs: options.vfs,
           encryptionKey: options.encryptionKey,

packages/web/src/db/adapters/wa-sqlite/WASQLiteOpenFactory.ts

@@ -4,7 +4,12 @@ import { openWorkerDatabasePort, resolveWorkerDatabasePortFactory } from '../../
 import { AbstractWebSQLOpenFactory } from '../AbstractWebSQLOpenFactory';
 import { AsyncDatabaseConnection, OpenAsyncDatabaseConnection } from '../AsyncDatabaseConnection';
 import { LockedAsyncDatabaseAdapter } from '../LockedAsyncDatabaseAdapter';
-import { ResolvedWebSQLOpenOptions, TemporaryStorageOption, WebSQLOpenFactoryOptions } from '../web-sql-flags';
+import {
+  DEFAULT_CACHE_SIZE_KB,
+  ResolvedWebSQLOpenOptions,
+  TemporaryStorageOption,
+  WebSQLOpenFactoryOptions
+} from '../web-sql-flags';
 import { WorkerWrappedAsyncDatabaseConnection } from '../WorkerWrappedAsyncDatabaseConnection';
 import { WASqliteConnection, WASQLiteVFS } from './WASQLiteConnection';
 
@@ -44,6 +49,7 @@ export class WASQLiteOpenFactory extends AbstractWebSQLOpenFactory {
     const {
       vfs = WASQLiteVFS.IDBBatchAtomicVFS,
       temporaryStorage = TemporaryStorageOption.MEMORY,
+      cacheSizeKb = DEFAULT_CACHE_SIZE_KB,
       encryptionKey
     } = this.waOptions;
 
@@ -60,6 +66,7 @@ export class WASQLiteOpenFactory extends AbstractWebSQLOpenFactory {
               optionsDbWorker({
                 ...this.options,
                 temporaryStorage,
+                cacheSizeKb,
                 flags: this.resolvedFlags,
                 encryptionKey
               })
@@ -74,6 +81,7 @@ export class WASQLiteOpenFactory extends AbstractWebSQLOpenFactory {
           dbFilename: this.options.dbFilename,
           vfs,
           temporaryStorage,
+          cacheSizeKb,
           flags: this.resolvedFlags,
           encryptionKey: encryptionKey
         }),
@@ -94,6 +102,7 @@ export class WASQLiteOpenFactory extends AbstractWebSQLOpenFactory {
         debugMode: this.options.debugMode,
         vfs,
         temporaryStorage,
+        cacheSizeKb,
         flags: this.resolvedFlags,
         encryptionKey: encryptionKey
       });

packages/web/src/db/adapters/web-sql-flags.ts

@@ -47,6 +47,8 @@ export interface ResolvedWebSQLOpenOptions extends SQLOpenOptions {
    */
   temporaryStorage: TemporaryStorageOption;
 
+  cacheSizeKb: number;
+
   /**
    * Encryption key for the database.
    * If set, the database will be encrypted using ChaCha20.
@@ -59,6 +61,8 @@ export enum TemporaryStorageOption {
   FILESYSTEM = 'file'
 }
 
+export const DEFAULT_CACHE_SIZE_KB = 50 * 1024;
+
 /**
  * Options for opening a Web SQL connection
  */
@@ -74,12 +78,22 @@ export interface WebSQLOpenFactoryOptions extends SQLOpenOptions {
   worker?: string | URL | ((options: ResolvedWebSQLOpenOptions) => Worker | SharedWorker);
 
   logger?: ILogger;
+
   /**
    * Where to store SQLite temporary files. Defaults to 'MEMORY'.
    * Setting this to `FILESYSTEM` can cause issues with larger queries or datasets.
+   *
+   * For details, see: https://www.sqlite.org/pragma.html#pragma_temp_store
    */
   temporaryStorage?: TemporaryStorageOption;
 
+  /**
+   * Maximum SQLite cache size. Defaults to 50MB.
+   *
+   * For details, see: https://www.sqlite.org/pragma.html#pragma_cache_size
+   */
+  cacheSizeKb?: number;
+
   /**
    * Encryption key for the database.
    * If set, the database will be encrypted using ChaCha20.

packages/web/src/db/sync/SharedWebStreamingSyncImplementation.ts

@@ -6,7 +6,7 @@ import {
   SharedSyncClientEvent,
   SharedSyncImplementation
 } from '../../worker/sync/SharedSyncImplementation';
-import { resolveWebSQLFlags, TemporaryStorageOption } from '../adapters/web-sql-flags';
+import { DEFAULT_CACHE_SIZE_KB, resolveWebSQLFlags, TemporaryStorageOption } from '../adapters/web-sql-flags';
 import { WebDBAdapter } from '../adapters/WebDBAdapter';
 import {
   WebStreamingSyncImplementation,
@@ -106,10 +106,10 @@ export class SharedWebStreamingSyncImplementation extends WebStreamingSyncImplem
      * This worker will manage all syncing operations remotely.
      */
     const resolvedWorkerOptions = {
-      ...options,
       dbFilename: this.options.identifier!,
-      // TODO
       temporaryStorage: TemporaryStorageOption.MEMORY,
+      cacheSizeKb: DEFAULT_CACHE_SIZE_KB,
+      ...options,
       flags: resolveWebSQLFlags(options.flags)
     };
 

tools/diagnostics-app/src/library/powersync/ConnectionManager.ts

@@ -4,7 +4,10 @@ import {
   PowerSyncDatabase,
   WebRemote,
   WebStreamingSyncImplementation,
-  WebStreamingSyncImplementationOptions
+  WebStreamingSyncImplementationOptions,
+  WASQLiteOpenFactory,
+  TemporaryStorageOption,
+  WASQLiteVFS
 } from '@powersync/web';
 import Logger from 'js-logger';
 import { DynamicSchemaManager } from './DynamicSchemaManager';
@@ -25,14 +28,18 @@ export const getParams = () => {
 
 export const schemaManager = new DynamicSchemaManager();
 
+const openFactory = new WASQLiteOpenFactory({
+  dbFilename: 'diagnostics.db',
+  debugMode: true,
+  cacheSizeKb: 500 * 1024,
+  temporaryStorage: TemporaryStorageOption.MEMORY,
+  vfs: WASQLiteVFS.OPFSCoopSyncVFS
+});
+
 export const db = new PowerSyncDatabase({
-  database: {
-    dbFilename: 'example.db',
-    debugMode: true
-  },
+  database: openFactory,
   schema: schemaManager.buildSchema()
 });
-db.execute('PRAGMA cache_size=-500000');
 
 export const connector = new TokenConnector();