Refactoring and fixes.

rkistner · rkistner · commit f79ff7e14abe · 2024-08-16T15:00:52.000+02:00
diff --git a/package.json b/package.json
@@ -1,6 +1,7 @@
 {
   "scripts": {
     "build": "pnpm run -r build",
+    "clean": "pnpm run -r clean",
     "test": "CI=1 pnpm run -r test",
     "format": "prettier . --write"
   },
diff --git a/packages/sqlite-js-api/package.json b/packages/sqlite-js-api/package.json
@@ -5,7 +5,8 @@
   "type": "module",
   "scripts": {
     "build": "tsc -b",
-    "test": "vitest"
+    "test": "vitest",
+    "clean": "tsc -b --clean && rm -rf lib"
   },
   "exports": {
     ".": "./lib/index.js"
diff --git a/packages/sqlite-js-api/test/src/node-sqlite.test.ts b/packages/sqlite-js-api/test/src/node-sqlite.test.ts
@@ -1,8 +1,8 @@
-import { nodeSqliteAsyncPool } from '@powersync/sqlite-js-driver/node';
+import { NodeSqliteDriver } from '@powersync/sqlite-js-driver/node';
 import { ConnectionPoolImpl } from '../../lib/impl.js';
 import { describeImplTests } from './impl-tests.js';
 
 describeImplTests(
   'node:sqlite',
-  (path) => new ConnectionPoolImpl(nodeSqliteAsyncPool(path))
+  (path) => new ConnectionPoolImpl(NodeSqliteDriver.open(path))
 );
diff --git a/packages/sqlite-js-better-sqlite3/package.json b/packages/sqlite-js-better-sqlite3/package.json
@@ -5,7 +5,8 @@
   "type": "module",
   "scripts": {
     "build": "tsc -b",
-    "test": "pnpm build && vitest"
+    "test": "pnpm build && vitest",
+    "clean": "tsc -b --clean && tsc -b ./test/tsconfig.json --clean && rm -rf lib test/lib"
   },
   "exports": {
     ".": "./lib/index.js",
diff --git a/packages/sqlite-js-better-sqlite3/src/worker-driver.ts b/packages/sqlite-js-better-sqlite3/src/worker-driver.ts
@@ -11,7 +11,7 @@ export function betterSqliteAsyncPool(
   return new ReadWriteConnectionPool({
     async openConnection(options) {
       return new WorkerDriverConnection(
-        new URL('./better-sqlite3-worker.js', import.meta.url),
+        new URL('./worker.js', import.meta.url),
         path,
         {
           ...poolOptions,
diff --git a/packages/sqlite-js-better-sqlite3/test/src/better-sqlite3-async.test.ts b/packages/sqlite-js-better-sqlite3/test/src/better-sqlite3-async.test.ts
@@ -1,4 +1,4 @@
-import { betterSqliteAsyncPool } from '../../lib/better-sqlite3-async-driver.js';
+import { betterSqliteAsyncPool } from '../../lib/worker-driver.js';
 
 import { describeDriverTests } from '@powersync/sqlite-js-driver-tests';
 
diff --git a/packages/sqlite-js-better-sqlite3/test/src/better-sqlite3.test.ts b/packages/sqlite-js-better-sqlite3/test/src/better-sqlite3.test.ts
@@ -1,4 +1,4 @@
-import { betterSqlitePool } from '../../lib/better-sqlite3-driver.js';
+import { betterSqlitePool } from '../../lib/sync-driver.js';
 import { describeDriverTests } from '@powersync/sqlite-js-driver-tests';
 
 describeDriverTests(
diff --git a/packages/sqlite-js-driver-tests/package.json b/packages/sqlite-js-driver-tests/package.json
@@ -10,6 +10,7 @@
   },
   "scripts": {
     "build": "tsc -b",
+    "clean": "tsc -b --clean && tsc -b ./test/tsconfig.json --clean && rm -rf lib test/lib",
     "test": "tsc -b ./test/tsconfig.json && NODE_OPTIONS=\"--experimental-sqlite --disable-warning=ExperimentalWarning\" mocha"
   },
   "keywords": [],
diff --git a/packages/sqlite-js-driver-tests/test/src/node-sqlite-async.test.ts b/packages/sqlite-js-driver-tests/test/src/node-sqlite-async.test.ts
@@ -1,8 +1,8 @@
-import { nodeSqliteAsyncPool } from '@powersync/sqlite-js-driver/node';
+import { NodeSqliteDriver } from '@powersync/sqlite-js-driver/node';
 import { describeDriverTests } from '../../lib/index.js';
 
 describeDriverTests(
   'node:sqlite worker',
   { getColumns: false, rawResults: false, allowsMissingParameters: true },
-  nodeSqliteAsyncPool
+  (path) => NodeSqliteDriver.open(path)
 );
diff --git a/packages/sqlite-js-driver-tests/test/src/node-sqlite-sync.test.ts b/packages/sqlite-js-driver-tests/test/src/node-sqlite-sync.test.ts
@@ -1,13 +1,13 @@
-import { nodeSqlitePool } from '@powersync/sqlite-js-driver/node';
+import { NodeSqliteDriver } from '@powersync/sqlite-js-driver/node';
 import { describeDriverTests } from '../../lib/index.js';
 
-import { test, isMocha } from '../../lib/test.js';
+import { isMocha, test } from '../../lib/test.js';
 
 if (isMocha) {
   describeDriverTests(
     'node:sqlite direct',
     { getColumns: false, rawResults: false, allowsMissingParameters: true },
-    nodeSqlitePool
+    (path) => NodeSqliteDriver.openInProcess(path)
   );
 } else {
   test.skip('only running in mocha');
diff --git a/packages/sqlite-js-driver-tests/test/tsconfig.json b/packages/sqlite-js-driver-tests/test/tsconfig.json
@@ -2,7 +2,8 @@
   "extends": "../../../tsconfig.base.json",
   "compilerOptions": {
     "lib": ["ES2022", "DOM"],
-    "outDir": "lib"
+    "outDir": "lib",
+    "rootDir": "src"
   },
   "include": ["src"],
   "references": [{ "path": "../" }]
diff --git a/packages/sqlite-js-driver/package.json b/packages/sqlite-js-driver/package.json
@@ -5,7 +5,8 @@
   "main": "index.js",
   "type": "module",
   "scripts": {
-    "build": "tsc -b"
+    "build": "tsc -b",
+    "clean": "tsc -b --clean && rm -rf lib"
   },
   "exports": {
     ".": "./lib/index.js",
diff --git a/packages/sqlite-js-driver/src/node/driver.ts b/packages/sqlite-js-driver/src/node/driver.ts
@@ -0,0 +1,91 @@
+import {
+  ReserveConnectionOptions,
+  ReservedConnection,
+  SqliteDriverConnectionPool,
+  UpdateListener
+} from '../driver-api.js';
+import { ConnectionPoolOptions } from '../util/connection-pools.js';
+import { LazyConnectionPool } from '../util/LazyConnectionPool.js';
+import { ReadWriteConnectionPool } from '../util/ReadWriteConnectionPool.js';
+import { WorkerDriverConnection } from '../worker_threads/worker-driver.js';
+import { NodeSqliteConnection } from './impl.js';
+import { loadNodeSqlite } from './node-sqlite.js';
+
+export interface NodeSqliteDriverOptions extends ConnectionPoolOptions {
+  /**
+   * Specify a custom path to a worker script, to customize the loading process.
+   */
+  workerPath?: string | URL;
+}
+
+export class NodeSqliteDriver implements SqliteDriverConnectionPool {
+  /**
+   * Opens a single in-process connection.
+   *
+   * Uses blocking I/O.
+   *
+   * This requires `NODE_OPTIONS=--experimental-sqlite`.
+   */
+  static openInProcess(path: string): NodeSqliteDriver {
+    const connection = new LazyConnectionPool(async () => {
+      const sqlite = await loadNodeSqlite();
+      const db = new sqlite.DatabaseSync(path);
+      return new NodeSqliteConnection(db);
+    });
+    return new NodeSqliteDriver(connection);
+  }
+
+  /**
+   * Opens a connection pool with non-blocking I/O using worker_threads.
+   */
+  static open(
+    path: string,
+    options?: NodeSqliteDriverOptions
+  ): NodeSqliteDriver {
+    const workerPath =
+      options?.workerPath ?? new URL('./worker.js', import.meta.url);
+
+    const connection = new ReadWriteConnectionPool(
+      {
+        openConnection: async (connectionOptions) => {
+          return new WorkerDriverConnection(workerPath, path, {
+            readonly: connectionOptions?.readonly ?? false,
+            connectionName: connectionOptions?.connectionName,
+            workerOptions: {
+              env: {
+                ...process.env,
+                NODE_OPTIONS:
+                  '--experimental-sqlite --disable-warning=ExperimentalWarning'
+              }
+            }
+          });
+        }
+      },
+      options
+    );
+    return new NodeSqliteDriver(connection);
+  }
+
+  private constructor(private connection: SqliteDriverConnectionPool) {}
+
+  reserveConnection(
+    options?: ReserveConnectionOptions
+  ): Promise<ReservedConnection> {
+    return this.connection.reserveConnection(options);
+  }
+
+  close(): Promise<void> {
+    return this.connection.close();
+  }
+
+  onUpdate(
+    listener: UpdateListener,
+    options?: { tables?: string[]; batchLimit?: number }
+  ): () => void {
+    return this.connection.onUpdate(listener, options);
+  }
+
+  [Symbol.asyncDispose](): Promise<void> {
+    return this.connection[Symbol.asyncDispose]();
+  }
+}
diff --git a/packages/sqlite-js-driver/src/node/impl.ts b/packages/sqlite-js-driver/src/node/impl.ts
@@ -40,7 +40,7 @@ export function nodeSqlitePool(path: string): SqliteDriverConnectionPool {
       const db = new sqlite.DatabaseSync(path);
       return new NodeSqliteConnection(db, {
         readonly: options?.readonly,
-        name: options?.name
+        name: options?.connectionName
       });
     }
   });
diff --git a/packages/sqlite-js-driver/src/node/index.ts b/packages/sqlite-js-driver/src/node/index.ts
@@ -1,2 +1 @@
-export * from './node-sqlite-driver.js';
-export * from './node-sqlite-worker-driver.js';
+export * from './driver.js';
diff --git a/packages/sqlite-js-driver/src/node/node-sqlite-worker-driver.ts b/packages/sqlite-js-driver/src/node/node-sqlite-worker-driver.ts
diff --git a/packages/sqlite-js-driver/src/node/worker.ts b/packages/sqlite-js-driver/src/node/worker.ts
@@ -1,4 +1,4 @@
-import { NodeSqliteConnection } from './node-sqlite-driver.js';
+import { NodeSqliteConnection } from './impl.js';
 import { retriedOpen, setupDriverWorker } from '../worker_threads/setup.js';
 import { loadNodeSqlite } from './node-sqlite.js';
 
diff --git a/packages/sqlite-js-driver/src/util/LazyConnectionPool.ts b/packages/sqlite-js-driver/src/util/LazyConnectionPool.ts
@@ -0,0 +1,46 @@
+import {
+  ReserveConnectionOptions,
+  ReservedConnection,
+  SqliteDriverConnection,
+  SqliteDriverConnectionPool,
+  UpdateListener
+} from '../driver-api.js';
+import { SingleConnectionPool } from './SingleConnectionPool.js';
+
+/**
+ * Manages locks for a single connection, created asynchronously.
+ */
+export class LazyConnectionPool implements SqliteDriverConnectionPool {
+  private initPromise: Promise<void>;
+  private connection?: SingleConnectionPool;
+
+  [Symbol.asyncDispose]: () => Promise<void> = undefined as any;
+  constructor(open: () => Promise<SqliteDriverConnection>) {
+    if (typeof Symbol.asyncDispose != 'undefined') {
+      this[Symbol.asyncDispose] = () => this.close();
+    }
+
+    this.initPromise = open().then((c) => {
+      this.connection = new SingleConnectionPool(c);
+    });
+  }
+
+  async reserveConnection(
+    options?: ReserveConnectionOptions
+  ): Promise<ReservedConnection> {
+    await this.initPromise;
+    return this.connection!.reserveConnection(options);
+  }
+
+  async close(): Promise<void> {
+    await this.initPromise;
+    await this.connection!.close();
+  }
+
+  onUpdate(
+    listener: UpdateListener,
+    options?: { tables?: string[]; batchLimit?: number }
+  ): () => void {
+    throw new Error('Method not implemented.');
+  }
+}
diff --git a/packages/sqlite-js-driver/src/util/MultiConnectionPool.ts b/packages/sqlite-js-driver/src/util/MultiConnectionPool.ts
diff --git a/packages/sqlite-js-driver/src/util/ReadWriteConnectionPool.ts b/packages/sqlite-js-driver/src/util/ReadWriteConnectionPool.ts
diff --git a/packages/sqlite-js-driver/src/util/SingleConnectionPool.ts b/packages/sqlite-js-driver/src/util/SingleConnectionPool.ts
diff --git a/packages/sqlite-js-driver/src/util/connection-pools.ts b/packages/sqlite-js-driver/src/util/connection-pools.ts
diff --git a/packages/sqlite-js-driver/src/util/index.ts b/packages/sqlite-js-driver/src/util/index.ts
diff --git a/packages/sqlite-js-driver/src/worker_threads/worker-driver.ts b/packages/sqlite-js-driver/src/worker_threads/worker-driver.ts

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,7 @@`
`1`	`1`	`{`
`2`	`2`	`"scripts": {`
`3`	`3`	`"build": "pnpm run -r build",`
	`4`	`+ "clean": "pnpm run -r clean",`
`4`	`5`	`"test": "CI=1 pnpm run -r test",`
`5`	`6`	`"format": "prettier . --write"`
`6`	`7`	`},`
Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@ export function betterSqliteAsyncPool(`
`11`	`11`	`return new ReadWriteConnectionPool({`
`12`	`12`	`async openConnection(options) {`
`13`	`13`	`return new WorkerDriverConnection(`
`14`		`- new URL('./better-sqlite3-worker.js', import.meta.url),`
	`14`	`+ new URL('./worker.js', import.meta.url),`
`15`	`15`	`path,`
`16`	`16`	`{`
`17`	`17`	`...poolOptions,`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-import { betterSqliteAsyncPool } from '../../lib/better-sqlite3-async-driver.js';`
	`1`	`+import { betterSqliteAsyncPool } from '../../lib/worker-driver.js';`
`2`	`2`
`3`	`3`	`import { describeDriverTests } from '@powersync/sqlite-js-driver-tests';`
`4`	`4`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-import { betterSqlitePool } from '../../lib/better-sqlite3-driver.js';`
	`1`	`+import { betterSqlitePool } from '../../lib/sync-driver.js';`
`2`	`2`	`import { describeDriverTests } from '@powersync/sqlite-js-driver-tests';`
`3`	`3`
`4`	`4`	`describeDriverTests(`