Tidyup

packages/pglite/src/extensionUtils.ts

@@ -1,6 +1,13 @@
 import { IN_NODE } from "./utils.js";
+import type { PostgresMod } from "./postgres.js";
+import type { PGliteInterface } from "./interface.js";
 
-export async function loadExtensionBundle(bundlePath: URL): Promise<Blob> {
+// @ts-ignore - tinytar has no types, we will probably replace this with another library
+import tinyTar from "tinytar";
+
+export async function loadExtensionBundle(
+  bundlePath: URL,
+): Promise<Blob | null> {
   // Async load the extension bundle tar file
   // could be from a URL or a file
   if (IN_NODE) {
@@ -24,7 +31,7 @@ export async function loadExtensionBundle(bundlePath: URL): Promise<Blob> {
           chunks.push(chunk);
           callback();
         },
-      })
+      }),
     );
     return new Blob(chunks);
   } else {
@@ -34,8 +41,77 @@ export async function loadExtensionBundle(bundlePath: URL): Promise<Blob> {
     }
     const decompressionStream = new DecompressionStream("gzip");
     const decompressedStream = new Response(
-      response.body.pipeThrough(decompressionStream)
+      response.body.pipeThrough(decompressionStream),
     );
     return decompressedStream.blob();
   }
 }
+
+export async function loadExtensions(
+  mod: PostgresMod,
+  log: (...args: any[]) => void,
+) {
+  for (const ext in mod.pg_extensions) {
+    let blob;
+    try {
+      blob = await mod.pg_extensions[ext];
+    } catch (err) {
+      console.error("Failed to fetch extension:", ext, err);
+      continue;
+    }
+    if (blob) {
+      const bytes = new Uint8Array(await blob.arrayBuffer());
+      loadExtension(mod, ext, bytes, log);
+    } else {
+      console.error("Could not get binary data for extension:", ext);
+    }
+  }
+}
+
+interface UntarFile {
+  name: string;
+  data: Uint8Array;
+}
+
+function loadExtension(
+  mod: PostgresMod,
+  ext: string,
+  bytes: Uint8Array,
+  log: (...args: any[]) => void,
+) {
+  const data: UntarFile[] = tinyTar.untar(bytes);
+  data.forEach((file) => {
+    if (!file.name.startsWith(".")) {
+      const filePath = mod.WASM_PREFIX + "/" + file.name;
+      if (file.name.endsWith(".so")) {
+        const extOk = (...args: any[]) => {
+          log("pgfs:ext OK", filePath, args);
+        };
+        const extFail = (...args: any[]) => {
+          log("pgfs:ext FAIL", filePath, args);
+        };
+        mod.FS.createPreloadedFile(
+          dirname(filePath),
+          file.name.split("/").pop()!.slice(0, -3),
+          file.data as any, // There is a type error in Emscripten's FS.createPreloadedFile, this excepts a Uint8Array, but the type is defined as any
+          true,
+          true,
+          extOk,
+          extFail,
+          false,
+        );
+      } else {
+        mod.FS.writeFile(filePath, file.data);
+      }
+    }
+  });
+}
+
+function dirname(path: string) {
+  const last = path.lastIndexOf("/");
+  if (last > 0) {
+    return path.slice(0, last);
+  } else {
+    return path;
+  }
+}

packages/pglite/src/fs/idbfs.ts

@@ -1,10 +1,10 @@
 import { FilesystemBase } from "./types.js";
-import type { FS, EmPostgres } from "../postgres.js";
-import { PGDATA, loadExtensions } from "./index.js";
+import type { FS, PostgresMod } from "../postgres.js";
+import { PGDATA } from "./index.js";
 
 export class IdbFs extends FilesystemBase {
-  async emscriptenOpts(opts: Partial<EmPostgres>) {
-    const options: Partial<EmPostgres> = {
+  async emscriptenOpts(opts: Partial<PostgresMod>) {
+    const options: Partial<PostgresMod> = {
       ...opts,
       preRun: [
         ...(opts.preRun || []),
@@ -31,7 +31,6 @@ export class IdbFs extends FilesystemBase {
         if (err) {
           reject(err);
         } else {
-          loadExtensions("idbfs", fs);
           resolve();
         }
       });

packages/pglite/src/fs/index.ts

@@ -1,11 +1,6 @@
-// @ts-nocheck
-
 import type { FsType, Filesystem } from "./types.js";
-import type { FS } from "../postgres.js";
 import { IdbFs } from "./idbfs.js";
 import { MemoryFS } from "./memoryfs.js";
-import tinyTar from "tinytar";
-
 
 export type * from "./types.js";
 
@@ -37,73 +32,14 @@ export function parseDataDir(dataDir?: string) {
 
 export async function loadFs(dataDir?: string, fsType?: FsType) {
   let fs: Filesystem;
-  if (dataDir && (fsType === "nodefs")) {
+  if (dataDir && fsType === "nodefs") {
     // Lazy load the nodefs to avoid bundling it in the browser
     const { NodeFS } = await import("./nodefs.js");
     fs = new NodeFS(dataDir);
-  } else if (dataDir && (fsType === "idbfs")) {
+  } else if (dataDir && fsType === "idbfs") {
     fs = new IdbFs(dataDir);
   } else {
     fs = new MemoryFS();
   }
   return fs;
 }
-
-function load_pg_extension(Module, ext, bytes) {
-    var data = tinyTar.untar(bytes);
-    data.forEach(function(file) {
-          if (!file.name.startsWith(".")) {
-              const _file = Module.WASM_PREFIX + "/" + file.name;
-              console.log("    + ", _file);
-              if (file.name.endsWith(".so")) {
-                console.warn(_file, "scheduled for wasm streaming compilation");
-                const ext_ok = (...args) => {
-                  console.log("pgfs:ext OK", _file, args);
-                };
-
-                const ext_fail = (...args) => {
-                  console.log("pgfs:ext FAIL", _file, args);
-                };
-
-                function dirname(p) {
-                    const last = p.lastIndexOf("/");
-                    if (last>0)
-                        return p.substr(0,last)
-                    return p;
-                }
-
-                Module.FS.createPreloadedFile(dirname(_file), file.name.split("/").pop().slice(0, -3), file.data, true, true, ext_ok, ext_fail, false);
-                console.log("createPreloadedFile called for :", _file);
-              } else {
-                Module.FS.writeFile(_file, file.data);
-              }
-          }
-    });
-    console.warn("pgfs ext:end", ext);
-}
-
-
-export async function loadExtensions(fsType: FsType, fs: FS) {
-    const Module = fs.Module;
-    console.warn("fs/index.ts: loadExtensions for FS :", fsType);
-
-    for (const ext in Module.pg_extensions) {
-        var blob;
-        try {
-            blob = await Module.pg_extensions[ext]
-        } catch (x) {
-            console.error("failed to fetch extension :", ext)
-            continue
-        }
-        if (blob) {
-            const bytes = new Uint8Array(await blob.arrayBuffer())
-            console.log("  +", ext,"tardata:", bytes.length )
-            if (ext=="quack")
-               console.warn(ext,"skipped !")
-            else
-               load_pg_extension(Module, ext, bytes)
-        } else {
-           console.error("could not get binary data for extension :", ext);
-        }
-    }
-}

packages/pglite/src/fs/memoryfs.ts

@@ -1,8 +1,8 @@
 import { FilesystemBase } from "./types.js";
-import type { EmPostgres } from "../postgres.js";
+import type { PostgresMod } from "../postgres.js";
 
 export class MemoryFS extends FilesystemBase {
-  async emscriptenOpts(opts: Partial<EmPostgres>) {
+  async emscriptenOpts(opts: Partial<PostgresMod>) {
     // Nothing to do for memoryfs
     return opts;
   }

packages/pglite/src/fs/nodefs.ts

@@ -2,7 +2,7 @@ import * as fs from "fs";
 import * as path from "path";
 import { FilesystemBase } from "./types.js";
 import { PGDATA } from "./index.js";
-import type { EmPostgres } from "../postgres.js";
+import type { PostgresMod } from "../postgres.js";
 
 export class NodeFS extends FilesystemBase {
   protected rootDir: string;
@@ -15,8 +15,8 @@ export class NodeFS extends FilesystemBase {
     }
   }
 
-  async emscriptenOpts(opts: Partial<EmPostgres>) {
-    const options: Partial<EmPostgres> = {
+  async emscriptenOpts(opts: Partial<PostgresMod>) {
+    const options: Partial<PostgresMod> = {
       ...opts,
       preRun: [
         ...(opts.preRun || []),

packages/pglite/src/fs/types.ts

@@ -1,4 +1,4 @@
-import type { EmPostgres, FS } from "../postgres.js";
+import type { PostgresMod, FS } from "../postgres.js";
 
 export type FsType = "nodefs" | "idbfs" | "memoryfs";
 
@@ -10,7 +10,7 @@ export interface Filesystem {
   /**
    * Returns the options to pass to the emscripten module.
    */
-  emscriptenOpts(opts: Partial<EmPostgres>): Promise<Partial<EmPostgres>>;
+  emscriptenOpts(opts: Partial<PostgresMod>): Promise<Partial<PostgresMod>>;
 
   /**
    * Sync the filesystem to the emscripten filesystem.
@@ -22,10 +22,8 @@ export interface Filesystem {
    */
   initialSyncFs(mod: FS): Promise<void>;
 
-
-//  on_mount(): Function<void>;
+  //  on_mount(): Function<void>;
   // load_extension(ext: string): Promise<void>;
-
 }
 
 export abstract class FilesystemBase implements Filesystem {
@@ -34,8 +32,8 @@ export abstract class FilesystemBase implements Filesystem {
     this.dataDir = dataDir;
   }
   abstract emscriptenOpts(
-    opts: Partial<EmPostgres>,
-  ): Promise<Partial<EmPostgres>>;
+    opts: Partial<PostgresMod>,
+  ): Promise<Partial<PostgresMod>>;
   async syncToFs(mod: FS) {}
   async initialSyncFs(mod: FS) {}
 }

packages/pglite/src/interface.ts

@@ -31,7 +31,7 @@ export interface ExtensionSetupResult {
 
 export type ExtensionSetup = (
   pg: PGliteInterface,
-  emscriptenOpts: any
+  emscriptenOpts: any,
 ) => Promise<ExtensionSetupResult>;
 
 export interface Extension {
@@ -61,26 +61,26 @@ export type PGliteInterface = {
   query<T>(
     query: string,
     params?: any[],
-    options?: QueryOptions
+    options?: QueryOptions,
   ): Promise<Results<T>>;
   exec(query: string, options?: QueryOptions): Promise<Array<Results>>;
   transaction<T>(
-    callback: (tx: Transaction) => Promise<T>
+    callback: (tx: Transaction) => Promise<T>,
   ): Promise<T | undefined>;
   execProtocol(
     message: Uint8Array,
-    options?: ExecProtocolOptions
+    options?: ExecProtocolOptions,
   ): Promise<Array<[BackendMessage, Uint8Array]>>;
   listen(
     channel: string,
-    callback: (payload: string) => void
+    callback: (payload: string) => void,
   ): Promise<() => Promise<void>>;
   unlisten(
     channel: string,
-    callback?: (payload: string) => void
+    callback?: (payload: string) => void,
   ): Promise<void>;
   onNotification(
-    callback: (channel: string, payload: string) => void
+    callback: (channel: string, payload: string) => void,
   ): () => void;
   offNotification(callback: (channel: string, payload: string) => void): void;
 };
@@ -110,7 +110,7 @@ export interface Transaction {
   query<T>(
     query: string,
     params?: any[],
-    options?: QueryOptions
+    options?: QueryOptions,
   ): Promise<Results<T>>;
   exec(query: string, options?: QueryOptions): Promise<Array<Results>>;
   rollback(): Promise<void>;

packages/pglite/src/parse.ts

@@ -14,7 +14,7 @@ import { parseType } from "./types.js";
 export function parseResults(
   messages: Array<BackendMessage>,
   options?: QueryOptions,
-  blob?: Blob
+  blob?: Blob,
 ): Array<Results> {
   const resultSets: Results[] = [];
   let currentResultSet: Results = { rows: [], fields: [] };
@@ -24,7 +24,7 @@ export function parseResults(
     (msg) =>
       msg instanceof RowDescriptionMessage ||
       msg instanceof DataRowMessage ||
-      msg instanceof CommandCompleteMessage
+      msg instanceof CommandCompleteMessage,
   );
 
   filteredMessages.forEach((msg, index) => {
@@ -40,9 +40,9 @@ export function parseResults(
             parseType(
               field,
               currentResultSet!.fields[i].dataTypeID,
-              options?.parsers
-            )
-          )
+              options?.parsers,
+            ),
+          ),
         );
       } else {
         // rowMode === "object"
@@ -53,10 +53,10 @@ export function parseResults(
               parseType(
                 field,
                 currentResultSet!.fields[i].dataTypeID,
-                options?.parsers
+                options?.parsers,
               ),
-            ])
-          )
+            ]),
+          ),
         );
       }
     } else if (msg instanceof CommandCompleteMessage) {