From 04da9854b3fd61dbd293cc0ec91ee983cb763790 Mon Sep 17 00:00:00 2001 From: James Long Date: Tue, 20 Jul 2021 22:38:17 -0400 Subject: [PATCH 1/5] Implement changes required for IndexedDB-backed filesystem --- Makefile | 17 ++++++---- src/api.js | 56 ++++++++++++++++++++++++++++--- src/exported_functions.json | 2 ++ src/exported_runtime_methods.json | 3 +- src/fs-externs.js | 48 ++++++++++++++++++++++++++ src/vfs.c | 49 +++++++++++++++++++++++++++ 6 files changed, 163 insertions(+), 12 deletions(-) create mode 100644 src/fs-externs.js create mode 100644 src/vfs.c diff --git a/Makefile b/Makefile index a4ce5ebc..326e8d23 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ EMFLAGS_DEBUG = \ -s ASSERTIONS=1 \ -O1 -BITCODE_FILES = out/sqlite3.bc out/extension-functions.bc +BITCODE_FILES = out/sqlite3.bc out/extension-functions.bc out/vfs.bc OUTPUT_WRAPPER_FILES = src/shell-pre.js src/shell-post.js @@ -79,13 +79,13 @@ all: optimized debug worker debug: dist/sql-asm-debug.js dist/sql-wasm-debug.js dist/sql-asm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) - $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ + EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ mv $@ out/tmp-raw.js cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ rm out/tmp-raw.js dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) - $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ + EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ mv $@ out/tmp-raw.js cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ rm out/tmp-raw.js @@ -94,19 +94,19 @@ dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FI optimized: dist/sql-asm.js dist/sql-wasm.js dist/sql-asm-memory-growth.js dist/sql-asm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) - $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ + EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ mv $@ out/tmp-raw.js cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ rm out/tmp-raw.js dist/sql-wasm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) - $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ + EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ mv $@ out/tmp-raw.js cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ rm out/tmp-raw.js dist/sql-asm-memory-growth.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) - $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM_MEMORY_GROWTH) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ + EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM_MEMORY_GROWTH) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ mv $@ out/tmp-raw.js cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ rm out/tmp-raw.js @@ -156,6 +156,11 @@ out/extension-functions.bc: sqlite-src/$(SQLITE_AMALGAMATION) # Generate llvm bitcode $(EMCC) $(CFLAGS) -c sqlite-src/$(SQLITE_AMALGAMATION)/extension-functions.c -o $@ +out/vfs.bc: src/vfs.c sqlite-src/$(SQLITE_AMALGAMATION) + mkdir -p out + # Generate llvm bitcode + $(EMCC) $(CFLAGS) -s LINKABLE=1 -I sqlite-src/$(SQLITE_AMALGAMATION) -c src/vfs.c -o $@ + # TODO: This target appears to be unused. If we re-instatate it, we'll need to add more files inside of the JS folder # module.tar.gz: test package.json AUTHORS README.md dist/sql-asm.js # tar --create --gzip $^ > $@ diff --git a/src/api.js b/src/api.js index 854bf32e..88fd91a4 100644 --- a/src/api.js +++ b/src/api.js @@ -811,13 +811,19 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { * @memberof module:SqlJs * Open a new database either by creating a new one or opening an existing * one stored in the byte array passed in first argument - * @param {number[]} data An array of bytes representing - * an SQLite database file + * @param {number[]|string} data An array of bytes representing + * an SQLite database file or a path + * @param {Object} opts Options to specify a filename */ - function Database(data) { - this.filename = "dbfile_" + (0xffffffff * Math.random() >>> 0); - if (data != null) { + function Database(data, { filename = false } = {}) { + if(filename === false) { + this.filename = "dbfile_" + (0xffffffff * Math.random() >>> 0); + if (data != null) { FS.createDataFile("/", this.filename, data, true, true); + } + } + else { + this.filename = data; } this.handleError(sqlite3_open(this.filename, apiTemp)); this.db = getValue(apiTemp, "i32"); @@ -1231,4 +1237,44 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { // export Database to Module Module.Database = Database; + + // Because emscripten doesn't allow us to handle `ioctl`, we need + // to manually install lock/unlock methods. Unfortunately we need + // to keep track of a mapping of `sqlite_file*` pointers to filename + // so that we can tell our filesystem which files to lock/unlock + var sqliteFiles = new Map(); + + Module["register_for_idb"] = (customFS) => { + var SQLITE_BUSY = 5; + + function open(namePtr, file) { + var path = UTF8ToString(namePtr); + sqliteFiles.set(file, path); + } + + function lock(file, lockType) { + var path = sqliteFiles.get(file); + var success = customFS.lock(path, lockType) + return success? 0 : SQLITE_BUSY; + } + + function unlock(file,lockType) { + var path = sqliteFiles.get(file); + customFS.unlock(path, lockType) + return 0; + } + + let lockPtr = addFunction(lock, 'iii'); + let unlockPtr = addFunction(unlock, 'iii'); + let openPtr = addFunction(open, 'vii'); + Module["_register_for_idb"](lockPtr, unlockPtr, openPtr) + } + + // TODO: This isn't called from anywhere yet. We need to + // somehow cleanup closed files from `sqliteFiles` + Module["cleanup_file"] = (path) => { + let filesInfo = [...sqliteFiles.entries()] + let fileInfo = filesInfo.find(f => f[1] === path); + sqliteFiles.delete(fileInfo[0]) + } }; diff --git a/src/exported_functions.json b/src/exported_functions.json index b93b07d2..c792b657 100644 --- a/src/exported_functions.json +++ b/src/exported_functions.json @@ -41,5 +41,7 @@ "_sqlite3_result_int", "_sqlite3_result_int64", "_sqlite3_result_error", +"_sqlite3_vfs_find", +"_register_for_idb", "_RegisterExtensionFunctions" ] diff --git a/src/exported_runtime_methods.json b/src/exported_runtime_methods.json index 13a8efb8..e8915a6f 100644 --- a/src/exported_runtime_methods.json +++ b/src/exported_runtime_methods.json @@ -3,5 +3,6 @@ "stackAlloc", "stackSave", "stackRestore", -"UTF8ToString" +"UTF8ToString", +"FS" ] diff --git a/src/fs-externs.js b/src/fs-externs.js new file mode 100644 index 00000000..a82e2463 --- /dev/null +++ b/src/fs-externs.js @@ -0,0 +1,48 @@ +/** + * @externs + */ + +Module.FS = class { + constructor() { + this.ErrnoError = class {}; + } + mount() {} + isRoot() {} + isFile() {} + isDir() {} + stat() {} + /** @return {FSNode} */ + lookupPath() {} + /** @return {FSNode} */ + lookupNode() {} + /** @return {FSNode} */ + createNode() {} + /** @return {FSNode} */ + mknod() {} +}; + +Module.FS.FSNode = class { + constructor() { + this.node_ops = { + getattr: () => {}, + setattr: () => {}, + lookup: () => {}, + mknod: () => {}, + rename: () => {}, + unlink: () => {}, + rmdir: () => {}, + reaaddir: () => {}, + symlink: () => {}, + readlink: () => {} + }; + + this.stream_ops = { + llseek: () => {}, + read: () => {}, + write: () => {}, + allocate: () => {}, + mmap: () => {}, + msync: () => {} + }; + } +}; diff --git a/src/vfs.c b/src/vfs.c new file mode 100644 index 00000000..30b19ca7 --- /dev/null +++ b/src/vfs.c @@ -0,0 +1,49 @@ +#include +#include + +static int (*defaultOpen)(sqlite3_vfs *vfs, const char *zName, sqlite3_file *file, int flags, int *pOutFlags); + +static void (*fsOpen)(const char *, void*); +static int (*fsLock)(sqlite3_file *file, int); +static int (*fsUnlock)(sqlite3_file *file, int); + +static int blockDeviceCharacteristics(sqlite3_file* file) { + return SQLITE_IOCAP_SAFE_APPEND | + SQLITE_IOCAP_SEQUENTIAL | + SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; +} + +static int block_lock(sqlite3_file *file, int lock) { + return fsLock(file, lock); +} + +static int block_unlock(sqlite3_file *file, int lock) { + return fsUnlock(file, lock); +} + +static int block_open(sqlite3_vfs *vfs, const char *zName, sqlite3_file *file, int flags, int *pOutFlags) { + int res = defaultOpen(vfs, zName, file, flags, pOutFlags); + + printf("Opened!\n"); + + sqlite3_io_methods* methods = (sqlite3_io_methods*)file->pMethods; + methods->xDeviceCharacteristics = blockDeviceCharacteristics; + methods->xLock = block_lock; + methods->xUnlock = block_unlock; + + fsOpen(zName, (void*)file); + + return res; +} + +void register_for_idb(int(*lockFile)(sqlite3_file*,int), int(*unlockFile)(sqlite3_file*,int), void(*openFile)(const char*, void*)) { + sqlite3_vfs *vfs = sqlite3_vfs_find("unix"); + defaultOpen = vfs->xOpen; + + vfs->xOpen = block_open; + sqlite3_vfs_register(vfs, 1); + + fsLock = lockFile; + fsUnlock = unlockFile; + fsOpen = openFile; +} From 1acb3f4d1135f282ab0349a2393b5921f0389f8f Mon Sep 17 00:00:00 2001 From: James Long Date: Mon, 2 Aug 2021 10:40:39 -0400 Subject: [PATCH 2/5] Improved exports for testing --- src/fs-externs.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/fs-externs.js b/src/fs-externs.js index a82e2463..c2e1a329 100644 --- a/src/fs-externs.js +++ b/src/fs-externs.js @@ -5,8 +5,10 @@ Module.FS = class { constructor() { this.ErrnoError = class {}; + this.filesystems = {} } mount() {} + unmount() {} isRoot() {} isFile() {} isDir() {} From ebc92a09b1ab8a3fee84d83b893a8de98b55c544 Mon Sep 17 00:00:00 2001 From: James Long Date: Mon, 2 Aug 2021 10:51:12 -0400 Subject: [PATCH 3/5] Add reset_filesystem --- src/api.js | 5 +++++ src/fs-externs.js | 2 -- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/api.js b/src/api.js index 88fd91a4..4f8641db 100644 --- a/src/api.js +++ b/src/api.js @@ -1277,4 +1277,9 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { let fileInfo = filesInfo.find(f => f[1] === path); sqliteFiles.delete(fileInfo[0]) } + + Module["reset_filesystem"] = () => { + FS.root = null; + FS.staticInit(); + } }; diff --git a/src/fs-externs.js b/src/fs-externs.js index c2e1a329..a82e2463 100644 --- a/src/fs-externs.js +++ b/src/fs-externs.js @@ -5,10 +5,8 @@ Module.FS = class { constructor() { this.ErrnoError = class {}; - this.filesystems = {} } mount() {} - unmount() {} isRoot() {} isFile() {} isDir() {} From c1aea3e81c006ebab5b6ad7f9f2c09b398833cce Mon Sep 17 00:00:00 2001 From: James Long Date: Fri, 6 Aug 2021 09:25:58 -0400 Subject: [PATCH 4/5] Don't delete file on close --- src/api.js | 6 +++++- src/vfs.c | 2 -- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/api.js b/src/api.js index 4f8641db..d2a20dfb 100644 --- a/src/api.js +++ b/src/api.js @@ -818,6 +818,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { function Database(data, { filename = false } = {}) { if(filename === false) { this.filename = "dbfile_" + (0xffffffff * Math.random() >>> 0); + this.memoryFile = true; if (data != null) { FS.createDataFile("/", this.filename, data, true, true); } @@ -1109,7 +1110,10 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { Object.values(this.functions).forEach(removeFunction); this.functions = {}; this.handleError(sqlite3_close_v2(this.db)); - FS.unlink("/" + this.filename); + + if(this.memoryFile) { + FS.unlink("/" + this.filename); + } this.db = null; }; diff --git a/src/vfs.c b/src/vfs.c index 30b19ca7..412bad87 100644 --- a/src/vfs.c +++ b/src/vfs.c @@ -24,8 +24,6 @@ static int block_unlock(sqlite3_file *file, int lock) { static int block_open(sqlite3_vfs *vfs, const char *zName, sqlite3_file *file, int flags, int *pOutFlags) { int res = defaultOpen(vfs, zName, file, flags, pOutFlags); - printf("Opened!\n"); - sqlite3_io_methods* methods = (sqlite3_io_methods*)file->pMethods; methods->xDeviceCharacteristics = blockDeviceCharacteristics; methods->xLock = block_lock; From 54ec38785e3057a1fb85aa7008ff0a58c686cc1e Mon Sep 17 00:00:00 2001 From: Sergey Popov Date: Sat, 28 Aug 2021 17:49:46 +0300 Subject: [PATCH 5/5] Fix makefile fs-externs path --- Makefile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 326e8d23..a70c27c4 100644 --- a/Makefile +++ b/Makefile @@ -73,19 +73,21 @@ EMFLAGS_PRE_JS_FILES = \ EXPORTED_METHODS_JSON_FILES = src/exported_functions.json src/exported_runtime_methods.json +FS_EXTERN_PATH = "$(realpath -s ./src/fs-externs.js)" + all: optimized debug worker .PHONY: debug debug: dist/sql-asm-debug.js dist/sql-wasm-debug.js dist/sql-asm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) - EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ + EMCC_CLOSURE_ARGS="--externs ${FS_EXTERN_PATH}" $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ mv $@ out/tmp-raw.js cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ rm out/tmp-raw.js dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) - EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ + EMCC_CLOSURE_ARGS="--externs ${FS_EXTERN_PATH}" $(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ mv $@ out/tmp-raw.js cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ rm out/tmp-raw.js @@ -94,19 +96,19 @@ dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FI optimized: dist/sql-asm.js dist/sql-wasm.js dist/sql-asm-memory-growth.js dist/sql-asm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) - EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ + EMCC_CLOSURE_ARGS="--externs ${FS_EXTERN_PATH}" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ mv $@ out/tmp-raw.js cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ rm out/tmp-raw.js dist/sql-wasm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) - EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ + EMCC_CLOSURE_ARGS="--externs ${FS_EXTERN_PATH}" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ mv $@ out/tmp-raw.js cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ rm out/tmp-raw.js dist/sql-asm-memory-growth.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES) - EMCC_CLOSURE_ARGS="--externs src/fs-externs.js" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM_MEMORY_GROWTH) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ + EMCC_CLOSURE_ARGS="--externs ${FS_EXTERN_PATH}" $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM_MEMORY_GROWTH) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@ mv $@ out/tmp-raw.js cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@ rm out/tmp-raw.js