From 060f8152208f75bff691266f65e0d294f3491d77 Mon Sep 17 00:00:00 2001 From: Roy Hashimoto Date: Sat, 17 Apr 2021 13:16:46 -0700 Subject: [PATCH 1/4] Replace sqlite3_open with sqlite3_open_v2 --- package-lock.json | 2 +- src/api.js | 23 ++++++++++++++++++++--- src/exported_functions.json | 2 +- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 346d48ee..7b06628a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "sql.js", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/src/api.js b/src/api.js index 52bbab58..ba79313a 100644 --- a/src/api.js +++ b/src/api.js @@ -71,8 +71,15 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { var SQLITE_BLOB = 4; // var - Encodings, used for registering functions. var SQLITE_UTF8 = 1; + // var - Open flags + var SQLITE_OPEN_READWRITE = 0x00000002; + var SQLITE_OPEN_CREATE = 0x00000004; // var - cwrap function - var sqlite3_open = cwrap("sqlite3_open", "number", ["string", "number"]); + var sqlite3_open_v2 = cwrap( + "sqlite3_open_v2", + "number", + ["string", "number", "number", "string"] + ); var sqlite3_close_v2 = cwrap("sqlite3_close_v2", "number", ["number"]); var sqlite3_exec = cwrap( "sqlite3_exec", @@ -790,7 +797,12 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { if (data != null) { FS.createDataFile("/", this.filename, data, true, true); } - this.handleError(sqlite3_open(this.filename, apiTemp)); + this.handleError(sqlite3_open_v2( + this.filename, + apiTemp, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + "unix" + )); this.db = getValue(apiTemp, "i32"); registerExtensionFunctions(this.db); // A list of all prepared statements of the database @@ -1046,7 +1058,12 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { this.functions = {}; this.handleError(sqlite3_close_v2(this.db)); var binaryDb = FS.readFile(this.filename, { encoding: "binary" }); - this.handleError(sqlite3_open(this.filename, apiTemp)); + this.handleError(sqlite3_open_v2( + this.filename, + apiTemp, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + "unix" + )); this.db = getValue(apiTemp, "i32"); return binaryDb; }; diff --git a/src/exported_functions.json b/src/exported_functions.json index b93b07d2..29ce8d1e 100644 --- a/src/exported_functions.json +++ b/src/exported_functions.json @@ -1,7 +1,7 @@ [ "_malloc", "_free", -"_sqlite3_open", +"_sqlite3_open_v2", "_sqlite3_exec", "_sqlite3_free", "_sqlite3_errmsg", From 5652766f1ec7b2e6c6959a027c5ae67047598953 Mon Sep 17 00:00:00 2001 From: Roy Hashimoto Date: Sat, 17 Apr 2021 16:46:39 -0700 Subject: [PATCH 2/4] Add options argument to Database constructor --- .eslintrc.js | 4 ++ src/api.js | 72 ++++++++++++++++++++++++++++------- test/test_database_options.js | 53 ++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 14 deletions(-) create mode 100644 test/test_database_options.js diff --git a/.eslintrc.js b/.eslintrc.js index f730a261..22ef5812 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -54,9 +54,13 @@ module.exports = { // reason - parserOptions is set to es5 language-syntax "prefer-destructuring": "off", // reason - parserOptions is set to es5 language-syntax + "prefer-object-spread": "off", + // reason - parserOptions is set to es5 language-syntax "prefer-spread": "off", // reason - parserOptions is set to es5 language-syntax "prefer-template": "off", + // reason - Closure compiler renames unquoted properties + "quote-props": "off", // reason - sql.js frequently use sql-query-strings containing // single-quotes quotes: ["error", "double"], diff --git a/src/api.js b/src/api.js index ba79313a..1a196059 100644 --- a/src/api.js +++ b/src/api.js @@ -783,6 +783,19 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { }; } + /** + * @typedef {Object} Database.Options + * @property {string} [filename] Database filename. Can be ":memory:" + * to bypass the filesystem but "export" method should not be used. + * If unset, a random name will be generated. + * @property {number} [flags] Any combination of SQLite open flags. See + * https://www.sqlite.org/c3ref/c_open_autoproxy.html (default: + * SQLITE3_OPEN_CREATE | SQLITE3_OPEN_READWRITE). + * @property {string} [vfs] VFS name. If specified, an initialization + * array cannot be provided to the constructor and the "export" method + * should not be used. + */ + /** @classdesc * Represents an SQLite database * @constructs Database @@ -791,17 +804,33 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { * one stored in the byte array passed in first argument * @param {number[]} data An array of bytes representing * an SQLite database file + * @param {Database.Options=} options */ - function Database(data) { - this.filename = "dbfile_" + (0xffffffff * Math.random() >>> 0); - if (data != null) { - FS.createDataFile("/", this.filename, data, true, true); + function Database(data, options) { + this.options = Object.assign( + { + "filename": "dbfile_" + (0xffffffff * Math.random() >>> 0), + "flags": SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, + "vfs": undefined + }, + options + ); + if (data) { + if (this.options["vfs"]) { + throw new Error( + "Loading data is not supported if a VFS is specified" + ); + } + FS.createDataFile( + "/", + this.options["filename"], data, true, true + ); } this.handleError(sqlite3_open_v2( - this.filename, + this.options["filename"], apiTemp, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, - "unix" + this.options["flags"], + this.options["vfs"] || "unix" )); this.db = getValue(apiTemp, "i32"); registerExtensionFunctions(this.db); @@ -1047,22 +1076,35 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { return new StatementIterator(sql, this); }; - /** Exports the contents of the database to a binary array - @return {Uint8Array} An array of bytes of the SQLite3 database file + /** Exports the contents of the database to a binary array. + * Throws an error if the database filename is ":memory:" or + * a VFS is specified. + * @return {Uint8Array} An array of bytes of the SQLite3 database file */ Database.prototype["export"] = function exportDatabase() { + if (this.options["filename"] === ":memory:") { + throw new Error("Exporting a \":memory:\" DB is not supported"); + } + if (this.options["vfs"]) { + throw new Error( + "Exporting is not supported if a VFS is specified" + ); + } Object.values(this.statements).forEach(function each(stmt) { stmt["free"](); }); Object.values(this.functions).forEach(removeFunction); this.functions = {}; this.handleError(sqlite3_close_v2(this.db)); - var binaryDb = FS.readFile(this.filename, { encoding: "binary" }); + var binaryDb = FS.readFile( + this.options["filename"], + { encoding: "binary" } + ); this.handleError(sqlite3_open_v2( - this.filename, + this.options["filename"], apiTemp, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, - "unix" + this.options["flags"], + this.options["vfs"] || "unix" )); this.db = getValue(apiTemp, "i32"); return binaryDb; @@ -1089,7 +1131,9 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { Object.values(this.functions).forEach(removeFunction); this.functions = {}; this.handleError(sqlite3_close_v2(this.db)); - FS.unlink("/" + this.filename); + if (this.options["filename"] !== ":memory:" && !this.options["vfs"]) { + FS.unlink("/" + this.options["filename"]); + } this.db = null; }; diff --git a/test/test_database_options.js b/test/test_database_options.js new file mode 100644 index 00000000..0b756b34 --- /dev/null +++ b/test/test_database_options.js @@ -0,0 +1,53 @@ +exports.test = async function(sql, assert) { + async function doVFS(vfs) { + var db = new sql.Database(null, { vfs }); + db.run("CREATE TABLE tbl (x, y);"); + db.run("INSERT INTO tbl VALUES ('foo', 0), ('bar', 1);"); + return db.exec("SELECT * FROM tbl;"); + } + + const vfsNone = await doVFS(); + assert.ok(vfsNone, 'no VFS executes SQL'); + + const vfsValid = await doVFS("unix-none") + assert.ok(vfsValid, 'valid VFS executes SQL'); + + const vfsInvalid = await doVFS("not a vfs").catch(e => e); + assert.ok(vfsInvalid instanceof Error, 'invalid VFS throws'); + + async function doExport(filename) { + // Specify database name + var db = new sql.Database(null, { filename }); + db.run("CREATE TABLE tbl (x, y);"); + db.run("INSERT INTO tbl VALUES ('foo', 0), ('bar', 1);"); + + // Check that export works. + var binaryArray = db.export(); + db.close(); + return binaryArray; + } + + const exportFile = await doExport('foo'); + assert.strictEqual( + String.fromCharCode.apply(null, exportFile.subarray(0,6)), 'SQLite', + 'export custom filename valid'); + + const exportMem = await doExport(':memory:').catch(e => e); + assert.strictEqual(exportMem instanceof Error, true, 'export :memory: throws'); +}; + +if (module == require.main) { + const target_file = process.argv[2]; + const sql_loader = require('./load_sql_lib'); + sql_loader(target_file).then((sql)=>{ + require('test').run({ + 'test database options': function(assert){ + return exports.test(sql, assert); + } + }); + }) + .catch((e)=>{ + console.error(e); + assert.fail(e); + }); +} From cec3733b6cbb42759bd0835861d47fe544a5a218 Mon Sep 17 00:00:00 2001 From: Roy Hashimoto Date: Sat, 17 Apr 2021 16:58:51 -0700 Subject: [PATCH 3/4] Clean up test_database_options resources. --- test/test_database_options.js | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/test/test_database_options.js b/test/test_database_options.js index 0b756b34..e1769fb8 100644 --- a/test/test_database_options.js +++ b/test/test_database_options.js @@ -1,9 +1,14 @@ exports.test = async function(sql, assert) { + // Use database with a specified VFS. async function doVFS(vfs) { var db = new sql.Database(null, { vfs }); - db.run("CREATE TABLE tbl (x, y);"); - db.run("INSERT INTO tbl VALUES ('foo', 0), ('bar', 1);"); - return db.exec("SELECT * FROM tbl;"); + try { + db.run("CREATE TABLE tbl (x, y);"); + db.run("INSERT INTO tbl VALUES ('foo', 0), ('bar', 1);"); + return db.exec("SELECT * FROM tbl;"); + } finally { + db.close(); + } } const vfsNone = await doVFS(); @@ -15,16 +20,16 @@ exports.test = async function(sql, assert) { const vfsInvalid = await doVFS("not a vfs").catch(e => e); assert.ok(vfsInvalid instanceof Error, 'invalid VFS throws'); + // Attempt export of database with specified filename. async function doExport(filename) { - // Specify database name var db = new sql.Database(null, { filename }); - db.run("CREATE TABLE tbl (x, y);"); - db.run("INSERT INTO tbl VALUES ('foo', 0), ('bar', 1);"); - - // Check that export works. - var binaryArray = db.export(); - db.close(); - return binaryArray; + try { + db.run("CREATE TABLE tbl (x, y);"); + db.run("INSERT INTO tbl VALUES ('foo', 0), ('bar', 1);"); + return db.export(); + } finally { + db.close(); + } } const exportFile = await doExport('foo'); From 0179590426edc2cfbe761972e78a3c7c37eb4682 Mon Sep 17 00:00:00 2001 From: Roy Hashimoto Date: Sun, 18 Apr 2021 11:37:03 -0700 Subject: [PATCH 4/4] Implement as Database.with_vfs. --- src/api.js | 100 +++++++++++++++------------------- test/test_database_options.js | 58 -------------------- test/test_with_vfs.js | 58 ++++++++++++++++++++ 3 files changed, 101 insertions(+), 115 deletions(-) delete mode 100644 test/test_database_options.js create mode 100644 test/test_with_vfs.js diff --git a/src/api.js b/src/api.js index 1a196059..1fbdeab4 100644 --- a/src/api.js +++ b/src/api.js @@ -783,19 +783,6 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { }; } - /** - * @typedef {Object} Database.Options - * @property {string} [filename] Database filename. Can be ":memory:" - * to bypass the filesystem but "export" method should not be used. - * If unset, a random name will be generated. - * @property {number} [flags] Any combination of SQLite open flags. See - * https://www.sqlite.org/c3ref/c_open_autoproxy.html (default: - * SQLITE3_OPEN_CREATE | SQLITE3_OPEN_READWRITE). - * @property {string} [vfs] VFS name. If specified, an initialization - * array cannot be provided to the constructor and the "export" method - * should not be used. - */ - /** @classdesc * Represents an SQLite database * @constructs Database @@ -804,33 +791,17 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { * one stored in the byte array passed in first argument * @param {number[]} data An array of bytes representing * an SQLite database file - * @param {Database.Options=} options */ - function Database(data, options) { - this.options = Object.assign( - { - "filename": "dbfile_" + (0xffffffff * Math.random() >>> 0), - "flags": SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, - "vfs": undefined - }, - options - ); - if (data) { - if (this.options["vfs"]) { - throw new Error( - "Loading data is not supported if a VFS is specified" - ); - } - FS.createDataFile( - "/", - this.options["filename"], data, true, true - ); + function Database(data) { + this.filename = "dbfile_" + (0xffffffff * Math.random() >>> 0); + if (data != null) { + FS.createDataFile("/", this.filename, data, true, true); } this.handleError(sqlite3_open_v2( - this.options["filename"], + this.filename, apiTemp, - this.options["flags"], - this.options["vfs"] || "unix" + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + "unix" )); this.db = getValue(apiTemp, "i32"); registerExtensionFunctions(this.db); @@ -1076,19 +1047,12 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { return new StatementIterator(sql, this); }; - /** Exports the contents of the database to a binary array. - * Throws an error if the database filename is ":memory:" or - * a VFS is specified. - * @return {Uint8Array} An array of bytes of the SQLite3 database file + /** Exports the contents of the database to a binary array + @return {Uint8Array} An array of bytes of the SQLite3 database file */ Database.prototype["export"] = function exportDatabase() { - if (this.options["filename"] === ":memory:") { - throw new Error("Exporting a \":memory:\" DB is not supported"); - } - if (this.options["vfs"]) { - throw new Error( - "Exporting is not supported if a VFS is specified" - ); + if (this.filename === ":memory:") { + throw new Error("export on ':memory:' database is not supported"); } Object.values(this.statements).forEach(function each(stmt) { stmt["free"](); @@ -1096,15 +1060,12 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { Object.values(this.functions).forEach(removeFunction); this.functions = {}; this.handleError(sqlite3_close_v2(this.db)); - var binaryDb = FS.readFile( - this.options["filename"], - { encoding: "binary" } - ); + var binaryDb = FS.readFile(this.filename, { encoding: "binary" }); this.handleError(sqlite3_open_v2( - this.options["filename"], + this.filename, apiTemp, - this.options["flags"], - this.options["vfs"] || "unix" + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + "unix" )); this.db = getValue(apiTemp, "i32"); return binaryDb; @@ -1131,9 +1092,7 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { Object.values(this.functions).forEach(removeFunction); this.functions = {}; this.handleError(sqlite3_close_v2(this.db)); - if (this.options["filename"] !== ":memory:" && !this.options["vfs"]) { - FS.unlink("/" + this.options["filename"]); - } + FS.unlink("/" + this.filename); this.db = null; }; @@ -1259,6 +1218,33 @@ Module["onRuntimeInitialized"] = function onRuntimeInitialized() { return this; }; + /** + * Open a database accessed with a specified VFS. + * @param {string} vfs Name of a registered VFS + * @param {string} filename Database filename + * @param {number} [flags] Open flags for sqlite3_open_v2, described at + * https://www.sqlite.org/c3ref/c_open_autoproxy.html + * @returns {Database} + */ + Database["with_vfs"] = function with_vfs(vfs, filename, flags) { + var instance = Object.create(Database.prototype); + instance.filename = filename; + instance.vfs = vfs; + instance.statements = {}; + instance.functions = {}; + + instance.handleError(sqlite3_open_v2( + instance.filename, + apiTemp, + flags || (SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE), + vfs + )); + instance.db = getValue(apiTemp, "i32"); + registerExtensionFunctions(instance.db); + + return instance; + }; + // export Database to Module Module.Database = Database; }; diff --git a/test/test_database_options.js b/test/test_database_options.js deleted file mode 100644 index e1769fb8..00000000 --- a/test/test_database_options.js +++ /dev/null @@ -1,58 +0,0 @@ -exports.test = async function(sql, assert) { - // Use database with a specified VFS. - async function doVFS(vfs) { - var db = new sql.Database(null, { vfs }); - try { - db.run("CREATE TABLE tbl (x, y);"); - db.run("INSERT INTO tbl VALUES ('foo', 0), ('bar', 1);"); - return db.exec("SELECT * FROM tbl;"); - } finally { - db.close(); - } - } - - const vfsNone = await doVFS(); - assert.ok(vfsNone, 'no VFS executes SQL'); - - const vfsValid = await doVFS("unix-none") - assert.ok(vfsValid, 'valid VFS executes SQL'); - - const vfsInvalid = await doVFS("not a vfs").catch(e => e); - assert.ok(vfsInvalid instanceof Error, 'invalid VFS throws'); - - // Attempt export of database with specified filename. - async function doExport(filename) { - var db = new sql.Database(null, { filename }); - try { - db.run("CREATE TABLE tbl (x, y);"); - db.run("INSERT INTO tbl VALUES ('foo', 0), ('bar', 1);"); - return db.export(); - } finally { - db.close(); - } - } - - const exportFile = await doExport('foo'); - assert.strictEqual( - String.fromCharCode.apply(null, exportFile.subarray(0,6)), 'SQLite', - 'export custom filename valid'); - - const exportMem = await doExport(':memory:').catch(e => e); - assert.strictEqual(exportMem instanceof Error, true, 'export :memory: throws'); -}; - -if (module == require.main) { - const target_file = process.argv[2]; - const sql_loader = require('./load_sql_lib'); - sql_loader(target_file).then((sql)=>{ - require('test').run({ - 'test database options': function(assert){ - return exports.test(sql, assert); - } - }); - }) - .catch((e)=>{ - console.error(e); - assert.fail(e); - }); -} diff --git a/test/test_with_vfs.js b/test/test_with_vfs.js new file mode 100644 index 00000000..cc51f4a9 --- /dev/null +++ b/test/test_with_vfs.js @@ -0,0 +1,58 @@ +exports.test = async function(sql, assert) { + // Use database with a specified VFS. + async function doVFS(vfs, filename) { + const db = sql.Database.with_vfs(vfs, filename); + try { + db.run("CREATE TABLE tbl (x, y);"); + db.run("INSERT INTO tbl VALUES ('foo', 0), ('bar', 1), ('baz', 2);"); + return db.exec("SELECT SUM(y) FROM tbl;"); + } finally { + db.close(); + } + } + + let result; + result = await doVFS("unix", "db"); + assert.strictEqual(result[0].values[0][0], 3, 'built-in unix VFS executes SQL'); + + result = await doVFS("unix-none", "db"); + assert.strictEqual(result[0].values[0][0], 3, 'built-in unix-none VFS executes SQL'); + + result = await doVFS("unregistered", "db").catch(e => e); + assert.ok(result instanceof Error, 'unregistered VFS throws'); + + // Export database with a specified filename. + async function doExport(vfs, filename) { + const db = sql.Database.with_vfs(vfs, filename); + try { + db.run("CREATE TABLE tbl (x, y);"); + db.run("INSERT INTO tbl VALUES ('foo', 0), ('bar', 1);"); + const exported = db.export(); + return String.fromCharCode.apply(null, exported.subarray(0,6)); + } finally { + db.close(); + } + } + + result = await doExport("unix", "foo"); + assert.strictEqual(result, "SQLite", 'built-in VFS exports'); + + result = await doExport("unix", ":memory:").catch(e => e); + assert.ok(result instanceof Error, ':memory: throws on export'); +}; + +if (module == require.main) { + const target_file = process.argv[2]; + const sql_loader = require('./load_sql_lib'); + sql_loader(target_file).then((sql)=>{ + require('test').run({ + 'test with_vfs': function(assert){ + return exports.test(sql, assert); + } + }); + }) + .catch((e)=>{ + console.error(e); + assert.fail(e); + }); +}