Merge pull request #83 from Mytherin/busytimeout

Add more options for opening SQLite databases - busy_timeout and journal_mode - and set busy time-out by default

Author: Mytherin

src/include/sqlite_db.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "sqlite_utils.hpp"
+#include "storage/sqlite_options.hpp"
 
 namespace duckdb {
 class SQLiteStatement;
@@ -29,7 +30,7 @@ class SQLiteDB {
 	sqlite3 *db;
 
 public:
-	static SQLiteDB Open(const string &path, bool is_read_only = true, bool is_shared = false);
+	static SQLiteDB Open(const string &path, const SQLiteOpenOptions &options, bool is_shared = false);
 	bool TryPrepare(const string &query, SQLiteStatement &result);
 	SQLiteStatement Prepare(const string &query);
 	void Execute(const string &query);

src/include/storage/sqlite_catalog.hpp

@@ -9,19 +9,19 @@
 #pragma once
 
 #include "duckdb/catalog/catalog.hpp"
-#include "duckdb/common/enums/access_mode.hpp"
+#include "sqlite_options.hpp"
 #include "sqlite_db.hpp"
 
 namespace duckdb {
 class SQLiteSchemaEntry;
 
 class SQLiteCatalog : public Catalog {
 public:
-	explicit SQLiteCatalog(AttachedDatabase &db_p, const string &path, AccessMode access_mode);
+	explicit SQLiteCatalog(AttachedDatabase &db_p, const string &path, SQLiteOpenOptions options);
 	~SQLiteCatalog();
 
 	string path;
-	AccessMode access_mode;
+	SQLiteOpenOptions options;
 
 public:
 	void Initialize(bool load_builtin) override;

src/include/storage/sqlite_options.hpp

@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// storage/sqlite_options.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/enums/access_mode.hpp"
+
+namespace duckdb {
+
+struct SQLiteOpenOptions {
+	// access mode
+	AccessMode access_mode = AccessMode::READ_WRITE;
+	// busy time-out in ms
+	idx_t busy_timeout = 5000;
+	// journal mode
+	string journal_mode;
+};
+
+
+} // namespace duckdb

src/sqlite_db.cpp

@@ -28,10 +28,10 @@ SQLiteDB &SQLiteDB::operator=(SQLiteDB &&other) noexcept {
 	return *this;
 }
 
-SQLiteDB SQLiteDB::Open(const string &path, bool is_read_only, bool is_shared) {
+SQLiteDB SQLiteDB::Open(const string &path, const SQLiteOpenOptions &options, bool is_shared) {
 	SQLiteDB result;
 	int flags = SQLITE_OPEN_PRIVATECACHE;
-	if (is_read_only) {
+	if (options.access_mode == AccessMode::READ_ONLY) {
 		flags |= SQLITE_OPEN_READONLY;
 	} else {
 		flags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE;
@@ -46,6 +46,19 @@ SQLiteDB SQLiteDB::Open(const string &path, bool is_read_only, bool is_shared) {
 	if (rc != SQLITE_OK) {
 		throw std::runtime_error("Unable to open database \"" + path + "\": " + string(sqlite3_errstr(rc)));
 	}
+	// default busy time-out of 5 seconds
+	if (options.busy_timeout > 0) {
+		if (options.busy_timeout > NumericLimits<int>::Maximum()) {
+			throw std::runtime_error("busy_timeout out of range - must be within valid range for type int");
+		}
+		rc = sqlite3_busy_timeout(result.db, int(options.busy_timeout));
+		if (rc != SQLITE_OK) {
+			throw std::runtime_error("Failed to set busy timeout");
+		}
+	}
+	if (!options.journal_mode.empty()) {
+		result.Execute("PRAGMA journal_mode=" + KeywordHelper::EscapeQuotes(options.journal_mode, '\''));
+	}
 	return result;
 }
 

src/sqlite_scanner.cpp

@@ -50,7 +50,9 @@ static unique_ptr<FunctionData> SqliteBind(ClientContext &context, TableFunction
 
 	SQLiteDB db;
 	SQLiteStatement stmt;
-	db = SQLiteDB::Open(result->file_name);
+	SQLiteOpenOptions options;
+	options.access_mode = AccessMode::READ_ONLY;
+	db = SQLiteDB::Open(result->file_name, options);
 
 	ColumnList columns;
 	vector<unique_ptr<Constraint>> constraints;
@@ -90,7 +92,9 @@ static void SqliteInitInternal(ClientContext &context, const SqliteBindData &bin
 	// function
 	local_state.stmt.Close();
 	if (!local_state.db) {
-		local_state.owned_db = SQLiteDB::Open(bind_data.file_name.c_str());
+		SQLiteOpenOptions options;
+		options.access_mode = AccessMode::READ_ONLY;
+		local_state.owned_db = SQLiteDB::Open(bind_data.file_name.c_str(), options);
 		local_state.db = &local_state.owned_db;
 	}
 
@@ -292,7 +296,9 @@ static void AttachFunction(ClientContext &context, TableFunctionInput &data_p, D
 		return;
 	}
 
-	SQLiteDB db = SQLiteDB::Open(data.file_name);
+	SQLiteOpenOptions options;
+	options.access_mode = AccessMode::READ_ONLY;
+	SQLiteDB db = SQLiteDB::Open(data.file_name, options);
 	auto dconn = Connection(context.db->GetDatabase(context));
 	{
 		auto tables = db.GetTables();

src/sqlite_storage.cpp

@@ -15,7 +15,16 @@ namespace duckdb {
 static unique_ptr<Catalog> SQLiteAttach(StorageExtensionInfo *storage_info, ClientContext &context,
                                         AttachedDatabase &db, const string &name, AttachInfo &info,
                                         AccessMode access_mode) {
-	return make_uniq<SQLiteCatalog>(db, info.path, access_mode);
+	SQLiteOpenOptions options;
+	options.access_mode = access_mode;
+	for(auto &entry : info.options) {
+		if (StringUtil::CIEquals(entry.first, "busy_timeout")) {
+			options.busy_timeout = entry.second.GetValue<uint64_t>();
+		} else if (StringUtil::CIEquals(entry.first, "journal_mode")) {
+			options.journal_mode = entry.second.ToString();
+		}
+	}
+	return make_uniq<SQLiteCatalog>(db, info.path, std::move(options));
 }
 
 static unique_ptr<TransactionManager> SQLiteCreateTransactionManager(StorageExtensionInfo *storage_info,

src/storage/sqlite_catalog.cpp

@@ -8,10 +8,10 @@
 
 namespace duckdb {
 
-SQLiteCatalog::SQLiteCatalog(AttachedDatabase &db_p, const string &path, AccessMode access_mode)
-    : Catalog(db_p), path(path), access_mode(access_mode), in_memory(path == ":memory:"), active_in_memory(false) {
+SQLiteCatalog::SQLiteCatalog(AttachedDatabase &db_p, const string &path, SQLiteOpenOptions options_p)
+    : Catalog(db_p), path(path), options(std::move(options_p)), in_memory(path == ":memory:"), active_in_memory(false) {
 	if (InMemory()) {
-		in_memory_db = SQLiteDB::Open(path, false, true);
+		in_memory_db = SQLiteDB::Open(path, options, true);
 	}
 }
 

src/storage/sqlite_transaction.cpp

@@ -17,8 +17,7 @@ SQLiteTransaction::SQLiteTransaction(SQLiteCatalog &sqlite_catalog, TransactionM
 		db = sqlite_catalog.GetInMemoryDatabase();
 	} else {
 		// on-disk database - open a new database connection
-		owned_db = SQLiteDB::Open(sqlite_catalog.path,
-		                          sqlite_catalog.access_mode == AccessMode::READ_ONLY ? true : false, true);
+		owned_db = SQLiteDB::Open(sqlite_catalog.path, sqlite_catalog.options, true);
 		db = &owned_db;
 	}
 }

test/sql/storage/attach_options.test

@@ -0,0 +1,24 @@
+# name: test/sql/storage/attach_options.test
+# description:
+# group: [sqlite_storage]
+
+require sqlite_scanner
+
+statement error
+ATTACH ':memory:' AS mem (TYPE SQLITE, BUSY_TIMEOUT 'hello')
+----
+Could not convert string
+
+statement error
+ATTACH ':memory:' AS mem (TYPE SQLITE, BUSY_TIMEOUT 99999999999)
+----
+busy_timeout out of range
+
+statement ok
+ATTACH ':memory:' AS mem (TYPE SQLITE, BUSY_TIMEOUT 0)
+
+statement ok
+DETACH mem
+
+statement ok
+ATTACH ':memory:' AS mem (TYPE SQLITE, JOURNAL_MODE 'WAL')