From bdd2d408c8c90bfab9e782fab3c30d0e27c332f4 Mon Sep 17 00:00:00 2001 From: Piotr Sarna Date: Mon, 8 Jan 2024 11:04:41 +0100 Subject: [PATCH] treewide: add opt-in `passphrase` param for encryption at rest You can now choose a passphrase and use it (plain text for now, sorry) to set up an encryption-at-rest key. Example: cargo run -F encryption-at-rest -- --passphrase pekka --- libsql-replication/Cargo.toml | 3 ++ libsql-replication/src/injector/mod.rs | 2 + libsql-server/Cargo.toml | 2 +- libsql-server/src/config.rs | 4 ++ libsql-server/src/connection/libsql.rs | 38 +++++++++++-- libsql-server/src/connection/write_proxy.rs | 13 ++++- libsql-server/src/lib.rs | 4 ++ libsql-server/src/main.rs | 6 +++ libsql-server/src/namespace/meta_store.rs | 9 +++- libsql-server/src/namespace/mod.rs | 54 +++++++++++++++++-- libsql-server/src/query_result_builder.rs | 4 +- .../src/replication/primary/logger.rs | 19 +++++-- .../primary/replication_logger_wal.rs | 4 ++ libsql-server/src/test/bottomless.rs | 2 + libsql-sys/src/connection.rs | 12 +++-- 15 files changed, 157 insertions(+), 19 deletions(-) diff --git a/libsql-replication/Cargo.toml b/libsql-replication/Cargo.toml index c877ca915ad..cfa90afc4cf 100644 --- a/libsql-replication/Cargo.toml +++ b/libsql-replication/Cargo.toml @@ -29,3 +29,6 @@ bincode = "1.3.3" tempfile = "3.8.0" prost-build = "0.12.0" tonic-build = "0.10" + +[features] +encryption-at-rest = ["libsql-sys/encryption-at-rest"] diff --git a/libsql-replication/src/injector/mod.rs b/libsql-replication/src/injector/mod.rs index e36f1a4fc1b..6321938e98c 100644 --- a/libsql-replication/src/injector/mod.rs +++ b/libsql-replication/src/injector/mod.rs @@ -56,6 +56,8 @@ impl Injector { | OpenFlags::SQLITE_OPEN_NO_MUTEX, wal_manager, auto_checkpoint, + #[cfg(feature = "encryption-at-rest")] + None, )?; Ok(Self { diff --git a/libsql-server/Cargo.toml b/libsql-server/Cargo.toml index b08edb0ccc8..20f711404d0 100644 --- a/libsql-server/Cargo.toml +++ b/libsql-server/Cargo.toml @@ -107,4 +107,4 @@ default = [] debug-tools = ["console-subscriber", "rusqlite/trace", "tokio/tracing"] wasm-udfs = ["rusqlite/libsql-wasm-experimental"] unix-excl-vfs = ["libsql-sys/unix-excl-vfs"] -encryption-at-rest = ["libsql-sys/encryption-at-rest"] +encryption-at-rest = ["libsql-sys/encryption-at-rest", "libsql-replication/encryption-at-rest"] diff --git a/libsql-server/src/config.rs b/libsql-server/src/config.rs index ac7699f91d7..fe0f797546a 100644 --- a/libsql-server/src/config.rs +++ b/libsql-server/src/config.rs @@ -125,6 +125,8 @@ pub struct DbConfig { pub snapshot_exec: Option, pub checkpoint_interval: Option, pub snapshot_at_shutdown: bool, + #[cfg(feature = "encryption-at-rest")] + pub passphrase: Option, } impl Default for DbConfig { @@ -141,6 +143,8 @@ impl Default for DbConfig { snapshot_exec: None, checkpoint_interval: None, snapshot_at_shutdown: false, + #[cfg(feature = "encryption-at-rest")] + passphrase: None, } } } diff --git a/libsql-server/src/connection/libsql.rs b/libsql-server/src/connection/libsql.rs index 3f7217e25ad..b8f11f593e7 100644 --- a/libsql-server/src/connection/libsql.rs +++ b/libsql-server/src/connection/libsql.rs @@ -44,6 +44,8 @@ pub struct MakeLibSqlConn { /// In wal mode, closing the last database takes time, and causes other databases creation to /// return sqlite busy. To mitigate that, we hold on to one connection _db: Option>, + #[cfg(feature = "encryption-at-rest")] + passphrase: Option, } impl MakeLibSqlConn @@ -62,6 +64,7 @@ where max_total_response_size: u64, auto_checkpoint: u32, current_frame_no_receiver: watch::Receiver>, + #[cfg(feature = "encryption-at-rest")] passphrase: Option, ) -> Result { let mut this = Self { db_path, @@ -75,6 +78,8 @@ where _db: None, state: Default::default(), wal_manager, + #[cfg(feature = "encryption-at-rest")] + passphrase, }; let db = this.try_create_db().await?; @@ -123,6 +128,8 @@ where max_size: Some(self.max_response_size), max_total_size: Some(self.max_total_response_size), auto_checkpoint: self.auto_checkpoint, + #[cfg(feature = "encryption-at-rest")] + passphrase: self.passphrase.clone(), }, self.current_frame_no_receiver.clone(), self.state.clone(), @@ -207,6 +214,7 @@ pub fn open_conn( path: &Path, wal_manager: T, flags: Option, + #[cfg(feature = "encryption-at-rest")] passphrase: Option, ) -> Result>, rusqlite::Error> where T: WalManager, @@ -223,6 +231,8 @@ where flags, WalWrapper::new(InhibitCheckpointWalWrapper, wal_manager), u32::MAX, + #[cfg(feature = "encryption-at-rest")] + passphrase, ) } @@ -232,6 +242,7 @@ pub fn open_conn_active_checkpoint( wal_manager: T, flags: Option, auto_checkpoint: u32, + #[cfg(feature = "encryption-at-rest")] passphrase: Option, ) -> Result, rusqlite::Error> where T: WalManager, @@ -243,7 +254,14 @@ where | OpenFlags::SQLITE_OPEN_NO_MUTEX, ); - libsql_sys::Connection::open(path.join("data"), flags, wal_manager, auto_checkpoint) + libsql_sys::Connection::open( + path.join("data"), + flags, + wal_manager, + auto_checkpoint, + #[cfg(feature = "encryption-at-rest")] + passphrase, + ) } impl LibSqlConnection @@ -506,8 +524,14 @@ impl Connection { current_frame_no_receiver: watch::Receiver>, state: Arc>, ) -> Result { - let conn = - open_conn_active_checkpoint(path, wal_manager, None, builder_config.auto_checkpoint)?; + let conn = open_conn_active_checkpoint( + path, + wal_manager, + None, + builder_config.auto_checkpoint, + #[cfg(feature = "encryption-at-rest")] + builder_config.passphrase.clone(), + )?; // register the lock-stealing busy handler unsafe { @@ -1049,6 +1073,8 @@ mod test { 100000000, DEFAULT_AUTO_CHECKPOINT, watch::channel(None).1, + #[cfg(feature = "encryption-at-rest")] + None, ) .await .unwrap(); @@ -1090,6 +1116,8 @@ mod test { 100000000, DEFAULT_AUTO_CHECKPOINT, watch::channel(None).1, + #[cfg(feature = "encryption-at-rest")] + None, ) .await .unwrap(); @@ -1132,6 +1160,8 @@ mod test { 100000000, DEFAULT_AUTO_CHECKPOINT, watch::channel(None).1, + #[cfg(feature = "encryption-at-rest")] + None, ) .await .unwrap(); @@ -1210,6 +1240,8 @@ mod test { 100000000, DEFAULT_AUTO_CHECKPOINT, watch::channel(None).1, + #[cfg(feature = "encryption-at-rest")] + None, ) .await .unwrap(); diff --git a/libsql-server/src/connection/write_proxy.rs b/libsql-server/src/connection/write_proxy.rs index a3c9893db1f..c9ea0114a9c 100644 --- a/libsql-server/src/connection/write_proxy.rs +++ b/libsql-server/src/connection/write_proxy.rs @@ -44,6 +44,8 @@ pub struct MakeWriteProxyConn { namespace: NamespaceName, primary_replication_index: Option, make_read_only_conn: MakeLibSqlConn, + #[cfg(feature = "encryption-at-rest")] + passphrase: Option, } impl MakeWriteProxyConn { @@ -60,6 +62,7 @@ impl MakeWriteProxyConn { max_total_response_size: u64, namespace: NamespaceName, primary_replication_index: Option, + #[cfg(feature = "encryption-at-rest")] passphrase: Option, ) -> crate::Result { let client = ProxyClient::with_origin(channel, uri); let make_read_only_conn = MakeLibSqlConn::new( @@ -72,6 +75,8 @@ impl MakeWriteProxyConn { max_total_response_size, DEFAULT_AUTO_CHECKPOINT, applied_frame_no_receiver.clone(), + #[cfg(feature = "encryption-at-rest")] + passphrase.clone(), ) .await?; @@ -84,6 +89,8 @@ impl MakeWriteProxyConn { namespace, make_read_only_conn, primary_replication_index, + #[cfg(feature = "encryption-at-rest")] + passphrase, }) } } @@ -100,6 +107,8 @@ impl MakeConnection for MakeWriteProxyConn { max_size: Some(self.max_response_size), max_total_size: Some(self.max_total_response_size), auto_checkpoint: DEFAULT_AUTO_CHECKPOINT, + #[cfg(feature = "encryption-at-rest")] + passphrase: self.passphrase.clone(), }, self.namespace.clone(), self.primary_replication_index, @@ -188,7 +197,7 @@ impl WriteProxyConnection { self.stats.inc_write_requests_delegated(); *status = TxnStatus::Invalid; let res = self - .with_remote_conn(auth, self.builder_config, |conn| { + .with_remote_conn(auth, self.builder_config.clone(), |conn| { Box::pin(conn.execute(pgm, builder)) }) .await; @@ -375,7 +384,7 @@ where ) -> crate::Result<(B, TxnStatus, Option)> { let mut txn_status = TxnStatus::Invalid; let mut new_frame_no = None; - let builder_config = self.builder_config; + let builder_config = self.builder_config.clone(); let cb = move |response: exec_resp::Response, builder: &mut B| match response { exec_resp::Response::ProgramResp(resp) => { crate::rpc::streaming_exec::apply_program_resp_to_builder( diff --git a/libsql-server/src/lib.rs b/libsql-server/src/lib.rs index f0f93d887b5..b7ec3832d48 100644 --- a/libsql-server/src/lib.rs +++ b/libsql-server/src/lib.rs @@ -528,6 +528,8 @@ where max_total_response_size: self.db_config.max_total_response_size, checkpoint_interval: self.db_config.checkpoint_interval, disable_namespace: self.disable_namespaces, + #[cfg(feature = "encryption-at-rest")] + passphrase: self.db_config.passphrase.clone(), }; let factory = PrimaryNamespaceMaker::new(conf); @@ -636,6 +638,8 @@ impl Replica { base_path: self.base_path.clone(), max_response_size: self.db_config.max_response_size, max_total_response_size: self.db_config.max_total_response_size, + #[cfg(feature = "encryption-at-rest")] + passphrase: self.db_config.passphrase.clone(), }; let factory = ReplicaNamespaceMaker::new(conf); diff --git a/libsql-server/src/main.rs b/libsql-server/src/main.rs index eaf30a95cf8..48c61ff6825 100644 --- a/libsql-server/src/main.rs +++ b/libsql-server/src/main.rs @@ -224,6 +224,10 @@ struct Cli { /// S3 endpoint for the meta store backups #[clap(long)] meta_store_bucket_endpoint: Option, + /// Passphrase for encryption at rest + #[cfg(feature = "encryption-at-rest")] + #[clap(long)] + passphrase: Option, } #[derive(clap::Subcommand, Debug)] @@ -338,6 +342,8 @@ fn make_db_config(config: &Cli) -> anyhow::Result { snapshot_exec: config.snapshot_exec.clone(), checkpoint_interval: config.checkpoint_interval_s.map(Duration::from_secs), snapshot_at_shutdown: config.snapshot_at_shutdown, + #[cfg(feature = "encryption-at-rest")] + passphrase: config.passphrase.clone(), }) } diff --git a/libsql-server/src/namespace/meta_store.rs b/libsql-server/src/namespace/meta_store.rs index fa5ae63e465..d463bcca656 100644 --- a/libsql-server/src/namespace/meta_store.rs +++ b/libsql-server/src/namespace/meta_store.rs @@ -198,7 +198,14 @@ impl MetaStore { replicator.map(BottomlessWalWrapper::new), Sqlite3WalManager::default(), ); - let conn = open_conn_active_checkpoint(&db_path, wal_manager.clone(), None, 1000)?; + let conn = open_conn_active_checkpoint( + &db_path, + wal_manager.clone(), + None, + 1000, + #[cfg(feature = "encryption-at-rest")] + None, + )?; let configs = restore(&conn)?; diff --git a/libsql-server/src/namespace/mod.rs b/libsql-server/src/namespace/mod.rs index d3f35fa305e..13a76e8168b 100644 --- a/libsql-server/src/namespace/mod.rs +++ b/libsql-server/src/namespace/mod.rs @@ -817,6 +817,8 @@ pub struct ReplicaNamespaceConfig { pub extensions: Arc<[PathBuf]>, /// Stats monitor pub stats_sender: StatsSender, + #[cfg(feature = "encryption-at-rest")] + pub passphrase: Option, } impl Namespace { @@ -911,6 +913,8 @@ impl Namespace { config.stats_sender.clone(), name.clone(), applied_frame_no_receiver.clone(), + #[cfg(feature = "encryption-at-rest")] + config.passphrase.clone(), ) .await?; @@ -926,6 +930,8 @@ impl Namespace { config.max_total_response_size, name.clone(), primary_current_replicatio_index, + #[cfg(feature = "encryption-at-rest")] + config.passphrase.clone(), ) .await? .throttled( @@ -959,6 +965,8 @@ pub struct PrimaryNamespaceConfig { pub max_total_response_size: u64, pub checkpoint_interval: Option, pub disable_namespace: bool, + #[cfg(feature = "encryption-at-rest")] + pub passphrase: Option, } pub type DumpStream = @@ -1088,6 +1096,8 @@ impl Namespace { config.stats_sender.clone(), name.clone(), logger.new_frame_notifier.subscribe(), + #[cfg(feature = "encryption-at-rest")] + config.passphrase.clone(), ) .await?; @@ -1102,6 +1112,8 @@ impl Namespace { config.max_total_response_size, auto_checkpoint, logger.new_frame_notifier.subscribe(), + #[cfg(feature = "encryption-at-rest")] + config.passphrase.clone(), ) .await? .throttled( @@ -1119,7 +1131,14 @@ impl Namespace { Err(LoadDumpError::LoadDumpExistingDb)?; } RestoreOption::Dump(dump) => { - load_dump(&db_path, dump, wal_manager.clone()).await?; + load_dump( + &db_path, + dump, + wal_manager.clone(), + #[cfg(feature = "encryption-at-rest")] + config.passphrase.clone(), + ) + .await?; } _ => { /* other cases were already handled when creating bottomless */ } } @@ -1152,6 +1171,7 @@ async fn make_stats( stats_sender: StatsSender, name: NamespaceName, mut current_frame_no: watch::Receiver>, + #[cfg(feature = "encryption-at-rest")] passphrase: Option, ) -> anyhow::Result> { let stats = Stats::new(name.clone(), db_path, join_set).await?; @@ -1176,7 +1196,12 @@ async fn make_stats( } }); - join_set.spawn(run_storage_monitor(db_path.into(), Arc::downgrade(&stats))); + join_set.spawn(run_storage_monitor( + db_path.into(), + Arc::downgrade(&stats), + #[cfg(feature = "encryption-at-rest")] + passphrase, + )); Ok(stats) } @@ -1202,6 +1227,7 @@ async fn load_dump( db_path: &Path, dump: S, wal_manager: C, + #[cfg(feature = "encryption-at-rest")] passphrase: Option, ) -> crate::Result<(), LoadDumpError> where S: Stream> + Unpin, @@ -1213,7 +1239,19 @@ where let conn = loop { let db_path = db_path.to_path_buf(); let wal_manager = wal_manager.clone(); - match tokio::task::spawn_blocking(move || open_conn(&db_path, wal_manager, None)).await? { + #[cfg(feature = "encryption-at-rest")] + let passphrase = passphrase.clone(); + match tokio::task::spawn_blocking(move || { + open_conn( + &db_path, + wal_manager, + None, + #[cfg(feature = "encryption-at-rest")] + passphrase, + ) + }) + .await? + { Ok(conn) => { break conn; } @@ -1357,7 +1395,11 @@ fn check_fresh_db(path: &Path) -> crate::Result { // Periodically check the storage used by the database and save it in the Stats structure. // TODO: Once we have a separate fiber that does WAL checkpoints, running this routine // right after checkpointing is exactly where it should be done. -async fn run_storage_monitor(db_path: PathBuf, stats: Weak) -> anyhow::Result<()> { +async fn run_storage_monitor( + db_path: PathBuf, + stats: Weak, + #[cfg(feature = "encryption-at-rest")] passphrase: Option, +) -> anyhow::Result<()> { // on initialization, the database file doesn't exist yet, so we wait a bit for it to be // created tokio::time::sleep(Duration::from_secs(1)).await; @@ -1369,11 +1411,13 @@ async fn run_storage_monitor(db_path: PathBuf, stats: Weak) -> anyhow::Re let Some(stats) = stats.upgrade() else { return Ok(()); }; + #[cfg(feature = "encryption-at-rest")] + let passphrase = passphrase.clone(); let _ = tokio::task::spawn_blocking(move || { // because closing the last connection interferes with opening a new one, we lazily // initialize a connection here, and keep it alive for the entirety of the program. If we // fail to open it, we wait for `duration` and try again later. - match open_conn(&db_path, Sqlite3WalManager::new(), Some(rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)) { + match open_conn(&db_path, Sqlite3WalManager::new(), Some(rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY), #[cfg(feature = "encryption-at-rest")] passphrase) { Ok(conn) => { if let Ok(storage_bytes_used) = conn.query_row("select sum(pgsize) from dbstat;", [], |row| { diff --git a/libsql-server/src/query_result_builder.rs b/libsql-server/src/query_result_builder.rs index 16efa08fd3a..0ad7c4f1673 100644 --- a/libsql-server/src/query_result_builder.rs +++ b/libsql-server/src/query_result_builder.rs @@ -82,11 +82,13 @@ impl<'a> From<&'a rusqlite::Column<'a>> for Column<'a> { } } -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Default)] pub struct QueryBuilderConfig { pub max_size: Option, pub max_total_size: Option, pub auto_checkpoint: u32, + #[cfg(feature = "encryption-at-rest")] + pub passphrase: Option, } pub trait QueryResultBuilder: Send + 'static { diff --git a/libsql-server/src/replication/primary/logger.rs b/libsql-server/src/replication/primary/logger.rs index 88ea2440b02..75645cbbdd9 100644 --- a/libsql-server/src/replication/primary/logger.rs +++ b/libsql-server/src/replication/primary/logger.rs @@ -922,8 +922,14 @@ mod test { ) .unwrap(), ); - let mut conn = - open_conn(tmp.path(), ReplicationLoggerWalManager::new(logger), None).unwrap(); + let mut conn = open_conn( + tmp.path(), + ReplicationLoggerWalManager::new(logger), + None, + #[cfg(feature = "encryption-at-rest")] + None, + ) + .unwrap(); conn.execute("BEGIN", ()).unwrap(); conn.execute("CREATE TABLE test (x)", ()).unwrap(); @@ -963,7 +969,14 @@ mod test { new_db_file.flush().unwrap(); - let conn2 = open_conn(tmp2.path(), Sqlite3WalManager::new(), None).unwrap(); + let conn2 = open_conn( + tmp2.path(), + Sqlite3WalManager::new(), + None, + #[cfg(feature = "encryption-at-rest")] + None, + ) + .unwrap(); conn2 .query_row("SELECT count(*) FROM test", (), |row| { diff --git a/libsql-server/src/replication/primary/replication_logger_wal.rs b/libsql-server/src/replication/primary/replication_logger_wal.rs index fee58ad2945..fbad359988e 100644 --- a/libsql-server/src/replication/primary/replication_logger_wal.rs +++ b/libsql-server/src/replication/primary/replication_logger_wal.rs @@ -373,6 +373,8 @@ mod test { wal_manager, None, u32::MAX, + #[cfg(feature = "encryption-at-rest")] + None, ) .unwrap(); @@ -421,6 +423,8 @@ mod test { wal_manager, None, u32::MAX, + #[cfg(feature = "encryption-at-rest")] + None, ) .unwrap(); diff --git a/libsql-server/src/test/bottomless.rs b/libsql-server/src/test/bottomless.rs index da284221c8d..3e48a5dbda7 100644 --- a/libsql-server/src/test/bottomless.rs +++ b/libsql-server/src/test/bottomless.rs @@ -92,6 +92,8 @@ async fn configure_server( snapshot_exec: None, checkpoint_interval: Some(Duration::from_secs(3)), snapshot_at_shutdown: false, + #[cfg(feature = "encryption-at-rest")] + passphrase: None, }, admin_api_config: None, disable_namespaces: true, diff --git a/libsql-sys/src/connection.rs b/libsql-sys/src/connection.rs index af2cc40ec4f..a38c5a19c21 100644 --- a/libsql-sys/src/connection.rs +++ b/libsql-sys/src/connection.rs @@ -59,6 +59,7 @@ impl Connection { flags: OpenFlags, wal_manager: T, auto_checkpoint: u32, + #[cfg(feature = "encryption-at-rest")] passphrase: Option, ) -> Result where T: WalManager, @@ -84,10 +85,15 @@ impl Connection { make_wal_manager(wal_manager), ) }?; - if cfg!(feature = "encryption-at-rest") { - conn.pragma_update(None, "key", "s3cr3t")?; - tracing::debug!("KEY set to s3cr3t: don't tell anyone, SOC2 compliance, shhh"); + + #[cfg(feature = "encryption-at-rest")] + if let Some(passphrase) = passphrase { + conn.pragma_update(None, "key", &passphrase)?; + tracing::debug!( + "KEY set to {passphrase}: don't tell anyone, SOC2 compliance, shhh" + ); } + conn.pragma_update(None, "journal_mode", "WAL")?; unsafe { let rc =