diff --git a/.gitmodules b/.gitmodules index 282ed3a..707c4e6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "sqlite-rs-embedded"] path = sqlite-rs-embedded - url = https://github.com/vlcn-io/sqlite-rs-embedded.git + url = https://github.com/powersync-ja/sqlite-rs-embedded.git diff --git a/Cargo.lock b/Cargo.lock index 5751c1e..56edbe8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "bindgen" -version = "0.63.0" +version = "0.68.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36d860121800b2a9a94f9b5604b332d5cffb234ce17609ea479d723dbc9d3885" +checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" dependencies = [ "bitflags", "cexpr", @@ -30,20 +30,21 @@ dependencies = [ "lazycell", "log", "peeking_take_while", + "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", - "syn 1.0.109", + "syn 2.0.100", "which", ] [[package]] name = "bitflags" -version = "1.3.2" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" [[package]] name = "bytes" @@ -182,11 +183,22 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] @@ -208,11 +220,12 @@ name = "powersync_core" version = "0.3.9" dependencies = [ "bytes", - "num-derive", + "num-derive 0.3.3", "num-traits", "serde", "serde_json", "sqlite_nostd", + "streaming-iterator", "uuid", ] @@ -233,20 +246,30 @@ dependencies = [ "sqlite_nostd", ] +[[package]] +name = "prettyplease" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" +dependencies = [ + "proc-macro2", + "syn 2.0.100", +] + [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.33" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] @@ -309,7 +332,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn 2.0.100", ] [[package]] @@ -354,12 +377,18 @@ dependencies = [ name = "sqlite_nostd" version = "0.1.0" dependencies = [ - "num-derive", + "num-derive 0.4.2", "num-traits", "sqlite3_allocator", "sqlite3_capi", ] +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + [[package]] name = "syn" version = "1.0.109" @@ -373,9 +402,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.29" +version = "2.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" dependencies = [ "proc-macro2", "quote", diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 0f0f012..9c1882b 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -19,6 +19,7 @@ num-traits = { version = "0.2.15", default-features = false } num-derive = "0.3" serde_json = { version = "1.0", default-features = false, features = ["alloc"] } serde = { version = "1.0", default-features = false, features = ["alloc", "derive"] } +streaming-iterator = { version = "0.1.9", default-features = false, features = ["alloc"] } [dependencies.uuid] version = "1.4.1" diff --git a/crates/core/src/checkpoint.rs b/crates/core/src/checkpoint.rs index 30cf11c..1e6527d 100644 --- a/crates/core/src/checkpoint.rs +++ b/crates/core/src/checkpoint.rs @@ -4,7 +4,6 @@ use alloc::format; use alloc::string::String; use alloc::vec::Vec; use core::ffi::c_int; -use core::slice; use serde::{Deserialize, Serialize}; use serde_json as json; @@ -59,9 +58,9 @@ GROUP BY bucket_list.bucket", while statement.step()? == ResultCode::ROW { let name = statement.column_text(0)?; // checksums with column_int are wrapped to i32 by SQLite - let add_checksum = statement.column_int(1)?; - let oplog_checksum = statement.column_int(2)?; - let expected_checksum = statement.column_int(3)?; + let add_checksum = statement.column_int(1); + let oplog_checksum = statement.column_int(2); + let expected_checksum = statement.column_int(3); // wrapping add is like +, but safely overflows let checksum = oplog_checksum.wrapping_add(add_checksum); diff --git a/crates/core/src/crud_vtab.rs b/crates/core/src/crud_vtab.rs index 4f4fc40..95cc0dc 100644 --- a/crates/core/src/crud_vtab.rs +++ b/crates/core/src/crud_vtab.rs @@ -3,7 +3,6 @@ extern crate alloc; use alloc::boxed::Box; use alloc::string::String; use core::ffi::{c_char, c_int, c_void}; -use core::slice; use sqlite::{Connection, ResultCode, Value}; use sqlite_nostd as sqlite; @@ -29,7 +28,7 @@ struct VirtualTable { base: sqlite::vtab, db: *mut sqlite::sqlite3, current_tx: Option, - insert_statement: Option + insert_statement: Option, } extern "C" fn connect( @@ -40,8 +39,7 @@ extern "C" fn connect( vtab: *mut *mut sqlite::vtab, _err: *mut *mut c_char, ) -> c_int { - if let Err(rc) = sqlite::declare_vtab(db, "CREATE TABLE powersync_crud_(data TEXT);") - { + if let Err(rc) = sqlite::declare_vtab(db, "CREATE TABLE powersync_crud_(data TEXT);") { return rc as c_int; } @@ -54,7 +52,7 @@ extern "C" fn connect( }, db, current_tx: None, - insert_statement: None + insert_statement: None, })); *vtab = tab.cast::(); let _ = sqlite::vtab_config(db, 0); @@ -69,7 +67,6 @@ extern "C" fn disconnect(vtab: *mut sqlite::vtab) -> c_int { ResultCode::OK as c_int } - fn begin_impl(tab: &mut VirtualTable) -> Result<(), SQLiteError> { let db = tab.db; @@ -77,9 +74,10 @@ fn begin_impl(tab: &mut VirtualTable) -> Result<(), SQLiteError> { tab.insert_statement = Some(insert_statement); // language=SQLite - let statement = db.prepare_v2("UPDATE ps_tx SET next_tx = next_tx + 1 WHERE id = 1 RETURNING next_tx")?; + let statement = + db.prepare_v2("UPDATE ps_tx SET next_tx = next_tx + 1 WHERE id = 1 RETURNING next_tx")?; if statement.step()? == ResultCode::ROW { - let tx_id = statement.column_int64(0)? - 1; + let tx_id = statement.column_int64(0) - 1; tab.current_tx = Some(tx_id); } else { return Err(SQLiteError::from(ResultCode::ABORT)); @@ -109,15 +107,20 @@ extern "C" fn rollback(vtab: *mut sqlite::vtab) -> c_int { ResultCode::OK as c_int } -fn insert_operation( - vtab: *mut sqlite::vtab, data: &str) -> Result<(), SQLiteError> { +fn insert_operation(vtab: *mut sqlite::vtab, data: &str) -> Result<(), SQLiteError> { let tab = unsafe { &mut *(vtab.cast::()) }; if tab.current_tx.is_none() { - return Err(SQLiteError(ResultCode::MISUSE, Some(String::from("No tx_id")))); + return Err(SQLiteError( + ResultCode::MISUSE, + Some(String::from("No tx_id")), + )); } let current_tx = tab.current_tx.unwrap(); // language=SQLite - let statement = tab.insert_statement.as_ref().ok_or(SQLiteError::from(NULL))?; + let statement = tab + .insert_statement + .as_ref() + .ok_or(SQLiteError::from(NULL))?; statement.bind_int64(1, current_tx)?; statement.bind_text(2, data, sqlite::Destructor::STATIC)?; statement.exec()?; @@ -125,7 +128,6 @@ fn insert_operation( Ok(()) } - extern "C" fn update( vtab: *mut sqlite::vtab, argc: c_int, @@ -178,6 +180,7 @@ static MODULE: sqlite_nostd::module = sqlite_nostd::module { xRelease: None, xRollbackTo: None, xShadowName: None, + xIntegrity: None, }; pub fn register(db: *mut sqlite::sqlite3) -> Result<(), ResultCode> { diff --git a/crates/core/src/diff.rs b/crates/core/src/diff.rs index a244c58..52ede13 100644 --- a/crates/core/src/diff.rs +++ b/crates/core/src/diff.rs @@ -3,7 +3,6 @@ extern crate alloc; use alloc::format; use alloc::string::{String, ToString}; use core::ffi::c_int; -use core::slice; use sqlite::ResultCode; use sqlite_nostd as sqlite; @@ -20,11 +19,20 @@ fn powersync_diff_impl( ) -> Result { let data_old = args[0].text(); let data_new = args[1].text(); + let ignore_removed = args.get(2).map_or(false, |v| v.int() != 0); - diff_objects(data_old, data_new) + diff_objects_with_options(data_old, data_new, ignore_removed) } -pub fn diff_objects(data_old: &str, data_new: &str) -> Result { +/// Returns a JSON object containing entries from [data_new] that are not present in [data_old]. +/// +/// When [ignore_removed_columns] is set, columns that are present in [data_old] but not in +/// [data_new] will not be present in the returned object. Otherwise, they will be set to `null`. +fn diff_objects_with_options( + data_old: &str, + data_new: &str, + ignore_removed_columns: bool, +) -> Result { let v_new: json::Value = json::from_str(data_new)?; let v_old: json::Value = json::from_str(data_old)?; @@ -39,9 +47,11 @@ pub fn diff_objects(data_old: &str, data_new: &str) -> Result Result<(), ResultCode> { None, )?; + db.create_function_v2( + "powersync_diff", + 3, + sqlite::UTF8 | sqlite::DETERMINISTIC, + None, + Some(powersync_diff), + None, + None, + None, + )?; + Ok(()) } @@ -83,6 +104,10 @@ pub fn register(db: *mut sqlite::sqlite3) -> Result<(), ResultCode> { mod tests { use super::*; + fn diff_objects(data_old: &str, data_new: &str) -> Result { + diff_objects_with_options(data_old, data_new, false) + } + #[test] fn basic_diff_test() { assert_eq!(diff_objects("{}", "{}").unwrap(), "{}"); diff --git a/crates/core/src/json_merge.rs b/crates/core/src/json_merge.rs index 6332326..80c1687 100644 --- a/crates/core/src/json_merge.rs +++ b/crates/core/src/json_merge.rs @@ -3,7 +3,6 @@ extern crate alloc; use alloc::format; use alloc::string::{String, ToString}; use core::ffi::c_int; -use core::slice; use sqlite::ResultCode; use sqlite_nostd as sqlite; diff --git a/crates/core/src/kv.rs b/crates/core/src/kv.rs index c5b7bbc..811409d 100644 --- a/crates/core/src/kv.rs +++ b/crates/core/src/kv.rs @@ -3,7 +3,6 @@ extern crate alloc; use alloc::format; use alloc::string::{String, ToString}; use core::ffi::c_int; -use core::slice; use sqlite::ResultCode; use sqlite_nostd as sqlite; diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 972f21d..b191fd0 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -25,7 +25,7 @@ mod macros; mod migrations; mod operations; mod operations_vtab; -mod schema_management; +mod schema; mod sync_local; mod sync_types; mod util; @@ -62,7 +62,7 @@ fn init_extension(db: *mut sqlite::sqlite3) -> Result<(), ResultCode> { crate::checkpoint::register(db)?; crate::kv::register(db)?; - crate::schema_management::register(db)?; + crate::schema::register(db)?; crate::operations_vtab::register(db)?; crate::crud_vtab::register(db)?; diff --git a/crates/core/src/migrations.rs b/crates/core/src/migrations.rs index 0bfbb03..193de21 100644 --- a/crates/core/src/migrations.rs +++ b/crates/core/src/migrations.rs @@ -32,7 +32,7 @@ CREATE TABLE IF NOT EXISTS ps_migration(id INTEGER PRIMARY KEY, down_migrations return Err(SQLiteError::from(ResultCode::ABORT)); } - let mut current_version = current_version_stmt.column_int(0)?; + let mut current_version = current_version_stmt.column_int(0); while current_version > target_version { // Run down migrations. @@ -76,7 +76,7 @@ CREATE TABLE IF NOT EXISTS ps_migration(id INTEGER PRIMARY KEY, down_migrations Some("Down migration failed - could not get version".to_string()), )); } - let new_version = current_version_stmt.column_int(0)?; + let new_version = current_version_stmt.column_int(0); if new_version >= current_version { // Database down from version $currentVersion to $version failed - version not updated after dow migration return Err(SQLiteError( diff --git a/crates/core/src/operations.rs b/crates/core/src/operations.rs index 2de3afe..58f5f64 100644 --- a/crates/core/src/operations.rs +++ b/crates/core/src/operations.rs @@ -69,13 +69,13 @@ FROM json_each(?) e", bucket_statement.bind_text(1, bucket, sqlite::Destructor::STATIC)?; bucket_statement.step()?; - let bucket_id = bucket_statement.column_int64(0)?; + let bucket_id = bucket_statement.column_int64(0); // This is an optimization for initial sync - we can avoid persisting individual REMOVE // operations when last_applied_op = 0. // We do still need to do the "supersede_statement" step for this case, since a REMOVE // operation can supersede another PUT operation we're syncing at the same time. - let mut is_empty = bucket_statement.column_int64(1)? == 0; + let mut is_empty = bucket_statement.column_int64(1) == 0; // Statement to supersede (replace) operations with the same key. // language=SQLite @@ -105,11 +105,11 @@ INSERT OR IGNORE INTO ps_updated_rows(row_type, row_id) VALUES(?1, ?2)", let mut op_checksum: i32 = 0; while iterate_statement.step()? == ResultCode::ROW { - let op_id = iterate_statement.column_int64(0)?; + let op_id = iterate_statement.column_int64(0); let op = iterate_statement.column_text(1)?; let object_type = iterate_statement.column_text(2); let object_id = iterate_statement.column_text(3); - let checksum = iterate_statement.column_int(4)?; + let checksum = iterate_statement.column_int(4); let op_data = iterate_statement.column_text(5); last_op = Some(op_id); @@ -129,7 +129,7 @@ INSERT OR IGNORE INTO ps_updated_rows(row_type, row_id) VALUES(?1, ?2)", while supersede_statement.step()? == ResultCode::ROW { // Superseded (deleted) a previous operation, add the checksum - let supersede_checksum = supersede_statement.column_int(1)?; + let supersede_checksum = supersede_statement.column_int(1); add_checksum = add_checksum.wrapping_add(supersede_checksum); op_checksum = op_checksum.wrapping_sub(supersede_checksum); @@ -268,7 +268,7 @@ pub fn delete_bucket(db: *mut sqlite::sqlite3, name: &str) -> Result<(), SQLiteE statement.bind_text(1, name, sqlite::Destructor::STATIC)?; if statement.step()? == ResultCode::ROW { - let bucket_id = statement.column_int64(0)?; + let bucket_id = statement.column_int64(0); // language=SQLite let updated_statement = db.prepare_v2( diff --git a/crates/core/src/operations_vtab.rs b/crates/core/src/operations_vtab.rs index 4ac441b..96b5506 100644 --- a/crates/core/src/operations_vtab.rs +++ b/crates/core/src/operations_vtab.rs @@ -2,7 +2,6 @@ extern crate alloc; use alloc::boxed::Box; use core::ffi::{c_char, c_int, c_void}; -use core::slice; use sqlite::{Connection, ResultCode, Value}; use sqlite_nostd as sqlite; @@ -137,6 +136,7 @@ static MODULE: sqlite_nostd::module = sqlite_nostd::module { xRelease: None, xRollbackTo: None, xShadowName: None, + xIntegrity: None, }; pub fn register(db: *mut sqlite::sqlite3) -> Result<(), ResultCode> { diff --git a/crates/core/src/schema_management.rs b/crates/core/src/schema/management.rs similarity index 98% rename from crates/core/src/schema_management.rs rename to crates/core/src/schema/management.rs index 2a46473..6f66d03 100644 --- a/crates/core/src/schema_management.rs +++ b/crates/core/src/schema/management.rs @@ -4,7 +4,6 @@ use alloc::format; use alloc::string::String; use alloc::vec::Vec; use core::ffi::c_int; -use core::slice; use sqlite::{Connection, ResultCode, Value}; use sqlite_nostd as sqlite; @@ -35,7 +34,7 @@ SELECT while statement.step().into_db_result(db)? == ResultCode::ROW { let name = statement.column_text(0)?; let internal_name = statement.column_text(1)?; - let local_only = statement.column_int(2)? != 0; + let local_only = statement.column_int(2) != 0; db.exec_safe(&format!( "CREATE TABLE {:}(id TEXT PRIMARY KEY NOT NULL, data TEXT)", @@ -110,7 +109,7 @@ SELECT name, internal_name, local_only FROM powersync_tables WHERE name NOT IN ( while statement.step()? == ResultCode::ROW { let name = statement.column_text(0)?; let internal_name = statement.column_text(1)?; - let local_only = statement.column_int(2)? != 0; + let local_only = statement.column_int(2) != 0; tables_to_drop.push(String::from(internal_name)); @@ -169,7 +168,7 @@ SELECT while stmt2.step()? == ResultCode::ROW { let name = stmt2.column_text(0)?; let type_name = stmt2.column_text(1)?; - let ascending = stmt2.column_int(2)? != 0; + let ascending = stmt2.column_int(2) != 0; if ascending { let value = format!( diff --git a/crates/core/src/schema/mod.rs b/crates/core/src/schema/mod.rs new file mode 100644 index 0000000..46ab69a --- /dev/null +++ b/crates/core/src/schema/mod.rs @@ -0,0 +1,10 @@ +mod management; +mod table_info; + +use sqlite::ResultCode; +use sqlite_nostd as sqlite; +pub use table_info::{ColumnInfo, ColumnNameAndTypeStatement, DiffIncludeOld, TableInfo}; + +pub fn register(db: *mut sqlite::sqlite3) -> Result<(), ResultCode> { + management::register(db) +} diff --git a/crates/core/src/schema/table_info.rs b/crates/core/src/schema/table_info.rs new file mode 100644 index 0000000..8b3ca4f --- /dev/null +++ b/crates/core/src/schema/table_info.rs @@ -0,0 +1,197 @@ +use core::marker::PhantomData; + +use alloc::{ + string::{String, ToString}, + vec::Vec, +}; +use streaming_iterator::StreamingIterator; + +use crate::error::SQLiteError; +use sqlite::{Connection, ResultCode}; +use sqlite_nostd::{self as sqlite, ManagedStmt}; + +pub struct TableInfo { + pub name: String, + pub view_name: String, + pub diff_include_old: Option, + pub flags: TableInfoFlags, +} + +impl TableInfo { + pub fn parse_from(db: *mut sqlite::sqlite3, data: &str) -> Result { + // language=SQLite + let statement = db.prepare_v2( + "SELECT + json_extract(?1, '$.name'), + ifnull(json_extract(?1, '$.view_name'), json_extract(?1, '$.name')), + json_extract(?1, '$.local_only'), + json_extract(?1, '$.insert_only'), + json_extract(?1, '$.include_old'), + json_extract(?1, '$.include_metadata'), + json_extract(?1, '$.include_old_only_when_changed')", + )?; + statement.bind_text(1, data, sqlite::Destructor::STATIC)?; + + let step_result = statement.step()?; + if step_result != ResultCode::ROW { + return Err(SQLiteError::from(ResultCode::SCHEMA)); + } + + let name = statement.column_text(0)?.to_string(); + let view_name = statement.column_text(1)?.to_string(); + let flags = { + let local_only = statement.column_int(2) != 0; + let insert_only = statement.column_int(3) != 0; + let include_metadata = statement.column_int(5) != 0; + let include_old_only_when_changed = statement.column_int(6) != 0; + + let mut flags = TableInfoFlags::default(); + flags = flags.set_flag(TableInfoFlags::LOCAL_ONLY, local_only); + flags = flags.set_flag(TableInfoFlags::INSERT_ONLY, insert_only); + flags = flags.set_flag(TableInfoFlags::INCLUDE_METADATA, include_metadata); + flags = flags.set_flag( + TableInfoFlags::INCLUDE_OLD_ONLY_IF_CHANGED, + include_old_only_when_changed, + ); + + flags + }; + + let include_old = match statement.column_type(4)? { + sqlite_nostd::ColumnType::Text => { + let columns: Vec = serde_json::from_str(statement.column_text(4)?)?; + Some(DiffIncludeOld::OnlyForColumns { columns }) + } + + sqlite_nostd::ColumnType::Integer => { + if statement.column_int(4) != 0 { + Some(DiffIncludeOld::ForAllColumns) + } else { + None + } + } + _ => None, + }; + + // Don't allow include_metadata for local_only tables, it breaks our trigger setup and makes + // no sense because these changes are never inserted into ps_crud. + if flags.include_metadata() && flags.local_only() { + return Err(SQLiteError( + ResultCode::ERROR, + Some("include_metadata and local_only are incompatible".to_string()), + )); + } + + return Ok(TableInfo { + name, + view_name, + diff_include_old: include_old, + flags, + }); + } +} + +pub enum DiffIncludeOld { + OnlyForColumns { columns: Vec }, + ForAllColumns, +} + +#[derive(Clone, Copy)] +#[repr(transparent)] +pub struct TableInfoFlags(u32); + +impl TableInfoFlags { + pub const LOCAL_ONLY: u32 = 1; + pub const INSERT_ONLY: u32 = 2; + pub const INCLUDE_METADATA: u32 = 4; + pub const INCLUDE_OLD_ONLY_IF_CHANGED: u32 = 8; + + pub const fn local_only(self) -> bool { + self.0 & Self::LOCAL_ONLY != 0 + } + + pub const fn insert_only(self) -> bool { + self.0 & Self::INSERT_ONLY != 0 + } + + pub const fn include_metadata(self) -> bool { + self.0 & Self::INCLUDE_METADATA != 0 + } + + pub const fn include_old_only_if_changed(self) -> bool { + self.0 & Self::INCLUDE_OLD_ONLY_IF_CHANGED != 0 + } + + const fn with_flag(self, flag: u32) -> Self { + Self(self.0 | flag) + } + + const fn without_flag(self, flag: u32) -> Self { + Self(self.0 & !flag) + } + + const fn set_flag(self, flag: u32, enable: bool) -> Self { + if enable { + self.with_flag(flag) + } else { + self.without_flag(flag) + } + } +} + +impl Default for TableInfoFlags { + fn default() -> Self { + Self(0) + } +} + +pub struct ColumnNameAndTypeStatement<'a> { + pub stmt: ManagedStmt, + table: PhantomData<&'a str>, +} + +impl ColumnNameAndTypeStatement<'_> { + pub fn new(db: *mut sqlite::sqlite3, table: &str) -> Result { + let stmt = db.prepare_v2("select json_extract(e.value, '$.name'), json_extract(e.value, '$.type') from json_each(json_extract(?, '$.columns')) e")?; + stmt.bind_text(1, table, sqlite::Destructor::STATIC)?; + + Ok(Self { + stmt, + table: PhantomData, + }) + } + + fn step(stmt: &ManagedStmt) -> Result, ResultCode> { + if stmt.step()? == ResultCode::ROW { + let name = stmt.column_text(0)?; + let type_name = stmt.column_text(1)?; + + return Ok(Some(ColumnInfo { name, type_name })); + } + + Ok(None) + } + + pub fn streaming_iter( + &mut self, + ) -> impl StreamingIterator> { + streaming_iterator::from_fn(|| match Self::step(&self.stmt) { + Err(e) => Some(Err(e)), + Ok(Some(other)) => Some(Ok(other)), + Ok(None) => None, + }) + } + + pub fn names_iter(&mut self) -> impl StreamingIterator> { + self.streaming_iter().map(|item| match item { + Ok(row) => Ok(row.name), + Err(e) => Err(*e), + }) + } +} + +#[derive(Clone)] +pub struct ColumnInfo<'a> { + pub name: &'a str, + pub type_name: &'a str, +} diff --git a/crates/core/src/sync_local.rs b/crates/core/src/sync_local.rs index 89646f4..6cd8068 100644 --- a/crates/core/src/sync_local.rs +++ b/crates/core/src/sync_local.rs @@ -107,7 +107,7 @@ impl<'a> SyncOperation<'a> { while statement.step().into_db_result(self.db)? == ResultCode::ROW { let type_name = statement.column_text(0)?; let id = statement.column_text(1)?; - let buckets = statement.column_int(3)?; + let buckets = statement.column_int(3); let data = statement.column_text(2); let table_name = internal_table_name(type_name); diff --git a/crates/core/src/util.rs b/crates/core/src/util.rs index 8bbe7b6..2e50951 100644 --- a/crates/core/src/util.rs +++ b/crates/core/src/util.rs @@ -6,16 +6,14 @@ use alloc::string::String; use serde::Deserialize; use serde_json as json; -use sqlite::{Connection, ResultCode}; -use sqlite_nostd as sqlite; -use sqlite_nostd::ManagedStmt; +#[cfg(not(feature = "getrandom"))] +use crate::sqlite; + use uuid::Uuid; #[cfg(not(feature = "getrandom"))] use uuid::Builder; -use crate::error::SQLiteError; - pub fn quote_string(s: &str) -> String { format!("'{:}'", s.replace("'", "''")) } @@ -44,29 +42,6 @@ pub fn quote_identifier_prefixed(prefix: &str, name: &str) -> String { return format!("\"{:}{:}\"", prefix, name.replace("\"", "\"\"")); } -pub fn extract_table_info( - db: *mut sqlite::sqlite3, - data: &str, -) -> Result { - // language=SQLite - let statement = db.prepare_v2( - "SELECT - json_extract(?1, '$.name') as name, - ifnull(json_extract(?1, '$.view_name'), json_extract(?1, '$.name')) as view_name, - json_extract(?1, '$.local_only') as local_only, - json_extract(?1, '$.insert_only') as insert_only, - json_extract(?1, '$.include_old') as include_old, - json_extract(?1, '$.include_metadata') as include_metadata", - )?; - statement.bind_text(1, data, sqlite::Destructor::STATIC)?; - - let step_result = statement.step()?; - if step_result != ResultCode::ROW { - return Err(SQLiteError::from(ResultCode::SCHEMA)); - } - Ok(statement) -} - pub fn deserialize_string_to_i64<'de, D>(deserializer: D) -> Result where D: serde::Deserializer<'de>, diff --git a/crates/core/src/uuid.rs b/crates/core/src/uuid.rs index dd797c4..db617f9 100644 --- a/crates/core/src/uuid.rs +++ b/crates/core/src/uuid.rs @@ -4,7 +4,6 @@ use alloc::format; use alloc::string::String; use alloc::string::ToString; use core::ffi::c_int; -use core::slice; use sqlite::ResultCode; use sqlite_nostd as sqlite; @@ -14,7 +13,6 @@ use crate::create_sqlite_text_fn; use crate::error::SQLiteError; use crate::util::*; - fn uuid_v4_impl( _ctx: *mut sqlite::context, _args: &[*mut sqlite::value], diff --git a/crates/core/src/version.rs b/crates/core/src/version.rs index e9bdf76..6a39ad3 100644 --- a/crates/core/src/version.rs +++ b/crates/core/src/version.rs @@ -3,7 +3,6 @@ extern crate alloc; use alloc::format; use alloc::string::String; use core::ffi::c_int; -use core::slice; use sqlite::ResultCode; use sqlite_nostd as sqlite; @@ -22,7 +21,11 @@ fn powersync_rs_version_impl( Ok(version) } -create_sqlite_text_fn!(powersync_rs_version, powersync_rs_version_impl, "powersync_rs_version"); +create_sqlite_text_fn!( + powersync_rs_version, + powersync_rs_version_impl, + "powersync_rs_version" +); pub fn register(db: *mut sqlite::sqlite3) -> Result<(), ResultCode> { db.create_function_v2( diff --git a/crates/core/src/view_admin.rs b/crates/core/src/view_admin.rs index 16f7a7f..dd110ad 100644 --- a/crates/core/src/view_admin.rs +++ b/crates/core/src/view_admin.rs @@ -4,7 +4,6 @@ use alloc::format; use alloc::string::{String, ToString}; use alloc::vec::Vec; use core::ffi::c_int; -use core::slice; use sqlite::{ResultCode, Value}; use sqlite_nostd as sqlite; @@ -75,7 +74,7 @@ fn powersync_internal_table_name_impl( } let name = stmt1.column_text(0)?; - let local_only = stmt1.column_int(1)? != 0; + let local_only = stmt1.column_int(1) != 0; if local_only { Ok(format!("ps_data_local__{:}", name)) diff --git a/crates/core/src/views.rs b/crates/core/src/views.rs index f196be7..d4941db 100644 --- a/crates/core/src/views.rs +++ b/crates/core/src/views.rs @@ -1,16 +1,19 @@ extern crate alloc; +use alloc::borrow::Cow; use alloc::format; use alloc::string::String; use alloc::vec::Vec; use core::ffi::c_int; -use core::slice; +use core::fmt::Write; +use streaming_iterator::StreamingIterator; use sqlite::{Connection, Context, ResultCode, Value}; -use sqlite_nostd::{self as sqlite, ManagedStmt}; +use sqlite_nostd::{self as sqlite}; use crate::create_sqlite_text_fn; -use crate::error::{PSResult, SQLiteError}; +use crate::error::SQLiteError; +use crate::schema::{ColumnInfo, ColumnNameAndTypeStatement, DiffIncludeOld, TableInfo}; use crate::util::*; fn powersync_view_sql_impl( @@ -19,26 +22,25 @@ fn powersync_view_sql_impl( ) -> Result { let db = ctx.db_handle(); let table = args[0].text(); - let statement = extract_table_info(db, table)?; + let table_info = TableInfo::parse_from(db, table)?; - let name = statement.column_text(0)?; - let view_name = statement.column_text(1)?; - let local_only = statement.column_int(2)? != 0; - let include_metadata = statement.column_int(5)? != 0; + let name = &table_info.name; + let view_name = &table_info.view_name; + let local_only = table_info.flags.local_only(); + let include_metadata = table_info.flags.include_metadata(); let quoted_name = quote_identifier(view_name); let internal_name = quote_internal_name(name, local_only); - let stmt2 = db.prepare_v2("select json_extract(e.value, '$.name') as name, json_extract(e.value, '$.type') as type from json_each(json_extract(?, '$.columns')) e")?; - stmt2.bind_text(1, table, sqlite::Destructor::STATIC)?; + let mut columns = ColumnNameAndTypeStatement::new(db, table)?; + let mut iter = columns.streaming_iter(); let mut column_names_quoted: Vec = alloc::vec![]; let mut column_values: Vec = alloc::vec![]; column_names_quoted.push(quote_identifier("id")); column_values.push(String::from("id")); - while stmt2.step()? == ResultCode::ROW { - let name = stmt2.column_text(0)?; - let type_name = stmt2.column_text(1)?; + while let Some(row) = iter.next() { + let ColumnInfo { name, type_name } = row.clone()?; column_names_quoted.push(quote_identifier(name)); let foo = format!( @@ -52,6 +54,9 @@ fn powersync_view_sql_impl( if include_metadata { column_names_quoted.push(quote_identifier("_metadata")); column_values.push(String::from("NULL")); + + column_names_quoted.push(quote_identifier("_deleted")); + column_values.push(String::from("NULL")); } let view_statement = format!( @@ -76,43 +81,87 @@ fn powersync_trigger_delete_sql_impl( args: &[*mut sqlite::value], ) -> Result { let table = args[0].text(); - let statement = extract_table_info(ctx.db_handle(), table)?; + let table_info = TableInfo::parse_from(ctx.db_handle(), table)?; - let name = statement.column_text(0)?; - let view_name = statement.column_text(1)?; - let local_only = statement.column_int(2)? != 0; - let insert_only = statement.column_int(3)? != 0; + let name = &table_info.name; + let view_name = &table_info.view_name; + let local_only = table_info.flags.local_only(); + let insert_only = table_info.flags.insert_only(); let quoted_name = quote_identifier(view_name); let internal_name = quote_internal_name(name, local_only); let trigger_name = quote_identifier_prefixed("ps_view_delete_", view_name); let type_string = quote_string(name); + let db = ctx.db_handle(); + let old_fragment: Cow<'static, str> = match table_info.diff_include_old { + Some(include_old) => { + let mut columns = ColumnNameAndTypeStatement::new(db, table)?; + + let json = match include_old { + DiffIncludeOld::OnlyForColumns { columns } => { + let mut iterator = columns.iter(); + let mut columns = + streaming_iterator::from_fn(|| -> Option> { + Some(Ok(iterator.next()?.as_str())) + }); + + json_object_fragment("OLD", &mut columns) + } + DiffIncludeOld::ForAllColumns => { + json_object_fragment("OLD", &mut columns.names_iter()) + } + }?; + + format!(", 'old', {json}").into() + } + None => "".into(), + }; + return if !local_only && !insert_only { - let trigger = format!( + let mut trigger = format!( "\ -CREATE TRIGGER {:} -INSTEAD OF DELETE ON {:} +CREATE TRIGGER {trigger_name} +INSTEAD OF DELETE ON {quoted_name} FOR EACH ROW BEGIN -DELETE FROM {:} WHERE id = OLD.id; -INSERT INTO powersync_crud_(data) VALUES(json_object('op', 'DELETE', 'type', {:}, 'id', OLD.id)); -INSERT OR IGNORE INTO ps_updated_rows(row_type, row_id) VALUES({:}, OLD.id); -INSERT OR REPLACE INTO ps_buckets(name, last_op, target_op) VALUES('$local', 0, {:}); -END", - trigger_name, quoted_name, internal_name, type_string, type_string, MAX_OP_ID +DELETE FROM {internal_name} WHERE id = OLD.id; +INSERT INTO powersync_crud_(data) VALUES(json_object('op', 'DELETE', 'type', {type_string}, 'id', OLD.id{old_fragment})); +INSERT OR IGNORE INTO ps_updated_rows(row_type, row_id) VALUES({type_string}, OLD.id); +INSERT OR REPLACE INTO ps_buckets(name, last_op, target_op) VALUES('$local', 0, {MAX_OP_ID}); +END;" ); + + // The DELETE statement can't include metadata for the delete operation, so we create + // another trigger to delete with a fake UPDATE syntax. + if table_info.flags.include_metadata() { + let trigger_name = quote_identifier_prefixed("ps_view_delete2_", view_name); + write!(&mut trigger, "\ +CREATE TRIGGER {trigger_name} +INSTEAD OF UPDATE ON {quoted_name} +FOR EACH ROW +WHEN NEW._deleted IS TRUE +BEGIN +DELETE FROM {internal_name} WHERE id = NEW.id; +INSERT INTO powersync_crud_(data) VALUES(json_object('op', 'DELETE', 'type', {type_string}, 'id', NEW.id{old_fragment}, 'metadata', NEW._metadata)); +INSERT OR IGNORE INTO ps_updated_rows(row_type, row_id) VALUES({type_string}, NEW.id); +INSERT OR REPLACE INTO ps_buckets(name, last_op, target_op) VALUES('$local', 0, {MAX_OP_ID}); +END;" + ).expect("writing to string should be infallible"); + } + Ok(trigger) } else if local_only { + debug_assert!(!table_info.flags.include_metadata()); + let trigger = format!( "\ -CREATE TRIGGER {:} -INSTEAD OF DELETE ON {:} +CREATE TRIGGER {trigger_name} +INSTEAD OF DELETE ON {quoted_name} FOR EACH ROW BEGIN -DELETE FROM {:} WHERE id = OLD.id; +DELETE FROM {internal_name} WHERE id = OLD.id; END", - trigger_name, quoted_name, internal_name ); Ok(trigger) } else if insert_only { @@ -134,12 +183,12 @@ fn powersync_trigger_insert_sql_impl( ) -> Result { let table = args[0].text(); - let statement = extract_table_info(ctx.db_handle(), table)?; + let table_info = TableInfo::parse_from(ctx.db_handle(), table)?; - let name = statement.column_text(0)?; - let view_name = statement.column_text(1)?; - let local_only = statement.column_int(2)? != 0; - let insert_only = statement.column_int(3)? != 0; + let name = &table_info.name; + let view_name = &table_info.view_name; + let local_only = table_info.flags.local_only(); + let insert_only = table_info.flags.insert_only(); let quoted_name = quote_identifier(view_name); let internal_name = quote_internal_name(name, local_only); @@ -147,47 +196,52 @@ fn powersync_trigger_insert_sql_impl( let type_string = quote_string(name); let local_db = ctx.db_handle(); - let stmt2 = local_db.prepare_v2("select json_extract(e.value, '$.name') as name from json_each(json_extract(?, '$.columns')) e")?; - stmt2.bind_text(1, table, sqlite::Destructor::STATIC)?; - let json_fragment = json_object_fragment("NEW", &stmt2)?; + + let mut columns = ColumnNameAndTypeStatement::new(local_db, table)?; + let json_fragment = json_object_fragment("NEW", &mut columns.names_iter())?; + + let metadata_fragment = if table_info.flags.include_metadata() { + ", 'metadata', NEW._metadata" + } else { + "" + }; return if !local_only && !insert_only { let trigger = format!("\ - CREATE TRIGGER {:} - INSTEAD OF INSERT ON {:} + CREATE TRIGGER {trigger_name} + INSTEAD OF INSERT ON {quoted_name} FOR EACH ROW BEGIN SELECT CASE WHEN (NEW.id IS NULL) THEN RAISE (FAIL, 'id is required') END; - INSERT INTO {:} - SELECT NEW.id, {:}; - INSERT INTO powersync_crud_(data) VALUES(json_object('op', 'PUT', 'type', {:}, 'id', NEW.id, 'data', json(powersync_diff('{{}}', {:})))); - INSERT OR IGNORE INTO ps_updated_rows(row_type, row_id) VALUES({:}, NEW.id); - INSERT OR REPLACE INTO ps_buckets(name, last_op, target_op) VALUES('$local', 0, {:}); - END", trigger_name, quoted_name, internal_name, json_fragment, type_string, json_fragment, type_string, MAX_OP_ID); + INSERT INTO {internal_name} + SELECT NEW.id, {json_fragment}; + INSERT INTO powersync_crud_(data) VALUES(json_object('op', 'PUT', 'type', {:}, 'id', NEW.id, 'data', json(powersync_diff('{{}}', {:})){metadata_fragment})); + INSERT OR IGNORE INTO ps_updated_rows(row_type, row_id) VALUES({type_string}, NEW.id); + INSERT OR REPLACE INTO ps_buckets(name, last_op, target_op) VALUES('$local', 0, {MAX_OP_ID}); + END", type_string, json_fragment); Ok(trigger) } else if local_only { let trigger = format!( "\ - CREATE TRIGGER {:} - INSTEAD OF INSERT ON {:} + CREATE TRIGGER {trigger_name} + INSTEAD OF INSERT ON {quoted_name} FOR EACH ROW BEGIN - INSERT INTO {:} SELECT NEW.id, {:}; + INSERT INTO {internal_name} SELECT NEW.id, {json_fragment}; END", - trigger_name, quoted_name, internal_name, json_fragment ); Ok(trigger) } else if insert_only { let trigger = format!("\ - CREATE TRIGGER {:} - INSTEAD OF INSERT ON {:} + CREATE TRIGGER {trigger_name} + INSTEAD OF INSERT ON {quoted_name} FOR EACH ROW BEGIN INSERT INTO powersync_crud_(data) VALUES(json_object('op', 'PUT', 'type', {}, 'id', NEW.id, 'data', json(powersync_diff('{{}}', {:})))); - END", trigger_name, quoted_name, type_string, json_fragment); + END", type_string, json_fragment); Ok(trigger) } else { Err(SQLiteError::from(ResultCode::MISUSE)) @@ -206,15 +260,12 @@ fn powersync_trigger_update_sql_impl( ) -> Result { let table = args[0].text(); - let statement = extract_table_info(ctx.db_handle(), table)?; + let table_info = TableInfo::parse_from(ctx.db_handle(), table)?; - let name = statement.column_text(0)?; - let view_name = statement.column_text(1)?; - let local_only = statement.column_int(2)? != 0; - let insert_only = statement.column_int(3)? != 0; - // TODO: allow accepting a column list - let include_old = statement.column_type(4)? == sqlite::ColumnType::Text; - let include_metadata = statement.column_int(5)? != 0; + let name = &table_info.name; + let view_name = &table_info.view_name; + let insert_only = table_info.flags.insert_only(); + let local_only = table_info.flags.local_only(); let quoted_name = quote_identifier(view_name); let internal_name = quote_internal_name(name, local_only); @@ -222,60 +273,85 @@ fn powersync_trigger_update_sql_impl( let type_string = quote_string(name); let db = ctx.db_handle(); - let stmt2 = db.prepare_v2("select json_extract(e.value, '$.name') as name from json_each(json_extract(?, '$.columns')) e").into_db_result(db)?; - stmt2.bind_text(1, table, sqlite::Destructor::STATIC)?; - let json_fragment_new = json_object_fragment("NEW", &stmt2)?; - stmt2.reset()?; - let json_fragment_old = json_object_fragment("OLD", &stmt2)?; - let old_fragment: String; - let metadata_fragment: &str; - - if include_old { - old_fragment = format!(", 'old', {:}", json_fragment_old); - } else { - old_fragment = String::from(""); + let mut columns = ColumnNameAndTypeStatement::new(db, table)?; + let json_fragment_new = json_object_fragment("NEW", &mut columns.names_iter())?; + let json_fragment_old = json_object_fragment("OLD", &mut columns.names_iter())?; + + let mut old_values_fragment = match &table_info.diff_include_old { + None => None, + Some(DiffIncludeOld::ForAllColumns) => Some(json_fragment_old.clone()), + Some(DiffIncludeOld::OnlyForColumns { columns }) => { + let mut iterator = columns.iter(); + let mut columns = + streaming_iterator::from_fn(|| -> Option> { + Some(Ok(iterator.next()?.as_str())) + }); + + Some(json_object_fragment("OLD", &mut columns)?) + } + }; + + if table_info.flags.include_old_only_if_changed() { + // We're setting ignore_removed_columns here because values that are present in the new + // values but not in the old values should not lead to a null entry. + old_values_fragment = old_values_fragment + .map(|f| format!("json(powersync_diff({json_fragment_new}, {f}, TRUE))")); } - if include_metadata { - metadata_fragment = ", 'metadata', NEW._metadata"; + let old_fragment: Cow<'static, str> = match old_values_fragment { + Some(f) => format!(", 'old', {f}").into(), + None => "".into(), + }; + + let metadata_fragment = if table_info.flags.include_metadata() { + ", 'metadata', NEW._metadata" } else { - metadata_fragment = ""; - } + "" + }; return if !local_only && !insert_only { + // If we're supposed to include metadata, we support UPDATE ... SET _deleted = TRUE with + // another trigger (because there's no way to attach data to DELETE statements otherwise). + let when = if table_info.flags.include_metadata() { + " WHEN NEW._deleted IS NOT TRUE" + } else { + "" + }; + let trigger = format!("\ -CREATE TRIGGER {:} -INSTEAD OF UPDATE ON {:} -FOR EACH ROW +CREATE TRIGGER {trigger_name} +INSTEAD OF UPDATE ON {quoted_name} +FOR EACH ROW{when} BEGIN SELECT CASE WHEN (OLD.id != NEW.id) THEN RAISE (FAIL, 'Cannot update id') END; - UPDATE {:} - SET data = {:} + UPDATE {internal_name} + SET data = {json_fragment_new} WHERE id = NEW.id; INSERT INTO powersync_crud_(data) VALUES(json_object('op', 'PATCH', 'type', {:}, 'id', NEW.id, 'data', json(powersync_diff({:}, {:})){:}{:})); - INSERT OR IGNORE INTO ps_updated_rows(row_type, row_id) VALUES({:}, NEW.id); - INSERT OR REPLACE INTO ps_buckets(name, last_op, target_op) VALUES('$local', 0, {:}); -END", trigger_name, quoted_name, internal_name, json_fragment_new, type_string, json_fragment_old, json_fragment_new, old_fragment, metadata_fragment, type_string, MAX_OP_ID); + INSERT OR IGNORE INTO ps_updated_rows(row_type, row_id) VALUES({type_string}, NEW.id); + INSERT OR REPLACE INTO ps_buckets(name, last_op, target_op) VALUES('$local', 0, {MAX_OP_ID}); +END", type_string, json_fragment_old, json_fragment_new, old_fragment, metadata_fragment); Ok(trigger) } else if local_only { + debug_assert!(!table_info.flags.include_metadata()); + let trigger = format!( "\ -CREATE TRIGGER {:} -INSTEAD OF UPDATE ON {:} +CREATE TRIGGER {trigger_name} +INSTEAD OF UPDATE ON {quoted_name} FOR EACH ROW BEGIN SELECT CASE WHEN (OLD.id != NEW.id) THEN RAISE (FAIL, 'Cannot update id') END; - UPDATE {:} - SET data = {:} + UPDATE {internal_name} + SET data = {json_fragment_new} WHERE id = NEW.id; -END", - trigger_name, quoted_name, internal_name, json_fragment_new +END" ); Ok(trigger) } else if insert_only { @@ -342,15 +418,18 @@ pub fn register(db: *mut sqlite::sqlite3) -> Result<(), ResultCode> { /// Given a query returning column names, return a JSON object fragment for a trigger. /// /// Example output with prefix "NEW": "json_object('id', NEW.id, 'name', NEW.name, 'age', NEW.age)". -fn json_object_fragment(prefix: &str, name_results: &ManagedStmt) -> Result { +fn json_object_fragment<'a>( + prefix: &str, + name_results: &mut dyn StreamingIterator>, +) -> Result { // floor(SQLITE_MAX_FUNCTION_ARG / 2). // To keep databases portable, we use the default limit of 100 args for this, // and don't try to query the limit dynamically. const MAX_ARG_COUNT: usize = 50; let mut column_names_quoted: Vec = alloc::vec![]; - while name_results.step()? == ResultCode::ROW { - let name = name_results.column_text(0)?; + while let Some(row) = name_results.next() { + let name = (*row)?; let quoted: String = format!( "{:}, {:}.{:}", diff --git a/dart/test/crud_test.dart b/dart/test/crud_test.dart index 4ec7615..ff8475c 100644 --- a/dart/test/crud_test.dart +++ b/dart/test/crud_test.dart @@ -226,5 +226,252 @@ void main() { runCrudTestInsertOnly(numberOfColumns); }); } + + group('tracks previous values', () { + void createTable([Map options = const {}]) { + final tableSchema = { + 'tables': [ + { + 'name': 'test', + 'columns': [ + {'name': 'name', 'type': 'text'}, + {'name': 'name2', 'type': 'text'}, + ], + ...options, + } + ] + }; + + db.select('select powersync_init()'); + db.select( + 'select powersync_replace_schema(?)', [jsonEncode(tableSchema)]); + } + + group('for updates', () { + void insertThenUpdate() { + db + ..execute('insert into test (id, name, name2) values (?, ?, ?)', + ['id', 'name', 'name2']) + ..execute('delete from ps_crud') + ..execute('update test set name = name || ?', ['.']); + } + + test('is not tracked by default', () { + createTable(); + insertThenUpdate(); + + final [row] = db.select('select data from ps_crud'); + expect(jsonDecode(row[0] as String), isNot(contains('old'))); + }); + + test('can be disabled', () { + createTable({'include_old': false}); + insertThenUpdate(); + + final [row] = db.select('select data from ps_crud'); + expect(jsonDecode(row[0] as String), isNot(contains('old'))); + }); + + test('can be enabled for all columns', () { + createTable({'include_old': true}); + insertThenUpdate(); + + final [row] = db.select('select data from ps_crud'); + final op = jsonDecode(row[0] as String); + expect(op['data'], {'name': 'name.'}); + expect(op['old'], {'name': 'name', 'name2': 'name2'}); + }); + + test('can be enabled for some columns', () { + createTable({ + 'include_old': ['name'] + }); + insertThenUpdate(); + + final [row] = db.select('select data from ps_crud'); + final op = jsonDecode(row[0] as String); + expect(op['data'], {'name': 'name.'}); + expect(op['old'], {'name': 'name'}); + }); + + test('can track changed values only', () { + createTable({ + 'include_old': true, + 'include_old_only_when_changed': true, + }); + insertThenUpdate(); + + final [row] = db.select('select data from ps_crud'); + final op = jsonDecode(row[0] as String); + expect(op['data'], {'name': 'name.'}); + expect(op['old'], {'name': 'name'}); + }); + + test('combined column filter and only tracking changes', () { + createTable({ + 'include_old': ['name2'], + 'include_old_only_when_changed': true, + }); + insertThenUpdate(); + + final [row] = db.select('select data from ps_crud'); + final op = jsonDecode(row[0] as String); + expect(op['data'], {'name': 'name.'}); + expect(op['old'], {}); + }); + }); + + group('for deletes', () { + void insertThenDelete() { + db + ..execute('insert into test (id, name, name2) values (?, ?, ?)', + ['id', 'name', 'name2']) + ..execute('delete from ps_crud') + ..execute('delete from test'); + } + + test('is not tracked by default', () { + createTable(); + insertThenDelete(); + + final [row] = db.select('select data from ps_crud'); + expect(jsonDecode(row[0] as String), isNot(contains('old'))); + }); + + test('can be disabled', () { + createTable({'include_old': false}); + insertThenDelete(); + + final [row] = db.select('select data from ps_crud'); + expect(jsonDecode(row[0] as String), isNot(contains('old'))); + }); + + test('can be enabled for all columns', () { + createTable({'include_old': true}); + insertThenDelete(); + + final [row] = db.select('select data from ps_crud'); + final op = jsonDecode(row[0] as String); + expect(op['data'], null); + expect(op['old'], {'name': 'name', 'name2': 'name2'}); + }); + + test('can be enabled for some columns', () { + createTable({ + 'include_old': ['name'] + }); + insertThenDelete(); + + final [row] = db.select('select data from ps_crud'); + final op = jsonDecode(row[0] as String); + expect(op['data'], null); + expect(op['old'], {'name': 'name'}); + }); + }); + }); + + group('including metadata', () { + void createTable([Map options = const {}]) { + final tableSchema = { + 'tables': [ + { + 'name': 'test', + 'columns': [ + {'name': 'name', 'type': 'text'}, + ], + ...options, + } + ] + }; + + db.select('select powersync_init()'); + db.select( + 'select powersync_replace_schema(?)', [jsonEncode(tableSchema)]); + } + + test('is disabled by default', () { + createTable(); + expect( + () => db.execute( + 'INSERT INTO test (id, name, _metadata) VALUES (?, ?, ?)', + ['id', 'name', 'test insert'], + ), + throwsA(isA()), + ); + }); + + test('can be disabled', () { + createTable({'include_metadata': false}); + expect( + () => db.execute( + 'INSERT INTO test (id, name, _metadata) VALUES (?, ?, ?)', + ['id', 'name', 'test insert'], + ), + throwsA(isA()), + ); + }); + + test('supports insert statements', () { + createTable({'include_metadata': true}); + db.execute( + 'INSERT INTO test (id, name, _metadata) VALUES (?, ?, ?)', + ['id', 'name', 'test insert'], + ); + + final [row] = db.select('select data from ps_crud'); + final op = jsonDecode(row[0] as String); + expect(op['data'], {'name': 'name'}); + expect(op['metadata'], 'test insert'); + }); + + test('supports update statements', () { + createTable({'include_metadata': true}); + db.execute( + 'INSERT INTO test (id, name, _metadata) VALUES (?, ?, ?)', + ['id', 'name', 'test insert'], + ); + db.execute('delete from ps_crud;'); + db.execute( + 'update test set name = name || ?, _metadata = ?', ['.', 'update']); + + final [row] = db.select('select data from ps_crud'); + final op = jsonDecode(row[0] as String); + expect(op['data'], {'name': 'name.'}); + expect(op['metadata'], 'update'); + }); + + test('supports regular delete statements', () { + createTable({'include_metadata': true}); + db.execute( + 'INSERT INTO test (id, name, _metadata) VALUES (?, ?, ?)', + ['id', 'name', 'test insert'], + ); + db.execute('delete from ps_crud;'); + db.execute('delete from test'); + + final [row] = db.select('select data from ps_crud'); + final op = jsonDecode(row[0] as String); + expect(op['op'], 'DELETE'); + expect(op['metadata'], null); + }); + + test('supports deleting updates with metadata', () { + createTable({'include_metadata': true}); + db.execute( + 'INSERT INTO test (id, name, _metadata) VALUES (?, ?, ?)', + ['id', 'name', 'test insert'], + ); + db.execute('delete from ps_crud;'); + db.execute('update test set _deleted = TRUE, _metadata = ?', + ['custom delete']); + + expect(db.select('select * from test'), hasLength(0)); + + final [row] = db.select('select data from ps_crud'); + final op = jsonDecode(row[0] as String); + expect(op['op'], 'DELETE'); + expect(op['metadata'], 'custom delete'); + }); + }); }); } diff --git a/dart/test/utils/native_test_utils.dart b/dart/test/utils/native_test_utils.dart index bc8b637..0ccc408 100644 --- a/dart/test/utils/native_test_utils.dart +++ b/dart/test/utils/native_test_utils.dart @@ -14,7 +14,7 @@ CommonDatabase openTestDatabase() { return DynamicLibrary.open('libsqlite3.so.0'); }); sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.macOS, () { - return DynamicLibrary.open('libsqlite3.dylib'); + return DynamicLibrary.open('/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib'); }); var lib = DynamicLibrary.open(getLibraryForPlatform(path: libPath)); var extension = SqliteExtension.inLibrary(lib, 'sqlite3_powersync_init'); diff --git a/sqlite-rs-embedded b/sqlite-rs-embedded index b0ced62..5d35c28 160000 --- a/sqlite-rs-embedded +++ b/sqlite-rs-embedded @@ -1 +1 @@ -Subproject commit b0ced62a9dfe4d9eecfd7f1a22136dccf3d75198 +Subproject commit 5d35c2883d9889f01dee010223d94570c70039b7