From 443c4fc8df82ef6f302d4dfd9dba7a47b688d594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 14:41:13 +0200 Subject: [PATCH 01/51] First version that seems to work great --- Cargo.toml | 2 +- examples/all-types-heed3.rs | 111 +++++++++++++++ {heed/examples => examples}/all-types.rs | 0 {heed/examples => examples}/clear-database.rs | 0 {heed/examples => examples}/cursor-append.rs | 0 .../custom-comparator.rs | 0 {heed/examples => examples}/multi-env.rs | 0 {heed/examples => examples}/nested.rs | 0 {heed/examples => examples}/rmp-serde.rs | 0 heed-master3-proc-macro/Cargo.toml | 12 ++ heed-master3-proc-macro/src/lib.rs | 73 ++++++++++ heed/Cargo.toml | 27 ++++ heed/build.rs | 17 +++ heed/src/cursor.rs | 1 + heed/src/database.rs | 17 +++ heed/src/txn.rs | 8 ++ heed3/Cargo.toml | 128 ++++++++++++++++++ 17 files changed, 395 insertions(+), 1 deletion(-) create mode 100644 examples/all-types-heed3.rs rename {heed/examples => examples}/all-types.rs (100%) rename {heed/examples => examples}/clear-database.rs (100%) rename {heed/examples => examples}/cursor-append.rs (100%) rename {heed/examples => examples}/custom-comparator.rs (100%) rename {heed/examples => examples}/multi-env.rs (100%) rename {heed/examples => examples}/nested.rs (100%) rename {heed/examples => examples}/rmp-serde.rs (100%) create mode 100644 heed-master3-proc-macro/Cargo.toml create mode 100644 heed-master3-proc-macro/src/lib.rs create mode 100644 heed/build.rs create mode 100644 heed3/Cargo.toml diff --git a/Cargo.toml b/Cargo.toml index b860d3cb..e780b535 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["lmdb-master-sys", "heed", "heed-traits", "heed-types"] +members = ["lmdb-master-sys", "heed", "heed-traits", "heed-types", "heed-master3-proc-macro"] resolver = "2" diff --git a/examples/all-types-heed3.rs b/examples/all-types-heed3.rs new file mode 100644 index 00000000..7ec698df --- /dev/null +++ b/examples/all-types-heed3.rs @@ -0,0 +1,111 @@ +use std::error::Error; +use std::fs; +use std::path::Path; + +use heed3::byteorder::BE; +use heed3::types::*; +use heed3::{Database, EnvOpenOptions}; +use serde::{Deserialize, Serialize}; + +fn main() -> Result<(), Box> { + let path = Path::new("target").join("heed.mdb"); + + fs::create_dir_all(&path)?; + + let env = unsafe { + EnvOpenOptions::new() + .map_size(10 * 1024 * 1024) // 10MB + .max_dbs(3000) + .open(path)? + }; + + // here the key will be an str and the data will be a slice of u8 + let mut wtxn = env.write_txn()?; + let db: Database = env.create_database(&mut wtxn, Some("kiki"))?; + + db.put(&mut wtxn, "hello", &[2, 3][..])?; + let ret: Option<&[u8]> = db.get(&mut wtxn, "hello")?; + + println!("{:?}", ret); + wtxn.commit()?; + + // serde types are also supported!!! + #[derive(Debug, Serialize, Deserialize)] + struct Hello<'a> { + string: &'a str, + } + + let mut wtxn = env.write_txn()?; + let db: Database> = + env.create_database(&mut wtxn, Some("serde-bincode"))?; + + let hello = Hello { string: "hi" }; + db.put(&mut wtxn, "hello", &hello)?; + + let ret: Option = db.get(&mut wtxn, "hello")?; + println!("serde-bincode:\t{:?}", ret); + + wtxn.commit()?; + + let mut wtxn = env.write_txn()?; + let db: Database> = env.create_database(&mut wtxn, Some("serde-json"))?; + + let hello = Hello { string: "hi" }; + db.put(&mut wtxn, "hello", &hello)?; + + let ret: Option = db.get(&mut wtxn, "hello")?; + println!("serde-json:\t{:?}", ret); + + wtxn.commit()?; + + // you can ignore the data + let mut wtxn = env.write_txn()?; + let db: Database = env.create_database(&mut wtxn, Some("ignored-data"))?; + + db.put(&mut wtxn, "hello", &())?; + let ret: Option<()> = db.get(&mut wtxn, "hello")?; + + println!("{:?}", ret); + + let ret: Option<()> = db.get(&mut wtxn, "non-existant")?; + + println!("{:?}", ret); + wtxn.commit()?; + + // database opening and types are tested in a safe way + // + // we try to open a database twice with the same types + let mut wtxn = env.write_txn()?; + let _db: Database = env.create_database(&mut wtxn, Some("ignored-data"))?; + + // you can iterate over keys in order + type BEI64 = I64; + + let db: Database = env.create_database(&mut wtxn, Some("big-endian-iter"))?; + + db.put(&mut wtxn, &0, &())?; + db.put(&mut wtxn, &68, &())?; + db.put(&mut wtxn, &35, &())?; + db.put(&mut wtxn, &42, &())?; + + let rets: Result, _> = db.iter(&mut wtxn)?.collect(); + + println!("{:?}", rets); + + // or iterate over ranges too!!! + let range = 35..=42; + let rets: Result, _> = db.range(&mut wtxn, &range)?.collect(); + + println!("{:?}", rets); + + // delete a range of key + let range = 35..=42; + let deleted: usize = db.delete_range(&mut wtxn, &range)?; + + let rets: Result, _> = db.iter(&mut wtxn)?.collect(); + + println!("deleted: {:?}, {:?}", deleted, rets); + wtxn.commit()?; + + Ok(()) +} diff --git a/heed/examples/all-types.rs b/examples/all-types.rs similarity index 100% rename from heed/examples/all-types.rs rename to examples/all-types.rs diff --git a/heed/examples/clear-database.rs b/examples/clear-database.rs similarity index 100% rename from heed/examples/clear-database.rs rename to examples/clear-database.rs diff --git a/heed/examples/cursor-append.rs b/examples/cursor-append.rs similarity index 100% rename from heed/examples/cursor-append.rs rename to examples/cursor-append.rs diff --git a/heed/examples/custom-comparator.rs b/examples/custom-comparator.rs similarity index 100% rename from heed/examples/custom-comparator.rs rename to examples/custom-comparator.rs diff --git a/heed/examples/multi-env.rs b/examples/multi-env.rs similarity index 100% rename from heed/examples/multi-env.rs rename to examples/multi-env.rs diff --git a/heed/examples/nested.rs b/examples/nested.rs similarity index 100% rename from heed/examples/nested.rs rename to examples/nested.rs diff --git a/heed/examples/rmp-serde.rs b/examples/rmp-serde.rs similarity index 100% rename from heed/examples/rmp-serde.rs rename to examples/rmp-serde.rs diff --git a/heed-master3-proc-macro/Cargo.toml b/heed-master3-proc-macro/Cargo.toml new file mode 100644 index 00000000..e8d7a76a --- /dev/null +++ b/heed-master3-proc-macro/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "heed-master3-proc-macro" +version = "0.1.0" +edition = "2021" + +[dependencies] +syn = { version = "1.0", features = ["full"] } +quote = "1.0" +proc-macro2 = "1.0" + +[lib] +proc-macro = true diff --git a/heed-master3-proc-macro/src/lib.rs b/heed-master3-proc-macro/src/lib.rs new file mode 100644 index 00000000..e29019f2 --- /dev/null +++ b/heed-master3-proc-macro/src/lib.rs @@ -0,0 +1,73 @@ +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, parse_quote, FnArg, Ident, ItemFn, Pat, PatType, Type}; + +#[proc_macro_attribute] +pub fn mut_read_txn(attr: TokenStream, item: TokenStream) -> TokenStream { + // Parse the function + let mut input_fn = parse_macro_input!(item as ItemFn); + + // Get the parameter name from the attribute + let param_name = parse_macro_input!(attr as Ident); + + // Flag to check if we found and modified the parameter + let mut found_and_modified = false; + + // Iterate through the function arguments + for arg in &mut input_fn.sig.inputs { + if let FnArg::Typed(PatType { pat, ty, .. }) = arg { + // Check if this is the parameter we want to modify + if let Pat::Ident(pat_ident) = pat.as_ref() { + if pat_ident.ident == param_name { + if let Type::Reference(type_reference) = ty.as_mut() { + if let Type::Path(type_path) = type_reference.elem.as_mut() { + if let Some(segment) = type_path.path.segments.last() { + if segment.ident == "RoTxn" { + // Check if it's non-mutable + if type_reference.mutability.is_none() { + // Add the `mut` keyword + type_reference.mutability = Some(parse_quote!(mut)); + found_and_modified = true; + } else { + // If it's already mutable, return an error + return syn::Error::new_spanned( + type_reference, + "The specified parameter is already mutable", + ) + .to_compile_error() + .into(); + } + } else { + // If it's not RoTxn, return an error + return syn::Error::new_spanned( + type_path, + "The specified parameter is not of type RoTxn", + ) + .to_compile_error() + .into(); + } + } + } + } + } + } + } + } + + // If we didn't find and modify the parameter, return an error + if !found_and_modified { + return syn::Error::new_spanned( + input_fn.sig, + format!("Could not find non-mutable parameter '{}' of type RoTxn", param_name), + ) + .to_compile_error() + .into(); + } + + // Generate the modified function + let output = quote! { + #input_fn + }; + + output.into() +} diff --git a/heed/Cargo.toml b/heed/Cargo.toml index f0df6524..967510e6 100644 --- a/heed/Cargo.toml +++ b/heed/Cargo.toml @@ -110,6 +110,33 @@ mdb_idl_logn_16 = ["lmdb-master-sys/mdb_idl_logn_16"] # computers then you need to keep your keys within the smaller 1982 byte limit. longer-keys = ["lmdb-master-sys/longer-keys"] +# Examples are located outside the standard heed/examples directory to prevent +# conflicts between heed3 and heed examples when working on both crates. +[[example]] +name = "all-types" +path = "../examples/all-types.rs" + +[[example]] +name = "clear-database" +path = "../examples/clear-database.rs" + +[[example]] +name = "cursor-append" +path = "../examples/cursor-append.rs" + +[[example]] +name = "custom-comparator" +path = "../examples/custom-comparator.rs" + +[[example]] +name = "multi-env" +path = "../examples/multi-env.rs" + +[[example]] +name = "nested" +path = "../examples/nested.rs" + [[example]] name = "rmp-serde" +path = "../examples/rmp-serde.rs" required-features = ["serde-rmp"] diff --git a/heed/build.rs b/heed/build.rs new file mode 100644 index 00000000..b85c1b9d --- /dev/null +++ b/heed/build.rs @@ -0,0 +1,17 @@ +use std::env; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo::rustc-check-cfg=cfg(master3)"); + // if let Some(channel) = version_check::Channel::read() { + // if channel.supports_features() { + // println!("cargo:rustc-cfg=has_specialisation"); + // } + // } + let pkgname = env::var("CARGO_PKG_NAME").expect("Cargo didn't set the CARGO_PKG_NAME env var!"); + match pkgname.as_str() { + "heed3" => println!("cargo:rustc-cfg=master3"), + "heed" => (), + _ => panic!("unexpected package name!"), + } +} diff --git a/heed/src/cursor.rs b/heed/src/cursor.rs index d22afee6..d77508e5 100644 --- a/heed/src/cursor.rs +++ b/heed/src/cursor.rs @@ -11,6 +11,7 @@ pub struct RoCursor<'txn> { } impl<'txn> RoCursor<'txn> { + // TODO should I ask for a &mut RoTxn, here? pub(crate) fn new(txn: &'txn RoTxn, dbi: ffi::MDB_dbi) -> Result> { let mut cursor: *mut ffi::MDB_cursor = ptr::null_mut(); let mut txn = txn.txn.unwrap(); diff --git a/heed/src/database.rs b/heed/src/database.rs index 2696d5be..d4b2955c 100644 --- a/heed/src/database.rs +++ b/heed/src/database.rs @@ -339,6 +339,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn get<'a, 'txn>(&self, txn: &'txn RoTxn, key: &'a KC::EItem) -> Result> where KC: BytesEncode<'a>, @@ -419,6 +420,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_duplicates<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -482,6 +484,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_lower_than<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -551,6 +554,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_lower_than_or_equal_to<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -624,6 +628,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_greater_than<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -696,6 +701,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_greater_than_or_equal_to<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -755,6 +761,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn first<'txn>(&self, txn: &'txn RoTxn) -> Result> where KC: BytesDecode<'txn>, @@ -809,6 +816,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn last<'txn>(&self, txn: &'txn RoTxn) -> Result> where KC: BytesDecode<'txn>, @@ -866,6 +874,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn len(&self, txn: &RoTxn) -> Result { self.stat(txn).map(|stat| stat.entries as u64) } @@ -909,6 +918,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn is_empty(&self, txn: &RoTxn) -> Result { self.len(txn).map(|l| l == 0) } @@ -951,6 +961,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn stat(&self, txn: &RoTxn) -> Result { assert_eq_env_db_txn!(self, txn); @@ -1015,6 +1026,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn iter<'txn>(&self, txn: &'txn RoTxn) -> Result> { assert_eq_env_db_txn!(self, txn); RoCursor::new(txn, self.dbi).map(|cursor| RoIter::new(cursor)) @@ -1116,6 +1128,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn rev_iter<'txn>(&self, txn: &'txn RoTxn) -> Result> { assert_eq_env_db_txn!(self, txn); @@ -1222,6 +1235,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn range<'a, 'txn, R>( &self, txn: &'txn RoTxn, @@ -1394,6 +1408,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn rev_range<'a, 'txn, R>( &self, txn: &'txn RoTxn, @@ -1568,6 +1583,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn prefix_iter<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -1700,6 +1716,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` + #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] pub fn rev_prefix_iter<'a, 'txn>( &self, txn: &'txn RoTxn, diff --git a/heed/src/txn.rs b/heed/src/txn.rs index 327ab0fd..5d648de6 100644 --- a/heed/src/txn.rs +++ b/heed/src/txn.rs @@ -207,3 +207,11 @@ impl<'p> Deref for RwTxn<'p> { &self.txn } } + +// TODO can't we just always implement it? +#[cfg(all(master3, feature = "encryption"))] +impl<'p> std::ops::DerefMut for RwTxn<'p> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.txn + } +} diff --git a/heed3/Cargo.toml b/heed3/Cargo.toml new file mode 100644 index 00000000..a17505f3 --- /dev/null +++ b/heed3/Cargo.toml @@ -0,0 +1,128 @@ +[package] +name = "heed3" +version = "0.20.5" +authors = ["Kerollmops "] +description = "A fully typed LMDB wrapper with minimum overhead and optional support for encryption" +license = "MIT" +repository = "https://github.com/Kerollmops/heed" +keywords = ["lmdb", "database", "storage", "typed", "encryption"] +categories = ["database", "data-structures"] +readme = "../README.md" +edition = "2021" + +[dependencies] +bitflags = { version = "2.6.0", features = ["serde"] } +byteorder = { version = "1.5.0", default-features = false } +heed-traits = { version = "0.20.0", path = "../heed-traits" } +heed-types = { version = "0.20.1", default-features = false, path = "../heed-types" } +libc = "0.2.155" +# TODO depend on lmdb-master3-sys +lmdb-master-sys = { version = "0.2.4", path = "../lmdb-master-sys" } +once_cell = "1.19.0" +page_size = "0.6.0" +serde = { version = "1.0.203", features = ["derive"], optional = true } +synchronoise = "1.0.1" +# TODO find a better name, depend on that only with heed3 +# move the proc macro directly in the heed3 crate (not possible?) +# Set a cfg to react on the crate name: https://github.com/bodil/im-rs/blob/master/build.rs +heed-master3-proc-macro = { path = "../heed-master3-proc-macro", optional = true } + +[dev-dependencies] +serde = { version = "1.0.203", features = ["derive"] } +tempfile = "3.10.1" + +[target.'cfg(windows)'.dependencies] +url = "2.5.2" + +[features] +# The `serde` feature makes some types serializable, +# like the `EnvOpenOptions` struct. +# TODO remove encryption from defaults +default = ["serde", "serde-bincode", "serde-json", "encryption"] + +# Enable the LMDB encryption feature +# TODO add more information here +encryption = ["dep:heed-master3-proc-macro"] + +serde = ["bitflags/serde", "dep:serde"] + +# The #MDB_NOTLS flag is automatically set on Env opening, +# RoTxn and RoCursors implements the Send trait. This allows the +# user to move RoTxns and RoCursors between threads as read transactions +# will no more use thread local storage and will tie reader locktable +# slots to #MDB_txn objects instead of to threads. +# +# According to the LMDB documentation, when this feature is not enabled: +# A thread can only use one transaction at a time, plus any child +# transactions. Each transaction belongs to one thread. [...] +# The #MDB_NOTLS flag changes this for read-only transactions. +# +# And a #MDB_BAD_RSLOT error will be thrown when multiple read +# transactions exists on the same thread +read-txn-no-tls = [] + +# Enable the serde en/decoders for bincode, serde_json, or rmp_serde +serde-bincode = ["heed-types/serde-bincode"] +serde-json = ["heed-types/serde-json"] +serde-rmp = ["heed-types/serde-rmp"] + +# serde_json features +preserve_order = ["heed-types/preserve_order"] +arbitrary_precision = ["heed-types/arbitrary_precision"] +raw_value = ["heed-types/raw_value"] +unbounded_depth = ["heed-types/unbounded_depth"] + +# Whether to tell LMDB to use POSIX semaphores during compilation +# (instead of the default, which are System V semaphores). +# POSIX semaphores are required for Apple's App Sandbox on iOS & macOS, +# and are possibly faster and more appropriate for single-process use. +# There are tradeoffs for both POSIX and SysV semaphores; which you +# should look into before enabling this feature. Also, see here: +# +posix-sem = ["lmdb-master-sys/posix-sem"] + +# These features configure the MDB_IDL_LOGN macro, which determines +# the size of the free and dirty page lists (and thus the amount of memory +# allocated when opening an LMDB environment in read-write mode). +# +# Each feature defines MDB_IDL_LOGN as the value in the name of the feature. +# That means these features are mutually exclusive, and you must not specify +# more than one at the same time (or the crate will fail to compile). +# +# For more information on the motivation for these features (and their effect), +# see https://github.com/mozilla/lmdb/pull/2. +mdb_idl_logn_8 = ["lmdb-master-sys/mdb_idl_logn_8"] +mdb_idl_logn_9 = ["lmdb-master-sys/mdb_idl_logn_9"] +mdb_idl_logn_10 = ["lmdb-master-sys/mdb_idl_logn_10"] +mdb_idl_logn_11 = ["lmdb-master-sys/mdb_idl_logn_11"] +mdb_idl_logn_12 = ["lmdb-master-sys/mdb_idl_logn_12"] +mdb_idl_logn_13 = ["lmdb-master-sys/mdb_idl_logn_13"] +mdb_idl_logn_14 = ["lmdb-master-sys/mdb_idl_logn_14"] +mdb_idl_logn_15 = ["lmdb-master-sys/mdb_idl_logn_15"] +mdb_idl_logn_16 = ["lmdb-master-sys/mdb_idl_logn_16"] + +# Setting this enables you to use keys longer than 511 bytes. The exact limit +# is computed by LMDB at compile time. You can find the exact value by calling +# Env::max_key_size(). This value varies by architecture. +# +# Example max key sizes: +# - Apple M1 (ARM64): 8126 bytes +# - Apple Intel (AMD64): 1982 bytes +# - Linux Intel (AMD64): 1982 bytes +# +# Setting this also enables you to use values larger than 511 bytes when using +# a Database with the DatabaseFlags::DUP_SORT flag. +# +# This builds LMDB with the -DMDB_MAXKEYSIZE=0 option. +# +# Note: If you are moving database files between architectures then your longest +# stored key must fit within the smallest limit of all architectures used. For +# example, if you are moving databases between Apple M1 and Apple Intel +# computers then you need to keep your keys within the smaller 1982 byte limit. +longer-keys = ["lmdb-master-sys/longer-keys"] + +# Examples are located outside the standard heed/examples directory to prevent +# conflicts between heed3 and heed examples when working on both crates. +[[example]] +name = "all-types-heed3" +path = "../examples/all-types-heed3.rs" From 96482426673cdaadce340db921b1b404cb1609f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 14:55:33 +0200 Subject: [PATCH 02/51] Ignore the encryption feature when using heed --- heed/build.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/heed/build.rs b/heed/build.rs index b85c1b9d..0835bfae 100644 --- a/heed/build.rs +++ b/heed/build.rs @@ -11,7 +11,8 @@ fn main() { let pkgname = env::var("CARGO_PKG_NAME").expect("Cargo didn't set the CARGO_PKG_NAME env var!"); match pkgname.as_str() { "heed3" => println!("cargo:rustc-cfg=master3"), - "heed" => (), + // Ignore the absence of the encryption feature when not using heed3 + "heed" => println!("cargo::rustc-check-cfg=cfg(feature, values(\"encryption\"))"), _ => panic!("unexpected package name!"), } } From 17283846d27610ddeaec941d4ee1af3eab849e4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 15:15:07 +0200 Subject: [PATCH 03/51] Check the heed3 crate in the CI --- .github/workflows/rust.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a990c879..fd16ada0 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -28,6 +28,32 @@ jobs: cargo clean cargo test + check: + name: Check the heed3 project + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + include: + - os: ubuntu-latest + - os: windows-latest + - os: macos-latest + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - name: Run cargo check + run: | + cargo clean + cp heed3/Cargo.toml heed/ + cargo check -p heed3 + check_all_features: name: Check all the features of the heed project runs-on: ${{ matrix.os }} From a0c57ed48fdd586ff7d1c67186af47bc6e498eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 15:26:53 +0200 Subject: [PATCH 04/51] Run all the heed3 examples in the CI --- .github/workflows/rust.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index fd16ada0..cca1bbfe 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -107,6 +107,31 @@ jobs: cargo run --example 2>&1 | grep -E '^ ' | awk '!/rmp-serde/' | xargs -n1 cargo run --example cargo run --example rmp-serde --features serde-rmp + heed3-examples: + name: Run the heed3 examples + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + include: + - os: ubuntu-latest + - os: macos-latest + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - name: Run the examples + run: | + cargo clean + cp heed3/Cargo.toml heed/ + cargo run --example 2>&1 | grep -E '^ '| xargs -n1 cargo run --example + fmt: name: Ensure the heed project is formatted runs-on: ubuntu-latest From e4ba9790e7857d73a1ba9c0cf02c17b874a2d107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 15:41:31 +0200 Subject: [PATCH 05/51] Introduce the lmdb-master3-sys crate based on the mdb.master3 branch --- .gitmodules | 4 + Cargo.toml | 2 +- heed3/Cargo.toml | 3 +- lmdb-master3-sys/.rustfmt.toml | 3 + lmdb-master3-sys/Cargo.toml | 82 +++++ lmdb-master3-sys/README.md | 3 + lmdb-master3-sys/bindgen.rs | 76 +++++ lmdb-master3-sys/build.rs | 155 +++++++++ lmdb-master3-sys/lmdb | 1 + lmdb-master3-sys/src/bindings.rs | 552 +++++++++++++++++++++++++++++++ lmdb-master3-sys/src/lib.rs | 24 ++ lmdb-master3-sys/tests/lmdb.rs | 26 ++ lmdb-master3-sys/tests/simple.rs | 95 ++++++ 13 files changed, 1023 insertions(+), 3 deletions(-) create mode 100644 lmdb-master3-sys/.rustfmt.toml create mode 100644 lmdb-master3-sys/Cargo.toml create mode 100644 lmdb-master3-sys/README.md create mode 100644 lmdb-master3-sys/bindgen.rs create mode 100644 lmdb-master3-sys/build.rs create mode 160000 lmdb-master3-sys/lmdb create mode 100644 lmdb-master3-sys/src/bindings.rs create mode 100644 lmdb-master3-sys/src/lib.rs create mode 100644 lmdb-master3-sys/tests/lmdb.rs create mode 100644 lmdb-master3-sys/tests/simple.rs diff --git a/.gitmodules b/.gitmodules index e946c61f..31da67d9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,7 @@ path = lmdb-master-sys/lmdb url = https://github.com/LMDB/lmdb branch = mdb.master +[submodule "lmdb-master3-sys/lmdb"] + path = lmdb-master3-sys/lmdb + url = https://github.com/LMDB/lmdb.git + branch = mdb.master3 diff --git a/Cargo.toml b/Cargo.toml index e780b535..68ccde2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["lmdb-master-sys", "heed", "heed-traits", "heed-types", "heed-master3-proc-macro"] +members = ["lmdb-master-sys", "lmdb-master3-sys", "heed", "heed-traits", "heed-types", "heed-master3-proc-macro"] resolver = "2" diff --git a/heed3/Cargo.toml b/heed3/Cargo.toml index a17505f3..a384af83 100644 --- a/heed3/Cargo.toml +++ b/heed3/Cargo.toml @@ -16,8 +16,7 @@ byteorder = { version = "1.5.0", default-features = false } heed-traits = { version = "0.20.0", path = "../heed-traits" } heed-types = { version = "0.20.1", default-features = false, path = "../heed-types" } libc = "0.2.155" -# TODO depend on lmdb-master3-sys -lmdb-master-sys = { version = "0.2.4", path = "../lmdb-master-sys" } +lmdb-master3-sys = { version = "0.2.4", path = "../lmdb-master3-sys" } once_cell = "1.19.0" page_size = "0.6.0" serde = { version = "1.0.203", features = ["derive"], optional = true } diff --git a/lmdb-master3-sys/.rustfmt.toml b/lmdb-master3-sys/.rustfmt.toml new file mode 100644 index 00000000..fc441bba --- /dev/null +++ b/lmdb-master3-sys/.rustfmt.toml @@ -0,0 +1,3 @@ +ignore = [ + "src/bindings.rs" +] \ No newline at end of file diff --git a/lmdb-master3-sys/Cargo.toml b/lmdb-master3-sys/Cargo.toml new file mode 100644 index 00000000..cabc2c7e --- /dev/null +++ b/lmdb-master3-sys/Cargo.toml @@ -0,0 +1,82 @@ +[package] +name = "lmdb-master3-sys" +# NB: When modifying, also modify html_root_url in lib.rs +version = "0.2.4" +authors = [ + "Kerollmops ", + "Dan Burkert ", + "Victor Porof ", +] +license = "Apache-2.0" +description = "Rust bindings for liblmdb on the mdb.master3 branch." +documentation = "https://docs.rs/lmdb-master3-sys" +repository = "https://github.com/meilisearch/heed/tree/main/lmdb-master3-sys" +readme = "README.md" +keywords = ["LMDB", "database", "storage-engine", "bindings", "library"] +categories = ["database", "external-ffi-bindings"] +edition = "2021" + +# NB: Use "--features bindgen" to generate bindings. +build = "build.rs" + +[lib] +name = "lmdb_master3_sys" +doctest = false + +[dependencies] +libc = "0.2.155" + +[build-dependencies] +bindgen = { version = "0.69.4", default-features = false, optional = true, features = ["runtime"] } +cc = "1.0.104" +doxygen-rs = "0.4.2" + +[dev-dependencies] +cstr = "0.2.12" + +[features] +default = [] +asan = [] +fuzzer = [] +fuzzer-no-link = [] +posix-sem = [] + +# These features configure the MDB_IDL_LOGN macro, which determines +# the size of the free and dirty page lists (and thus the amount of memory +# allocated when opening an LMDB environment in read-write mode). +# +# Each feature defines MDB_IDL_LOGN as the value in the name of the feature. +# That means these features are mutually exclusive, and you must not specify +# more than one at the same time (or the crate will fail to compile). +# +# For more information on the motivation for these features (and their effect), +# see https://github.com/mozilla/lmdb/pull/2. +mdb_idl_logn_8 = [] +mdb_idl_logn_9 = [] +mdb_idl_logn_10 = [] +mdb_idl_logn_11 = [] +mdb_idl_logn_12 = [] +mdb_idl_logn_13 = [] +mdb_idl_logn_14 = [] +mdb_idl_logn_15 = [] +mdb_idl_logn_16 = [] + +# Setting this enables you to use keys longer than 511 bytes. The exact limit +# is computed by LMDB at compile time. You can find the exact value by calling +# Env::max_key_size(). This value varies by architecture. +# +# Example max key sizes: +# - Apple M1 (ARM64): 8126 bytes +# - Apple Intel (AMD64): 1982 bytes +# - Linux Intel (AMD64): 1982 bytes +# +# Setting this also enables you to use values larger than 511 bytes when using +# a Database with the DatabaseFlags::DUP_SORT flag. +# +# This builds LMDB with the -DMDB_MAXKEYSIZE=0 option. +# +# Note: If you are moving database files between architectures then your longest +# stored key must fit within the smallest limit of all architectures used. For +# example, if you are moving databases between Apple M1 and Apple Intel +# computers then you need to keep your keys within the smaller 1982 byte limit. +longer-keys = [] diff --git a/lmdb-master3-sys/README.md b/lmdb-master3-sys/README.md new file mode 100644 index 00000000..7612705c --- /dev/null +++ b/lmdb-master3-sys/README.md @@ -0,0 +1,3 @@ +# lmdb-master-sys + +Rust bindings for liblmdb on the mdb.master branch. diff --git a/lmdb-master3-sys/bindgen.rs b/lmdb-master3-sys/bindgen.rs new file mode 100644 index 00000000..64b8339f --- /dev/null +++ b/lmdb-master3-sys/bindgen.rs @@ -0,0 +1,76 @@ +use std::env; +use std::path::PathBuf; + +use bindgen::callbacks::{IntKind, ParseCallbacks}; + +#[derive(Debug)] +struct Callbacks; + +impl ParseCallbacks for Callbacks { + fn process_comment(&self, comment: &str) -> Option { + Some(doxygen_rs::transform(comment)) + } + + fn int_macro(&self, name: &str, _value: i64) -> Option { + match name { + "MDB_SUCCESS" + | "MDB_KEYEXIST" + | "MDB_NOTFOUND" + | "MDB_PAGE_NOTFOUND" + | "MDB_CORRUPTED" + | "MDB_PANIC" + | "MDB_VERSION_MISMATCH" + | "MDB_INVALID" + | "MDB_MAP_FULL" + | "MDB_DBS_FULL" + | "MDB_READERS_FULL" + | "MDB_TLS_FULL" + | "MDB_TXN_FULL" + | "MDB_CURSOR_FULL" + | "MDB_PAGE_FULL" + | "MDB_MAP_RESIZED" + | "MDB_INCOMPATIBLE" + | "MDB_BAD_RSLOT" + | "MDB_BAD_TXN" + | "MDB_BAD_VALSIZE" + | "MDB_BAD_DBI" + | "MDB_LAST_ERRCODE" => Some(IntKind::Int), + "MDB_SIZE_MAX" => Some(IntKind::U64), + "MDB_PROBLEM" | "MDB_BAD_CHECKSUM" | "MDB_CRYPTO_FAIL" | "MDB_ENV_ENCRYPTION" => { + Some(IntKind::Int) + } + _ => Some(IntKind::UInt), + } + } +} + +pub fn generate() { + let mut lmdb = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); + lmdb.push("lmdb"); + lmdb.push("libraries"); + lmdb.push("liblmdb"); + + let mut out_path = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); + out_path.push("src"); + + let bindings = bindgen::Builder::default() + .header(lmdb.join("lmdb.h").to_string_lossy()) + .allowlist_var("^(MDB|mdb)_.*") + .allowlist_type("^(MDB|mdb)_.*") + .allowlist_function("^(MDB|mdb)_.*") + .size_t_is_usize(true) + .ctypes_prefix("::libc") + .blocklist_item("mode_t") + .blocklist_item("mdb_mode_t") + .blocklist_item("mdb_filehandle_t") + .blocklist_item("^__.*") + .parse_callbacks(Box::new(Callbacks {})) + .layout_tests(false) + .prepend_enum_name(false) + .generate() + .expect("Unable to generate bindings"); + + bindings + .write_to_file(out_path.join("bindings.rs")) + .expect("Couldn't write bindings!"); +} diff --git a/lmdb-master3-sys/build.rs b/lmdb-master3-sys/build.rs new file mode 100644 index 00000000..9bff3df5 --- /dev/null +++ b/lmdb-master3-sys/build.rs @@ -0,0 +1,155 @@ +extern crate cc; + +#[cfg(feature = "bindgen")] +extern crate bindgen; + +#[cfg(feature = "bindgen")] +#[path = "bindgen.rs"] +mod generate; + +use std::env; +use std::path::PathBuf; + +#[cfg(all( + feature = "mdb_idl_logn_8", + not(any( + feature = "mdb_idl_logn_9", + feature = "mdb_idl_logn_10", + feature = "mdb_idl_logn_11", + feature = "mdb_idl_logn_12", + feature = "mdb_idl_logn_13", + feature = "mdb_idl_logn_14", + feature = "mdb_idl_logn_15", + feature = "mdb_idl_logn_16" + )) +))] +const MDB_IDL_LOGN: u8 = 8; +#[cfg(all( + feature = "mdb_idl_logn_9", + not(any( + feature = "mdb_idl_logn_10", + feature = "mdb_idl_logn_11", + feature = "mdb_idl_logn_12", + feature = "mdb_idl_logn_13", + feature = "mdb_idl_logn_14", + feature = "mdb_idl_logn_15", + feature = "mdb_idl_logn_16" + )) +))] +const MDB_IDL_LOGN: u8 = 9; +#[cfg(all( + feature = "mdb_idl_logn_10", + not(any( + feature = "mdb_idl_logn_11", + feature = "mdb_idl_logn_12", + feature = "mdb_idl_logn_13", + feature = "mdb_idl_logn_14", + feature = "mdb_idl_logn_15", + feature = "mdb_idl_logn_16" + )) +))] +const MDB_IDL_LOGN: u8 = 10; +#[cfg(all( + feature = "mdb_idl_logn_11", + not(any( + feature = "mdb_idl_logn_12", + feature = "mdb_idl_logn_13", + feature = "mdb_idl_logn_14", + feature = "mdb_idl_logn_15", + feature = "mdb_idl_logn_16" + )) +))] +const MDB_IDL_LOGN: u8 = 11; +#[cfg(all( + feature = "mdb_idl_logn_12", + not(any( + feature = "mdb_idl_logn_13", + feature = "mdb_idl_logn_14", + feature = "mdb_idl_logn_15", + feature = "mdb_idl_logn_16" + )) +))] +const MDB_IDL_LOGN: u8 = 12; +#[cfg(all( + feature = "mdb_idl_logn_13", + not(any( + feature = "mdb_idl_logn_14", + feature = "mdb_idl_logn_15", + feature = "mdb_idl_logn_16" + )) +))] +const MDB_IDL_LOGN: u8 = 13; +#[cfg(all( + feature = "mdb_idl_logn_14", + not(any(feature = "mdb_idl_logn_15", feature = "mdb_idl_logn_16")) +))] +const MDB_IDL_LOGN: u8 = 14; +#[cfg(all(feature = "mdb_idl_logn_15", not(any(feature = "mdb_idl_logn_16"))))] +const MDB_IDL_LOGN: u8 = 15; +#[cfg(any( + feature = "mdb_idl_logn_16", + not(any( + feature = "mdb_idl_logn_8", + feature = "mdb_idl_logn_9", + feature = "mdb_idl_logn_10", + feature = "mdb_idl_logn_11", + feature = "mdb_idl_logn_12", + feature = "mdb_idl_logn_13", + feature = "mdb_idl_logn_14", + feature = "mdb_idl_logn_15", + feature = "mdb_idl_logn_16", + )) +))] +const MDB_IDL_LOGN: u8 = 16; + +macro_rules! warn { + ($message:expr) => { + println!("cargo:warning={}", $message); + }; +} + +fn main() { + #[cfg(feature = "bindgen")] + generate::generate(); + + let mut lmdb = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); + lmdb.push("lmdb"); + lmdb.push("libraries"); + lmdb.push("liblmdb"); + + if cfg!(feature = "fuzzer") && cfg!(feature = "fuzzer-no-link") { + warn!("Features `fuzzer` and `fuzzer-no-link` are mutually exclusive."); + warn!("Building with `-fsanitize=fuzzer`."); + } + + let mut builder = cc::Build::new(); + + builder + .define("MDB_IDL_LOGN", Some(MDB_IDL_LOGN.to_string().as_str())) + .file(lmdb.join("mdb.c")) + .file(lmdb.join("midl.c")) + // https://github.com/mozilla/lmdb/blob/b7df2cac50fb41e8bd16aab4cc5fd167be9e032a/libraries/liblmdb/Makefile#L23 + .flag_if_supported("-Wno-unused-parameter") + .flag_if_supported("-Wbad-function-cast") + .flag_if_supported("-Wuninitialized"); + + if cfg!(feature = "posix-sem") { + builder.define("MDB_USE_POSIX_SEM", None); + } + + if cfg!(feature = "asan") { + builder.flag("-fsanitize=address"); + } + + if cfg!(feature = "fuzzer") { + builder.flag("-fsanitize=fuzzer"); + } else if cfg!(feature = "fuzzer-no-link") { + builder.flag("-fsanitize=fuzzer-no-link"); + } + + if cfg!(feature = "longer-keys") { + builder.define("MDB_MAXKEYSIZE", "0"); + } + + builder.compile("liblmdb.a") +} diff --git a/lmdb-master3-sys/lmdb b/lmdb-master3-sys/lmdb new file mode 160000 index 00000000..fd3c2ada --- /dev/null +++ b/lmdb-master3-sys/lmdb @@ -0,0 +1 @@ +Subproject commit fd3c2adae70d2ed65017100db45e0b3babfe342a diff --git a/lmdb-master3-sys/src/bindings.rs b/lmdb-master3-sys/src/bindings.rs new file mode 100644 index 00000000..b1af47d7 --- /dev/null +++ b/lmdb-master3-sys/src/bindings.rs @@ -0,0 +1,552 @@ +/* automatically generated by rust-bindgen 0.69.4 */ + +pub const MDB_FMT_Z: &[u8; 2] = b"z\0"; +pub const MDB_RPAGE_CACHE: ::libc::c_uint = 1; +pub const MDB_SIZE_MAX: u64 = 18446744073709551615; +pub const MDB_VERSION_MAJOR: ::libc::c_uint = 0; +pub const MDB_VERSION_MINOR: ::libc::c_uint = 9; +pub const MDB_VERSION_PATCH: ::libc::c_uint = 90; +pub const MDB_VERSION_DATE: &[u8; 12] = b"May 1, 2017\0"; +pub const MDB_FIXEDMAP: ::libc::c_uint = 1; +pub const MDB_ENCRYPT: ::libc::c_uint = 8192; +pub const MDB_NOSUBDIR: ::libc::c_uint = 16384; +pub const MDB_NOSYNC: ::libc::c_uint = 65536; +pub const MDB_RDONLY: ::libc::c_uint = 131072; +pub const MDB_NOMETASYNC: ::libc::c_uint = 262144; +pub const MDB_WRITEMAP: ::libc::c_uint = 524288; +pub const MDB_MAPASYNC: ::libc::c_uint = 1048576; +pub const MDB_NOTLS: ::libc::c_uint = 2097152; +pub const MDB_NOLOCK: ::libc::c_uint = 4194304; +pub const MDB_NORDAHEAD: ::libc::c_uint = 8388608; +pub const MDB_NOMEMINIT: ::libc::c_uint = 16777216; +pub const MDB_PREVSNAPSHOT: ::libc::c_uint = 33554432; +pub const MDB_REMAP_CHUNKS: ::libc::c_uint = 67108864; +pub const MDB_REVERSEKEY: ::libc::c_uint = 2; +pub const MDB_DUPSORT: ::libc::c_uint = 4; +pub const MDB_INTEGERKEY: ::libc::c_uint = 8; +pub const MDB_DUPFIXED: ::libc::c_uint = 16; +pub const MDB_INTEGERDUP: ::libc::c_uint = 32; +pub const MDB_REVERSEDUP: ::libc::c_uint = 64; +pub const MDB_CREATE: ::libc::c_uint = 262144; +pub const MDB_NOOVERWRITE: ::libc::c_uint = 16; +pub const MDB_NODUPDATA: ::libc::c_uint = 32; +pub const MDB_CURRENT: ::libc::c_uint = 64; +pub const MDB_RESERVE: ::libc::c_uint = 65536; +pub const MDB_APPEND: ::libc::c_uint = 131072; +pub const MDB_APPENDDUP: ::libc::c_uint = 262144; +pub const MDB_MULTIPLE: ::libc::c_uint = 524288; +pub const MDB_CP_COMPACT: ::libc::c_uint = 1; +pub const MDB_SUCCESS: ::libc::c_int = 0; +pub const MDB_KEYEXIST: ::libc::c_int = -30799; +pub const MDB_NOTFOUND: ::libc::c_int = -30798; +pub const MDB_PAGE_NOTFOUND: ::libc::c_int = -30797; +pub const MDB_CORRUPTED: ::libc::c_int = -30796; +pub const MDB_PANIC: ::libc::c_int = -30795; +pub const MDB_VERSION_MISMATCH: ::libc::c_int = -30794; +pub const MDB_INVALID: ::libc::c_int = -30793; +pub const MDB_MAP_FULL: ::libc::c_int = -30792; +pub const MDB_DBS_FULL: ::libc::c_int = -30791; +pub const MDB_READERS_FULL: ::libc::c_int = -30790; +pub const MDB_TLS_FULL: ::libc::c_int = -30789; +pub const MDB_TXN_FULL: ::libc::c_int = -30788; +pub const MDB_CURSOR_FULL: ::libc::c_int = -30787; +pub const MDB_PAGE_FULL: ::libc::c_int = -30786; +pub const MDB_MAP_RESIZED: ::libc::c_int = -30785; +pub const MDB_INCOMPATIBLE: ::libc::c_int = -30784; +pub const MDB_BAD_RSLOT: ::libc::c_int = -30783; +pub const MDB_BAD_TXN: ::libc::c_int = -30782; +pub const MDB_BAD_VALSIZE: ::libc::c_int = -30781; +pub const MDB_BAD_DBI: ::libc::c_int = -30780; +pub const MDB_PROBLEM: ::libc::c_int = -30779; +pub const MDB_BAD_CHECKSUM: ::libc::c_int = -30778; +pub const MDB_CRYPTO_FAIL: ::libc::c_int = -30777; +pub const MDB_ENV_ENCRYPTION: ::libc::c_int = -30776; +pub const MDB_LAST_ERRCODE: ::libc::c_int = -30776; +#[doc = "Unsigned type used for mapsize, entry counts and page/transaction IDs.\n\n\tIt is normally size_t, hence the name. Defining MDB_VL32 makes it\n\tuint64_t, but do not try this unless you know what you are doing."] +pub type mdb_size_t = usize; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct MDB_env { + _unused: [u8; 0], +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct MDB_txn { + _unused: [u8; 0], +} +#[doc = "A handle for an individual database in the DB environment."] +pub type MDB_dbi = ::libc::c_uint; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct MDB_cursor { + _unused: [u8; 0], +} +#[doc = "Generic structure used for passing keys and data in and out\n of the database.\n\n Values returned from the database are valid only until a subsequent\n update operation, or the end of the transaction. Do not modify or\n free them, they commonly point into the database itself.\n\n Key sizes must be between 1 and #mdb_env_get_maxkeysize() inclusive.\n The same applies to data sizes in databases with the #MDB_DUPSORT flag.\n Other data items can in theory be from 0 to 0xffffffff bytes long."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct MDB_val { + #[doc = "< size of the data item"] + pub mv_size: usize, + #[doc = "< address of the data item"] + pub mv_data: *mut ::libc::c_void, +} +#[doc = "A callback function used to compare two keys in a database"] +pub type MDB_cmp_func = ::std::option::Option< + unsafe extern "C" fn(a: *const MDB_val, b: *const MDB_val) -> ::libc::c_int, +>; +#[doc = "A callback function used to relocate a position-dependent data item\n in a fixed-address database.\n\n The **newptr** gives the item's desired address in\n the memory map, and **oldptr** gives its previous address. The item's actual\n data resides at the address in **item.** This callback is expected to walk\n through the fields of the record in **item** and modify any\n values based at the **oldptr** address to be relative to the **newptr** address.\n # Arguments\n\n* `item` (direction in, out) - The item that is to be relocated.\n * `oldptr` (direction in) - The previous address.\n * `newptr` (direction in) - The new address to relocate to.\n * `relctx` (direction in) - An application-provided context, set by #mdb_set_relctx().\n This feature is currently unimplemented."] +pub type MDB_rel_func = ::std::option::Option< + unsafe extern "C" fn( + item: *mut MDB_val, + oldptr: *mut ::libc::c_void, + newptr: *mut ::libc::c_void, + relctx: *mut ::libc::c_void, + ), +>; +#[doc = "A callback function used to encrypt/decrypt pages in the env.\n\n Encrypt or decrypt the data in src and store the result in dst using the\n provided key. The result must be the same number of bytes as the input.\n # Arguments\n\n* `src` (direction in) - The input data to be transformed.\n * `dst` (direction out) - Storage for the result.\n * `key` (direction in) - An array of three values: key[0] is the encryption key,\n key[1] is the initialization vector, and key[2] is the authentication\n data, if any.\n * `encdec` (direction in) - 1 to encrypt, 0 to decrypt.\n # Returns\n\nA non-zero error value on failure and 0 on success."] +pub type MDB_enc_func = ::std::option::Option< + unsafe extern "C" fn( + src: *const MDB_val, + dst: *mut MDB_val, + key: *const MDB_val, + encdec: ::libc::c_int, + ) -> ::libc::c_int, +>; +#[doc = "A callback function used to checksum pages in the env.\n\n Compute the checksum of the data in src and store the result in dst,\n An optional key may be used with keyed hash algorithms.\n # Arguments\n\n* `src` (direction in) - The input data to be transformed.\n * `dst` (direction out) - Storage for the result.\n * `key` (direction in) - An encryption key, if encryption was configured. This\n parameter will be NULL if there is no key."] +pub type MDB_sum_func = ::std::option::Option< + unsafe extern "C" fn(src: *const MDB_val, dst: *mut MDB_val, key: *const MDB_val), +>; +#[doc = "< Position at first key/data item"] +pub const MDB_FIRST: MDB_cursor_op = 0; +#[doc = "< Position at first data item of current key.\nOnly for #MDB_DUPSORT"] +pub const MDB_FIRST_DUP: MDB_cursor_op = 1; +#[doc = "< Position at key/data pair. Only for #MDB_DUPSORT"] +pub const MDB_GET_BOTH: MDB_cursor_op = 2; +#[doc = "< position at key, nearest data. Only for #MDB_DUPSORT"] +pub const MDB_GET_BOTH_RANGE: MDB_cursor_op = 3; +#[doc = "< Return key/data at current cursor position"] +pub const MDB_GET_CURRENT: MDB_cursor_op = 4; +#[doc = "< Return up to a page of duplicate data items\nfrom current cursor position. Move cursor to prepare\nfor #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED"] +pub const MDB_GET_MULTIPLE: MDB_cursor_op = 5; +#[doc = "< Position at last key/data item"] +pub const MDB_LAST: MDB_cursor_op = 6; +#[doc = "< Position at last data item of current key.\nOnly for #MDB_DUPSORT"] +pub const MDB_LAST_DUP: MDB_cursor_op = 7; +#[doc = "< Position at next data item"] +pub const MDB_NEXT: MDB_cursor_op = 8; +#[doc = "< Position at next data item of current key.\nOnly for #MDB_DUPSORT"] +pub const MDB_NEXT_DUP: MDB_cursor_op = 9; +#[doc = "< Return up to a page of duplicate data items\nfrom next cursor position. Move cursor to prepare\nfor #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED"] +pub const MDB_NEXT_MULTIPLE: MDB_cursor_op = 10; +#[doc = "< Position at first data item of next key"] +pub const MDB_NEXT_NODUP: MDB_cursor_op = 11; +#[doc = "< Position at previous data item"] +pub const MDB_PREV: MDB_cursor_op = 12; +#[doc = "< Position at previous data item of current key.\nOnly for #MDB_DUPSORT"] +pub const MDB_PREV_DUP: MDB_cursor_op = 13; +#[doc = "< Position at last data item of previous key"] +pub const MDB_PREV_NODUP: MDB_cursor_op = 14; +#[doc = "< Position at specified key"] +pub const MDB_SET: MDB_cursor_op = 15; +#[doc = "< Position at specified key, return key + data"] +pub const MDB_SET_KEY: MDB_cursor_op = 16; +#[doc = "< Position at first key greater than or equal to specified key."] +pub const MDB_SET_RANGE: MDB_cursor_op = 17; +#[doc = "< Position at previous page and return up to\na page of duplicate data items. Only for #MDB_DUPFIXED"] +pub const MDB_PREV_MULTIPLE: MDB_cursor_op = 18; +#[doc = "Cursor Get operations.\n\n\tThis is the set of all operations for retrieving data\n\tusing a cursor."] +pub type MDB_cursor_op = ::libc::c_uint; +#[doc = "Statistics for a database in the environment"] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct MDB_stat { + #[doc = "< Size of a database page.\nThis is currently the same for all databases."] + pub ms_psize: ::libc::c_uint, + #[doc = "< Depth (height) of the B-tree"] + pub ms_depth: ::libc::c_uint, + #[doc = "< Number of internal (non-leaf) pages"] + pub ms_branch_pages: mdb_size_t, + #[doc = "< Number of leaf pages"] + pub ms_leaf_pages: mdb_size_t, + #[doc = "< Number of overflow pages"] + pub ms_overflow_pages: mdb_size_t, + #[doc = "< Number of data items"] + pub ms_entries: mdb_size_t, +} +#[doc = "Information about the environment"] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct MDB_envinfo { + #[doc = "< Address of map, if fixed"] + pub me_mapaddr: *mut ::libc::c_void, + #[doc = "< Size of the data memory map"] + pub me_mapsize: mdb_size_t, + #[doc = "< ID of the last used page"] + pub me_last_pgno: mdb_size_t, + #[doc = "< ID of the last committed transaction"] + pub me_last_txnid: mdb_size_t, + #[doc = "< max reader slots in the environment"] + pub me_maxreaders: ::libc::c_uint, + #[doc = "< max reader slots used in the environment"] + pub me_numreaders: ::libc::c_uint, +} +extern "C" { + #[doc = "Return the LMDB library version information.\n\n # Arguments\n\n* `major` (direction out) - if non-NULL, the library major version number is copied here\n * `minor` (direction out) - if non-NULL, the library minor version number is copied here\n * `patch` (direction out) - if non-NULL, the library patch version number is copied here\n # Returns\n\n* `\"version` - string\" The library version as a string"] + pub fn mdb_version( + major: *mut ::libc::c_int, + minor: *mut ::libc::c_int, + patch: *mut ::libc::c_int, + ) -> *mut ::libc::c_char; +} +extern "C" { + #[doc = "Return a string describing a given error code.\n\n This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3)\n function. If the error code is greater than or equal to 0, then the string\n returned by the system function strerror(3) is returned. If the error code\n is less than 0, an error string corresponding to the LMDB library error is\n returned. See errors for a list of LMDB-specific error codes.\n # Arguments\n\n* `err` (direction in) - The error code\n # Returns\n\n* `\"error` - message\" The description of the error"] + pub fn mdb_strerror(err: ::libc::c_int) -> *mut ::libc::c_char; +} +extern "C" { + #[doc = "Create an LMDB environment handle.\n\n This function allocates memory for a #MDB_env structure. To release\n the allocated memory and discard the handle, call #mdb_env_close().\n Before the handle may be used, it must be opened using #mdb_env_open().\n Various other options may also need to be set before opening the handle,\n e.g. #mdb_env_set_mapsize(), #mdb_env_set_maxreaders(), #mdb_env_set_maxdbs(),\n depending on usage requirements.\n # Arguments\n\n* `env` (direction out) - The address where the new handle will be stored\n # Returns\n\nA non-zero error value on failure and 0 on success."] + pub fn mdb_env_create(env: *mut *mut MDB_env) -> ::libc::c_int; +} +extern "C" { + #[doc = "Open an environment handle.\n\n If this function fails, #mdb_env_close() must be called to discard the #MDB_env handle.\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `path` (direction in) - The directory in which the database files reside. This\n directory must already exist and be writable.\n * `flags` (direction in) - Special options for this environment. This parameter\n must be set to 0 or by bitwise OR'ing together one or more of the\n values described here.\n Flags set by mdb_env_set_flags() are also used.\n
    \n\t
  • #MDB_FIXEDMAP\n use a fixed address for the mmap region. This flag must be specified\n when creating the environment, and is stored persistently in the environment.\n\t\tIf successful, the memory map will always reside at the same virtual address\n\t\tand pointers used to reference data items in the database will be constant\n\t\tacross multiple invocations. This option may not always work, depending on\n\t\thow the operating system has allocated memory to shared libraries and other uses.\n\t\tThe feature is highly experimental.\n\t
  • #MDB_NOSUBDIR\n\t\tBy default, LMDB creates its environment in a directory whose\n\t\tpathname is given in **path,** and creates its data and lock files\n\t\tunder that directory. With this option, **path** is used as-is for\n\t\tthe database main data file. The database lock file is the **path**\n\t\twith \"-lock\" appended.\n\t
  • #MDB_RDONLY\n\t\tOpen the environment in read-only mode. No write operations will be\n\t\tallowed. LMDB will still modify the lock file - except on read-only\n\t\tfilesystems, where LMDB does not use locks.\n\t
  • #MDB_WRITEMAP\n\t\tUse a writeable memory map unless MDB_RDONLY is set. This uses\n\t\tfewer mallocs but loses protection from application bugs\n\t\tlike wild pointer writes and other bad updates into the database.\n\t\tThis may be slightly faster for DBs that fit entirely in RAM, but\n\t\tis slower for DBs larger than RAM.\n\t\tIncompatible with nested transactions.\n\t\tDo not mix processes with and without MDB_WRITEMAP on the same\n\t\tenvironment. This can defeat durability (#mdb_env_sync etc).\n\t
  • #MDB_NOMETASYNC\n\t\tFlush system buffers to disk only once per transaction, omit the\n\t\tmetadata flush. Defer that until the system flushes files to disk,\n\t\tor next non-MDB_RDONLY commit or #mdb_env_sync(). This optimization\n\t\tmaintains database integrity, but a system crash may undo the last\n\t\tcommitted transaction. I.e. it preserves the ACI (atomicity,\n\t\tconsistency, isolation) but not D (durability) database property.\n\t\tThis flag may be changed at any time using #mdb_env_set_flags().\n\t
  • #MDB_NOSYNC\n\t\tDon't flush system buffers to disk when committing a transaction.\n\t\tThis optimization means a system crash can corrupt the database or\n\t\tlose the last transactions if buffers are not yet flushed to disk.\n\t\tThe risk is governed by how often the system flushes dirty buffers\n\t\tto disk and how often #mdb_env_sync() is called. However, if the\n\t\tfilesystem preserves write order and the #MDB_WRITEMAP flag is not\n\t\tused, transactions exhibit ACI (atomicity, consistency, isolation)\n\t\tproperties and only lose D (durability). I.e. database integrity\n\t\tis maintained, but a system crash may undo the final transactions.\n\t\tNote that (#MDB_NOSYNC | #MDB_WRITEMAP) leaves the system with no\n\t\thint for when to write transactions to disk, unless #mdb_env_sync()\n\t\tis called. (#MDB_MAPASYNC | #MDB_WRITEMAP) may be preferable.\n\t\tThis flag may be changed at any time using #mdb_env_set_flags().\n\t
  • #MDB_MAPASYNC\n\t\tWhen using #MDB_WRITEMAP, use asynchronous flushes to disk.\n\t\tAs with #MDB_NOSYNC, a system crash can then corrupt the\n\t\tdatabase or lose the last transactions. Calling #mdb_env_sync()\n\t\tensures on-disk database integrity until next commit.\n\t\tThis flag may be changed at any time using #mdb_env_set_flags().\n\t
  • #MDB_NOTLS\n\t\tDon't use Thread-Local Storage. Tie reader locktable slots to\n\t\t#MDB_txn objects instead of to threads. I.e. #mdb_txn_reset() keeps\n\t\tthe slot reserved for the #MDB_txn object. A thread may use parallel\n\t\tread-only transactions. A read-only transaction may span threads if\n\t\tthe user synchronizes its use. Applications that multiplex many\n\t\tuser threads over individual OS threads need this option. Such an\n\t\tapplication must also serialize the write transactions in an OS\n\t\tthread, since LMDB's write locking is unaware of the user threads.\n\t
  • #MDB_NOLOCK\n\t\tDon't do any locking. If concurrent access is anticipated, the\n\t\tcaller must manage all concurrency itself. For proper operation\n\t\tthe caller must enforce single-writer semantics, and must ensure\n\t\tthat no readers are using old transactions while a writer is\n\t\tactive. The simplest approach is to use an exclusive lock so that\n\t\tno readers may be active at all when a writer begins.\n\t
  • #MDB_NORDAHEAD\n\t\tTurn off readahead. Most operating systems perform readahead on\n\t\tread requests by default. This option turns it off if the OS\n\t\tsupports it. Turning it off may help random read performance\n\t\twhen the DB is larger than RAM and system RAM is full.\n\t\tThe option is not implemented on Windows.\n\t
  • #MDB_NOMEMINIT\n\t\tDon't initialize malloc'd memory before writing to unused spaces\n\t\tin the data file. By default, memory for pages written to the data\n\t\tfile is obtained using malloc. While these pages may be reused in\n\t\tsubsequent transactions, freshly malloc'd pages will be initialized\n\t\tto zeroes before use. This avoids persisting leftover data from other\n\t\tcode (that used the heap and subsequently freed the memory) into the\n\t\tdata file. Note that many other system libraries may allocate\n\t\tand free memory from the heap for arbitrary uses. E.g., stdio may\n\t\tuse the heap for file I/O buffers. This initialization step has a\n\t\tmodest performance cost so some applications may want to disable\n\t\tit using this flag. This option can be a problem for applications\n\t\twhich handle sensitive data like passwords, and it makes memory\n\t\tcheckers like Valgrind noisy. This flag is not needed with #MDB_WRITEMAP,\n\t\twhich writes directly to the mmap instead of using malloc for pages. The\n\t\tinitialization is also skipped if #MDB_RESERVE is used; the\n\t\tcaller is expected to overwrite all of the memory that was\n\t\treserved in that case.\n\t\tThis flag may be changed at any time using #mdb_env_set_flags().\n\t
  • #MDB_PREVSNAPSHOT\n\t\tOpen the environment with the previous snapshot rather than the latest\n\t\tone. This loses the latest transaction, but may help work around some\n\t\ttypes of corruption. If opened with write access, this must be the\n\t\tonly process using the environment. This flag is automatically reset\n\t\tafter a write transaction is successfully committed.\n
\n * `mode` (direction in) - The UNIX permissions to set on created files and semaphores.\n This parameter is ignored on Windows.\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • #MDB_VERSION_MISMATCH - the version of the LMDB library doesn't match the\n\tversion that created the database environment.\n\t
  • #MDB_INVALID - the environment file headers are corrupted.\n\t
  • ENOENT - the directory specified by the path parameter doesn't exist.\n\t
  • EACCES - the user didn't have permission to access the environment files.\n\t
  • EAGAIN - the environment was locked by another process.\n
"] + pub fn mdb_env_open( + env: *mut MDB_env, + path: *const ::libc::c_char, + flags: ::libc::c_uint, + mode: mdb_mode_t, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Copy an LMDB environment to the specified path.\n\n This function may be used to make a backup of an existing environment.\n No lockfile is created, since it gets recreated at need.\n > **Note:** This call can trigger significant file size growth if run in\n parallel with write transactions, because it employs a read-only\n transaction. See long-lived transactions under caveats_sec.\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create(). It\n must have already been opened successfully.\n * `path` (direction in) - The directory in which the copy will reside. This\n directory must already exist and be writable but must otherwise be\n empty.\n # Returns\n\nA non-zero error value on failure and 0 on success."] + pub fn mdb_env_copy(env: *mut MDB_env, path: *const ::libc::c_char) -> ::libc::c_int; +} +extern "C" { + #[doc = "Copy an LMDB environment to the specified file descriptor.\n\n This function may be used to make a backup of an existing environment.\n No lockfile is created, since it gets recreated at need.\n > **Note:** This call can trigger significant file size growth if run in\n parallel with write transactions, because it employs a read-only\n transaction. See long-lived transactions under caveats_sec.\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create(). It\n must have already been opened successfully.\n * `fd` (direction in) - The filedescriptor to write the copy to. It must\n have already been opened for Write access.\n # Returns\n\nA non-zero error value on failure and 0 on success."] + pub fn mdb_env_copyfd(env: *mut MDB_env, fd: mdb_filehandle_t) -> ::libc::c_int; +} +extern "C" { + #[doc = "Copy an LMDB environment to the specified path, with options.\n\n This function may be used to make a backup of an existing environment.\n No lockfile is created, since it gets recreated at need.\n > **Note:** This call can trigger significant file size growth if run in\n parallel with write transactions, because it employs a read-only\n transaction. See long-lived transactions under caveats_sec.\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create(). It\n must have already been opened successfully.\n * `path` (direction in) - The directory in which the copy will reside. This\n directory must already exist and be writable but must otherwise be\n empty.\n * `flags` (direction in) - Special options for this operation. This parameter\n must be set to 0 or by bitwise OR'ing together one or more of the\n values described here.\n
    \n\t
  • #MDB_CP_COMPACT - Perform compaction while copying: omit free\n\t\tpages and sequentially renumber all pages in output. This option\n\t\tconsumes more CPU and runs more slowly than the default.\n\t\tCurrently it fails if the environment has suffered a page leak.\n
\n # Returns\n\nA non-zero error value on failure and 0 on success."] + pub fn mdb_env_copy2( + env: *mut MDB_env, + path: *const ::libc::c_char, + flags: ::libc::c_uint, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Copy an LMDB environment to the specified file descriptor,\n\twith options.\n\n This function may be used to make a backup of an existing environment.\n No lockfile is created, since it gets recreated at need. See\n #mdb_env_copy2() for further details.\n > **Note:** This call can trigger significant file size growth if run in\n parallel with write transactions, because it employs a read-only\n transaction. See long-lived transactions under caveats_sec.\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create(). It\n must have already been opened successfully.\n * `fd` (direction in) - The filedescriptor to write the copy to. It must\n have already been opened for Write access.\n * `flags` (direction in) - Special options for this operation.\n See #mdb_env_copy2() for options.\n # Returns\n\nA non-zero error value on failure and 0 on success."] + pub fn mdb_env_copyfd2( + env: *mut MDB_env, + fd: mdb_filehandle_t, + flags: ::libc::c_uint, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Return statistics about the LMDB environment.\n\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `stat` (direction out) - The address of an #MDB_stat structure\n \twhere the statistics will be copied"] + pub fn mdb_env_stat(env: *mut MDB_env, stat: *mut MDB_stat) -> ::libc::c_int; +} +extern "C" { + #[doc = "Return information about the LMDB environment.\n\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `stat` (direction out) - The address of an #MDB_envinfo structure\n \twhere the information will be copied"] + pub fn mdb_env_info(env: *mut MDB_env, stat: *mut MDB_envinfo) -> ::libc::c_int; +} +extern "C" { + #[doc = "Flush the data buffers to disk.\n\n Data is always written to disk when #mdb_txn_commit() is called,\n but the operating system may keep it buffered. LMDB always flushes\n the OS buffers upon commit as well, unless the environment was\n opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. This call is\n not valid if the environment was opened with #MDB_RDONLY.\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `force` (direction in) - If non-zero, force a synchronous flush. Otherwise\n if the environment has the #MDB_NOSYNC flag set the flushes\n\twill be omitted, and with #MDB_MAPASYNC they will be asynchronous.\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EACCES - the environment is read-only.\n\t
  • EINVAL - an invalid parameter was specified.\n\t
  • EIO - an error occurred during synchronization.\n
"] + pub fn mdb_env_sync(env: *mut MDB_env, force: ::libc::c_int) -> ::libc::c_int; +} +extern "C" { + #[doc = "Close the environment and release the memory map.\n\n Only a single thread may call this function. All transactions, databases,\n and cursors must already be closed before calling this function. Attempts to\n use any such handles after calling this function will cause a SIGSEGV.\n The environment handle will be freed and must not be used again after this call.\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()"] + pub fn mdb_env_close(env: *mut MDB_env); +} +extern "C" { + #[doc = "Set environment flags.\n\n This may be used to set some flags in addition to those from\n #mdb_env_open(), or to unset these flags. If several threads\n change the flags at the same time, the result is undefined.\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `flags` (direction in) - The flags to change, bitwise OR'ed together\n * `onoff` (direction in) - A non-zero value sets the flags, zero clears them.\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_env_set_flags( + env: *mut MDB_env, + flags: ::libc::c_uint, + onoff: ::libc::c_int, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Get environment flags.\n\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `flags` (direction out) - The address of an integer to store the flags\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_env_get_flags(env: *mut MDB_env, flags: *mut ::libc::c_uint) -> ::libc::c_int; +} +extern "C" { + #[doc = "Return the path that was used in #mdb_env_open().\n\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `path` (direction out) - Address of a string pointer to contain the path. This\n is the actual string in the environment, not a copy. It should not be\n altered in any way.\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_env_get_path(env: *mut MDB_env, path: *mut *const ::libc::c_char) -> ::libc::c_int; +} +extern "C" { + #[doc = "Return the filedescriptor for the given environment.\n\n This function may be called after fork(), so the descriptor can be\n closed before exec*(). Other LMDB file descriptors have FD_CLOEXEC.\n (Until LMDB 0.9.18, only the lockfile had that.)\n\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `fd` (direction out) - Address of a mdb_filehandle_t to contain the descriptor.\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_env_get_fd(env: *mut MDB_env, fd: *mut mdb_filehandle_t) -> ::libc::c_int; +} +extern "C" { + #[doc = "Set the size of the memory map to use for this environment.\n\n The size should be a multiple of the OS page size. The default is\n 10485760 bytes. The size of the memory map is also the maximum size\n of the database. The value should be chosen as large as possible,\n to accommodate future growth of the database.\n This function should be called after #mdb_env_create() and before #mdb_env_open().\n It may be called at later times if no transactions are active in\n this process. Note that the library does not check for this condition,\n the caller must ensure it explicitly.\n\n The new size takes effect immediately for the current process but\n will not be persisted to any others until a write transaction has been\n committed by the current process. Also, only mapsize increases are\n persisted into the environment.\n\n If the mapsize is increased by another process, and data has grown\n beyond the range of the current mapsize, #mdb_txn_begin() will\n return #MDB_MAP_RESIZED. This function may be called with a size\n of zero to adopt the new size.\n\n Any attempt to set a size smaller than the space already consumed\n by the environment will be silently changed to the current size of the used space.\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `size` (direction in) - The size in bytes\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified, or the environment has\n \tan active write transaction.\n
"] + pub fn mdb_env_set_mapsize(env: *mut MDB_env, size: mdb_size_t) -> ::libc::c_int; +} +extern "C" { + #[doc = "Set the size of DB pages in bytes.\n\n The size defaults to the OS page size. Smaller or larger values may be\n desired depending on the size of keys and values being used. Also, an\n explicit size may need to be set when using filesystems like ZFS which\n don't use the OS page size."] + pub fn mdb_env_set_pagesize(env: *mut MDB_env, size: ::libc::c_int) -> ::libc::c_int; +} +extern "C" { + #[doc = "Set the maximum number of threads/reader slots for the environment.\n\n This defines the number of slots in the lock table that is used to track readers in the\n the environment. The default is 126.\n Starting a read-only transaction normally ties a lock table slot to the\n current thread until the environment closes or the thread exits. If\n MDB_NOTLS is in use, #mdb_txn_begin() instead ties the slot to the\n MDB_txn object until it or the #MDB_env object is destroyed.\n This function may only be called after #mdb_env_create() and before #mdb_env_open().\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `readers` (direction in) - The maximum number of reader lock table slots\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified, or the environment is already open.\n
"] + pub fn mdb_env_set_maxreaders(env: *mut MDB_env, readers: ::libc::c_uint) -> ::libc::c_int; +} +extern "C" { + #[doc = "Get the maximum number of threads/reader slots for the environment.\n\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `readers` (direction out) - Address of an integer to store the number of readers\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_env_get_maxreaders(env: *mut MDB_env, readers: *mut ::libc::c_uint) + -> ::libc::c_int; +} +extern "C" { + #[doc = "Set the maximum number of named databases for the environment.\n\n This function is only needed if multiple databases will be used in the\n environment. Simpler applications that use the environment as a single\n unnamed database can ignore this option.\n This function may only be called after #mdb_env_create() and before #mdb_env_open().\n\n Currently a moderate number of slots are cheap but a huge number gets\n expensive: 7-120 words per transaction, and every #mdb_dbi_open()\n does a linear search of the opened slots.\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `dbs` (direction in) - The maximum number of databases\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified, or the environment is already open.\n
"] + pub fn mdb_env_set_maxdbs(env: *mut MDB_env, dbs: MDB_dbi) -> ::libc::c_int; +} +extern "C" { + #[doc = "Get the maximum size of keys and #MDB_DUPSORT data we can write.\n\n Depends on the compile-time constant #MDB_MAXKEYSIZE. Default 511.\n See MDB_val.\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n # Returns\n\nThe maximum size of a key we can write"] + pub fn mdb_env_get_maxkeysize(env: *mut MDB_env) -> ::libc::c_int; +} +extern "C" { + #[doc = "Set application information associated with the #MDB_env.\n\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `ctx` (direction in) - An arbitrary pointer for whatever the application needs.\n # Returns\n\nA non-zero error value on failure and 0 on success."] + pub fn mdb_env_set_userctx(env: *mut MDB_env, ctx: *mut ::libc::c_void) -> ::libc::c_int; +} +extern "C" { + #[doc = "Get the application information associated with the #MDB_env.\n\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n # Returns\n\nThe pointer set by #mdb_env_set_userctx()."] + pub fn mdb_env_get_userctx(env: *mut MDB_env) -> *mut ::libc::c_void; +} +#[doc = "A callback function for most LMDB assert() failures,\n called before printing the message and aborting.\n\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create().\n * `msg` (direction in) - The assertion message, not including newline."] +pub type MDB_assert_func = + ::std::option::Option; +extern "C" { + #[doc = "Set or reset the assert() callback of the environment.\n Disabled if liblmdb is built with NDEBUG.\n > **Note:** This hack should become obsolete as lmdb's error handling matures.\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create().\n * `func` (direction in) - An #MDB_assert_func function, or 0.\n # Returns\n\nA non-zero error value on failure and 0 on success."] + pub fn mdb_env_set_assert(env: *mut MDB_env, func: MDB_assert_func) -> ::libc::c_int; +} +extern "C" { + #[doc = "Set encryption on an environment.\n\n This must be called before #mdb_env_open().\n It implicitly sets #MDB_REMAP_CHUNKS on the env.\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create().\n * `func` (direction in) - An #MDB_enc_func function.\n * `key` (direction in) - The encryption key.\n * `size` (direction in) - The size of authentication data in bytes, if any.\n Set this to zero for unauthenticated encryption mechanisms.\n # Returns\n\nA non-zero error value on failure and 0 on success."] + pub fn mdb_env_set_encrypt( + env: *mut MDB_env, + func: MDB_enc_func, + key: *const MDB_val, + size: ::libc::c_uint, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Set checksums on an environment.\n\n This must be called before #mdb_env_open().\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create().\n * `func` (direction in) - An #MDB_sum_func function.\n * `size` (direction in) - The size of computed checksum values, in bytes.\n # Returns\n\nA non-zero error value on failure and 0 on success."] + pub fn mdb_env_set_checksum( + env: *mut MDB_env, + func: MDB_sum_func, + size: ::libc::c_uint, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Create a transaction for use with the environment.\n\n The transaction handle may be discarded using #mdb_txn_abort() or #mdb_txn_commit().\n > **Note:** A transaction and its cursors must only be used by a single\n thread, and a thread may only have a single transaction at a time.\n If #MDB_NOTLS is in use, this does not apply to read-only transactions.\n > **Note:** Cursors may not span transactions.\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `parent` (direction in) - If this parameter is non-NULL, the new transaction\n will be a nested transaction, with the transaction indicated by **parent**\n as its parent. Transactions may be nested to any level. A parent\n transaction and its cursors may not issue any other operations than\n mdb_txn_commit and mdb_txn_abort while it has active child transactions.\n * `flags` (direction in) - Special options for this transaction. This parameter\n must be set to 0 or by bitwise OR'ing together one or more of the\n values described here.\n
    \n\t
  • #MDB_RDONLY\n\t\tThis transaction will not perform any write operations.\n\t
  • #MDB_NOSYNC\n\t\tDon't flush system buffers to disk when committing this transaction.\n\t
  • #MDB_NOMETASYNC\n\t\tFlush system buffers but omit metadata flush when committing this transaction.\n
\n * `txn` (direction out) - Address where the new #MDB_txn handle will be stored\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • #MDB_PANIC - a fatal error occurred earlier and the environment\n\t\tmust be shut down.\n\t
  • #MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's\n\t\tmapsize and this environment's map must be resized as well.\n\t\tSee #mdb_env_set_mapsize().\n\t
  • #MDB_READERS_FULL - a read-only transaction was requested and\n\t\tthe reader lock table is full. See #mdb_env_set_maxreaders().\n\t
  • ENOMEM - out of memory.\n
"] + pub fn mdb_txn_begin( + env: *mut MDB_env, + parent: *mut MDB_txn, + flags: ::libc::c_uint, + txn: *mut *mut MDB_txn, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Returns the transaction's #MDB_env\n\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()"] + pub fn mdb_txn_env(txn: *mut MDB_txn) -> *mut MDB_env; +} +extern "C" { + #[doc = "Return the transaction's ID.\n\n This returns the identifier associated with this transaction. For a\n read-only transaction, this corresponds to the snapshot being read;\n concurrent readers will frequently have the same transaction ID.\n\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n # Returns\n\nA transaction ID, valid if input is an active transaction."] + pub fn mdb_txn_id(txn: *mut MDB_txn) -> mdb_size_t; +} +extern "C" { + #[doc = "Commit all the operations of a transaction into the database.\n\n The transaction handle is freed. It and its cursors must not be used\n again after this call, except with #mdb_cursor_renew().\n > **Note:** Earlier documentation incorrectly said all cursors would be freed.\n Only write-transactions free cursors.\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified.\n\t
  • ENOSPC - no more disk space.\n\t
  • EIO - a low-level I/O error occurred while writing.\n\t
  • ENOMEM - out of memory.\n
"] + pub fn mdb_txn_commit(txn: *mut MDB_txn) -> ::libc::c_int; +} +extern "C" { + #[doc = "Abandon all the operations of the transaction instead of saving them.\n\n The transaction handle is freed. It and its cursors must not be used\n again after this call, except with #mdb_cursor_renew().\n > **Note:** Earlier documentation incorrectly said all cursors would be freed.\n Only write-transactions free cursors.\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()"] + pub fn mdb_txn_abort(txn: *mut MDB_txn); +} +extern "C" { + #[doc = "Reset a read-only transaction.\n\n Abort the transaction like #mdb_txn_abort(), but keep the transaction\n handle. #mdb_txn_renew() may reuse the handle. This saves allocation\n overhead if the process will start a new read-only transaction soon,\n and also locking overhead if #MDB_NOTLS is in use. The reader table\n lock is released, but the table slot stays tied to its thread or\n #MDB_txn. Use mdb_txn_abort() to discard a reset handle, and to free\n its lock table slot if MDB_NOTLS is in use.\n Cursors opened within the transaction must not be used\n again after this call, except with #mdb_cursor_renew().\n Reader locks generally don't interfere with writers, but they keep old\n versions of database pages allocated. Thus they prevent the old pages\n from being reused when writers commit new data, and so under heavy load\n the database size may grow much more rapidly than otherwise.\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()"] + pub fn mdb_txn_reset(txn: *mut MDB_txn); +} +extern "C" { + #[doc = "Renew a read-only transaction.\n\n This acquires a new reader lock for a transaction handle that had been\n released by #mdb_txn_reset(). It must be called before a reset transaction\n may be used again.\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • #MDB_PANIC - a fatal error occurred earlier and the environment\n\t\tmust be shut down.\n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_txn_renew(txn: *mut MDB_txn) -> ::libc::c_int; +} +extern "C" { + #[doc = "Open a database in the environment.\n\n A database handle denotes the name and parameters of a database,\n independently of whether such a database exists.\n The database handle may be discarded by calling #mdb_dbi_close().\n The old database handle is returned if the database was already open.\n The handle may only be closed once.\n\n The database handle will be private to the current transaction until\n the transaction is successfully committed. If the transaction is\n aborted the handle will be closed automatically.\n After a successful commit the handle will reside in the shared\n environment, and may be used by other transactions.\n\n This function must not be called from multiple concurrent\n transactions in the same process. A transaction that uses\n this function must finish (either commit or abort) before\n any other transaction in the process may use this function.\n\n To use named databases (with name != NULL), #mdb_env_set_maxdbs()\n must be called before opening the environment. Database names are\n keys in the unnamed database, and may be read but not written.\n > **Note:** Names are C strings and stored with their NUL terminator included.\n In LMDB 0.9 the NUL terminator was omitted.\n\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n * `name` (direction in) - The name of the database to open. If only a single\n \tdatabase is needed in the environment, this value may be NULL.\n * `flags` (direction in) - Special options for this database. This parameter\n must be set to 0 or by bitwise OR'ing together one or more of the\n values described here.\n
    \n\t
  • #MDB_REVERSEKEY\n\t\tKeys are strings to be compared in reverse order, from the end\n\t\tof the strings to the beginning. By default, Keys are treated as strings and\n\t\tcompared from beginning to end.\n\t
  • #MDB_DUPSORT\n\t\tDuplicate keys may be used in the database. (Or, from another perspective,\n\t\tkeys may have multiple data items, stored in sorted order.) By default\n\t\tkeys must be unique and may have only a single data item.\n\t
  • #MDB_INTEGERKEY\n\t\tKeys are binary integers in native byte order, either unsigned int\n\t\tor #mdb_size_t, and will be sorted as such.\n\t\t(lmdb expects 32-bit int <= size_t <= 32/64-bit mdb_size_t.)\n\t\tThe keys must all be of the same size.\n\t
  • #MDB_DUPFIXED\n\t\tThis flag may only be used in combination with #MDB_DUPSORT. This option\n\t\ttells the library that the data items for this database are all the same\n\t\tsize, which allows further optimizations in storage and retrieval. When\n\t\tall data items are the same size, the #MDB_GET_MULTIPLE, #MDB_NEXT_MULTIPLE\n\t\tand #MDB_PREV_MULTIPLE cursor operations may be used to retrieve multiple\n\t\titems at once.\n\t
  • #MDB_INTEGERDUP\n\t\tThis option specifies that duplicate data items are binary integers,\n\t\tsimilar to #MDB_INTEGERKEY keys.\n\t
  • #MDB_REVERSEDUP\n\t\tThis option specifies that duplicate data items should be compared as\n\t\tstrings in reverse order.\n\t
  • #MDB_CREATE\n\t\tCreate the named database if it doesn't exist. This option is not\n\t\tallowed in a read-only transaction or a read-only environment.\n
\n * `dbi` (direction out) - Address where the new #MDB_dbi handle will be stored\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • #MDB_NOTFOUND - the specified database doesn't exist in the environment\n\t\tand #MDB_CREATE was not specified.\n\t
  • #MDB_DBS_FULL - too many databases have been opened. See #mdb_env_set_maxdbs().\n
"] + pub fn mdb_dbi_open( + txn: *mut MDB_txn, + name: *const ::libc::c_char, + flags: ::libc::c_uint, + dbi: *mut MDB_dbi, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Retrieve statistics for a database.\n\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n * `dbi` (direction in) - A database handle returned by #mdb_dbi_open()\n * `stat` (direction out) - The address of an #MDB_stat structure\n \twhere the statistics will be copied\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_stat(txn: *mut MDB_txn, dbi: MDB_dbi, stat: *mut MDB_stat) -> ::libc::c_int; +} +extern "C" { + #[doc = "Retrieve the DB flags for a database handle.\n\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n * `dbi` (direction in) - A database handle returned by #mdb_dbi_open()\n * `flags` (direction out) - Address where the flags will be returned.\n # Returns\n\nA non-zero error value on failure and 0 on success."] + pub fn mdb_dbi_flags( + txn: *mut MDB_txn, + dbi: MDB_dbi, + flags: *mut ::libc::c_uint, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Close a database handle. Normally unnecessary. Use with care:\n\n This call is not mutex protected. Handles should only be closed by\n a single thread, and only if no other threads are going to reference\n the database handle or one of its cursors any further. Do not close\n a handle if an existing transaction has modified its database.\n Doing so can cause misbehavior from database corruption to errors\n like MDB_BAD_VALSIZE (since the DB name is gone).\n\n Closing a database handle is not necessary, but lets #mdb_dbi_open()\n reuse the handle value. Usually it's better to set a bigger\n #mdb_env_set_maxdbs(), unless that value would be large.\n\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `dbi` (direction in) - A database handle returned by #mdb_dbi_open()"] + pub fn mdb_dbi_close(env: *mut MDB_env, dbi: MDB_dbi); +} +extern "C" { + #[doc = "Empty or delete+close a database.\n\n See #mdb_dbi_close() for restrictions about closing the DB handle.\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n * `dbi` (direction in) - A database handle returned by #mdb_dbi_open()\n * `del` (direction in) - 0 to empty the DB, 1 to delete it from the\n environment and close the DB handle.\n # Returns\n\nA non-zero error value on failure and 0 on success."] + pub fn mdb_drop(txn: *mut MDB_txn, dbi: MDB_dbi, del: ::libc::c_int) -> ::libc::c_int; +} +extern "C" { + #[doc = "Set a custom key comparison function for a database.\n\n The comparison function is called whenever it is necessary to compare a\n key specified by the application with a key currently stored in the database.\n If no comparison function is specified, and no special key flags were specified\n with #mdb_dbi_open(), the keys are compared lexically, with shorter keys collating\n before longer keys.\n This function must be called before any data access functions are used,\n otherwise data corruption may occur. The same comparison function must be used by every\n program accessing the database, every time the database is used.\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n * `dbi` (direction in) - A database handle returned by #mdb_dbi_open()\n * `cmp` (direction in) - A #MDB_cmp_func function\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_set_compare(txn: *mut MDB_txn, dbi: MDB_dbi, cmp: MDB_cmp_func) -> ::libc::c_int; +} +extern "C" { + #[doc = "Set a custom data comparison function for a #MDB_DUPSORT database.\n\n This comparison function is called whenever it is necessary to compare a data\n item specified by the application with a data item currently stored in the database.\n This function only takes effect if the database was opened with the #MDB_DUPSORT\n flag.\n If no comparison function is specified, and no special key flags were specified\n with #mdb_dbi_open(), the data items are compared lexically, with shorter items collating\n before longer items.\n This function must be called before any data access functions are used,\n otherwise data corruption may occur. The same comparison function must be used by every\n program accessing the database, every time the database is used.\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n * `dbi` (direction in) - A database handle returned by #mdb_dbi_open()\n * `cmp` (direction in) - A #MDB_cmp_func function\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_set_dupsort(txn: *mut MDB_txn, dbi: MDB_dbi, cmp: MDB_cmp_func) -> ::libc::c_int; +} +extern "C" { + #[doc = "Set a relocation function for a #MDB_FIXEDMAP database.\n\n The relocation function is called whenever it is necessary to move the data\n of an item to a different position in the database (e.g. through tree\n balancing operations, shifts as a result of adds or deletes, etc.). It is\n intended to allow address/position-dependent data items to be stored in\n a database in an environment opened with the #MDB_FIXEDMAP option.\n Currently the relocation feature is unimplemented and setting\n this function has no effect.\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n * `dbi` (direction in) - A database handle returned by #mdb_dbi_open()\n * `rel` (direction in) - A #MDB_rel_func function\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_set_relfunc(txn: *mut MDB_txn, dbi: MDB_dbi, rel: MDB_rel_func) -> ::libc::c_int; +} +extern "C" { + #[doc = "Set a context pointer for a #MDB_FIXEDMAP database's relocation function.\n\n See #mdb_set_relfunc and #MDB_rel_func for more details.\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n * `dbi` (direction in) - A database handle returned by #mdb_dbi_open()\n * `ctx` (direction in) - An arbitrary pointer for whatever the application needs.\n It will be passed to the callback function set by #mdb_set_relfunc\n as its **relctx** parameter whenever the callback is invoked.\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_set_relctx( + txn: *mut MDB_txn, + dbi: MDB_dbi, + ctx: *mut ::libc::c_void, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Get items from a database.\n\n This function retrieves key/data pairs from the database. The address\n and length of the data associated with the specified **key** are returned\n in the structure to which **data** refers.\n If the database supports duplicate keys (#MDB_DUPSORT) then the\n first data item for the key will be returned. Retrieval of other\n items requires the use of #mdb_cursor_get().\n\n > **Note:** The memory pointed to by the returned values is owned by the\n database. The caller need not dispose of the memory, and may not\n modify it in any way. For values returned in a read-only transaction\n any modification attempts will cause a SIGSEGV.\n > **Note:** Values returned from the database are valid only until a\n subsequent update operation, or the end of the transaction.\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n * `dbi` (direction in) - A database handle returned by #mdb_dbi_open()\n * `key` (direction in) - The key to search for in the database\n * `data` (direction out) - The data corresponding to the key\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • #MDB_NOTFOUND - the key was not in the database.\n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_get( + txn: *mut MDB_txn, + dbi: MDB_dbi, + key: *mut MDB_val, + data: *mut MDB_val, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Store items into a database.\n\n This function stores key/data pairs in the database. The default behavior\n is to enter the new key/data pair, replacing any previously existing key\n if duplicates are disallowed, or adding a duplicate data item if\n duplicates are allowed (#MDB_DUPSORT).\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n * `dbi` (direction in) - A database handle returned by #mdb_dbi_open()\n * `key` (direction in) - The key to store in the database\n * `data` (direction in, out) - The data to store\n * `flags` (direction in) - Special options for this operation. This parameter\n must be set to 0 or by bitwise OR'ing together one or more of the\n values described here.\n
    \n\t
  • #MDB_NODUPDATA - enter the new key/data pair only if it does not\n\t\talready appear in the database. This flag may only be specified\n\t\tif the database was opened with #MDB_DUPSORT. The function will\n\t\treturn #MDB_KEYEXIST if the key/data pair already appears in the\n\t\tdatabase.\n\t
  • #MDB_NOOVERWRITE - enter the new key/data pair only if the key\n\t\tdoes not already appear in the database. The function will return\n\t\t#MDB_KEYEXIST if the key already appears in the database, even if\n\t\tthe database supports duplicates (#MDB_DUPSORT). The **data**\n\t\tparameter will be set to point to the existing item.\n\t
  • #MDB_RESERVE - reserve space for data of the given size, but\n\t\tdon't copy the given data. Instead, return a pointer to the\n\t\treserved space, which the caller can fill in later - before\n\t\tthe next update operation or the transaction ends. This saves\n\t\tan extra memcpy if the data is being generated later.\n\t\tLMDB does nothing else with this memory, the caller is expected\n\t\tto modify all of the space requested. This flag must not be\n\t\tspecified if the database was opened with #MDB_DUPSORT.\n\t
  • #MDB_APPEND - append the given key/data pair to the end of the\n\t\tdatabase. This option allows fast bulk loading when keys are\n\t\talready known to be in the correct order. Loading unsorted keys\n\t\twith this flag will cause a #MDB_KEYEXIST error.\n\t
  • #MDB_APPENDDUP - as above, but for sorted dup data.\n
\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize().\n\t
  • #MDB_TXN_FULL - the transaction has too many dirty pages.\n\t
  • EACCES - an attempt was made to write in a read-only transaction.\n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_put( + txn: *mut MDB_txn, + dbi: MDB_dbi, + key: *mut MDB_val, + data: *mut MDB_val, + flags: ::libc::c_uint, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Delete items from a database.\n\n This function removes key/data pairs from the database.\n If the database does not support sorted duplicate data items\n (#MDB_DUPSORT) the data parameter is ignored.\n If the database supports sorted duplicates and the data parameter\n is NULL, all of the duplicate data items for the key will be\n deleted. Otherwise, if the data parameter is non-NULL\n only the matching data item will be deleted.\n This function will return #MDB_NOTFOUND if the specified key/data\n pair is not in the database.\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n * `dbi` (direction in) - A database handle returned by #mdb_dbi_open()\n * `key` (direction in) - The key to delete from the database\n * `data` (direction in) - The data to delete\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EACCES - an attempt was made to write in a read-only transaction.\n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_del( + txn: *mut MDB_txn, + dbi: MDB_dbi, + key: *mut MDB_val, + data: *mut MDB_val, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Create a cursor handle.\n\n A cursor is associated with a specific transaction and database.\n A cursor cannot be used when its database handle is closed. Nor\n when its transaction has ended, except with #mdb_cursor_renew().\n It can be discarded with #mdb_cursor_close().\n A cursor in a write-transaction can be closed before its transaction\n ends, and will otherwise be closed when its transaction ends.\n A cursor in a read-only transaction must be closed explicitly, before\n or after its transaction ends. It can be reused with\n #mdb_cursor_renew() before finally closing it.\n > **Note:** Earlier documentation said that cursors in every transaction\n were closed when the transaction committed or aborted.\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n * `dbi` (direction in) - A database handle returned by #mdb_dbi_open()\n * `cursor` (direction out) - Address where the new #MDB_cursor handle will be stored\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_cursor_open( + txn: *mut MDB_txn, + dbi: MDB_dbi, + cursor: *mut *mut MDB_cursor, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Close a cursor handle.\n\n The cursor handle will be freed and must not be used again after this call.\n Its transaction must still be live if it is a write-transaction.\n # Arguments\n\n* `cursor` (direction in) - A cursor handle returned by #mdb_cursor_open()"] + pub fn mdb_cursor_close(cursor: *mut MDB_cursor); +} +extern "C" { + #[doc = "Renew a cursor handle.\n\n A cursor is associated with a specific transaction and database.\n Cursors that are only used in read-only\n transactions may be re-used, to avoid unnecessary malloc/free overhead.\n The cursor may be associated with a new read-only transaction, and\n referencing the same database handle as it was created with.\n This may be done whether the previous transaction is live or dead.\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n * `cursor` (direction in) - A cursor handle returned by #mdb_cursor_open()\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_cursor_renew(txn: *mut MDB_txn, cursor: *mut MDB_cursor) -> ::libc::c_int; +} +extern "C" { + #[doc = "Return the cursor's transaction handle.\n\n # Arguments\n\n* `cursor` (direction in) - A cursor handle returned by #mdb_cursor_open()"] + pub fn mdb_cursor_txn(cursor: *mut MDB_cursor) -> *mut MDB_txn; +} +extern "C" { + #[doc = "Return the cursor's database handle.\n\n # Arguments\n\n* `cursor` (direction in) - A cursor handle returned by #mdb_cursor_open()"] + pub fn mdb_cursor_dbi(cursor: *mut MDB_cursor) -> MDB_dbi; +} +extern "C" { + #[doc = "Check if the cursor is pointing to a named database record.\n\n # Arguments\n\n* `cursor` (direction in) - A cursor handle returned by #mdb_cursor_open()\n # Returns\n\n1 if current record is a named database, 0 otherwise."] + pub fn mdb_cursor_is_db(cursor: *mut MDB_cursor) -> ::libc::c_int; +} +extern "C" { + #[doc = "Retrieve by cursor.\n\n This function retrieves key/data pairs from the database. The address and length\n of the key are returned in the object to which **key** refers (except for the\n case of the #MDB_SET option, in which the **key** object is unchanged), and\n the address and length of the data are returned in the object to which **data**\n refers.\n See #mdb_get() for restrictions on using the output values.\n # Arguments\n\n* `cursor` (direction in) - A cursor handle returned by #mdb_cursor_open()\n * `key` (direction in, out) - The key for a retrieved item\n * `data` (direction in, out) - The data of a retrieved item\n * `op` (direction in) - A cursor operation #MDB_cursor_op\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • #MDB_NOTFOUND - no matching key found.\n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_cursor_get( + cursor: *mut MDB_cursor, + key: *mut MDB_val, + data: *mut MDB_val, + op: MDB_cursor_op, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Store by cursor.\n\n This function stores key/data pairs into the database.\n The cursor is positioned at the new item, or on failure usually near it.\n > **Note:** Earlier documentation incorrectly said errors would leave the\n state of the cursor unchanged.\n # Arguments\n\n* `cursor` (direction in) - A cursor handle returned by #mdb_cursor_open()\n * `key` (direction in) - The key operated on.\n * `data` (direction in) - The data operated on.\n * `flags` (direction in) - Options for this operation. This parameter\n must be set to 0 or one of the values described here.\n
    \n\t
  • #MDB_CURRENT - replace the item at the current cursor position.\n\t\tThe **key** parameter must still be provided, and must match it.\n\t\tIf using sorted duplicates (#MDB_DUPSORT) the data item must still\n\t\tsort into the same place. This is intended to be used when the\n\t\tnew data is the same size as the old. Otherwise it will simply\n\t\tperform a delete of the old record followed by an insert.\n\t
  • #MDB_NODUPDATA - enter the new key/data pair only if it does not\n\t\talready appear in the database. This flag may only be specified\n\t\tif the database was opened with #MDB_DUPSORT. The function will\n\t\treturn #MDB_KEYEXIST if the key/data pair already appears in the\n\t\tdatabase.\n\t
  • #MDB_NOOVERWRITE - enter the new key/data pair only if the key\n\t\tdoes not already appear in the database. The function will return\n\t\t#MDB_KEYEXIST if the key already appears in the database, even if\n\t\tthe database supports duplicates (#MDB_DUPSORT).\n\t
  • #MDB_RESERVE - reserve space for data of the given size, but\n\t\tdon't copy the given data. Instead, return a pointer to the\n\t\treserved space, which the caller can fill in later - before\n\t\tthe next update operation or the transaction ends. This saves\n\t\tan extra memcpy if the data is being generated later. This flag\n\t\tmust not be specified if the database was opened with #MDB_DUPSORT.\n\t
  • #MDB_APPEND - append the given key/data pair to the end of the\n\t\tdatabase. No key comparisons are performed. This option allows\n\t\tfast bulk loading when keys are already known to be in the\n\t\tcorrect order. Loading unsorted keys with this flag will cause\n\t\ta #MDB_KEYEXIST error.\n\t
  • #MDB_APPENDDUP - as above, but for sorted dup data.\n\t
  • #MDB_MULTIPLE - store multiple contiguous data elements in a\n\t\tsingle request. This flag may only be specified if the database\n\t\twas opened with #MDB_DUPFIXED. The **data** argument must be an\n\t\tarray of two MDB_vals. The mv_size of the first MDB_val must be\n\t\tthe size of a single data element. The mv_data of the first MDB_val\n\t\tmust point to the beginning of the array of contiguous data elements.\n\t\tThe mv_size of the second MDB_val must be the count of the number\n\t\tof data elements to store. On return this field will be set to\n\t\tthe count of the number of elements actually written. The mv_data\n\t\tof the second MDB_val is unused.\n
\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize().\n\t
  • #MDB_TXN_FULL - the transaction has too many dirty pages.\n\t
  • EACCES - an attempt was made to write in a read-only transaction.\n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_cursor_put( + cursor: *mut MDB_cursor, + key: *mut MDB_val, + data: *mut MDB_val, + flags: ::libc::c_uint, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Delete current key/data pair\n\n This function deletes the key/data pair to which the cursor refers.\n This does not invalidate the cursor, so operations such as MDB_NEXT\n can still be used on it.\n Both MDB_NEXT and MDB_GET_CURRENT will return the same record after\n this operation.\n # Arguments\n\n* `cursor` (direction in) - A cursor handle returned by #mdb_cursor_open()\n * `flags` (direction in) - Options for this operation. This parameter\n must be set to 0 or one of the values described here.\n
    \n\t
  • #MDB_NODUPDATA - delete all of the data items for the current key.\n\t\tThis flag may only be specified if the database was opened with #MDB_DUPSORT.\n
\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EACCES - an attempt was made to write in a read-only transaction.\n\t
  • EINVAL - an invalid parameter was specified.\n
"] + pub fn mdb_cursor_del(cursor: *mut MDB_cursor, flags: ::libc::c_uint) -> ::libc::c_int; +} +extern "C" { + #[doc = "Return count of duplicates for current key.\n\n This call is only valid on databases that support sorted duplicate\n data items #MDB_DUPSORT.\n # Arguments\n\n* `cursor` (direction in) - A cursor handle returned by #mdb_cursor_open()\n * `countp` (direction out) - Address where the count will be stored\n # Returns\n\nA non-zero error value on failure and 0 on success. Some possible\n errors are:\n
    \n\t
  • EINVAL - cursor is not initialized, or an invalid parameter was specified.\n
"] + pub fn mdb_cursor_count(cursor: *mut MDB_cursor, countp: *mut mdb_size_t) -> ::libc::c_int; +} +extern "C" { + #[doc = "Compare two data items according to a particular database.\n\n This returns a comparison as if the two data items were keys in the\n specified database.\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n * `dbi` (direction in) - A database handle returned by #mdb_dbi_open()\n * `a` (direction in) - The first item to compare\n * `b` (direction in) - The second item to compare\n # Returns\n\n< 0 if a < b, 0 if a == b, > 0 if a > b"] + pub fn mdb_cmp( + txn: *mut MDB_txn, + dbi: MDB_dbi, + a: *const MDB_val, + b: *const MDB_val, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Compare two data items according to a particular database.\n\n This returns a comparison as if the two items were data items of\n the specified database. The database must have the #MDB_DUPSORT flag.\n # Arguments\n\n* `txn` (direction in) - A transaction handle returned by #mdb_txn_begin()\n * `dbi` (direction in) - A database handle returned by #mdb_dbi_open()\n * `a` (direction in) - The first item to compare\n * `b` (direction in) - The second item to compare\n # Returns\n\n< 0 if a < b, 0 if a == b, > 0 if a > b"] + pub fn mdb_dcmp( + txn: *mut MDB_txn, + dbi: MDB_dbi, + a: *const MDB_val, + b: *const MDB_val, + ) -> ::libc::c_int; +} +#[doc = "A callback function used to print a message from the library.\n\n # Arguments\n\n* `msg` (direction in) - The string to be printed.\n * `ctx` (direction in) - An arbitrary context pointer for the callback.\n # Returns\n\n< 0 on failure, >= 0 on success."] +pub type MDB_msg_func = ::std::option::Option< + unsafe extern "C" fn(msg: *const ::libc::c_char, ctx: *mut ::libc::c_void) -> ::libc::c_int, +>; +extern "C" { + #[doc = "Dump the entries in the reader lock table.\n\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `func` (direction in) - A #MDB_msg_func function\n * `ctx` (direction in) - Anything the message function needs\n # Returns\n\n< 0 on failure, >= 0 on success."] + pub fn mdb_reader_list( + env: *mut MDB_env, + func: MDB_msg_func, + ctx: *mut ::libc::c_void, + ) -> ::libc::c_int; +} +extern "C" { + #[doc = "Check for stale entries in the reader lock table.\n\n # Arguments\n\n* `env` (direction in) - An environment handle returned by #mdb_env_create()\n * `dead` (direction out) - Number of stale slots that were cleared\n # Returns\n\n0 on success, non-zero on failure."] + pub fn mdb_reader_check(env: *mut MDB_env, dead: *mut ::libc::c_int) -> ::libc::c_int; +} +#[doc = "A function for converting a string into an encryption key.\n\n # Arguments\n\n* `passwd` (direction in) - The string to be converted.\n * `key` (direction in, out) - The resulting key. The caller must\n provide the space for the key.\n # Returns\n\n0 on success, non-zero on failure."] +pub type MDB_str2key_func = ::std::option::Option< + unsafe extern "C" fn(passwd: *const ::libc::c_char, key: *mut MDB_val) -> ::libc::c_int, +>; +#[doc = "A structure for dynamically loaded crypto modules.\n\n This is the information that the command line tools expect\n in order to operate on encrypted or checksummed environments."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct MDB_crypto_funcs { + pub mcf_str2key: MDB_str2key_func, + pub mcf_encfunc: MDB_enc_func, + pub mcf_sumfunc: MDB_sum_func, + #[doc = "< The size of an encryption key, in bytes"] + pub mcf_keysize: ::libc::c_int, + #[doc = "< The size of the MAC, for authenticated encryption"] + pub mcf_esumsize: ::libc::c_int, + #[doc = "< The size of the checksum, for plain checksums"] + pub mcf_sumsize: ::libc::c_int, +} +#[doc = "The function that returns the #MDB_crypto_funcs structure.\n\n The command line tools expect this function to be named \"MDB_crypto\".\n It must be exported by the dynamic module so that the tools can use it.\n # Returns\n\nA pointer to a #MDB_crypto_funcs structure."] +pub type MDB_crypto_hooks = ::std::option::Option *mut MDB_crypto_funcs>; diff --git a/lmdb-master3-sys/src/lib.rs b/lmdb-master3-sys/src/lib.rs new file mode 100644 index 00000000..d76518f0 --- /dev/null +++ b/lmdb-master3-sys/src/lib.rs @@ -0,0 +1,24 @@ +#![deny(warnings)] +#![allow(rustdoc::broken_intra_doc_links)] +#![allow(rustdoc::invalid_html_tags)] +#![allow(non_camel_case_types)] +#![allow(clippy::all)] +#![doc(html_root_url = "https://docs.rs/lmdb-master-sys/0.2.1")] + +extern crate libc; + +#[cfg(unix)] +#[allow(non_camel_case_types)] +pub type mdb_mode_t = ::libc::mode_t; +#[cfg(windows)] +#[allow(non_camel_case_types)] +pub type mdb_mode_t = ::libc::c_int; + +#[cfg(unix)] +#[allow(non_camel_case_types)] +pub type mdb_filehandle_t = ::libc::c_int; +#[cfg(windows)] +#[allow(non_camel_case_types)] +pub type mdb_filehandle_t = *mut ::libc::c_void; + +include!("bindings.rs"); diff --git a/lmdb-master3-sys/tests/lmdb.rs b/lmdb-master3-sys/tests/lmdb.rs new file mode 100644 index 00000000..f02284b4 --- /dev/null +++ b/lmdb-master3-sys/tests/lmdb.rs @@ -0,0 +1,26 @@ +use std::env; +use std::path::PathBuf; +use std::process::Command; + +#[test] +fn test_lmdb() { + let mut lmdb = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); + lmdb.push("lmdb"); + lmdb.push("libraries"); + lmdb.push("liblmdb"); + + let make_cmd = Command::new("make") + .current_dir(&lmdb) + .status() + .expect("failed to execute process"); + + assert!(make_cmd.success()); + + let make_test_cmd = Command::new("make") + .arg("test") + .current_dir(&lmdb) + .status() + .expect("failed to execute process"); + + assert!(make_test_cmd.success()); +} diff --git a/lmdb-master3-sys/tests/simple.rs b/lmdb-master3-sys/tests/simple.rs new file mode 100644 index 00000000..fe0da4dd --- /dev/null +++ b/lmdb-master3-sys/tests/simple.rs @@ -0,0 +1,95 @@ +use std::ffi::{c_void, CString}; +use std::fs::{self, File}; +use std::ptr; + +use cstr::cstr; +use lmdb_master3_sys::*; + +// https://github.com/victorporof/lmdb/blob/mdb.master/libraries/liblmdb/moz-test.c + +macro_rules! E { + ($expr:expr) => {{ + match $expr { + lmdb_master3_sys::MDB_SUCCESS => (), + err_code => assert!(false, "Failed with code {}", err_code), + } + }}; +} + +#[test] +#[cfg(target_pointer_width = "32")] +fn test_simple_32() { + test_simple("./tests/fixtures/testdb-32") +} + +#[test] +#[cfg(target_pointer_width = "64")] +fn test_simple_64() { + test_simple("./tests/fixtures/testdb") +} + +#[cfg(windows)] +fn get_file_fd(file: &File) -> std::os::windows::io::RawHandle { + use std::os::windows::io::AsRawHandle; + file.as_raw_handle() +} + +#[cfg(unix)] +fn get_file_fd(file: &File) -> std::os::unix::io::RawFd { + use std::os::unix::io::AsRawFd; + file.as_raw_fd() +} + +fn test_simple(env_path: &str) { + let _ = fs::remove_dir_all(env_path); + fs::create_dir_all(env_path).unwrap(); + + let mut env: *mut MDB_env = ptr::null_mut(); + let mut dbi: MDB_dbi = 0; + let mut key = MDB_val { mv_size: 0, mv_data: ptr::null_mut() }; + let mut data = MDB_val { mv_size: 0, mv_data: ptr::null_mut() }; + let mut txn: *mut MDB_txn = ptr::null_mut(); + let sval = cstr!("foo").as_ptr() as *mut c_void; + let dval = cstr!("bar").as_ptr() as *mut c_void; + + unsafe { + E!(mdb_env_create(&mut env)); + E!(mdb_env_set_maxdbs(env, 2)); + let env_path = CString::new(env_path).unwrap(); + E!(mdb_env_open(env, env_path.as_ptr(), 0, 0o664)); + + E!(mdb_txn_begin(env, ptr::null_mut(), 0, &mut txn)); + E!(mdb_dbi_open(txn, cstr!("subdb").as_ptr(), MDB_CREATE, &mut dbi)); + E!(mdb_txn_commit(txn)); + + key.mv_size = 3; + key.mv_data = sval; + data.mv_size = 3; + data.mv_data = dval; + + E!(mdb_txn_begin(env, ptr::null_mut(), 0, &mut txn)); + E!(mdb_put(txn, dbi, &mut key, &mut data, 0)); + + if cfg!(feature = "longer-keys") { + // Try storing a key larger than 511 bytes (the default if MDB_MAXKEYSIZE is not set) + let sval = cstr!("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut pharetra sit amet aliquam. Sit amet nisl purus in mollis nunc. Eget egestas purus viverra accumsan in nisl nisi scelerisque. Duis ultricies lacus sed turpis tincidunt. Sem nulla pharetra diam sit. Leo vel orci porta non pulvinar. Erat pellentesque adipiscing commodo elit at imperdiet dui. Suspendisse ultrices gravida dictum fusce ut placerat orci nulla. Diam donec adipiscing tristique risus nec feugiat. In fermentum et sollicitudin ac orci. Ut sem nulla pharetra diam sit amet. Aliquam purus sit amet luctus venenatis lectus. Erat pellentesque adipiscing commodo elit at imperdiet dui accumsan. Urna duis convallis convallis tellus id interdum velit laoreet id. Ac feugiat sed lectus vestibulum mattis ullamcorper velit sed. Tincidunt arcu non sodales neque. Habitant morbi tristique senectus et netus et malesuada fames.").as_ptr() as *mut c_void; + + key.mv_size = 952; + key.mv_data = sval; + + E!(mdb_put(txn, dbi, &mut key, &mut data, 0)); + } + + E!(mdb_txn_commit(txn)); + } + + let file = File::create("./tests/fixtures/copytestdb.mdb").unwrap(); + + unsafe { + let fd = get_file_fd(&file); + E!(mdb_env_copyfd(env, fd)); + + mdb_dbi_close(env, dbi); + mdb_env_close(env); + } +} From 27d9a98d422e427207c20770a4ab022592f818b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 16:09:26 +0200 Subject: [PATCH 06/51] Use master3 and imports everywhere needed --- heed/src/mdb/lmdb_error.rs | 24 ++++++++++++++++++++++++ heed/src/mdb/lmdb_ffi.rs | 5 +++++ heed/src/mdb/lmdb_flags.rs | 3 +++ heed3/Cargo.toml | 35 +++++++++++++++++++---------------- 4 files changed, 51 insertions(+), 16 deletions(-) diff --git a/heed/src/mdb/lmdb_error.rs b/heed/src/mdb/lmdb_error.rs index 36b62605..4dc74691 100644 --- a/heed/src/mdb/lmdb_error.rs +++ b/heed/src/mdb/lmdb_error.rs @@ -4,6 +4,9 @@ use std::os::raw::c_char; use std::{fmt, str}; use libc::c_int; +#[cfg(master3)] +use lmdb_master3_sys as ffi; +#[cfg(not(master3))] use lmdb_master_sys as ffi; /// An LMDB error kind. @@ -66,6 +69,15 @@ pub enum Error { BadDbi, /// Unexpected problem - transaction should abort. Problem, + /// Page checksum incorrect. + #[cfg(master3)] + BadChecksum, + /// Encryption/decryption failed. + #[cfg(master3)] + CryptoFail, + /// Environment encryption mismatch. + #[cfg(master3)] + EnvEncryption, /// Other error. Other(c_int), } @@ -100,6 +112,12 @@ impl Error { ffi::MDB_BAD_VALSIZE => Error::BadValSize, ffi::MDB_BAD_DBI => Error::BadDbi, ffi::MDB_PROBLEM => Error::Problem, + #[cfg(master3)] + ffi::MDB_BAD_CHECKSUM => Error::BadChecksum, + #[cfg(master3)] + ffi::MDB_CRYPTO_FAIL => Error::CryptoFail, + #[cfg(master3)] + ffi::MDB_ENV_ENCRYPTION => Error::EnvEncryption, other => Error::Other(other), } } @@ -129,6 +147,12 @@ impl Error { Error::BadValSize => ffi::MDB_BAD_VALSIZE, Error::BadDbi => ffi::MDB_BAD_DBI, Error::Problem => ffi::MDB_PROBLEM, + #[cfg(master3)] + Error::BadChecksum => ffi::MDB_BAD_CHECKSUM, + #[cfg(master3)] + Error::CryptoFail => ffi::MDB_CRYPTO_FAIL, + #[cfg(master3)] + Error::EnvEncryption => ffi::MDB_ENV_ENCRYPTION, Error::Other(err_code) => err_code, } } diff --git a/heed/src/mdb/lmdb_ffi.rs b/heed/src/mdb/lmdb_ffi.rs index 966a53e0..d387606b 100644 --- a/heed/src/mdb/lmdb_ffi.rs +++ b/heed/src/mdb/lmdb_ffi.rs @@ -10,6 +10,11 @@ pub use ffi::{ MDB_cursor, MDB_dbi, MDB_env, MDB_stat, MDB_txn, MDB_val, MDB_CP_COMPACT, MDB_CURRENT, MDB_RDONLY, MDB_RESERVE, }; +#[cfg(master3)] +pub use ffi::{mdb_env_set_checksum, mdb_env_set_encrypt}; +#[cfg(master3)] +use lmdb_master3_sys as ffi; +#[cfg(not(master3))] use lmdb_master_sys as ffi; pub mod cursor_op { diff --git a/heed/src/mdb/lmdb_flags.rs b/heed/src/mdb/lmdb_flags.rs index 3b72baca..f12464ac 100644 --- a/heed/src/mdb/lmdb_flags.rs +++ b/heed/src/mdb/lmdb_flags.rs @@ -1,4 +1,7 @@ use bitflags::bitflags; +#[cfg(master3)] +use lmdb_master3_sys as ffi; +#[cfg(not(master3))] use lmdb_master_sys as ffi; bitflags! { diff --git a/heed3/Cargo.toml b/heed3/Cargo.toml index a384af83..0578becb 100644 --- a/heed3/Cargo.toml +++ b/heed3/Cargo.toml @@ -11,8 +11,12 @@ readme = "../README.md" edition = "2021" [dependencies] +# TODO update dependencies +aead = { version = "0.5.1", default-features = false, optional = true } bitflags = { version = "2.6.0", features = ["serde"] } byteorder = { version = "1.5.0", default-features = false } +generic-array = { version = "0.14.6", features = ["serde"], optional = true } +heed-master3-proc-macro = { path = "../heed-master3-proc-macro", optional = true } heed-traits = { version = "0.20.0", path = "../heed-traits" } heed-types = { version = "0.20.1", default-features = false, path = "../heed-types" } libc = "0.2.155" @@ -21,13 +25,12 @@ once_cell = "1.19.0" page_size = "0.6.0" serde = { version = "1.0.203", features = ["derive"], optional = true } synchronoise = "1.0.1" -# TODO find a better name, depend on that only with heed3 -# move the proc macro directly in the heed3 crate (not possible?) -# Set a cfg to react on the crate name: https://github.com/bodil/im-rs/blob/master/build.rs -heed-master3-proc-macro = { path = "../heed-master3-proc-macro", optional = true } [dev-dependencies] +# TODO update dependencies +argon2 = { version = "0.4.1", features = ["std"] } serde = { version = "1.0.203", features = ["derive"] } +chacha20poly1305 = "0.10.1" tempfile = "3.10.1" [target.'cfg(windows)'.dependencies] @@ -41,7 +44,7 @@ default = ["serde", "serde-bincode", "serde-json", "encryption"] # Enable the LMDB encryption feature # TODO add more information here -encryption = ["dep:heed-master3-proc-macro"] +encryption = ["dep:heed-master3-proc-macro", "dep:aead", "dep:generic-array"] serde = ["bitflags/serde", "dep:serde"] @@ -78,7 +81,7 @@ unbounded_depth = ["heed-types/unbounded_depth"] # There are tradeoffs for both POSIX and SysV semaphores; which you # should look into before enabling this feature. Also, see here: # -posix-sem = ["lmdb-master-sys/posix-sem"] +posix-sem = ["lmdb-master3-sys/posix-sem"] # These features configure the MDB_IDL_LOGN macro, which determines # the size of the free and dirty page lists (and thus the amount of memory @@ -90,15 +93,15 @@ posix-sem = ["lmdb-master-sys/posix-sem"] # # For more information on the motivation for these features (and their effect), # see https://github.com/mozilla/lmdb/pull/2. -mdb_idl_logn_8 = ["lmdb-master-sys/mdb_idl_logn_8"] -mdb_idl_logn_9 = ["lmdb-master-sys/mdb_idl_logn_9"] -mdb_idl_logn_10 = ["lmdb-master-sys/mdb_idl_logn_10"] -mdb_idl_logn_11 = ["lmdb-master-sys/mdb_idl_logn_11"] -mdb_idl_logn_12 = ["lmdb-master-sys/mdb_idl_logn_12"] -mdb_idl_logn_13 = ["lmdb-master-sys/mdb_idl_logn_13"] -mdb_idl_logn_14 = ["lmdb-master-sys/mdb_idl_logn_14"] -mdb_idl_logn_15 = ["lmdb-master-sys/mdb_idl_logn_15"] -mdb_idl_logn_16 = ["lmdb-master-sys/mdb_idl_logn_16"] +mdb_idl_logn_8 = ["lmdb-master3-sys/mdb_idl_logn_8"] +mdb_idl_logn_9 = ["lmdb-master3-sys/mdb_idl_logn_9"] +mdb_idl_logn_10 = ["lmdb-master3-sys/mdb_idl_logn_10"] +mdb_idl_logn_11 = ["lmdb-master3-sys/mdb_idl_logn_11"] +mdb_idl_logn_12 = ["lmdb-master3-sys/mdb_idl_logn_12"] +mdb_idl_logn_13 = ["lmdb-master3-sys/mdb_idl_logn_13"] +mdb_idl_logn_14 = ["lmdb-master3-sys/mdb_idl_logn_14"] +mdb_idl_logn_15 = ["lmdb-master3-sys/mdb_idl_logn_15"] +mdb_idl_logn_16 = ["lmdb-master3-sys/mdb_idl_logn_16"] # Setting this enables you to use keys longer than 511 bytes. The exact limit # is computed by LMDB at compile time. You can find the exact value by calling @@ -118,7 +121,7 @@ mdb_idl_logn_16 = ["lmdb-master-sys/mdb_idl_logn_16"] # stored key must fit within the smallest limit of all architectures used. For # example, if you are moving databases between Apple M1 and Apple Intel # computers then you need to keep your keys within the smaller 1982 byte limit. -longer-keys = ["lmdb-master-sys/longer-keys"] +longer-keys = ["lmdb-master3-sys/longer-keys"] # Examples are located outside the standard heed/examples directory to prevent # conflicts between heed3 and heed examples when working on both crates. From b3041a8be049c900e25e940439cf2b8d1ca4c422 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 16:14:01 +0200 Subject: [PATCH 07/51] Do not use the mdb_master_sys crate directly but rather the ffi module --- heed/src/database.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heed/src/database.rs b/heed/src/database.rs index d4b2955c..97d1d8d9 100644 --- a/heed/src/database.rs +++ b/heed/src/database.rs @@ -2265,7 +2265,7 @@ impl Database { let mut key_val = unsafe { crate::into_val(&key_bytes) }; let mut reserved = ffi::reserve_size_val(data_size); - let flags = (flags | PutFlags::NO_OVERWRITE).bits() | lmdb_master_sys::MDB_RESERVE; + let flags = (flags | PutFlags::NO_OVERWRITE).bits() | ffi::MDB_RESERVE; let mut txn = txn.txn.txn.unwrap(); let result = unsafe { From 62fefaaf3882b5426f54eef0cf4c12d51cd152c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 16:17:29 +0200 Subject: [PATCH 08/51] Introduce a new encryption example for heed3 --- examples/encrypt-heed3.rs | 57 +++++++++++++++++++++++++++++++++++++++ heed3/Cargo.toml | 5 ++++ 2 files changed, 62 insertions(+) create mode 100644 examples/encrypt-heed3.rs diff --git a/examples/encrypt-heed3.rs b/examples/encrypt-heed3.rs new file mode 100644 index 00000000..a527a28d --- /dev/null +++ b/examples/encrypt-heed3.rs @@ -0,0 +1,57 @@ +use std::error::Error; +use std::fs; +use std::path::Path; + +use argon2::Argon2; +use chacha20poly1305::{ChaCha20Poly1305, Key}; +use heed3::types::*; +use heed3::{Database, EnvOpenOptions}; + +fn main() -> Result<(), Box> { + let env_path = Path::new("target").join("encrypt.mdb"); + let password = "This is the password that will be hashed by the argon2 algorithm"; + let salt = "The salt added to the password hashes to add more security when stored"; + + let _ = fs::remove_dir_all(&env_path); + fs::create_dir_all(&env_path)?; + + // We choose to use argon2 as our Key Derivation Function, but you can choose whatever you want. + // + let mut key = Key::default(); + Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; + + // We open the environment + let mut options = EnvOpenOptions::new().encrypt_with::(key); + let env = options + .map_size(10 * 1024 * 1024) // 10MB + .max_dbs(3) + .open(&env_path)?; + + let key1 = "first-key"; + let val1 = "this is a secret info"; + let key2 = "second-key"; + let val2 = "this is another secret info"; + + // We create database and write secret values in it + let mut wtxn = env.write_txn()?; + let db: Database = env.create_database(&mut wtxn, Some("first"))?; + db.put(&mut wtxn, key1, val1)?; + db.put(&mut wtxn, key2, val2)?; + wtxn.commit()?; + env.prepare_for_closing().wait(); + + // We reopen the environment now + let env = options.open(&env_path)?; + + // We check that the secret entries are correctly decrypted + let rtxn = env.read_txn()?; + let db: Database = env.open_database(&rtxn, Some("first"))?.unwrap(); + let mut iter = db.iter(&rtxn)?; + assert_eq!(iter.next().transpose()?, Some((key1, val1))); + assert_eq!(iter.next().transpose()?, Some((key2, val2))); + assert_eq!(iter.next().transpose()?, None); + + eprintln!("Successful test!"); + + Ok(()) +} diff --git a/heed3/Cargo.toml b/heed3/Cargo.toml index 0578becb..b6ebae5c 100644 --- a/heed3/Cargo.toml +++ b/heed3/Cargo.toml @@ -128,3 +128,8 @@ longer-keys = ["lmdb-master3-sys/longer-keys"] [[example]] name = "all-types-heed3" path = "../examples/all-types-heed3.rs" + +[[example]] +name = "encrypt-heed3" +path = "../examples/encrypt-heed3.rs" +required-features = ["encryption"] From 865b1b7ac77780691117e16dd030965f5075e774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 16:17:44 +0200 Subject: [PATCH 09/51] Fix formatting of some LMDB tests --- lmdb-master3-sys/tests/simple.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/lmdb-master3-sys/tests/simple.rs b/lmdb-master3-sys/tests/simple.rs index fe0da4dd..1eadaee1 100644 --- a/lmdb-master3-sys/tests/simple.rs +++ b/lmdb-master3-sys/tests/simple.rs @@ -46,8 +46,14 @@ fn test_simple(env_path: &str) { let mut env: *mut MDB_env = ptr::null_mut(); let mut dbi: MDB_dbi = 0; - let mut key = MDB_val { mv_size: 0, mv_data: ptr::null_mut() }; - let mut data = MDB_val { mv_size: 0, mv_data: ptr::null_mut() }; + let mut key = MDB_val { + mv_size: 0, + mv_data: ptr::null_mut(), + }; + let mut data = MDB_val { + mv_size: 0, + mv_data: ptr::null_mut(), + }; let mut txn: *mut MDB_txn = ptr::null_mut(); let sval = cstr!("foo").as_ptr() as *mut c_void; let dval = cstr!("bar").as_ptr() as *mut c_void; @@ -59,7 +65,12 @@ fn test_simple(env_path: &str) { E!(mdb_env_open(env, env_path.as_ptr(), 0, 0o664)); E!(mdb_txn_begin(env, ptr::null_mut(), 0, &mut txn)); - E!(mdb_dbi_open(txn, cstr!("subdb").as_ptr(), MDB_CREATE, &mut dbi)); + E!(mdb_dbi_open( + txn, + cstr!("subdb").as_ptr(), + MDB_CREATE, + &mut dbi + )); E!(mdb_txn_commit(txn)); key.mv_size = 3; From cf5f037be32a6254e07b66ba68d4c9a968d0e8e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 16:27:18 +0200 Subject: [PATCH 10/51] Remove useless test for master3 --- lmdb-master3-sys/tests/simple.rs | 106 ------------------------------- 1 file changed, 106 deletions(-) delete mode 100644 lmdb-master3-sys/tests/simple.rs diff --git a/lmdb-master3-sys/tests/simple.rs b/lmdb-master3-sys/tests/simple.rs deleted file mode 100644 index 1eadaee1..00000000 --- a/lmdb-master3-sys/tests/simple.rs +++ /dev/null @@ -1,106 +0,0 @@ -use std::ffi::{c_void, CString}; -use std::fs::{self, File}; -use std::ptr; - -use cstr::cstr; -use lmdb_master3_sys::*; - -// https://github.com/victorporof/lmdb/blob/mdb.master/libraries/liblmdb/moz-test.c - -macro_rules! E { - ($expr:expr) => {{ - match $expr { - lmdb_master3_sys::MDB_SUCCESS => (), - err_code => assert!(false, "Failed with code {}", err_code), - } - }}; -} - -#[test] -#[cfg(target_pointer_width = "32")] -fn test_simple_32() { - test_simple("./tests/fixtures/testdb-32") -} - -#[test] -#[cfg(target_pointer_width = "64")] -fn test_simple_64() { - test_simple("./tests/fixtures/testdb") -} - -#[cfg(windows)] -fn get_file_fd(file: &File) -> std::os::windows::io::RawHandle { - use std::os::windows::io::AsRawHandle; - file.as_raw_handle() -} - -#[cfg(unix)] -fn get_file_fd(file: &File) -> std::os::unix::io::RawFd { - use std::os::unix::io::AsRawFd; - file.as_raw_fd() -} - -fn test_simple(env_path: &str) { - let _ = fs::remove_dir_all(env_path); - fs::create_dir_all(env_path).unwrap(); - - let mut env: *mut MDB_env = ptr::null_mut(); - let mut dbi: MDB_dbi = 0; - let mut key = MDB_val { - mv_size: 0, - mv_data: ptr::null_mut(), - }; - let mut data = MDB_val { - mv_size: 0, - mv_data: ptr::null_mut(), - }; - let mut txn: *mut MDB_txn = ptr::null_mut(); - let sval = cstr!("foo").as_ptr() as *mut c_void; - let dval = cstr!("bar").as_ptr() as *mut c_void; - - unsafe { - E!(mdb_env_create(&mut env)); - E!(mdb_env_set_maxdbs(env, 2)); - let env_path = CString::new(env_path).unwrap(); - E!(mdb_env_open(env, env_path.as_ptr(), 0, 0o664)); - - E!(mdb_txn_begin(env, ptr::null_mut(), 0, &mut txn)); - E!(mdb_dbi_open( - txn, - cstr!("subdb").as_ptr(), - MDB_CREATE, - &mut dbi - )); - E!(mdb_txn_commit(txn)); - - key.mv_size = 3; - key.mv_data = sval; - data.mv_size = 3; - data.mv_data = dval; - - E!(mdb_txn_begin(env, ptr::null_mut(), 0, &mut txn)); - E!(mdb_put(txn, dbi, &mut key, &mut data, 0)); - - if cfg!(feature = "longer-keys") { - // Try storing a key larger than 511 bytes (the default if MDB_MAXKEYSIZE is not set) - let sval = cstr!("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut pharetra sit amet aliquam. Sit amet nisl purus in mollis nunc. Eget egestas purus viverra accumsan in nisl nisi scelerisque. Duis ultricies lacus sed turpis tincidunt. Sem nulla pharetra diam sit. Leo vel orci porta non pulvinar. Erat pellentesque adipiscing commodo elit at imperdiet dui. Suspendisse ultrices gravida dictum fusce ut placerat orci nulla. Diam donec adipiscing tristique risus nec feugiat. In fermentum et sollicitudin ac orci. Ut sem nulla pharetra diam sit amet. Aliquam purus sit amet luctus venenatis lectus. Erat pellentesque adipiscing commodo elit at imperdiet dui accumsan. Urna duis convallis convallis tellus id interdum velit laoreet id. Ac feugiat sed lectus vestibulum mattis ullamcorper velit sed. Tincidunt arcu non sodales neque. Habitant morbi tristique senectus et netus et malesuada fames.").as_ptr() as *mut c_void; - - key.mv_size = 952; - key.mv_data = sval; - - E!(mdb_put(txn, dbi, &mut key, &mut data, 0)); - } - - E!(mdb_txn_commit(txn)); - } - - let file = File::create("./tests/fixtures/copytestdb.mdb").unwrap(); - - unsafe { - let fd = get_file_fd(&file); - E!(mdb_env_copyfd(env, fd)); - - mdb_dbi_close(env, dbi); - mdb_env_close(env); - } -} From 932b591865b002e4cd5b47f57e8ffd66aa51a2f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 17:19:55 +0200 Subject: [PATCH 11/51] Import the work of encryption --- heed/src/env.rs | 507 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 492 insertions(+), 15 deletions(-) diff --git a/heed/src/env.rs b/heed/src/env.rs index 4b7bf98c..47c7b34e 100644 --- a/heed/src/env.rs +++ b/heed/src/env.rs @@ -22,6 +22,11 @@ use std::{ }; use std::{fmt, io, mem, ptr}; +#[cfg(master3)] +use aead::{ + consts::U0, generic_array::typenum::Unsigned, generic_array::GenericArray, AeadCore, + AeadMutInPlace, Key, KeyInit, KeySizeUser, Nonce, Tag, +}; use heed_traits::{Comparator, LexicographicComparator}; use once_cell::sync::Lazy; use synchronoise::event::SignalEvent; @@ -35,7 +40,7 @@ use crate::{Database, EnvFlags, Error, Result, RoCursor, RoTxn, RwTxn, Unspecifi /// The list of opened environments, the value is an optional environment, it is None /// when someone asks to close the environment, closing is a two-phase step, to make sure -/// noone tries to open the same environment between these two phases. +/// no one tries to open the same environment between these two phases. /// /// Trying to open a None marked environment returns an error to the user trying to open it. static OPENED_ENV: Lazy>> = Lazy::new(RwLock::default); @@ -43,7 +48,50 @@ static OPENED_ENV: Lazy>> = Lazy::new(RwLock:: struct EnvEntry { env: Option, signal_event: Arc, - options: EnvOpenOptions, + options: SimplifiedOpenOptions, +} + +/// A simplified version of the options that were used to open a given [`Env`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SimplifiedOpenOptions { + /// Weither this [`Env`] has been opened with an encryption/decryption algorithm. + #[cfg(master3)] + pub use_encryption: bool, + /// The maximum size this [`Env`] with take in bytes or [`None`] if it was not specified. + pub map_size: Option, + /// The maximum number of concurrent readers or [`None`] if it was not specified. + pub max_readers: Option, + /// The maximum number of opened database or [`None`] if it was not specified. + pub max_dbs: Option, + /// The raw flags enabled for this [`Env`] or [`None`] if it was not specified. + pub flags: u32, +} + +#[cfg(not(master3))] +impl From<&EnvOpenOptions> for SimplifiedOpenOptions { + fn from(eoo: &EnvOpenOptions) -> SimplifiedOpenOptions { + let EnvOpenOptions { map_size, max_readers, max_dbs, flags } = eoo; + SimplifiedOpenOptions { + map_size: *map_size, + max_readers: *max_readers, + max_dbs: *max_dbs, + flags: flags.bits(), + } + } +} + +#[cfg(master3)] +impl From<&EnvOpenOptions> for SimplifiedOpenOptions { + fn from(eoo: &EnvOpenOptions) -> SimplifiedOpenOptions { + let EnvOpenOptions { encrypt, map_size, max_readers, max_dbs, flags } = eoo; + SimplifiedOpenOptions { + use_encryption: encrypt.is_some(), + map_size: *map_size, + max_readers: *max_readers, + max_dbs: *max_dbs, + flags: flags.bits(), + } + } } // Thanks to the mozilla/rkv project @@ -102,8 +150,9 @@ unsafe fn metadata_from_fd(raw_fd: RawHandle) -> io::Result { } /// Options and flags which can be used to configure how an environment is opened. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg(not(master3))] pub struct EnvOpenOptions { map_size: Option, max_readers: Option, @@ -111,23 +160,54 @@ pub struct EnvOpenOptions { flags: EnvFlags, } +/// Options and flags which can be used to configure how an environment is opened. +#[derive(Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg(master3)] +pub struct EnvOpenOptions { + encrypt: Option<(std::marker::PhantomData, Key)>, + map_size: Option, + max_readers: Option, + max_dbs: Option, + flags: EnvFlags, +} + impl Default for EnvOpenOptions { fn default() -> Self { Self::new() } } +#[cfg(master3)] +impl fmt::Debug for EnvOpenOptions { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let EnvOpenOptions { encrypt, map_size, max_readers, max_dbs, flags } = self; + f.debug_struct("EnvOpenOptions") + .field("encrypted", &encrypt.is_some()) + .field("map_size", &map_size) + .field("max_readers", &max_readers) + .field("max_dbs", &max_dbs) + .field("flags", &flags) + .finish() + } +} + impl EnvOpenOptions { /// Creates a blank new set of options ready for configuration. pub fn new() -> EnvOpenOptions { EnvOpenOptions { + #[cfg(all(master3, feature = "encryption"))] + encrypt: None, map_size: None, max_readers: None, max_dbs: None, flags: EnvFlags::empty(), } } +} +#[cfg(master3)] +impl EnvOpenOptions { /// Set the size of the memory map to use for this environment. pub fn map_size(&mut self, size: usize) -> &mut Self { self.map_size = Some(size); @@ -146,7 +226,71 @@ impl EnvOpenOptions { self } - /// Set one or more [LMDB flags](http://www.lmdb.tech/doc/group__mdb__env.html). + /// Specifies that the [`Env`] will be encrypted using the `A` algorithm with the given `key`. + /// + /// You can find more compatible algorithms on [the RustCrypto/AEADs page](https://github.com/RustCrypto/AEADs#crates). + /// + /// Note that you cannot use any type of encryption algorithm as LMDB exposes a nonce of 16 bytes. + /// Heed makes sure to truncate it if necessary. + /// + /// As an example, XChaCha20 requires a 20 bytes long nonce. However, XChaCha20 is used to protect + /// against nonce misuse in systems that use randomly generated nonces i.e., to protect against + /// weak RNGs. There is no need to use this kind of algorithm in LMDB since LMDB nonces aren't + /// random and are guaranteed to be unique. + /// + /// ``` + /// use std::fs; + /// use std::path::Path; + /// use argon2::Argon2; + /// use chacha20poly1305::{ChaCha20Poly1305, Key}; + /// use heed3::types::*; + /// use heed3::{EnvOpenOptions, Database}; + /// + /// # fn main() -> Result<(), Box> { + /// let env_path = Path::new("target").join("encrypt.mdb"); + /// let password = "This is the password that will be hashed by the argon2 algorithm"; + /// let salt = "The salt added to the password hashes to add more security when stored"; + /// + /// let _ = fs::remove_dir_all(&env_path); + /// fs::create_dir_all(&env_path)?; + /// + /// let mut key = Key::default(); + /// Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; + /// + /// // We open the environment + /// let mut options = EnvOpenOptions::new().encrypt_with::(key); + /// let env = options + /// .map_size(10 * 1024 * 1024) // 10MB + /// .max_dbs(3) + /// .open(&env_path)?; + /// + /// let key1 = "first-key"; + /// let val1 = "this is a secret info"; + /// let key2 = "second-key"; + /// let val2 = "this is another secret info"; + /// + /// // We create database and write secret values in it + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("first"))?; + /// db.put(&mut wtxn, key1, val1)?; + /// db.put(&mut wtxn, key2, val2)?; + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + #[cfg(all(master3, feature = "encryption"))] + pub fn encrypt_with(self, key: Key) -> EnvOpenOptions { + let EnvOpenOptions { encrypt: _, map_size, max_readers, max_dbs, flags } = self; + EnvOpenOptions { + encrypt: Some((std::marker::PhantomData, key)), + map_size, + max_readers, + max_dbs, + flags, + } + } + + /// Set one or [more LMDB flags](http://www.lmdb.tech/doc/group__mdb__env.html). + /// /// ``` /// use std::fs; /// use std::path::Path; @@ -160,7 +304,7 @@ impl EnvOpenOptions { /// let dir = tempfile::tempdir().unwrap(); /// let env = unsafe { env_builder.open(dir.path())? }; /// - /// // we will open the default unnamed database + /// // we will open the default unamed database /// let mut wtxn = env.write_txn()?; /// let db: Database> = env.create_database(&mut wtxn, None)?; /// @@ -171,7 +315,7 @@ impl EnvOpenOptions { /// db.put(&mut wtxn, "three", &3)?; /// wtxn.commit()?; /// - /// // force the OS to flush the buffers (see Flag::NoSync and Flag::NoMetaSync). + /// // Force the OS to flush the buffers (see Flag::NoSync and Flag::NoMetaSync). /// env.force_sync(); /// /// // opening a read transaction @@ -220,19 +364,222 @@ impl EnvOpenOptions { /// For more details, it is highly recommended to read LMDB's official documentation. [^8] /// /// [^1]: - /// /// [^2]: - /// /// [^3]: - /// /// [^4]: - /// /// [^5]: - /// /// [^6]: - /// /// [^7]: + /// [^8]: + pub unsafe fn open>(&self, path: P) -> Result { + let mut lock = OPENED_ENV.write().unwrap(); + + let path = match canonicalize_path(path.as_ref()) { + Err(err) => { + if err.kind() == NotFound && self.flags.contains(EnvFlags::NO_SUB_DIR) { + let path = path.as_ref(); + match path.parent().zip(path.file_name()) { + Some((dir, file_name)) => canonicalize_path(dir)?.join(file_name), + None => return Err(err.into()), + } + } else { + return Err(err.into()); + } + } + Ok(path) => path, + }; + + let original_options = SimplifiedOpenOptions::from(self); + match lock.entry(path) { + Entry::Occupied(entry) => { + let env = entry.get().env.clone().ok_or(Error::DatabaseClosing)?; + let options = entry.get().options.clone(); + if options == original_options { + Ok(env) + } else { + // Err(Error::BadOpenOptions { env, original_options }) + todo!() + } + } + Entry::Vacant(entry) => { + let path = entry.key(); + let path_str = CString::new(path.as_os_str().as_bytes()).unwrap(); + + unsafe { + let mut env: *mut ffi::MDB_env = ptr::null_mut(); + mdb_result(ffi::mdb_env_create(&mut env))?; + + #[cfg(master3)] + if let Some((_marker, key)) = &self.encrypt { + let key = crate::into_val(key); + mdb_result(ffi::mdb_env_set_encrypt( + env, + Some(encrypt_func_wrapper::), + &key, + ::TagSize::U32, + ))?; + } + + if let Some(size) = self.map_size { + if size % page_size::get() != 0 { + let msg = format!( + "map size ({}) must be a multiple of the system page size ({})", + size, + page_size::get() + ); + return Err(Error::Io(io::Error::new( + io::ErrorKind::InvalidInput, + msg, + ))); + } + mdb_result(ffi::mdb_env_set_mapsize(env, size))?; + } + + if let Some(readers) = self.max_readers { + mdb_result(ffi::mdb_env_set_maxreaders(env, readers))?; + } + + if let Some(dbs) = self.max_dbs { + mdb_result(ffi::mdb_env_set_maxdbs(env, dbs))?; + } + + // When the `read-txn-no-tls` feature is enabled, we must force LMDB + // to avoid using the thread local storage, this way we allow users + // to use references of RoTxn between threads safely. + let flags = if cfg!(feature = "read-txn-no-tls") { + self.flags | EnvFlags::NO_TLS + } else { + self.flags + }; + + let result = + mdb_result(ffi::mdb_env_open(env, path_str.as_ptr(), flags.bits(), 0o600)); + + match result { + Ok(()) => { + let signal_event = Arc::new(SignalEvent::manual(false)); + let inner = EnvInner { env, path: path.clone() }; + let env = Env(Arc::new(inner)); + let cache_entry = EnvEntry { + env: Some(env.clone()), + options: original_options, + signal_event, + }; + entry.insert(cache_entry); + Ok(env) + } + Err(e) => { + ffi::mdb_env_close(env); + Err(e.into()) + } + } + } + } + } + } +} + +#[cfg(not(master3))] +impl EnvOpenOptions { + /// Set the size of the memory map to use for this environment. + pub fn map_size(&mut self, size: usize) -> &mut Self { + self.map_size = Some(size); + self + } + + /// Set the maximum number of threads/reader slots for the environment. + pub fn max_readers(&mut self, readers: u32) -> &mut Self { + self.max_readers = Some(readers); + self + } + + /// Set the maximum number of named databases for the environment. + pub fn max_dbs(&mut self, dbs: u32) -> &mut Self { + self.max_dbs = Some(dbs); + self + } + + /// Set one or [more LMDB flags](http://www.lmdb.tech/doc/group__mdb__env.html). + /// + /// ``` + /// use std::fs; + /// use std::path::Path; + /// use heed::{EnvOpenOptions, Database, EnvFlags}; + /// use heed::types::*; + /// + /// # fn main() -> Result<(), Box> { + /// fs::create_dir_all(Path::new("target").join("database.mdb"))?; + /// let mut env_builder = EnvOpenOptions::new(); + /// unsafe { env_builder.flags(EnvFlags::NO_TLS | EnvFlags::NO_META_SYNC); } + /// let dir = tempfile::tempdir().unwrap(); + /// let env = unsafe { env_builder.open(dir.path())? }; + /// + /// // we will open the default unamed database + /// let mut wtxn = env.write_txn()?; + /// let db: Database> = env.create_database(&mut wtxn, None)?; + /// + /// // opening a write transaction + /// db.put(&mut wtxn, "seven", &7)?; + /// db.put(&mut wtxn, "zero", &0)?; + /// db.put(&mut wtxn, "five", &5)?; + /// db.put(&mut wtxn, "three", &3)?; + /// wtxn.commit()?; + /// + /// // Force the OS to flush the buffers (see Flag::NoSync and Flag::NoMetaSync). + /// env.force_sync(); + /// + /// // opening a read transaction + /// // to check if those values are now available + /// let mut rtxn = env.read_txn()?; + /// + /// let ret = db.get(&rtxn, "zero")?; + /// assert_eq!(ret, Some(0)); + /// + /// let ret = db.get(&rtxn, "five")?; + /// assert_eq!(ret, Some(5)); + /// # Ok(()) } + /// ``` + /// + /// # Safety + /// + /// It is unsafe to use unsafe LMDB flags such as `NO_SYNC`, `NO_META_SYNC`, or `NO_LOCK`. + pub unsafe fn flags(&mut self, flags: EnvFlags) -> &mut Self { + self.flags |= flags; + self + } + + /// Open an environment that will be located at the specified path. + /// + /// # Safety + /// LMDB is backed by a memory map [^1] which comes with some safety precautions. + /// + /// Memory map constructors are marked `unsafe` because of the potential + /// for Undefined Behavior (UB) using the map if the underlying file is + /// subsequently modified, in or out of process. + /// + /// LMDB itself has a locking system that solves this problem, + /// but it will not save you from making mistakes yourself. + /// + /// These are some things to take note of: + /// + /// - Avoid long-lived transactions, they will cause the database to grow quickly [^2] + /// - Avoid aborting your process with an active transaction [^3] + /// - Do not use LMDB on remote filesystems, even between processes on the same host [^4] + /// - You must manage concurrent accesses yourself if using [`EnvFlags::NO_LOCK`] [^5] + /// - Anything that causes LMDB's lock file to be broken will cause synchronization issues and may introduce UB [^6] + /// + /// `heed` itself upholds some safety invariants, including but not limited to: + /// - Calling [`EnvOpenOptions::open`] twice in the same process, at the same time is OK [^7] /// + /// For more details, it is highly recommended to read LMDB's official documentation. [^8] + /// + /// [^1]: + /// [^2]: + /// [^3]: + /// [^4]: + /// [^5]: + /// [^6]: + /// [^7]: /// [^8]: pub unsafe fn open>(&self, path: P) -> Result { let mut lock = OPENED_ENV.write().unwrap(); @@ -252,14 +599,16 @@ impl EnvOpenOptions { Ok(path) => path, }; + let original_options = SimplifiedOpenOptions::from(self); match lock.entry(path) { Entry::Occupied(entry) => { let env = entry.get().env.clone().ok_or(Error::DatabaseClosing)?; let options = entry.get().options.clone(); - if &options == self { + if options == original_options { Ok(env) } else { - Err(Error::BadOpenOptions { env, options }) + // Err(Error::BadOpenOptions { env, original_options }) + todo!() } } Entry::Vacant(entry) => { @@ -312,7 +661,7 @@ impl EnvOpenOptions { let env = Env(Arc::new(inner)); let cache_entry = EnvEntry { env: Some(env.clone()), - options: self.clone(), + options: original_options, signal_event, }; entry.insert(cache_entry); @@ -329,6 +678,88 @@ impl EnvOpenOptions { } } +#[cfg(master3)] +fn encrypt( + key: &[u8], + nonce: &[u8], + aad: &[u8], + plaintext: &[u8], + chipertext_out: &mut [u8], + auth_out: &mut [u8], +) -> aead::Result<()> { + chipertext_out.copy_from_slice(plaintext); + let key: &Key = key.try_into().unwrap(); + let nonce: &Nonce = if nonce.len() >= A::NonceSize::USIZE { + nonce[..A::NonceSize::USIZE].into() + } else { + return Err(aead::Error); + }; + let mut aead = A::new(key); + let tag = aead.encrypt_in_place_detached(nonce, aad, chipertext_out)?; + auth_out.copy_from_slice(&tag); + Ok(()) +} + +#[cfg(master3)] +fn decrypt( + key: &[u8], + nonce: &[u8], + aad: &[u8], + chipher_text: &[u8], + output: &mut [u8], + auth_in: &[u8], +) -> aead::Result<()> { + output.copy_from_slice(chipher_text); + let key: &Key = key.try_into().unwrap(); + let nonce: &Nonce = if nonce.len() >= A::NonceSize::USIZE { + nonce[..A::NonceSize::USIZE].into() + } else { + return Err(aead::Error); + }; + let tag: &Tag = auth_in.try_into().unwrap(); + let mut aead = A::new(key); + aead.decrypt_in_place_detached(nonce, aad, output, tag) +} + +/// The wrapper function that is called by LMDB that directly calls +/// the Rust idiomatic function internally. +#[cfg(master3)] +unsafe extern "C" fn encrypt_func_wrapper( + src: *const ffi::MDB_val, + dst: *mut ffi::MDB_val, + key_ptr: *const ffi::MDB_val, + encdec: i32, +) -> i32 { + let result = catch_unwind(|| { + let input = std::slice::from_raw_parts((*src).mv_data as *const u8, (*src).mv_size); + let output = std::slice::from_raw_parts_mut((*dst).mv_data as *mut u8, (*dst).mv_size); + let key = std::slice::from_raw_parts((*key_ptr).mv_data as *const u8, (*key_ptr).mv_size); + let iv = std::slice::from_raw_parts( + (*key_ptr.offset(1)).mv_data as *const u8, + (*key_ptr.offset(1)).mv_size, + ); + let auth = std::slice::from_raw_parts_mut( + (*key_ptr.offset(2)).mv_data as *mut u8, + (*key_ptr.offset(2)).mv_size, + ); + + let aad = []; + let nonce = iv; + let result = if encdec == 1 { + encrypt::(&key, nonce, &aad, input, output, auth) + } else { + decrypt::(&key, nonce, &aad, input, output, auth) + }; + + result.is_err() as i32 + }); + + match result { + Ok(out) => out, + Err(_) => 1, + } +} + /// Returns a struct that allows to wait for the effective closing of an environment. pub fn env_closing_event>(path: P) -> Option { let lock = OPENED_ENV.read().unwrap(); @@ -1014,6 +1445,52 @@ impl fmt::Debug for EnvClosingEvent { } } +/// A dummy encryption/decryption algorithm that must never be used. +/// Only here for Rust API purposes. +#[cfg(master3)] +pub enum DummyEncrypt {} + +#[cfg(master3)] +impl AeadMutInPlace for DummyEncrypt { + fn encrypt_in_place_detached( + &mut self, + _nonce: &Nonce, + _associated_data: &[u8], + _buffer: &mut [u8], + ) -> aead::Result> { + Err(aead::Error) + } + + fn decrypt_in_place_detached( + &mut self, + _nonce: &Nonce, + _associated_data: &[u8], + _buffer: &mut [u8], + _tag: &Tag, + ) -> aead::Result<()> { + Err(aead::Error) + } +} + +#[cfg(master3)] +impl AeadCore for DummyEncrypt { + type NonceSize = U0; + type TagSize = U0; + type CiphertextOverhead = U0; +} + +#[cfg(master3)] +impl KeySizeUser for DummyEncrypt { + type KeySize = U0; +} + +#[cfg(master3)] +impl KeyInit for DummyEncrypt { + fn new(_key: &GenericArray) -> Self { + panic!("This DummyEncrypt type must not be used") + } +} + #[cfg(test)] mod tests { use std::io::ErrorKind; From a9d45732e864d95b3b00f8e3273ed4bf703be029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 17:22:07 +0200 Subject: [PATCH 12/51] Fix the encryption test to pass --- examples/encrypt-heed3.rs | 16 +++++++++------- heed/src/mdb/lmdb_ffi.rs | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/examples/encrypt-heed3.rs b/examples/encrypt-heed3.rs index a527a28d..9cd2841f 100644 --- a/examples/encrypt-heed3.rs +++ b/examples/encrypt-heed3.rs @@ -22,10 +22,12 @@ fn main() -> Result<(), Box> { // We open the environment let mut options = EnvOpenOptions::new().encrypt_with::(key); - let env = options - .map_size(10 * 1024 * 1024) // 10MB - .max_dbs(3) - .open(&env_path)?; + let env = unsafe { + options + .map_size(10 * 1024 * 1024) // 10MB + .max_dbs(3) + .open(&env_path)? + }; let key1 = "first-key"; let val1 = "this is a secret info"; @@ -41,12 +43,12 @@ fn main() -> Result<(), Box> { env.prepare_for_closing().wait(); // We reopen the environment now - let env = options.open(&env_path)?; + let env = unsafe { options.open(&env_path)? }; // We check that the secret entries are correctly decrypted - let rtxn = env.read_txn()?; + let mut rtxn = env.read_txn()?; let db: Database = env.open_database(&rtxn, Some("first"))?.unwrap(); - let mut iter = db.iter(&rtxn)?; + let mut iter = db.iter(&mut rtxn)?; assert_eq!(iter.next().transpose()?, Some((key1, val1))); assert_eq!(iter.next().transpose()?, Some((key2, val2))); assert_eq!(iter.next().transpose()?, None); diff --git a/heed/src/mdb/lmdb_ffi.rs b/heed/src/mdb/lmdb_ffi.rs index d387606b..cb8b3ba5 100644 --- a/heed/src/mdb/lmdb_ffi.rs +++ b/heed/src/mdb/lmdb_ffi.rs @@ -1,5 +1,7 @@ use std::ptr; +#[cfg(master3)] +pub use ffi::mdb_env_set_encrypt; pub use ffi::{ mdb_cursor_close, mdb_cursor_del, mdb_cursor_get, mdb_cursor_open, mdb_cursor_put, mdb_dbi_open, mdb_del, mdb_drop, mdb_env_close, mdb_env_copyfd2, mdb_env_create, @@ -11,8 +13,6 @@ pub use ffi::{ MDB_RDONLY, MDB_RESERVE, }; #[cfg(master3)] -pub use ffi::{mdb_env_set_checksum, mdb_env_set_encrypt}; -#[cfg(master3)] use lmdb_master3_sys as ffi; #[cfg(not(master3))] use lmdb_master_sys as ffi; From cc5f3dd199efb8938895edebd9eae75ac8efe8f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 17:28:26 +0200 Subject: [PATCH 13/51] Fix some issues in Env opening --- heed/src/env.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/heed/src/env.rs b/heed/src/env.rs index 47c7b34e..f665d8dc 100644 --- a/heed/src/env.rs +++ b/heed/src/env.rs @@ -48,7 +48,10 @@ static OPENED_ENV: Lazy>> = Lazy::new(RwLock:: struct EnvEntry { env: Option, signal_event: Arc, + #[cfg(master3)] options: SimplifiedOpenOptions, + #[cfg(not(master3))] + options: EnvOpenOptions, } /// A simplified version of the options that were used to open a given [`Env`]. @@ -397,8 +400,7 @@ impl EnvOpenOptions { if options == original_options { Ok(env) } else { - // Err(Error::BadOpenOptions { env, original_options }) - todo!() + Err(Error::BadOpenOptions { env, original_options }) } } Entry::Vacant(entry) => { @@ -599,16 +601,14 @@ impl EnvOpenOptions { Ok(path) => path, }; - let original_options = SimplifiedOpenOptions::from(self); match lock.entry(path) { Entry::Occupied(entry) => { let env = entry.get().env.clone().ok_or(Error::DatabaseClosing)?; let options = entry.get().options.clone(); - if options == original_options { + if &options == self { Ok(env) } else { - // Err(Error::BadOpenOptions { env, original_options }) - todo!() + Err(Error::BadOpenOptions { env, options }) } } Entry::Vacant(entry) => { @@ -661,7 +661,7 @@ impl EnvOpenOptions { let env = Env(Arc::new(inner)); let cache_entry = EnvEntry { env: Some(env.clone()), - options: original_options, + options: self.clone(), signal_event, }; entry.insert(cache_entry); From 116253a64eca0db2af9f9f4f94b1147f10ffbdcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 17:31:52 +0200 Subject: [PATCH 14/51] Fix simplified open options --- heed/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/heed/src/lib.rs b/heed/src/lib.rs index 906b1db3..3a32633c 100644 --- a/heed/src/lib.rs +++ b/heed/src/lib.rs @@ -154,6 +154,10 @@ pub enum Error { /// Attempt to open [`Env`] with different options. BadOpenOptions { /// The options that were used to originally open this env. + #[cfg(master3)] + original_options: env::SimplifiedOpenOptions, + /// The options that were used to originally open this env. + #[cfg(not(master3))] options: EnvOpenOptions, /// The env opened with the original options. env: Env, From 95a830d8aa37f26be9b001a5753138a1afef8e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 17:45:29 +0200 Subject: [PATCH 15/51] Add a test showing that encryption is limited --- heed/src/env.rs | 56 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/heed/src/env.rs b/heed/src/env.rs index f665d8dc..12fb5a54 100644 --- a/heed/src/env.rs +++ b/heed/src/env.rs @@ -241,6 +241,8 @@ impl EnvOpenOptions { /// weak RNGs. There is no need to use this kind of algorithm in LMDB since LMDB nonces aren't /// random and are guaranteed to be unique. /// + /// ## Basic Example + /// /// ``` /// use std::fs; /// use std::path::Path; @@ -262,10 +264,12 @@ impl EnvOpenOptions { /// /// // We open the environment /// let mut options = EnvOpenOptions::new().encrypt_with::(key); - /// let env = options - /// .map_size(10 * 1024 * 1024) // 10MB - /// .max_dbs(3) - /// .open(&env_path)?; + /// let env = unsafe { + /// options + /// .map_size(10 * 1024 * 1024) // 10MB + /// .max_dbs(3) + /// .open(&env_path)? + /// }; /// /// let key1 = "first-key"; /// let val1 = "this is a secret info"; @@ -280,6 +284,50 @@ impl EnvOpenOptions { /// wtxn.commit()?; /// # Ok(()) } /// ``` + /// + /// ## Example Showing limitations + /// + /// ```compile_fail + /// use std::fs; + /// use std::path::Path; + /// use argon2::Argon2; + /// use chacha20poly1305::{ChaCha20Poly1305, Key}; + /// use heed3::types::*; + /// use heed3::{EnvOpenOptions, Database}; + /// + /// # fn main() -> Result<(), Box> { + /// let env_path = Path::new("target").join("encrypt.mdb"); + /// let password = "This is the password that will be hashed by the argon2 algorithm"; + /// let salt = "The salt added to the password hashes to add more security when stored"; + /// + /// let _ = fs::remove_dir_all(&env_path); + /// fs::create_dir_all(&env_path)?; + /// + /// let mut key = Key::default(); + /// Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; + /// + /// // We open the environment + /// let mut options = EnvOpenOptions::new().encrypt_with::(key); + /// let env = unsafe { + /// options + /// .map_size(10 * 1024 * 1024) // 10MB + /// .max_dbs(3) + /// .open(&env_path)? + /// }; + /// + /// let key1 = "first-key"; + /// let key2 = "second-key"; + /// + /// // Declare the read transaction as mutable because LMDB, when using encryption, + /// // does not allow keeping keys between reads due to the use of an internal cache. + /// let mut rtxn = env.read_txn()?; + /// let val1 = db.get(&mut rtxn, key1)?; + /// let val2 = db.get(&mut rtxn, key2)?; + /// + /// // This example won't compile because val1 cannot be used for too long. + /// let _force_keep = val1; + /// # Ok(()) } + /// ``` #[cfg(all(master3, feature = "encryption"))] pub fn encrypt_with(self, key: Key) -> EnvOpenOptions { let EnvOpenOptions { encrypt: _, map_size, max_readers, max_dbs, flags } = self; From 195c2d9cb737ee83ea86c5ac2e12998159db782d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 20 Aug 2024 17:47:33 +0200 Subject: [PATCH 16/51] Remove useless master3 tests --- lmdb-master3-sys/tests/lmdb.rs | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 lmdb-master3-sys/tests/lmdb.rs diff --git a/lmdb-master3-sys/tests/lmdb.rs b/lmdb-master3-sys/tests/lmdb.rs deleted file mode 100644 index f02284b4..00000000 --- a/lmdb-master3-sys/tests/lmdb.rs +++ /dev/null @@ -1,26 +0,0 @@ -use std::env; -use std::path::PathBuf; -use std::process::Command; - -#[test] -fn test_lmdb() { - let mut lmdb = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); - lmdb.push("lmdb"); - lmdb.push("libraries"); - lmdb.push("liblmdb"); - - let make_cmd = Command::new("make") - .current_dir(&lmdb) - .status() - .expect("failed to execute process"); - - assert!(make_cmd.success()); - - let make_test_cmd = Command::new("make") - .arg("test") - .current_dir(&lmdb) - .status() - .expect("failed to execute process"); - - assert!(make_test_cmd.success()); -} From 8a5992f046104c89d10439cb446cfddf4fb0e4c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 21 Aug 2024 12:10:50 +0200 Subject: [PATCH 17/51] Create two different Env for master3 and the normal branch --- examples/all-types-heed3.rs | 2 +- heed/src/database.rs | 4 +- heed/src/env/clear.rs | 247 ++++++++++ heed/src/env/encrypted.rs | 532 +++++++++++++++++++++ heed/src/{env.rs => env/mod.rs} | 793 +------------------------------- heed/src/lib.rs | 2 +- 6 files changed, 803 insertions(+), 777 deletions(-) create mode 100644 heed/src/env/clear.rs create mode 100644 heed/src/env/encrypted.rs rename heed/src/{env.rs => env/mod.rs} (56%) diff --git a/examples/all-types-heed3.rs b/examples/all-types-heed3.rs index 7ec698df..bea3a015 100644 --- a/examples/all-types-heed3.rs +++ b/examples/all-types-heed3.rs @@ -8,7 +8,7 @@ use heed3::{Database, EnvOpenOptions}; use serde::{Deserialize, Serialize}; fn main() -> Result<(), Box> { - let path = Path::new("target").join("heed.mdb"); + let path = Path::new("target").join("heed3.mdb"); fs::create_dir_all(&path)?; diff --git a/heed/src/database.rs b/heed/src/database.rs index 97d1d8d9..1ba05341 100644 --- a/heed/src/database.rs +++ b/heed/src/database.rs @@ -2138,7 +2138,7 @@ impl Database { /// written to disk, or if a value already exists for the key, returns the previous value. /// /// The entry is always written with the [`NO_OVERWRITE`](PutFlags::NO_OVERWRITE) and - /// [`MDB_RESERVE`](lmdb_master_sys::MDB_RESERVE) flags. + /// [`MDB_RESERVE`](ffi::MDB_RESERVE) flags. /// /// ``` /// # use heed::EnvOpenOptions; @@ -2201,7 +2201,7 @@ impl Database { /// written to disk, or if a value already exists for the key, returns the previous value. /// /// The entry is written with the specified flags, in addition to - /// [`NO_OVERWRITE`](PutFlags::NO_OVERWRITE) and [`MDB_RESERVE`](lmdb_master_sys::MDB_RESERVE) + /// [`NO_OVERWRITE`](PutFlags::NO_OVERWRITE) and [`MDB_RESERVE`](ffi::MDB_RESERVE) /// which are always used. /// /// ``` diff --git a/heed/src/env/clear.rs b/heed/src/env/clear.rs new file mode 100644 index 00000000..ee985818 --- /dev/null +++ b/heed/src/env/clear.rs @@ -0,0 +1,247 @@ +use std::collections::hash_map::Entry; +use std::ffi::CString; +#[cfg(windows)] +use std::ffi::OsStr; +use std::io::ErrorKind::NotFound; +#[cfg(unix)] +use std::os::unix::ffi::OsStrExt; +use std::path::Path; +use std::sync::Arc; +use std::{io, ptr}; + +use synchronoise::SignalEvent; + +use crate::env::{canonicalize_path, Env, EnvFlags, EnvInner, OPENED_ENV}; +use crate::mdb::ffi; +use crate::mdb::lmdb_error::mdb_result; +use crate::{Error, Result}; + +pub struct EnvEntry { + pub(super) env: Option, + pub(super) signal_event: Arc, + pub(super) options: EnvOpenOptions, +} + +/// Options and flags which can be used to configure how an environment is opened. +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct EnvOpenOptions { + map_size: Option, + max_readers: Option, + max_dbs: Option, + flags: EnvFlags, +} + +impl Default for EnvOpenOptions { + fn default() -> Self { + Self::new() + } +} + +impl EnvOpenOptions { + /// Creates a blank new set of options ready for configuration. + pub fn new() -> EnvOpenOptions { + EnvOpenOptions { + map_size: None, + max_readers: None, + max_dbs: None, + flags: EnvFlags::empty(), + } + } +} + +impl EnvOpenOptions { + /// Set the size of the memory map to use for this environment. + pub fn map_size(&mut self, size: usize) -> &mut Self { + self.map_size = Some(size); + self + } + + /// Set the maximum number of threads/reader slots for the environment. + pub fn max_readers(&mut self, readers: u32) -> &mut Self { + self.max_readers = Some(readers); + self + } + + /// Set the maximum number of named databases for the environment. + pub fn max_dbs(&mut self, dbs: u32) -> &mut Self { + self.max_dbs = Some(dbs); + self + } + + /// Set one or [more LMDB flags](http://www.lmdb.tech/doc/group__mdb__env.html). + /// + /// ``` + /// use std::fs; + /// use std::path::Path; + /// use heed::{EnvOpenOptions, Database, EnvFlags}; + /// use heed::types::*; + /// + /// # fn main() -> Result<(), Box> { + /// fs::create_dir_all(Path::new("target").join("database.mdb"))?; + /// let mut env_builder = EnvOpenOptions::new(); + /// unsafe { env_builder.flags(EnvFlags::NO_TLS | EnvFlags::NO_META_SYNC); } + /// let dir = tempfile::tempdir().unwrap(); + /// let env = unsafe { env_builder.open(dir.path())? }; + /// + /// // we will open the default unamed database + /// let mut wtxn = env.write_txn()?; + /// let db: Database> = env.create_database(&mut wtxn, None)?; + /// + /// // opening a write transaction + /// db.put(&mut wtxn, "seven", &7)?; + /// db.put(&mut wtxn, "zero", &0)?; + /// db.put(&mut wtxn, "five", &5)?; + /// db.put(&mut wtxn, "three", &3)?; + /// wtxn.commit()?; + /// + /// // Force the OS to flush the buffers (see Flag::NoSync and Flag::NoMetaSync). + /// env.force_sync(); + /// + /// // opening a read transaction + /// // to check if those values are now available + /// let mut rtxn = env.read_txn()?; + /// + /// let ret = db.get(&rtxn, "zero")?; + /// assert_eq!(ret, Some(0)); + /// + /// let ret = db.get(&rtxn, "five")?; + /// assert_eq!(ret, Some(5)); + /// # Ok(()) } + /// ``` + /// + /// # Safety + /// + /// It is unsafe to use unsafe LMDB flags such as `NO_SYNC`, `NO_META_SYNC`, or `NO_LOCK`. + pub unsafe fn flags(&mut self, flags: EnvFlags) -> &mut Self { + self.flags |= flags; + self + } + + /// Open an environment that will be located at the specified path. + /// + /// # Safety + /// LMDB is backed by a memory map [^1] which comes with some safety precautions. + /// + /// Memory map constructors are marked `unsafe` because of the potential + /// for Undefined Behavior (UB) using the map if the underlying file is + /// subsequently modified, in or out of process. + /// + /// LMDB itself has a locking system that solves this problem, + /// but it will not save you from making mistakes yourself. + /// + /// These are some things to take note of: + /// + /// - Avoid long-lived transactions, they will cause the database to grow quickly [^2] + /// - Avoid aborting your process with an active transaction [^3] + /// - Do not use LMDB on remote filesystems, even between processes on the same host [^4] + /// - You must manage concurrent accesses yourself if using [`EnvFlags::NO_LOCK`] [^5] + /// - Anything that causes LMDB's lock file to be broken will cause synchronization issues and may introduce UB [^6] + /// + /// `heed` itself upholds some safety invariants, including but not limited to: + /// - Calling [`EnvOpenOptions::open`] twice in the same process, at the same time is OK [^7] + /// + /// For more details, it is highly recommended to read LMDB's official documentation. [^8] + /// + /// [^1]: + /// [^2]: + /// [^3]: + /// [^4]: + /// [^5]: + /// [^6]: + /// [^7]: + /// [^8]: + pub unsafe fn open>(&self, path: P) -> Result { + let mut lock = OPENED_ENV.write().unwrap(); + + let path = match canonicalize_path(path.as_ref()) { + Err(err) => { + if err.kind() == NotFound && self.flags.contains(EnvFlags::NO_SUB_DIR) { + let path = path.as_ref(); + match path.parent().zip(path.file_name()) { + Some((dir, file_name)) => canonicalize_path(dir)?.join(file_name), + None => return Err(err.into()), + } + } else { + return Err(err.into()); + } + } + Ok(path) => path, + }; + + match lock.entry(path) { + Entry::Occupied(entry) => { + let env = entry.get().env.clone().ok_or(Error::DatabaseClosing)?; + let options = entry.get().options.clone(); + if &options == self { + Ok(env) + } else { + Err(Error::BadOpenOptions { env, options }) + } + } + Entry::Vacant(entry) => { + let path = entry.key(); + let path_str = CString::new(path.as_os_str().as_bytes()).unwrap(); + + unsafe { + let mut env: *mut ffi::MDB_env = ptr::null_mut(); + mdb_result(ffi::mdb_env_create(&mut env))?; + + if let Some(size) = self.map_size { + if size % page_size::get() != 0 { + let msg = format!( + "map size ({}) must be a multiple of the system page size ({})", + size, + page_size::get() + ); + return Err(Error::Io(io::Error::new( + io::ErrorKind::InvalidInput, + msg, + ))); + } + mdb_result(ffi::mdb_env_set_mapsize(env, size))?; + } + + if let Some(readers) = self.max_readers { + mdb_result(ffi::mdb_env_set_maxreaders(env, readers))?; + } + + if let Some(dbs) = self.max_dbs { + mdb_result(ffi::mdb_env_set_maxdbs(env, dbs))?; + } + + // When the `read-txn-no-tls` feature is enabled, we must force LMDB + // to avoid using the thread local storage, this way we allow users + // to use references of RoTxn between threads safely. + let flags = if cfg!(feature = "read-txn-no-tls") { + self.flags | EnvFlags::NO_TLS + } else { + self.flags + }; + + let result = + mdb_result(ffi::mdb_env_open(env, path_str.as_ptr(), flags.bits(), 0o600)); + + match result { + Ok(()) => { + let signal_event = Arc::new(SignalEvent::manual(false)); + let inner = EnvInner { env, path: path.clone() }; + let env = Env(Arc::new(inner)); + let cache_entry = EnvEntry { + env: Some(env.clone()), + options: self.clone(), + signal_event, + }; + entry.insert(cache_entry); + Ok(env) + } + Err(e) => { + ffi::mdb_env_close(env); + Err(e.into()) + } + } + } + } + } + } +} diff --git a/heed/src/env/encrypted.rs b/heed/src/env/encrypted.rs new file mode 100644 index 00000000..95257b21 --- /dev/null +++ b/heed/src/env/encrypted.rs @@ -0,0 +1,532 @@ +use std::collections::hash_map::Entry; +use std::ffi::CString; +#[cfg(windows)] +use std::ffi::OsStr; +use std::io::ErrorKind::NotFound; +#[cfg(unix)] +use std::os::unix::ffi::OsStrExt; +use std::panic::catch_unwind; +use std::path::Path; +use std::sync::Arc; +use std::{fmt, io, ptr}; + +use aead::consts::U0; +use aead::generic_array::typenum::Unsigned; +use aead::generic_array::GenericArray; +use aead::{AeadCore, AeadMutInPlace, Key, KeyInit, KeySizeUser, Nonce, Tag}; +use synchronoise::SignalEvent; + +use crate::env::{canonicalize_path, Env, EnvFlags, EnvInner, OPENED_ENV}; +use crate::mdb::ffi; +use crate::mdb::lmdb_error::mdb_result; +use crate::{Error, Result}; + +pub struct EnvEntry { + pub(super) env: Option, + pub(super) signal_event: Arc, + pub(super) options: SimplifiedOpenOptions, +} + +/// A simplified version of the options that were used to open a given [`Env`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SimplifiedOpenOptions { + /// Weither this [`Env`] has been opened with an encryption/decryption algorithm. + pub use_encryption: bool, + /// The maximum size this [`Env`] with take in bytes or [`None`] if it was not specified. + pub map_size: Option, + /// The maximum number of concurrent readers or [`None`] if it was not specified. + pub max_readers: Option, + /// The maximum number of opened database or [`None`] if it was not specified. + pub max_dbs: Option, + /// The raw flags enabled for this [`Env`] or [`None`] if it was not specified. + pub flags: u32, +} + +impl From<&EnvOpenOptions> for SimplifiedOpenOptions { + fn from(eoo: &EnvOpenOptions) -> SimplifiedOpenOptions { + let EnvOpenOptions { encrypt, map_size, max_readers, max_dbs, flags } = eoo; + SimplifiedOpenOptions { + use_encryption: encrypt.is_some(), + map_size: *map_size, + max_readers: *max_readers, + max_dbs: *max_dbs, + flags: flags.bits(), + } + } +} + +/// Options and flags which can be used to configure how an environment is opened. +#[derive(Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct EnvOpenOptions { + encrypt: Option>, + map_size: Option, + max_readers: Option, + max_dbs: Option, + flags: EnvFlags, +} + +impl Default for EnvOpenOptions { + fn default() -> Self { + Self::new() + } +} + +impl EnvOpenOptions { + /// Creates a blank new set of options ready for configuration. + pub fn new() -> EnvOpenOptions { + EnvOpenOptions { + encrypt: None, + map_size: None, + max_readers: None, + max_dbs: None, + flags: EnvFlags::empty(), + } + } +} + +impl EnvOpenOptions { + /// Set the size of the memory map to use for this environment. + pub fn map_size(&mut self, size: usize) -> &mut Self { + self.map_size = Some(size); + self + } + + /// Set the maximum number of threads/reader slots for the environment. + pub fn max_readers(&mut self, readers: u32) -> &mut Self { + self.max_readers = Some(readers); + self + } + + /// Set the maximum number of named databases for the environment. + pub fn max_dbs(&mut self, dbs: u32) -> &mut Self { + self.max_dbs = Some(dbs); + self + } + + /// Specifies that the [`Env`] will be encrypted using the `A` algorithm with the given `key`. + /// + /// You can find more compatible algorithms on [the RustCrypto/AEADs page](https://github.com/RustCrypto/AEADs#crates). + /// + /// Note that you cannot use any type of encryption algorithm as LMDB exposes a nonce of 16 bytes. + /// Heed makes sure to truncate it if necessary. + /// + /// As an example, XChaCha20 requires a 20 bytes long nonce. However, XChaCha20 is used to protect + /// against nonce misuse in systems that use randomly generated nonces i.e., to protect against + /// weak RNGs. There is no need to use this kind of algorithm in LMDB since LMDB nonces aren't + /// random and are guaranteed to be unique. + /// + /// ## Basic Example + /// + /// ``` + /// use std::fs; + /// use std::path::Path; + /// use argon2::Argon2; + /// use chacha20poly1305::{ChaCha20Poly1305, Key}; + /// use heed3::types::*; + /// use heed3::{EnvOpenOptions, Database}; + /// + /// # fn main() -> Result<(), Box> { + /// let env_path = Path::new("target").join("encrypt.mdb"); + /// let password = "This is the password that will be hashed by the argon2 algorithm"; + /// let salt = "The salt added to the password hashes to add more security when stored"; + /// + /// let _ = fs::remove_dir_all(&env_path); + /// fs::create_dir_all(&env_path)?; + /// + /// let mut key = Key::default(); + /// Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; + /// + /// // We open the environment + /// let mut options = EnvOpenOptions::new().encrypt_with::(key); + /// let env = unsafe { + /// options + /// .map_size(10 * 1024 * 1024) // 10MB + /// .max_dbs(3) + /// .open(&env_path)? + /// }; + /// + /// let key1 = "first-key"; + /// let val1 = "this is a secret info"; + /// let key2 = "second-key"; + /// let val2 = "this is another secret info"; + /// + /// // We create database and write secret values in it + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("first"))?; + /// db.put(&mut wtxn, key1, val1)?; + /// db.put(&mut wtxn, key2, val2)?; + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + /// + /// ## Example Showing limitations + /// + /// ```compile_fail + /// use std::fs; + /// use std::path::Path; + /// use argon2::Argon2; + /// use chacha20poly1305::{ChaCha20Poly1305, Key}; + /// use heed3::types::*; + /// use heed3::{EnvOpenOptions, Database}; + /// + /// # fn main() -> Result<(), Box> { + /// let env_path = Path::new("target").join("encrypt.mdb"); + /// let password = "This is the password that will be hashed by the argon2 algorithm"; + /// let salt = "The salt added to the password hashes to add more security when stored"; + /// + /// let _ = fs::remove_dir_all(&env_path); + /// fs::create_dir_all(&env_path)?; + /// + /// let mut key = Key::default(); + /// Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; + /// + /// // We open the environment + /// let mut options = EnvOpenOptions::new().encrypt_with::(key); + /// let env = unsafe { + /// options + /// .map_size(10 * 1024 * 1024) // 10MB + /// .max_dbs(3) + /// .open(&env_path)? + /// }; + /// + /// let key1 = "first-key"; + /// let key2 = "second-key"; + /// + /// // Declare the read transaction as mutable because LMDB, when using encryption, + /// // does not allow keeping keys between reads due to the use of an internal cache. + /// let mut rtxn = env.read_txn()?; + /// let val1 = db.get(&mut rtxn, key1)?; + /// let val2 = db.get(&mut rtxn, key2)?; + /// + /// // This example won't compile because val1 cannot be used for too long. + /// let _force_keep = val1; + /// # Ok(()) } + /// ``` + #[cfg(feature = "encryption")] + pub fn encrypt_with(self, key: Key) -> EnvOpenOptions { + let EnvOpenOptions { encrypt: _, map_size, max_readers, max_dbs, flags } = self; + EnvOpenOptions { encrypt: Some(key), map_size, max_readers, max_dbs, flags } + } + + /// Set one or [more LMDB flags](http://www.lmdb.tech/doc/group__mdb__env.html). + /// + /// ``` + /// use std::fs; + /// use std::path::Path; + /// use heed::{EnvOpenOptions, Database, EnvFlags}; + /// use heed::types::*; + /// + /// # fn main() -> Result<(), Box> { + /// fs::create_dir_all(Path::new("target").join("database.mdb"))?; + /// let mut env_builder = EnvOpenOptions::new(); + /// unsafe { env_builder.flags(EnvFlags::NO_TLS | EnvFlags::NO_META_SYNC); } + /// let dir = tempfile::tempdir().unwrap(); + /// let env = unsafe { env_builder.open(dir.path())? }; + /// + /// // we will open the default unamed database + /// let mut wtxn = env.write_txn()?; + /// let db: Database> = env.create_database(&mut wtxn, None)?; + /// + /// // opening a write transaction + /// db.put(&mut wtxn, "seven", &7)?; + /// db.put(&mut wtxn, "zero", &0)?; + /// db.put(&mut wtxn, "five", &5)?; + /// db.put(&mut wtxn, "three", &3)?; + /// wtxn.commit()?; + /// + /// // Force the OS to flush the buffers (see Flag::NoSync and Flag::NoMetaSync). + /// env.force_sync(); + /// + /// // opening a read transaction + /// // to check if those values are now available + /// let mut rtxn = env.read_txn()?; + /// + /// let ret = db.get(&rtxn, "zero")?; + /// assert_eq!(ret, Some(0)); + /// + /// let ret = db.get(&rtxn, "five")?; + /// assert_eq!(ret, Some(5)); + /// # Ok(()) } + /// ``` + /// + /// # Safety + /// + /// It is unsafe to use unsafe LMDB flags such as `NO_SYNC`, `NO_META_SYNC`, or `NO_LOCK`. + pub unsafe fn flags(&mut self, flags: EnvFlags) -> &mut Self { + self.flags |= flags; + self + } + + /// Open an environment that will be located at the specified path. + /// + /// # Safety + /// LMDB is backed by a memory map [^1] which comes with some safety precautions. + /// + /// Memory map constructors are marked `unsafe` because of the potential + /// for Undefined Behavior (UB) using the map if the underlying file is + /// subsequently modified, in or out of process. + /// + /// LMDB itself has a locking system that solves this problem, + /// but it will not save you from making mistakes yourself. + /// + /// These are some things to take note of: + /// + /// - Avoid long-lived transactions, they will cause the database to grow quickly [^2] + /// - Avoid aborting your process with an active transaction [^3] + /// - Do not use LMDB on remote filesystems, even between processes on the same host [^4] + /// - You must manage concurrent accesses yourself if using [`EnvFlags::NO_LOCK`] [^5] + /// - Anything that causes LMDB's lock file to be broken will cause synchronization issues and may introduce UB [^6] + /// + /// `heed` itself upholds some safety invariants, including but not limited to: + /// - Calling [`EnvOpenOptions::open`] twice in the same process, at the same time is OK [^7] + /// + /// For more details, it is highly recommended to read LMDB's official documentation. [^8] + /// + /// [^1]: + /// [^2]: + /// [^3]: + /// [^4]: + /// [^5]: + /// [^6]: + /// [^7]: + /// [^8]: + pub unsafe fn open>(&self, path: P) -> Result { + let mut lock = OPENED_ENV.write().unwrap(); + + let path = match canonicalize_path(path.as_ref()) { + Err(err) => { + if err.kind() == NotFound && self.flags.contains(EnvFlags::NO_SUB_DIR) { + let path = path.as_ref(); + match path.parent().zip(path.file_name()) { + Some((dir, file_name)) => canonicalize_path(dir)?.join(file_name), + None => return Err(err.into()), + } + } else { + return Err(err.into()); + } + } + Ok(path) => path, + }; + + let original_options = SimplifiedOpenOptions::from(self); + match lock.entry(path) { + Entry::Occupied(entry) => { + let env = entry.get().env.clone().ok_or(Error::DatabaseClosing)?; + let options = entry.get().options.clone(); + if options == original_options { + Ok(env) + } else { + Err(Error::BadOpenOptions { env, options: original_options }) + } + } + Entry::Vacant(entry) => { + let path = entry.key(); + let path_str = CString::new(path.as_os_str().as_bytes()).unwrap(); + + unsafe { + let mut env: *mut ffi::MDB_env = ptr::null_mut(); + mdb_result(ffi::mdb_env_create(&mut env))?; + + // This block will only be executed if the "encryption" feature is enabled. + if let Some(key) = &self.encrypt { + let key = crate::into_val(key); + mdb_result(ffi::mdb_env_set_encrypt( + env, + Some(encrypt_func_wrapper::), + &key, + ::TagSize::U32, + ))?; + } + + if let Some(size) = self.map_size { + if size % page_size::get() != 0 { + let msg = format!( + "map size ({}) must be a multiple of the system page size ({})", + size, + page_size::get() + ); + return Err(Error::Io(io::Error::new( + io::ErrorKind::InvalidInput, + msg, + ))); + } + mdb_result(ffi::mdb_env_set_mapsize(env, size))?; + } + + if let Some(readers) = self.max_readers { + mdb_result(ffi::mdb_env_set_maxreaders(env, readers))?; + } + + if let Some(dbs) = self.max_dbs { + mdb_result(ffi::mdb_env_set_maxdbs(env, dbs))?; + } + + // When the `read-txn-no-tls` feature is enabled, we must force LMDB + // to avoid using the thread local storage, this way we allow users + // to use references of RoTxn between threads safely. + let flags = if cfg!(feature = "read-txn-no-tls") { + self.flags | EnvFlags::NO_TLS + } else { + self.flags + }; + + let result = + mdb_result(ffi::mdb_env_open(env, path_str.as_ptr(), flags.bits(), 0o600)); + + match result { + Ok(()) => { + let signal_event = Arc::new(SignalEvent::manual(false)); + let inner = EnvInner { env, path: path.clone() }; + let env = Env(Arc::new(inner)); + let cache_entry = EnvEntry { + env: Some(env.clone()), + options: original_options, + signal_event, + }; + entry.insert(cache_entry); + Ok(env) + } + Err(e) => { + ffi::mdb_env_close(env); + Err(e.into()) + } + } + } + } + } + } +} + +impl fmt::Debug for EnvOpenOptions { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let EnvOpenOptions { encrypt, map_size, max_readers, max_dbs, flags } = self; + f.debug_struct("EnvOpenOptions") + .field("encrypted", &encrypt.is_some()) + .field("map_size", &map_size) + .field("max_readers", &max_readers) + .field("max_dbs", &max_dbs) + .field("flags", &flags) + .finish() + } +} + +fn encrypt( + key: &[u8], + nonce: &[u8], + aad: &[u8], + plaintext: &[u8], + chipertext_out: &mut [u8], + auth_out: &mut [u8], +) -> aead::Result<()> { + chipertext_out.copy_from_slice(plaintext); + let key: &Key = key.try_into().unwrap(); + let nonce: &Nonce = if nonce.len() >= A::NonceSize::USIZE { + nonce[..A::NonceSize::USIZE].into() + } else { + return Err(aead::Error); + }; + let mut aead = A::new(key); + let tag = aead.encrypt_in_place_detached(nonce, aad, chipertext_out)?; + auth_out.copy_from_slice(&tag); + Ok(()) +} + +fn decrypt( + key: &[u8], + nonce: &[u8], + aad: &[u8], + chipher_text: &[u8], + output: &mut [u8], + auth_in: &[u8], +) -> aead::Result<()> { + output.copy_from_slice(chipher_text); + let key: &Key = key.try_into().unwrap(); + let nonce: &Nonce = if nonce.len() >= A::NonceSize::USIZE { + nonce[..A::NonceSize::USIZE].into() + } else { + return Err(aead::Error); + }; + let tag: &Tag = auth_in.try_into().unwrap(); + let mut aead = A::new(key); + aead.decrypt_in_place_detached(nonce, aad, output, tag) +} + +/// The wrapper function that is called by LMDB that directly calls +/// the Rust idiomatic function internally. +unsafe extern "C" fn encrypt_func_wrapper( + src: *const ffi::MDB_val, + dst: *mut ffi::MDB_val, + key_ptr: *const ffi::MDB_val, + encdec: i32, +) -> i32 { + let result = catch_unwind(|| { + let input = std::slice::from_raw_parts((*src).mv_data as *const u8, (*src).mv_size); + let output = std::slice::from_raw_parts_mut((*dst).mv_data as *mut u8, (*dst).mv_size); + let key = std::slice::from_raw_parts((*key_ptr).mv_data as *const u8, (*key_ptr).mv_size); + let iv = std::slice::from_raw_parts( + (*key_ptr.offset(1)).mv_data as *const u8, + (*key_ptr.offset(1)).mv_size, + ); + let auth = std::slice::from_raw_parts_mut( + (*key_ptr.offset(2)).mv_data as *mut u8, + (*key_ptr.offset(2)).mv_size, + ); + + let aad = []; + let nonce = iv; + let result = if encdec == 1 { + encrypt::(&key, nonce, &aad, input, output, auth) + } else { + decrypt::(&key, nonce, &aad, input, output, auth) + }; + + result.is_err() as i32 + }); + + match result { + Ok(out) => out, + Err(_) => 1, + } +} + +/// A dummy encryption/decryption algorithm that must never be used. +/// Only here for Rust API purposes. +pub enum DummyEncrypt {} + +impl AeadMutInPlace for DummyEncrypt { + fn encrypt_in_place_detached( + &mut self, + _nonce: &Nonce, + _associated_data: &[u8], + _buffer: &mut [u8], + ) -> aead::Result> { + Err(aead::Error) + } + + fn decrypt_in_place_detached( + &mut self, + _nonce: &Nonce, + _associated_data: &[u8], + _buffer: &mut [u8], + _tag: &Tag, + ) -> aead::Result<()> { + Err(aead::Error) + } +} + +impl AeadCore for DummyEncrypt { + type NonceSize = U0; + type TagSize = U0; + type CiphertextOverhead = U0; +} + +impl KeySizeUser for DummyEncrypt { + type KeySize = U0; +} + +impl KeyInit for DummyEncrypt { + fn new(_key: &GenericArray) -> Self { + panic!("This DummyEncrypt type must not be used") + } +} diff --git a/heed/src/env.rs b/heed/src/env/mod.rs similarity index 56% rename from heed/src/env.rs rename to heed/src/env/mod.rs index 12fb5a54..18213327 100644 --- a/heed/src/env.rs +++ b/heed/src/env/mod.rs @@ -1,34 +1,23 @@ use std::any::TypeId; use std::cmp::Ordering; -use std::collections::hash_map::{Entry, HashMap}; +use std::collections::hash_map::HashMap; use std::ffi::{c_void, CString}; use std::fs::{File, Metadata}; -use std::io::ErrorKind::NotFound; #[cfg(unix)] -use std::os::unix::{ - ffi::OsStrExt, - io::{AsRawFd, BorrowedFd, RawFd}, -}; +use std::os::unix::io::{AsRawFd, BorrowedFd, RawFd}; +#[cfg(windows)] +use std::os::windows::io::{AsRawHandle, BorrowedHandle, RawHandle}; use std::panic::catch_unwind; use std::path::{Path, PathBuf}; use std::process::abort; use std::ptr::NonNull; use std::sync::{Arc, RwLock}; use std::time::Duration; -#[cfg(windows)] -use std::{ - ffi::OsStr, - os::windows::io::{AsRawHandle, BorrowedHandle, RawHandle}, -}; use std::{fmt, io, mem, ptr}; -#[cfg(master3)] -use aead::{ - consts::U0, generic_array::typenum::Unsigned, generic_array::GenericArray, AeadCore, - AeadMutInPlace, Key, KeyInit, KeySizeUser, Nonce, Tag, -}; use heed_traits::{Comparator, LexicographicComparator}; use once_cell::sync::Lazy; +pub use sub_env::EnvOpenOptions; use synchronoise::event::SignalEvent; use crate::cursor::MoveOperation; @@ -38,64 +27,24 @@ use crate::mdb::ffi; use crate::mdb::lmdb_flags::AllDatabaseFlags; use crate::{Database, EnvFlags, Error, Result, RoCursor, RoTxn, RwTxn, Unspecified}; +#[cfg(not(master3))] +mod clear; +#[cfg(not(master3))] +use clear as sub_env; + +#[cfg(master3)] +mod encrypted; +#[cfg(master3)] +use encrypted as sub_env; +#[cfg(master3)] +pub use sub_env::SimplifiedOpenOptions; + /// The list of opened environments, the value is an optional environment, it is None /// when someone asks to close the environment, closing is a two-phase step, to make sure /// no one tries to open the same environment between these two phases. /// /// Trying to open a None marked environment returns an error to the user trying to open it. -static OPENED_ENV: Lazy>> = Lazy::new(RwLock::default); - -struct EnvEntry { - env: Option, - signal_event: Arc, - #[cfg(master3)] - options: SimplifiedOpenOptions, - #[cfg(not(master3))] - options: EnvOpenOptions, -} - -/// A simplified version of the options that were used to open a given [`Env`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SimplifiedOpenOptions { - /// Weither this [`Env`] has been opened with an encryption/decryption algorithm. - #[cfg(master3)] - pub use_encryption: bool, - /// The maximum size this [`Env`] with take in bytes or [`None`] if it was not specified. - pub map_size: Option, - /// The maximum number of concurrent readers or [`None`] if it was not specified. - pub max_readers: Option, - /// The maximum number of opened database or [`None`] if it was not specified. - pub max_dbs: Option, - /// The raw flags enabled for this [`Env`] or [`None`] if it was not specified. - pub flags: u32, -} - -#[cfg(not(master3))] -impl From<&EnvOpenOptions> for SimplifiedOpenOptions { - fn from(eoo: &EnvOpenOptions) -> SimplifiedOpenOptions { - let EnvOpenOptions { map_size, max_readers, max_dbs, flags } = eoo; - SimplifiedOpenOptions { - map_size: *map_size, - max_readers: *max_readers, - max_dbs: *max_dbs, - flags: flags.bits(), - } - } -} - -#[cfg(master3)] -impl From<&EnvOpenOptions> for SimplifiedOpenOptions { - fn from(eoo: &EnvOpenOptions) -> SimplifiedOpenOptions { - let EnvOpenOptions { encrypt, map_size, max_readers, max_dbs, flags } = eoo; - SimplifiedOpenOptions { - use_encryption: encrypt.is_some(), - map_size: *map_size, - max_readers: *max_readers, - max_dbs: *max_dbs, - flags: flags.bits(), - } - } -} +static OPENED_ENV: Lazy>> = Lazy::new(RwLock::default); // Thanks to the mozilla/rkv project // Workaround the UNC path on Windows, see https://github.com/rust-lang/rust/issues/42869. @@ -152,662 +101,6 @@ unsafe fn metadata_from_fd(raw_fd: RawHandle) -> io::Result { File::from(owned).metadata() } -/// Options and flags which can be used to configure how an environment is opened. -#[derive(Clone, Debug, PartialEq, Eq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg(not(master3))] -pub struct EnvOpenOptions { - map_size: Option, - max_readers: Option, - max_dbs: Option, - flags: EnvFlags, -} - -/// Options and flags which can be used to configure how an environment is opened. -#[derive(Clone, PartialEq, Eq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg(master3)] -pub struct EnvOpenOptions { - encrypt: Option<(std::marker::PhantomData, Key)>, - map_size: Option, - max_readers: Option, - max_dbs: Option, - flags: EnvFlags, -} - -impl Default for EnvOpenOptions { - fn default() -> Self { - Self::new() - } -} - -#[cfg(master3)] -impl fmt::Debug for EnvOpenOptions { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let EnvOpenOptions { encrypt, map_size, max_readers, max_dbs, flags } = self; - f.debug_struct("EnvOpenOptions") - .field("encrypted", &encrypt.is_some()) - .field("map_size", &map_size) - .field("max_readers", &max_readers) - .field("max_dbs", &max_dbs) - .field("flags", &flags) - .finish() - } -} - -impl EnvOpenOptions { - /// Creates a blank new set of options ready for configuration. - pub fn new() -> EnvOpenOptions { - EnvOpenOptions { - #[cfg(all(master3, feature = "encryption"))] - encrypt: None, - map_size: None, - max_readers: None, - max_dbs: None, - flags: EnvFlags::empty(), - } - } -} - -#[cfg(master3)] -impl EnvOpenOptions { - /// Set the size of the memory map to use for this environment. - pub fn map_size(&mut self, size: usize) -> &mut Self { - self.map_size = Some(size); - self - } - - /// Set the maximum number of threads/reader slots for the environment. - pub fn max_readers(&mut self, readers: u32) -> &mut Self { - self.max_readers = Some(readers); - self - } - - /// Set the maximum number of named databases for the environment. - pub fn max_dbs(&mut self, dbs: u32) -> &mut Self { - self.max_dbs = Some(dbs); - self - } - - /// Specifies that the [`Env`] will be encrypted using the `A` algorithm with the given `key`. - /// - /// You can find more compatible algorithms on [the RustCrypto/AEADs page](https://github.com/RustCrypto/AEADs#crates). - /// - /// Note that you cannot use any type of encryption algorithm as LMDB exposes a nonce of 16 bytes. - /// Heed makes sure to truncate it if necessary. - /// - /// As an example, XChaCha20 requires a 20 bytes long nonce. However, XChaCha20 is used to protect - /// against nonce misuse in systems that use randomly generated nonces i.e., to protect against - /// weak RNGs. There is no need to use this kind of algorithm in LMDB since LMDB nonces aren't - /// random and are guaranteed to be unique. - /// - /// ## Basic Example - /// - /// ``` - /// use std::fs; - /// use std::path::Path; - /// use argon2::Argon2; - /// use chacha20poly1305::{ChaCha20Poly1305, Key}; - /// use heed3::types::*; - /// use heed3::{EnvOpenOptions, Database}; - /// - /// # fn main() -> Result<(), Box> { - /// let env_path = Path::new("target").join("encrypt.mdb"); - /// let password = "This is the password that will be hashed by the argon2 algorithm"; - /// let salt = "The salt added to the password hashes to add more security when stored"; - /// - /// let _ = fs::remove_dir_all(&env_path); - /// fs::create_dir_all(&env_path)?; - /// - /// let mut key = Key::default(); - /// Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; - /// - /// // We open the environment - /// let mut options = EnvOpenOptions::new().encrypt_with::(key); - /// let env = unsafe { - /// options - /// .map_size(10 * 1024 * 1024) // 10MB - /// .max_dbs(3) - /// .open(&env_path)? - /// }; - /// - /// let key1 = "first-key"; - /// let val1 = "this is a secret info"; - /// let key2 = "second-key"; - /// let val2 = "this is another secret info"; - /// - /// // We create database and write secret values in it - /// let mut wtxn = env.write_txn()?; - /// let db: Database = env.create_database(&mut wtxn, Some("first"))?; - /// db.put(&mut wtxn, key1, val1)?; - /// db.put(&mut wtxn, key2, val2)?; - /// wtxn.commit()?; - /// # Ok(()) } - /// ``` - /// - /// ## Example Showing limitations - /// - /// ```compile_fail - /// use std::fs; - /// use std::path::Path; - /// use argon2::Argon2; - /// use chacha20poly1305::{ChaCha20Poly1305, Key}; - /// use heed3::types::*; - /// use heed3::{EnvOpenOptions, Database}; - /// - /// # fn main() -> Result<(), Box> { - /// let env_path = Path::new("target").join("encrypt.mdb"); - /// let password = "This is the password that will be hashed by the argon2 algorithm"; - /// let salt = "The salt added to the password hashes to add more security when stored"; - /// - /// let _ = fs::remove_dir_all(&env_path); - /// fs::create_dir_all(&env_path)?; - /// - /// let mut key = Key::default(); - /// Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; - /// - /// // We open the environment - /// let mut options = EnvOpenOptions::new().encrypt_with::(key); - /// let env = unsafe { - /// options - /// .map_size(10 * 1024 * 1024) // 10MB - /// .max_dbs(3) - /// .open(&env_path)? - /// }; - /// - /// let key1 = "first-key"; - /// let key2 = "second-key"; - /// - /// // Declare the read transaction as mutable because LMDB, when using encryption, - /// // does not allow keeping keys between reads due to the use of an internal cache. - /// let mut rtxn = env.read_txn()?; - /// let val1 = db.get(&mut rtxn, key1)?; - /// let val2 = db.get(&mut rtxn, key2)?; - /// - /// // This example won't compile because val1 cannot be used for too long. - /// let _force_keep = val1; - /// # Ok(()) } - /// ``` - #[cfg(all(master3, feature = "encryption"))] - pub fn encrypt_with(self, key: Key) -> EnvOpenOptions { - let EnvOpenOptions { encrypt: _, map_size, max_readers, max_dbs, flags } = self; - EnvOpenOptions { - encrypt: Some((std::marker::PhantomData, key)), - map_size, - max_readers, - max_dbs, - flags, - } - } - - /// Set one or [more LMDB flags](http://www.lmdb.tech/doc/group__mdb__env.html). - /// - /// ``` - /// use std::fs; - /// use std::path::Path; - /// use heed::{EnvOpenOptions, Database, EnvFlags}; - /// use heed::types::*; - /// - /// # fn main() -> Result<(), Box> { - /// fs::create_dir_all(Path::new("target").join("database.mdb"))?; - /// let mut env_builder = EnvOpenOptions::new(); - /// unsafe { env_builder.flags(EnvFlags::NO_TLS | EnvFlags::NO_META_SYNC); } - /// let dir = tempfile::tempdir().unwrap(); - /// let env = unsafe { env_builder.open(dir.path())? }; - /// - /// // we will open the default unamed database - /// let mut wtxn = env.write_txn()?; - /// let db: Database> = env.create_database(&mut wtxn, None)?; - /// - /// // opening a write transaction - /// db.put(&mut wtxn, "seven", &7)?; - /// db.put(&mut wtxn, "zero", &0)?; - /// db.put(&mut wtxn, "five", &5)?; - /// db.put(&mut wtxn, "three", &3)?; - /// wtxn.commit()?; - /// - /// // Force the OS to flush the buffers (see Flag::NoSync and Flag::NoMetaSync). - /// env.force_sync(); - /// - /// // opening a read transaction - /// // to check if those values are now available - /// let mut rtxn = env.read_txn()?; - /// - /// let ret = db.get(&rtxn, "zero")?; - /// assert_eq!(ret, Some(0)); - /// - /// let ret = db.get(&rtxn, "five")?; - /// assert_eq!(ret, Some(5)); - /// # Ok(()) } - /// ``` - /// - /// # Safety - /// - /// It is unsafe to use unsafe LMDB flags such as `NO_SYNC`, `NO_META_SYNC`, or `NO_LOCK`. - pub unsafe fn flags(&mut self, flags: EnvFlags) -> &mut Self { - self.flags |= flags; - self - } - - /// Open an environment that will be located at the specified path. - /// - /// # Safety - /// LMDB is backed by a memory map [^1] which comes with some safety precautions. - /// - /// Memory map constructors are marked `unsafe` because of the potential - /// for Undefined Behavior (UB) using the map if the underlying file is - /// subsequently modified, in or out of process. - /// - /// LMDB itself has a locking system that solves this problem, - /// but it will not save you from making mistakes yourself. - /// - /// These are some things to take note of: - /// - /// - Avoid long-lived transactions, they will cause the database to grow quickly [^2] - /// - Avoid aborting your process with an active transaction [^3] - /// - Do not use LMDB on remote filesystems, even between processes on the same host [^4] - /// - You must manage concurrent accesses yourself if using [`EnvFlags::NO_LOCK`] [^5] - /// - Anything that causes LMDB's lock file to be broken will cause synchronization issues and may introduce UB [^6] - /// - /// `heed` itself upholds some safety invariants, including but not limited to: - /// - Calling [`EnvOpenOptions::open`] twice in the same process, at the same time is OK [^7] - /// - /// For more details, it is highly recommended to read LMDB's official documentation. [^8] - /// - /// [^1]: - /// [^2]: - /// [^3]: - /// [^4]: - /// [^5]: - /// [^6]: - /// [^7]: - /// [^8]: - pub unsafe fn open>(&self, path: P) -> Result { - let mut lock = OPENED_ENV.write().unwrap(); - - let path = match canonicalize_path(path.as_ref()) { - Err(err) => { - if err.kind() == NotFound && self.flags.contains(EnvFlags::NO_SUB_DIR) { - let path = path.as_ref(); - match path.parent().zip(path.file_name()) { - Some((dir, file_name)) => canonicalize_path(dir)?.join(file_name), - None => return Err(err.into()), - } - } else { - return Err(err.into()); - } - } - Ok(path) => path, - }; - - let original_options = SimplifiedOpenOptions::from(self); - match lock.entry(path) { - Entry::Occupied(entry) => { - let env = entry.get().env.clone().ok_or(Error::DatabaseClosing)?; - let options = entry.get().options.clone(); - if options == original_options { - Ok(env) - } else { - Err(Error::BadOpenOptions { env, original_options }) - } - } - Entry::Vacant(entry) => { - let path = entry.key(); - let path_str = CString::new(path.as_os_str().as_bytes()).unwrap(); - - unsafe { - let mut env: *mut ffi::MDB_env = ptr::null_mut(); - mdb_result(ffi::mdb_env_create(&mut env))?; - - #[cfg(master3)] - if let Some((_marker, key)) = &self.encrypt { - let key = crate::into_val(key); - mdb_result(ffi::mdb_env_set_encrypt( - env, - Some(encrypt_func_wrapper::), - &key, - ::TagSize::U32, - ))?; - } - - if let Some(size) = self.map_size { - if size % page_size::get() != 0 { - let msg = format!( - "map size ({}) must be a multiple of the system page size ({})", - size, - page_size::get() - ); - return Err(Error::Io(io::Error::new( - io::ErrorKind::InvalidInput, - msg, - ))); - } - mdb_result(ffi::mdb_env_set_mapsize(env, size))?; - } - - if let Some(readers) = self.max_readers { - mdb_result(ffi::mdb_env_set_maxreaders(env, readers))?; - } - - if let Some(dbs) = self.max_dbs { - mdb_result(ffi::mdb_env_set_maxdbs(env, dbs))?; - } - - // When the `read-txn-no-tls` feature is enabled, we must force LMDB - // to avoid using the thread local storage, this way we allow users - // to use references of RoTxn between threads safely. - let flags = if cfg!(feature = "read-txn-no-tls") { - self.flags | EnvFlags::NO_TLS - } else { - self.flags - }; - - let result = - mdb_result(ffi::mdb_env_open(env, path_str.as_ptr(), flags.bits(), 0o600)); - - match result { - Ok(()) => { - let signal_event = Arc::new(SignalEvent::manual(false)); - let inner = EnvInner { env, path: path.clone() }; - let env = Env(Arc::new(inner)); - let cache_entry = EnvEntry { - env: Some(env.clone()), - options: original_options, - signal_event, - }; - entry.insert(cache_entry); - Ok(env) - } - Err(e) => { - ffi::mdb_env_close(env); - Err(e.into()) - } - } - } - } - } - } -} - -#[cfg(not(master3))] -impl EnvOpenOptions { - /// Set the size of the memory map to use for this environment. - pub fn map_size(&mut self, size: usize) -> &mut Self { - self.map_size = Some(size); - self - } - - /// Set the maximum number of threads/reader slots for the environment. - pub fn max_readers(&mut self, readers: u32) -> &mut Self { - self.max_readers = Some(readers); - self - } - - /// Set the maximum number of named databases for the environment. - pub fn max_dbs(&mut self, dbs: u32) -> &mut Self { - self.max_dbs = Some(dbs); - self - } - - /// Set one or [more LMDB flags](http://www.lmdb.tech/doc/group__mdb__env.html). - /// - /// ``` - /// use std::fs; - /// use std::path::Path; - /// use heed::{EnvOpenOptions, Database, EnvFlags}; - /// use heed::types::*; - /// - /// # fn main() -> Result<(), Box> { - /// fs::create_dir_all(Path::new("target").join("database.mdb"))?; - /// let mut env_builder = EnvOpenOptions::new(); - /// unsafe { env_builder.flags(EnvFlags::NO_TLS | EnvFlags::NO_META_SYNC); } - /// let dir = tempfile::tempdir().unwrap(); - /// let env = unsafe { env_builder.open(dir.path())? }; - /// - /// // we will open the default unamed database - /// let mut wtxn = env.write_txn()?; - /// let db: Database> = env.create_database(&mut wtxn, None)?; - /// - /// // opening a write transaction - /// db.put(&mut wtxn, "seven", &7)?; - /// db.put(&mut wtxn, "zero", &0)?; - /// db.put(&mut wtxn, "five", &5)?; - /// db.put(&mut wtxn, "three", &3)?; - /// wtxn.commit()?; - /// - /// // Force the OS to flush the buffers (see Flag::NoSync and Flag::NoMetaSync). - /// env.force_sync(); - /// - /// // opening a read transaction - /// // to check if those values are now available - /// let mut rtxn = env.read_txn()?; - /// - /// let ret = db.get(&rtxn, "zero")?; - /// assert_eq!(ret, Some(0)); - /// - /// let ret = db.get(&rtxn, "five")?; - /// assert_eq!(ret, Some(5)); - /// # Ok(()) } - /// ``` - /// - /// # Safety - /// - /// It is unsafe to use unsafe LMDB flags such as `NO_SYNC`, `NO_META_SYNC`, or `NO_LOCK`. - pub unsafe fn flags(&mut self, flags: EnvFlags) -> &mut Self { - self.flags |= flags; - self - } - - /// Open an environment that will be located at the specified path. - /// - /// # Safety - /// LMDB is backed by a memory map [^1] which comes with some safety precautions. - /// - /// Memory map constructors are marked `unsafe` because of the potential - /// for Undefined Behavior (UB) using the map if the underlying file is - /// subsequently modified, in or out of process. - /// - /// LMDB itself has a locking system that solves this problem, - /// but it will not save you from making mistakes yourself. - /// - /// These are some things to take note of: - /// - /// - Avoid long-lived transactions, they will cause the database to grow quickly [^2] - /// - Avoid aborting your process with an active transaction [^3] - /// - Do not use LMDB on remote filesystems, even between processes on the same host [^4] - /// - You must manage concurrent accesses yourself if using [`EnvFlags::NO_LOCK`] [^5] - /// - Anything that causes LMDB's lock file to be broken will cause synchronization issues and may introduce UB [^6] - /// - /// `heed` itself upholds some safety invariants, including but not limited to: - /// - Calling [`EnvOpenOptions::open`] twice in the same process, at the same time is OK [^7] - /// - /// For more details, it is highly recommended to read LMDB's official documentation. [^8] - /// - /// [^1]: - /// [^2]: - /// [^3]: - /// [^4]: - /// [^5]: - /// [^6]: - /// [^7]: - /// [^8]: - pub unsafe fn open>(&self, path: P) -> Result { - let mut lock = OPENED_ENV.write().unwrap(); - - let path = match canonicalize_path(path.as_ref()) { - Err(err) => { - if err.kind() == NotFound && self.flags.contains(EnvFlags::NO_SUB_DIR) { - let path = path.as_ref(); - match path.parent().zip(path.file_name()) { - Some((dir, file_name)) => canonicalize_path(dir)?.join(file_name), - None => return Err(err.into()), - } - } else { - return Err(err.into()); - } - } - Ok(path) => path, - }; - - match lock.entry(path) { - Entry::Occupied(entry) => { - let env = entry.get().env.clone().ok_or(Error::DatabaseClosing)?; - let options = entry.get().options.clone(); - if &options == self { - Ok(env) - } else { - Err(Error::BadOpenOptions { env, options }) - } - } - Entry::Vacant(entry) => { - let path = entry.key(); - let path_str = CString::new(path.as_os_str().as_bytes()).unwrap(); - - unsafe { - let mut env: *mut ffi::MDB_env = ptr::null_mut(); - mdb_result(ffi::mdb_env_create(&mut env))?; - - if let Some(size) = self.map_size { - if size % page_size::get() != 0 { - let msg = format!( - "map size ({}) must be a multiple of the system page size ({})", - size, - page_size::get() - ); - return Err(Error::Io(io::Error::new( - io::ErrorKind::InvalidInput, - msg, - ))); - } - mdb_result(ffi::mdb_env_set_mapsize(env, size))?; - } - - if let Some(readers) = self.max_readers { - mdb_result(ffi::mdb_env_set_maxreaders(env, readers))?; - } - - if let Some(dbs) = self.max_dbs { - mdb_result(ffi::mdb_env_set_maxdbs(env, dbs))?; - } - - // When the `read-txn-no-tls` feature is enabled, we must force LMDB - // to avoid using the thread local storage, this way we allow users - // to use references of RoTxn between threads safely. - let flags = if cfg!(feature = "read-txn-no-tls") { - self.flags | EnvFlags::NO_TLS - } else { - self.flags - }; - - let result = - mdb_result(ffi::mdb_env_open(env, path_str.as_ptr(), flags.bits(), 0o600)); - - match result { - Ok(()) => { - let signal_event = Arc::new(SignalEvent::manual(false)); - let inner = EnvInner { env, path: path.clone() }; - let env = Env(Arc::new(inner)); - let cache_entry = EnvEntry { - env: Some(env.clone()), - options: self.clone(), - signal_event, - }; - entry.insert(cache_entry); - Ok(env) - } - Err(e) => { - ffi::mdb_env_close(env); - Err(e.into()) - } - } - } - } - } - } -} - -#[cfg(master3)] -fn encrypt( - key: &[u8], - nonce: &[u8], - aad: &[u8], - plaintext: &[u8], - chipertext_out: &mut [u8], - auth_out: &mut [u8], -) -> aead::Result<()> { - chipertext_out.copy_from_slice(plaintext); - let key: &Key = key.try_into().unwrap(); - let nonce: &Nonce = if nonce.len() >= A::NonceSize::USIZE { - nonce[..A::NonceSize::USIZE].into() - } else { - return Err(aead::Error); - }; - let mut aead = A::new(key); - let tag = aead.encrypt_in_place_detached(nonce, aad, chipertext_out)?; - auth_out.copy_from_slice(&tag); - Ok(()) -} - -#[cfg(master3)] -fn decrypt( - key: &[u8], - nonce: &[u8], - aad: &[u8], - chipher_text: &[u8], - output: &mut [u8], - auth_in: &[u8], -) -> aead::Result<()> { - output.copy_from_slice(chipher_text); - let key: &Key = key.try_into().unwrap(); - let nonce: &Nonce = if nonce.len() >= A::NonceSize::USIZE { - nonce[..A::NonceSize::USIZE].into() - } else { - return Err(aead::Error); - }; - let tag: &Tag = auth_in.try_into().unwrap(); - let mut aead = A::new(key); - aead.decrypt_in_place_detached(nonce, aad, output, tag) -} - -/// The wrapper function that is called by LMDB that directly calls -/// the Rust idiomatic function internally. -#[cfg(master3)] -unsafe extern "C" fn encrypt_func_wrapper( - src: *const ffi::MDB_val, - dst: *mut ffi::MDB_val, - key_ptr: *const ffi::MDB_val, - encdec: i32, -) -> i32 { - let result = catch_unwind(|| { - let input = std::slice::from_raw_parts((*src).mv_data as *const u8, (*src).mv_size); - let output = std::slice::from_raw_parts_mut((*dst).mv_data as *mut u8, (*dst).mv_size); - let key = std::slice::from_raw_parts((*key_ptr).mv_data as *const u8, (*key_ptr).mv_size); - let iv = std::slice::from_raw_parts( - (*key_ptr.offset(1)).mv_data as *const u8, - (*key_ptr.offset(1)).mv_size, - ); - let auth = std::slice::from_raw_parts_mut( - (*key_ptr.offset(2)).mv_data as *mut u8, - (*key_ptr.offset(2)).mv_size, - ); - - let aad = []; - let nonce = iv; - let result = if encdec == 1 { - encrypt::(&key, nonce, &aad, input, output, auth) - } else { - decrypt::(&key, nonce, &aad, input, output, auth) - }; - - result.is_err() as i32 - }); - - match result { - Ok(out) => out, - Err(_) => 1, - } -} - /// Returns a struct that allows to wait for the effective closing of an environment. pub fn env_closing_event>(path: P) -> Option { let lock = OPENED_ENV.read().unwrap(); @@ -840,7 +133,7 @@ impl Drop for EnvInner { match lock.remove(&self.path) { None => panic!("It seems another env closed this env before"), - Some(EnvEntry { signal_event, .. }) => { + Some(sub_env::EnvEntry { signal_event, .. }) => { unsafe { ffi::mdb_env_close(self.env); } @@ -1391,7 +684,7 @@ impl Env { let mut lock = OPENED_ENV.write().unwrap(); match lock.get_mut(self.path()) { None => panic!("cannot find the env that we are trying to close"), - Some(EnvEntry { env, signal_event, .. }) => { + Some(sub_env::EnvEntry { env, signal_event, .. }) => { // We remove the env from the global list and replace it with a None. let _env = env.take(); let signal_event = signal_event.clone(); @@ -1493,52 +786,6 @@ impl fmt::Debug for EnvClosingEvent { } } -/// A dummy encryption/decryption algorithm that must never be used. -/// Only here for Rust API purposes. -#[cfg(master3)] -pub enum DummyEncrypt {} - -#[cfg(master3)] -impl AeadMutInPlace for DummyEncrypt { - fn encrypt_in_place_detached( - &mut self, - _nonce: &Nonce, - _associated_data: &[u8], - _buffer: &mut [u8], - ) -> aead::Result> { - Err(aead::Error) - } - - fn decrypt_in_place_detached( - &mut self, - _nonce: &Nonce, - _associated_data: &[u8], - _buffer: &mut [u8], - _tag: &Tag, - ) -> aead::Result<()> { - Err(aead::Error) - } -} - -#[cfg(master3)] -impl AeadCore for DummyEncrypt { - type NonceSize = U0; - type TagSize = U0; - type CiphertextOverhead = U0; -} - -#[cfg(master3)] -impl KeySizeUser for DummyEncrypt { - type KeySize = U0; -} - -#[cfg(master3)] -impl KeyInit for DummyEncrypt { - fn new(_key: &GenericArray) -> Self { - panic!("This DummyEncrypt type must not be used") - } -} - #[cfg(test)] mod tests { use std::io::ErrorKind; diff --git a/heed/src/lib.rs b/heed/src/lib.rs index 3a32633c..136430af 100644 --- a/heed/src/lib.rs +++ b/heed/src/lib.rs @@ -155,7 +155,7 @@ pub enum Error { BadOpenOptions { /// The options that were used to originally open this env. #[cfg(master3)] - original_options: env::SimplifiedOpenOptions, + options: env::SimplifiedOpenOptions, /// The options that were used to originally open this env. #[cfg(not(master3))] options: EnvOpenOptions, From dfc275035ecfffe1624fb88e7a600cf29f29021f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 21 Aug 2024 12:25:25 +0200 Subject: [PATCH 18/51] Prepare the heed3 crates to be published on crates.io --- heed-master3-proc-macro/Cargo.toml | 3 +++ heed3/Cargo.toml | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/heed-master3-proc-macro/Cargo.toml b/heed-master3-proc-macro/Cargo.toml index e8d7a76a..d41973b3 100644 --- a/heed-master3-proc-macro/Cargo.toml +++ b/heed-master3-proc-macro/Cargo.toml @@ -1,5 +1,8 @@ [package] name = "heed-master3-proc-macro" +authors = ["Kerollmops "] +description = "A proc macro to simplify the maintenance of the heed and heed3 crates" +license = "MIT" version = "0.1.0" edition = "2021" diff --git a/heed3/Cargo.toml b/heed3/Cargo.toml index b6ebae5c..d023d9a7 100644 --- a/heed3/Cargo.toml +++ b/heed3/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "heed3" -version = "0.20.5" +version = "0.20.5-beta.1" authors = ["Kerollmops "] description = "A fully typed LMDB wrapper with minimum overhead and optional support for encryption" license = "MIT" @@ -16,7 +16,7 @@ aead = { version = "0.5.1", default-features = false, optional = true } bitflags = { version = "2.6.0", features = ["serde"] } byteorder = { version = "1.5.0", default-features = false } generic-array = { version = "0.14.6", features = ["serde"], optional = true } -heed-master3-proc-macro = { path = "../heed-master3-proc-macro", optional = true } +heed-master3-proc-macro = { version = "0.1.0", path = "../heed-master3-proc-macro", optional = true } heed-traits = { version = "0.20.0", path = "../heed-traits" } heed-types = { version = "0.20.1", default-features = false, path = "../heed-types" } libc = "0.2.155" From c4b832acc1b031143cfb3609f4e5d05a4f94264e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 21 Aug 2024 12:29:37 +0200 Subject: [PATCH 19/51] Add a little more documentation to the proc macro --- heed-master3-proc-macro/src/lib.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/heed-master3-proc-macro/src/lib.rs b/heed-master3-proc-macro/src/lib.rs index e29019f2..2f3f656d 100644 --- a/heed-master3-proc-macro/src/lib.rs +++ b/heed-master3-proc-macro/src/lib.rs @@ -2,6 +2,17 @@ use proc_macro::TokenStream; use quote::quote; use syn::{parse_macro_input, parse_quote, FnArg, Ident, ItemFn, Pat, PatType, Type}; +/// By annotating all the heed methods that use a `&RoTxn` this way : +/// +/// ```ignore +/// #[uniq(rtxn)] +/// fn get<'t>(&self, rtxn: &'t RoTxn, key: &[u8]) -> heed::Result<&'t [u8]>; +/// ``` +/// +/// It transforms the function signature to use a `&mut RoTxn`: +/// ```ignore +/// fn get<'t>(&self, rtxn: &'t mut RoTxn, key: &[u8]) -> heed::Result<&'t [u8]>; +/// ``` #[proc_macro_attribute] pub fn mut_read_txn(attr: TokenStream, item: TokenStream) -> TokenStream { // Parse the function From 7771aa4c47381c7882fdc2b50035aaa10a57209c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 21 Aug 2024 12:55:10 +0200 Subject: [PATCH 20/51] Fix missing imports for Windows --- heed/src/env/clear.rs | 2 ++ heed/src/env/encrypted.rs | 2 ++ heed/src/env/mod.rs | 7 +++++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/heed/src/env/clear.rs b/heed/src/env/clear.rs index ee985818..3e1ee899 100644 --- a/heed/src/env/clear.rs +++ b/heed/src/env/clear.rs @@ -11,6 +11,8 @@ use std::{io, ptr}; use synchronoise::SignalEvent; +#[cfg(windows)] +use crate::env::OsStrExtLmdb as _; use crate::env::{canonicalize_path, Env, EnvFlags, EnvInner, OPENED_ENV}; use crate::mdb::ffi; use crate::mdb::lmdb_error::mdb_result; diff --git a/heed/src/env/encrypted.rs b/heed/src/env/encrypted.rs index 95257b21..4feacd8d 100644 --- a/heed/src/env/encrypted.rs +++ b/heed/src/env/encrypted.rs @@ -16,6 +16,8 @@ use aead::generic_array::GenericArray; use aead::{AeadCore, AeadMutInPlace, Key, KeyInit, KeySizeUser, Nonce, Tag}; use synchronoise::SignalEvent; +#[cfg(windows)] +use crate::env::OsStrExtLmdb as _; use crate::env::{canonicalize_path, Env, EnvFlags, EnvInner, OPENED_ENV}; use crate::mdb::ffi; use crate::mdb::lmdb_error::mdb_result; diff --git a/heed/src/env/mod.rs b/heed/src/env/mod.rs index 18213327..1499c246 100644 --- a/heed/src/env/mod.rs +++ b/heed/src/env/mod.rs @@ -5,14 +5,17 @@ use std::ffi::{c_void, CString}; use std::fs::{File, Metadata}; #[cfg(unix)] use std::os::unix::io::{AsRawFd, BorrowedFd, RawFd}; -#[cfg(windows)] -use std::os::windows::io::{AsRawHandle, BorrowedHandle, RawHandle}; use std::panic::catch_unwind; use std::path::{Path, PathBuf}; use std::process::abort; use std::ptr::NonNull; use std::sync::{Arc, RwLock}; use std::time::Duration; +#[cfg(windows)] +use std::{ + ffi::OsStr, + os::windows::io::{AsRawHandle, BorrowedHandle, RawHandle}, +}; use std::{fmt, io, mem, ptr}; use heed_traits::{Comparator, LexicographicComparator}; From 85d52b3243f0120dbcfd0a22b72d9dea9b31142a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 21 Aug 2024 12:56:12 +0200 Subject: [PATCH 21/51] Fix some doc --- heed-master3-proc-macro/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heed-master3-proc-macro/src/lib.rs b/heed-master3-proc-macro/src/lib.rs index 2f3f656d..3ccf022b 100644 --- a/heed-master3-proc-macro/src/lib.rs +++ b/heed-master3-proc-macro/src/lib.rs @@ -5,7 +5,7 @@ use syn::{parse_macro_input, parse_quote, FnArg, Ident, ItemFn, Pat, PatType, Ty /// By annotating all the heed methods that use a `&RoTxn` this way : /// /// ```ignore -/// #[uniq(rtxn)] +/// #[heed_master3_proc_macro::mut_read_txn(rtxn)] /// fn get<'t>(&self, rtxn: &'t RoTxn, key: &[u8]) -> heed::Result<&'t [u8]>; /// ``` /// From f31c16d3aee3eda1196e38db34d7a8c924c1e227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 21 Aug 2024 14:46:04 +0200 Subject: [PATCH 22/51] Introduce the last crate heed3-encryption --- examples/all-types-heed3.rs | 111 ------------ .../{encrypt-heed3.rs => heed3-encryption.rs} | 4 +- heed/Cargo.toml | 2 +- heed/build.rs | 14 +- heed/src/database.rs | 34 ++-- heed/src/env/clear.rs | 8 +- heed/src/env/encrypted.rs | 171 ++++-------------- heed/src/env/mod.rs | 22 ++- heed/src/lib.rs | 4 - heed/src/txn.rs | 2 +- heed3-encryption/Cargo.toml | 124 +++++++++++++ heed3/Cargo.toml | 46 +++-- 12 files changed, 232 insertions(+), 310 deletions(-) delete mode 100644 examples/all-types-heed3.rs rename examples/{encrypt-heed3.rs => heed3-encryption.rs} (96%) create mode 100644 heed3-encryption/Cargo.toml diff --git a/examples/all-types-heed3.rs b/examples/all-types-heed3.rs deleted file mode 100644 index bea3a015..00000000 --- a/examples/all-types-heed3.rs +++ /dev/null @@ -1,111 +0,0 @@ -use std::error::Error; -use std::fs; -use std::path::Path; - -use heed3::byteorder::BE; -use heed3::types::*; -use heed3::{Database, EnvOpenOptions}; -use serde::{Deserialize, Serialize}; - -fn main() -> Result<(), Box> { - let path = Path::new("target").join("heed3.mdb"); - - fs::create_dir_all(&path)?; - - let env = unsafe { - EnvOpenOptions::new() - .map_size(10 * 1024 * 1024) // 10MB - .max_dbs(3000) - .open(path)? - }; - - // here the key will be an str and the data will be a slice of u8 - let mut wtxn = env.write_txn()?; - let db: Database = env.create_database(&mut wtxn, Some("kiki"))?; - - db.put(&mut wtxn, "hello", &[2, 3][..])?; - let ret: Option<&[u8]> = db.get(&mut wtxn, "hello")?; - - println!("{:?}", ret); - wtxn.commit()?; - - // serde types are also supported!!! - #[derive(Debug, Serialize, Deserialize)] - struct Hello<'a> { - string: &'a str, - } - - let mut wtxn = env.write_txn()?; - let db: Database> = - env.create_database(&mut wtxn, Some("serde-bincode"))?; - - let hello = Hello { string: "hi" }; - db.put(&mut wtxn, "hello", &hello)?; - - let ret: Option = db.get(&mut wtxn, "hello")?; - println!("serde-bincode:\t{:?}", ret); - - wtxn.commit()?; - - let mut wtxn = env.write_txn()?; - let db: Database> = env.create_database(&mut wtxn, Some("serde-json"))?; - - let hello = Hello { string: "hi" }; - db.put(&mut wtxn, "hello", &hello)?; - - let ret: Option = db.get(&mut wtxn, "hello")?; - println!("serde-json:\t{:?}", ret); - - wtxn.commit()?; - - // you can ignore the data - let mut wtxn = env.write_txn()?; - let db: Database = env.create_database(&mut wtxn, Some("ignored-data"))?; - - db.put(&mut wtxn, "hello", &())?; - let ret: Option<()> = db.get(&mut wtxn, "hello")?; - - println!("{:?}", ret); - - let ret: Option<()> = db.get(&mut wtxn, "non-existant")?; - - println!("{:?}", ret); - wtxn.commit()?; - - // database opening and types are tested in a safe way - // - // we try to open a database twice with the same types - let mut wtxn = env.write_txn()?; - let _db: Database = env.create_database(&mut wtxn, Some("ignored-data"))?; - - // you can iterate over keys in order - type BEI64 = I64; - - let db: Database = env.create_database(&mut wtxn, Some("big-endian-iter"))?; - - db.put(&mut wtxn, &0, &())?; - db.put(&mut wtxn, &68, &())?; - db.put(&mut wtxn, &35, &())?; - db.put(&mut wtxn, &42, &())?; - - let rets: Result, _> = db.iter(&mut wtxn)?.collect(); - - println!("{:?}", rets); - - // or iterate over ranges too!!! - let range = 35..=42; - let rets: Result, _> = db.range(&mut wtxn, &range)?.collect(); - - println!("{:?}", rets); - - // delete a range of key - let range = 35..=42; - let deleted: usize = db.delete_range(&mut wtxn, &range)?; - - let rets: Result, _> = db.iter(&mut wtxn)?.collect(); - - println!("deleted: {:?}, {:?}", deleted, rets); - wtxn.commit()?; - - Ok(()) -} diff --git a/examples/encrypt-heed3.rs b/examples/heed3-encryption.rs similarity index 96% rename from examples/encrypt-heed3.rs rename to examples/heed3-encryption.rs index 9cd2841f..99f7a09d 100644 --- a/examples/encrypt-heed3.rs +++ b/examples/heed3-encryption.rs @@ -4,8 +4,8 @@ use std::path::Path; use argon2::Argon2; use chacha20poly1305::{ChaCha20Poly1305, Key}; -use heed3::types::*; -use heed3::{Database, EnvOpenOptions}; +use heed3_encryption::types::*; +use heed3_encryption::{Database, EnvOpenOptions}; fn main() -> Result<(), Box> { let env_path = Path::new("target").join("encrypt.mdb"); diff --git a/heed/Cargo.toml b/heed/Cargo.toml index 967510e6..fb8eae96 100644 --- a/heed/Cargo.toml +++ b/heed/Cargo.toml @@ -2,7 +2,7 @@ name = "heed" version = "0.20.5" authors = ["Kerollmops "] -description = "A fully typed LMDB wrapper with minimum overhead" +description = "A fully typed LMDB (mdb.master) wrapper with minimum overhead" license = "MIT" repository = "https://github.com/Kerollmops/heed" keywords = ["lmdb", "database", "storage", "typed"] diff --git a/heed/build.rs b/heed/build.rs index 0835bfae..3c3f804e 100644 --- a/heed/build.rs +++ b/heed/build.rs @@ -3,16 +3,16 @@ use std::env; fn main() { println!("cargo:rerun-if-changed=build.rs"); println!("cargo::rustc-check-cfg=cfg(master3)"); - // if let Some(channel) = version_check::Channel::read() { - // if channel.supports_features() { - // println!("cargo:rustc-cfg=has_specialisation"); - // } - // } + println!("cargo::rustc-check-cfg=cfg(encryption)"); + let pkgname = env::var("CARGO_PKG_NAME").expect("Cargo didn't set the CARGO_PKG_NAME env var!"); match pkgname.as_str() { + "heed" => (), "heed3" => println!("cargo:rustc-cfg=master3"), - // Ignore the absence of the encryption feature when not using heed3 - "heed" => println!("cargo::rustc-check-cfg=cfg(feature, values(\"encryption\"))"), + "heed3-encryption" => { + println!("cargo:rustc-cfg=master3"); + println!("cargo:rustc-cfg=encryption"); + } _ => panic!("unexpected package name!"), } } diff --git a/heed/src/database.rs b/heed/src/database.rs index 1ba05341..bf794171 100644 --- a/heed/src/database.rs +++ b/heed/src/database.rs @@ -339,7 +339,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn get<'a, 'txn>(&self, txn: &'txn RoTxn, key: &'a KC::EItem) -> Result> where KC: BytesEncode<'a>, @@ -420,7 +420,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_duplicates<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -484,7 +484,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_lower_than<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -554,7 +554,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_lower_than_or_equal_to<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -628,7 +628,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_greater_than<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -701,7 +701,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_greater_than_or_equal_to<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -761,7 +761,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn first<'txn>(&self, txn: &'txn RoTxn) -> Result> where KC: BytesDecode<'txn>, @@ -816,7 +816,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn last<'txn>(&self, txn: &'txn RoTxn) -> Result> where KC: BytesDecode<'txn>, @@ -874,7 +874,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn len(&self, txn: &RoTxn) -> Result { self.stat(txn).map(|stat| stat.entries as u64) } @@ -918,7 +918,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn is_empty(&self, txn: &RoTxn) -> Result { self.len(txn).map(|l| l == 0) } @@ -961,7 +961,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn stat(&self, txn: &RoTxn) -> Result { assert_eq_env_db_txn!(self, txn); @@ -1026,7 +1026,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn iter<'txn>(&self, txn: &'txn RoTxn) -> Result> { assert_eq_env_db_txn!(self, txn); RoCursor::new(txn, self.dbi).map(|cursor| RoIter::new(cursor)) @@ -1128,7 +1128,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn rev_iter<'txn>(&self, txn: &'txn RoTxn) -> Result> { assert_eq_env_db_txn!(self, txn); @@ -1235,7 +1235,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn range<'a, 'txn, R>( &self, txn: &'txn RoTxn, @@ -1408,7 +1408,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn rev_range<'a, 'txn, R>( &self, txn: &'txn RoTxn, @@ -1583,7 +1583,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn prefix_iter<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -1716,7 +1716,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, feature = "encryption"), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] pub fn rev_prefix_iter<'a, 'txn>( &self, txn: &'txn RoTxn, diff --git a/heed/src/env/clear.rs b/heed/src/env/clear.rs index 3e1ee899..6780a47a 100644 --- a/heed/src/env/clear.rs +++ b/heed/src/env/clear.rs @@ -13,17 +13,11 @@ use synchronoise::SignalEvent; #[cfg(windows)] use crate::env::OsStrExtLmdb as _; -use crate::env::{canonicalize_path, Env, EnvFlags, EnvInner, OPENED_ENV}; +use crate::env::{canonicalize_path, Env, EnvEntry, EnvFlags, EnvInner, OPENED_ENV}; use crate::mdb::ffi; use crate::mdb::lmdb_error::mdb_result; use crate::{Error, Result}; -pub struct EnvEntry { - pub(super) env: Option, - pub(super) signal_event: Arc, - pub(super) options: EnvOpenOptions, -} - /// Options and flags which can be used to configure how an environment is opened. #[derive(Clone, Debug, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] diff --git a/heed/src/env/encrypted.rs b/heed/src/env/encrypted.rs index 4feacd8d..9d9af96a 100644 --- a/heed/src/env/encrypted.rs +++ b/heed/src/env/encrypted.rs @@ -18,95 +18,25 @@ use synchronoise::SignalEvent; #[cfg(windows)] use crate::env::OsStrExtLmdb as _; -use crate::env::{canonicalize_path, Env, EnvFlags, EnvInner, OPENED_ENV}; +use crate::env::{canonicalize_path, Env, EnvEntry, EnvFlags, EnvInner, OPENED_ENV}; use crate::mdb::ffi; use crate::mdb::lmdb_error::mdb_result; use crate::{Error, Result}; -pub struct EnvEntry { - pub(super) env: Option, - pub(super) signal_event: Arc, - pub(super) options: SimplifiedOpenOptions, -} - -/// A simplified version of the options that were used to open a given [`Env`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SimplifiedOpenOptions { - /// Weither this [`Env`] has been opened with an encryption/decryption algorithm. - pub use_encryption: bool, - /// The maximum size this [`Env`] with take in bytes or [`None`] if it was not specified. - pub map_size: Option, - /// The maximum number of concurrent readers or [`None`] if it was not specified. - pub max_readers: Option, - /// The maximum number of opened database or [`None`] if it was not specified. - pub max_dbs: Option, - /// The raw flags enabled for this [`Env`] or [`None`] if it was not specified. - pub flags: u32, -} - -impl From<&EnvOpenOptions> for SimplifiedOpenOptions { - fn from(eoo: &EnvOpenOptions) -> SimplifiedOpenOptions { - let EnvOpenOptions { encrypt, map_size, max_readers, max_dbs, flags } = eoo; - SimplifiedOpenOptions { - use_encryption: encrypt.is_some(), - map_size: *map_size, - max_readers: *max_readers, - max_dbs: *max_dbs, - flags: flags.bits(), - } - } -} - /// Options and flags which can be used to configure how an environment is opened. #[derive(Clone, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct EnvOpenOptions { - encrypt: Option>, +pub struct EnvOpenOptions { + encrypt: Key, map_size: Option, max_readers: Option, max_dbs: Option, flags: EnvFlags, } -impl Default for EnvOpenOptions { - fn default() -> Self { - Self::new() - } -} - -impl EnvOpenOptions { - /// Creates a blank new set of options ready for configuration. - pub fn new() -> EnvOpenOptions { - EnvOpenOptions { - encrypt: None, - map_size: None, - max_readers: None, - max_dbs: None, - flags: EnvFlags::empty(), - } - } -} - impl EnvOpenOptions { - /// Set the size of the memory map to use for this environment. - pub fn map_size(&mut self, size: usize) -> &mut Self { - self.map_size = Some(size); - self - } - - /// Set the maximum number of threads/reader slots for the environment. - pub fn max_readers(&mut self, readers: u32) -> &mut Self { - self.max_readers = Some(readers); - self - } - - /// Set the maximum number of named databases for the environment. - pub fn max_dbs(&mut self, dbs: u32) -> &mut Self { - self.max_dbs = Some(dbs); - self - } - - /// Specifies that the [`Env`] will be encrypted using the `A` algorithm with the given `key`. + /// Creates a blank new set of options ready for configuration and specifies that + /// the [`Env`] will be encrypted using the `E` algorithm with the given `key`. /// /// You can find more compatible algorithms on [the RustCrypto/AEADs page](https://github.com/RustCrypto/AEADs#crates). /// @@ -205,10 +135,32 @@ impl EnvOpenOptions { /// let _force_keep = val1; /// # Ok(()) } /// ``` - #[cfg(feature = "encryption")] - pub fn encrypt_with(self, key: Key) -> EnvOpenOptions { - let EnvOpenOptions { encrypt: _, map_size, max_readers, max_dbs, flags } = self; - EnvOpenOptions { encrypt: Some(key), map_size, max_readers, max_dbs, flags } + pub fn new_encrypted_with(key: Key) -> EnvOpenOptions { + EnvOpenOptions { + encrypt: key, + map_size: None, + max_readers: None, + max_dbs: None, + flags: EnvFlags::empty(), + } + } + + /// Set the size of the memory map to use for this environment. + pub fn map_size(&mut self, size: usize) -> &mut Self { + self.map_size = Some(size); + self + } + + /// Set the maximum number of threads/reader slots for the environment. + pub fn max_readers(&mut self, readers: u32) -> &mut Self { + self.max_readers = Some(readers); + self + } + + /// Set the maximum number of named databases for the environment. + pub fn max_dbs(&mut self, dbs: u32) -> &mut Self { + self.max_dbs = Some(dbs); + self } /// Set one or [more LMDB flags](http://www.lmdb.tech/doc/group__mdb__env.html). @@ -330,16 +282,13 @@ impl EnvOpenOptions { let mut env: *mut ffi::MDB_env = ptr::null_mut(); mdb_result(ffi::mdb_env_create(&mut env))?; - // This block will only be executed if the "encryption" feature is enabled. - if let Some(key) = &self.encrypt { - let key = crate::into_val(key); - mdb_result(ffi::mdb_env_set_encrypt( - env, - Some(encrypt_func_wrapper::), - &key, - ::TagSize::U32, - ))?; - } + let encrypt_key = crate::into_val(self.encrypt); + mdb_result(ffi::mdb_env_set_encrypt( + env, + Some(encrypt_func_wrapper::), + &encrypt_key, + ::TagSize::U32, + ))?; if let Some(size) = self.map_size { if size % page_size::get() != 0 { @@ -402,9 +351,8 @@ impl EnvOpenOptions { impl fmt::Debug for EnvOpenOptions { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let EnvOpenOptions { encrypt, map_size, max_readers, max_dbs, flags } = self; + let EnvOpenOptions { encrypt: _, map_size, max_readers, max_dbs, flags } = self; f.debug_struct("EnvOpenOptions") - .field("encrypted", &encrypt.is_some()) .field("map_size", &map_size) .field("max_readers", &max_readers) .field("max_dbs", &max_dbs) @@ -491,44 +439,3 @@ unsafe extern "C" fn encrypt_func_wrapper( Err(_) => 1, } } - -/// A dummy encryption/decryption algorithm that must never be used. -/// Only here for Rust API purposes. -pub enum DummyEncrypt {} - -impl AeadMutInPlace for DummyEncrypt { - fn encrypt_in_place_detached( - &mut self, - _nonce: &Nonce, - _associated_data: &[u8], - _buffer: &mut [u8], - ) -> aead::Result> { - Err(aead::Error) - } - - fn decrypt_in_place_detached( - &mut self, - _nonce: &Nonce, - _associated_data: &[u8], - _buffer: &mut [u8], - _tag: &Tag, - ) -> aead::Result<()> { - Err(aead::Error) - } -} - -impl AeadCore for DummyEncrypt { - type NonceSize = U0; - type TagSize = U0; - type CiphertextOverhead = U0; -} - -impl KeySizeUser for DummyEncrypt { - type KeySize = U0; -} - -impl KeyInit for DummyEncrypt { - fn new(_key: &GenericArray) -> Self { - panic!("This DummyEncrypt type must not be used") - } -} diff --git a/heed/src/env/mod.rs b/heed/src/env/mod.rs index 1499c246..0c8f00ad 100644 --- a/heed/src/env/mod.rs +++ b/heed/src/env/mod.rs @@ -30,24 +30,28 @@ use crate::mdb::ffi; use crate::mdb::lmdb_flags::AllDatabaseFlags; use crate::{Database, EnvFlags, Error, Result, RoCursor, RoTxn, RwTxn, Unspecified}; -#[cfg(not(master3))] +#[cfg(not(all(master3, encryption)))] mod clear; -#[cfg(not(master3))] +#[cfg(not(all(master3, encryption)))] use clear as sub_env; -#[cfg(master3)] +#[cfg(all(master3, encryption))] mod encrypted; -#[cfg(master3)] +#[cfg(all(master3, encryption))] use encrypted as sub_env; -#[cfg(master3)] -pub use sub_env::SimplifiedOpenOptions; /// The list of opened environments, the value is an optional environment, it is None /// when someone asks to close the environment, closing is a two-phase step, to make sure /// no one tries to open the same environment between these two phases. /// /// Trying to open a None marked environment returns an error to the user trying to open it. -static OPENED_ENV: Lazy>> = Lazy::new(RwLock::default); +static OPENED_ENV: Lazy>> = Lazy::new(RwLock::default); + +pub struct EnvEntry { + env: Option, + signal_event: Arc, + options: EnvOpenOptions, +} // Thanks to the mozilla/rkv project // Workaround the UNC path on Windows, see https://github.com/rust-lang/rust/issues/42869. @@ -136,7 +140,7 @@ impl Drop for EnvInner { match lock.remove(&self.path) { None => panic!("It seems another env closed this env before"), - Some(sub_env::EnvEntry { signal_event, .. }) => { + Some(EnvEntry { signal_event, .. }) => { unsafe { ffi::mdb_env_close(self.env); } @@ -687,7 +691,7 @@ impl Env { let mut lock = OPENED_ENV.write().unwrap(); match lock.get_mut(self.path()) { None => panic!("cannot find the env that we are trying to close"), - Some(sub_env::EnvEntry { env, signal_event, .. }) => { + Some(EnvEntry { env, signal_event, .. }) => { // We remove the env from the global list and replace it with a None. let _env = env.take(); let signal_event = signal_event.clone(); diff --git a/heed/src/lib.rs b/heed/src/lib.rs index 136430af..906b1db3 100644 --- a/heed/src/lib.rs +++ b/heed/src/lib.rs @@ -154,10 +154,6 @@ pub enum Error { /// Attempt to open [`Env`] with different options. BadOpenOptions { /// The options that were used to originally open this env. - #[cfg(master3)] - options: env::SimplifiedOpenOptions, - /// The options that were used to originally open this env. - #[cfg(not(master3))] options: EnvOpenOptions, /// The env opened with the original options. env: Env, diff --git a/heed/src/txn.rs b/heed/src/txn.rs index 5d648de6..87da9f43 100644 --- a/heed/src/txn.rs +++ b/heed/src/txn.rs @@ -209,7 +209,7 @@ impl<'p> Deref for RwTxn<'p> { } // TODO can't we just always implement it? -#[cfg(all(master3, feature = "encryption"))] +#[cfg(all(master3, encryption))] impl<'p> std::ops::DerefMut for RwTxn<'p> { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.txn diff --git a/heed3-encryption/Cargo.toml b/heed3-encryption/Cargo.toml new file mode 100644 index 00000000..9c8d61d5 --- /dev/null +++ b/heed3-encryption/Cargo.toml @@ -0,0 +1,124 @@ +[package] +name = "heed3-encryption" +version = "0.20.5-beta.1" +authors = ["Kerollmops "] +description = "A fully typed LMDB (mdb.master3) wrapper with minimum overhead with support for encryption" +license = "MIT" +repository = "https://github.com/Kerollmops/heed" +keywords = ["lmdb", "database", "storage", "typed", "encryption"] +categories = ["database", "data-structures"] +readme = "../README.md" +edition = "2021" + +[dependencies] +# TODO update dependencies +aead = { version = "0.5.1", default-features = false } +bitflags = { version = "2.6.0", features = ["serde"] } +byteorder = { version = "1.5.0", default-features = false } +generic-array = { version = "0.14.6", features = ["serde"] } +heed-master3-proc-macro = { version = "0.1.0", path = "../heed-master3-proc-macro" } +heed-traits = { version = "0.20.0", path = "../heed-traits" } +heed-types = { version = "0.20.1", default-features = false, path = "../heed-types" } +libc = "0.2.155" +lmdb-master3-sys = { version = "0.2.4", path = "../lmdb-master3-sys" } +once_cell = "1.19.0" +page_size = "0.6.0" +serde = { version = "1.0.203", features = ["derive"], optional = true } +synchronoise = "1.0.1" + +[dev-dependencies] +# TODO update dependencies +argon2 = { version = "0.4.1", features = ["std"] } +serde = { version = "1.0.203", features = ["derive"] } +chacha20poly1305 = "0.10.1" +tempfile = "3.10.1" + +[target.'cfg(windows)'.dependencies] +url = "2.5.2" + +[features] +# The `serde` feature makes some types serializable, +# like the `EnvOpenOptions` struct. +default = ["serde", "serde-bincode", "serde-json"] +serde = ["bitflags/serde", "dep:serde"] + +# The #MDB_NOTLS flag is automatically set on Env opening, +# RoTxn and RoCursors implements the Send trait. This allows the +# user to move RoTxns and RoCursors between threads as read transactions +# will no more use thread local storage and will tie reader locktable +# slots to #MDB_txn objects instead of to threads. +# +# According to the LMDB documentation, when this feature is not enabled: +# A thread can only use one transaction at a time, plus any child +# transactions. Each transaction belongs to one thread. [...] +# The #MDB_NOTLS flag changes this for read-only transactions. +# +# And a #MDB_BAD_RSLOT error will be thrown when multiple read +# transactions exists on the same thread +read-txn-no-tls = [] + +# Enable the serde en/decoders for bincode, serde_json, or rmp_serde +serde-bincode = ["heed-types/serde-bincode"] +serde-json = ["heed-types/serde-json"] +serde-rmp = ["heed-types/serde-rmp"] + +# serde_json features +preserve_order = ["heed-types/preserve_order"] +arbitrary_precision = ["heed-types/arbitrary_precision"] +raw_value = ["heed-types/raw_value"] +unbounded_depth = ["heed-types/unbounded_depth"] + +# Whether to tell LMDB to use POSIX semaphores during compilation +# (instead of the default, which are System V semaphores). +# POSIX semaphores are required for Apple's App Sandbox on iOS & macOS, +# and are possibly faster and more appropriate for single-process use. +# There are tradeoffs for both POSIX and SysV semaphores; which you +# should look into before enabling this feature. Also, see here: +# +posix-sem = ["lmdb-master3-sys/posix-sem"] + +# These features configure the MDB_IDL_LOGN macro, which determines +# the size of the free and dirty page lists (and thus the amount of memory +# allocated when opening an LMDB environment in read-write mode). +# +# Each feature defines MDB_IDL_LOGN as the value in the name of the feature. +# That means these features are mutually exclusive, and you must not specify +# more than one at the same time (or the crate will fail to compile). +# +# For more information on the motivation for these features (and their effect), +# see https://github.com/mozilla/lmdb/pull/2. +mdb_idl_logn_8 = ["lmdb-master3-sys/mdb_idl_logn_8"] +mdb_idl_logn_9 = ["lmdb-master3-sys/mdb_idl_logn_9"] +mdb_idl_logn_10 = ["lmdb-master3-sys/mdb_idl_logn_10"] +mdb_idl_logn_11 = ["lmdb-master3-sys/mdb_idl_logn_11"] +mdb_idl_logn_12 = ["lmdb-master3-sys/mdb_idl_logn_12"] +mdb_idl_logn_13 = ["lmdb-master3-sys/mdb_idl_logn_13"] +mdb_idl_logn_14 = ["lmdb-master3-sys/mdb_idl_logn_14"] +mdb_idl_logn_15 = ["lmdb-master3-sys/mdb_idl_logn_15"] +mdb_idl_logn_16 = ["lmdb-master3-sys/mdb_idl_logn_16"] + +# Setting this enables you to use keys longer than 511 bytes. The exact limit +# is computed by LMDB at compile time. You can find the exact value by calling +# Env::max_key_size(). This value varies by architecture. +# +# Example max key sizes: +# - Apple M1 (ARM64): 8126 bytes +# - Apple Intel (AMD64): 1982 bytes +# - Linux Intel (AMD64): 1982 bytes +# +# Setting this also enables you to use values larger than 511 bytes when using +# a Database with the DatabaseFlags::DUP_SORT flag. +# +# This builds LMDB with the -DMDB_MAXKEYSIZE=0 option. +# +# Note: If you are moving database files between architectures then your longest +# stored key must fit within the smallest limit of all architectures used. For +# example, if you are moving databases between Apple M1 and Apple Intel +# computers then you need to keep your keys within the smaller 1982 byte limit. +longer-keys = ["lmdb-master3-sys/longer-keys"] + +# Examples are located outside the standard heed/examples directory to prevent +# conflicts between heed3 and heed examples when working on both crates. +[[example]] +name = "heed3-encryption" +path = "../examples/heed3-encryption.rs" diff --git a/heed3/Cargo.toml b/heed3/Cargo.toml index d023d9a7..04b1ebdd 100644 --- a/heed3/Cargo.toml +++ b/heed3/Cargo.toml @@ -2,7 +2,7 @@ name = "heed3" version = "0.20.5-beta.1" authors = ["Kerollmops "] -description = "A fully typed LMDB wrapper with minimum overhead and optional support for encryption" +description = "A fully typed LMDB (mdb.master3) wrapper with minimum overhead" license = "MIT" repository = "https://github.com/Kerollmops/heed" keywords = ["lmdb", "database", "storage", "typed", "encryption"] @@ -11,12 +11,8 @@ readme = "../README.md" edition = "2021" [dependencies] -# TODO update dependencies -aead = { version = "0.5.1", default-features = false, optional = true } bitflags = { version = "2.6.0", features = ["serde"] } byteorder = { version = "1.5.0", default-features = false } -generic-array = { version = "0.14.6", features = ["serde"], optional = true } -heed-master3-proc-macro = { version = "0.1.0", path = "../heed-master3-proc-macro", optional = true } heed-traits = { version = "0.20.0", path = "../heed-traits" } heed-types = { version = "0.20.1", default-features = false, path = "../heed-types" } libc = "0.2.155" @@ -28,9 +24,7 @@ synchronoise = "1.0.1" [dev-dependencies] # TODO update dependencies -argon2 = { version = "0.4.1", features = ["std"] } serde = { version = "1.0.203", features = ["derive"] } -chacha20poly1305 = "0.10.1" tempfile = "3.10.1" [target.'cfg(windows)'.dependencies] @@ -39,13 +33,7 @@ url = "2.5.2" [features] # The `serde` feature makes some types serializable, # like the `EnvOpenOptions` struct. -# TODO remove encryption from defaults -default = ["serde", "serde-bincode", "serde-json", "encryption"] - -# Enable the LMDB encryption feature -# TODO add more information here -encryption = ["dep:heed-master3-proc-macro", "dep:aead", "dep:generic-array"] - +default = ["serde", "serde-bincode", "serde-json"] serde = ["bitflags/serde", "dep:serde"] # The #MDB_NOTLS flag is automatically set on Env opening, @@ -126,10 +114,30 @@ longer-keys = ["lmdb-master3-sys/longer-keys"] # Examples are located outside the standard heed/examples directory to prevent # conflicts between heed3 and heed examples when working on both crates. [[example]] -name = "all-types-heed3" -path = "../examples/all-types-heed3.rs" +name = "all-types" +path = "../examples/all-types.rs" + +[[example]] +name = "clear-database" +path = "../examples/clear-database.rs" + +[[example]] +name = "cursor-append" +path = "../examples/cursor-append.rs" + +[[example]] +name = "custom-comparator" +path = "../examples/custom-comparator.rs" + +[[example]] +name = "multi-env" +path = "../examples/multi-env.rs" + +[[example]] +name = "nested" +path = "../examples/nested.rs" [[example]] -name = "encrypt-heed3" -path = "../examples/encrypt-heed3.rs" -required-features = ["encryption"] +name = "rmp-serde" +path = "../examples/rmp-serde.rs" +required-features = ["serde-rmp"] From 1057112717d823c4f921031ec1fc5717b7a92da0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 21 Aug 2024 14:48:54 +0200 Subject: [PATCH 23/51] Test the heed3 project and check the heed3-encryption one in the CI --- .github/workflows/rust.yml | 87 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index cca1bbfe..81dd945d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -28,8 +28,8 @@ jobs: cargo clean cargo test - check: - name: Check the heed3 project + test-heed3: + name: Test the heed3 project runs-on: ${{ matrix.os }} strategy: matrix: @@ -48,11 +48,37 @@ jobs: profile: minimal toolchain: stable override: true - - name: Run cargo check + - name: Run cargo test run: | cargo clean cp heed3/Cargo.toml heed/ - cargo check -p heed3 + cargo test -p heed3 + + check-heed3-encryption: + name: Check the heed3-encryption project + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + include: + - os: ubuntu-latest + - os: windows-latest + - os: macos-latest + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - name: Run cargo check + run: | + cargo clean + cp heed3-encryption/Cargo.toml heed/ + cargo check -p heed3-encryption check_all_features: name: Check all the features of the heed project @@ -81,6 +107,34 @@ jobs: cargo clean cargo check --all-features + check_all_features-heed3: + name: Check all the features of the heed3 project + runs-on: ${{ matrix.os }} + env: + RUSTFLAGS: -D warnings + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + include: + - os: ubuntu-latest + - os: macos-latest + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - name: Run cargo test + run: | + cd heed + cargo clean + cp heed3-encryption/Cargo.toml heed/ + cargo check --all-features -p heed3 + examples: name: Run the heed examples runs-on: ${{ matrix.os }} @@ -132,6 +186,31 @@ jobs: cp heed3/Cargo.toml heed/ cargo run --example 2>&1 | grep -E '^ '| xargs -n1 cargo run --example + heed3-encryption-examples: + name: Run the heed3-encryption examples + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + include: + - os: ubuntu-latest + - os: macos-latest + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - name: Run the examples + run: | + cargo clean + cp heed3-encryption/Cargo.toml heed/ + cargo run --example 2>&1 | grep -E '^ '| xargs -n1 cargo run --example + fmt: name: Ensure the heed project is formatted runs-on: ubuntu-latest From 7b3d7868204423a77600d365acd5234cb1ac126b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 21 Aug 2024 14:51:58 +0200 Subject: [PATCH 24/51] Make the heed3 examples pass --- examples/heed3-all-types.rs | 111 ++++++++++++++++++++++++++++++++++++ heed3/Cargo.toml | 29 +--------- 2 files changed, 113 insertions(+), 27 deletions(-) create mode 100644 examples/heed3-all-types.rs diff --git a/examples/heed3-all-types.rs b/examples/heed3-all-types.rs new file mode 100644 index 00000000..3fb77be5 --- /dev/null +++ b/examples/heed3-all-types.rs @@ -0,0 +1,111 @@ +use std::error::Error; +use std::fs; +use std::path::Path; + +use heed3::byteorder::BE; +use heed3::types::*; +use heed3::{Database, EnvOpenOptions}; +use serde::{Deserialize, Serialize}; + +fn main() -> Result<(), Box> { + let path = Path::new("target").join("heed3.mdb"); + + fs::create_dir_all(&path)?; + + let env = unsafe { + EnvOpenOptions::new() + .map_size(10 * 1024 * 1024) // 10MB + .max_dbs(3000) + .open(path)? + }; + + // here the key will be an str and the data will be a slice of u8 + let mut wtxn = env.write_txn()?; + let db: Database = env.create_database(&mut wtxn, Some("kiki"))?; + + db.put(&mut wtxn, "hello", &[2, 3][..])?; + let ret: Option<&[u8]> = db.get(&wtxn, "hello")?; + + println!("{:?}", ret); + wtxn.commit()?; + + // serde types are also supported!!! + #[derive(Debug, Serialize, Deserialize)] + struct Hello<'a> { + string: &'a str, + } + + let mut wtxn = env.write_txn()?; + let db: Database> = + env.create_database(&mut wtxn, Some("serde-bincode"))?; + + let hello = Hello { string: "hi" }; + db.put(&mut wtxn, "hello", &hello)?; + + let ret: Option = db.get(&wtxn, "hello")?; + println!("serde-bincode:\t{:?}", ret); + + wtxn.commit()?; + + let mut wtxn = env.write_txn()?; + let db: Database> = env.create_database(&mut wtxn, Some("serde-json"))?; + + let hello = Hello { string: "hi" }; + db.put(&mut wtxn, "hello", &hello)?; + + let ret: Option = db.get(&wtxn, "hello")?; + println!("serde-json:\t{:?}", ret); + + wtxn.commit()?; + + // you can ignore the data + let mut wtxn = env.write_txn()?; + let db: Database = env.create_database(&mut wtxn, Some("ignored-data"))?; + + db.put(&mut wtxn, "hello", &())?; + let ret: Option<()> = db.get(&wtxn, "hello")?; + + println!("{:?}", ret); + + let ret: Option<()> = db.get(&wtxn, "non-existant")?; + + println!("{:?}", ret); + wtxn.commit()?; + + // database opening and types are tested in a safe way + // + // we try to open a database twice with the same types + let mut wtxn = env.write_txn()?; + let _db: Database = env.create_database(&mut wtxn, Some("ignored-data"))?; + + // you can iterate over keys in order + type BEI64 = I64; + + let db: Database = env.create_database(&mut wtxn, Some("big-endian-iter"))?; + + db.put(&mut wtxn, &0, &())?; + db.put(&mut wtxn, &68, &())?; + db.put(&mut wtxn, &35, &())?; + db.put(&mut wtxn, &42, &())?; + + let rets: Result, _> = db.iter(&wtxn)?.collect(); + + println!("{:?}", rets); + + // or iterate over ranges too!!! + let range = 35..=42; + let rets: Result, _> = db.range(&wtxn, &range)?.collect(); + + println!("{:?}", rets); + + // delete a range of key + let range = 35..=42; + let deleted: usize = db.delete_range(&mut wtxn, &range)?; + + let rets: Result, _> = db.iter(&wtxn)?.collect(); + + println!("deleted: {:?}, {:?}", deleted, rets); + wtxn.commit()?; + + Ok(()) +} diff --git a/heed3/Cargo.toml b/heed3/Cargo.toml index 04b1ebdd..80a2be18 100644 --- a/heed3/Cargo.toml +++ b/heed3/Cargo.toml @@ -114,30 +114,5 @@ longer-keys = ["lmdb-master3-sys/longer-keys"] # Examples are located outside the standard heed/examples directory to prevent # conflicts between heed3 and heed examples when working on both crates. [[example]] -name = "all-types" -path = "../examples/all-types.rs" - -[[example]] -name = "clear-database" -path = "../examples/clear-database.rs" - -[[example]] -name = "cursor-append" -path = "../examples/cursor-append.rs" - -[[example]] -name = "custom-comparator" -path = "../examples/custom-comparator.rs" - -[[example]] -name = "multi-env" -path = "../examples/multi-env.rs" - -[[example]] -name = "nested" -path = "../examples/nested.rs" - -[[example]] -name = "rmp-serde" -path = "../examples/rmp-serde.rs" -required-features = ["serde-rmp"] +name = "heed3-all-types" +path = "../examples/heed3-all-types.rs" From 8a16b12fcb34b0c32a032547e07f75f7748ee8b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 21 Aug 2024 14:54:21 +0200 Subject: [PATCH 25/51] Do not import the encrypt LMDB function for heed3 --- heed/src/mdb/lmdb_ffi.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heed/src/mdb/lmdb_ffi.rs b/heed/src/mdb/lmdb_ffi.rs index cb8b3ba5..36f399ed 100644 --- a/heed/src/mdb/lmdb_ffi.rs +++ b/heed/src/mdb/lmdb_ffi.rs @@ -1,6 +1,6 @@ use std::ptr; -#[cfg(master3)] +#[cfg(all(master3, encryption))] pub use ffi::mdb_env_set_encrypt; pub use ffi::{ mdb_cursor_close, mdb_cursor_del, mdb_cursor_get, mdb_cursor_open, mdb_cursor_put, From c4f4b45fa8f3e047ea40360a26f3f3b40611d819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 21 Aug 2024 15:12:44 +0200 Subject: [PATCH 26/51] Reintroduce the Simplified Open Options --- examples/heed3-encryption.rs | 2 +- heed/src/env/clear.rs | 8 ++++- heed/src/env/encrypted.rs | 60 ++++++++++++++++++++++++++++-------- heed/src/env/mod.rs | 14 +++------ heed/src/lib.rs | 4 +++ 5 files changed, 64 insertions(+), 24 deletions(-) diff --git a/examples/heed3-encryption.rs b/examples/heed3-encryption.rs index 99f7a09d..da16491f 100644 --- a/examples/heed3-encryption.rs +++ b/examples/heed3-encryption.rs @@ -21,7 +21,7 @@ fn main() -> Result<(), Box> { Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; // We open the environment - let mut options = EnvOpenOptions::new().encrypt_with::(key); + let mut options = EnvOpenOptions::::new_encrypted_with(key); let env = unsafe { options .map_size(10 * 1024 * 1024) // 10MB diff --git a/heed/src/env/clear.rs b/heed/src/env/clear.rs index 6780a47a..3e1ee899 100644 --- a/heed/src/env/clear.rs +++ b/heed/src/env/clear.rs @@ -13,11 +13,17 @@ use synchronoise::SignalEvent; #[cfg(windows)] use crate::env::OsStrExtLmdb as _; -use crate::env::{canonicalize_path, Env, EnvEntry, EnvFlags, EnvInner, OPENED_ENV}; +use crate::env::{canonicalize_path, Env, EnvFlags, EnvInner, OPENED_ENV}; use crate::mdb::ffi; use crate::mdb::lmdb_error::mdb_result; use crate::{Error, Result}; +pub struct EnvEntry { + pub(super) env: Option, + pub(super) signal_event: Arc, + pub(super) options: EnvOpenOptions, +} + /// Options and flags which can be used to configure how an environment is opened. #[derive(Clone, Debug, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] diff --git a/heed/src/env/encrypted.rs b/heed/src/env/encrypted.rs index 9d9af96a..b5e9bd05 100644 --- a/heed/src/env/encrypted.rs +++ b/heed/src/env/encrypted.rs @@ -10,19 +10,48 @@ use std::path::Path; use std::sync::Arc; use std::{fmt, io, ptr}; -use aead::consts::U0; use aead::generic_array::typenum::Unsigned; -use aead::generic_array::GenericArray; -use aead::{AeadCore, AeadMutInPlace, Key, KeyInit, KeySizeUser, Nonce, Tag}; +use aead::{AeadCore, AeadMutInPlace, Key, KeyInit, Nonce, Tag}; use synchronoise::SignalEvent; #[cfg(windows)] use crate::env::OsStrExtLmdb as _; -use crate::env::{canonicalize_path, Env, EnvEntry, EnvFlags, EnvInner, OPENED_ENV}; +use crate::env::{canonicalize_path, Env, EnvFlags, EnvInner, OPENED_ENV}; use crate::mdb::ffi; use crate::mdb::lmdb_error::mdb_result; use crate::{Error, Result}; +pub struct EnvEntry { + pub(super) env: Option, + pub(super) signal_event: Arc, + pub(super) options: SimplifiedEncryptedOpenOptions, +} + +/// A simplified version of the options that were used to open a given [`Env`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SimplifiedEncryptedOpenOptions { + /// The maximum size this [`Env`] with take in bytes or [`None`] if it was not specified. + pub map_size: Option, + /// The maximum number of concurrent readers or [`None`] if it was not specified. + pub max_readers: Option, + /// The maximum number of opened database or [`None`] if it was not specified. + pub max_dbs: Option, + /// The raw flags enabled for this [`Env`] or [`None`] if it was not specified. + pub flags: u32, +} + +impl From<&EnvOpenOptions> for SimplifiedEncryptedOpenOptions { + fn from(eoo: &EnvOpenOptions) -> SimplifiedEncryptedOpenOptions { + let EnvOpenOptions { encrypt: _, map_size, max_readers, max_dbs, flags } = eoo; + SimplifiedEncryptedOpenOptions { + map_size: *map_size, + max_readers: *max_readers, + max_dbs: *max_dbs, + flags: flags.bits(), + } + } +} + /// Options and flags which can be used to configure how an environment is opened. #[derive(Clone, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] @@ -55,8 +84,8 @@ impl EnvOpenOptions { /// use std::path::Path; /// use argon2::Argon2; /// use chacha20poly1305::{ChaCha20Poly1305, Key}; - /// use heed3::types::*; - /// use heed3::{EnvOpenOptions, Database}; + /// use heed3_encryption::types::*; + /// use heed3_encryption::{EnvOpenOptions, Database}; /// /// # fn main() -> Result<(), Box> { /// let env_path = Path::new("target").join("encrypt.mdb"); @@ -70,7 +99,7 @@ impl EnvOpenOptions { /// Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; /// /// // We open the environment - /// let mut options = EnvOpenOptions::new().encrypt_with::(key); + /// let mut options = EnvOpenOptions::::new_encrypted_with(key); /// let env = unsafe { /// options /// .map_size(10 * 1024 * 1024) // 10MB @@ -94,13 +123,13 @@ impl EnvOpenOptions { /// /// ## Example Showing limitations /// - /// ```compile_fail + /// ```compile_fail,E0499 /// use std::fs; /// use std::path::Path; /// use argon2::Argon2; /// use chacha20poly1305::{ChaCha20Poly1305, Key}; - /// use heed3::types::*; - /// use heed3::{EnvOpenOptions, Database}; + /// use heed3_encryption::types::*; + /// use heed3_encryption::{EnvOpenOptions, Database}; /// /// # fn main() -> Result<(), Box> { /// let env_path = Path::new("target").join("encrypt.mdb"); @@ -114,7 +143,7 @@ impl EnvOpenOptions { /// Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; /// /// // We open the environment - /// let mut options = EnvOpenOptions::new().encrypt_with::(key); + /// let mut options = EnvOpenOptions::::new_encrypted_with(key); /// let env = unsafe { /// options /// .map_size(10 * 1024 * 1024) // 10MB @@ -125,6 +154,11 @@ impl EnvOpenOptions { /// let key1 = "first-key"; /// let key2 = "second-key"; /// + /// // We create the database + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("first"))?; + /// wtxn.commit()?; + /// /// // Declare the read transaction as mutable because LMDB, when using encryption, /// // does not allow keeping keys between reads due to the use of an internal cache. /// let mut rtxn = env.read_txn()?; @@ -263,7 +297,7 @@ impl EnvOpenOptions { Ok(path) => path, }; - let original_options = SimplifiedOpenOptions::from(self); + let original_options = SimplifiedEncryptedOpenOptions::from(self); match lock.entry(path) { Entry::Occupied(entry) => { let env = entry.get().env.clone().ok_or(Error::DatabaseClosing)?; @@ -282,7 +316,7 @@ impl EnvOpenOptions { let mut env: *mut ffi::MDB_env = ptr::null_mut(); mdb_result(ffi::mdb_env_create(&mut env))?; - let encrypt_key = crate::into_val(self.encrypt); + let encrypt_key = crate::into_val(&self.encrypt); mdb_result(ffi::mdb_env_set_encrypt( env, Some(encrypt_func_wrapper::), diff --git a/heed/src/env/mod.rs b/heed/src/env/mod.rs index 0c8f00ad..30c83fe0 100644 --- a/heed/src/env/mod.rs +++ b/heed/src/env/mod.rs @@ -39,19 +39,15 @@ use clear as sub_env; mod encrypted; #[cfg(all(master3, encryption))] use encrypted as sub_env; +#[cfg(all(master3, encryption))] +pub use encrypted::SimplifiedEncryptedOpenOptions; /// The list of opened environments, the value is an optional environment, it is None /// when someone asks to close the environment, closing is a two-phase step, to make sure /// no one tries to open the same environment between these two phases. /// /// Trying to open a None marked environment returns an error to the user trying to open it. -static OPENED_ENV: Lazy>> = Lazy::new(RwLock::default); - -pub struct EnvEntry { - env: Option, - signal_event: Arc, - options: EnvOpenOptions, -} +static OPENED_ENV: Lazy>> = Lazy::new(RwLock::default); // Thanks to the mozilla/rkv project // Workaround the UNC path on Windows, see https://github.com/rust-lang/rust/issues/42869. @@ -140,7 +136,7 @@ impl Drop for EnvInner { match lock.remove(&self.path) { None => panic!("It seems another env closed this env before"), - Some(EnvEntry { signal_event, .. }) => { + Some(sub_env::EnvEntry { signal_event, .. }) => { unsafe { ffi::mdb_env_close(self.env); } @@ -691,7 +687,7 @@ impl Env { let mut lock = OPENED_ENV.write().unwrap(); match lock.get_mut(self.path()) { None => panic!("cannot find the env that we are trying to close"), - Some(EnvEntry { env, signal_event, .. }) => { + Some(sub_env::EnvEntry { env, signal_event, .. }) => { // We remove the env from the global list and replace it with a None. let _env = env.take(); let signal_event = signal_event.clone(); diff --git a/heed/src/lib.rs b/heed/src/lib.rs index 906b1db3..a3abcd81 100644 --- a/heed/src/lib.rs +++ b/heed/src/lib.rs @@ -153,7 +153,11 @@ pub enum Error { DatabaseClosing, /// Attempt to open [`Env`] with different options. BadOpenOptions { + /// The simplified options that were used to originally open this env. + #[cfg(all(master3, encryption))] + options: env::SimplifiedEncryptedOpenOptions, /// The options that were used to originally open this env. + #[cfg(not(all(master3, encryption)))] options: EnvOpenOptions, /// The env opened with the original options. env: Env, From 727aba2981794d11a652cc3b73703f0b22ccf4ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 21 Aug 2024 15:28:26 +0200 Subject: [PATCH 27/51] Make sure we only check heed3 not test it --- .github/workflows/rust.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 81dd945d..1059e2c6 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -28,8 +28,8 @@ jobs: cargo clean cargo test - test-heed3: - name: Test the heed3 project + check-heed3: + name: Check the heed3 project runs-on: ${{ matrix.os }} strategy: matrix: @@ -48,11 +48,11 @@ jobs: profile: minimal toolchain: stable override: true - - name: Run cargo test + - name: Run cargo check run: | cargo clean cp heed3/Cargo.toml heed/ - cargo test -p heed3 + cargo check -p heed3 check-heed3-encryption: name: Check the heed3-encryption project From fab327c151aa99e4c005a97c07a961a610a66bf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 21 Aug 2024 18:45:35 +0200 Subject: [PATCH 28/51] Fix CI for heed3-encryption --- .github/workflows/rust.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1059e2c6..5b385798 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -103,9 +103,8 @@ jobs: override: true - name: Run cargo test run: | - cd heed cargo clean - cargo check --all-features + cargo check --all-features -p heed check_all_features-heed3: name: Check all the features of the heed3 project @@ -130,10 +129,9 @@ jobs: override: true - name: Run cargo test run: | - cd heed cargo clean cp heed3-encryption/Cargo.toml heed/ - cargo check --all-features -p heed3 + cargo check --all-features -p heed3-encryption examples: name: Run the heed examples From 5b15f9df408cfd8b7d2bb3742ea5bac9b9ef3b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Wed, 16 Oct 2024 20:00:48 +0200 Subject: [PATCH 29/51] Change the submodule upstream url --- .gitmodules | 2 +- lmdb-master3-sys/lmdb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 31da67d9..ad5c36af 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,5 +4,5 @@ branch = mdb.master [submodule "lmdb-master3-sys/lmdb"] path = lmdb-master3-sys/lmdb - url = https://github.com/LMDB/lmdb.git + url = https://github.com/LMDB/lmdb branch = mdb.master3 diff --git a/lmdb-master3-sys/lmdb b/lmdb-master3-sys/lmdb index fd3c2ada..1e7891c0 160000 --- a/lmdb-master3-sys/lmdb +++ b/lmdb-master3-sys/lmdb @@ -1 +1 @@ -Subproject commit fd3c2adae70d2ed65017100db45e0b3babfe342a +Subproject commit 1e7891c016a4518eaf0aa29f959b0cc8a22d4111 From b47a7d7013cfa0bab0c25df651df3285dc33fa38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sat, 16 Nov 2024 16:20:09 +0100 Subject: [PATCH 30/51] Create basic encrypted versions of the env and database --- Cargo.toml | 2 +- heed-master3-proc-macro/Cargo.toml | 15 - heed-master3-proc-macro/src/lib.rs | 84 - heed/Cargo.toml | 1 - heed/build.rs | 5 - heed/src/{ => database}/database.rs | 52 +- heed/src/database/encrypted_database.rs | 2249 +++++++++++++++++ heed/src/database/mod.rs | 25 + heed/src/env/clear.rs | 249 -- heed/src/env/encrypted_env.rs | 430 ++++ heed/src/env/env.rs | 478 ++++ .../env/{encrypted.rs => env_open_options.rs} | 554 ++-- heed/src/env/mod.rs | 968 +------ heed/src/lib.rs | 30 +- heed/src/mdb/lmdb_ffi.rs | 2 +- heed/src/txn.rs | 5 +- heed3-encryption/Cargo.toml | 124 - heed3/Cargo.toml | 14 +- 18 files changed, 3495 insertions(+), 1792 deletions(-) delete mode 100644 heed-master3-proc-macro/Cargo.toml delete mode 100644 heed-master3-proc-macro/src/lib.rs rename heed/src/{ => database}/database.rs (97%) create mode 100644 heed/src/database/encrypted_database.rs create mode 100644 heed/src/database/mod.rs delete mode 100644 heed/src/env/clear.rs create mode 100644 heed/src/env/encrypted_env.rs create mode 100644 heed/src/env/env.rs rename heed/src/env/{encrypted.rs => env_open_options.rs} (56%) delete mode 100644 heed3-encryption/Cargo.toml diff --git a/Cargo.toml b/Cargo.toml index 68ccde2b..e2ee0886 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["lmdb-master-sys", "lmdb-master3-sys", "heed", "heed-traits", "heed-types", "heed-master3-proc-macro"] +members = ["lmdb-master-sys", "lmdb-master3-sys", "heed", "heed-traits", "heed-types"] resolver = "2" diff --git a/heed-master3-proc-macro/Cargo.toml b/heed-master3-proc-macro/Cargo.toml deleted file mode 100644 index d41973b3..00000000 --- a/heed-master3-proc-macro/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "heed-master3-proc-macro" -authors = ["Kerollmops "] -description = "A proc macro to simplify the maintenance of the heed and heed3 crates" -license = "MIT" -version = "0.1.0" -edition = "2021" - -[dependencies] -syn = { version = "1.0", features = ["full"] } -quote = "1.0" -proc-macro2 = "1.0" - -[lib] -proc-macro = true diff --git a/heed-master3-proc-macro/src/lib.rs b/heed-master3-proc-macro/src/lib.rs deleted file mode 100644 index 3ccf022b..00000000 --- a/heed-master3-proc-macro/src/lib.rs +++ /dev/null @@ -1,84 +0,0 @@ -use proc_macro::TokenStream; -use quote::quote; -use syn::{parse_macro_input, parse_quote, FnArg, Ident, ItemFn, Pat, PatType, Type}; - -/// By annotating all the heed methods that use a `&RoTxn` this way : -/// -/// ```ignore -/// #[heed_master3_proc_macro::mut_read_txn(rtxn)] -/// fn get<'t>(&self, rtxn: &'t RoTxn, key: &[u8]) -> heed::Result<&'t [u8]>; -/// ``` -/// -/// It transforms the function signature to use a `&mut RoTxn`: -/// ```ignore -/// fn get<'t>(&self, rtxn: &'t mut RoTxn, key: &[u8]) -> heed::Result<&'t [u8]>; -/// ``` -#[proc_macro_attribute] -pub fn mut_read_txn(attr: TokenStream, item: TokenStream) -> TokenStream { - // Parse the function - let mut input_fn = parse_macro_input!(item as ItemFn); - - // Get the parameter name from the attribute - let param_name = parse_macro_input!(attr as Ident); - - // Flag to check if we found and modified the parameter - let mut found_and_modified = false; - - // Iterate through the function arguments - for arg in &mut input_fn.sig.inputs { - if let FnArg::Typed(PatType { pat, ty, .. }) = arg { - // Check if this is the parameter we want to modify - if let Pat::Ident(pat_ident) = pat.as_ref() { - if pat_ident.ident == param_name { - if let Type::Reference(type_reference) = ty.as_mut() { - if let Type::Path(type_path) = type_reference.elem.as_mut() { - if let Some(segment) = type_path.path.segments.last() { - if segment.ident == "RoTxn" { - // Check if it's non-mutable - if type_reference.mutability.is_none() { - // Add the `mut` keyword - type_reference.mutability = Some(parse_quote!(mut)); - found_and_modified = true; - } else { - // If it's already mutable, return an error - return syn::Error::new_spanned( - type_reference, - "The specified parameter is already mutable", - ) - .to_compile_error() - .into(); - } - } else { - // If it's not RoTxn, return an error - return syn::Error::new_spanned( - type_path, - "The specified parameter is not of type RoTxn", - ) - .to_compile_error() - .into(); - } - } - } - } - } - } - } - } - - // If we didn't find and modify the parameter, return an error - if !found_and_modified { - return syn::Error::new_spanned( - input_fn.sig, - format!("Could not find non-mutable parameter '{}' of type RoTxn", param_name), - ) - .to_compile_error() - .into(); - } - - // Generate the modified function - let output = quote! { - #input_fn - }; - - output.into() -} diff --git a/heed/Cargo.toml b/heed/Cargo.toml index fb8eae96..35374010 100644 --- a/heed/Cargo.toml +++ b/heed/Cargo.toml @@ -20,7 +20,6 @@ lmdb-master-sys = { version = "0.2.4", path = "../lmdb-master-sys" } once_cell = "1.19.0" page_size = "0.6.0" serde = { version = "1.0.203", features = ["derive"], optional = true } -synchronoise = "1.0.1" [dev-dependencies] serde = { version = "1.0.203", features = ["derive"] } diff --git a/heed/build.rs b/heed/build.rs index 3c3f804e..d3fdf7dd 100644 --- a/heed/build.rs +++ b/heed/build.rs @@ -3,16 +3,11 @@ use std::env; fn main() { println!("cargo:rerun-if-changed=build.rs"); println!("cargo::rustc-check-cfg=cfg(master3)"); - println!("cargo::rustc-check-cfg=cfg(encryption)"); let pkgname = env::var("CARGO_PKG_NAME").expect("Cargo didn't set the CARGO_PKG_NAME env var!"); match pkgname.as_str() { "heed" => (), "heed3" => println!("cargo:rustc-cfg=master3"), - "heed3-encryption" => { - println!("cargo:rustc-cfg=master3"); - println!("cargo:rustc-cfg=encryption"); - } _ => panic!("unexpected package name!"), } } diff --git a/heed/src/database.rs b/heed/src/database/database.rs similarity index 97% rename from heed/src/database.rs rename to heed/src/database/database.rs index bf794171..5d373cfd 100644 --- a/heed/src/database.rs +++ b/heed/src/database/database.rs @@ -339,7 +339,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn get<'a, 'txn>(&self, txn: &'txn RoTxn, key: &'a KC::EItem) -> Result> where KC: BytesEncode<'a>, @@ -420,7 +420,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_duplicates<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -484,7 +484,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_lower_than<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -554,7 +554,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_lower_than_or_equal_to<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -628,7 +628,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_greater_than<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -701,7 +701,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_greater_than_or_equal_to<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -761,7 +761,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn first<'txn>(&self, txn: &'txn RoTxn) -> Result> where KC: BytesDecode<'txn>, @@ -816,7 +816,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn last<'txn>(&self, txn: &'txn RoTxn) -> Result> where KC: BytesDecode<'txn>, @@ -874,7 +874,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn len(&self, txn: &RoTxn) -> Result { self.stat(txn).map(|stat| stat.entries as u64) } @@ -918,7 +918,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn is_empty(&self, txn: &RoTxn) -> Result { self.len(txn).map(|l| l == 0) } @@ -961,7 +961,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn stat(&self, txn: &RoTxn) -> Result { assert_eq_env_db_txn!(self, txn); @@ -1026,7 +1026,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn iter<'txn>(&self, txn: &'txn RoTxn) -> Result> { assert_eq_env_db_txn!(self, txn); RoCursor::new(txn, self.dbi).map(|cursor| RoIter::new(cursor)) @@ -1128,7 +1128,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn rev_iter<'txn>(&self, txn: &'txn RoTxn) -> Result> { assert_eq_env_db_txn!(self, txn); @@ -1235,7 +1235,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn range<'a, 'txn, R>( &self, txn: &'txn RoTxn, @@ -1408,7 +1408,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn rev_range<'a, 'txn, R>( &self, txn: &'txn RoTxn, @@ -1583,7 +1583,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn prefix_iter<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -1716,7 +1716,7 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(all(master3, encryption), heed_master3_proc_macro::mut_read_txn(txn))] + #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn rev_prefix_iter<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -2643,24 +2643,6 @@ impl fmt::Debug for Database { } } -/// Statistics for a database in the environment. -#[derive(Debug, Clone, Copy)] -pub struct DatabaseStat { - /// Size of a database page. - /// This is currently the same for all databases. - pub page_size: u32, - /// Depth (height) of the B-tree. - pub depth: u32, - /// Number of internal (non-leaf) pages - pub branch_pages: usize, - /// Number of leaf pages. - pub leaf_pages: usize, - /// Number of overflow pages. - pub overflow_pages: usize, - /// Number of data items. - pub entries: usize, -} - #[cfg(test)] mod tests { use byteorder::*; diff --git a/heed/src/database/encrypted_database.rs b/heed/src/database/encrypted_database.rs new file mode 100644 index 00000000..60b75b20 --- /dev/null +++ b/heed/src/database/encrypted_database.rs @@ -0,0 +1,2249 @@ +use std::borrow::Cow; +use std::ops::{Bound, RangeBounds}; +use std::{any, fmt, marker, mem, ptr}; + +use heed_traits::{Comparator, LexicographicComparator}; +use types::{DecodeIgnore, LazyDecode}; + +use crate::cursor::MoveOperation; +use crate::env::DefaultComparator; +use crate::iteration_method::MoveOnCurrentKeyDuplicates; +use crate::mdb::error::mdb_result; +use crate::mdb::ffi; +use crate::mdb::lmdb_flags::{AllDatabaseFlags, DatabaseFlags}; +use crate::*; + +/// Options and flags which can be used to configure how a [`Database`] is opened. +/// +/// # Examples +/// +/// Opening a file to read: +/// +/// ``` +/// # use std::fs; +/// # use std::path::Path; +/// # use heed::EnvOpenOptions; +/// use heed::types::*; +/// use heed::byteorder::BigEndian; +/// +/// # fn main() -> Result<(), Box> { +/// # let dir = tempfile::tempdir()?; +/// # let env = unsafe { EnvOpenOptions::new() +/// # .map_size(10 * 1024 * 1024) // 10MB +/// # .max_dbs(3000) +/// # .open(dir.path())? +/// # }; +/// type BEI64 = I64; +/// +/// // Imagine you have an optional name +/// let conditional_name = Some("big-endian-iter"); +/// +/// let mut wtxn = env.write_txn()?; +/// let mut options = env.database_options().types::(); +/// if let Some(name) = conditional_name { +/// options.name(name); +/// } +/// let db = options.create(&mut wtxn)?; +/// +/// # db.clear(&mut wtxn)?; +/// db.put(&mut wtxn, &68, &())?; +/// db.put(&mut wtxn, &35, &())?; +/// db.put(&mut wtxn, &0, &())?; +/// db.put(&mut wtxn, &42, &())?; +/// +/// wtxn.commit()?; +/// # Ok(()) } +/// ``` +#[derive(Debug)] +pub struct EncryptedDatabaseOpenOptions<'e, 'n, KC, DC, C = DefaultComparator> { + inner: DatabaseOpenOptions<'e, 'n, KC, DC, C>, +} + +impl<'e> EncryptedDatabaseOpenOptions<'e, 'static, Unspecified, Unspecified> { + /// Create an options struct to open/create a database with specific flags. + pub fn new(env: &'e Env) -> Self { + EncryptedDatabaseOpenOptions { inner: DatabaseOpenOptions::new(env) } + } +} + +impl<'e, 'n, KC, DC, C> EncryptedDatabaseOpenOptions<'e, 'n, KC, DC, C> { + /// Change the type of the database. + /// + /// The default types are [`Unspecified`] and require a call to [`Database::remap_types`] + /// to use the [`Database`]. + pub fn types(self) -> EncryptedDatabaseOpenOptions<'e, 'n, NKC, NDC> { + EncryptedDatabaseOpenOptions { inner: self.inner.types() } + } + /// Change the customized key compare function of the database. + /// + /// By default no customized compare function will be set when opening a database. + pub fn key_comparator(self) -> EncryptedDatabaseOpenOptions<'e, 'n, KC, DC, NC> { + EncryptedDatabaseOpenOptions { inner: self.inner.key_comparator() } + } + + /// Change the name of the database. + /// + /// By default the database is unnamed and there only is a single unnamed database. + pub fn name(&mut self, name: &'n str) -> &mut Self { + self.inner.name(name); + self + } + + /// Specify the set of flags used to open the database. + pub fn flags(&mut self, flags: DatabaseFlags) -> &mut Self { + self.inner.flags(flags); + self + } + + /// Opens a typed database that already exists in this environment. + /// + /// If the database was previously opened in this program run, types will be checked. + /// + /// ## Important Information + /// + /// LMDB has an important restriction on the unnamed database when named ones are opened. + /// The names of the named databases are stored as keys in the unnamed one and are immutable, + /// and these keys can only be read and not written. + /// + /// ## LMDB read-only access of existing database + /// + /// In the case of accessing a database in a read-only manner from another process + /// where you wrote, you might need to manually call [`RoTxn::commit`] to get metadata + /// and the database handles opened and shared with the global [`Env`] handle. + /// + /// If not done, you might raise `Io(Os { code: 22, kind: InvalidInput, message: "Invalid argument" })` + /// known as `EINVAL`. + pub fn open(&self, rtxn: &RoTxn) -> Result>> + where + KC: 'static, + DC: 'static, + C: Comparator + 'static, + { + self.inner.open(rtxn) + } + + /// Creates a typed database that can already exist in this environment. + /// + /// If the database was previously opened in this program run, types will be checked. + /// + /// ## Important Information + /// + /// LMDB has an important restriction on the unnamed database when named ones are opened. + /// The names of the named databases are stored as keys in the unnamed one and are immutable, + /// and these keys can only be read and not written. + pub fn create(&self, wtxn: &mut RwTxn) -> Result> + where + KC: 'static, + DC: 'static, + C: Comparator + 'static, + { + self.inner.create(wtxn) + } +} + +impl Clone for EncryptedDatabaseOpenOptions<'_, '_, KC, DC, C> { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for EncryptedDatabaseOpenOptions<'_, '_, KC, DC, C> {} + +/// A typed database that accepts only the types it was created with. +/// +/// # Example: Iterate over databases entries +/// +/// In this example we store numbers in big endian this way those are ordered. +/// Thanks to their bytes representation, heed is able to iterate over them +/// from the lowest to the highest. +/// +/// ``` +/// # use std::fs; +/// # use std::path::Path; +/// # use heed::EnvOpenOptions; +/// use heed::Database; +/// use heed::types::*; +/// use heed::byteorder::BigEndian; +/// +/// # fn main() -> Result<(), Box> { +/// # let dir = tempfile::tempdir()?; +/// # let env = unsafe { EnvOpenOptions::new() +/// # .map_size(10 * 1024 * 1024) // 10MB +/// # .max_dbs(3000) +/// # .open(dir.path())? +/// # }; +/// type BEI64 = I64; +/// +/// let mut wtxn = env.write_txn()?; +/// let db: Database = env.create_database(&mut wtxn, Some("big-endian-iter"))?; +/// +/// # db.clear(&mut wtxn)?; +/// db.put(&mut wtxn, &68, &())?; +/// db.put(&mut wtxn, &35, &())?; +/// db.put(&mut wtxn, &0, &())?; +/// db.put(&mut wtxn, &42, &())?; +/// +/// // you can iterate over database entries in order +/// let rets: Result<_, _> = db.iter(&wtxn)?.collect(); +/// let rets: Vec<(i64, _)> = rets?; +/// +/// let expected = vec![ +/// (0, ()), +/// (35, ()), +/// (42, ()), +/// (68, ()), +/// ]; +/// +/// assert_eq!(rets, expected); +/// wtxn.commit()?; +/// # Ok(()) } +/// ``` +/// +/// # Example: Iterate over and delete ranges of entries +/// +/// Discern also support ranges and ranges deletions. +/// Same configuration as above, numbers are ordered, therefore it is safe to specify +/// a range and be able to iterate over and/or delete it. +/// +/// ``` +/// # use std::fs; +/// # use std::path::Path; +/// # use heed::EnvOpenOptions; +/// use heed::Database; +/// use heed::types::*; +/// use heed::byteorder::BigEndian; +/// +/// # fn main() -> Result<(), Box> { +/// # let dir = tempfile::tempdir()?; +/// # let env = unsafe { EnvOpenOptions::new() +/// # .map_size(10 * 1024 * 1024) // 10MB +/// # .max_dbs(3000) +/// # .open(dir.path())? +/// # }; +/// type BEI64 = I64; +/// +/// let mut wtxn = env.write_txn()?; +/// let db: Database = env.create_database(&mut wtxn, Some("big-endian-iter"))?; +/// +/// # db.clear(&mut wtxn)?; +/// db.put(&mut wtxn, &0, &())?; +/// db.put(&mut wtxn, &68, &())?; +/// db.put(&mut wtxn, &35, &())?; +/// db.put(&mut wtxn, &42, &())?; +/// +/// // you can iterate over ranges too!!! +/// let range = 35..=42; +/// let rets: Result<_, _> = db.range(&wtxn, &range)?.collect(); +/// let rets: Vec<(i64, _)> = rets?; +/// +/// let expected = vec![ +/// (35, ()), +/// (42, ()), +/// ]; +/// +/// assert_eq!(rets, expected); +/// +/// // even delete a range of keys +/// let range = 35..=42; +/// let deleted: usize = db.delete_range(&mut wtxn, &range)?; +/// +/// let rets: Result<_, _> = db.iter(&wtxn)?.collect(); +/// let rets: Vec<(i64, _)> = rets?; +/// +/// let expected = vec![ +/// (0, ()), +/// (68, ()), +/// ]; +/// +/// assert_eq!(deleted, 2); +/// assert_eq!(rets, expected); +/// +/// wtxn.commit()?; +/// # Ok(()) } +/// ``` +pub struct EncryptedDatabase { + inner: Database, +} + +impl EncryptedDatabase { + pub(crate) fn new(env_ident: usize, dbi: ffi::MDB_dbi) -> Database { + EncryptedDatabase { inner: Database::mew(env_ident, dbi) } + } + + /// Retrieves the value associated with a key. + /// + /// If the key does not exist, then `None` is returned. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32= U32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("get-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, "i-am-forty-two", &42)?; + /// db.put(&mut wtxn, "i-am-twenty-seven", &27)?; + /// + /// let ret = db.get(&wtxn, "i-am-forty-two")?; + /// assert_eq!(ret, Some(42)); + /// + /// let ret = db.get(&wtxn, "i-am-twenty-one")?; + /// assert_eq!(ret, None); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn get<'a, 'txn>( + &self, + txn: &'txn mut RoTxn, + key: &'a KC::EItem, + ) -> Result> + where + KC: BytesEncode<'a>, + DC: BytesDecode<'txn>, + { + self.inner.get(txn, key) + } + + /// Returns an iterator over all of the values of a single key. + /// + /// You can make this iterator `Send`able between threads by + /// using the `read-txn-no-tls` crate feature. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::{DatabaseFlags, EnvOpenOptions}; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI64 = I64; + /// + /// let mut wtxn = env.write_txn()?; + /// let db = env.database_options() + /// .types::() + /// .flags(DatabaseFlags::DUP_SORT) + /// .name("dup-sort") + /// .create(&mut wtxn)?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &68, &120)?; + /// db.put(&mut wtxn, &68, &121)?; + /// db.put(&mut wtxn, &68, &122)?; + /// db.put(&mut wtxn, &68, &123)?; + /// db.put(&mut wtxn, &92, &32)?; + /// db.put(&mut wtxn, &35, &120)?; + /// db.put(&mut wtxn, &0, &120)?; + /// db.put(&mut wtxn, &42, &120)?; + /// + /// let mut iter = db.get_duplicates(&wtxn, &68)?.expect("the key exists"); + /// assert_eq!(iter.next().transpose()?, Some((68, 120))); + /// assert_eq!(iter.next().transpose()?, Some((68, 121))); + /// assert_eq!(iter.next().transpose()?, Some((68, 122))); + /// assert_eq!(iter.next().transpose()?, Some((68, 123))); + /// assert_eq!(iter.next().transpose()?, None); + /// drop(iter); + /// + /// let mut iter = db.get_duplicates(&wtxn, &68)?.expect("the key exists"); + /// assert_eq!(iter.last().transpose()?, Some((68, 123))); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn get_duplicates<'a, 'txn>( + &self, + txn: &'txn mut RoTxn, + key: &'a KC::EItem, + ) -> Result>> + where + KC: BytesEncode<'a>, + { + self.inner.get_duplicates(txn, key) + } + + /// Retrieves the key/value pair lower than the given one in this database. + /// + /// If the database if empty or there is no key lower than the given one, + /// then `None` is returned. + /// + /// Comparisons are made by using the bytes representation of the key. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEU32 = U32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db = env.create_database::(&mut wtxn, Some("get-lt-u32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &27, &())?; + /// db.put(&mut wtxn, &42, &())?; + /// db.put(&mut wtxn, &43, &())?; + /// + /// let ret = db.get_lower_than(&wtxn, &4404)?; + /// assert_eq!(ret, Some((43, ()))); + /// + /// let ret = db.get_lower_than(&wtxn, &43)?; + /// assert_eq!(ret, Some((42, ()))); + /// + /// let ret = db.get_lower_than(&wtxn, &27)?; + /// assert_eq!(ret, None); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn get_lower_than<'a, 'txn>( + &self, + txn: &'txn mut RoTxn, + key: &'a KC::EItem, + ) -> Result> + where + KC: BytesEncode<'a> + BytesDecode<'txn>, + DC: BytesDecode<'txn>, + { + self.inner.get_lower_than(txn, key) + } + + /// Retrieves the key/value pair lower than or equal to the given one in this database. + /// + /// If the database if empty or there is no key lower than or equal to the given one, + /// then `None` is returned. + /// + /// Comparisons are made by using the bytes representation of the key. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEU32 = U32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db = env.create_database::(&mut wtxn, Some("get-lt-u32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &27, &())?; + /// db.put(&mut wtxn, &42, &())?; + /// db.put(&mut wtxn, &43, &())?; + /// + /// let ret = db.get_lower_than_or_equal_to(&wtxn, &4404)?; + /// assert_eq!(ret, Some((43, ()))); + /// + /// let ret = db.get_lower_than_or_equal_to(&wtxn, &43)?; + /// assert_eq!(ret, Some((43, ()))); + /// + /// let ret = db.get_lower_than_or_equal_to(&wtxn, &26)?; + /// assert_eq!(ret, None); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn get_lower_than_or_equal_to<'a, 'txn>( + &self, + txn: &'txn mut RoTxn, + key: &'a KC::EItem, + ) -> Result> + where + KC: BytesEncode<'a> + BytesDecode<'txn>, + DC: BytesDecode<'txn>, + { + self.inner.get_lower_than_or_equal_to(txn, key) + } + + /// Retrieves the key/value pair greater than the given one in this database. + /// + /// If the database if empty or there is no key greater than the given one, + /// then `None` is returned. + /// + /// Comparisons are made by using the bytes representation of the key. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEU32 = U32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db = env.create_database::(&mut wtxn, Some("get-lt-u32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &27, &())?; + /// db.put(&mut wtxn, &42, &())?; + /// db.put(&mut wtxn, &43, &())?; + /// + /// let ret = db.get_greater_than(&wtxn, &0)?; + /// assert_eq!(ret, Some((27, ()))); + /// + /// let ret = db.get_greater_than(&wtxn, &42)?; + /// assert_eq!(ret, Some((43, ()))); + /// + /// let ret = db.get_greater_than(&wtxn, &43)?; + /// assert_eq!(ret, None); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn get_greater_than<'a, 'txn>( + &self, + txn: &'txn mut RoTxn, + key: &'a KC::EItem, + ) -> Result> + where + KC: BytesEncode<'a> + BytesDecode<'txn>, + DC: BytesDecode<'txn>, + { + self.inner.get_greater_than(txn, key) + } + + /// Retrieves the key/value pair greater than or equal to the given one in this database. + /// + /// If the database if empty or there is no key greater than or equal to the given one, + /// then `None` is returned. + /// + /// Comparisons are made by using the bytes representation of the key. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEU32 = U32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db = env.create_database::(&mut wtxn, Some("get-lt-u32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &27, &())?; + /// db.put(&mut wtxn, &42, &())?; + /// db.put(&mut wtxn, &43, &())?; + /// + /// let ret = db.get_greater_than_or_equal_to(&wtxn, &0)?; + /// assert_eq!(ret, Some((27, ()))); + /// + /// let ret = db.get_greater_than_or_equal_to(&wtxn, &42)?; + /// assert_eq!(ret, Some((42, ()))); + /// + /// let ret = db.get_greater_than_or_equal_to(&wtxn, &44)?; + /// assert_eq!(ret, None); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn get_greater_than_or_equal_to<'a, 'txn>( + &self, + txn: &'txn mut RoTxn, + key: &'a KC::EItem, + ) -> Result> + where + KC: BytesEncode<'a> + BytesDecode<'txn>, + DC: BytesDecode<'txn>, + { + self.inner.get_greater_than_or_equal_to(txn, key) + } + + /// Retrieves the first key/value pair of this database. + /// + /// If the database if empty, then `None` is returned. + /// + /// Comparisons are made by using the bytes representation of the key. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("first-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// + /// let ret = db.first(&wtxn)?; + /// assert_eq!(ret, Some((27, "i-am-twenty-seven"))); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn first<'txn>(&self, txn: &'txn mut RoTxn) -> Result> + where + KC: BytesDecode<'txn>, + DC: BytesDecode<'txn>, + { + self.inner.first(txn) + } + + /// Retrieves the last key/value pair of this database. + /// + /// If the database if empty, then `None` is returned. + /// + /// Comparisons are made by using the bytes representation of the key. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("last-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// + /// let ret = db.last(&wtxn)?; + /// assert_eq!(ret, Some((42, "i-am-forty-two"))); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn last<'txn>(&self, txn: &'txn mut RoTxn) -> Result> + where + KC: BytesDecode<'txn>, + DC: BytesDecode<'txn>, + { + self.inner.last(txn) + } + + /// Returns the number of elements in this database. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// db.put(&mut wtxn, &521, "i-am-five-hundred-and-twenty-one")?; + /// + /// let ret = db.len(&wtxn)?; + /// assert_eq!(ret, 4); + /// + /// db.delete(&mut wtxn, &27)?; + /// + /// let ret = db.len(&wtxn)?; + /// assert_eq!(ret, 3); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn len(&self, txn: &RoTxn) -> Result { + self.inner.len(txn) + } + + /// Returns `true` if and only if this database is empty. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// db.put(&mut wtxn, &521, "i-am-five-hundred-and-twenty-one")?; + /// + /// let ret = db.is_empty(&wtxn)?; + /// assert_eq!(ret, false); + /// + /// db.clear(&mut wtxn)?; + /// + /// let ret = db.is_empty(&wtxn)?; + /// assert_eq!(ret, true); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn is_empty(&self, txn: &RoTxn) -> Result { + self.inner.is_empty(txn) + } + + /// Returns some statistics for this database. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// db.put(&mut wtxn, &521, "i-am-five-hundred-and-twenty-one")?; + /// + /// let stat = db.stat(&wtxn)?; + /// assert_eq!(stat.depth, 1); + /// assert_eq!(stat.branch_pages, 0); + /// assert_eq!(stat.leaf_pages, 1); + /// assert_eq!(stat.overflow_pages, 0); + /// assert_eq!(stat.entries, 4); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn stat(&self, txn: &RoTxn) -> Result { + self.inner.stat(txn) + } + + /// Return a lexicographically ordered iterator of all key-value pairs in this database. + /// + /// You can make this iterator `Send`able between threads by + /// using the `read-txn-no-tls` crate feature. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// + /// let mut iter = db.iter(&wtxn)?; + /// assert_eq!(iter.next().transpose()?, Some((13, "i-am-thirteen"))); + /// assert_eq!(iter.next().transpose()?, Some((27, "i-am-twenty-seven"))); + /// assert_eq!(iter.next().transpose()?, Some((42, "i-am-forty-two"))); + /// assert_eq!(iter.next().transpose()?, None); + /// + /// drop(iter); + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn iter<'txn>(&self, txn: &'txn mut RoTxn) -> Result> { + self.inner.iter(txn) + } + + /// Return a mutable lexicographically ordered iterator of all key-value pairs in this database. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// + /// let mut iter = db.iter_mut(&mut wtxn)?; + /// assert_eq!(iter.next().transpose()?, Some((13, "i-am-thirteen"))); + /// let ret = unsafe { iter.del_current()? }; + /// assert!(ret); + /// + /// assert_eq!(iter.next().transpose()?, Some((27, "i-am-twenty-seven"))); + /// assert_eq!(iter.next().transpose()?, Some((42, "i-am-forty-two"))); + /// let ret = unsafe { iter.put_current(&42, "i-am-the-new-forty-two")? }; + /// assert!(ret); + /// + /// assert_eq!(iter.next().transpose()?, None); + /// + /// drop(iter); + /// + /// let ret = db.get(&wtxn, &13)?; + /// assert_eq!(ret, None); + /// + /// let ret = db.get(&wtxn, &42)?; + /// assert_eq!(ret, Some("i-am-the-new-forty-two")); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn iter_mut<'txn>(&self, txn: &'txn mut RwTxn) -> Result> { + self.inner.iter_mut(txn) + } + + /// Return a reversed lexicographically ordered iterator of all key-value pairs in this database. + /// + /// You can make this iterator `Send`able between threads by + /// using the `read-txn-no-tls` crate feature. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// + /// let mut iter = db.rev_iter(&wtxn)?; + /// assert_eq!(iter.next().transpose()?, Some((42, "i-am-forty-two"))); + /// assert_eq!(iter.next().transpose()?, Some((27, "i-am-twenty-seven"))); + /// assert_eq!(iter.next().transpose()?, Some((13, "i-am-thirteen"))); + /// assert_eq!(iter.next().transpose()?, None); + /// + /// drop(iter); + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn rev_iter<'txn>(&self, txn: &'txn mut RoTxn) -> Result> { + self.inner.rev_iter(txn) + } + + /// Return a mutable reversed lexicographically ordered iterator of all key-value\ + /// pairs in this database. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// + /// let mut iter = db.rev_iter_mut(&mut wtxn)?; + /// assert_eq!(iter.next().transpose()?, Some((42, "i-am-forty-two"))); + /// let ret = unsafe { iter.del_current()? }; + /// assert!(ret); + /// + /// assert_eq!(iter.next().transpose()?, Some((27, "i-am-twenty-seven"))); + /// assert_eq!(iter.next().transpose()?, Some((13, "i-am-thirteen"))); + /// let ret = unsafe { iter.put_current(&13, "i-am-the-new-thirteen")? }; + /// assert!(ret); + /// + /// assert_eq!(iter.next().transpose()?, None); + /// + /// drop(iter); + /// + /// let ret = db.get(&wtxn, &42)?; + /// assert_eq!(ret, None); + /// + /// let ret = db.get(&wtxn, &13)?; + /// assert_eq!(ret, Some("i-am-the-new-thirteen")); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn rev_iter_mut<'txn>(&self, txn: &'txn mut RwTxn) -> Result> { + self.inner.rev_iter_mut(txn) + } + + /// Return a lexicographically ordered iterator of a range of key-value pairs in this database. + /// + /// Comparisons are made by using the bytes representation of the key. + /// + /// You can make this iterator `Send`able between threads by + /// using the `read-txn-no-tls` crate feature. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// db.put(&mut wtxn, &521, "i-am-five-hundred-and-twenty-one")?; + /// + /// let range = 27..=42; + /// let mut iter = db.range(&wtxn, &range)?; + /// assert_eq!(iter.next().transpose()?, Some((27, "i-am-twenty-seven"))); + /// assert_eq!(iter.next().transpose()?, Some((42, "i-am-forty-two"))); + /// assert_eq!(iter.next().transpose()?, None); + /// + /// drop(iter); + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn range<'a, 'txn, R>( + &self, + txn: &'txn mut RoTxn, + range: &'a R, + ) -> Result> + where + KC: BytesEncode<'a>, + R: RangeBounds, + { + self.inner.range(txn, range) + } + + /// Return a mutable lexicographically ordered iterator of a range of + /// key-value pairs in this database. + /// + /// Comparisons are made by using the bytes representation of the key. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// db.put(&mut wtxn, &521, "i-am-five-hundred-and-twenty-one")?; + /// + /// let range = 27..=42; + /// let mut range = db.range_mut(&mut wtxn, &range)?; + /// assert_eq!(range.next().transpose()?, Some((27, "i-am-twenty-seven"))); + /// let ret = unsafe { range.del_current()? }; + /// assert!(ret); + /// assert_eq!(range.next().transpose()?, Some((42, "i-am-forty-two"))); + /// let ret = unsafe { range.put_current(&42, "i-am-the-new-forty-two")? }; + /// assert!(ret); + /// + /// assert_eq!(range.next().transpose()?, None); + /// drop(range); + /// + /// + /// let mut iter = db.iter(&wtxn)?; + /// assert_eq!(iter.next().transpose()?, Some((13, "i-am-thirteen"))); + /// assert_eq!(iter.next().transpose()?, Some((42, "i-am-the-new-forty-two"))); + /// assert_eq!(iter.next().transpose()?, Some((521, "i-am-five-hundred-and-twenty-one"))); + /// assert_eq!(iter.next().transpose()?, None); + /// + /// drop(iter); + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn range_mut<'a, 'txn, R>( + &self, + txn: &'txn mut RwTxn, + range: &'a R, + ) -> Result> + where + KC: BytesEncode<'a>, + R: RangeBounds, + { + self.inner.range_mut(txn, range) + } + + /// Return a reversed lexicographically ordered iterator of a range of key-value + /// pairs in this database. + /// + /// Comparisons are made by using the bytes representation of the key. + /// + /// You can make this iterator `Send`able between threads by + /// using the `read-txn-no-tls` crate feature. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// db.put(&mut wtxn, &521, "i-am-five-hundred-and-twenty-one")?; + /// + /// let range = 27..=43; + /// let mut iter = db.rev_range(&wtxn, &range)?; + /// assert_eq!(iter.next().transpose()?, Some((42, "i-am-forty-two"))); + /// assert_eq!(iter.next().transpose()?, Some((27, "i-am-twenty-seven"))); + /// assert_eq!(iter.next().transpose()?, None); + /// + /// drop(iter); + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn rev_range<'a, 'txn, R>( + &self, + txn: &'txn mut RoTxn, + range: &'a R, + ) -> Result> + where + KC: BytesEncode<'a>, + R: RangeBounds, + { + self.inner.rev_range(txn, range) + } + + /// Return a mutable reversed lexicographically ordered iterator of a range of + /// key-value pairs in this database. + /// + /// Comparisons are made by using the bytes representation of the key. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// db.put(&mut wtxn, &521, "i-am-five-hundred-and-twenty-one")?; + /// + /// let range = 27..=42; + /// let mut range = db.rev_range_mut(&mut wtxn, &range)?; + /// assert_eq!(range.next().transpose()?, Some((42, "i-am-forty-two"))); + /// let ret = unsafe { range.del_current()? }; + /// assert!(ret); + /// assert_eq!(range.next().transpose()?, Some((27, "i-am-twenty-seven"))); + /// let ret = unsafe { range.put_current(&27, "i-am-the-new-twenty-seven")? }; + /// assert!(ret); + /// + /// assert_eq!(range.next().transpose()?, None); + /// drop(range); + /// + /// + /// let mut iter = db.iter(&wtxn)?; + /// assert_eq!(iter.next().transpose()?, Some((13, "i-am-thirteen"))); + /// assert_eq!(iter.next().transpose()?, Some((27, "i-am-the-new-twenty-seven"))); + /// assert_eq!(iter.next().transpose()?, Some((521, "i-am-five-hundred-and-twenty-one"))); + /// assert_eq!(iter.next().transpose()?, None); + /// + /// drop(iter); + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn rev_range_mut<'a, 'txn, R>( + &self, + txn: &'txn mut RwTxn, + range: &'a R, + ) -> Result> + where + KC: BytesEncode<'a>, + R: RangeBounds, + { + self.inner.rev_range_mut(txn, range) + } + + /// Return a lexicographically ordered iterator of all key-value pairs + /// in this database that starts with the given prefix. + /// + /// Comparisons are made by using the bytes representation of the key. + /// + /// You can make this iterator `Send`able between threads by + /// using the `read-txn-no-tls` crate feature. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, "i-am-twenty-eight", &28)?; + /// db.put(&mut wtxn, "i-am-twenty-seven", &27)?; + /// db.put(&mut wtxn, "i-am-twenty-nine", &29)?; + /// db.put(&mut wtxn, "i-am-forty-one", &41)?; + /// db.put(&mut wtxn, "i-am-forty-two", &42)?; + /// + /// let mut iter = db.prefix_iter(&mut wtxn, "i-am-twenty")?; + /// assert_eq!(iter.next().transpose()?, Some(("i-am-twenty-eight", 28))); + /// assert_eq!(iter.next().transpose()?, Some(("i-am-twenty-nine", 29))); + /// assert_eq!(iter.next().transpose()?, Some(("i-am-twenty-seven", 27))); + /// assert_eq!(iter.next().transpose()?, None); + /// + /// drop(iter); + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn prefix_iter<'a, 'txn>( + &self, + txn: &'txn mut RoTxn, + prefix: &'a KC::EItem, + ) -> Result> + where + KC: BytesEncode<'a>, + C: LexicographicComparator, + { + self.inner.prefix_iter(txn, prefix) + } + + /// Return a mutable lexicographically ordered iterator of all key-value pairs + /// in this database that starts with the given prefix. + /// + /// Comparisons are made by using the bytes representation of the key. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, "i-am-twenty-eight", &28)?; + /// db.put(&mut wtxn, "i-am-twenty-seven", &27)?; + /// db.put(&mut wtxn, "i-am-twenty-nine", &29)?; + /// db.put(&mut wtxn, "i-am-forty-one", &41)?; + /// db.put(&mut wtxn, "i-am-forty-two", &42)?; + /// + /// let mut iter = db.prefix_iter_mut(&mut wtxn, "i-am-twenty")?; + /// assert_eq!(iter.next().transpose()?, Some(("i-am-twenty-eight", 28))); + /// let ret = unsafe { iter.del_current()? }; + /// assert!(ret); + /// + /// assert_eq!(iter.next().transpose()?, Some(("i-am-twenty-nine", 29))); + /// assert_eq!(iter.next().transpose()?, Some(("i-am-twenty-seven", 27))); + /// let ret = unsafe { iter.put_current("i-am-twenty-seven", &27000)? }; + /// assert!(ret); + /// + /// assert_eq!(iter.next().transpose()?, None); + /// + /// drop(iter); + /// + /// let ret = db.get(&wtxn, "i-am-twenty-eight")?; + /// assert_eq!(ret, None); + /// + /// let ret = db.get(&wtxn, "i-am-twenty-seven")?; + /// assert_eq!(ret, Some(27000)); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn prefix_iter_mut<'a, 'txn>( + &self, + txn: &'txn mut RwTxn, + prefix: &'a KC::EItem, + ) -> Result> + where + KC: BytesEncode<'a>, + C: LexicographicComparator, + { + self.inner.prefix_iter_mut(txn, prefix) + } + + /// Return a reversed lexicographically ordered iterator of all key-value pairs + /// in this database that starts with the given prefix. + /// + /// Comparisons are made by using the bytes representation of the key. + /// + /// You can make this iterator `Send`able between threads by + /// using the `read-txn-no-tls` crate feature. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, "i-am-twenty-eight", &28)?; + /// db.put(&mut wtxn, "i-am-twenty-seven", &27)?; + /// db.put(&mut wtxn, "i-am-twenty-nine", &29)?; + /// db.put(&mut wtxn, "i-am-forty-one", &41)?; + /// db.put(&mut wtxn, "i-am-forty-two", &42)?; + /// + /// let mut iter = db.rev_prefix_iter(&mut wtxn, "i-am-twenty")?; + /// assert_eq!(iter.next().transpose()?, Some(("i-am-twenty-seven", 27))); + /// assert_eq!(iter.next().transpose()?, Some(("i-am-twenty-nine", 29))); + /// assert_eq!(iter.next().transpose()?, Some(("i-am-twenty-eight", 28))); + /// assert_eq!(iter.next().transpose()?, None); + /// + /// drop(iter); + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn rev_prefix_iter<'a, 'txn>( + &self, + txn: &'txn mut RoTxn, + prefix: &'a KC::EItem, + ) -> Result> + where + KC: BytesEncode<'a>, + C: LexicographicComparator, + { + self.inner.rev_prefix_iter(txn, prefix) + } + + /// Return a mutable reversed lexicographically ordered iterator of all key-value pairs + /// in this database that starts with the given prefix. + /// + /// Comparisons are made by using the bytes representation of the key. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, "i-am-twenty-eight", &28)?; + /// db.put(&mut wtxn, "i-am-twenty-seven", &27)?; + /// db.put(&mut wtxn, "i-am-twenty-nine", &29)?; + /// db.put(&mut wtxn, "i-am-forty-one", &41)?; + /// db.put(&mut wtxn, "i-am-forty-two", &42)?; + /// + /// let mut iter = db.rev_prefix_iter_mut(&mut wtxn, "i-am-twenty")?; + /// assert_eq!(iter.next().transpose()?, Some(("i-am-twenty-seven", 27))); + /// let ret = unsafe { iter.del_current()? }; + /// assert!(ret); + /// + /// assert_eq!(iter.next().transpose()?, Some(("i-am-twenty-nine", 29))); + /// assert_eq!(iter.next().transpose()?, Some(("i-am-twenty-eight", 28))); + /// let ret = unsafe { iter.put_current("i-am-twenty-eight", &28000)? }; + /// assert!(ret); + /// + /// assert_eq!(iter.next().transpose()?, None); + /// + /// drop(iter); + /// + /// let ret = db.get(&wtxn, "i-am-twenty-seven")?; + /// assert_eq!(ret, None); + /// + /// let ret = db.get(&wtxn, "i-am-twenty-eight")?; + /// assert_eq!(ret, Some(28000)); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn rev_prefix_iter_mut<'a, 'txn>( + &self, + txn: &'txn mut RwTxn, + prefix: &'a KC::EItem, + ) -> Result> + where + KC: BytesEncode<'a>, + C: LexicographicComparator, + { + self.inner.rev_prefix_iter_mut(txn, prefix) + } + + /// Insert a key-value pair in this database, replacing any previous value. The entry is + /// written with no specific flag. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// db.put(&mut wtxn, &521, "i-am-five-hundred-and-twenty-one")?; + /// + /// let ret = db.get(&mut wtxn, &27)?; + /// assert_eq!(ret, Some("i-am-twenty-seven")); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn put<'a>(&self, txn: &mut RwTxn, key: &'a KC::EItem, data: &'a DC::EItem) -> Result<()> + where + KC: BytesEncode<'a>, + DC: BytesEncode<'a>, + { + self.inner.put(txn, key, data) + } + + /// Insert a key-value pair where the value can directly be written to disk, replacing any + /// previous value. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use std::io::Write; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db = env.create_database::(&mut wtxn, Some("number-string"))?; + /// + /// # db.clear(&mut wtxn)?; + /// let value = "I am a long long long value"; + /// db.put_reserved(&mut wtxn, &42, value.len(), |reserved| { + /// reserved.write_all(value.as_bytes()) + /// })?; + /// + /// let ret = db.get(&mut wtxn, &42)?; + /// assert_eq!(ret, Some(value)); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn put_reserved<'a, F>( + &self, + txn: &mut RwTxn, + key: &'a KC::EItem, + data_size: usize, + write_func: F, + ) -> Result<()> + where + KC: BytesEncode<'a>, + F: FnOnce(&mut ReservedSpace) -> io::Result<()>, + { + self.inner.put_reserved(txn, key, data_size, write_func) + } + + /// Insert a key-value pair in this database, replacing any previous value. The entry is + /// written with the specified flags. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::{Database, PutFlags, DatabaseFlags, Error, MdbError}; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db = env.database_options() + /// .types::() + /// .name("dup-i32") + /// .flags(DatabaseFlags::DUP_SORT) + /// .create(&mut wtxn)?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &42, "i-am-so-cool")?; + /// db.put(&mut wtxn, &42, "i-am-the-king")?; + /// db.put(&mut wtxn, &42, "i-am-fun")?; + /// db.put_with_flags(&mut wtxn, PutFlags::APPEND, &54, "i-am-older-than-you")?; + /// db.put_with_flags(&mut wtxn, PutFlags::APPEND_DUP, &54, "ok-but-i-am-better-than-you")?; + /// // You can compose flags by OR'ing them + /// db.put_with_flags(&mut wtxn, PutFlags::APPEND_DUP | PutFlags::NO_OVERWRITE, &55, "welcome")?; + /// + /// // The NO_DUP_DATA flag will return KeyExist if we try to insert the exact same key/value pair. + /// let ret = db.put_with_flags(&mut wtxn, PutFlags::NO_DUP_DATA, &54, "ok-but-i-am-better-than-you"); + /// assert!(matches!(ret, Err(Error::Mdb(MdbError::KeyExist)))); + /// + /// // The NO_OVERWRITE flag will return KeyExist if we try to insert something with an already existing key. + /// let ret = db.put_with_flags(&mut wtxn, PutFlags::NO_OVERWRITE, &54, "there-can-be-only-one-data"); + /// assert!(matches!(ret, Err(Error::Mdb(MdbError::KeyExist)))); + /// + /// let mut iter = db.iter(&wtxn)?; + /// assert_eq!(iter.next().transpose()?, Some((42, "i-am-forty-two"))); + /// assert_eq!(iter.next().transpose()?, Some((42, "i-am-fun"))); + /// assert_eq!(iter.next().transpose()?, Some((42, "i-am-so-cool"))); + /// assert_eq!(iter.next().transpose()?, Some((42, "i-am-the-king"))); + /// assert_eq!(iter.next().transpose()?, Some((54, "i-am-older-than-you"))); + /// assert_eq!(iter.next().transpose()?, Some((54, "ok-but-i-am-better-than-you"))); + /// assert_eq!(iter.next().transpose()?, Some((55, "welcome"))); + /// assert_eq!(iter.next().transpose()?, None); + /// + /// drop(iter); + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn put_with_flags<'a>( + &self, + txn: &mut RwTxn, + flags: PutFlags, + key: &'a KC::EItem, + data: &'a DC::EItem, + ) -> Result<()> + where + KC: BytesEncode<'a>, + DC: BytesEncode<'a>, + { + self.inner.put_with_flags(txn, flags, key, data) + } + + /// Attempt to insert a key-value pair in this database, or if a value already exists for the + /// key, returns the previous value. + /// + /// The entry is always written with the [`NO_OVERWRITE`](PutFlags::NO_OVERWRITE) flag. + /// + /// ``` + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// assert_eq!(db.get_or_put(&mut wtxn, &42, "i-am-forty-two")?, None); + /// assert_eq!(db.get_or_put(&mut wtxn, &42, "the meaning of life")?, Some("i-am-forty-two")); + /// + /// let ret = db.get(&mut wtxn, &42)?; + /// assert_eq!(ret, Some("i-am-forty-two")); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn get_or_put<'a, 'txn>( + &'txn self, + txn: &mut RwTxn, + key: &'a KC::EItem, + data: &'a DC::EItem, + ) -> Result> + where + KC: BytesEncode<'a>, + DC: BytesEncode<'a> + BytesDecode<'a>, + { + self.inner.get_or_put(txn, key, data) + } + + /// Attempt to insert a key-value pair in this database, or if a value already exists for the + /// key, returns the previous value. + /// + /// The entry is written with the specified flags, in addition to + /// [`NO_OVERWRITE`](PutFlags::NO_OVERWRITE) which is always used. + /// + /// ``` + /// # use heed::EnvOpenOptions; + /// use heed::{Database, PutFlags}; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// assert_eq!(db.get_or_put_with_flags(&mut wtxn, PutFlags::empty(), &42, "i-am-forty-two")?, None); + /// assert_eq!(db.get_or_put_with_flags(&mut wtxn, PutFlags::empty(), &42, "the meaning of life")?, Some("i-am-forty-two")); + /// + /// let ret = db.get(&mut wtxn, &42)?; + /// assert_eq!(ret, Some("i-am-forty-two")); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn get_or_put_with_flags<'a, 'txn>( + &'txn self, + txn: &mut RwTxn, + flags: PutFlags, + key: &'a KC::EItem, + data: &'a DC::EItem, + ) -> Result> + where + KC: BytesEncode<'a>, + DC: BytesEncode<'a> + BytesDecode<'a>, + { + self.inner.get_or_put_with_flags(txn, flags, key, data) + } + + /// Attempt to insert a key-value pair in this database, where the value can be directly + /// written to disk, or if a value already exists for the key, returns the previous value. + /// + /// The entry is always written with the [`NO_OVERWRITE`](PutFlags::NO_OVERWRITE) and + /// [`MDB_RESERVE`](ffi::MDB_RESERVE) flags. + /// + /// ``` + /// # use heed::EnvOpenOptions; + /// use std::io::Write; + /// use heed::{Database, PutFlags}; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db = env.create_database::(&mut wtxn, Some("number-string"))?; + /// + /// # db.clear(&mut wtxn)?; + /// let long = "I am a long long long value"; + /// assert_eq!( + /// db.get_or_put_reserved(&mut wtxn, &42, long.len(), |reserved| { + /// reserved.write_all(long.as_bytes()) + /// })?, + /// None + /// ); + /// + /// let longer = "I am an even longer long long long value"; + /// assert_eq!( + /// db.get_or_put_reserved(&mut wtxn, &42, longer.len(), |reserved| { + /// unreachable!() + /// })?, + /// Some(long) + /// ); + /// + /// let ret = db.get(&mut wtxn, &42)?; + /// assert_eq!(ret, Some(long)); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn get_or_put_reserved<'a, 'txn, F>( + &'txn self, + txn: &mut RwTxn, + key: &'a KC::EItem, + data_size: usize, + write_func: F, + ) -> Result> + where + KC: BytesEncode<'a>, + F: FnOnce(&mut ReservedSpace) -> io::Result<()>, + DC: BytesDecode<'a>, + { + self.inner.get_or_put_reserved(txn, key, data_size, write_func) + } + + /// Attempt to insert a key-value pair in this database, where the value can be directly + /// written to disk, or if a value already exists for the key, returns the previous value. + /// + /// The entry is written with the specified flags, in addition to + /// [`NO_OVERWRITE`](PutFlags::NO_OVERWRITE) and [`MDB_RESERVE`](ffi::MDB_RESERVE) + /// which are always used. + /// + /// ``` + /// # use heed::EnvOpenOptions; + /// use std::io::Write; + /// use heed::{Database, PutFlags}; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db = env.create_database::(&mut wtxn, Some("number-string"))?; + /// + /// # db.clear(&mut wtxn)?; + /// let long = "I am a long long long value"; + /// assert_eq!( + /// db.get_or_put_reserved_with_flags(&mut wtxn, PutFlags::empty(), &42, long.len(), |reserved| { + /// reserved.write_all(long.as_bytes()) + /// })?, + /// None + /// ); + /// + /// let longer = "I am an even longer long long long value"; + /// assert_eq!( + /// db.get_or_put_reserved_with_flags(&mut wtxn, PutFlags::empty(), &42, longer.len(), |reserved| { + /// unreachable!() + /// })?, + /// Some(long) + /// ); + /// + /// let ret = db.get(&mut wtxn, &42)?; + /// assert_eq!(ret, Some(long)); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn get_or_put_reserved_with_flags<'a, 'txn, F>( + &'txn self, + txn: &mut RwTxn, + flags: PutFlags, + key: &'a KC::EItem, + data_size: usize, + write_func: F, + ) -> Result> + where + KC: BytesEncode<'a>, + F: FnOnce(&mut ReservedSpace) -> io::Result<()>, + DC: BytesDecode<'a>, + { + self.inner.get_or_put_reserved_with_flags(txn, flags, key, data_size, write_func) + } + + /// Deletes an entry or every duplicate data items of a key + /// if the database supports duplicate data items. + /// + /// If the entry does not exist, then `false` is returned. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// db.put(&mut wtxn, &521, "i-am-five-hundred-and-twenty-one")?; + /// + /// let ret = db.delete(&mut wtxn, &27)?; + /// assert_eq!(ret, true); + /// + /// let ret = db.get(&mut wtxn, &27)?; + /// assert_eq!(ret, None); + /// + /// let ret = db.delete(&mut wtxn, &467)?; + /// assert_eq!(ret, false); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn delete<'a>(&self, txn: &mut RwTxn, key: &'a KC::EItem) -> Result + where + KC: BytesEncode<'a>, + { + self.inner.delete(txn, key) + } + + /// Deletes a single key-value pair in this database. + /// + /// If the database doesn't support duplicate data items the data is ignored. + /// If the key does not exist, then `false` is returned. + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::{DatabaseFlags, EnvOpenOptions}; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI64 = I64; + /// + /// let mut wtxn = env.write_txn()?; + /// let db = env.database_options() + /// .types::() + /// .flags(DatabaseFlags::DUP_SORT) + /// .name("dup-sort") + /// .create(&mut wtxn)?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &68, &120)?; + /// db.put(&mut wtxn, &68, &121)?; + /// db.put(&mut wtxn, &68, &122)?; + /// db.put(&mut wtxn, &68, &123)?; + /// db.put(&mut wtxn, &92, &32)?; + /// db.put(&mut wtxn, &35, &120)?; + /// db.put(&mut wtxn, &0, &120)?; + /// db.put(&mut wtxn, &42, &120)?; + /// + /// let mut iter = db.get_duplicates(&wtxn, &68)?.expect("the key exists"); + /// assert_eq!(iter.next().transpose()?, Some((68, 120))); + /// assert_eq!(iter.next().transpose()?, Some((68, 121))); + /// assert_eq!(iter.next().transpose()?, Some((68, 122))); + /// assert_eq!(iter.next().transpose()?, Some((68, 123))); + /// assert_eq!(iter.next().transpose()?, None); + /// drop(iter); + /// + /// assert!(db.delete_one_duplicate(&mut wtxn, &68, &121)?, "The entry must exist"); + /// + /// let mut iter = db.get_duplicates(&wtxn, &68)?.expect("the key exists"); + /// assert_eq!(iter.next().transpose()?, Some((68, 120))); + /// // No more (68, 121) returned here! + /// assert_eq!(iter.next().transpose()?, Some((68, 122))); + /// assert_eq!(iter.next().transpose()?, Some((68, 123))); + /// assert_eq!(iter.next().transpose()?, None); + /// drop(iter); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn delete_one_duplicate<'a>( + &self, + txn: &mut RwTxn, + key: &'a KC::EItem, + data: &'a DC::EItem, + ) -> Result + where + KC: BytesEncode<'a>, + DC: BytesEncode<'a>, + { + self.inner.delete_one_duplicate(txn, key, data) + } + + /// Deletes a range of key-value pairs in this database. + /// + /// Prefer using [`clear`] instead of a call to this method with a full range ([`..`]). + /// + /// Comparisons are made by using the bytes representation of the key. + /// + /// [`clear`]: crate::Database::clear + /// [`..`]: std::ops::RangeFull + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// db.put(&mut wtxn, &521, "i-am-five-hundred-and-twenty-one")?; + /// + /// let range = 27..=42; + /// let ret = db.delete_range(&mut wtxn, &range)?; + /// assert_eq!(ret, 2); + /// + /// + /// let mut iter = db.iter(&wtxn)?; + /// assert_eq!(iter.next().transpose()?, Some((13, "i-am-thirteen"))); + /// assert_eq!(iter.next().transpose()?, Some((521, "i-am-five-hundred-and-twenty-one"))); + /// assert_eq!(iter.next().transpose()?, None); + /// + /// drop(iter); + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn delete_range<'a, 'txn, R>(&self, txn: &'txn mut RwTxn, range: &'a R) -> Result + where + KC: BytesEncode<'a> + BytesDecode<'txn>, + R: RangeBounds, + { + self.inner.delete_range(range) + } + + /// Deletes all key/value pairs in this database. + /// + /// Prefer using this method instead of a call to [`delete_range`] with a full range ([`..`]). + /// + /// [`delete_range`]: crate::Database::delete_range + /// [`..`]: std::ops::RangeFull + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// db.put(&mut wtxn, &521, "i-am-five-hundred-and-twenty-one")?; + /// + /// db.clear(&mut wtxn)?; + /// + /// let ret = db.is_empty(&wtxn)?; + /// assert!(ret); + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn clear(&self, txn: &mut RwTxn) -> Result<()> { + self.inner.clear() + } + + /// Change the codec types of this database, specifying the codecs. + /// + /// # Safety + /// + /// It is up to you to ensure that the data read and written using the polymorphic + /// handle correspond to the the typed, uniform one. If an invalid write is made, + /// it can corrupt the database from the eyes of heed. + /// + /// # Example + /// + /// ``` + /// # use std::fs; + /// # use std::path::Path; + /// # use heed::EnvOpenOptions; + /// use heed::Database; + /// use heed::types::*; + /// use heed::byteorder::BigEndian; + /// + /// # fn main() -> Result<(), Box> { + /// # let dir = tempfile::tempdir()?; + /// # let env = unsafe { EnvOpenOptions::new() + /// # .map_size(10 * 1024 * 1024) // 10MB + /// # .max_dbs(3000) + /// # .open(dir.path())? + /// # }; + /// type BEI32 = I32; + /// + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("iter-i32"))?; + /// + /// # db.clear(&mut wtxn)?; + /// // We remap the types for ease of use. + /// let db = db.remap_types::(); + /// db.put(&mut wtxn, &42, "i-am-forty-two")?; + /// db.put(&mut wtxn, &27, "i-am-twenty-seven")?; + /// db.put(&mut wtxn, &13, "i-am-thirteen")?; + /// db.put(&mut wtxn, &521, "i-am-five-hundred-and-twenty-one")?; + /// + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + pub fn remap_types(&self) -> EncryptedDatabase { + EncryptedDatabase::new(self.inner.env_ident, self.inner.dbi) + } + + /// Change the key codec type of this database, specifying the new codec. + pub fn remap_key_type(&self) -> EncryptedDatabase { + self.remap_types::() + } + + /// Change the data codec type of this database, specifying the new codec. + pub fn remap_data_type(&self) -> EncryptedDatabase { + self.remap_types::() + } + + /// Wrap the data bytes into a lazy decoder. + pub fn lazily_decode_data(&self) -> EncryptedDatabase, C> { + self.remap_types::>() + } +} + +impl Clone for EncryptedDatabase { + fn clone(&self) -> EncryptedDatabase { + *self + } +} + +impl Copy for EncryptedDatabase {} + +impl fmt::Debug for EncryptedDatabase { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("EncryptedDatabase") + .field("key_codec", &any::type_name::()) + .field("data_codec", &any::type_name::()) + .field("comparator", &any::type_name::()) + .finish() + } +} + +#[cfg(test)] +mod tests { + use heed_types::*; + + use super::*; + + #[test] + fn put_overwrite() -> Result<()> { + let dir = tempfile::tempdir()?; + let env = unsafe { EnvOpenOptions::new().open(dir.path())? }; + let mut txn = env.write_txn()?; + let db = env.create_database::(&mut txn, None)?; + + assert_eq!(db.get(&txn, b"hello").unwrap(), None); + + db.put(&mut txn, b"hello", b"hi").unwrap(); + assert_eq!(db.get(&txn, b"hello").unwrap(), Some(&b"hi"[..])); + + db.put(&mut txn, b"hello", b"bye").unwrap(); + assert_eq!(db.get(&txn, b"hello").unwrap(), Some(&b"bye"[..])); + + Ok(()) + } + + #[test] + #[cfg(feature = "longer-keys")] + fn longer_keys() -> Result<()> { + let dir = tempfile::tempdir()?; + let env = unsafe { EnvOpenOptions::new().open(dir.path())? }; + let mut txn = env.write_txn()?; + let db = env.create_database::(&mut txn, None)?; + + // Try storing a key larger than 511 bytes (the default if MDB_MAXKEYSIZE is not set) + let long_key = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut pharetra sit amet aliquam. Sit amet nisl purus in mollis nunc. Eget egestas purus viverra accumsan in nisl nisi scelerisque. Duis ultricies lacus sed turpis tincidunt. Sem nulla pharetra diam sit. Leo vel orci porta non pulvinar. Erat pellentesque adipiscing commodo elit at imperdiet dui. Suspendisse ultrices gravida dictum fusce ut placerat orci nulla. Diam donec adipiscing tristique risus nec feugiat. In fermentum et sollicitudin ac orci. Ut sem nulla pharetra diam sit amet. Aliquam purus sit amet luctus venenatis lectus. Erat pellentesque adipiscing commodo elit at imperdiet dui accumsan. Urna duis convallis convallis tellus id interdum velit laoreet id. Ac feugiat sed lectus vestibulum mattis ullamcorper velit sed. Tincidunt arcu non sodales neque. Habitant morbi tristique senectus et netus et malesuada fames."; + + assert_eq!(db.get(&txn, long_key).unwrap(), None); + + db.put(&mut txn, long_key, b"hi").unwrap(); + assert_eq!(db.get(&txn, long_key).unwrap(), Some(&b"hi"[..])); + + db.put(&mut txn, long_key, b"bye").unwrap(); + assert_eq!(db.get(&txn, long_key).unwrap(), Some(&b"bye"[..])); + + Ok(()) + } +} diff --git a/heed/src/database/mod.rs b/heed/src/database/mod.rs new file mode 100644 index 00000000..b32f16ea --- /dev/null +++ b/heed/src/database/mod.rs @@ -0,0 +1,25 @@ +pub use database::{Database, DatabaseOpenOptions}; +#[cfg(master3)] +pub use encrypted_database::{EncryptedDatabase, EncryptedDatabaseOpenOptions}; + +mod database; +#[cfg(master3)] +mod encrypted_database; + +/// Statistics for a database in the environment. +#[derive(Debug, Clone, Copy)] +pub struct DatabaseStat { + /// Size of a database page. + /// This is currently the same for all databases. + pub page_size: u32, + /// Depth (height) of the B-tree. + pub depth: u32, + /// Number of internal (non-leaf) pages + pub branch_pages: usize, + /// Number of leaf pages. + pub leaf_pages: usize, + /// Number of overflow pages. + pub overflow_pages: usize, + /// Number of data items. + pub entries: usize, +} diff --git a/heed/src/env/clear.rs b/heed/src/env/clear.rs deleted file mode 100644 index 3e1ee899..00000000 --- a/heed/src/env/clear.rs +++ /dev/null @@ -1,249 +0,0 @@ -use std::collections::hash_map::Entry; -use std::ffi::CString; -#[cfg(windows)] -use std::ffi::OsStr; -use std::io::ErrorKind::NotFound; -#[cfg(unix)] -use std::os::unix::ffi::OsStrExt; -use std::path::Path; -use std::sync::Arc; -use std::{io, ptr}; - -use synchronoise::SignalEvent; - -#[cfg(windows)] -use crate::env::OsStrExtLmdb as _; -use crate::env::{canonicalize_path, Env, EnvFlags, EnvInner, OPENED_ENV}; -use crate::mdb::ffi; -use crate::mdb::lmdb_error::mdb_result; -use crate::{Error, Result}; - -pub struct EnvEntry { - pub(super) env: Option, - pub(super) signal_event: Arc, - pub(super) options: EnvOpenOptions, -} - -/// Options and flags which can be used to configure how an environment is opened. -#[derive(Clone, Debug, PartialEq, Eq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct EnvOpenOptions { - map_size: Option, - max_readers: Option, - max_dbs: Option, - flags: EnvFlags, -} - -impl Default for EnvOpenOptions { - fn default() -> Self { - Self::new() - } -} - -impl EnvOpenOptions { - /// Creates a blank new set of options ready for configuration. - pub fn new() -> EnvOpenOptions { - EnvOpenOptions { - map_size: None, - max_readers: None, - max_dbs: None, - flags: EnvFlags::empty(), - } - } -} - -impl EnvOpenOptions { - /// Set the size of the memory map to use for this environment. - pub fn map_size(&mut self, size: usize) -> &mut Self { - self.map_size = Some(size); - self - } - - /// Set the maximum number of threads/reader slots for the environment. - pub fn max_readers(&mut self, readers: u32) -> &mut Self { - self.max_readers = Some(readers); - self - } - - /// Set the maximum number of named databases for the environment. - pub fn max_dbs(&mut self, dbs: u32) -> &mut Self { - self.max_dbs = Some(dbs); - self - } - - /// Set one or [more LMDB flags](http://www.lmdb.tech/doc/group__mdb__env.html). - /// - /// ``` - /// use std::fs; - /// use std::path::Path; - /// use heed::{EnvOpenOptions, Database, EnvFlags}; - /// use heed::types::*; - /// - /// # fn main() -> Result<(), Box> { - /// fs::create_dir_all(Path::new("target").join("database.mdb"))?; - /// let mut env_builder = EnvOpenOptions::new(); - /// unsafe { env_builder.flags(EnvFlags::NO_TLS | EnvFlags::NO_META_SYNC); } - /// let dir = tempfile::tempdir().unwrap(); - /// let env = unsafe { env_builder.open(dir.path())? }; - /// - /// // we will open the default unamed database - /// let mut wtxn = env.write_txn()?; - /// let db: Database> = env.create_database(&mut wtxn, None)?; - /// - /// // opening a write transaction - /// db.put(&mut wtxn, "seven", &7)?; - /// db.put(&mut wtxn, "zero", &0)?; - /// db.put(&mut wtxn, "five", &5)?; - /// db.put(&mut wtxn, "three", &3)?; - /// wtxn.commit()?; - /// - /// // Force the OS to flush the buffers (see Flag::NoSync and Flag::NoMetaSync). - /// env.force_sync(); - /// - /// // opening a read transaction - /// // to check if those values are now available - /// let mut rtxn = env.read_txn()?; - /// - /// let ret = db.get(&rtxn, "zero")?; - /// assert_eq!(ret, Some(0)); - /// - /// let ret = db.get(&rtxn, "five")?; - /// assert_eq!(ret, Some(5)); - /// # Ok(()) } - /// ``` - /// - /// # Safety - /// - /// It is unsafe to use unsafe LMDB flags such as `NO_SYNC`, `NO_META_SYNC`, or `NO_LOCK`. - pub unsafe fn flags(&mut self, flags: EnvFlags) -> &mut Self { - self.flags |= flags; - self - } - - /// Open an environment that will be located at the specified path. - /// - /// # Safety - /// LMDB is backed by a memory map [^1] which comes with some safety precautions. - /// - /// Memory map constructors are marked `unsafe` because of the potential - /// for Undefined Behavior (UB) using the map if the underlying file is - /// subsequently modified, in or out of process. - /// - /// LMDB itself has a locking system that solves this problem, - /// but it will not save you from making mistakes yourself. - /// - /// These are some things to take note of: - /// - /// - Avoid long-lived transactions, they will cause the database to grow quickly [^2] - /// - Avoid aborting your process with an active transaction [^3] - /// - Do not use LMDB on remote filesystems, even between processes on the same host [^4] - /// - You must manage concurrent accesses yourself if using [`EnvFlags::NO_LOCK`] [^5] - /// - Anything that causes LMDB's lock file to be broken will cause synchronization issues and may introduce UB [^6] - /// - /// `heed` itself upholds some safety invariants, including but not limited to: - /// - Calling [`EnvOpenOptions::open`] twice in the same process, at the same time is OK [^7] - /// - /// For more details, it is highly recommended to read LMDB's official documentation. [^8] - /// - /// [^1]: - /// [^2]: - /// [^3]: - /// [^4]: - /// [^5]: - /// [^6]: - /// [^7]: - /// [^8]: - pub unsafe fn open>(&self, path: P) -> Result { - let mut lock = OPENED_ENV.write().unwrap(); - - let path = match canonicalize_path(path.as_ref()) { - Err(err) => { - if err.kind() == NotFound && self.flags.contains(EnvFlags::NO_SUB_DIR) { - let path = path.as_ref(); - match path.parent().zip(path.file_name()) { - Some((dir, file_name)) => canonicalize_path(dir)?.join(file_name), - None => return Err(err.into()), - } - } else { - return Err(err.into()); - } - } - Ok(path) => path, - }; - - match lock.entry(path) { - Entry::Occupied(entry) => { - let env = entry.get().env.clone().ok_or(Error::DatabaseClosing)?; - let options = entry.get().options.clone(); - if &options == self { - Ok(env) - } else { - Err(Error::BadOpenOptions { env, options }) - } - } - Entry::Vacant(entry) => { - let path = entry.key(); - let path_str = CString::new(path.as_os_str().as_bytes()).unwrap(); - - unsafe { - let mut env: *mut ffi::MDB_env = ptr::null_mut(); - mdb_result(ffi::mdb_env_create(&mut env))?; - - if let Some(size) = self.map_size { - if size % page_size::get() != 0 { - let msg = format!( - "map size ({}) must be a multiple of the system page size ({})", - size, - page_size::get() - ); - return Err(Error::Io(io::Error::new( - io::ErrorKind::InvalidInput, - msg, - ))); - } - mdb_result(ffi::mdb_env_set_mapsize(env, size))?; - } - - if let Some(readers) = self.max_readers { - mdb_result(ffi::mdb_env_set_maxreaders(env, readers))?; - } - - if let Some(dbs) = self.max_dbs { - mdb_result(ffi::mdb_env_set_maxdbs(env, dbs))?; - } - - // When the `read-txn-no-tls` feature is enabled, we must force LMDB - // to avoid using the thread local storage, this way we allow users - // to use references of RoTxn between threads safely. - let flags = if cfg!(feature = "read-txn-no-tls") { - self.flags | EnvFlags::NO_TLS - } else { - self.flags - }; - - let result = - mdb_result(ffi::mdb_env_open(env, path_str.as_ptr(), flags.bits(), 0o600)); - - match result { - Ok(()) => { - let signal_event = Arc::new(SignalEvent::manual(false)); - let inner = EnvInner { env, path: path.clone() }; - let env = Env(Arc::new(inner)); - let cache_entry = EnvEntry { - env: Some(env.clone()), - options: self.clone(), - signal_event, - }; - entry.insert(cache_entry); - Ok(env) - } - Err(e) => { - ffi::mdb_env_close(env); - Err(e.into()) - } - } - } - } - } - } -} diff --git a/heed/src/env/encrypted_env.rs b/heed/src/env/encrypted_env.rs new file mode 100644 index 00000000..2c38e651 --- /dev/null +++ b/heed/src/env/encrypted_env.rs @@ -0,0 +1,430 @@ +use std::any::TypeId; +use std::ffi::CString; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::ptr::{self, NonNull}; +use std::{fmt, io, mem}; + +use heed_traits::Comparator; +use lmdb_master_sys::mdb_env_close; + +use super::{ + custom_key_cmp_wrapper, get_file_fd, metadata_from_fd, DefaultComparator, EnvInfo, FlagSetMode, + OPENED_ENV, +}; +use crate::cursor::{MoveOperation, RoCursor}; +use crate::mdb::ffi::{self, MDB_env}; +use crate::mdb::lmdb_error::mdb_result; +use crate::mdb::lmdb_flags::AllDatabaseFlags; +use crate::{ + CompactionOption, Database, DatabaseOpenOptions, EnvFlags, Error, Result, RoTxn, RwTxn, + Unspecified, +}; + +/// An environment handle constructed by using [`EnvOpenOptions::open_encrypted`]. +pub struct EncryptedEnv { + inner: Env, +} + +impl Env { + pub(crate) fn new(env_ptr: NonNull, path: PathBuf) -> Env { + Env { env_ptr, path } + } + + pub(crate) fn env_mut_ptr(&self) -> NonNull { + self.inner.env_mut_ptr() + } + + /// The size of the data file on disk. + /// + /// # Example + /// + /// ``` + /// use heed::EnvOpenOptions; + /// + /// # fn main() -> Result<(), Box> { + /// let dir = tempfile::tempdir()?; + /// let size_in_bytes = 1024 * 1024; + /// let env = unsafe { EnvOpenOptions::new().map_size(size_in_bytes).open(dir.path())? }; + /// + /// let actual_size = env.real_disk_size()? as usize; + /// assert!(actual_size < size_in_bytes); + /// # Ok(()) } + /// ``` + pub fn real_disk_size(&self) -> Result { + self.inner.real_disk_size() + } + + /// Return the raw flags the environment was opened with. + /// + /// Returns `None` if the environment flags are different from the [`EnvFlags`] set. + pub fn flags(&self) -> Result> { + self.inner.flags() + } + + /// Enable or disable the environment's currently active [`EnvFlags`]. + /// + /// ``` + /// use std::fs; + /// use std::path::Path; + /// use heed::{EnvOpenOptions, Database, EnvFlags, FlagSetMode}; + /// use heed::types::*; + /// + /// # fn main() -> Result<(), Box> { + /// fs::create_dir_all(Path::new("target").join("database.mdb"))?; + /// let mut env_builder = EnvOpenOptions::new(); + /// let dir = tempfile::tempdir().unwrap(); + /// let env = unsafe { env_builder.open(dir.path())? }; + /// + /// // Env was opened without flags. + /// assert_eq!(env.get_flags().unwrap(), EnvFlags::empty().bits()); + /// + /// // Enable a flag after opening. + /// unsafe { env.set_flags(EnvFlags::NO_SYNC, FlagSetMode::Enable).unwrap(); } + /// assert_eq!(env.get_flags().unwrap(), EnvFlags::NO_SYNC.bits()); + /// + /// // Disable a flag after opening. + /// unsafe { env.set_flags(EnvFlags::NO_SYNC, FlagSetMode::Disable).unwrap(); } + /// assert_eq!(env.get_flags().unwrap(), EnvFlags::empty().bits()); + /// # Ok(()) } + /// ``` + /// + /// # Safety + /// + /// It is unsafe to use unsafe LMDB flags such as `NO_SYNC`, `NO_META_SYNC`, or `NO_LOCK`. + /// + /// LMDB also requires that only 1 thread calls this function at any given moment. + /// Neither `heed` or LMDB check for this condition, so the caller must ensure it explicitly. + pub unsafe fn set_flags(&self, flags: EnvFlags, mode: FlagSetMode) -> Result<()> { + self.inner.set_flags(flags, mode) + } + + /// Return the raw flags the environment is currently set with. + pub fn get_flags(&self) -> Result { + self.inner.get_flags() + } + + /// Returns some basic informations about this environment. + pub fn info(&self) -> EnvInfo { + self.inner.info() + } + + /// Returns the size used by all the databases in the environment without the free pages. + /// + /// It is crucial to configure [`EnvOpenOptions::max_dbs`] with a sufficiently large value + /// before invoking this function. All databases within the environment will be opened + /// and remain so. + pub fn non_free_pages_size(&self) -> Result { + self.inner.non_free_pages_size() + } + + /// Options and flags which can be used to configure how a [`Database`] is opened. + pub fn database_options(&self) -> EncryptedDatabaseOpenOptions { + EncryptedDatabaseOpenOptions::new(self) + } + + /// Opens a typed database that already exists in this environment. + /// + /// If the database was previously opened in this program run, types will be checked. + /// + /// ## Important Information + /// + /// LMDB has an important restriction on the unnamed database when named ones are opened. + /// The names of the named databases are stored as keys in the unnamed one and are immutable, + /// and these keys can only be read and not written. + /// + /// ## LMDB read-only access of existing database + /// + /// In the case of accessing a database in a read-only manner from another process + /// where you wrote, you might need to manually call [`RoTxn::commit`] to get metadata + /// and the database handles opened and shared with the global [`Env`] handle. + /// + /// If not done, you might raise `Io(Os { code: 22, kind: InvalidInput, message: "Invalid argument" })` + /// known as `EINVAL`. + pub fn open_database( + &self, + rtxn: &RoTxn, + name: Option<&str>, + ) -> Result>> + where + KC: 'static, + DC: 'static, + { + let mut options = self.database_options().types::(); + if let Some(name) = name { + options.name(name); + } + options.open(rtxn) + } + + /// Creates a typed database that can already exist in this environment. + /// + /// If the database was previously opened during this program run, types will be checked. + /// + /// ## Important Information + /// + /// LMDB has an important restriction on the unnamed database when named ones are opened. + /// The names of the named databases are stored as keys in the unnamed one and are immutable, + /// and these keys can only be read and not written. + pub fn create_database( + &self, + wtxn: &mut RwTxn, + name: Option<&str>, + ) -> Result> + where + KC: 'static, + DC: 'static, + { + let mut options = self.database_options().types::(); + if let Some(name) = name { + options.name(name); + } + options.create(wtxn) + } + + pub(crate) fn raw_init_database( + &self, + raw_txn: *mut ffi::MDB_txn, + name: Option<&str>, + flags: AllDatabaseFlags, + ) -> Result { + self.inner.raw_init_database(raw_txn, name, flags) + } + + fn raw_open_dbi( + &self, + raw_txn: *mut ffi::MDB_txn, + name: Option<&str>, + flags: u32, + ) -> std::result::Result { + self.inner.raw_open_dbi(raw_txn, name, flags) + } + + /// Create a transaction with read and write access for use with the environment. + /// + /// ## LMDB Limitations + /// + /// Only one [`RwTxn`] may exist simultaneously in the current environment. + /// If another write transaction is initiated, while another write transaction exists + /// the thread initiating the new one will wait on a mutex upon completion of the previous + /// transaction. + pub fn write_txn(&self) -> Result { + self.inner.write_txn() + } + + /// Create a nested transaction with read and write access for use with the environment. + /// + /// The new transaction will be a nested transaction, with the transaction indicated by parent + /// as its parent. Transactions may be nested to any level. + /// + /// A parent transaction and its cursors may not issue any other operations than _commit_ and + /// _abort_ while it has active child transactions. + pub fn nested_write_txn<'p>(&'p self, parent: &'p mut RwTxn) -> Result> { + self.inner.nested_write_txn(parent) + } + + /// Create a transaction with read-only access for use with the environment. + /// + /// You can make this transaction `Send`able between threads by + /// using the `read-txn-no-tls` crate feature. + /// See [`Self::static_read_txn`] if you want the txn to own the environment. + /// + /// ## LMDB Limitations + /// + /// It's possible to have multiple read transactions in the same environment + /// while there is a write transaction ongoing. + /// + /// But read transactions prevent reuse of pages freed by newer write transactions, + /// thus the database can grow quickly. Write transactions prevent other write transactions, + /// since writes are serialized. + /// + /// So avoid long-lived read transactions. + /// + /// ## Errors + /// + /// * [`crate::MdbError::Panic`]: A fatal error occurred earlier, and the environment must be shut down + /// * [`crate::MdbError::MapResized`]: Another process wrote data beyond this [`Env`] mapsize and this env + /// map must be resized + /// * [`crate::MdbError::ReadersFull`]: a read-only transaction was requested, and the reader lock table is + /// full + pub fn read_txn(&self) -> Result { + self.inner.read_txn() + } + + /// Create a transaction with read-only access for use with the environment. + /// Contrary to [`Self::read_txn`], this version **owns** the environment, which + /// means you won't be able to close the environment while this transaction is alive. + /// + /// You can make this transaction `Send`able between threads by + /// using the `read-txn-no-tls` crate feature. + /// + /// ## LMDB Limitations + /// + /// It's possible to have multiple read transactions in the same environment + /// while there is a write transaction ongoing. + /// + /// But read transactions prevent reuse of pages freed by newer write transactions, + /// thus the database can grow quickly. Write transactions prevent other write transactions, + /// since writes are serialized. + /// + /// So avoid long-lived read transactions. + /// + /// ## Errors + /// + /// * [`crate::MdbError::Panic`]: A fatal error occurred earlier, and the environment must be shut down + /// * [`crate::MdbError::MapResized`]: Another process wrote data beyond this [`Env`] mapsize and this env + /// map must be resized + /// * [`crate::MdbError::ReadersFull`]: a read-only transaction was requested, and the reader lock table is + /// full + pub fn static_read_txn(self) -> Result> { + self.inner.static_read_txn() + } + + /// Copy an LMDB environment to the specified path, with options. + /// + /// This function may be used to make a backup of an existing environment. + /// No lockfile is created, since it gets recreated at need. + pub fn copy_to_file>(&self, path: P, option: CompactionOption) -> Result { + self.inner.copy_to_file(path, option) + } + + /// Copy an LMDB environment to the specified file descriptor, with compaction option. + /// + /// This function may be used to make a backup of an existing environment. + /// No lockfile is created, since it gets recreated at need. + /// + /// # Safety + /// + /// The [`ffi::mdb_filehandle_t`] must have already been opened for Write access. + pub unsafe fn copy_to_fd( + &self, + fd: ffi::mdb_filehandle_t, + option: CompactionOption, + ) -> Result<()> { + self.inner.copy_to_fd(fd, option) + } + + /// Flush the data buffers to disk. + pub fn force_sync(&self) -> Result<()> { + self.inner.force_sync() + } + + /// Returns the canonicalized path where this env lives. + pub fn path(&self) -> &Path { + self.inner.path() + } + + /// Check for stale entries in the reader lock table and clear them. + /// + /// Returns the number of stale readers cleared. + pub fn clear_stale_readers(&self) -> Result { + self.inner.clear_stale_readers() + } + + /// Resize the memory map to a new size. + /// + /// # Safety + /// + /// According to the [LMDB documentation](http://www.lmdb.tech/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5), + /// it is okay to call `mdb_env_set_mapsize` for an open environment as long as no transactions are active, + /// but the library does not check for this condition, so the caller must ensure it explicitly. + pub unsafe fn resize(&self, new_size: usize) -> Result<()> { + self.inner.resize(new_size) + } + + /// Get the maximum size of keys and MDB_DUPSORT data we can write. + /// + /// Depends on the compile-time constant MDB_MAXKEYSIZE. Default 511 + pub fn max_key_size(&self) -> usize { + self.inner.max_key_size() + } +} + +unsafe impl Send for EncryptedEnv {} + +unsafe impl Sync for EncryptedEnv {} + +impl fmt::Debug for EncryptedEnv { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("EncryptedEnv").field("path", &self.path.display()).finish_non_exhaustive() + } +} + +fn encrypt( + key: &[u8], + nonce: &[u8], + aad: &[u8], + plaintext: &[u8], + chipertext_out: &mut [u8], + auth_out: &mut [u8], +) -> aead::Result<()> { + chipertext_out.copy_from_slice(plaintext); + let key: &Key = key.try_into().unwrap(); + let nonce: &Nonce = if nonce.len() >= A::NonceSize::USIZE { + nonce[..A::NonceSize::USIZE].into() + } else { + return Err(aead::Error); + }; + let mut aead = A::new(key); + let tag = aead.encrypt_in_place_detached(nonce, aad, chipertext_out)?; + auth_out.copy_from_slice(&tag); + Ok(()) +} + +fn decrypt( + key: &[u8], + nonce: &[u8], + aad: &[u8], + chipher_text: &[u8], + output: &mut [u8], + auth_in: &[u8], +) -> aead::Result<()> { + output.copy_from_slice(chipher_text); + let key: &Key = key.try_into().unwrap(); + let nonce: &Nonce = if nonce.len() >= A::NonceSize::USIZE { + nonce[..A::NonceSize::USIZE].into() + } else { + return Err(aead::Error); + }; + let tag: &Tag = auth_in.try_into().unwrap(); + let mut aead = A::new(key); + aead.decrypt_in_place_detached(nonce, aad, output, tag) +} + +/// The wrapper function that is called by LMDB that directly calls +/// the Rust idiomatic function internally. +unsafe extern "C" fn encrypt_func_wrapper( + src: *const ffi::MDB_val, + dst: *mut ffi::MDB_val, + key_ptr: *const ffi::MDB_val, + encdec: i32, +) -> i32 { + let result = catch_unwind(|| { + let input = std::slice::from_raw_parts((*src).mv_data as *const u8, (*src).mv_size); + let output = std::slice::from_raw_parts_mut((*dst).mv_data as *mut u8, (*dst).mv_size); + let key = std::slice::from_raw_parts((*key_ptr).mv_data as *const u8, (*key_ptr).mv_size); + let iv = std::slice::from_raw_parts( + (*key_ptr.offset(1)).mv_data as *const u8, + (*key_ptr.offset(1)).mv_size, + ); + let auth = std::slice::from_raw_parts_mut( + (*key_ptr.offset(2)).mv_data as *mut u8, + (*key_ptr.offset(2)).mv_size, + ); + + let aad = []; + let nonce = iv; + let result = if encdec == 1 { + encrypt::(&key, nonce, &aad, input, output, auth) + } else { + decrypt::(&key, nonce, &aad, input, output, auth) + }; + + result.is_err() as i32 + }); + + match result { + Ok(out) => out, + Err(_) => 1, + } +} diff --git a/heed/src/env/env.rs b/heed/src/env/env.rs new file mode 100644 index 00000000..29de0269 --- /dev/null +++ b/heed/src/env/env.rs @@ -0,0 +1,478 @@ +use std::any::TypeId; +use std::ffi::CString; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::ptr::{self, NonNull}; +use std::{fmt, io, mem}; + +use heed_traits::Comparator; +use lmdb_master_sys::mdb_env_close; + +use super::{ + custom_key_cmp_wrapper, get_file_fd, metadata_from_fd, DefaultComparator, EnvInfo, FlagSetMode, + OPENED_ENV, +}; +use crate::cursor::{MoveOperation, RoCursor}; +use crate::mdb::ffi::{self, MDB_env}; +use crate::mdb::lmdb_error::mdb_result; +use crate::mdb::lmdb_flags::AllDatabaseFlags; +use crate::{ + CompactionOption, Database, DatabaseOpenOptions, EnvFlags, Error, Result, RoTxn, RwTxn, + Unspecified, +}; + +/// An environment handle constructed by using [`EnvOpenOptions::open`]. +pub struct Env { + env_ptr: NonNull, + path: PathBuf, +} + +impl Env { + pub(crate) fn new(env_ptr: NonNull, path: PathBuf) -> Env { + Env { env_ptr, path } + } + + pub(crate) fn env_mut_ptr(&self) -> NonNull { + self.env_ptr + } + + /// The size of the data file on disk. + /// + /// # Example + /// + /// ``` + /// use heed::EnvOpenOptions; + /// + /// # fn main() -> Result<(), Box> { + /// let dir = tempfile::tempdir()?; + /// let size_in_bytes = 1024 * 1024; + /// let env = unsafe { EnvOpenOptions::new().map_size(size_in_bytes).open(dir.path())? }; + /// + /// let actual_size = env.real_disk_size()? as usize; + /// assert!(actual_size < size_in_bytes); + /// # Ok(()) } + /// ``` + pub fn real_disk_size(&self) -> Result { + let mut fd = mem::MaybeUninit::uninit(); + unsafe { mdb_result(ffi::mdb_env_get_fd(self.env_mut_ptr(), fd.as_mut_ptr()))? }; + let fd = unsafe { fd.assume_init() }; + let metadata = unsafe { metadata_from_fd(fd)? }; + Ok(metadata.len()) + } + + /// Return the raw flags the environment was opened with. + /// + /// Returns `None` if the environment flags are different from the [`EnvFlags`] set. + pub fn flags(&self) -> Result> { + self.get_flags().map(EnvFlags::from_bits) + } + + /// Enable or disable the environment's currently active [`EnvFlags`]. + /// + /// ``` + /// use std::fs; + /// use std::path::Path; + /// use heed::{EnvOpenOptions, Database, EnvFlags, FlagSetMode}; + /// use heed::types::*; + /// + /// # fn main() -> Result<(), Box> { + /// fs::create_dir_all(Path::new("target").join("database.mdb"))?; + /// let mut env_builder = EnvOpenOptions::new(); + /// let dir = tempfile::tempdir().unwrap(); + /// let env = unsafe { env_builder.open(dir.path())? }; + /// + /// // Env was opened without flags. + /// assert_eq!(env.get_flags().unwrap(), EnvFlags::empty().bits()); + /// + /// // Enable a flag after opening. + /// unsafe { env.set_flags(EnvFlags::NO_SYNC, FlagSetMode::Enable).unwrap(); } + /// assert_eq!(env.get_flags().unwrap(), EnvFlags::NO_SYNC.bits()); + /// + /// // Disable a flag after opening. + /// unsafe { env.set_flags(EnvFlags::NO_SYNC, FlagSetMode::Disable).unwrap(); } + /// assert_eq!(env.get_flags().unwrap(), EnvFlags::empty().bits()); + /// # Ok(()) } + /// ``` + /// + /// # Safety + /// + /// It is unsafe to use unsafe LMDB flags such as `NO_SYNC`, `NO_META_SYNC`, or `NO_LOCK`. + /// + /// LMDB also requires that only 1 thread calls this function at any given moment. + /// Neither `heed` or LMDB check for this condition, so the caller must ensure it explicitly. + pub unsafe fn set_flags(&self, flags: EnvFlags, mode: FlagSetMode) -> Result<()> { + // safety: caller must ensure no other thread is calling this function. + // + mdb_result(unsafe { + ffi::mdb_env_set_flags( + self.env_mut_ptr(), + flags.bits(), + mode.as_mdb_env_set_flags_input(), + ) + }) + .map_err(Into::into) + } + + /// Return the raw flags the environment is currently set with. + pub fn get_flags(&self) -> Result { + let mut flags = mem::MaybeUninit::uninit(); + unsafe { mdb_result(ffi::mdb_env_get_flags(self.env_mut_ptr(), flags.as_mut_ptr()))? }; + let flags = unsafe { flags.assume_init() }; + Ok(flags) + } + + /// Returns some basic informations about this environment. + pub fn info(&self) -> EnvInfo { + let mut raw_info = mem::MaybeUninit::uninit(); + unsafe { ffi::mdb_env_info(self.0.env, raw_info.as_mut_ptr()) }; + let raw_info = unsafe { raw_info.assume_init() }; + + EnvInfo { + map_addr: raw_info.me_mapaddr, + map_size: raw_info.me_mapsize, + last_page_number: raw_info.me_last_pgno, + last_txn_id: raw_info.me_last_txnid, + maximum_number_of_readers: raw_info.me_maxreaders, + number_of_readers: raw_info.me_numreaders, + } + } + + /// Returns the size used by all the databases in the environment without the free pages. + /// + /// It is crucial to configure [`EnvOpenOptions::max_dbs`] with a sufficiently large value + /// before invoking this function. All databases within the environment will be opened + /// and remain so. + pub fn non_free_pages_size(&self) -> Result { + let compute_size = |stat: ffi::MDB_stat| { + (stat.ms_leaf_pages + stat.ms_branch_pages + stat.ms_overflow_pages) as u64 + * stat.ms_psize as u64 + }; + + let mut size = 0; + + let mut stat = mem::MaybeUninit::uninit(); + unsafe { mdb_result(ffi::mdb_env_stat(self.env_mut_ptr(), stat.as_mut_ptr()))? }; + let stat = unsafe { stat.assume_init() }; + size += compute_size(stat); + + let rtxn = self.read_txn()?; + // Open the main database + let dbi = self.raw_open_dbi::(rtxn.txn.unwrap(), None, 0)?; + + // We're going to iterate on the unnamed database + let mut cursor = RoCursor::new(&rtxn, dbi)?; + + while let Some((key, _value)) = cursor.move_on_next(MoveOperation::NoDup)? { + if key.contains(&0) { + continue; + } + + let key = String::from_utf8(key.to_vec()).unwrap(); + // Calling `ffi::db_stat` on a database instance does not involve key comparison + // in LMDB, so it's safe to specify a noop key compare function for it. + if let Ok(dbi) = + self.raw_open_dbi::(rtxn.txn.unwrap(), Some(&key), 0) + { + let mut stat = mem::MaybeUninit::uninit(); + let mut txn = rtxn.txn.unwrap(); + unsafe { mdb_result(ffi::mdb_stat(txn.as_mut(), dbi, stat.as_mut_ptr()))? }; + let stat = unsafe { stat.assume_init() }; + size += compute_size(stat); + } + } + + Ok(size) + } + + /// Options and flags which can be used to configure how a [`Database`] is opened. + pub fn database_options(&self) -> DatabaseOpenOptions { + DatabaseOpenOptions::new(self) + } + + /// Opens a typed database that already exists in this environment. + /// + /// If the database was previously opened in this program run, types will be checked. + /// + /// ## Important Information + /// + /// LMDB has an important restriction on the unnamed database when named ones are opened. + /// The names of the named databases are stored as keys in the unnamed one and are immutable, + /// and these keys can only be read and not written. + /// + /// ## LMDB read-only access of existing database + /// + /// In the case of accessing a database in a read-only manner from another process + /// where you wrote, you might need to manually call [`RoTxn::commit`] to get metadata + /// and the database handles opened and shared with the global [`Env`] handle. + /// + /// If not done, you might raise `Io(Os { code: 22, kind: InvalidInput, message: "Invalid argument" })` + /// known as `EINVAL`. + pub fn open_database( + &self, + rtxn: &RoTxn, + name: Option<&str>, + ) -> Result>> + where + KC: 'static, + DC: 'static, + { + let mut options = self.database_options().types::(); + if let Some(name) = name { + options.name(name); + } + options.open(rtxn) + } + + /// Creates a typed database that can already exist in this environment. + /// + /// If the database was previously opened during this program run, types will be checked. + /// + /// ## Important Information + /// + /// LMDB has an important restriction on the unnamed database when named ones are opened. + /// The names of the named databases are stored as keys in the unnamed one and are immutable, + /// and these keys can only be read and not written. + pub fn create_database( + &self, + wtxn: &mut RwTxn, + name: Option<&str>, + ) -> Result> + where + KC: 'static, + DC: 'static, + { + let mut options = self.database_options().types::(); + if let Some(name) = name { + options.name(name); + } + options.create(wtxn) + } + + pub(crate) fn raw_init_database( + &self, + raw_txn: NonNull, + name: Option<&str>, + mut flags: AllDatabaseFlags, + ) -> Result { + if TypeId::of::() == TypeId::of::() { + flags.insert(AllDatabaseFlags::INTEGER_KEY); + } + + match self.raw_open_dbi::(raw_txn, name, flags.bits()) { + Ok(dbi) => Ok(dbi), + Err(e) => Err(e.into()), + } + } + + fn raw_open_dbi( + &self, + raw_txn: NonNull, + name: Option<&str>, + flags: u32, + ) -> std::result::Result { + let mut dbi = 0; + let name = name.map(|n| CString::new(n).unwrap()); + let name_ptr = match name { + Some(ref name) => name.as_bytes_with_nul().as_ptr() as *const _, + None => ptr::null(), + }; + + // safety: The name cstring is cloned by LMDB, we can drop it after. + // If a read-only is used with the MDB_CREATE flag, LMDB will throw an error. + unsafe { + mdb_result(ffi::mdb_dbi_open(raw_txn, name_ptr, flags, &mut dbi))?; + let cmp_type_id = TypeId::of::(); + + if cmp_type_id != TypeId::of::() + && cmp_type_id != TypeId::of::() + { + mdb_result(ffi::mdb_set_compare(raw_txn, dbi, Some(custom_key_cmp_wrapper::)))?; + } + }; + + Ok(dbi) + } + + /// Create a transaction with read and write access for use with the environment. + /// + /// ## LMDB Limitations + /// + /// Only one [`RwTxn`] may exist simultaneously in the current environment. + /// If another write transaction is initiated, while another write transaction exists + /// the thread initiating the new one will wait on a mutex upon completion of the previous + /// transaction. + pub fn write_txn(&self) -> Result { + RwTxn::new(self) + } + + /// Create a nested transaction with read and write access for use with the environment. + /// + /// The new transaction will be a nested transaction, with the transaction indicated by parent + /// as its parent. Transactions may be nested to any level. + /// + /// A parent transaction and its cursors may not issue any other operations than _commit_ and + /// _abort_ while it has active child transactions. + pub fn nested_write_txn<'p>(&'p self, parent: &'p mut RwTxn) -> Result> { + RwTxn::nested(self, parent) + } + + /// Create a transaction with read-only access for use with the environment. + /// + /// You can make this transaction `Send`able between threads by + /// using the `read-txn-no-tls` crate feature. + /// See [`Self::static_read_txn`] if you want the txn to own the environment. + /// + /// ## LMDB Limitations + /// + /// It's possible to have multiple read transactions in the same environment + /// while there is a write transaction ongoing. + /// + /// But read transactions prevent reuse of pages freed by newer write transactions, + /// thus the database can grow quickly. Write transactions prevent other write transactions, + /// since writes are serialized. + /// + /// So avoid long-lived read transactions. + /// + /// ## Errors + /// + /// * [`crate::MdbError::Panic`]: A fatal error occurred earlier, and the environment must be shut down + /// * [`crate::MdbError::MapResized`]: Another process wrote data beyond this [`Env`] mapsize and this env + /// map must be resized + /// * [`crate::MdbError::ReadersFull`]: a read-only transaction was requested, and the reader lock table is + /// full + pub fn read_txn(&self) -> Result { + RoTxn::new(self) + } + + /// Create a transaction with read-only access for use with the environment. + /// Contrary to [`Self::read_txn`], this version **owns** the environment, which + /// means you won't be able to close the environment while this transaction is alive. + /// + /// You can make this transaction `Send`able between threads by + /// using the `read-txn-no-tls` crate feature. + /// + /// ## LMDB Limitations + /// + /// It's possible to have multiple read transactions in the same environment + /// while there is a write transaction ongoing. + /// + /// But read transactions prevent reuse of pages freed by newer write transactions, + /// thus the database can grow quickly. Write transactions prevent other write transactions, + /// since writes are serialized. + /// + /// So avoid long-lived read transactions. + /// + /// ## Errors + /// + /// * [`crate::MdbError::Panic`]: A fatal error occurred earlier, and the environment must be shut down + /// * [`crate::MdbError::MapResized`]: Another process wrote data beyond this [`Env`] mapsize and this env + /// map must be resized + /// * [`crate::MdbError::ReadersFull`]: a read-only transaction was requested, and the reader lock table is + /// full + pub fn static_read_txn(self) -> Result> { + RoTxn::static_read_txn(self) + } + + /// Copy an LMDB environment to the specified path, with options. + /// + /// This function may be used to make a backup of an existing environment. + /// No lockfile is created, since it gets recreated at need. + pub fn copy_to_file>(&self, path: P, option: CompactionOption) -> Result { + let file = File::options().create_new(true).write(true).open(&path)?; + let fd = get_file_fd(&file); + + unsafe { self.copy_to_fd(fd, option)? }; + + // We reopen the file to make sure the cursor is at the start, + // even a seek to start doesn't work properly. + let file = File::open(path)?; + + Ok(file) + } + + /// Copy an LMDB environment to the specified file descriptor, with compaction option. + /// + /// This function may be used to make a backup of an existing environment. + /// No lockfile is created, since it gets recreated at need. + /// + /// # Safety + /// + /// The [`ffi::mdb_filehandle_t`] must have already been opened for Write access. + pub unsafe fn copy_to_fd( + &self, + fd: ffi::mdb_filehandle_t, + option: CompactionOption, + ) -> Result<()> { + let flags = if let CompactionOption::Enabled = option { ffi::MDB_CP_COMPACT } else { 0 }; + mdb_result(ffi::mdb_env_copyfd2(self.0.env, fd, flags))?; + Ok(()) + } + + /// Flush the data buffers to disk. + pub fn force_sync(&self) -> Result<()> { + unsafe { mdb_result(ffi::mdb_env_sync(self.0.env, 1))? } + Ok(()) + } + + /// Returns the canonicalized path where this env lives. + pub fn path(&self) -> &Path { + &self.path + } + + /// Check for stale entries in the reader lock table and clear them. + /// + /// Returns the number of stale readers cleared. + pub fn clear_stale_readers(&self) -> Result { + let mut dead: i32 = 0; + unsafe { mdb_result(ffi::mdb_reader_check(self.0.env, &mut dead))? } + // safety: The reader_check function asks for an i32, initialize it to zero + // and never decrements it. It is safe to use either an u32 or u64 (usize). + Ok(dead as usize) + } + + /// Resize the memory map to a new size. + /// + /// # Safety + /// + /// According to the [LMDB documentation](http://www.lmdb.tech/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5), + /// it is okay to call `mdb_env_set_mapsize` for an open environment as long as no transactions are active, + /// but the library does not check for this condition, so the caller must ensure it explicitly. + pub unsafe fn resize(&self, new_size: usize) -> Result<()> { + if new_size % page_size::get() != 0 { + let msg = format!( + "map size ({}) must be a multiple of the system page size ({})", + new_size, + page_size::get() + ); + return Err(Error::Io(io::Error::new(io::ErrorKind::InvalidInput, msg))); + } + mdb_result(unsafe { ffi::mdb_env_set_mapsize(self.env_mut_ptr(), new_size) }) + .map_err(Into::into) + } + + /// Get the maximum size of keys and MDB_DUPSORT data we can write. + /// + /// Depends on the compile-time constant MDB_MAXKEYSIZE. Default 511 + pub fn max_key_size(&self) -> usize { + let maxsize: i32 = unsafe { ffi::mdb_env_get_maxkeysize(self.env_mut_ptr()) }; + maxsize as usize + } +} + +unsafe impl Send for Env {} + +unsafe impl Sync for Env {} + +impl fmt::Debug for Env { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("Env").field("path", &self.path.display()).finish_non_exhaustive() + } +} + +impl Drop for Env { + fn drop(&mut self) { + unsafe { mdb_env_close(self.env_ptr.as_mut()) }; + let mut lock = OPENED_ENV.write().unwrap(); + debug_assert!(lock.remove(&self.path)); + } +} diff --git a/heed/src/env/encrypted.rs b/heed/src/env/env_open_options.rs similarity index 56% rename from heed/src/env/encrypted.rs rename to heed/src/env/env_open_options.rs index b5e9bd05..f7f5ce5f 100644 --- a/heed/src/env/encrypted.rs +++ b/heed/src/env/env_open_options.rs @@ -1,184 +1,50 @@ -use std::collections::hash_map::Entry; use std::ffi::CString; #[cfg(windows)] use std::ffi::OsStr; use std::io::ErrorKind::NotFound; #[cfg(unix)] use std::os::unix::ffi::OsStrExt; -use std::panic::catch_unwind; use std::path::Path; -use std::sync::Arc; -use std::{fmt, io, ptr}; - -use aead::generic_array::typenum::Unsigned; -use aead::{AeadCore, AeadMutInPlace, Key, KeyInit, Nonce, Tag}; -use synchronoise::SignalEvent; +use std::ptr::NonNull; +use std::{io, ptr}; +use super::env::Env; +use super::{canonicalize_path, OPENED_ENV}; #[cfg(windows)] use crate::env::OsStrExtLmdb as _; -use crate::env::{canonicalize_path, Env, EnvFlags, EnvInner, OPENED_ENV}; +use crate::mdb::error::mdb_result; use crate::mdb::ffi; -use crate::mdb::lmdb_error::mdb_result; -use crate::{Error, Result}; - -pub struct EnvEntry { - pub(super) env: Option, - pub(super) signal_event: Arc, - pub(super) options: SimplifiedEncryptedOpenOptions, -} - -/// A simplified version of the options that were used to open a given [`Env`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SimplifiedEncryptedOpenOptions { - /// The maximum size this [`Env`] with take in bytes or [`None`] if it was not specified. - pub map_size: Option, - /// The maximum number of concurrent readers or [`None`] if it was not specified. - pub max_readers: Option, - /// The maximum number of opened database or [`None`] if it was not specified. - pub max_dbs: Option, - /// The raw flags enabled for this [`Env`] or [`None`] if it was not specified. - pub flags: u32, -} - -impl From<&EnvOpenOptions> for SimplifiedEncryptedOpenOptions { - fn from(eoo: &EnvOpenOptions) -> SimplifiedEncryptedOpenOptions { - let EnvOpenOptions { encrypt: _, map_size, max_readers, max_dbs, flags } = eoo; - SimplifiedEncryptedOpenOptions { - map_size: *map_size, - max_readers: *max_readers, - max_dbs: *max_dbs, - flags: flags.bits(), - } - } -} +use crate::{EnvFlags, Error, Result}; /// Options and flags which can be used to configure how an environment is opened. -#[derive(Clone, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct EnvOpenOptions { - encrypt: Key, +pub struct EnvOpenOptions { map_size: Option, max_readers: Option, max_dbs: Option, flags: EnvFlags, } -impl EnvOpenOptions { - /// Creates a blank new set of options ready for configuration and specifies that - /// the [`Env`] will be encrypted using the `E` algorithm with the given `key`. - /// - /// You can find more compatible algorithms on [the RustCrypto/AEADs page](https://github.com/RustCrypto/AEADs#crates). - /// - /// Note that you cannot use any type of encryption algorithm as LMDB exposes a nonce of 16 bytes. - /// Heed makes sure to truncate it if necessary. - /// - /// As an example, XChaCha20 requires a 20 bytes long nonce. However, XChaCha20 is used to protect - /// against nonce misuse in systems that use randomly generated nonces i.e., to protect against - /// weak RNGs. There is no need to use this kind of algorithm in LMDB since LMDB nonces aren't - /// random and are guaranteed to be unique. - /// - /// ## Basic Example - /// - /// ``` - /// use std::fs; - /// use std::path::Path; - /// use argon2::Argon2; - /// use chacha20poly1305::{ChaCha20Poly1305, Key}; - /// use heed3_encryption::types::*; - /// use heed3_encryption::{EnvOpenOptions, Database}; - /// - /// # fn main() -> Result<(), Box> { - /// let env_path = Path::new("target").join("encrypt.mdb"); - /// let password = "This is the password that will be hashed by the argon2 algorithm"; - /// let salt = "The salt added to the password hashes to add more security when stored"; - /// - /// let _ = fs::remove_dir_all(&env_path); - /// fs::create_dir_all(&env_path)?; - /// - /// let mut key = Key::default(); - /// Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; - /// - /// // We open the environment - /// let mut options = EnvOpenOptions::::new_encrypted_with(key); - /// let env = unsafe { - /// options - /// .map_size(10 * 1024 * 1024) // 10MB - /// .max_dbs(3) - /// .open(&env_path)? - /// }; - /// - /// let key1 = "first-key"; - /// let val1 = "this is a secret info"; - /// let key2 = "second-key"; - /// let val2 = "this is another secret info"; - /// - /// // We create database and write secret values in it - /// let mut wtxn = env.write_txn()?; - /// let db: Database = env.create_database(&mut wtxn, Some("first"))?; - /// db.put(&mut wtxn, key1, val1)?; - /// db.put(&mut wtxn, key2, val2)?; - /// wtxn.commit()?; - /// # Ok(()) } - /// ``` - /// - /// ## Example Showing limitations - /// - /// ```compile_fail,E0499 - /// use std::fs; - /// use std::path::Path; - /// use argon2::Argon2; - /// use chacha20poly1305::{ChaCha20Poly1305, Key}; - /// use heed3_encryption::types::*; - /// use heed3_encryption::{EnvOpenOptions, Database}; - /// - /// # fn main() -> Result<(), Box> { - /// let env_path = Path::new("target").join("encrypt.mdb"); - /// let password = "This is the password that will be hashed by the argon2 algorithm"; - /// let salt = "The salt added to the password hashes to add more security when stored"; - /// - /// let _ = fs::remove_dir_all(&env_path); - /// fs::create_dir_all(&env_path)?; - /// - /// let mut key = Key::default(); - /// Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; - /// - /// // We open the environment - /// let mut options = EnvOpenOptions::::new_encrypted_with(key); - /// let env = unsafe { - /// options - /// .map_size(10 * 1024 * 1024) // 10MB - /// .max_dbs(3) - /// .open(&env_path)? - /// }; - /// - /// let key1 = "first-key"; - /// let key2 = "second-key"; - /// - /// // We create the database - /// let mut wtxn = env.write_txn()?; - /// let db: Database = env.create_database(&mut wtxn, Some("first"))?; - /// wtxn.commit()?; - /// - /// // Declare the read transaction as mutable because LMDB, when using encryption, - /// // does not allow keeping keys between reads due to the use of an internal cache. - /// let mut rtxn = env.read_txn()?; - /// let val1 = db.get(&mut rtxn, key1)?; - /// let val2 = db.get(&mut rtxn, key2)?; - /// - /// // This example won't compile because val1 cannot be used for too long. - /// let _force_keep = val1; - /// # Ok(()) } - /// ``` - pub fn new_encrypted_with(key: Key) -> EnvOpenOptions { +impl Default for EnvOpenOptions { + fn default() -> Self { + Self::new() + } +} + +impl EnvOpenOptions { + /// Creates a blank new set of options ready for configuration. + pub fn new() -> EnvOpenOptions { EnvOpenOptions { - encrypt: key, map_size: None, max_readers: None, max_dbs: None, flags: EnvFlags::empty(), } } +} +impl EnvOpenOptions { /// Set the size of the memory map to use for this environment. pub fn map_size(&mut self, size: usize) -> &mut Self { self.map_size = Some(size); @@ -280,6 +146,8 @@ impl EnvOpenOptions { /// [^7]: /// [^8]: pub unsafe fn open>(&self, path: P) -> Result { + /// TODO change the function description + /// and deduplicate the code let mut lock = OPENED_ENV.write().unwrap(); let path = match canonicalize_path(path.as_ref()) { @@ -297,179 +165,249 @@ impl EnvOpenOptions { Ok(path) => path, }; - let original_options = SimplifiedEncryptedOpenOptions::from(self); - match lock.entry(path) { - Entry::Occupied(entry) => { - let env = entry.get().env.clone().ok_or(Error::DatabaseClosing)?; - let options = entry.get().options.clone(); - if options == original_options { - Ok(env) - } else { - Err(Error::BadOpenOptions { env, options: original_options }) - } - } - Entry::Vacant(entry) => { - let path = entry.key(); - let path_str = CString::new(path.as_os_str().as_bytes()).unwrap(); - - unsafe { - let mut env: *mut ffi::MDB_env = ptr::null_mut(); - mdb_result(ffi::mdb_env_create(&mut env))?; + if lock.contains(&path) { + Err(Error::EnvAlreadyOpened) + } else { + let path_str = CString::new(path.as_os_str().as_bytes()).unwrap(); - let encrypt_key = crate::into_val(&self.encrypt); - mdb_result(ffi::mdb_env_set_encrypt( - env, - Some(encrypt_func_wrapper::), - &encrypt_key, - ::TagSize::U32, - ))?; + unsafe { + let mut env: *mut ffi::MDB_env = ptr::null_mut(); + mdb_result(ffi::mdb_env_create(&mut env))?; - if let Some(size) = self.map_size { - if size % page_size::get() != 0 { - let msg = format!( - "map size ({}) must be a multiple of the system page size ({})", - size, - page_size::get() - ); - return Err(Error::Io(io::Error::new( - io::ErrorKind::InvalidInput, - msg, - ))); - } - mdb_result(ffi::mdb_env_set_mapsize(env, size))?; - } - - if let Some(readers) = self.max_readers { - mdb_result(ffi::mdb_env_set_maxreaders(env, readers))?; + if let Some(size) = self.map_size { + if size % page_size::get() != 0 { + let msg = format!( + "map size ({}) must be a multiple of the system page size ({})", + size, + page_size::get() + ); + return Err(Error::Io(io::Error::new(io::ErrorKind::InvalidInput, msg))); } + mdb_result(ffi::mdb_env_set_mapsize(env, size))?; + } - if let Some(dbs) = self.max_dbs { - mdb_result(ffi::mdb_env_set_maxdbs(env, dbs))?; - } + if let Some(readers) = self.max_readers { + mdb_result(ffi::mdb_env_set_maxreaders(env, readers))?; + } - // When the `read-txn-no-tls` feature is enabled, we must force LMDB - // to avoid using the thread local storage, this way we allow users - // to use references of RoTxn between threads safely. - let flags = if cfg!(feature = "read-txn-no-tls") { - self.flags | EnvFlags::NO_TLS - } else { - self.flags - }; + if let Some(dbs) = self.max_dbs { + mdb_result(ffi::mdb_env_set_maxdbs(env, dbs))?; + } - let result = - mdb_result(ffi::mdb_env_open(env, path_str.as_ptr(), flags.bits(), 0o600)); + // When the `read-txn-no-tls` feature is enabled, we must force LMDB + // to avoid using the thread local storage, this way we allow users + // to use references of RoTxn between threads safely. + let flags = if cfg!(feature = "read-txn-no-tls") { + // TODO make this a ZST flag on the Env and on RoTxn (make them Send when we can) + self.flags | EnvFlags::NO_TLS + } else { + self.flags + }; - match result { - Ok(()) => { - let signal_event = Arc::new(SignalEvent::manual(false)); - let inner = EnvInner { env, path: path.clone() }; - let env = Env(Arc::new(inner)); - let cache_entry = EnvEntry { - env: Some(env.clone()), - options: original_options, - signal_event, - }; - entry.insert(cache_entry); - Ok(env) - } - Err(e) => { - ffi::mdb_env_close(env); - Err(e.into()) - } + let result = ffi::mdb_env_open(env, path_str.as_ptr(), flags.bits(), 0o600); + match mdb_result(result) { + Ok(()) => { + let env_ptr = NonNull::new(env).unwrap(); + debug_assert!(lock.insert(path.clone())); + Ok(Env::new(env_ptr, path)) + } + Err(e) => { + ffi::mdb_env_close(env); + Err(e.into()) } } } } } -} -impl fmt::Debug for EnvOpenOptions { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let EnvOpenOptions { encrypt: _, map_size, max_readers, max_dbs, flags } = self; - f.debug_struct("EnvOpenOptions") - .field("map_size", &map_size) - .field("max_readers", &max_readers) - .field("max_dbs", &max_dbs) - .field("flags", &flags) - .finish() - } -} + /// Creates a blank new set of options ready for configuration and specifies that + /// the [`Env`] will be encrypted using the `E` algorithm with the given `key`. + /// + /// You can find more compatible algorithms on [the RustCrypto/AEADs page](https://github.com/RustCrypto/AEADs#crates). + /// + /// Note that you cannot use any type of encryption algorithm as LMDB exposes a nonce of 16 bytes. + /// Heed makes sure to truncate it if necessary. + /// + /// As an example, XChaCha20 requires a 20 bytes long nonce. However, XChaCha20 is used to protect + /// against nonce misuse in systems that use randomly generated nonces i.e., to protect against + /// weak RNGs. There is no need to use this kind of algorithm in LMDB since LMDB nonces aren't + /// random and are guaranteed to be unique. + /// + /// ## Basic Example + /// + /// ``` + /// use std::fs; + /// use std::path::Path; + /// use argon2::Argon2; + /// use chacha20poly1305::{ChaCha20Poly1305, Key}; + /// use heed3_encryption::types::*; + /// use heed3_encryption::{EnvOpenOptions, Database}; + /// + /// # fn main() -> Result<(), Box> { + /// let env_path = Path::new("target").join("encrypt.mdb"); + /// let password = "This is the password that will be hashed by the argon2 algorithm"; + /// let salt = "The salt added to the password hashes to add more security when stored"; + /// + /// let _ = fs::remove_dir_all(&env_path); + /// fs::create_dir_all(&env_path)?; + /// + /// let mut key = Key::default(); + /// Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; + /// + /// // We open the environment + /// let mut options = EnvOpenOptions::::new_encrypted_with(key); + /// let env = unsafe { + /// options + /// .map_size(10 * 1024 * 1024) // 10MB + /// .max_dbs(3) + /// .open(&env_path)? + /// }; + /// + /// let key1 = "first-key"; + /// let val1 = "this is a secret info"; + /// let key2 = "second-key"; + /// let val2 = "this is another secret info"; + /// + /// // We create database and write secret values in it + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("first"))?; + /// db.put(&mut wtxn, key1, val1)?; + /// db.put(&mut wtxn, key2, val2)?; + /// wtxn.commit()?; + /// # Ok(()) } + /// ``` + /// + /// ## Example Showing limitations + /// + /// ```compile_fail,E0499 + /// use std::fs; + /// use std::path::Path; + /// use argon2::Argon2; + /// use chacha20poly1305::{ChaCha20Poly1305, Key}; + /// use heed3_encryption::types::*; + /// use heed3_encryption::{EnvOpenOptions, Database}; + /// + /// # fn main() -> Result<(), Box> { + /// let env_path = Path::new("target").join("encrypt.mdb"); + /// let password = "This is the password that will be hashed by the argon2 algorithm"; + /// let salt = "The salt added to the password hashes to add more security when stored"; + /// + /// let _ = fs::remove_dir_all(&env_path); + /// fs::create_dir_all(&env_path)?; + /// + /// let mut key = Key::default(); + /// Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; + /// + /// // We open the environment + /// let mut options = EnvOpenOptions::::new_encrypted_with(key); + /// let env = unsafe { + /// options + /// .map_size(10 * 1024 * 1024) // 10MB + /// .max_dbs(3) + /// .open(&env_path)? + /// }; + /// + /// let key1 = "first-key"; + /// let key2 = "second-key"; + /// + /// // We create the database + /// let mut wtxn = env.write_txn()?; + /// let db: Database = env.create_database(&mut wtxn, Some("first"))?; + /// wtxn.commit()?; + /// + /// // Declare the read transaction as mutable because LMDB, when using encryption, + /// // does not allow keeping keys between reads due to the use of an internal cache. + /// let mut rtxn = env.read_txn()?; + /// let val1 = db.get(&mut rtxn, key1)?; + /// let val2 = db.get(&mut rtxn, key2)?; + /// + /// // This example won't compile because val1 cannot be used for too long. + /// let _force_keep = val1; + /// # Ok(()) } + /// ``` + #[cfg(master3)] + pub unsafe fn open_encrypted(&self, key: Key, path: P) -> Result + where + E: AeadMutInPlace + KeyInit, + P: AsRef, + { + let mut lock = OPENED_ENV.write().unwrap(); -fn encrypt( - key: &[u8], - nonce: &[u8], - aad: &[u8], - plaintext: &[u8], - chipertext_out: &mut [u8], - auth_out: &mut [u8], -) -> aead::Result<()> { - chipertext_out.copy_from_slice(plaintext); - let key: &Key = key.try_into().unwrap(); - let nonce: &Nonce = if nonce.len() >= A::NonceSize::USIZE { - nonce[..A::NonceSize::USIZE].into() - } else { - return Err(aead::Error); - }; - let mut aead = A::new(key); - let tag = aead.encrypt_in_place_detached(nonce, aad, chipertext_out)?; - auth_out.copy_from_slice(&tag); - Ok(()) -} + let path = match canonicalize_path(path.as_ref()) { + Err(err) => { + if err.kind() == NotFound && self.flags.contains(EnvFlags::NO_SUB_DIR) { + let path = path.as_ref(); + match path.parent().zip(path.file_name()) { + Some((dir, file_name)) => canonicalize_path(dir)?.join(file_name), + None => return Err(err.into()), + } + } else { + return Err(err.into()); + } + } + Ok(path) => path, + }; -fn decrypt( - key: &[u8], - nonce: &[u8], - aad: &[u8], - chipher_text: &[u8], - output: &mut [u8], - auth_in: &[u8], -) -> aead::Result<()> { - output.copy_from_slice(chipher_text); - let key: &Key = key.try_into().unwrap(); - let nonce: &Nonce = if nonce.len() >= A::NonceSize::USIZE { - nonce[..A::NonceSize::USIZE].into() - } else { - return Err(aead::Error); - }; - let tag: &Tag = auth_in.try_into().unwrap(); - let mut aead = A::new(key); - aead.decrypt_in_place_detached(nonce, aad, output, tag) -} + if lock.contains(&path) { + Err(Error::EnvAlreadyOpened) + } else { + let path_str = CString::new(path.as_os_str().as_bytes()).unwrap(); -/// The wrapper function that is called by LMDB that directly calls -/// the Rust idiomatic function internally. -unsafe extern "C" fn encrypt_func_wrapper( - src: *const ffi::MDB_val, - dst: *mut ffi::MDB_val, - key_ptr: *const ffi::MDB_val, - encdec: i32, -) -> i32 { - let result = catch_unwind(|| { - let input = std::slice::from_raw_parts((*src).mv_data as *const u8, (*src).mv_size); - let output = std::slice::from_raw_parts_mut((*dst).mv_data as *mut u8, (*dst).mv_size); - let key = std::slice::from_raw_parts((*key_ptr).mv_data as *const u8, (*key_ptr).mv_size); - let iv = std::slice::from_raw_parts( - (*key_ptr.offset(1)).mv_data as *const u8, - (*key_ptr.offset(1)).mv_size, - ); - let auth = std::slice::from_raw_parts_mut( - (*key_ptr.offset(2)).mv_data as *mut u8, - (*key_ptr.offset(2)).mv_size, - ); + unsafe { + let mut env: *mut ffi::MDB_env = ptr::null_mut(); + mdb_result(ffi::mdb_env_create(&mut env))?; - let aad = []; - let nonce = iv; - let result = if encdec == 1 { - encrypt::(&key, nonce, &aad, input, output, auth) - } else { - decrypt::(&key, nonce, &aad, input, output, auth) - }; + let encrypt_key = crate::into_val(&self.encrypt); + mdb_result(ffi::mdb_env_set_encrypt( + env, + Some(encrypt_func_wrapper::), + &encrypt_key, + ::TagSize::U32, + ))?; - result.is_err() as i32 - }); + if let Some(size) = self.map_size { + if size % page_size::get() != 0 { + let msg = format!( + "map size ({}) must be a multiple of the system page size ({})", + size, + page_size::get() + ); + return Err(Error::Io(io::Error::new(io::ErrorKind::InvalidInput, msg))); + } + mdb_result(ffi::mdb_env_set_mapsize(env, size))?; + } + + if let Some(readers) = self.max_readers { + mdb_result(ffi::mdb_env_set_maxreaders(env, readers))?; + } - match result { - Ok(out) => out, - Err(_) => 1, + if let Some(dbs) = self.max_dbs { + mdb_result(ffi::mdb_env_set_maxdbs(env, dbs))?; + } + + // When the `read-txn-no-tls` feature is enabled, we must force LMDB + // to avoid using the thread local storage, this way we allow users + // to use references of RoTxn between threads safely. + let flags = if cfg!(feature = "read-txn-no-tls") { + // TODO make this a ZST flag on the Env and on RoTxn (make them Send when we can) + self.flags | EnvFlags::NO_TLS + } else { + self.flags + }; + + let result = ffi::mdb_env_open(env, path_str.as_ptr(), flags.bits(), 0o600); + match mdb_result(result) { + Ok(()) => { + let env_ptr = NonNull::new(env).unwrap(); + debug_assert!(lock.insert(path.clone())); + Ok(Env::new(env_ptr, path)) + } + Err(e) => { + ffi::mdb_env_close(env); + Err(e.into()) + } + } + } + } } } diff --git a/heed/src/env/mod.rs b/heed/src/env/mod.rs index 30c83fe0..adb2bb6a 100644 --- a/heed/src/env/mod.rs +++ b/heed/src/env/mod.rs @@ -1,53 +1,51 @@ -use std::any::TypeId; use std::cmp::Ordering; -use std::collections::hash_map::HashMap; -use std::ffi::{c_void, CString}; +use std::collections::HashSet; +use std::ffi::c_void; use std::fs::{File, Metadata}; +use std::io; #[cfg(unix)] use std::os::unix::io::{AsRawFd, BorrowedFd, RawFd}; use std::panic::catch_unwind; use std::path::{Path, PathBuf}; use std::process::abort; use std::ptr::NonNull; -use std::sync::{Arc, RwLock}; +use std::sync::{LazyLock, RwLock}; use std::time::Duration; #[cfg(windows)] use std::{ ffi::OsStr, - os::windows::io::{AsRawHandle, BorrowedHandle, RawHandle}, + os::windows::io::{AsRawHandle as _, BorrowedHandle, RawHandle}, }; -use std::{fmt, io, mem, ptr}; use heed_traits::{Comparator, LexicographicComparator}; -use once_cell::sync::Lazy; -pub use sub_env::EnvOpenOptions; -use synchronoise::event::SignalEvent; -use crate::cursor::MoveOperation; -use crate::database::DatabaseOpenOptions; -use crate::mdb::error::mdb_result; use crate::mdb::ffi; -use crate::mdb::lmdb_flags::AllDatabaseFlags; -use crate::{Database, EnvFlags, Error, Result, RoCursor, RoTxn, RwTxn, Unspecified}; -#[cfg(not(all(master3, encryption)))] -mod clear; -#[cfg(not(all(master3, encryption)))] -use clear as sub_env; +#[cfg(master3)] +mod encrypted_env; +mod env; +mod env_open_options; -#[cfg(all(master3, encryption))] -mod encrypted; -#[cfg(all(master3, encryption))] -use encrypted as sub_env; -#[cfg(all(master3, encryption))] -pub use encrypted::SimplifiedEncryptedOpenOptions; +/// Records the current list of opened environments for tracking purposes. The canonical +/// path of an environment is removed when either an `Env` or `EncryptedEnv` is closed. +static OPENED_ENV: LazyLock>> = LazyLock::new(RwLock::default); -/// The list of opened environments, the value is an optional environment, it is None -/// when someone asks to close the environment, closing is a two-phase step, to make sure -/// no one tries to open the same environment between these two phases. -/// -/// Trying to open a None marked environment returns an error to the user trying to open it. -static OPENED_ENV: Lazy>> = Lazy::new(RwLock::default); +/// Contains information about the environment. +#[derive(Debug, Clone, Copy)] +pub struct EnvInfo { + /// Address of the map, if fixed. + pub map_addr: *mut c_void, + /// Size of the data memory map. + pub map_size: usize, + /// ID of the last used page. + pub last_page_number: usize, + /// ID of the last committed transaction. + pub last_txn_id: usize, + /// Maximum number of reader slots in the environment. + pub maximum_number_of_readers: u32, + /// Number of reader slots used in the environment. + pub number_of_readers: u32, +} // Thanks to the mozilla/rkv project // Workaround the UNC path on Windows, see https://github.com/rust-lang/rust/issues/42869. @@ -104,49 +102,6 @@ unsafe fn metadata_from_fd(raw_fd: RawHandle) -> io::Result { File::from(owned).metadata() } -/// Returns a struct that allows to wait for the effective closing of an environment. -pub fn env_closing_event>(path: P) -> Option { - let lock = OPENED_ENV.read().unwrap(); - lock.get(path.as_ref()).map(|e| EnvClosingEvent(e.signal_event.clone())) -} - -/// An environment handle constructed by using [`EnvOpenOptions`]. -#[derive(Clone)] -pub struct Env(Arc); - -impl fmt::Debug for Env { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let EnvInner { env: _, path } = self.0.as_ref(); - f.debug_struct("Env").field("path", &path.display()).finish_non_exhaustive() - } -} - -struct EnvInner { - env: *mut ffi::MDB_env, - path: PathBuf, -} - -unsafe impl Send for EnvInner {} - -unsafe impl Sync for EnvInner {} - -impl Drop for EnvInner { - fn drop(&mut self) { - let mut lock = OPENED_ENV.write().unwrap(); - - match lock.remove(&self.path) { - None => panic!("It seems another env closed this env before"), - Some(sub_env::EnvEntry { signal_event, .. }) => { - unsafe { - ffi::mdb_env_close(self.env); - } - // We signal to all the waiters that the env is closed now. - signal_event.signal(); - } - } - } -} - /// A helper function that transforms the LMDB types into Rust types (`MDB_val` into slices) /// and vice versa, the Rust types into C types (`Ordering` into an integer). /// @@ -221,50 +176,6 @@ impl LexicographicComparator for DefaultComparator { } } -/// A representation of LMDB's `MDB_INTEGERKEY` comparator behavior. -/// -/// This enum is used to indicate a table should be sorted by the keys numeric -/// value in native byte order. When a [`Database`] is created or opened with -/// [`IntegerComparator`], it signifies that the comparator should not be explicitly -/// set via [`ffi::mdb_set_compare`], instead the flag [`AllDatabaseFlags::INTEGER_KEY`] -/// is set on the table. -/// -/// This can only be used on certain types: either `u32` or `usize`. The keys must all be of the same size. -pub enum IntegerComparator {} -impl Comparator for IntegerComparator { - fn compare(a: &[u8], b: &[u8]) -> Ordering { - #[cfg(target_endian = "big")] - return a.cmp(b); - - #[cfg(target_endian = "little")] - { - let len = a.len(); - - for i in (0..len).rev() { - match a[i].cmp(&b[i]) { - Ordering::Equal => continue, - other => return other, - } - } - - Ordering::Equal - } - } -} - -/// Whether to perform compaction while copying an environment. -#[derive(Debug, Copy, Clone)] -pub enum CompactionOption { - /// Omit free pages and sequentially renumber all pages in output. - /// - /// This option consumes more CPU and runs more slowly than the default. - /// Currently it fails if the environment has suffered a page leak. - Enabled, - - /// Copy everything without taking any special action about free pages. - Disabled, -} - /// Whether to enable or disable flags in [`Env::set_flags`]. #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum FlagSetMode { @@ -285,826 +196,3 @@ impl FlagSetMode { } } } - -impl Env { - pub(crate) fn env_mut_ptr(&self) -> *mut ffi::MDB_env { - self.0.env - } - - /// The size of the data file on disk. - /// - /// # Example - /// - /// ``` - /// use heed::EnvOpenOptions; - /// - /// # fn main() -> Result<(), Box> { - /// let dir = tempfile::tempdir()?; - /// let size_in_bytes = 1024 * 1024; - /// let env = unsafe { EnvOpenOptions::new().map_size(size_in_bytes).open(dir.path())? }; - /// - /// let actual_size = env.real_disk_size()? as usize; - /// assert!(actual_size < size_in_bytes); - /// # Ok(()) } - /// ``` - pub fn real_disk_size(&self) -> Result { - let mut fd = mem::MaybeUninit::uninit(); - unsafe { mdb_result(ffi::mdb_env_get_fd(self.env_mut_ptr(), fd.as_mut_ptr()))? }; - let fd = unsafe { fd.assume_init() }; - let metadata = unsafe { metadata_from_fd(fd)? }; - Ok(metadata.len()) - } - - /// Return the raw flags the environment was opened with. - /// - /// Returns `None` if the environment flags are different from the [`EnvFlags`] set. - pub fn flags(&self) -> Result> { - self.get_flags().map(EnvFlags::from_bits) - } - - /// Enable or disable the environment's currently active [`EnvFlags`]. - /// - /// ``` - /// use std::fs; - /// use std::path::Path; - /// use heed::{EnvOpenOptions, Database, EnvFlags, FlagSetMode}; - /// use heed::types::*; - /// - /// # fn main() -> Result<(), Box> { - /// fs::create_dir_all(Path::new("target").join("database.mdb"))?; - /// let mut env_builder = EnvOpenOptions::new(); - /// let dir = tempfile::tempdir().unwrap(); - /// let env = unsafe { env_builder.open(dir.path())? }; - /// - /// // Env was opened without flags. - /// assert_eq!(env.get_flags().unwrap(), EnvFlags::empty().bits()); - /// - /// // Enable a flag after opening. - /// unsafe { env.set_flags(EnvFlags::NO_SYNC, FlagSetMode::Enable).unwrap(); } - /// assert_eq!(env.get_flags().unwrap(), EnvFlags::NO_SYNC.bits()); - /// - /// // Disable a flag after opening. - /// unsafe { env.set_flags(EnvFlags::NO_SYNC, FlagSetMode::Disable).unwrap(); } - /// assert_eq!(env.get_flags().unwrap(), EnvFlags::empty().bits()); - /// # Ok(()) } - /// ``` - /// - /// # Safety - /// - /// It is unsafe to use unsafe LMDB flags such as `NO_SYNC`, `NO_META_SYNC`, or `NO_LOCK`. - /// - /// LMDB also requires that only 1 thread calls this function at any given moment. - /// Neither `heed` or LMDB check for this condition, so the caller must ensure it explicitly. - pub unsafe fn set_flags(&self, flags: EnvFlags, mode: FlagSetMode) -> Result<()> { - // safety: caller must ensure no other thread is calling this function. - // - mdb_result(unsafe { - ffi::mdb_env_set_flags( - self.env_mut_ptr(), - flags.bits(), - mode.as_mdb_env_set_flags_input(), - ) - }) - .map_err(Into::into) - } - - /// Return the raw flags the environment is currently set with. - pub fn get_flags(&self) -> Result { - let mut flags = mem::MaybeUninit::uninit(); - unsafe { mdb_result(ffi::mdb_env_get_flags(self.env_mut_ptr(), flags.as_mut_ptr()))? }; - let flags = unsafe { flags.assume_init() }; - Ok(flags) - } - - /// Returns some basic informations about this environment. - pub fn info(&self) -> EnvInfo { - let mut raw_info = mem::MaybeUninit::uninit(); - unsafe { ffi::mdb_env_info(self.0.env, raw_info.as_mut_ptr()) }; - let raw_info = unsafe { raw_info.assume_init() }; - - EnvInfo { - map_addr: raw_info.me_mapaddr, - map_size: raw_info.me_mapsize, - last_page_number: raw_info.me_last_pgno, - last_txn_id: raw_info.me_last_txnid, - maximum_number_of_readers: raw_info.me_maxreaders, - number_of_readers: raw_info.me_numreaders, - } - } - - /// Returns the size used by all the databases in the environment without the free pages. - /// - /// It is crucial to configure [`EnvOpenOptions::max_dbs`] with a sufficiently large value - /// before invoking this function. All databases within the environment will be opened - /// and remain so. - pub fn non_free_pages_size(&self) -> Result { - let compute_size = |stat: ffi::MDB_stat| { - (stat.ms_leaf_pages + stat.ms_branch_pages + stat.ms_overflow_pages) as u64 - * stat.ms_psize as u64 - }; - - let mut size = 0; - - let mut stat = mem::MaybeUninit::uninit(); - unsafe { mdb_result(ffi::mdb_env_stat(self.env_mut_ptr(), stat.as_mut_ptr()))? }; - let stat = unsafe { stat.assume_init() }; - size += compute_size(stat); - - let rtxn = self.read_txn()?; - // Open the main database - let dbi = self.raw_open_dbi::(rtxn.txn.unwrap(), None, 0)?; - - // We're going to iterate on the unnamed database - let mut cursor = RoCursor::new(&rtxn, dbi)?; - - while let Some((key, _value)) = cursor.move_on_next(MoveOperation::NoDup)? { - if key.contains(&0) { - continue; - } - - let key = String::from_utf8(key.to_vec()).unwrap(); - // Calling `ffi::db_stat` on a database instance does not involve key comparison - // in LMDB, so it's safe to specify a noop key compare function for it. - if let Ok(dbi) = - self.raw_open_dbi::(rtxn.txn.unwrap(), Some(&key), 0) - { - let mut stat = mem::MaybeUninit::uninit(); - let mut txn = rtxn.txn.unwrap(); - unsafe { mdb_result(ffi::mdb_stat(txn.as_mut(), dbi, stat.as_mut_ptr()))? }; - let stat = unsafe { stat.assume_init() }; - size += compute_size(stat); - } - } - - Ok(size) - } - - /// Options and flags which can be used to configure how a [`Database`] is opened. - pub fn database_options(&self) -> DatabaseOpenOptions { - DatabaseOpenOptions::new(self) - } - - /// Opens a typed database that already exists in this environment. - /// - /// If the database was previously opened in this program run, types will be checked. - /// - /// ## Important Information - /// - /// LMDB has an important restriction on the unnamed database when named ones are opened. - /// The names of the named databases are stored as keys in the unnamed one and are immutable, - /// and these keys can only be read and not written. - /// - /// ## LMDB read-only access of existing database - /// - /// In the case of accessing a database in a read-only manner from another process - /// where you wrote, you might need to manually call [`RoTxn::commit`] to get metadata - /// and the database handles opened and shared with the global [`Env`] handle. - /// - /// If not done, you might raise `Io(Os { code: 22, kind: InvalidInput, message: "Invalid argument" })` - /// known as `EINVAL`. - pub fn open_database( - &self, - rtxn: &RoTxn, - name: Option<&str>, - ) -> Result>> - where - KC: 'static, - DC: 'static, - { - let mut options = self.database_options().types::(); - if let Some(name) = name { - options.name(name); - } - options.open(rtxn) - } - - /// Creates a typed database that can already exist in this environment. - /// - /// If the database was previously opened during this program run, types will be checked. - /// - /// ## Important Information - /// - /// LMDB has an important restriction on the unnamed database when named ones are opened. - /// The names of the named databases are stored as keys in the unnamed one and are immutable, - /// and these keys can only be read and not written. - pub fn create_database( - &self, - wtxn: &mut RwTxn, - name: Option<&str>, - ) -> Result> - where - KC: 'static, - DC: 'static, - { - let mut options = self.database_options().types::(); - if let Some(name) = name { - options.name(name); - } - options.create(wtxn) - } - - pub(crate) fn raw_init_database( - &self, - raw_txn: NonNull, - name: Option<&str>, - mut flags: AllDatabaseFlags, - ) -> Result { - if TypeId::of::() == TypeId::of::() { - flags.insert(AllDatabaseFlags::INTEGER_KEY); - } - - match self.raw_open_dbi::(raw_txn, name, flags.bits()) { - Ok(dbi) => Ok(dbi), - Err(e) => Err(e.into()), - } - } - - fn raw_open_dbi( - &self, - mut raw_txn: NonNull, - name: Option<&str>, - flags: u32, - ) -> std::result::Result { - let mut dbi = 0; - let name = name.map(|n| CString::new(n).unwrap()); - let name_ptr = match name { - Some(ref name) => name.as_bytes_with_nul().as_ptr() as *const _, - None => ptr::null(), - }; - - // safety: The name cstring is cloned by LMDB, we can drop it after. - // If a read-only is used with the MDB_CREATE flag, LMDB will throw an error. - unsafe { - mdb_result(ffi::mdb_dbi_open(raw_txn.as_mut(), name_ptr, flags, &mut dbi))?; - let cmp_type_id = TypeId::of::(); - - if cmp_type_id != TypeId::of::() - && cmp_type_id != TypeId::of::() - { - mdb_result(ffi::mdb_set_compare( - raw_txn.as_mut(), - dbi, - Some(custom_key_cmp_wrapper::), - ))?; - } - }; - - Ok(dbi) - } - - /// Create a transaction with read and write access for use with the environment. - /// - /// ## LMDB Limitations - /// - /// Only one [`RwTxn`] may exist simultaneously in the current environment. - /// If another write transaction is initiated, while another write transaction exists - /// the thread initiating the new one will wait on a mutex upon completion of the previous - /// transaction. - pub fn write_txn(&self) -> Result { - RwTxn::new(self) - } - - /// Create a nested transaction with read and write access for use with the environment. - /// - /// The new transaction will be a nested transaction, with the transaction indicated by parent - /// as its parent. Transactions may be nested to any level. - /// - /// A parent transaction and its cursors may not issue any other operations than _commit_ and - /// _abort_ while it has active child transactions. - pub fn nested_write_txn<'p>(&'p self, parent: &'p mut RwTxn) -> Result> { - RwTxn::nested(self, parent) - } - - /// Create a transaction with read-only access for use with the environment. - /// - /// You can make this transaction `Send`able between threads by - /// using the `read-txn-no-tls` crate feature. - /// See [`Self::static_read_txn`] if you want the txn to own the environment. - /// - /// ## LMDB Limitations - /// - /// It's possible to have multiple read transactions in the same environment - /// while there is a write transaction ongoing. - /// - /// But read transactions prevent reuse of pages freed by newer write transactions, - /// thus the database can grow quickly. Write transactions prevent other write transactions, - /// since writes are serialized. - /// - /// So avoid long-lived read transactions. - /// - /// ## Errors - /// - /// * [`crate::MdbError::Panic`]: A fatal error occurred earlier, and the environment must be shut down - /// * [`crate::MdbError::MapResized`]: Another process wrote data beyond this [`Env`] mapsize and this env - /// map must be resized - /// * [`crate::MdbError::ReadersFull`]: a read-only transaction was requested, and the reader lock table is - /// full - pub fn read_txn(&self) -> Result { - RoTxn::new(self) - } - - /// Create a transaction with read-only access for use with the environment. - /// Contrary to [`Self::read_txn`], this version **owns** the environment, which - /// means you won't be able to close the environment while this transaction is alive. - /// - /// You can make this transaction `Send`able between threads by - /// using the `read-txn-no-tls` crate feature. - /// - /// ## LMDB Limitations - /// - /// It's possible to have multiple read transactions in the same environment - /// while there is a write transaction ongoing. - /// - /// But read transactions prevent reuse of pages freed by newer write transactions, - /// thus the database can grow quickly. Write transactions prevent other write transactions, - /// since writes are serialized. - /// - /// So avoid long-lived read transactions. - /// - /// ## Errors - /// - /// * [`crate::MdbError::Panic`]: A fatal error occurred earlier, and the environment must be shut down - /// * [`crate::MdbError::MapResized`]: Another process wrote data beyond this [`Env`] mapsize and this env - /// map must be resized - /// * [`crate::MdbError::ReadersFull`]: a read-only transaction was requested, and the reader lock table is - /// full - pub fn static_read_txn(self) -> Result> { - RoTxn::static_read_txn(self) - } - - /// Copy an LMDB environment to the specified path, with options. - /// - /// This function may be used to make a backup of an existing environment. - /// No lockfile is created, since it gets recreated at need. - pub fn copy_to_file>(&self, path: P, option: CompactionOption) -> Result { - let file = File::options().create_new(true).write(true).open(&path)?; - let fd = get_file_fd(&file); - - unsafe { self.copy_to_fd(fd, option)? }; - - // We reopen the file to make sure the cursor is at the start, - // even a seek to start doesn't work properly. - let file = File::open(path)?; - - Ok(file) - } - - /// Copy an LMDB environment to the specified file descriptor, with compaction option. - /// - /// This function may be used to make a backup of an existing environment. - /// No lockfile is created, since it gets recreated at need. - /// - /// # Safety - /// - /// The [`ffi::mdb_filehandle_t`] must have already been opened for Write access. - pub unsafe fn copy_to_fd( - &self, - fd: ffi::mdb_filehandle_t, - option: CompactionOption, - ) -> Result<()> { - let flags = if let CompactionOption::Enabled = option { ffi::MDB_CP_COMPACT } else { 0 }; - mdb_result(ffi::mdb_env_copyfd2(self.0.env, fd, flags))?; - Ok(()) - } - - /// Flush the data buffers to disk. - pub fn force_sync(&self) -> Result<()> { - unsafe { mdb_result(ffi::mdb_env_sync(self.0.env, 1))? } - Ok(()) - } - - /// Returns the canonicalized path where this env lives. - pub fn path(&self) -> &Path { - &self.0.path - } - - /// Returns an `EnvClosingEvent` that can be used to wait for the closing event, - /// multiple threads can wait on this event. - /// - /// Make sure that you drop all the copies of `Env`s you have, env closing are triggered - /// when all references are dropped, the last one will eventually close the environment. - pub fn prepare_for_closing(self) -> EnvClosingEvent { - let mut lock = OPENED_ENV.write().unwrap(); - match lock.get_mut(self.path()) { - None => panic!("cannot find the env that we are trying to close"), - Some(sub_env::EnvEntry { env, signal_event, .. }) => { - // We remove the env from the global list and replace it with a None. - let _env = env.take(); - let signal_event = signal_event.clone(); - - // we must make sure we release the lock before we drop the env - // as the drop of the EnvInner also tries to lock the OPENED_ENV - // global and we don't want to trigger a dead-lock. - drop(lock); - - EnvClosingEvent(signal_event) - } - } - } - - /// Check for stale entries in the reader lock table and clear them. - /// - /// Returns the number of stale readers cleared. - pub fn clear_stale_readers(&self) -> Result { - let mut dead: i32 = 0; - unsafe { mdb_result(ffi::mdb_reader_check(self.0.env, &mut dead))? } - // safety: The reader_check function asks for an i32, initialize it to zero - // and never decrements it. It is safe to use either an u32 or u64 (usize). - Ok(dead as usize) - } - - /// Resize the memory map to a new size. - /// - /// # Safety - /// - /// According to the [LMDB documentation](http://www.lmdb.tech/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5), - /// it is okay to call `mdb_env_set_mapsize` for an open environment as long as no transactions are active, - /// but the library does not check for this condition, so the caller must ensure it explicitly. - pub unsafe fn resize(&self, new_size: usize) -> Result<()> { - if new_size % page_size::get() != 0 { - let msg = format!( - "map size ({}) must be a multiple of the system page size ({})", - new_size, - page_size::get() - ); - return Err(Error::Io(io::Error::new(io::ErrorKind::InvalidInput, msg))); - } - mdb_result(unsafe { ffi::mdb_env_set_mapsize(self.env_mut_ptr(), new_size) }) - .map_err(Into::into) - } - - /// Get the maximum size of keys and MDB_DUPSORT data we can write. - /// - /// Depends on the compile-time constant MDB_MAXKEYSIZE. Default 511 - pub fn max_key_size(&self) -> usize { - let maxsize: i32 = unsafe { ffi::mdb_env_get_maxkeysize(self.env_mut_ptr()) }; - maxsize as usize - } -} - -/// Contains information about the environment. -#[derive(Debug, Clone, Copy)] -pub struct EnvInfo { - /// Address of the map, if fixed. - pub map_addr: *mut c_void, - /// Size of the data memory map. - pub map_size: usize, - /// ID of the last used page. - pub last_page_number: usize, - /// ID of the last committed transaction. - pub last_txn_id: usize, - /// Maximum number of reader slots in the environment. - pub maximum_number_of_readers: u32, - /// Number of reader slots used in the environment. - pub number_of_readers: u32, -} - -/// A structure that can be used to wait for the closing event. -/// Multiple threads can wait on this event. -#[derive(Clone)] -pub struct EnvClosingEvent(Arc); - -impl EnvClosingEvent { - /// Blocks this thread until the environment is effectively closed. - /// - /// # Safety - /// - /// Make sure that you don't have any copy of the environment in the thread - /// that is waiting for a close event. If you do, you will have a deadlock. - pub fn wait(&self) { - self.0.wait() - } - - /// Blocks this thread until either the environment has been closed - /// or until the timeout elapses. Returns `true` if the environment - /// has been effectively closed. - pub fn wait_timeout(&self, timeout: Duration) -> bool { - self.0.wait_timeout(timeout) - } -} - -impl fmt::Debug for EnvClosingEvent { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("EnvClosingEvent").finish() - } -} - -#[cfg(test)] -mod tests { - use std::io::ErrorKind; - use std::time::Duration; - use std::{fs, thread}; - - use crate::types::*; - use crate::{env_closing_event, EnvOpenOptions, Error}; - - #[test] - fn close_env() { - let dir = tempfile::tempdir().unwrap(); - let env = unsafe { - EnvOpenOptions::new() - .map_size(10 * 1024 * 1024) // 10MB - .max_dbs(30) - .open(dir.path()) - .unwrap() - }; - - // Force a thread to keep the env for 1 second. - let env_cloned = env.clone(); - thread::spawn(move || { - let _env = env_cloned; - thread::sleep(Duration::from_secs(1)); - }); - - let mut wtxn = env.write_txn().unwrap(); - let db = env.create_database::(&mut wtxn, None).unwrap(); - wtxn.commit().unwrap(); - - // Create an ordered list of keys... - let mut wtxn = env.write_txn().unwrap(); - db.put(&mut wtxn, "hello", "hello").unwrap(); - db.put(&mut wtxn, "world", "world").unwrap(); - - let mut iter = db.iter(&wtxn).unwrap(); - assert_eq!(iter.next().transpose().unwrap(), Some(("hello", "hello"))); - assert_eq!(iter.next().transpose().unwrap(), Some(("world", "world"))); - assert_eq!(iter.next().transpose().unwrap(), None); - drop(iter); - - wtxn.commit().unwrap(); - - let signal_event = env.prepare_for_closing(); - - eprintln!("waiting for the env to be closed"); - signal_event.wait(); - eprintln!("env closed successfully"); - - // Make sure we don't have a reference to the env - assert!(env_closing_event(dir.path()).is_none()); - } - - #[test] - fn reopen_env_with_different_options_is_err() { - let dir = tempfile::tempdir().unwrap(); - let _env = unsafe { - EnvOpenOptions::new() - .map_size(10 * 1024 * 1024) // 10MB - .open(dir.path()) - .unwrap() - }; - - let result = unsafe { - EnvOpenOptions::new() - .map_size(12 * 1024 * 1024) // 12MB - .open(dir.path()) - }; - - assert!(matches!(result, Err(Error::BadOpenOptions { .. }))); - } - - #[test] - fn open_env_with_named_path() { - let dir = tempfile::tempdir().unwrap(); - fs::create_dir_all(dir.path().join("babar.mdb")).unwrap(); - let _env = unsafe { - EnvOpenOptions::new() - .map_size(10 * 1024 * 1024) // 10MB - .open(dir.path().join("babar.mdb")) - .unwrap() - }; - - let _env = unsafe { - EnvOpenOptions::new() - .map_size(10 * 1024 * 1024) // 10MB - .open(dir.path().join("babar.mdb")) - .unwrap() - }; - } - - #[test] - #[cfg(not(windows))] - fn open_database_with_writemap_flag() { - let dir = tempfile::tempdir().unwrap(); - let mut envbuilder = EnvOpenOptions::new(); - envbuilder.map_size(10 * 1024 * 1024); // 10MB - envbuilder.max_dbs(10); - unsafe { envbuilder.flags(crate::EnvFlags::WRITE_MAP) }; - let env = unsafe { envbuilder.open(dir.path()).unwrap() }; - - let mut wtxn = env.write_txn().unwrap(); - let _db = env.create_database::(&mut wtxn, Some("my-super-db")).unwrap(); - wtxn.commit().unwrap(); - } - - #[test] - fn open_database_with_nosubdir() { - let dir = tempfile::tempdir().unwrap(); - let mut envbuilder = EnvOpenOptions::new(); - unsafe { envbuilder.flags(crate::EnvFlags::NO_SUB_DIR) }; - let _env = unsafe { envbuilder.open(dir.path().join("data.mdb")).unwrap() }; - } - - #[test] - fn create_database_without_commit() { - let dir = tempfile::tempdir().unwrap(); - let env = unsafe { - EnvOpenOptions::new() - .map_size(10 * 1024 * 1024) // 10MB - .max_dbs(10) - .open(dir.path()) - .unwrap() - }; - - let mut wtxn = env.write_txn().unwrap(); - let _db = env.create_database::(&mut wtxn, Some("my-super-db")).unwrap(); - wtxn.abort(); - - let rtxn = env.read_txn().unwrap(); - let option = env.open_database::(&rtxn, Some("my-super-db")).unwrap(); - assert!(option.is_none()); - } - - #[test] - fn open_already_existing_database() { - let dir = tempfile::tempdir().unwrap(); - let env = unsafe { - EnvOpenOptions::new() - .map_size(10 * 1024 * 1024) // 10MB - .max_dbs(10) - .open(dir.path()) - .unwrap() - }; - - // we first create a database - let mut wtxn = env.write_txn().unwrap(); - let _db = env.create_database::(&mut wtxn, Some("my-super-db")).unwrap(); - wtxn.commit().unwrap(); - - // Close the environement and reopen it, databases must not be loaded in memory. - env.prepare_for_closing().wait(); - let env = unsafe { - EnvOpenOptions::new() - .map_size(10 * 1024 * 1024) // 10MB - .max_dbs(10) - .open(dir.path()) - .unwrap() - }; - - let rtxn = env.read_txn().unwrap(); - let option = env.open_database::(&rtxn, Some("my-super-db")).unwrap(); - assert!(option.is_some()); - } - - #[test] - fn resize_database() { - let dir = tempfile::tempdir().unwrap(); - let page_size = page_size::get(); - let env = unsafe { - EnvOpenOptions::new().map_size(9 * page_size).max_dbs(1).open(dir.path()).unwrap() - }; - - let mut wtxn = env.write_txn().unwrap(); - let db = env.create_database::(&mut wtxn, Some("my-super-db")).unwrap(); - wtxn.commit().unwrap(); - - let mut wtxn = env.write_txn().unwrap(); - for i in 0..64 { - db.put(&mut wtxn, &i.to_string(), "world").unwrap(); - } - wtxn.commit().unwrap(); - - let mut wtxn = env.write_txn().unwrap(); - for i in 64..128 { - db.put(&mut wtxn, &i.to_string(), "world").unwrap(); - } - wtxn.commit().expect_err("cannot commit a transaction that would reach the map size limit"); - - unsafe { - env.resize(10 * page_size).unwrap(); - } - let mut wtxn = env.write_txn().unwrap(); - for i in 64..128 { - db.put(&mut wtxn, &i.to_string(), "world").unwrap(); - } - wtxn.commit().expect("transaction should commit after resizing the map size"); - - assert_eq!(10 * page_size, env.info().map_size); - } - - /// Non-regression test for - /// - /// - /// We should be able to open database Read-Only Env with - /// no prior Read-Write Env opening. And query data. - #[test] - fn open_read_only_without_no_env_opened_before() { - let expected_data0 = "Data Expected db0"; - let dir = tempfile::tempdir().unwrap(); - - { - // We really need this env to be dropped before the read-only access. - let env = unsafe { - EnvOpenOptions::new() - .map_size(16 * 1024 * 1024 * 1024) // 10MB - .max_dbs(32) - .open(dir.path()) - .unwrap() - }; - let mut wtxn = env.write_txn().unwrap(); - let database0 = env.create_database::(&mut wtxn, Some("shared0")).unwrap(); - - wtxn.commit().unwrap(); - let mut wtxn = env.write_txn().unwrap(); - database0.put(&mut wtxn, "shared0", expected_data0).unwrap(); - wtxn.commit().unwrap(); - // We also really need that no other env reside in memory in other thread doing tests. - env.prepare_for_closing().wait(); - } - - { - // Open now we do a read-only opening - let env = unsafe { - EnvOpenOptions::new() - .map_size(16 * 1024 * 1024 * 1024) // 10MB - .max_dbs(32) - .open(dir.path()) - .unwrap() - }; - let database0 = { - let rtxn = env.read_txn().unwrap(); - let database0 = - env.open_database::(&rtxn, Some("shared0")).unwrap().unwrap(); - // This commit is mandatory if not committed you might get - // Io(Os { code: 22, kind: InvalidInput, message: "Invalid argument" }) - rtxn.commit().unwrap(); - database0 - }; - - { - // If we didn't committed the opening it might fail with EINVAL. - let rtxn = env.read_txn().unwrap(); - let value = database0.get(&rtxn, "shared0").unwrap().unwrap(); - assert_eq!(value, expected_data0); - } - - env.prepare_for_closing().wait(); - } - - // To avoid reintroducing the bug let's try to open again but without the commit - { - // Open now we do a read-only opening - let env = unsafe { - EnvOpenOptions::new() - .map_size(16 * 1024 * 1024 * 1024) // 10MB - .max_dbs(32) - .open(dir.path()) - .unwrap() - }; - let database0 = { - let rtxn = env.read_txn().unwrap(); - let database0 = - env.open_database::(&rtxn, Some("shared0")).unwrap().unwrap(); - // No commit it's important, dropping explicitly - drop(rtxn); - database0 - }; - - { - // We didn't committed the opening we will get EINVAL. - let rtxn = env.read_txn().unwrap(); - // The dbg!() is intentional in case of a change in rust-std or in lmdb related - // to the windows error. - let err = dbg!(database0.get(&rtxn, "shared0")); - - // The error kind is still ErrorKind Uncategorized on windows. - // Behind it's a ERROR_BAD_COMMAND code 22 like EINVAL. - if cfg!(windows) { - assert!(err.is_err()); - } else { - assert!( - matches!(err, Err(Error::Io(ref e)) if e.kind() == ErrorKind::InvalidInput) - ); - } - } - - env.prepare_for_closing().wait(); - } - } - - #[test] - fn max_key_size() { - let dir = tempfile::tempdir().unwrap(); - let env = unsafe { EnvOpenOptions::new().open(dir.path().join(dir.path())).unwrap() }; - let maxkeysize = env.max_key_size(); - - eprintln!("maxkeysize: {}", maxkeysize); - - if cfg!(feature = "longer-keys") { - // Should be larger than the default of 511 - assert!(maxkeysize > 511); - } else { - // Should be the default of 511 - assert_eq!(maxkeysize, 511); - } - } -} diff --git a/heed/src/lib.rs b/heed/src/lib.rs index a3abcd81..fd172799 100644 --- a/heed/src/lib.rs +++ b/heed/src/lib.rs @@ -149,33 +149,19 @@ pub enum Error { Encoding(BoxedError), /// Decoding error. Decoding(BoxedError), - /// Database closing in progress. - DatabaseClosing, - /// Attempt to open [`Env`] with different options. - BadOpenOptions { - /// The simplified options that were used to originally open this env. - #[cfg(all(master3, encryption))] - options: env::SimplifiedEncryptedOpenOptions, - /// The options that were used to originally open this env. - #[cfg(not(all(master3, encryption)))] - options: EnvOpenOptions, - /// The env opened with the original options. - env: Env, - }, + /// The environment is already open; close it to be able to open it again. + EnvAlreadyOpened, } impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { - Error::Io(error) => write!(f, "{}", error), - Error::Mdb(error) => write!(f, "{}", error), - Error::Encoding(error) => write!(f, "error while encoding: {}", error), - Error::Decoding(error) => write!(f, "error while decoding: {}", error), - Error::DatabaseClosing => { - f.write_str("database is in a closing phase, you can't open it at the same time") - } - Error::BadOpenOptions { .. } => { - f.write_str("an environment is already opened with different options") + Error::Io(error) => write!(f, "{error}"), + Error::Mdb(error) => write!(f, "{error}"), + Error::Encoding(error) => write!(f, "error while encoding: {error}"), + Error::Decoding(error) => write!(f, "error while decoding: {error}"), + Error::EnvAlreadyOpened => { + f.write_str("environment already open; close it to be able to open it again") } } } diff --git a/heed/src/mdb/lmdb_ffi.rs b/heed/src/mdb/lmdb_ffi.rs index 36f399ed..cb8b3ba5 100644 --- a/heed/src/mdb/lmdb_ffi.rs +++ b/heed/src/mdb/lmdb_ffi.rs @@ -1,6 +1,6 @@ use std::ptr; -#[cfg(all(master3, encryption))] +#[cfg(master3)] pub use ffi::mdb_env_set_encrypt; pub use ffi::{ mdb_cursor_close, mdb_cursor_del, mdb_cursor_get, mdb_cursor_open, mdb_cursor_put, diff --git a/heed/src/txn.rs b/heed/src/txn.rs index 87da9f43..18e111f0 100644 --- a/heed/src/txn.rs +++ b/heed/src/txn.rs @@ -2,9 +2,10 @@ use std::borrow::Cow; use std::ops::Deref; use std::ptr::{self, NonNull}; +use crate::env2::Env; use crate::mdb::error::mdb_result; use crate::mdb::ffi; -use crate::{Env, Result}; +use crate::Result; /// A read-only transaction. /// @@ -209,7 +210,7 @@ impl<'p> Deref for RwTxn<'p> { } // TODO can't we just always implement it? -#[cfg(all(master3, encryption))] +#[cfg(master3)] impl<'p> std::ops::DerefMut for RwTxn<'p> { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.txn diff --git a/heed3-encryption/Cargo.toml b/heed3-encryption/Cargo.toml deleted file mode 100644 index 9c8d61d5..00000000 --- a/heed3-encryption/Cargo.toml +++ /dev/null @@ -1,124 +0,0 @@ -[package] -name = "heed3-encryption" -version = "0.20.5-beta.1" -authors = ["Kerollmops "] -description = "A fully typed LMDB (mdb.master3) wrapper with minimum overhead with support for encryption" -license = "MIT" -repository = "https://github.com/Kerollmops/heed" -keywords = ["lmdb", "database", "storage", "typed", "encryption"] -categories = ["database", "data-structures"] -readme = "../README.md" -edition = "2021" - -[dependencies] -# TODO update dependencies -aead = { version = "0.5.1", default-features = false } -bitflags = { version = "2.6.0", features = ["serde"] } -byteorder = { version = "1.5.0", default-features = false } -generic-array = { version = "0.14.6", features = ["serde"] } -heed-master3-proc-macro = { version = "0.1.0", path = "../heed-master3-proc-macro" } -heed-traits = { version = "0.20.0", path = "../heed-traits" } -heed-types = { version = "0.20.1", default-features = false, path = "../heed-types" } -libc = "0.2.155" -lmdb-master3-sys = { version = "0.2.4", path = "../lmdb-master3-sys" } -once_cell = "1.19.0" -page_size = "0.6.0" -serde = { version = "1.0.203", features = ["derive"], optional = true } -synchronoise = "1.0.1" - -[dev-dependencies] -# TODO update dependencies -argon2 = { version = "0.4.1", features = ["std"] } -serde = { version = "1.0.203", features = ["derive"] } -chacha20poly1305 = "0.10.1" -tempfile = "3.10.1" - -[target.'cfg(windows)'.dependencies] -url = "2.5.2" - -[features] -# The `serde` feature makes some types serializable, -# like the `EnvOpenOptions` struct. -default = ["serde", "serde-bincode", "serde-json"] -serde = ["bitflags/serde", "dep:serde"] - -# The #MDB_NOTLS flag is automatically set on Env opening, -# RoTxn and RoCursors implements the Send trait. This allows the -# user to move RoTxns and RoCursors between threads as read transactions -# will no more use thread local storage and will tie reader locktable -# slots to #MDB_txn objects instead of to threads. -# -# According to the LMDB documentation, when this feature is not enabled: -# A thread can only use one transaction at a time, plus any child -# transactions. Each transaction belongs to one thread. [...] -# The #MDB_NOTLS flag changes this for read-only transactions. -# -# And a #MDB_BAD_RSLOT error will be thrown when multiple read -# transactions exists on the same thread -read-txn-no-tls = [] - -# Enable the serde en/decoders for bincode, serde_json, or rmp_serde -serde-bincode = ["heed-types/serde-bincode"] -serde-json = ["heed-types/serde-json"] -serde-rmp = ["heed-types/serde-rmp"] - -# serde_json features -preserve_order = ["heed-types/preserve_order"] -arbitrary_precision = ["heed-types/arbitrary_precision"] -raw_value = ["heed-types/raw_value"] -unbounded_depth = ["heed-types/unbounded_depth"] - -# Whether to tell LMDB to use POSIX semaphores during compilation -# (instead of the default, which are System V semaphores). -# POSIX semaphores are required for Apple's App Sandbox on iOS & macOS, -# and are possibly faster and more appropriate for single-process use. -# There are tradeoffs for both POSIX and SysV semaphores; which you -# should look into before enabling this feature. Also, see here: -# -posix-sem = ["lmdb-master3-sys/posix-sem"] - -# These features configure the MDB_IDL_LOGN macro, which determines -# the size of the free and dirty page lists (and thus the amount of memory -# allocated when opening an LMDB environment in read-write mode). -# -# Each feature defines MDB_IDL_LOGN as the value in the name of the feature. -# That means these features are mutually exclusive, and you must not specify -# more than one at the same time (or the crate will fail to compile). -# -# For more information on the motivation for these features (and their effect), -# see https://github.com/mozilla/lmdb/pull/2. -mdb_idl_logn_8 = ["lmdb-master3-sys/mdb_idl_logn_8"] -mdb_idl_logn_9 = ["lmdb-master3-sys/mdb_idl_logn_9"] -mdb_idl_logn_10 = ["lmdb-master3-sys/mdb_idl_logn_10"] -mdb_idl_logn_11 = ["lmdb-master3-sys/mdb_idl_logn_11"] -mdb_idl_logn_12 = ["lmdb-master3-sys/mdb_idl_logn_12"] -mdb_idl_logn_13 = ["lmdb-master3-sys/mdb_idl_logn_13"] -mdb_idl_logn_14 = ["lmdb-master3-sys/mdb_idl_logn_14"] -mdb_idl_logn_15 = ["lmdb-master3-sys/mdb_idl_logn_15"] -mdb_idl_logn_16 = ["lmdb-master3-sys/mdb_idl_logn_16"] - -# Setting this enables you to use keys longer than 511 bytes. The exact limit -# is computed by LMDB at compile time. You can find the exact value by calling -# Env::max_key_size(). This value varies by architecture. -# -# Example max key sizes: -# - Apple M1 (ARM64): 8126 bytes -# - Apple Intel (AMD64): 1982 bytes -# - Linux Intel (AMD64): 1982 bytes -# -# Setting this also enables you to use values larger than 511 bytes when using -# a Database with the DatabaseFlags::DUP_SORT flag. -# -# This builds LMDB with the -DMDB_MAXKEYSIZE=0 option. -# -# Note: If you are moving database files between architectures then your longest -# stored key must fit within the smallest limit of all architectures used. For -# example, if you are moving databases between Apple M1 and Apple Intel -# computers then you need to keep your keys within the smaller 1982 byte limit. -longer-keys = ["lmdb-master3-sys/longer-keys"] - -# Examples are located outside the standard heed/examples directory to prevent -# conflicts between heed3 and heed examples when working on both crates. -[[example]] -name = "heed3-encryption" -path = "../examples/heed3-encryption.rs" diff --git a/heed3/Cargo.toml b/heed3/Cargo.toml index 80a2be18..a1fc9660 100644 --- a/heed3/Cargo.toml +++ b/heed3/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "heed3" +name = "heed3-encryption" version = "0.20.5-beta.1" authors = ["Kerollmops "] -description = "A fully typed LMDB (mdb.master3) wrapper with minimum overhead" +description = "A fully typed LMDB (mdb.master3) wrapper with minimum overhead with support for encryption" license = "MIT" repository = "https://github.com/Kerollmops/heed" keywords = ["lmdb", "database", "storage", "typed", "encryption"] @@ -11,8 +11,11 @@ readme = "../README.md" edition = "2021" [dependencies] +# TODO update dependencies +aead = { version = "0.5.1", default-features = false } bitflags = { version = "2.6.0", features = ["serde"] } byteorder = { version = "1.5.0", default-features = false } +generic-array = { version = "0.14.6", features = ["serde"] } heed-traits = { version = "0.20.0", path = "../heed-traits" } heed-types = { version = "0.20.1", default-features = false, path = "../heed-types" } libc = "0.2.155" @@ -20,11 +23,12 @@ lmdb-master3-sys = { version = "0.2.4", path = "../lmdb-master3-sys" } once_cell = "1.19.0" page_size = "0.6.0" serde = { version = "1.0.203", features = ["derive"], optional = true } -synchronoise = "1.0.1" [dev-dependencies] # TODO update dependencies +argon2 = { version = "0.4.1", features = ["std"] } serde = { version = "1.0.203", features = ["derive"] } +chacha20poly1305 = "0.10.1" tempfile = "3.10.1" [target.'cfg(windows)'.dependencies] @@ -114,5 +118,5 @@ longer-keys = ["lmdb-master3-sys/longer-keys"] # Examples are located outside the standard heed/examples directory to prevent # conflicts between heed3 and heed examples when working on both crates. [[example]] -name = "heed3-all-types" -path = "../examples/heed3-all-types.rs" +name = "heed3-encryption" +path = "../examples/heed3-encryption.rs" From cac91d944b5c42bbde762bab0aeb47fe22b5ca1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sun, 17 Nov 2024 18:42:45 +0100 Subject: [PATCH 31/51] Add a script to setup heed3 --- convert-to-heed3.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100755 convert-to-heed3.sh diff --git a/convert-to-heed3.sh b/convert-to-heed3.sh new file mode 100755 index 00000000..f92ef4b2 --- /dev/null +++ b/convert-to-heed3.sh @@ -0,0 +1,20 @@ +# This script is meant to setup the heed3 crate. +# + +if [[ -n $(git status -s) ]]; then + echo "Error: Repository is git dirty, please commit or stash changes before running this script." + exit 1 +fi + +set -e + +# It basically copy the heed3/Cargo.toml file into +# the heed folder... +cp heed3/Cargo.toml heed/Cargo.toml + +# ...and replaces the `heed::` string by the `heed3::` one. +for file in $(find heed/src -type f -name "*.rs"); do + sed -i '' 's/heed::/heed3::/g' "$file" +done + +echo "Heed3 crate setup completed successfully. Copied and modified configurations for the heed crate." From 38471d96ad48ad14debee77003948222c1824f48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sun, 17 Nov 2024 19:23:56 +0100 Subject: [PATCH 32/51] Use NonNull envs in the Txns and Databases --- heed/src/cookbook.rs | 3 +- heed/src/{database => databases}/database.rs | 88 +++++++++++++++---- .../encrypted_database.rs | 0 heed/src/{database => databases}/mod.rs | 0 heed/src/{env => envs}/encrypted_env.rs | 0 heed/src/{env => envs}/env.rs | 35 ++++---- heed/src/{env => envs}/env_open_options.rs | 2 +- heed/src/{env => envs}/mod.rs | 18 ++++ heed/src/iterator/prefix.rs | 2 +- heed/src/lib.rs | 14 +-- heed/src/txn.rs | 57 +++++++----- 11 files changed, 150 insertions(+), 69 deletions(-) rename heed/src/{database => databases}/database.rs (97%) rename heed/src/{database => databases}/encrypted_database.rs (100%) rename heed/src/{database => databases}/mod.rs (100%) rename heed/src/{env => envs}/encrypted_env.rs (100%) rename heed/src/{env => envs}/env.rs (94%) rename heed/src/{env => envs}/env_open_options.rs (99%) rename heed/src/{env => envs}/mod.rs (91%) diff --git a/heed/src/cookbook.rs b/heed/src/cookbook.rs index f2cfd905..0be8a53a 100644 --- a/heed/src/cookbook.rs +++ b/heed/src/cookbook.rs @@ -449,4 +449,5 @@ // To let cargo generate doc links #![allow(unused_imports)] -use crate::{BytesDecode, BytesEncode, Database, EnvOpenOptions}; +use crate::envs::EnvOpenOptions; +use crate::{BytesDecode, BytesEncode, Database}; diff --git a/heed/src/database/database.rs b/heed/src/databases/database.rs similarity index 97% rename from heed/src/database/database.rs rename to heed/src/databases/database.rs index 5d373cfd..fcfcb36c 100644 --- a/heed/src/database/database.rs +++ b/heed/src/databases/database.rs @@ -6,7 +6,7 @@ use heed_traits::{Comparator, LexicographicComparator}; use types::{DecodeIgnore, LazyDecode}; use crate::cursor::MoveOperation; -use crate::env::DefaultComparator; +use crate::envs::DefaultComparator; use crate::iteration_method::MoveOnCurrentKeyDuplicates; use crate::mdb::error::mdb_result; use crate::mdb::ffi; @@ -139,8 +139,9 @@ impl<'e, 'n, KC, DC, C> DatabaseOpenOptions<'e, 'n, KC, DC, C> { { assert_eq_env_txn!(self.env, rtxn); - match self.env.raw_init_database::(rtxn.txn.unwrap(), self.name, self.flags) { - Ok(dbi) => Ok(Some(Database::new(self.env.env_mut_ptr() as _, dbi))), + let raw_txn = unsafe { rtxn.txn.unwrap().as_mut() }; + match self.env.raw_init_database::(raw_txn, self.name, self.flags) { + Ok(dbi) => Ok(Some(Database::new(self.env.env_mut_ptr().as_ptr() as _, dbi))), Err(Error::Mdb(e)) if e.not_found() => Ok(None), Err(e) => Err(e), } @@ -164,8 +165,9 @@ impl<'e, 'n, KC, DC, C> DatabaseOpenOptions<'e, 'n, KC, DC, C> { assert_eq_env_txn!(self.env, wtxn); let flags = self.flags | AllDatabaseFlags::CREATE; - match self.env.raw_init_database::(wtxn.txn.txn.unwrap(), self.name, flags) { - Ok(dbi) => Ok(Database::new(self.env.env_mut_ptr() as _, dbi)), + let raw_txn = unsafe { wtxn.txn.txn.unwrap().as_mut() }; + match self.env.raw_init_database::(raw_txn, self.name, flags) { + Ok(dbi) => Ok(Database::new(self.env.env_mut_ptr().as_ptr() as _, dbi)), Err(e) => Err(e), } } @@ -354,7 +356,12 @@ impl Database { let mut txn = txn.txn.unwrap(); let result = unsafe { - mdb_result(ffi::mdb_get(txn.as_mut(), self.dbi, &mut key_val, data_val.as_mut_ptr())) + mdb_result(ffi::mdb_get( + txn.txn.unwrap().as_mut(), + self.dbi, + &mut key_val, + data_val.as_mut_ptr(), + )) }; match result { @@ -966,9 +973,9 @@ impl Database { assert_eq_env_db_txn!(self, txn); let mut db_stat = mem::MaybeUninit::uninit(); - let mut txn = txn.txn.unwrap(); - let result = - unsafe { mdb_result(ffi::mdb_stat(txn.as_mut(), self.dbi, db_stat.as_mut_ptr())) }; + let result = unsafe { + mdb_result(ffi::mdb_stat(txn.txn.unwrap().as_mut(), self.dbi, db_stat.as_mut_ptr())) + }; match result { Ok(()) => { @@ -1855,7 +1862,13 @@ impl Database { let mut txn = txn.txn.txn.unwrap(); unsafe { - mdb_result(ffi::mdb_put(txn.as_mut(), self.dbi, &mut key_val, &mut data_val, flags))? + mdb_result(ffi::mdb_put( + txn.txn.txn.unwrap().as_mut(), + self.dbi, + &mut key_val, + &mut data_val, + flags, + ))? } Ok(()) @@ -1917,7 +1930,13 @@ impl Database { let mut txn = txn.txn.txn.unwrap(); unsafe { - mdb_result(ffi::mdb_put(txn.as_mut(), self.dbi, &mut key_val, &mut reserved, flags))? + mdb_result(ffi::mdb_put( + txn.txn.txn.unwrap().as_mut(), + self.dbi, + &mut key_val, + &mut reserved, + flags, + ))? } let mut reserved = unsafe { ReservedSpace::from_val(reserved) }; @@ -2010,7 +2029,13 @@ impl Database { let mut txn = txn.txn.txn.unwrap(); unsafe { - mdb_result(ffi::mdb_put(txn.as_mut(), self.dbi, &mut key_val, &mut data_val, flags))? + mdb_result(ffi::mdb_put( + txn.txn.txn.unwrap().as_mut(), + self.dbi, + &mut key_val, + &mut data_val, + flags, + ))? } Ok(()) @@ -2118,7 +2143,13 @@ impl Database { let mut txn = txn.txn.txn.unwrap(); let result = unsafe { - mdb_result(ffi::mdb_put(txn.as_mut(), self.dbi, &mut key_val, &mut data_val, flags)) + mdb_result(ffi::mdb_put( + txn.txn.txn.unwrap().as_mut(), + self.dbi, + &mut key_val, + &mut data_val, + flags, + )) }; match result { @@ -2269,7 +2300,13 @@ impl Database { let mut txn = txn.txn.txn.unwrap(); let result = unsafe { - mdb_result(ffi::mdb_put(txn.as_mut(), self.dbi, &mut key_val, &mut reserved, flags)) + mdb_result(ffi::mdb_put( + txn.txn.txn.unwrap().as_mut(), + self.dbi, + &mut key_val, + &mut reserved, + flags, + )) }; match result { @@ -2347,7 +2384,12 @@ impl Database { let mut txn = txn.txn.txn.unwrap(); let result = unsafe { - mdb_result(ffi::mdb_del(txn.as_mut(), self.dbi, &mut key_val, ptr::null_mut())) + mdb_result(ffi::mdb_del( + txn.txn.txn.unwrap().as_mut(), + self.dbi, + &mut key_val, + ptr::null_mut(), + )) }; match result { @@ -2435,7 +2477,12 @@ impl Database { let mut txn = txn.txn.txn.unwrap(); let result = unsafe { - mdb_result(ffi::mdb_del(txn.as_mut(), self.dbi, &mut key_val, &mut data_val)) + mdb_result(ffi::mdb_del( + txn.txn.txn.unwrap().as_mut(), + self.dbi, + &mut key_val, + &mut data_val, + )) }; match result { @@ -2561,7 +2608,10 @@ impl Database { assert_eq_env_db_txn!(self, txn); let mut txn = txn.txn.txn.unwrap(); - unsafe { mdb_result(ffi::mdb_drop(txn.as_mut(), self.dbi, 0)).map_err(Into::into) } + unsafe { + mdb_result(ffi::mdb_drop(txn.txn.txn.unwrap().as_mut(), self.dbi, 0)) + .map_err(Into::into) + } } /// Change the codec types of this database, specifying the codecs. @@ -2674,8 +2724,8 @@ mod tests { fn longer_keys() -> Result<()> { let dir = tempfile::tempdir()?; let env = unsafe { EnvOpenOptions::new().open(dir.path())? }; - let mut txn = env.write_txn()?; - let db = env.create_database::(&mut txn, None)?; + let mut txn = envs.write_txn()?; + let db = envs.create_database::(&mut txn, None)?; // Try storing a key larger than 511 bytes (the default if MDB_MAXKEYSIZE is not set) let long_key = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut pharetra sit amet aliquam. Sit amet nisl purus in mollis nunc. Eget egestas purus viverra accumsan in nisl nisi scelerisque. Duis ultricies lacus sed turpis tincidunt. Sem nulla pharetra diam sit. Leo vel orci porta non pulvinar. Erat pellentesque adipiscing commodo elit at imperdiet dui. Suspendisse ultrices gravida dictum fusce ut placerat orci nulla. Diam donec adipiscing tristique risus nec feugiat. In fermentum et sollicitudin ac orci. Ut sem nulla pharetra diam sit amet. Aliquam purus sit amet luctus venenatis lectus. Erat pellentesque adipiscing commodo elit at imperdiet dui accumsan. Urna duis convallis convallis tellus id interdum velit laoreet id. Ac feugiat sed lectus vestibulum mattis ullamcorper velit sed. Tincidunt arcu non sodales neque. Habitant morbi tristique senectus et netus et malesuada fames."; diff --git a/heed/src/database/encrypted_database.rs b/heed/src/databases/encrypted_database.rs similarity index 100% rename from heed/src/database/encrypted_database.rs rename to heed/src/databases/encrypted_database.rs diff --git a/heed/src/database/mod.rs b/heed/src/databases/mod.rs similarity index 100% rename from heed/src/database/mod.rs rename to heed/src/databases/mod.rs diff --git a/heed/src/env/encrypted_env.rs b/heed/src/envs/encrypted_env.rs similarity index 100% rename from heed/src/env/encrypted_env.rs rename to heed/src/envs/encrypted_env.rs diff --git a/heed/src/env/env.rs b/heed/src/envs/env.rs similarity index 94% rename from heed/src/env/env.rs rename to heed/src/envs/env.rs index 29de0269..426dee93 100644 --- a/heed/src/env/env.rs +++ b/heed/src/envs/env.rs @@ -54,7 +54,7 @@ impl Env { /// ``` pub fn real_disk_size(&self) -> Result { let mut fd = mem::MaybeUninit::uninit(); - unsafe { mdb_result(ffi::mdb_env_get_fd(self.env_mut_ptr(), fd.as_mut_ptr()))? }; + unsafe { mdb_result(ffi::mdb_env_get_fd(self.env_mut_ptr().as_mut(), fd.as_mut_ptr()))? }; let fd = unsafe { fd.assume_init() }; let metadata = unsafe { metadata_from_fd(fd)? }; Ok(metadata.len()) @@ -105,7 +105,7 @@ impl Env { // mdb_result(unsafe { ffi::mdb_env_set_flags( - self.env_mut_ptr(), + self.env_mut_ptr().as_mut(), flags.bits(), mode.as_mdb_env_set_flags_input(), ) @@ -116,7 +116,9 @@ impl Env { /// Return the raw flags the environment is currently set with. pub fn get_flags(&self) -> Result { let mut flags = mem::MaybeUninit::uninit(); - unsafe { mdb_result(ffi::mdb_env_get_flags(self.env_mut_ptr(), flags.as_mut_ptr()))? }; + unsafe { + mdb_result(ffi::mdb_env_get_flags(self.env_mut_ptr().as_mut(), flags.as_mut_ptr()))? + }; let flags = unsafe { flags.assume_init() }; Ok(flags) } @@ -124,7 +126,7 @@ impl Env { /// Returns some basic informations about this environment. pub fn info(&self) -> EnvInfo { let mut raw_info = mem::MaybeUninit::uninit(); - unsafe { ffi::mdb_env_info(self.0.env, raw_info.as_mut_ptr()) }; + unsafe { ffi::mdb_env_info(self.env_ptr.as_ptr(), raw_info.as_mut_ptr()) }; let raw_info = unsafe { raw_info.assume_init() }; EnvInfo { @@ -151,13 +153,14 @@ impl Env { let mut size = 0; let mut stat = mem::MaybeUninit::uninit(); - unsafe { mdb_result(ffi::mdb_env_stat(self.env_mut_ptr(), stat.as_mut_ptr()))? }; + unsafe { mdb_result(ffi::mdb_env_stat(self.env_mut_ptr().as_mut(), stat.as_mut_ptr()))? }; let stat = unsafe { stat.assume_init() }; size += compute_size(stat); let rtxn = self.read_txn()?; // Open the main database - let dbi = self.raw_open_dbi::(rtxn.txn.unwrap(), None, 0)?; + let raw_txn = unsafe { rtxn.txn.unwrap().as_mut() }; + let dbi = self.raw_open_dbi::(raw_txn, None, 0)?; // We're going to iterate on the unnamed database let mut cursor = RoCursor::new(&rtxn, dbi)?; @@ -170,12 +173,12 @@ impl Env { let key = String::from_utf8(key.to_vec()).unwrap(); // Calling `ffi::db_stat` on a database instance does not involve key comparison // in LMDB, so it's safe to specify a noop key compare function for it. - if let Ok(dbi) = - self.raw_open_dbi::(rtxn.txn.unwrap(), Some(&key), 0) - { + let raw_txn = unsafe { rtxn.txn.unwrap().as_mut() }; + if let Ok(dbi) = self.raw_open_dbi::(raw_txn, Some(&key), 0) { let mut stat = mem::MaybeUninit::uninit(); - let mut txn = rtxn.txn.unwrap(); - unsafe { mdb_result(ffi::mdb_stat(txn.as_mut(), dbi, stat.as_mut_ptr()))? }; + unsafe { + mdb_result(ffi::mdb_stat(rtxn.txn.unwrap().as_mut(), dbi, stat.as_mut_ptr()))? + }; let stat = unsafe { stat.assume_init() }; size += compute_size(stat); } @@ -404,13 +407,13 @@ impl Env { option: CompactionOption, ) -> Result<()> { let flags = if let CompactionOption::Enabled = option { ffi::MDB_CP_COMPACT } else { 0 }; - mdb_result(ffi::mdb_env_copyfd2(self.0.env, fd, flags))?; + mdb_result(ffi::mdb_env_copyfd2(self.env_ptr.as_ptr(), fd, flags))?; Ok(()) } /// Flush the data buffers to disk. pub fn force_sync(&self) -> Result<()> { - unsafe { mdb_result(ffi::mdb_env_sync(self.0.env, 1))? } + unsafe { mdb_result(ffi::mdb_env_sync(self.env_ptr.as_ptr(), 1))? } Ok(()) } @@ -424,7 +427,7 @@ impl Env { /// Returns the number of stale readers cleared. pub fn clear_stale_readers(&self) -> Result { let mut dead: i32 = 0; - unsafe { mdb_result(ffi::mdb_reader_check(self.0.env, &mut dead))? } + unsafe { mdb_result(ffi::mdb_reader_check(self.env_ptr.as_ptr(), &mut dead))? } // safety: The reader_check function asks for an i32, initialize it to zero // and never decrements it. It is safe to use either an u32 or u64 (usize). Ok(dead as usize) @@ -446,7 +449,7 @@ impl Env { ); return Err(Error::Io(io::Error::new(io::ErrorKind::InvalidInput, msg))); } - mdb_result(unsafe { ffi::mdb_env_set_mapsize(self.env_mut_ptr(), new_size) }) + mdb_result(unsafe { ffi::mdb_env_set_mapsize(self.env_mut_ptr().as_mut(), new_size) }) .map_err(Into::into) } @@ -454,7 +457,7 @@ impl Env { /// /// Depends on the compile-time constant MDB_MAXKEYSIZE. Default 511 pub fn max_key_size(&self) -> usize { - let maxsize: i32 = unsafe { ffi::mdb_env_get_maxkeysize(self.env_mut_ptr()) }; + let maxsize: i32 = unsafe { ffi::mdb_env_get_maxkeysize(self.env_mut_ptr().as_mut()) }; maxsize as usize } } diff --git a/heed/src/env/env_open_options.rs b/heed/src/envs/env_open_options.rs similarity index 99% rename from heed/src/env/env_open_options.rs rename to heed/src/envs/env_open_options.rs index f7f5ce5f..0515e406 100644 --- a/heed/src/env/env_open_options.rs +++ b/heed/src/envs/env_open_options.rs @@ -11,7 +11,7 @@ use std::{io, ptr}; use super::env::Env; use super::{canonicalize_path, OPENED_ENV}; #[cfg(windows)] -use crate::env::OsStrExtLmdb as _; +use crate::envs::OsStrExtLmdb as _; use crate::mdb::error::mdb_result; use crate::mdb::ffi; use crate::{EnvFlags, Error, Result}; diff --git a/heed/src/env/mod.rs b/heed/src/envs/mod.rs similarity index 91% rename from heed/src/env/mod.rs rename to heed/src/envs/mod.rs index adb2bb6a..01f5f44b 100644 --- a/heed/src/env/mod.rs +++ b/heed/src/envs/mod.rs @@ -26,6 +26,11 @@ mod encrypted_env; mod env; mod env_open_options; +#[cfg(master3)] +pub use env::EncryptedEnv; +pub use env::Env; +pub use env_open_options::EnvOpenOptions; + /// Records the current list of opened environments for tracking purposes. The canonical /// path of an environment is removed when either an `Env` or `EncryptedEnv` is closed. static OPENED_ENV: LazyLock>> = LazyLock::new(RwLock::default); @@ -176,6 +181,19 @@ impl LexicographicComparator for DefaultComparator { } } +/// Whether to perform compaction while copying an environment. +#[derive(Debug, Copy, Clone)] +pub enum CompactionOption { + /// Omit free pages and sequentially renumber all pages in output. + /// + /// This option consumes more CPU and runs more slowly than the default. + /// Currently it fails if the environment has suffered a page leak. + Enabled, + + /// Copy everything without taking any special action about free pages. + Disabled, +} + /// Whether to enable or disable flags in [`Env::set_flags`]. #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum FlagSetMode { diff --git a/heed/src/iterator/prefix.rs b/heed/src/iterator/prefix.rs index fabd2f49..854177e3 100644 --- a/heed/src/iterator/prefix.rs +++ b/heed/src/iterator/prefix.rs @@ -5,7 +5,7 @@ use heed_traits::LexicographicComparator; use types::LazyDecode; use crate::cursor::MoveOperation; -use crate::env::DefaultComparator; +use crate::envs::DefaultComparator; use crate::iteration_method::{IterationMethod, MoveBetweenKeys, MoveThroughDuplicateValues}; use crate::*; diff --git a/heed/src/lib.rs b/heed/src/lib.rs index fd172799..254d7fef 100644 --- a/heed/src/lib.rs +++ b/heed/src/lib.rs @@ -64,8 +64,8 @@ pub mod cookbook; mod cursor; -mod database; -mod env; +mod databases; +mod envs; pub mod iteration_method; mod iterator; mod mdb; @@ -79,10 +79,10 @@ use heed_traits as traits; pub use {byteorder, heed_types as types}; use self::cursor::{RoCursor, RwCursor}; -pub use self::database::{Database, DatabaseOpenOptions, DatabaseStat}; -pub use self::env::{ - env_closing_event, CompactionOption, DefaultComparator, Env, EnvClosingEvent, EnvInfo, - EnvOpenOptions, FlagSetMode, IntegerComparator, +pub use self::databases::{Database, DatabaseOpenOptions, DatabaseStat}; +pub use self::envs::{ + CompactionOption, DefaultComparator, Env, EnvInfo, EnvOpenOptions, FlagSetMode, + IntegerComparator, }; pub use self::iterator::{ RoIter, RoPrefix, RoRange, RoRevIter, RoRevPrefix, RoRevRange, RwIter, RwPrefix, RwRange, @@ -198,7 +198,7 @@ pub enum Unspecified {} macro_rules! assert_eq_env_db_txn { ($database:ident, $txn:ident) => { assert!( - $database.env_ident == $txn.env_mut_ptr() as usize, + $database.env_ident == unsafe { $txn.env_mut_ptr().as_mut() as *mut _ as usize }, "The database environment doesn't match the transaction's environment" ); }; diff --git a/heed/src/txn.rs b/heed/src/txn.rs index 18e111f0..7e0d47de 100644 --- a/heed/src/txn.rs +++ b/heed/src/txn.rs @@ -1,8 +1,7 @@ -use std::borrow::Cow; use std::ops::Deref; use std::ptr::{self, NonNull}; -use crate::env2::Env; +use crate::envs::Env; use crate::mdb::error::mdb_result; use crate::mdb::ffi; use crate::Result; @@ -57,32 +56,34 @@ impl<'e> RoTxn<'e> { unsafe { mdb_result(ffi::mdb_txn_begin( - env.env_mut_ptr(), + env.env_mut_ptr().as_mut(), ptr::null_mut(), ffi::MDB_RDONLY, &mut txn, ))? }; - Ok(RoTxn { txn: NonNull::new(txn), env: Cow::Borrowed(env) }) + Ok(RoTxn { txn: NonNull::new(txn), env }) } + // TODO replace this by an ArcRoTxn pub(crate) fn static_read_txn(env: Env) -> Result> { - let mut txn: *mut ffi::MDB_txn = ptr::null_mut(); - - unsafe { - mdb_result(ffi::mdb_txn_begin( - env.env_mut_ptr(), - ptr::null_mut(), - ffi::MDB_RDONLY, - &mut txn, - ))? - }; - - Ok(RoTxn { txn: NonNull::new(txn), env: Cow::Owned(env) }) + // let mut txn: *mut ffi::MDB_txn = ptr::null_mut(); + + // unsafe { + // mdb_result(ffi::mdb_txn_begin( + // env.env_mut_ptr(), + // ptr::null_mut(), + // ffi::MDB_RDONLY, + // &mut txn, + // ))? + // }; + + // Ok(RoTxn { txn, env: Cow::Owned(env) }) + todo!() } - pub(crate) fn env_mut_ptr(&self) -> *mut ffi::MDB_env { + pub(crate) fn env_mut_ptr(&self) -> NonNull { self.env.env_mut_ptr() } @@ -162,22 +163,30 @@ impl<'p> RwTxn<'p> { pub(crate) fn new(env: &'p Env) -> Result> { let mut txn: *mut ffi::MDB_txn = ptr::null_mut(); - unsafe { mdb_result(ffi::mdb_txn_begin(env.env_mut_ptr(), ptr::null_mut(), 0, &mut txn))? }; + unsafe { + mdb_result(ffi::mdb_txn_begin( + env.env_mut_ptr().as_mut(), + ptr::null_mut(), + 0, + &mut txn, + ))? + }; - Ok(RwTxn { txn: RoTxn { txn: NonNull::new(txn), env: Cow::Borrowed(env) } }) + Ok(RwTxn { txn: RoTxn { txn: NonNull::new(txn), env } }) } pub(crate) fn nested(env: &'p Env, parent: &'p mut RwTxn) -> Result> { let mut txn: *mut ffi::MDB_txn = ptr::null_mut(); - let mut parent_txn = parent.txn.txn.unwrap(); - let parent_ptr: *mut ffi::MDB_txn = unsafe { parent_txn.as_mut() }; + let parent_ptr: *mut ffi::MDB_txn = unsafe { parent.txn.txn.unwrap().as_mut() }; - unsafe { mdb_result(ffi::mdb_txn_begin(env.env_mut_ptr(), parent_ptr, 0, &mut txn))? }; + unsafe { + mdb_result(ffi::mdb_txn_begin(env.env_mut_ptr().as_mut(), parent_ptr, 0, &mut txn))? + }; - Ok(RwTxn { txn: RoTxn { txn: NonNull::new(txn), env: Cow::Borrowed(env) } }) + Ok(RwTxn { txn: RoTxn { txn: NonNull::new(txn), env } }) } - pub(crate) fn env_mut_ptr(&self) -> *mut ffi::MDB_env { + pub(crate) fn env_mut_ptr(&self) -> NonNull { self.txn.env.env_mut_ptr() } From 5b657ef0579d89a237c2e8b073ecfa513071a899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sun, 17 Nov 2024 19:24:58 +0100 Subject: [PATCH 33/51] Rename the heed3-encryption crate into heed3 --- heed3/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heed3/Cargo.toml b/heed3/Cargo.toml index a1fc9660..1176d5e0 100644 --- a/heed3/Cargo.toml +++ b/heed3/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "heed3-encryption" +name = "heed3" version = "0.20.5-beta.1" authors = ["Kerollmops "] description = "A fully typed LMDB (mdb.master3) wrapper with minimum overhead with support for encryption" From c8efde1ef8061d49b822554ba68010d84e88f41b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sun, 17 Nov 2024 19:50:14 +0100 Subject: [PATCH 34/51] Fix all errors and most warnings --- heed/src/databases/database.rs | 17 ----- heed/src/databases/encrypted_database.rs | 32 ++++----- heed/src/envs/encrypted_env.rs | 83 +++++++----------------- heed/src/envs/env.rs | 5 +- heed/src/envs/env_open_options.rs | 7 +- heed/src/envs/mod.rs | 2 +- heed/src/lib.rs | 4 ++ 7 files changed, 50 insertions(+), 100 deletions(-) diff --git a/heed/src/databases/database.rs b/heed/src/databases/database.rs index fcfcb36c..43e9c0f1 100644 --- a/heed/src/databases/database.rs +++ b/heed/src/databases/database.rs @@ -341,7 +341,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn get<'a, 'txn>(&self, txn: &'txn RoTxn, key: &'a KC::EItem) -> Result> where KC: BytesEncode<'a>, @@ -427,7 +426,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_duplicates<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -491,7 +489,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_lower_than<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -561,7 +558,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_lower_than_or_equal_to<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -635,7 +631,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_greater_than<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -708,7 +703,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn get_greater_than_or_equal_to<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -768,7 +762,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn first<'txn>(&self, txn: &'txn RoTxn) -> Result> where KC: BytesDecode<'txn>, @@ -823,7 +816,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn last<'txn>(&self, txn: &'txn RoTxn) -> Result> where KC: BytesDecode<'txn>, @@ -881,7 +873,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn len(&self, txn: &RoTxn) -> Result { self.stat(txn).map(|stat| stat.entries as u64) } @@ -925,7 +916,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn is_empty(&self, txn: &RoTxn) -> Result { self.len(txn).map(|l| l == 0) } @@ -968,7 +958,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn stat(&self, txn: &RoTxn) -> Result { assert_eq_env_db_txn!(self, txn); @@ -1033,7 +1022,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn iter<'txn>(&self, txn: &'txn RoTxn) -> Result> { assert_eq_env_db_txn!(self, txn); RoCursor::new(txn, self.dbi).map(|cursor| RoIter::new(cursor)) @@ -1135,7 +1123,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn rev_iter<'txn>(&self, txn: &'txn RoTxn) -> Result> { assert_eq_env_db_txn!(self, txn); @@ -1242,7 +1229,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn range<'a, 'txn, R>( &self, txn: &'txn RoTxn, @@ -1415,7 +1401,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn rev_range<'a, 'txn, R>( &self, txn: &'txn RoTxn, @@ -1590,7 +1575,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn prefix_iter<'a, 'txn>( &self, txn: &'txn RoTxn, @@ -1723,7 +1707,6 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - #[cfg_attr(master3, heed_master3_proc_macro::mut_read_txn(txn))] pub fn rev_prefix_iter<'a, 'txn>( &self, txn: &'txn RoTxn, diff --git a/heed/src/databases/encrypted_database.rs b/heed/src/databases/encrypted_database.rs index 60b75b20..d55a61da 100644 --- a/heed/src/databases/encrypted_database.rs +++ b/heed/src/databases/encrypted_database.rs @@ -1,16 +1,12 @@ -use std::borrow::Cow; -use std::ops::{Bound, RangeBounds}; -use std::{any, fmt, marker, mem, ptr}; +use std::ops::RangeBounds; +use std::{any, fmt}; use heed_traits::{Comparator, LexicographicComparator}; -use types::{DecodeIgnore, LazyDecode}; +use types::LazyDecode; -use crate::cursor::MoveOperation; -use crate::env::DefaultComparator; +use crate::envs::DefaultComparator; use crate::iteration_method::MoveOnCurrentKeyDuplicates; -use crate::mdb::error::mdb_result; -use crate::mdb::ffi; -use crate::mdb::lmdb_flags::{AllDatabaseFlags, DatabaseFlags}; +use crate::mdb::lmdb_flags::DatabaseFlags; use crate::*; /// Options and flags which can be used to configure how a [`Database`] is opened. @@ -61,8 +57,8 @@ pub struct EncryptedDatabaseOpenOptions<'e, 'n, KC, DC, C = DefaultComparator> { impl<'e> EncryptedDatabaseOpenOptions<'e, 'static, Unspecified, Unspecified> { /// Create an options struct to open/create a database with specific flags. - pub fn new(env: &'e Env) -> Self { - EncryptedDatabaseOpenOptions { inner: DatabaseOpenOptions::new(env) } + pub fn new(env: &'e EncryptedEnv) -> Self { + EncryptedDatabaseOpenOptions { inner: DatabaseOpenOptions::new(&env.inner) } } } @@ -119,7 +115,7 @@ impl<'e, 'n, KC, DC, C> EncryptedDatabaseOpenOptions<'e, 'n, KC, DC, C> { DC: 'static, C: Comparator + 'static, { - self.inner.open(rtxn) + self.inner.open(rtxn).map(|opt| opt.map(EncryptedDatabase::new)) } /// Creates a typed database that can already exist in this environment. @@ -137,7 +133,7 @@ impl<'e, 'n, KC, DC, C> EncryptedDatabaseOpenOptions<'e, 'n, KC, DC, C> { DC: 'static, C: Comparator + 'static, { - self.inner.create(wtxn) + self.inner.create(wtxn).map(EncryptedDatabase::new) } } @@ -266,8 +262,8 @@ pub struct EncryptedDatabase { } impl EncryptedDatabase { - pub(crate) fn new(env_ident: usize, dbi: ffi::MDB_dbi) -> Database { - EncryptedDatabase { inner: Database::mew(env_ident, dbi) } + pub(crate) fn new(inner: Database) -> EncryptedDatabase { + EncryptedDatabase { inner } } /// Retrieves the value associated with a key. @@ -2074,7 +2070,7 @@ impl EncryptedDatabase { KC: BytesEncode<'a> + BytesDecode<'txn>, R: RangeBounds, { - self.inner.delete_range(range) + self.inner.delete_range(txn, range) } /// Deletes all key/value pairs in this database. @@ -2119,7 +2115,7 @@ impl EncryptedDatabase { /// # Ok(()) } /// ``` pub fn clear(&self, txn: &mut RwTxn) -> Result<()> { - self.inner.clear() + self.inner.clear(txn) } /// Change the codec types of this database, specifying the codecs. @@ -2164,7 +2160,7 @@ impl EncryptedDatabase { /// # Ok(()) } /// ``` pub fn remap_types(&self) -> EncryptedDatabase { - EncryptedDatabase::new(self.inner.env_ident, self.inner.dbi) + EncryptedDatabase::new(self.inner.remap_types::()) } /// Change the key codec type of this database, specifying the new codec. diff --git a/heed/src/envs/encrypted_env.rs b/heed/src/envs/encrypted_env.rs index 2c38e651..b37d5b2c 100644 --- a/heed/src/envs/encrypted_env.rs +++ b/heed/src/envs/encrypted_env.rs @@ -1,40 +1,22 @@ -use std::any::TypeId; -use std::ffi::CString; +use std::fmt; use std::fs::File; -use std::path::{Path, PathBuf}; -use std::ptr::{self, NonNull}; -use std::{fmt, io, mem}; - -use heed_traits::Comparator; -use lmdb_master_sys::mdb_env_close; - -use super::{ - custom_key_cmp_wrapper, get_file_fd, metadata_from_fd, DefaultComparator, EnvInfo, FlagSetMode, - OPENED_ENV, -}; -use crate::cursor::{MoveOperation, RoCursor}; -use crate::mdb::ffi::{self, MDB_env}; -use crate::mdb::lmdb_error::mdb_result; -use crate::mdb::lmdb_flags::AllDatabaseFlags; -use crate::{ - CompactionOption, Database, DatabaseOpenOptions, EnvFlags, Error, Result, RoTxn, RwTxn, - Unspecified, -}; +use std::panic::catch_unwind; +use std::path::Path; + +use aead::generic_array::typenum::Unsigned; +use aead::{AeadMutInPlace, Key, KeyInit, Nonce, Tag}; + +use super::{Env, EnvInfo, FlagSetMode}; +use crate::databases::{EncryptedDatabase, EncryptedDatabaseOpenOptions}; +use crate::mdb::ffi::{self}; +use crate::{CompactionOption, EnvFlags, Result, RoTxn, RwTxn, Unspecified}; /// An environment handle constructed by using [`EnvOpenOptions::open_encrypted`]. pub struct EncryptedEnv { - inner: Env, + pub(crate) inner: Env, } -impl Env { - pub(crate) fn new(env_ptr: NonNull, path: PathBuf) -> Env { - Env { env_ptr, path } - } - - pub(crate) fn env_mut_ptr(&self) -> NonNull { - self.inner.env_mut_ptr() - } - +impl EncryptedEnv { /// The size of the data file on disk. /// /// # Example @@ -182,24 +164,6 @@ impl Env { options.create(wtxn) } - pub(crate) fn raw_init_database( - &self, - raw_txn: *mut ffi::MDB_txn, - name: Option<&str>, - flags: AllDatabaseFlags, - ) -> Result { - self.inner.raw_init_database(raw_txn, name, flags) - } - - fn raw_open_dbi( - &self, - raw_txn: *mut ffi::MDB_txn, - name: Option<&str>, - flags: u32, - ) -> std::result::Result { - self.inner.raw_open_dbi(raw_txn, name, flags) - } - /// Create a transaction with read and write access for use with the environment. /// /// ## LMDB Limitations @@ -346,7 +310,9 @@ unsafe impl Sync for EncryptedEnv {} impl fmt::Debug for EncryptedEnv { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("EncryptedEnv").field("path", &self.path.display()).finish_non_exhaustive() + f.debug_struct("EncryptedEnv") + .field("path", &self.inner.path.display()) + .finish_non_exhaustive() } } @@ -359,7 +325,7 @@ fn encrypt( auth_out: &mut [u8], ) -> aead::Result<()> { chipertext_out.copy_from_slice(plaintext); - let key: &Key = key.try_into().unwrap(); + let key: &Key = key.into(); let nonce: &Nonce = if nonce.len() >= A::NonceSize::USIZE { nonce[..A::NonceSize::USIZE].into() } else { @@ -380,20 +346,20 @@ fn decrypt( auth_in: &[u8], ) -> aead::Result<()> { output.copy_from_slice(chipher_text); - let key: &Key = key.try_into().unwrap(); + let key: &Key = key.into(); let nonce: &Nonce = if nonce.len() >= A::NonceSize::USIZE { nonce[..A::NonceSize::USIZE].into() } else { return Err(aead::Error); }; - let tag: &Tag = auth_in.try_into().unwrap(); + let tag: &Tag = auth_in.into(); let mut aead = A::new(key); aead.decrypt_in_place_detached(nonce, aad, output, tag) } /// The wrapper function that is called by LMDB that directly calls /// the Rust idiomatic function internally. -unsafe extern "C" fn encrypt_func_wrapper( +pub(crate) unsafe extern "C" fn encrypt_func_wrapper( src: *const ffi::MDB_val, dst: *mut ffi::MDB_val, key_ptr: *const ffi::MDB_val, @@ -415,16 +381,13 @@ unsafe extern "C" fn encrypt_func_wrapper( let aad = []; let nonce = iv; let result = if encdec == 1 { - encrypt::(&key, nonce, &aad, input, output, auth) + encrypt::(key, nonce, &aad, input, output, auth) } else { - decrypt::(&key, nonce, &aad, input, output, auth) + decrypt::(key, nonce, &aad, input, output, auth) }; result.is_err() as i32 }); - match result { - Ok(out) => out, - Err(_) => 1, - } + result.unwrap_or(1) } diff --git a/heed/src/envs/env.rs b/heed/src/envs/env.rs index 426dee93..a178747c 100644 --- a/heed/src/envs/env.rs +++ b/heed/src/envs/env.rs @@ -6,7 +6,6 @@ use std::ptr::{self, NonNull}; use std::{fmt, io, mem}; use heed_traits::Comparator; -use lmdb_master_sys::mdb_env_close; use super::{ custom_key_cmp_wrapper, get_file_fd, metadata_from_fd, DefaultComparator, EnvInfo, FlagSetMode, @@ -24,7 +23,7 @@ use crate::{ /// An environment handle constructed by using [`EnvOpenOptions::open`]. pub struct Env { env_ptr: NonNull, - path: PathBuf, + pub(crate) path: PathBuf, } impl Env { @@ -474,7 +473,7 @@ impl fmt::Debug for Env { impl Drop for Env { fn drop(&mut self) { - unsafe { mdb_env_close(self.env_ptr.as_mut()) }; + unsafe { ffi::mdb_env_close(self.env_ptr.as_mut()) }; let mut lock = OPENED_ENV.write().unwrap(); debug_assert!(lock.remove(&self.path)); } diff --git a/heed/src/envs/env_open_options.rs b/heed/src/envs/env_open_options.rs index 0515e406..b31cf217 100644 --- a/heed/src/envs/env_open_options.rs +++ b/heed/src/envs/env_open_options.rs @@ -8,6 +8,11 @@ use std::path::Path; use std::ptr::NonNull; use std::{io, ptr}; +#[cfg(master3)] +use aead::{generic_array::typenum::Unsigned, AeadCore, AeadMutInPlace, Key, KeyInit}; + +#[cfg(master3)] +use super::encrypted_env::encrypt_func_wrapper; use super::env::Env; use super::{canonicalize_path, OPENED_ENV}; #[cfg(windows)] @@ -357,7 +362,7 @@ impl EnvOpenOptions { let mut env: *mut ffi::MDB_env = ptr::null_mut(); mdb_result(ffi::mdb_env_create(&mut env))?; - let encrypt_key = crate::into_val(&self.encrypt); + let encrypt_key = crate::into_val(&key); mdb_result(ffi::mdb_env_set_encrypt( env, Some(encrypt_func_wrapper::), diff --git a/heed/src/envs/mod.rs b/heed/src/envs/mod.rs index 01f5f44b..a5769213 100644 --- a/heed/src/envs/mod.rs +++ b/heed/src/envs/mod.rs @@ -27,7 +27,7 @@ mod env; mod env_open_options; #[cfg(master3)] -pub use env::EncryptedEnv; +pub use encrypted_env::EncryptedEnv; pub use env::Env; pub use env_open_options::EnvOpenOptions; diff --git a/heed/src/lib.rs b/heed/src/lib.rs index 254d7fef..90a6d618 100644 --- a/heed/src/lib.rs +++ b/heed/src/lib.rs @@ -80,6 +80,10 @@ pub use {byteorder, heed_types as types}; use self::cursor::{RoCursor, RwCursor}; pub use self::databases::{Database, DatabaseOpenOptions, DatabaseStat}; +#[cfg(master3)] +pub use self::databases::{EncryptedDatabase, EncryptedDatabaseOpenOptions}; +#[cfg(master3)] +pub use self::envs::EncryptedEnv; pub use self::envs::{ CompactionOption, DefaultComparator, Env, EnvInfo, EnvOpenOptions, FlagSetMode, IntegerComparator, From ee91f89b61942d0065db3f2c0759bad0fb505103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sat, 30 Nov 2024 11:30:36 +0100 Subject: [PATCH 35/51] Make the Env::open_encrypted method create an EncryptedEnv --- heed/src/envs/env_open_options.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/heed/src/envs/env_open_options.rs b/heed/src/envs/env_open_options.rs index b31cf217..72418c2e 100644 --- a/heed/src/envs/env_open_options.rs +++ b/heed/src/envs/env_open_options.rs @@ -12,7 +12,7 @@ use std::{io, ptr}; use aead::{generic_array::typenum::Unsigned, AeadCore, AeadMutInPlace, Key, KeyInit}; #[cfg(master3)] -use super::encrypted_env::encrypt_func_wrapper; +use super::encrypted_env::{encrypt_func_wrapper, EncryptedEnv}; use super::env::Env; use super::{canonicalize_path, OPENED_ENV}; #[cfg(windows)] @@ -331,7 +331,7 @@ impl EnvOpenOptions { /// # Ok(()) } /// ``` #[cfg(master3)] - pub unsafe fn open_encrypted(&self, key: Key, path: P) -> Result + pub unsafe fn open_encrypted(&self, key: Key, path: P) -> Result where E: AeadMutInPlace + KeyInit, P: AsRef, @@ -405,7 +405,7 @@ impl EnvOpenOptions { Ok(()) => { let env_ptr = NonNull::new(env).unwrap(); debug_assert!(lock.insert(path.clone())); - Ok(Env::new(env_ptr, path)) + Ok(EncryptedEnv { inner: Env::new(env_ptr, path) }) } Err(e) => { ffi::mdb_env_close(env); From ea897593788aebed734454056acb709190143186 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sat, 30 Nov 2024 12:10:38 +0100 Subject: [PATCH 36/51] Reintroduce #289 --- ...heed3-encryption.rs => heed3-encrypted.rs} | 12 +++---- heed/src/databases/database.rs | 19 +++------- heed/src/databases/encrypted_database.rs | 27 +++++++------- heed/src/envs/env.rs | 20 ++++++----- heed/src/envs/mod.rs | 35 +++++++++++++++++-- heed/src/txn.rs | 2 +- heed3/Cargo.toml | 4 +-- 7 files changed, 72 insertions(+), 47 deletions(-) rename examples/{heed3-encryption.rs => heed3-encrypted.rs} (85%) diff --git a/examples/heed3-encryption.rs b/examples/heed3-encrypted.rs similarity index 85% rename from examples/heed3-encryption.rs rename to examples/heed3-encrypted.rs index da16491f..acb3c671 100644 --- a/examples/heed3-encryption.rs +++ b/examples/heed3-encrypted.rs @@ -4,8 +4,8 @@ use std::path::Path; use argon2::Argon2; use chacha20poly1305::{ChaCha20Poly1305, Key}; -use heed3_encryption::types::*; -use heed3_encryption::{Database, EnvOpenOptions}; +use heed3::types::*; +use heed3::{Database, EnvOpenOptions}; fn main() -> Result<(), Box> { let env_path = Path::new("target").join("encrypt.mdb"); @@ -21,12 +21,12 @@ fn main() -> Result<(), Box> { Argon2::default().hash_password_into(password.as_bytes(), salt.as_bytes(), &mut key)?; // We open the environment - let mut options = EnvOpenOptions::::new_encrypted_with(key); + let mut options = EnvOpenOptions::new(); let env = unsafe { options .map_size(10 * 1024 * 1024) // 10MB .max_dbs(3) - .open(&env_path)? + .open_encrypted::(key, &env_path)? }; let key1 = "first-key"; @@ -36,11 +36,11 @@ fn main() -> Result<(), Box> { // We create database and write secret values in it let mut wtxn = env.write_txn()?; - let db: Database = env.create_database(&mut wtxn, Some("first"))?; + let db = env.create_database::(&mut wtxn, Some("first"))?; db.put(&mut wtxn, key1, val1)?; db.put(&mut wtxn, key2, val2)?; wtxn.commit()?; - env.prepare_for_closing().wait(); + // env.prepare_for_closing().wait(); // We reopen the environment now let env = unsafe { options.open(&env_path)? }; diff --git a/heed/src/databases/database.rs b/heed/src/databases/database.rs index 43e9c0f1..df8295ae 100644 --- a/heed/src/databases/database.rs +++ b/heed/src/databases/database.rs @@ -139,8 +139,7 @@ impl<'e, 'n, KC, DC, C> DatabaseOpenOptions<'e, 'n, KC, DC, C> { { assert_eq_env_txn!(self.env, rtxn); - let raw_txn = unsafe { rtxn.txn.unwrap().as_mut() }; - match self.env.raw_init_database::(raw_txn, self.name, self.flags) { + match self.env.raw_init_database::(rtxn.txn.unwrap(), self.name, self.flags) { Ok(dbi) => Ok(Some(Database::new(self.env.env_mut_ptr().as_ptr() as _, dbi))), Err(Error::Mdb(e)) if e.not_found() => Ok(None), Err(e) => Err(e), @@ -165,8 +164,7 @@ impl<'e, 'n, KC, DC, C> DatabaseOpenOptions<'e, 'n, KC, DC, C> { assert_eq_env_txn!(self.env, wtxn); let flags = self.flags | AllDatabaseFlags::CREATE; - let raw_txn = unsafe { wtxn.txn.txn.unwrap().as_mut() }; - match self.env.raw_init_database::(raw_txn, self.name, flags) { + match self.env.raw_init_database::(wtxn.txn.txn.unwrap(), self.name, flags) { Ok(dbi) => Ok(Database::new(self.env.env_mut_ptr().as_ptr() as _, dbi)), Err(e) => Err(e), } @@ -352,7 +350,6 @@ impl Database { let mut key_val = unsafe { crate::into_val(&key_bytes) }; let mut data_val = mem::MaybeUninit::uninit(); - let mut txn = txn.txn.unwrap(); let result = unsafe { mdb_result(ffi::mdb_get( @@ -1129,7 +1126,7 @@ impl Database { RoCursor::new(txn, self.dbi).map(|cursor| RoRevIter::new(cursor)) } - /// Return a mutable reversed lexicographically ordered iterator of all key-value\ + /// Return a mutable reverse ordered iterator of all key-value\ /// pairs in this database. /// /// ``` @@ -1842,7 +1839,6 @@ impl Database { let mut key_val = unsafe { crate::into_val(&key_bytes) }; let mut data_val = unsafe { crate::into_val(&data_bytes) }; let flags = 0; - let mut txn = txn.txn.txn.unwrap(); unsafe { mdb_result(ffi::mdb_put( @@ -1910,7 +1906,6 @@ impl Database { let mut key_val = unsafe { crate::into_val(&key_bytes) }; let mut reserved = ffi::reserve_size_val(data_size); let flags = ffi::MDB_RESERVE; - let mut txn = txn.txn.txn.unwrap(); unsafe { mdb_result(ffi::mdb_put( @@ -2009,7 +2004,6 @@ impl Database { let mut key_val = unsafe { crate::into_val(&key_bytes) }; let mut data_val = unsafe { crate::into_val(&data_bytes) }; let flags = flags.bits(); - let mut txn = txn.txn.txn.unwrap(); unsafe { mdb_result(ffi::mdb_put( @@ -2123,7 +2117,6 @@ impl Database { let mut key_val = unsafe { crate::into_val(&key_bytes) }; let mut data_val = unsafe { crate::into_val(&data_bytes) }; let flags = (flags | PutFlags::NO_OVERWRITE).bits(); - let mut txn = txn.txn.txn.unwrap(); let result = unsafe { mdb_result(ffi::mdb_put( @@ -2280,7 +2273,6 @@ impl Database { let mut key_val = unsafe { crate::into_val(&key_bytes) }; let mut reserved = ffi::reserve_size_val(data_size); let flags = (flags | PutFlags::NO_OVERWRITE).bits() | ffi::MDB_RESERVE; - let mut txn = txn.txn.txn.unwrap(); let result = unsafe { mdb_result(ffi::mdb_put( @@ -2364,7 +2356,6 @@ impl Database { let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; let mut key_val = unsafe { crate::into_val(&key_bytes) }; - let mut txn = txn.txn.txn.unwrap(); let result = unsafe { mdb_result(ffi::mdb_del( @@ -2457,7 +2448,6 @@ impl Database { let data_bytes: Cow<[u8]> = DC::bytes_encode(data).map_err(Error::Encoding)?; let mut key_val = unsafe { crate::into_val(&key_bytes) }; let mut data_val = unsafe { crate::into_val(&data_bytes) }; - let mut txn = txn.txn.txn.unwrap(); let result = unsafe { mdb_result(ffi::mdb_del( @@ -2589,7 +2579,6 @@ impl Database { /// ``` pub fn clear(&self, txn: &mut RwTxn) -> Result<()> { assert_eq_env_db_txn!(self, txn); - let mut txn = txn.txn.txn.unwrap(); unsafe { mdb_result(ffi::mdb_drop(txn.txn.txn.unwrap().as_mut(), self.dbi, 0)) @@ -2682,7 +2671,7 @@ mod tests { use heed_types::*; use super::*; - use crate::env::IntegerComparator; + use crate::IntegerComparator; #[test] fn put_overwrite() -> Result<()> { diff --git a/heed/src/databases/encrypted_database.rs b/heed/src/databases/encrypted_database.rs index d55a61da..f67522e7 100644 --- a/heed/src/databases/encrypted_database.rs +++ b/heed/src/databases/encrypted_database.rs @@ -1015,9 +1015,9 @@ impl EncryptedDatabase { self.inner.rev_iter_mut(txn) } - /// Return a lexicographically ordered iterator of a range of key-value pairs in this database. + /// Return an ordered iterator of a range of key-value pairs in this database. /// - /// Comparisons are made by using the bytes representation of the key. + /// Comparisons are made by using the comparator `C`. /// /// You can make this iterator `Send`able between threads by /// using the `read-txn-no-tls` crate feature. @@ -1062,7 +1062,7 @@ impl EncryptedDatabase { &self, txn: &'txn mut RoTxn, range: &'a R, - ) -> Result> + ) -> Result> where KC: BytesEncode<'a>, R: RangeBounds, @@ -1070,10 +1070,10 @@ impl EncryptedDatabase { self.inner.range(txn, range) } - /// Return a mutable lexicographically ordered iterator of a range of + /// Return a mutable ordered iterator of a range of /// key-value pairs in this database. /// - /// Comparisons are made by using the bytes representation of the key. + /// Comparisons are made by using the comparator `C`. /// /// ``` /// # use std::fs; @@ -1128,7 +1128,7 @@ impl EncryptedDatabase { &self, txn: &'txn mut RwTxn, range: &'a R, - ) -> Result> + ) -> Result> where KC: BytesEncode<'a>, R: RangeBounds, @@ -1136,10 +1136,10 @@ impl EncryptedDatabase { self.inner.range_mut(txn, range) } - /// Return a reversed lexicographically ordered iterator of a range of key-value + /// Return a reverse ordered iterator of a range of key-value /// pairs in this database. /// - /// Comparisons are made by using the bytes representation of the key. + /// Comparisons are made by using the comparator `C`. /// /// You can make this iterator `Send`able between threads by /// using the `read-txn-no-tls` crate feature. @@ -1184,7 +1184,7 @@ impl EncryptedDatabase { &self, txn: &'txn mut RoTxn, range: &'a R, - ) -> Result> + ) -> Result> where KC: BytesEncode<'a>, R: RangeBounds, @@ -1192,10 +1192,10 @@ impl EncryptedDatabase { self.inner.rev_range(txn, range) } - /// Return a mutable reversed lexicographically ordered iterator of a range of + /// Return a mutable reverse ordered iterator of a range of /// key-value pairs in this database. /// - /// Comparisons are made by using the bytes representation of the key. + /// Comparisons are made by using the comparator `C`. /// /// ``` /// # use std::fs; @@ -1250,7 +1250,7 @@ impl EncryptedDatabase { &self, txn: &'txn mut RwTxn, range: &'a R, - ) -> Result> + ) -> Result> where KC: BytesEncode<'a>, R: RangeBounds, @@ -2020,7 +2020,7 @@ impl EncryptedDatabase { /// /// Prefer using [`clear`] instead of a call to this method with a full range ([`..`]). /// - /// Comparisons are made by using the bytes representation of the key. + /// Comparisons are made by using the comparator `C`. /// /// [`clear`]: crate::Database::clear /// [`..`]: std::ops::RangeFull @@ -2068,6 +2068,7 @@ impl EncryptedDatabase { pub fn delete_range<'a, 'txn, R>(&self, txn: &'txn mut RwTxn, range: &'a R) -> Result where KC: BytesEncode<'a> + BytesDecode<'txn>, + C: Comparator, R: RangeBounds, { self.inner.delete_range(txn, range) diff --git a/heed/src/envs/env.rs b/heed/src/envs/env.rs index a178747c..3d8760d8 100644 --- a/heed/src/envs/env.rs +++ b/heed/src/envs/env.rs @@ -9,7 +9,7 @@ use heed_traits::Comparator; use super::{ custom_key_cmp_wrapper, get_file_fd, metadata_from_fd, DefaultComparator, EnvInfo, FlagSetMode, - OPENED_ENV, + IntegerComparator, OPENED_ENV, }; use crate::cursor::{MoveOperation, RoCursor}; use crate::mdb::ffi::{self, MDB_env}; @@ -158,8 +158,7 @@ impl Env { let rtxn = self.read_txn()?; // Open the main database - let raw_txn = unsafe { rtxn.txn.unwrap().as_mut() }; - let dbi = self.raw_open_dbi::(raw_txn, None, 0)?; + let dbi = self.raw_open_dbi::(rtxn.txn.unwrap(), None, 0)?; // We're going to iterate on the unnamed database let mut cursor = RoCursor::new(&rtxn, dbi)?; @@ -172,8 +171,9 @@ impl Env { let key = String::from_utf8(key.to_vec()).unwrap(); // Calling `ffi::db_stat` on a database instance does not involve key comparison // in LMDB, so it's safe to specify a noop key compare function for it. - let raw_txn = unsafe { rtxn.txn.unwrap().as_mut() }; - if let Ok(dbi) = self.raw_open_dbi::(raw_txn, Some(&key), 0) { + if let Ok(dbi) = + self.raw_open_dbi::(rtxn.txn.unwrap(), Some(&key), 0) + { let mut stat = mem::MaybeUninit::uninit(); unsafe { mdb_result(ffi::mdb_stat(rtxn.txn.unwrap().as_mut(), dbi, stat.as_mut_ptr()))? @@ -268,7 +268,7 @@ impl Env { fn raw_open_dbi( &self, - raw_txn: NonNull, + mut raw_txn: NonNull, name: Option<&str>, flags: u32, ) -> std::result::Result { @@ -282,13 +282,17 @@ impl Env { // safety: The name cstring is cloned by LMDB, we can drop it after. // If a read-only is used with the MDB_CREATE flag, LMDB will throw an error. unsafe { - mdb_result(ffi::mdb_dbi_open(raw_txn, name_ptr, flags, &mut dbi))?; + mdb_result(ffi::mdb_dbi_open(raw_txn.as_mut(), name_ptr, flags, &mut dbi))?; let cmp_type_id = TypeId::of::(); if cmp_type_id != TypeId::of::() && cmp_type_id != TypeId::of::() { - mdb_result(ffi::mdb_set_compare(raw_txn, dbi, Some(custom_key_cmp_wrapper::)))?; + mdb_result(ffi::mdb_set_compare( + raw_txn.as_mut(), + dbi, + Some(custom_key_cmp_wrapper::), + ))?; } }; diff --git a/heed/src/envs/mod.rs b/heed/src/envs/mod.rs index a5769213..9eb31dd3 100644 --- a/heed/src/envs/mod.rs +++ b/heed/src/envs/mod.rs @@ -8,9 +8,7 @@ use std::os::unix::io::{AsRawFd, BorrowedFd, RawFd}; use std::panic::catch_unwind; use std::path::{Path, PathBuf}; use std::process::abort; -use std::ptr::NonNull; use std::sync::{LazyLock, RwLock}; -use std::time::Duration; #[cfg(windows)] use std::{ ffi::OsStr, @@ -181,6 +179,39 @@ impl LexicographicComparator for DefaultComparator { } } +/// A representation of LMDB's `MDB_INTEGERKEY` comparator behavior. +/// +/// This enum is used to indicate a table should be sorted by the keys numeric +/// value in native byte order. When a [`Database`] is created or opened with +/// [`IntegerComparator`], it signifies that the comparator should not be explicitly +/// set via [`ffi::mdb_set_compare`], instead the flag [`AllDatabaseFlags::INTEGER_KEY`] +/// is set on the table. +/// +/// This can only be used on certain types: either `u32` or `usize`. +/// The keys must all be of the same size. +pub enum IntegerComparator {} + +impl Comparator for IntegerComparator { + fn compare(a: &[u8], b: &[u8]) -> Ordering { + #[cfg(target_endian = "big")] + return a.cmp(b); + + #[cfg(target_endian = "little")] + { + let len = a.len(); + + for i in (0..len).rev() { + match a[i].cmp(&b[i]) { + Ordering::Equal => continue, + other => return other, + } + } + + Ordering::Equal + } + } +} + /// Whether to perform compaction while copying an environment. #[derive(Debug, Copy, Clone)] pub enum CompactionOption { diff --git a/heed/src/txn.rs b/heed/src/txn.rs index 7e0d47de..07ba369e 100644 --- a/heed/src/txn.rs +++ b/heed/src/txn.rs @@ -47,7 +47,7 @@ use crate::Result; pub struct RoTxn<'e> { /// Makes the struct covariant and !Sync pub(crate) txn: Option>, - env: Cow<'e, Env>, + env: &'e Env, } impl<'e> RoTxn<'e> { diff --git a/heed3/Cargo.toml b/heed3/Cargo.toml index 1176d5e0..42a6a7b2 100644 --- a/heed3/Cargo.toml +++ b/heed3/Cargo.toml @@ -118,5 +118,5 @@ longer-keys = ["lmdb-master3-sys/longer-keys"] # Examples are located outside the standard heed/examples directory to prevent # conflicts between heed3 and heed examples when working on both crates. [[example]] -name = "heed3-encryption" -path = "../examples/heed3-encryption.rs" +name = "heed3-encrypted" +path = "../examples/heed3-encrypted.rs" From 889b19bd24ab26432f719e112d96131e76cced10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sat, 30 Nov 2024 12:30:32 +0100 Subject: [PATCH 37/51] Fix an elided names lifetime --- heed-types/src/bytes.rs | 2 +- heed-types/src/serde_bincode.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/heed-types/src/bytes.rs b/heed-types/src/bytes.rs index ae602ba0..5f2fba49 100644 --- a/heed-types/src/bytes.rs +++ b/heed-types/src/bytes.rs @@ -11,7 +11,7 @@ pub enum Bytes {} impl<'a> BytesEncode<'a> for Bytes { type EItem = [u8]; - fn bytes_encode(item: &'a Self::EItem) -> Result, BoxedError> { + fn bytes_encode(item: &'a Self::EItem) -> Result, BoxedError> { Ok(Cow::Borrowed(item)) } } diff --git a/heed-types/src/serde_bincode.rs b/heed-types/src/serde_bincode.rs index 37fefe01..b4b42916 100644 --- a/heed-types/src/serde_bincode.rs +++ b/heed-types/src/serde_bincode.rs @@ -14,7 +14,7 @@ where { type EItem = T; - fn bytes_encode(item: &'a Self::EItem) -> Result, BoxedError> { + fn bytes_encode(item: &'a Self::EItem) -> Result, BoxedError> { bincode::serialize(item).map(Cow::Owned).map_err(Into::into) } } From 4a078b047cc998ac3b00a8715de5e0676a76d051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sat, 30 Nov 2024 12:31:00 +0100 Subject: [PATCH 38/51] Bump heed-types to 0.20.2 --- heed-types/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heed-types/Cargo.toml b/heed-types/Cargo.toml index 059e69a8..482ba94b 100644 --- a/heed-types/Cargo.toml +++ b/heed-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "heed-types" -version = "0.20.1" +version = "0.20.2" authors = ["Kerollmops "] description = "The types used with the fully typed LMDB wrapper, heed" license = "MIT" From a9f5a10cb4f244f8f59768a2653cc850c5ea098f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sat, 30 Nov 2024 12:37:42 +0100 Subject: [PATCH 39/51] Improve env already opened error message --- heed/src/lib.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/heed/src/lib.rs b/heed/src/lib.rs index 90a6d618..bcbc19d7 100644 --- a/heed/src/lib.rs +++ b/heed/src/lib.rs @@ -153,7 +153,8 @@ pub enum Error { Encoding(BoxedError), /// Decoding error. Decoding(BoxedError), - /// The environment is already open; close it to be able to open it again. + /// The environment is already open in this program; + /// close it to be able to open it again with different options. EnvAlreadyOpened, } @@ -164,9 +165,10 @@ impl fmt::Display for Error { Error::Mdb(error) => write!(f, "{error}"), Error::Encoding(error) => write!(f, "error while encoding: {error}"), Error::Decoding(error) => write!(f, "error while decoding: {error}"), - Error::EnvAlreadyOpened => { - f.write_str("environment already open; close it to be able to open it again") - } + Error::EnvAlreadyOpened => f.write_str( + "environment already open in this program; \ + close it to be able to open it again with different options", + ), } } } From 6e71efb44dddce41f575874bca6d5d4cd5f2f492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sat, 30 Nov 2024 13:14:39 +0100 Subject: [PATCH 40/51] Reintroduce the prepare_for_closing function and Env/EncryptedEnv: Clone --- examples/heed3-encrypted.rs | 8 ++--- heed/Cargo.toml | 1 + heed/src/envs/encrypted_env.rs | 14 ++++++-- heed/src/envs/env.rs | 53 +++++++++++++++++++++++-------- heed/src/envs/env_open_options.rs | 6 ++-- heed/src/envs/mod.rs | 36 +++++++++++++++++++-- heed/src/lib.rs | 4 +-- heed/src/txn.rs | 35 ++++++++++---------- heed3/Cargo.toml | 1 + 9 files changed, 114 insertions(+), 44 deletions(-) diff --git a/examples/heed3-encrypted.rs b/examples/heed3-encrypted.rs index acb3c671..7f7b9332 100644 --- a/examples/heed3-encrypted.rs +++ b/examples/heed3-encrypted.rs @@ -5,7 +5,7 @@ use std::path::Path; use argon2::Argon2; use chacha20poly1305::{ChaCha20Poly1305, Key}; use heed3::types::*; -use heed3::{Database, EnvOpenOptions}; +use heed3::EnvOpenOptions; fn main() -> Result<(), Box> { let env_path = Path::new("target").join("encrypt.mdb"); @@ -40,14 +40,14 @@ fn main() -> Result<(), Box> { db.put(&mut wtxn, key1, val1)?; db.put(&mut wtxn, key2, val2)?; wtxn.commit()?; - // env.prepare_for_closing().wait(); + env.prepare_for_closing().wait(); // We reopen the environment now - let env = unsafe { options.open(&env_path)? }; + let env = unsafe { options.open_encrypted::(key, &env_path)? }; // We check that the secret entries are correctly decrypted let mut rtxn = env.read_txn()?; - let db: Database = env.open_database(&rtxn, Some("first"))?.unwrap(); + let db = env.open_database::(&rtxn, Some("first"))?.unwrap(); let mut iter = db.iter(&mut rtxn)?; assert_eq!(iter.next().transpose()?, Some((key1, val1))); assert_eq!(iter.next().transpose()?, Some((key2, val2))); diff --git a/heed/Cargo.toml b/heed/Cargo.toml index 35374010..fb8eae96 100644 --- a/heed/Cargo.toml +++ b/heed/Cargo.toml @@ -20,6 +20,7 @@ lmdb-master-sys = { version = "0.2.4", path = "../lmdb-master-sys" } once_cell = "1.19.0" page_size = "0.6.0" serde = { version = "1.0.203", features = ["derive"], optional = true } +synchronoise = "1.0.1" [dev-dependencies] serde = { version = "1.0.203", features = ["derive"] } diff --git a/heed/src/envs/encrypted_env.rs b/heed/src/envs/encrypted_env.rs index b37d5b2c..f78a8fad 100644 --- a/heed/src/envs/encrypted_env.rs +++ b/heed/src/envs/encrypted_env.rs @@ -6,12 +6,13 @@ use std::path::Path; use aead::generic_array::typenum::Unsigned; use aead::{AeadMutInPlace, Key, KeyInit, Nonce, Tag}; -use super::{Env, EnvInfo, FlagSetMode}; +use super::{Env, EnvClosingEvent, EnvInfo, FlagSetMode}; use crate::databases::{EncryptedDatabase, EncryptedDatabaseOpenOptions}; use crate::mdb::ffi::{self}; use crate::{CompactionOption, EnvFlags, Result, RoTxn, RwTxn, Unspecified}; /// An environment handle constructed by using [`EnvOpenOptions::open_encrypted`]. +#[derive(Clone)] pub struct EncryptedEnv { pub(crate) inner: Env, } @@ -278,6 +279,15 @@ impl EncryptedEnv { self.inner.path() } + /// Returns an `EnvClosingEvent` that can be used to wait for the closing event, + /// multiple threads can wait on this event. + /// + /// Make sure that you drop all the copies of `Env`s you have, env closing are triggered + /// when all references are dropped, the last one will eventually close the environment. + pub fn prepare_for_closing(self) -> EnvClosingEvent { + self.inner.prepare_for_closing() + } + /// Check for stale entries in the reader lock table and clear them. /// /// Returns the number of stale readers cleared. @@ -311,7 +321,7 @@ unsafe impl Sync for EncryptedEnv {} impl fmt::Debug for EncryptedEnv { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("EncryptedEnv") - .field("path", &self.inner.path.display()) + .field("path", &self.inner.path().display()) .finish_non_exhaustive() } } diff --git a/heed/src/envs/env.rs b/heed/src/envs/env.rs index 3d8760d8..c7e60b41 100644 --- a/heed/src/envs/env.rs +++ b/heed/src/envs/env.rs @@ -3,13 +3,15 @@ use std::ffi::CString; use std::fs::File; use std::path::{Path, PathBuf}; use std::ptr::{self, NonNull}; +use std::sync::Arc; use std::{fmt, io, mem}; use heed_traits::Comparator; +use synchronoise::SignalEvent; use super::{ - custom_key_cmp_wrapper, get_file_fd, metadata_from_fd, DefaultComparator, EnvInfo, FlagSetMode, - IntegerComparator, OPENED_ENV, + custom_key_cmp_wrapper, get_file_fd, metadata_from_fd, DefaultComparator, EnvClosingEvent, + EnvInfo, FlagSetMode, IntegerComparator, OPENED_ENV, }; use crate::cursor::{MoveOperation, RoCursor}; use crate::mdb::ffi::{self, MDB_env}; @@ -21,18 +23,24 @@ use crate::{ }; /// An environment handle constructed by using [`EnvOpenOptions::open`]. +#[derive(Clone)] pub struct Env { - env_ptr: NonNull, - pub(crate) path: PathBuf, + inner: Arc, } impl Env { pub(crate) fn new(env_ptr: NonNull, path: PathBuf) -> Env { - Env { env_ptr, path } + Env { + inner: Arc::new(EnvInner { + env_ptr, + path, + signal_event: Arc::new(SignalEvent::manual(false)), + }), + } } pub(crate) fn env_mut_ptr(&self) -> NonNull { - self.env_ptr + self.inner.env_ptr } /// The size of the data file on disk. @@ -125,7 +133,7 @@ impl Env { /// Returns some basic informations about this environment. pub fn info(&self) -> EnvInfo { let mut raw_info = mem::MaybeUninit::uninit(); - unsafe { ffi::mdb_env_info(self.env_ptr.as_ptr(), raw_info.as_mut_ptr()) }; + unsafe { ffi::mdb_env_info(self.inner.env_ptr.as_ptr(), raw_info.as_mut_ptr()) }; let raw_info = unsafe { raw_info.assume_init() }; EnvInfo { @@ -410,19 +418,28 @@ impl Env { option: CompactionOption, ) -> Result<()> { let flags = if let CompactionOption::Enabled = option { ffi::MDB_CP_COMPACT } else { 0 }; - mdb_result(ffi::mdb_env_copyfd2(self.env_ptr.as_ptr(), fd, flags))?; + mdb_result(ffi::mdb_env_copyfd2(self.inner.env_ptr.as_ptr(), fd, flags))?; Ok(()) } /// Flush the data buffers to disk. pub fn force_sync(&self) -> Result<()> { - unsafe { mdb_result(ffi::mdb_env_sync(self.env_ptr.as_ptr(), 1))? } + unsafe { mdb_result(ffi::mdb_env_sync(self.inner.env_ptr.as_ptr(), 1))? } Ok(()) } /// Returns the canonicalized path where this env lives. pub fn path(&self) -> &Path { - &self.path + &self.inner.path + } + + /// Returns an `EnvClosingEvent` that can be used to wait for the closing event, + /// multiple threads can wait on this event. + /// + /// Make sure that you drop all the copies of `Env`s you have, env closing are triggered + /// when all references are dropped, the last one will eventually close the environment. + pub fn prepare_for_closing(self) -> EnvClosingEvent { + EnvClosingEvent(self.inner.signal_event.clone()) } /// Check for stale entries in the reader lock table and clear them. @@ -430,7 +447,7 @@ impl Env { /// Returns the number of stale readers cleared. pub fn clear_stale_readers(&self) -> Result { let mut dead: i32 = 0; - unsafe { mdb_result(ffi::mdb_reader_check(self.env_ptr.as_ptr(), &mut dead))? } + unsafe { mdb_result(ffi::mdb_reader_check(self.inner.env_ptr.as_ptr(), &mut dead))? } // safety: The reader_check function asks for an i32, initialize it to zero // and never decrements it. It is safe to use either an u32 or u64 (usize). Ok(dead as usize) @@ -471,14 +488,22 @@ unsafe impl Sync for Env {} impl fmt::Debug for Env { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("Env").field("path", &self.path.display()).finish_non_exhaustive() + f.debug_struct("Env").field("path", &self.inner.path.display()).finish_non_exhaustive() } } -impl Drop for Env { +#[derive(Clone)] +pub(crate) struct EnvInner { + env_ptr: NonNull, + signal_event: Arc, + pub(crate) path: PathBuf, +} + +impl Drop for EnvInner { fn drop(&mut self) { unsafe { ffi::mdb_env_close(self.env_ptr.as_mut()) }; let mut lock = OPENED_ENV.write().unwrap(); - debug_assert!(lock.remove(&self.path)); + let removed = lock.remove(&self.path); + debug_assert!(removed); } } diff --git a/heed/src/envs/env_open_options.rs b/heed/src/envs/env_open_options.rs index 72418c2e..859a91a2 100644 --- a/heed/src/envs/env_open_options.rs +++ b/heed/src/envs/env_open_options.rs @@ -213,7 +213,8 @@ impl EnvOpenOptions { match mdb_result(result) { Ok(()) => { let env_ptr = NonNull::new(env).unwrap(); - debug_assert!(lock.insert(path.clone())); + let inserted = lock.insert(path.clone()); + debug_assert!(inserted); Ok(Env::new(env_ptr, path)) } Err(e) => { @@ -404,7 +405,8 @@ impl EnvOpenOptions { match mdb_result(result) { Ok(()) => { let env_ptr = NonNull::new(env).unwrap(); - debug_assert!(lock.insert(path.clone())); + let inserted = lock.insert(path.clone()); + debug_assert!(inserted); Ok(EncryptedEnv { inner: Env::new(env_ptr, path) }) } Err(e) => { diff --git a/heed/src/envs/mod.rs b/heed/src/envs/mod.rs index 9eb31dd3..3454099a 100644 --- a/heed/src/envs/mod.rs +++ b/heed/src/envs/mod.rs @@ -2,20 +2,22 @@ use std::cmp::Ordering; use std::collections::HashSet; use std::ffi::c_void; use std::fs::{File, Metadata}; -use std::io; #[cfg(unix)] use std::os::unix::io::{AsRawFd, BorrowedFd, RawFd}; use std::panic::catch_unwind; use std::path::{Path, PathBuf}; use std::process::abort; -use std::sync::{LazyLock, RwLock}; +use std::sync::{Arc, LazyLock, RwLock}; +use std::time::Duration; #[cfg(windows)] use std::{ ffi::OsStr, os::windows::io::{AsRawHandle as _, BorrowedHandle, RawHandle}, }; +use std::{fmt, io}; use heed_traits::{Comparator, LexicographicComparator}; +use synchronoise::event::SignalEvent; use crate::mdb::ffi; @@ -50,6 +52,36 @@ pub struct EnvInfo { pub number_of_readers: u32, } +/// A structure that can be used to wait for the closing event. +/// Multiple threads can wait on this event. +#[derive(Clone)] +pub struct EnvClosingEvent(Arc); + +impl EnvClosingEvent { + /// Blocks this thread until the environment is effectively closed. + /// + /// # Safety + /// + /// Make sure that you don't have any copy of the environment in the thread + /// that is waiting for a close event. If you do, you will have a deadlock. + pub fn wait(&self) { + self.0.wait() + } + + /// Blocks this thread until either the environment has been closed + /// or until the timeout elapses. Returns `true` if the environment + /// has been effectively closed. + pub fn wait_timeout(&self, timeout: Duration) -> bool { + self.0.wait_timeout(timeout) + } +} + +impl fmt::Debug for EnvClosingEvent { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("EnvClosingEvent").finish() + } +} + // Thanks to the mozilla/rkv project // Workaround the UNC path on Windows, see https://github.com/rust-lang/rust/issues/42869. // Otherwise, `Env::from_env()` will panic with error_no(123). diff --git a/heed/src/lib.rs b/heed/src/lib.rs index bcbc19d7..0ce180e7 100644 --- a/heed/src/lib.rs +++ b/heed/src/lib.rs @@ -85,8 +85,8 @@ pub use self::databases::{EncryptedDatabase, EncryptedDatabaseOpenOptions}; #[cfg(master3)] pub use self::envs::EncryptedEnv; pub use self::envs::{ - CompactionOption, DefaultComparator, Env, EnvInfo, EnvOpenOptions, FlagSetMode, - IntegerComparator, + CompactionOption, DefaultComparator, Env, EnvClosingEvent, EnvInfo, EnvOpenOptions, + FlagSetMode, IntegerComparator, }; pub use self::iterator::{ RoIter, RoPrefix, RoRange, RoRevIter, RoRevPrefix, RoRevRange, RwIter, RwPrefix, RwRange, diff --git a/heed/src/txn.rs b/heed/src/txn.rs index 07ba369e..657f1262 100644 --- a/heed/src/txn.rs +++ b/heed/src/txn.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::ops::Deref; use std::ptr::{self, NonNull}; @@ -47,7 +48,7 @@ use crate::Result; pub struct RoTxn<'e> { /// Makes the struct covariant and !Sync pub(crate) txn: Option>, - env: &'e Env, + env: Cow<'e, Env>, } impl<'e> RoTxn<'e> { @@ -63,24 +64,22 @@ impl<'e> RoTxn<'e> { ))? }; - Ok(RoTxn { txn: NonNull::new(txn), env }) + Ok(RoTxn { txn: NonNull::new(txn), env: Cow::Borrowed(env) }) } - // TODO replace this by an ArcRoTxn pub(crate) fn static_read_txn(env: Env) -> Result> { - // let mut txn: *mut ffi::MDB_txn = ptr::null_mut(); - - // unsafe { - // mdb_result(ffi::mdb_txn_begin( - // env.env_mut_ptr(), - // ptr::null_mut(), - // ffi::MDB_RDONLY, - // &mut txn, - // ))? - // }; - - // Ok(RoTxn { txn, env: Cow::Owned(env) }) - todo!() + let mut txn: *mut ffi::MDB_txn = ptr::null_mut(); + + unsafe { + mdb_result(ffi::mdb_txn_begin( + env.env_mut_ptr().as_mut(), + ptr::null_mut(), + ffi::MDB_RDONLY, + &mut txn, + ))? + }; + + Ok(RoTxn { txn: NonNull::new(txn), env: Cow::Owned(env) }) } pub(crate) fn env_mut_ptr(&self) -> NonNull { @@ -172,7 +171,7 @@ impl<'p> RwTxn<'p> { ))? }; - Ok(RwTxn { txn: RoTxn { txn: NonNull::new(txn), env } }) + Ok(RwTxn { txn: RoTxn { txn: NonNull::new(txn), env: Cow::Borrowed(env) } }) } pub(crate) fn nested(env: &'p Env, parent: &'p mut RwTxn) -> Result> { @@ -183,7 +182,7 @@ impl<'p> RwTxn<'p> { mdb_result(ffi::mdb_txn_begin(env.env_mut_ptr().as_mut(), parent_ptr, 0, &mut txn))? }; - Ok(RwTxn { txn: RoTxn { txn: NonNull::new(txn), env } }) + Ok(RwTxn { txn: RoTxn { txn: NonNull::new(txn), env: Cow::Borrowed(env) } }) } pub(crate) fn env_mut_ptr(&self) -> NonNull { diff --git a/heed3/Cargo.toml b/heed3/Cargo.toml index 42a6a7b2..3ebac3c3 100644 --- a/heed3/Cargo.toml +++ b/heed3/Cargo.toml @@ -23,6 +23,7 @@ lmdb-master3-sys = { version = "0.2.4", path = "../lmdb-master3-sys" } once_cell = "1.19.0" page_size = "0.6.0" serde = { version = "1.0.203", features = ["derive"], optional = true } +synchronoise = "1.0.1" [dev-dependencies] # TODO update dependencies From 9cc209c26202efb4f493475879146ce7601f12a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sat, 30 Nov 2024 13:30:40 +0100 Subject: [PATCH 41/51] Factorize Env opening function --- heed/src/envs/env_open_options.rs | 112 ++++++++---------------------- heed/src/mdb/lmdb_ffi.rs | 4 +- 2 files changed, 31 insertions(+), 85 deletions(-) diff --git a/heed/src/envs/env_open_options.rs b/heed/src/envs/env_open_options.rs index 859a91a2..1ec1a8f8 100644 --- a/heed/src/envs/env_open_options.rs +++ b/heed/src/envs/env_open_options.rs @@ -18,6 +18,7 @@ use super::{canonicalize_path, OPENED_ENV}; #[cfg(windows)] use crate::envs::OsStrExtLmdb as _; use crate::mdb::error::mdb_result; +#[cfg(master3)] use crate::mdb::ffi; use crate::{EnvFlags, Error, Result}; @@ -151,79 +152,11 @@ impl EnvOpenOptions { /// [^7]: /// [^8]: pub unsafe fn open>(&self, path: P) -> Result { - /// TODO change the function description - /// and deduplicate the code - let mut lock = OPENED_ENV.write().unwrap(); - - let path = match canonicalize_path(path.as_ref()) { - Err(err) => { - if err.kind() == NotFound && self.flags.contains(EnvFlags::NO_SUB_DIR) { - let path = path.as_ref(); - match path.parent().zip(path.file_name()) { - Some((dir, file_name)) => canonicalize_path(dir)?.join(file_name), - None => return Err(err.into()), - } - } else { - return Err(err.into()); - } - } - Ok(path) => path, - }; - - if lock.contains(&path) { - Err(Error::EnvAlreadyOpened) - } else { - let path_str = CString::new(path.as_os_str().as_bytes()).unwrap(); - - unsafe { - let mut env: *mut ffi::MDB_env = ptr::null_mut(); - mdb_result(ffi::mdb_env_create(&mut env))?; - - if let Some(size) = self.map_size { - if size % page_size::get() != 0 { - let msg = format!( - "map size ({}) must be a multiple of the system page size ({})", - size, - page_size::get() - ); - return Err(Error::Io(io::Error::new(io::ErrorKind::InvalidInput, msg))); - } - mdb_result(ffi::mdb_env_set_mapsize(env, size))?; - } - - if let Some(readers) = self.max_readers { - mdb_result(ffi::mdb_env_set_maxreaders(env, readers))?; - } - - if let Some(dbs) = self.max_dbs { - mdb_result(ffi::mdb_env_set_maxdbs(env, dbs))?; - } - - // When the `read-txn-no-tls` feature is enabled, we must force LMDB - // to avoid using the thread local storage, this way we allow users - // to use references of RoTxn between threads safely. - let flags = if cfg!(feature = "read-txn-no-tls") { - // TODO make this a ZST flag on the Env and on RoTxn (make them Send when we can) - self.flags | EnvFlags::NO_TLS - } else { - self.flags - }; - - let result = ffi::mdb_env_open(env, path_str.as_ptr(), flags.bits(), 0o600); - match mdb_result(result) { - Ok(()) => { - let env_ptr = NonNull::new(env).unwrap(); - let inserted = lock.insert(path.clone()); - debug_assert!(inserted); - Ok(Env::new(env_ptr, path)) - } - Err(e) => { - ffi::mdb_env_close(env); - Err(e.into()) - } - } - } - } + self.raw_open_with_encryption( + path.as_ref(), + #[cfg(master3)] + None, + ) } /// Creates a blank new set of options ready for configuration and specifies that @@ -337,12 +270,23 @@ impl EnvOpenOptions { E: AeadMutInPlace + KeyInit, P: AsRef, { + self.raw_open_with_encryption( + path.as_ref(), + Some((Some(encrypt_func_wrapper::), &key, ::TagSize::U32)), + ) + .map(|inner| EncryptedEnv { inner }) + } + + fn raw_open_with_encryption( + &self, + path: &Path, + #[cfg(master3)] enc: Option<(ffi::MDB_enc_func, &[u8], u32)>, + ) -> Result { let mut lock = OPENED_ENV.write().unwrap(); - let path = match canonicalize_path(path.as_ref()) { + let path = match canonicalize_path(path) { Err(err) => { if err.kind() == NotFound && self.flags.contains(EnvFlags::NO_SUB_DIR) { - let path = path.as_ref(); match path.parent().zip(path.file_name()) { Some((dir, file_name)) => canonicalize_path(dir)?.join(file_name), None => return Err(err.into()), @@ -363,13 +307,15 @@ impl EnvOpenOptions { let mut env: *mut ffi::MDB_env = ptr::null_mut(); mdb_result(ffi::mdb_env_create(&mut env))?; - let encrypt_key = crate::into_val(&key); - mdb_result(ffi::mdb_env_set_encrypt( - env, - Some(encrypt_func_wrapper::), - &encrypt_key, - ::TagSize::U32, - ))?; + #[cfg(master3)] + if let Some((encrypt_func, key, tag_size)) = enc { + mdb_result(ffi::mdb_env_set_encrypt( + env, + encrypt_func, + &crate::into_val(key), + tag_size, + ))?; + } if let Some(size) = self.map_size { if size % page_size::get() != 0 { @@ -407,7 +353,7 @@ impl EnvOpenOptions { let env_ptr = NonNull::new(env).unwrap(); let inserted = lock.insert(path.clone()); debug_assert!(inserted); - Ok(EncryptedEnv { inner: Env::new(env_ptr, path) }) + Ok(Env::new(env_ptr, path)) } Err(e) => { ffi::mdb_env_close(env); diff --git a/heed/src/mdb/lmdb_ffi.rs b/heed/src/mdb/lmdb_ffi.rs index cb8b3ba5..1f68da0e 100644 --- a/heed/src/mdb/lmdb_ffi.rs +++ b/heed/src/mdb/lmdb_ffi.rs @@ -1,7 +1,5 @@ use std::ptr; -#[cfg(master3)] -pub use ffi::mdb_env_set_encrypt; pub use ffi::{ mdb_cursor_close, mdb_cursor_del, mdb_cursor_get, mdb_cursor_open, mdb_cursor_put, mdb_dbi_open, mdb_del, mdb_drop, mdb_env_close, mdb_env_copyfd2, mdb_env_create, @@ -13,6 +11,8 @@ pub use ffi::{ MDB_RDONLY, MDB_RESERVE, }; #[cfg(master3)] +pub use ffi::{mdb_env_set_encrypt, MDB_enc_func}; +#[cfg(master3)] use lmdb_master3_sys as ffi; #[cfg(not(master3))] use lmdb_master_sys as ffi; From 018ff7e3a1003ac5853fc3d79abe37f313024b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sat, 30 Nov 2024 15:46:43 +0100 Subject: [PATCH 42/51] Make the CI check heed3 --- .github/workflows/rust.yml | 59 +++----------------------------------- 1 file changed, 4 insertions(+), 55 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5b385798..dc5f2885 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -51,35 +51,9 @@ jobs: - name: Run cargo check run: | cargo clean - cp heed3/Cargo.toml heed/ + ./convert-to-heed3.sh cargo check -p heed3 - check-heed3-encryption: - name: Check the heed3-encryption project - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - include: - - os: ubuntu-latest - - os: windows-latest - - os: macos-latest - - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - name: Run cargo check - run: | - cargo clean - cp heed3-encryption/Cargo.toml heed/ - cargo check -p heed3-encryption - check_all_features: name: Check all the features of the heed project runs-on: ${{ matrix.os }} @@ -130,8 +104,8 @@ jobs: - name: Run cargo test run: | cargo clean - cp heed3-encryption/Cargo.toml heed/ - cargo check --all-features -p heed3-encryption + ./convert-to-heed3.sh + cargo check --all-features -p heed3 examples: name: Run the heed examples @@ -181,32 +155,7 @@ jobs: - name: Run the examples run: | cargo clean - cp heed3/Cargo.toml heed/ - cargo run --example 2>&1 | grep -E '^ '| xargs -n1 cargo run --example - - heed3-encryption-examples: - name: Run the heed3-encryption examples - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - include: - - os: ubuntu-latest - - os: macos-latest - - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - name: Run the examples - run: | - cargo clean - cp heed3-encryption/Cargo.toml heed/ + ./convert-to-heed3.sh cargo run --example 2>&1 | grep -E '^ '| xargs -n1 cargo run --example fmt: From 775a6986e612a6e0c99d308d08b3e6dd4a08fcc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sat, 30 Nov 2024 15:40:14 +0100 Subject: [PATCH 43/51] Fix imports --- heed/src/envs/env_open_options.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/heed/src/envs/env_open_options.rs b/heed/src/envs/env_open_options.rs index 1ec1a8f8..cbf0e500 100644 --- a/heed/src/envs/env_open_options.rs +++ b/heed/src/envs/env_open_options.rs @@ -18,7 +18,6 @@ use super::{canonicalize_path, OPENED_ENV}; #[cfg(windows)] use crate::envs::OsStrExtLmdb as _; use crate::mdb::error::mdb_result; -#[cfg(master3)] use crate::mdb::ffi; use crate::{EnvFlags, Error, Result}; From 36a785b7df50e261c7c0f8af033f1c75ed00d1e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sat, 30 Nov 2024 15:43:42 +0100 Subject: [PATCH 44/51] Fix the heed3 script on Linux --- convert-to-heed3.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/convert-to-heed3.sh b/convert-to-heed3.sh index f92ef4b2..16d7b720 100755 --- a/convert-to-heed3.sh +++ b/convert-to-heed3.sh @@ -14,7 +14,11 @@ cp heed3/Cargo.toml heed/Cargo.toml # ...and replaces the `heed::` string by the `heed3::` one. for file in $(find heed/src -type f -name "*.rs"); do - sed -i '' 's/heed::/heed3::/g' "$file" + if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i '' 's/heed::/heed3::/g' "$file" + else + sed -i 's/heed::/heed3::/g' "$file" + fi done echo "Heed3 crate setup completed successfully. Copied and modified configurations for the heed crate." From de53beca8928e12bba468e322a78b5a960ddf50b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sun, 1 Dec 2024 12:38:30 +0100 Subject: [PATCH 45/51] Fix the prepare_for_closing system --- heed/src/envs/env.rs | 342 +++++++++++++++++++++++++++++- heed/src/envs/env_open_options.rs | 11 +- heed/src/envs/mod.rs | 11 +- heed/src/lib.rs | 4 +- 4 files changed, 349 insertions(+), 19 deletions(-) diff --git a/heed/src/envs/env.rs b/heed/src/envs/env.rs index c7e60b41..91e30cc4 100644 --- a/heed/src/envs/env.rs +++ b/heed/src/envs/env.rs @@ -29,14 +29,12 @@ pub struct Env { } impl Env { - pub(crate) fn new(env_ptr: NonNull, path: PathBuf) -> Env { - Env { - inner: Arc::new(EnvInner { - env_ptr, - path, - signal_event: Arc::new(SignalEvent::manual(false)), - }), - } + pub(crate) fn new( + env_ptr: NonNull, + path: PathBuf, + signal_event: Arc, + ) -> Env { + Env { inner: Arc::new(EnvInner { env_ptr, path, signal_event }) } } pub(crate) fn env_mut_ptr(&self) -> NonNull { @@ -492,7 +490,6 @@ impl fmt::Debug for Env { } } -#[derive(Clone)] pub(crate) struct EnvInner { env_ptr: NonNull, signal_event: Arc, @@ -501,9 +498,332 @@ pub(crate) struct EnvInner { impl Drop for EnvInner { fn drop(&mut self) { - unsafe { ffi::mdb_env_close(self.env_ptr.as_mut()) }; let mut lock = OPENED_ENV.write().unwrap(); let removed = lock.remove(&self.path); - debug_assert!(removed); + debug_assert!(removed.is_some()); + unsafe { ffi::mdb_env_close(self.env_ptr.as_mut()) }; + self.signal_event.signal(); + } +} + +#[cfg(test)] +mod tests { + use std::io::ErrorKind; + use std::time::Duration; + use std::{fs, thread}; + + use crate::types::*; + use crate::{env_closing_event, EnvOpenOptions, Error}; + + #[test] + fn close_env() { + let dir = tempfile::tempdir().unwrap(); + let env = unsafe { + EnvOpenOptions::new() + .map_size(10 * 1024 * 1024) // 10MB + .max_dbs(30) + .open(dir.path()) + .unwrap() + }; + + // Force a thread to keep the env for 1 second. + let env_cloned = env.clone(); + thread::spawn(move || { + let _env = env_cloned; + thread::sleep(Duration::from_secs(1)); + }); + + let mut wtxn = env.write_txn().unwrap(); + let db = env.create_database::(&mut wtxn, None).unwrap(); + wtxn.commit().unwrap(); + + // Create an ordered list of keys... + let mut wtxn = env.write_txn().unwrap(); + db.put(&mut wtxn, "hello", "hello").unwrap(); + db.put(&mut wtxn, "world", "world").unwrap(); + + let mut iter = db.iter(&wtxn).unwrap(); + assert_eq!(iter.next().transpose().unwrap(), Some(("hello", "hello"))); + assert_eq!(iter.next().transpose().unwrap(), Some(("world", "world"))); + assert_eq!(iter.next().transpose().unwrap(), None); + drop(iter); + + wtxn.commit().unwrap(); + + let signal_event = env.prepare_for_closing(); + + eprintln!("waiting for the env to be closed"); + signal_event.wait(); + eprintln!("env closed successfully"); + + // Make sure we don't have a reference to the env + assert!(env_closing_event(dir.path()).is_none()); + } + + #[test] + fn reopen_env_with_different_options_is_err() { + let dir = tempfile::tempdir().unwrap(); + let _env = unsafe { + EnvOpenOptions::new() + .map_size(10 * 1024 * 1024) // 10MB + .open(dir.path()) + .unwrap() + }; + + let result = unsafe { + EnvOpenOptions::new() + .map_size(12 * 1024 * 1024) // 12MB + .open(dir.path()) + }; + + assert!(matches!(result, Err(Error::EnvAlreadyOpened))); + } + + #[test] + fn open_env_with_named_path() { + let dir = tempfile::tempdir().unwrap(); + fs::create_dir_all(dir.path().join("babar.mdb")).unwrap(); + let _env = unsafe { + EnvOpenOptions::new() + .map_size(10 * 1024 * 1024) // 10MB + .open(dir.path().join("babar.mdb")) + .unwrap() + }; + + let error = unsafe { + EnvOpenOptions::new() + .map_size(10 * 1024 * 1024) // 10MB + .open(dir.path().join("babar.mdb")) + .unwrap_err() + }; + + assert!(matches!(error, Error::EnvAlreadyOpened)); + } + + #[test] + #[cfg(not(windows))] + fn open_database_with_writemap_flag() { + let dir = tempfile::tempdir().unwrap(); + let mut envbuilder = EnvOpenOptions::new(); + envbuilder.map_size(10 * 1024 * 1024); // 10MB + envbuilder.max_dbs(10); + unsafe { envbuilder.flags(crate::EnvFlags::WRITE_MAP) }; + let env = unsafe { envbuilder.open(dir.path()).unwrap() }; + + let mut wtxn = env.write_txn().unwrap(); + let _db = env.create_database::(&mut wtxn, Some("my-super-db")).unwrap(); + wtxn.commit().unwrap(); + } + + #[test] + fn open_database_with_nosubdir() { + let dir = tempfile::tempdir().unwrap(); + let mut envbuilder = EnvOpenOptions::new(); + unsafe { envbuilder.flags(crate::EnvFlags::NO_SUB_DIR) }; + let _env = unsafe { envbuilder.open(dir.path().join("data.mdb")).unwrap() }; + } + + #[test] + fn create_database_without_commit() { + let dir = tempfile::tempdir().unwrap(); + let env = unsafe { + EnvOpenOptions::new() + .map_size(10 * 1024 * 1024) // 10MB + .max_dbs(10) + .open(dir.path()) + .unwrap() + }; + + let mut wtxn = env.write_txn().unwrap(); + let _db = env.create_database::(&mut wtxn, Some("my-super-db")).unwrap(); + wtxn.abort(); + + let rtxn = env.read_txn().unwrap(); + let option = env.open_database::(&rtxn, Some("my-super-db")).unwrap(); + assert!(option.is_none()); + } + + #[test] + fn open_already_existing_database() { + let dir = tempfile::tempdir().unwrap(); + let env = unsafe { + EnvOpenOptions::new() + .map_size(10 * 1024 * 1024) // 10MB + .max_dbs(10) + .open(dir.path()) + .unwrap() + }; + + // we first create a database + let mut wtxn = env.write_txn().unwrap(); + let _db = env.create_database::(&mut wtxn, Some("my-super-db")).unwrap(); + wtxn.commit().unwrap(); + + // Close the environement and reopen it, databases must not be loaded in memory. + env.prepare_for_closing().wait(); + let env = unsafe { + EnvOpenOptions::new() + .map_size(10 * 1024 * 1024) // 10MB + .max_dbs(10) + .open(dir.path()) + .unwrap() + }; + + let rtxn = env.read_txn().unwrap(); + let option = env.open_database::(&rtxn, Some("my-super-db")).unwrap(); + assert!(option.is_some()); + } + + #[test] + fn resize_database() { + let dir = tempfile::tempdir().unwrap(); + let page_size = page_size::get(); + let env = unsafe { + EnvOpenOptions::new().map_size(9 * page_size).max_dbs(1).open(dir.path()).unwrap() + }; + + let mut wtxn = env.write_txn().unwrap(); + let db = env.create_database::(&mut wtxn, Some("my-super-db")).unwrap(); + wtxn.commit().unwrap(); + + let mut wtxn = env.write_txn().unwrap(); + for i in 0..64 { + db.put(&mut wtxn, &i.to_string(), "world").unwrap(); + } + wtxn.commit().unwrap(); + + let mut wtxn = env.write_txn().unwrap(); + for i in 64..128 { + db.put(&mut wtxn, &i.to_string(), "world").unwrap(); + } + wtxn.commit().expect_err("cannot commit a transaction that would reach the map size limit"); + + unsafe { + env.resize(10 * page_size).unwrap(); + } + let mut wtxn = env.write_txn().unwrap(); + for i in 64..128 { + db.put(&mut wtxn, &i.to_string(), "world").unwrap(); + } + wtxn.commit().expect("transaction should commit after resizing the map size"); + + assert_eq!(10 * page_size, env.info().map_size); + } + + /// Non-regression test for + /// + /// + /// We should be able to open database Read-Only Env with + /// no prior Read-Write Env opening. And query data. + #[test] + fn open_read_only_without_no_env_opened_before() { + let expected_data0 = "Data Expected db0"; + let dir = tempfile::tempdir().unwrap(); + + { + // We really need this env to be dropped before the read-only access. + let env = unsafe { + EnvOpenOptions::new() + .map_size(16 * 1024 * 1024 * 1024) // 10MB + .max_dbs(32) + .open(dir.path()) + .unwrap() + }; + let mut wtxn = env.write_txn().unwrap(); + let database0 = env.create_database::(&mut wtxn, Some("shared0")).unwrap(); + + wtxn.commit().unwrap(); + let mut wtxn = env.write_txn().unwrap(); + database0.put(&mut wtxn, "shared0", expected_data0).unwrap(); + wtxn.commit().unwrap(); + // We also really need that no other env reside in memory in other thread doing tests. + env.prepare_for_closing().wait(); + } + + { + // Open now we do a read-only opening + let env = unsafe { + EnvOpenOptions::new() + .map_size(16 * 1024 * 1024 * 1024) // 10MB + .max_dbs(32) + .open(dir.path()) + .unwrap() + }; + let database0 = { + let rtxn = env.read_txn().unwrap(); + let database0 = + env.open_database::(&rtxn, Some("shared0")).unwrap().unwrap(); + // This commit is mandatory if not committed you might get + // Io(Os { code: 22, kind: InvalidInput, message: "Invalid argument" }) + rtxn.commit().unwrap(); + database0 + }; + + { + // If we didn't committed the opening it might fail with EINVAL. + let rtxn = env.read_txn().unwrap(); + let value = database0.get(&rtxn, "shared0").unwrap().unwrap(); + assert_eq!(value, expected_data0); + } + + env.prepare_for_closing().wait(); + } + + // To avoid reintroducing the bug let's try to open again but without the commit + { + // Open now we do a read-only opening + let env = unsafe { + EnvOpenOptions::new() + .map_size(16 * 1024 * 1024 * 1024) // 10MB + .max_dbs(32) + .open(dir.path()) + .unwrap() + }; + let database0 = { + let rtxn = env.read_txn().unwrap(); + let database0 = + env.open_database::(&rtxn, Some("shared0")).unwrap().unwrap(); + // No commit it's important, dropping explicitly + drop(rtxn); + database0 + }; + + { + // We didn't committed the opening we will get EINVAL. + let rtxn = env.read_txn().unwrap(); + // The dbg!() is intentional in case of a change in rust-std or in lmdb related + // to the windows error. + let err = dbg!(database0.get(&rtxn, "shared0")); + + // The error kind is still ErrorKind Uncategorized on windows. + // Behind it's a ERROR_BAD_COMMAND code 22 like EINVAL. + if cfg!(windows) { + assert!(err.is_err()); + } else { + assert!( + matches!(err, Err(Error::Io(ref e)) if e.kind() == ErrorKind::InvalidInput) + ); + } + } + + env.prepare_for_closing().wait(); + } + } + + #[test] + fn max_key_size() { + let dir = tempfile::tempdir().unwrap(); + let env = unsafe { EnvOpenOptions::new().open(dir.path().join(dir.path())).unwrap() }; + let maxkeysize = env.max_key_size(); + + eprintln!("maxkeysize: {}", maxkeysize); + + if cfg!(feature = "longer-keys") { + // Should be larger than the default of 511 + assert!(maxkeysize > 511); + } else { + // Should be the default of 511 + assert_eq!(maxkeysize, 511); + } } } diff --git a/heed/src/envs/env_open_options.rs b/heed/src/envs/env_open_options.rs index cbf0e500..f4c7c241 100644 --- a/heed/src/envs/env_open_options.rs +++ b/heed/src/envs/env_open_options.rs @@ -6,10 +6,12 @@ use std::io::ErrorKind::NotFound; use std::os::unix::ffi::OsStrExt; use std::path::Path; use std::ptr::NonNull; +use std::sync::Arc; use std::{io, ptr}; #[cfg(master3)] use aead::{generic_array::typenum::Unsigned, AeadCore, AeadMutInPlace, Key, KeyInit}; +use synchronoise::SignalEvent; #[cfg(master3)] use super::encrypted_env::{encrypt_func_wrapper, EncryptedEnv}; @@ -297,7 +299,7 @@ impl EnvOpenOptions { Ok(path) => path, }; - if lock.contains(&path) { + if lock.contains_key(&path) { Err(Error::EnvAlreadyOpened) } else { let path_str = CString::new(path.as_os_str().as_bytes()).unwrap(); @@ -350,9 +352,10 @@ impl EnvOpenOptions { match mdb_result(result) { Ok(()) => { let env_ptr = NonNull::new(env).unwrap(); - let inserted = lock.insert(path.clone()); - debug_assert!(inserted); - Ok(Env::new(env_ptr, path)) + let signal_event = Arc::new(SignalEvent::manual(false)); + let inserted = lock.insert(path.clone(), signal_event.clone()); + debug_assert!(inserted.is_none()); + Ok(Env::new(env_ptr, path, signal_event)) } Err(e) => { ffi::mdb_env_close(env); diff --git a/heed/src/envs/mod.rs b/heed/src/envs/mod.rs index 3454099a..de2f3c62 100644 --- a/heed/src/envs/mod.rs +++ b/heed/src/envs/mod.rs @@ -1,5 +1,5 @@ use std::cmp::Ordering; -use std::collections::HashSet; +use std::collections::HashMap; use std::ffi::c_void; use std::fs::{File, Metadata}; #[cfg(unix)] @@ -33,7 +33,14 @@ pub use env_open_options::EnvOpenOptions; /// Records the current list of opened environments for tracking purposes. The canonical /// path of an environment is removed when either an `Env` or `EncryptedEnv` is closed. -static OPENED_ENV: LazyLock>> = LazyLock::new(RwLock::default); +static OPENED_ENV: LazyLock>>> = + LazyLock::new(RwLock::default); + +/// Returns a struct that allows to wait for the effective closing of an environment. +pub fn env_closing_event>(path: P) -> Option { + let lock = OPENED_ENV.read().unwrap(); + lock.get(path.as_ref()).map(|signal_event| EnvClosingEvent(signal_event.clone())) +} /// Contains information about the environment. #[derive(Debug, Clone, Copy)] diff --git a/heed/src/lib.rs b/heed/src/lib.rs index 0ce180e7..355c0475 100644 --- a/heed/src/lib.rs +++ b/heed/src/lib.rs @@ -85,8 +85,8 @@ pub use self::databases::{EncryptedDatabase, EncryptedDatabaseOpenOptions}; #[cfg(master3)] pub use self::envs::EncryptedEnv; pub use self::envs::{ - CompactionOption, DefaultComparator, Env, EnvClosingEvent, EnvInfo, EnvOpenOptions, - FlagSetMode, IntegerComparator, + env_closing_event, CompactionOption, DefaultComparator, Env, EnvClosingEvent, EnvInfo, + EnvOpenOptions, FlagSetMode, IntegerComparator, }; pub use self::iterator::{ RoIter, RoPrefix, RoRange, RoRevIter, RoRevPrefix, RoRevRange, RwIter, RwPrefix, RwRange, From d8bcae1a5eb5ce78375de23635734fca14c6b48a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sun, 1 Dec 2024 12:44:40 +0100 Subject: [PATCH 46/51] Try to fix the Windows CI --- .github/workflows/rust.yml | 6 +++--- convert-to-heed3.sh | 8 +++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index dc5f2885..11c2db3f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -51,7 +51,7 @@ jobs: - name: Run cargo check run: | cargo clean - ./convert-to-heed3.sh + bash convert-to-heed3.sh cargo check -p heed3 check_all_features: @@ -104,7 +104,7 @@ jobs: - name: Run cargo test run: | cargo clean - ./convert-to-heed3.sh + bash convert-to-heed3.sh cargo check --all-features -p heed3 examples: @@ -155,7 +155,7 @@ jobs: - name: Run the examples run: | cargo clean - ./convert-to-heed3.sh + bash convert-to-heed3.sh cargo run --example 2>&1 | grep -E '^ '| xargs -n1 cargo run --example fmt: diff --git a/convert-to-heed3.sh b/convert-to-heed3.sh index 16d7b720..880fc0e9 100755 --- a/convert-to-heed3.sh +++ b/convert-to-heed3.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # This script is meant to setup the heed3 crate. # @@ -10,7 +12,11 @@ set -e # It basically copy the heed3/Cargo.toml file into # the heed folder... -cp heed3/Cargo.toml heed/Cargo.toml +if [[ "$OSTYPE" == "cygwin" || "$OSTYPE" == "msys" ]]; then + cp heed3\\Cargo.toml heed\\Cargo.toml +else + cp heed3/Cargo.toml heed/Cargo.toml +fi # ...and replaces the `heed::` string by the `heed3::` one. for file in $(find heed/src -type f -name "*.rs"); do From 6a6f4e59def5036ef3fc3a70064e4d0e6cd0434e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Sun, 1 Dec 2024 14:25:09 +0100 Subject: [PATCH 47/51] Document a bit the heed and heed3 crates differences --- README.md | 6 +++--- heed/src/envs/env_open_options.rs | 2 +- heed/src/lib.rs | 11 +++++++++-- lmdb-master3-sys/README.md | 4 ++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index f7395b70..4415d7f6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

-

heed

+

heed & heed3

[![License](https://img.shields.io/badge/license-MIT-green)](#LICENSE) [![Crates.io](https://img.shields.io/crates/v/heed)](https://crates.io/crates/heed) @@ -7,11 +7,11 @@ [![dependency status](https://deps.rs/repo/github/meilisearch/heed/status.svg)](https://deps.rs/repo/github/meilisearch/heed) [![Build](https://github.com/meilisearch/heed/actions/workflows/rust.yml/badge.svg)](https://github.com/meilisearch/heed/actions/workflows/rust.yml) -A Rust-centric [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database) abstraction with minimal overhead. This library enables the storage of various Rust types within LMDB, extending support to include Serde-compatible types. +A Rust-centric [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database) abstraction with minimal overhead. This library enables the storage of various Rust types within LMDB, extending support to include Serde-compatible types. It not only supports the LMDB `mdb.master` branch but also the `mdb.master3` branch which features encryption-at-rest and checksumming. ## Simple Example Usage -Here is an example on how to store and read entries into LMDB in a safe and ACID way. For usage examples, see [heed/examples/](heed/examples/). To see more advanced usage techniques go check our [Cookbook](https://docs.rs/heed/latest/heed/cookbook/index.html). +Here is an example on how to store and read entries into LMDB in a safe and ACID way. For usage examples, see [examples/](examples/). To see more advanced usage techniques go check our [Cookbook](https://docs.rs/heed/latest/heed/cookbook/index.html). ```rust use std::fs; diff --git a/heed/src/envs/env_open_options.rs b/heed/src/envs/env_open_options.rs index f4c7c241..6c5ead98 100644 --- a/heed/src/envs/env_open_options.rs +++ b/heed/src/envs/env_open_options.rs @@ -161,7 +161,7 @@ impl EnvOpenOptions { } /// Creates a blank new set of options ready for configuration and specifies that - /// the [`Env`] will be encrypted using the `E` algorithm with the given `key`. + /// the [`Env`] will be encrypted-at-rest using the `E` algorithm with the given `key`. /// /// You can find more compatible algorithms on [the RustCrypto/AEADs page](https://github.com/RustCrypto/AEADs#crates). /// diff --git a/heed/src/lib.rs b/heed/src/lib.rs index 355c0475..c26c669d 100644 --- a/heed/src/lib.rs +++ b/heed/src/lib.rs @@ -5,9 +5,16 @@ html_logo_url = "https://raw.githubusercontent.com/meilisearch/heed/main/assets/heed-pigeon-logo.png?raw=true" )] -//! `heed` is a high-level wrapper of [LMDB]. +//! `heed` and `heed3` are high-level wrappers of [LMDB]. //! -//! The [cookbook] will give you a variety of complete Rust programs to use with heed. +//! - `heed` is a wrapper around LMDB on the `mdb.master` branch, +//! - `heed3` derives from the `heed` wrapper but on the `mdb.master3` branch. +//! +//! The `heed3` crate will be stable once the LMDB version on the `mdb.master3` branch +//! will be officially released. It features encryption-at-rest and checksumming features +//! that the `heed` crate doesn't. +//! +//! The [cookbook] will give you a variety of complete Rust programs to use with `heed`. //! //! ---- //! diff --git a/lmdb-master3-sys/README.md b/lmdb-master3-sys/README.md index 7612705c..c3323820 100644 --- a/lmdb-master3-sys/README.md +++ b/lmdb-master3-sys/README.md @@ -1,3 +1,3 @@ -# lmdb-master-sys +# lmdb-master3-sys -Rust bindings for liblmdb on the mdb.master branch. +Rust bindings for liblmdb on the mdb.master3 branch. From c44f329afc072038dc25519501d8ef9a50ab323d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 3 Dec 2024 11:42:24 +0100 Subject: [PATCH 48/51] Make the script create a commit for easier removal of heed3 changes --- convert-to-heed3.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/convert-to-heed3.sh b/convert-to-heed3.sh index 880fc0e9..91db4add 100755 --- a/convert-to-heed3.sh +++ b/convert-to-heed3.sh @@ -27,4 +27,10 @@ for file in $(find heed/src -type f -name "*.rs"); do fi done -echo "Heed3 crate setup completed successfully. Copied and modified configurations for the heed crate." +# Make it easier to rollback by doing a commit +git config --global user.email "ci@github.com" +git config --global user.name "The CI" +git commit -am 'remove-me: heed3 changes generate by the convert-to-heed3.sh script' + +echo "Heed3 crate setup completed successfully. Configurations for the heed crate have been copied and modified." +echo "A commit (starting with remove-me) has been generated and must be deleted before merging into the main branch." From 73e8e31eefff0f8baee5280584d6d1ddc25eca50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 3 Dec 2024 11:59:16 +0100 Subject: [PATCH 49/51] Add a CI check to ensure heed3 is not erasing heed --- .github/workflows/rust.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 11c2db3f..10e97b4f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -54,7 +54,7 @@ jobs: bash convert-to-heed3.sh cargo check -p heed3 - check_all_features: + check-all-features: name: Check all the features of the heed project runs-on: ${{ matrix.os }} env: @@ -80,7 +80,7 @@ jobs: cargo clean cargo check --all-features -p heed - check_all_features-heed3: + check-all-features-heed3: name: Check all the features of the heed3 project runs-on: ${{ matrix.os }} env: @@ -171,3 +171,11 @@ jobs: components: rustfmt - name: Run cargo fmt run: cargo fmt --check + + no-heed3-in-heed-folder: + name: Ensure heed3 is not erasing heed + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Check name is heed with grep + run: grep -q 'name = "heed"' heed/Cargo.toml From 446d2bd8796097049228e4bc9c857f8a318a680f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 3 Dec 2024 12:16:52 +0100 Subject: [PATCH 50/51] Add more doc about the heed3 script --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 4415d7f6..e12548bf 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,13 @@ fn main() -> Result<(), Box> { } ``` +## Working with two Crates: heed and heed3 + +The heed and heed3 crates manage a shared codebase. Within the heed3 folder, you can find the Cargo.toml specific to the heed3 crate. +To facilitate work on heed3, utilize the `convert-to-heed3.sh` script. + +This script conveniently moves the `heed3/Cargo.toml` file to the `heed/` folder, updates the `heed::` references to `heed3::`, and generates a commit for easy rollback if needed. + ## Building from Source You can use this command to clone the repository: From b47b72fbcf3f8ed9c8ec0ee7ba6a556f828a8c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 3 Dec 2024 12:21:41 +0100 Subject: [PATCH 51/51] Bump heed, heed3 and heed-types to v0.21.0 --- heed-types/Cargo.toml | 2 +- heed/Cargo.toml | 4 ++-- heed3/Cargo.toml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/heed-types/Cargo.toml b/heed-types/Cargo.toml index 482ba94b..967c963f 100644 --- a/heed-types/Cargo.toml +++ b/heed-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "heed-types" -version = "0.20.2" +version = "0.21.0" authors = ["Kerollmops "] description = "The types used with the fully typed LMDB wrapper, heed" license = "MIT" diff --git a/heed/Cargo.toml b/heed/Cargo.toml index fb8eae96..64e58371 100644 --- a/heed/Cargo.toml +++ b/heed/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "heed" -version = "0.20.5" +version = "0.21.0" authors = ["Kerollmops "] description = "A fully typed LMDB (mdb.master) wrapper with minimum overhead" license = "MIT" @@ -14,7 +14,7 @@ edition = "2021" bitflags = { version = "2.6.0", features = ["serde"] } byteorder = { version = "1.5.0", default-features = false } heed-traits = { version = "0.20.0", path = "../heed-traits" } -heed-types = { version = "0.20.1", default-features = false, path = "../heed-types" } +heed-types = { version = "0.21.0", default-features = false, path = "../heed-types" } libc = "0.2.155" lmdb-master-sys = { version = "0.2.4", path = "../lmdb-master-sys" } once_cell = "1.19.0" diff --git a/heed3/Cargo.toml b/heed3/Cargo.toml index 3ebac3c3..531d4458 100644 --- a/heed3/Cargo.toml +++ b/heed3/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "heed3" -version = "0.20.5-beta.1" +version = "0.21.0" authors = ["Kerollmops "] description = "A fully typed LMDB (mdb.master3) wrapper with minimum overhead with support for encryption" license = "MIT" @@ -17,7 +17,7 @@ bitflags = { version = "2.6.0", features = ["serde"] } byteorder = { version = "1.5.0", default-features = false } generic-array = { version = "0.14.6", features = ["serde"] } heed-traits = { version = "0.20.0", path = "../heed-traits" } -heed-types = { version = "0.20.1", default-features = false, path = "../heed-types" } +heed-types = { version = "0.21.0", default-features = false, path = "../heed-types" } libc = "0.2.155" lmdb-master3-sys = { version = "0.2.4", path = "../lmdb-master3-sys" } once_cell = "1.19.0"