Skip to content

Commit de53bec

Browse files
KerollmopsKerollmops
committed
Fix the prepare_for_closing system
1 parent 36a785b commit de53bec

File tree

4 files changed

+349
-19
lines changed

4 files changed

+349
-19
lines changed

‎heed/src/envs/env.rs

Lines changed: 331 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,12 @@ pub struct Env {
2929
}
3030

3131
impl Env {
32-
pub(crate) fn new(env_ptr: NonNull<MDB_env>, path: PathBuf) -> Env {
33-
Env {
34-
inner: Arc::new(EnvInner {
35-
env_ptr,
36-
path,
37-
signal_event: Arc::new(SignalEvent::manual(false)),
38-
}),
39-
}
32+
pub(crate) fn new(
33+
env_ptr: NonNull<MDB_env>,
34+
path: PathBuf,
35+
signal_event: Arc<SignalEvent>,
36+
) -> Env {
37+
Env { inner: Arc::new(EnvInner { env_ptr, path, signal_event }) }
4038
}
4139

4240
pub(crate) fn env_mut_ptr(&self) -> NonNull<ffi::MDB_env> {
@@ -492,7 +490,6 @@ impl fmt::Debug for Env {
492490
}
493491
}
494492

495-
#[derive(Clone)]
496493
pub(crate) struct EnvInner {
497494
env_ptr: NonNull<MDB_env>,
498495
signal_event: Arc<SignalEvent>,
@@ -501,9 +498,332 @@ pub(crate) struct EnvInner {
501498

502499
impl Drop for EnvInner {
503500
fn drop(&mut self) {
504-
unsafe { ffi::mdb_env_close(self.env_ptr.as_mut()) };
505501
let mut lock = OPENED_ENV.write().unwrap();
506502
let removed = lock.remove(&self.path);
507-
debug_assert!(removed);
503+
debug_assert!(removed.is_some());
504+
unsafe { ffi::mdb_env_close(self.env_ptr.as_mut()) };
505+
self.signal_event.signal();
506+
}
507+
}
508+
509+
#[cfg(test)]
510+
mod tests {
511+
use std::io::ErrorKind;
512+
use std::time::Duration;
513+
use std::{fs, thread};
514+
515+
use crate::types::*;
516+
use crate::{env_closing_event, EnvOpenOptions, Error};
517+
518+
#[test]
519+
fn close_env() {
520+
let dir = tempfile::tempdir().unwrap();
521+
let env = unsafe {
522+
EnvOpenOptions::new()
523+
.map_size(10 * 1024 * 1024) // 10MB
524+
.max_dbs(30)
525+
.open(dir.path())
526+
.unwrap()
527+
};
528+
529+
// Force a thread to keep the env for 1 second.
530+
let env_cloned = env.clone();
531+
thread::spawn(move || {
532+
let _env = env_cloned;
533+
thread::sleep(Duration::from_secs(1));
534+
});
535+
536+
let mut wtxn = env.write_txn().unwrap();
537+
let db = env.create_database::<Str, Str>(&mut wtxn, None).unwrap();
538+
wtxn.commit().unwrap();
539+
540+
// Create an ordered list of keys...
541+
let mut wtxn = env.write_txn().unwrap();
542+
db.put(&mut wtxn, "hello", "hello").unwrap();
543+
db.put(&mut wtxn, "world", "world").unwrap();
544+
545+
let mut iter = db.iter(&wtxn).unwrap();
546+
assert_eq!(iter.next().transpose().unwrap(), Some(("hello", "hello")));
547+
assert_eq!(iter.next().transpose().unwrap(), Some(("world", "world")));
548+
assert_eq!(iter.next().transpose().unwrap(), None);
549+
drop(iter);
550+
551+
wtxn.commit().unwrap();
552+
553+
let signal_event = env.prepare_for_closing();
554+
555+
eprintln!("waiting for the env to be closed");
556+
signal_event.wait();
557+
eprintln!("env closed successfully");
558+
559+
// Make sure we don't have a reference to the env
560+
assert!(env_closing_event(dir.path()).is_none());
561+
}
562+
563+
#[test]
564+
fn reopen_env_with_different_options_is_err() {
565+
let dir = tempfile::tempdir().unwrap();
566+
let _env = unsafe {
567+
EnvOpenOptions::new()
568+
.map_size(10 * 1024 * 1024) // 10MB
569+
.open(dir.path())
570+
.unwrap()
571+
};
572+
573+
let result = unsafe {
574+
EnvOpenOptions::new()
575+
.map_size(12 * 1024 * 1024) // 12MB
576+
.open(dir.path())
577+
};
578+
579+
assert!(matches!(result, Err(Error::EnvAlreadyOpened)));
580+
}
581+
582+
#[test]
583+
fn open_env_with_named_path() {
584+
let dir = tempfile::tempdir().unwrap();
585+
fs::create_dir_all(dir.path().join("babar.mdb")).unwrap();
586+
let _env = unsafe {
587+
EnvOpenOptions::new()
588+
.map_size(10 * 1024 * 1024) // 10MB
589+
.open(dir.path().join("babar.mdb"))
590+
.unwrap()
591+
};
592+
593+
let error = unsafe {
594+
EnvOpenOptions::new()
595+
.map_size(10 * 1024 * 1024) // 10MB
596+
.open(dir.path().join("babar.mdb"))
597+
.unwrap_err()
598+
};
599+
600+
assert!(matches!(error, Error::EnvAlreadyOpened));
601+
}
602+
603+
#[test]
604+
#[cfg(not(windows))]
605+
fn open_database_with_writemap_flag() {
606+
let dir = tempfile::tempdir().unwrap();
607+
let mut envbuilder = EnvOpenOptions::new();
608+
envbuilder.map_size(10 * 1024 * 1024); // 10MB
609+
envbuilder.max_dbs(10);
610+
unsafe { envbuilder.flags(crate::EnvFlags::WRITE_MAP) };
611+
let env = unsafe { envbuilder.open(dir.path()).unwrap() };
612+
613+
let mut wtxn = env.write_txn().unwrap();
614+
let _db = env.create_database::<Str, Str>(&mut wtxn, Some("my-super-db")).unwrap();
615+
wtxn.commit().unwrap();
616+
}
617+
618+
#[test]
619+
fn open_database_with_nosubdir() {
620+
let dir = tempfile::tempdir().unwrap();
621+
let mut envbuilder = EnvOpenOptions::new();
622+
unsafe { envbuilder.flags(crate::EnvFlags::NO_SUB_DIR) };
623+
let _env = unsafe { envbuilder.open(dir.path().join("data.mdb")).unwrap() };
624+
}
625+
626+
#[test]
627+
fn create_database_without_commit() {
628+
let dir = tempfile::tempdir().unwrap();
629+
let env = unsafe {
630+
EnvOpenOptions::new()
631+
.map_size(10 * 1024 * 1024) // 10MB
632+
.max_dbs(10)
633+
.open(dir.path())
634+
.unwrap()
635+
};
636+
637+
let mut wtxn = env.write_txn().unwrap();
638+
let _db = env.create_database::<Str, Str>(&mut wtxn, Some("my-super-db")).unwrap();
639+
wtxn.abort();
640+
641+
let rtxn = env.read_txn().unwrap();
642+
let option = env.open_database::<Str, Str>(&rtxn, Some("my-super-db")).unwrap();
643+
assert!(option.is_none());
644+
}
645+
646+
#[test]
647+
fn open_already_existing_database() {
648+
let dir = tempfile::tempdir().unwrap();
649+
let env = unsafe {
650+
EnvOpenOptions::new()
651+
.map_size(10 * 1024 * 1024) // 10MB
652+
.max_dbs(10)
653+
.open(dir.path())
654+
.unwrap()
655+
};
656+
657+
// we first create a database
658+
let mut wtxn = env.write_txn().unwrap();
659+
let _db = env.create_database::<Str, Str>(&mut wtxn, Some("my-super-db")).unwrap();
660+
wtxn.commit().unwrap();
661+
662+
// Close the environement and reopen it, databases must not be loaded in memory.
663+
env.prepare_for_closing().wait();
664+
let env = unsafe {
665+
EnvOpenOptions::new()
666+
.map_size(10 * 1024 * 1024) // 10MB
667+
.max_dbs(10)
668+
.open(dir.path())
669+
.unwrap()
670+
};
671+
672+
let rtxn = env.read_txn().unwrap();
673+
let option = env.open_database::<Str, Str>(&rtxn, Some("my-super-db")).unwrap();
674+
assert!(option.is_some());
675+
}
676+
677+
#[test]
678+
fn resize_database() {
679+
let dir = tempfile::tempdir().unwrap();
680+
let page_size = page_size::get();
681+
let env = unsafe {
682+
EnvOpenOptions::new().map_size(9 * page_size).max_dbs(1).open(dir.path()).unwrap()
683+
};
684+
685+
let mut wtxn = env.write_txn().unwrap();
686+
let db = env.create_database::<Str, Str>(&mut wtxn, Some("my-super-db")).unwrap();
687+
wtxn.commit().unwrap();
688+
689+
let mut wtxn = env.write_txn().unwrap();
690+
for i in 0..64 {
691+
db.put(&mut wtxn, &i.to_string(), "world").unwrap();
692+
}
693+
wtxn.commit().unwrap();
694+
695+
let mut wtxn = env.write_txn().unwrap();
696+
for i in 64..128 {
697+
db.put(&mut wtxn, &i.to_string(), "world").unwrap();
698+
}
699+
wtxn.commit().expect_err("cannot commit a transaction that would reach the map size limit");
700+
701+
unsafe {
702+
env.resize(10 * page_size).unwrap();
703+
}
704+
let mut wtxn = env.write_txn().unwrap();
705+
for i in 64..128 {
706+
db.put(&mut wtxn, &i.to_string(), "world").unwrap();
707+
}
708+
wtxn.commit().expect("transaction should commit after resizing the map size");
709+
710+
assert_eq!(10 * page_size, env.info().map_size);
711+
}
712+
713+
/// Non-regression test for
714+
/// <https://github.com/meilisearch/heed/issues/183>
715+
///
716+
/// We should be able to open database Read-Only Env with
717+
/// no prior Read-Write Env opening. And query data.
718+
#[test]
719+
fn open_read_only_without_no_env_opened_before() {
720+
let expected_data0 = "Data Expected db0";
721+
let dir = tempfile::tempdir().unwrap();
722+
723+
{
724+
// We really need this env to be dropped before the read-only access.
725+
let env = unsafe {
726+
EnvOpenOptions::new()
727+
.map_size(16 * 1024 * 1024 * 1024) // 10MB
728+
.max_dbs(32)
729+
.open(dir.path())
730+
.unwrap()
731+
};
732+
let mut wtxn = env.write_txn().unwrap();
733+
let database0 = env.create_database::<Str, Str>(&mut wtxn, Some("shared0")).unwrap();
734+
735+
wtxn.commit().unwrap();
736+
let mut wtxn = env.write_txn().unwrap();
737+
database0.put(&mut wtxn, "shared0", expected_data0).unwrap();
738+
wtxn.commit().unwrap();
739+
// We also really need that no other env reside in memory in other thread doing tests.
740+
env.prepare_for_closing().wait();
741+
}
742+
743+
{
744+
// Open now we do a read-only opening
745+
let env = unsafe {
746+
EnvOpenOptions::new()
747+
.map_size(16 * 1024 * 1024 * 1024) // 10MB
748+
.max_dbs(32)
749+
.open(dir.path())
750+
.unwrap()
751+
};
752+
let database0 = {
753+
let rtxn = env.read_txn().unwrap();
754+
let database0 =
755+
env.open_database::<Str, Str>(&rtxn, Some("shared0")).unwrap().unwrap();
756+
// This commit is mandatory if not committed you might get
757+
// Io(Os { code: 22, kind: InvalidInput, message: "Invalid argument" })
758+
rtxn.commit().unwrap();
759+
database0
760+
};
761+
762+
{
763+
// If we didn't committed the opening it might fail with EINVAL.
764+
let rtxn = env.read_txn().unwrap();
765+
let value = database0.get(&rtxn, "shared0").unwrap().unwrap();
766+
assert_eq!(value, expected_data0);
767+
}
768+
769+
env.prepare_for_closing().wait();
770+
}
771+
772+
// To avoid reintroducing the bug let's try to open again but without the commit
773+
{
774+
// Open now we do a read-only opening
775+
let env = unsafe {
776+
EnvOpenOptions::new()
777+
.map_size(16 * 1024 * 1024 * 1024) // 10MB
778+
.max_dbs(32)
779+
.open(dir.path())
780+
.unwrap()
781+
};
782+
let database0 = {
783+
let rtxn = env.read_txn().unwrap();
784+
let database0 =
785+
env.open_database::<Str, Str>(&rtxn, Some("shared0")).unwrap().unwrap();
786+
// No commit it's important, dropping explicitly
787+
drop(rtxn);
788+
database0
789+
};
790+
791+
{
792+
// We didn't committed the opening we will get EINVAL.
793+
let rtxn = env.read_txn().unwrap();
794+
// The dbg!() is intentional in case of a change in rust-std or in lmdb related
795+
// to the windows error.
796+
let err = dbg!(database0.get(&rtxn, "shared0"));
797+
798+
// The error kind is still ErrorKind Uncategorized on windows.
799+
// Behind it's a ERROR_BAD_COMMAND code 22 like EINVAL.
800+
if cfg!(windows) {
801+
assert!(err.is_err());
802+
} else {
803+
assert!(
804+
matches!(err, Err(Error::Io(ref e)) if e.kind() == ErrorKind::InvalidInput)
805+
);
806+
}
807+
}
808+
809+
env.prepare_for_closing().wait();
810+
}
811+
}
812+
813+
#[test]
814+
fn max_key_size() {
815+
let dir = tempfile::tempdir().unwrap();
816+
let env = unsafe { EnvOpenOptions::new().open(dir.path().join(dir.path())).unwrap() };
817+
let maxkeysize = env.max_key_size();
818+
819+
eprintln!("maxkeysize: {}", maxkeysize);
820+
821+
if cfg!(feature = "longer-keys") {
822+
// Should be larger than the default of 511
823+
assert!(maxkeysize > 511);
824+
} else {
825+
// Should be the default of 511
826+
assert_eq!(maxkeysize, 511);
827+
}
508828
}
509829
}

‎heed/src/envs/env_open_options.rs

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@ use std::io::ErrorKind::NotFound;
66
use std::os::unix::ffi::OsStrExt;
77
use std::path::Path;
88
use std::ptr::NonNull;
9+
use std::sync::Arc;
910
use std::{io, ptr};
1011

1112
#[cfg(master3)]
1213
use aead::{generic_array::typenum::Unsigned, AeadCore, AeadMutInPlace, Key, KeyInit};
14+
use synchronoise::SignalEvent;
1315

1416
#[cfg(master3)]
1517
use super::encrypted_env::{encrypt_func_wrapper, EncryptedEnv};
@@ -297,7 +299,7 @@ impl EnvOpenOptions {
297299
Ok(path) => path,
298300
};
299301

300-
if lock.contains(&path) {
302+
if lock.contains_key(&path) {
301303
Err(Error::EnvAlreadyOpened)
302304
} else {
303305
let path_str = CString::new(path.as_os_str().as_bytes()).unwrap();
@@ -350,9 +352,10 @@ impl EnvOpenOptions {
350352
match mdb_result(result) {
351353
Ok(()) => {
352354
let env_ptr = NonNull::new(env).unwrap();
353-
let inserted = lock.insert(path.clone());
354-
debug_assert!(inserted);
355-
Ok(Env::new(env_ptr, path))
355+
let signal_event = Arc::new(SignalEvent::manual(false));
356+
let inserted = lock.insert(path.clone(), signal_event.clone());
357+
debug_assert!(inserted.is_none());
358+
Ok(Env::new(env_ptr, path, signal_event))
356359
}
357360
Err(e) => {
358361
ffi::mdb_env_close(env);

0 commit comments

Comments
 (0)