From 0baef3be363a2da165649b45e90c9767988de161 Mon Sep 17 00:00:00 2001 From: Christofer Nolander Date: Mon, 29 Apr 2024 22:15:56 +0200 Subject: [PATCH 1/5] make `ReservedSpace` more flexible --- heed/src/reserved_space.rs | 50 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/heed/src/reserved_space.rs b/heed/src/reserved_space.rs index dd19223a..ed992666 100644 --- a/heed/src/reserved_space.rs +++ b/heed/src/reserved_space.rs @@ -1,3 +1,4 @@ +use std::mem::MaybeUninit; use std::{fmt, io}; use crate::mdb::ffi; @@ -25,6 +26,55 @@ impl ReservedSpace { pub fn remaining(&self) -> usize { self.size - self.written } + + /// Get a slice of all the bytes that have previously been written. + /// + /// This can be used to write information which cannot be known until the very end of + /// serialization. For example, this method can be used to serialize a value, then compute a + /// checksum over the bytes, and then write that checksum to a header at the start of the + /// reserved space. + pub fn written_mut(&mut self) -> &mut [u8] { + unsafe { std::slice::from_raw_parts_mut(self.start_ptr, self.written) } + } + + /// Fills the remaining reserved space with zeroes. + /// + /// ### Note + /// + /// After calling this function, the entire space is considered to be filled and any + /// further attempt to [`write`](std::io::Write::write) anything else will fail. + pub fn fill_zeroes(&mut self) { + for i in self.written..self.size { + unsafe { self.start_ptr.add(i).write(0) }; + } + self.written = self.size; + } + + /// Get a slice of bytes corresponding to the *entire* reserved space. + /// + /// It is safe to write to any byte within the slice. However, for a write past the end of the + /// prevously written bytes to take effect, [`assume_written`](Self::assume_written) has to be + /// called to mark those bytes as initialized. + /// + /// # Safety + /// + /// As the memory comes from within the database itself, the bytes may not yet be + /// initialized. Thus, it is up to the caller to ensure that only initialized memory is read + /// (ensured by the [`MaybeUninit`] API). + pub fn as_uninit_mut(&mut self) -> &mut [MaybeUninit] { + unsafe { std::slice::from_raw_parts_mut(self.start_ptr.cast(), self.size) } + } + + /// Marks the bytes in the range `0..len` as being initialized by advancing the internal write + /// pointer. + /// + /// # Safety + /// + /// The caller guarantees that all bytes in the range have been initialized. + pub unsafe fn assume_written(&mut self, len: usize) { + debug_assert!(len <= self.size); + self.written = len; + } } impl io::Write for ReservedSpace { From a36a40d2fb8f85c27e75b459c6056fde74ae43d3 Mon Sep 17 00:00:00 2001 From: Christofer Nolander Date: Mon, 29 Apr 2024 22:19:23 +0200 Subject: [PATCH 2/5] add documentation --- heed/src/reserved_space.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/heed/src/reserved_space.rs b/heed/src/reserved_space.rs index ed992666..343787bf 100644 --- a/heed/src/reserved_space.rs +++ b/heed/src/reserved_space.rs @@ -39,6 +39,9 @@ impl ReservedSpace { /// Fills the remaining reserved space with zeroes. /// + /// This can be used together with [`written_mut`](Self::written_mut) to get a mutable view of + /// the entire reserved space. + /// /// ### Note /// /// After calling this function, the entire space is considered to be filled and any From 6057c4e9c42edad5b877f1214161fffe75593063 Mon Sep 17 00:00:00 2001 From: Christofer Nolander Date: Mon, 29 Apr 2024 23:43:06 +0200 Subject: [PATCH 3/5] implement `std::io::Seek` for `ReservedSpace` --- heed/src/reserved_space.rs | 65 +++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/heed/src/reserved_space.rs b/heed/src/reserved_space.rs index 343787bf..aa99a92b 100644 --- a/heed/src/reserved_space.rs +++ b/heed/src/reserved_space.rs @@ -7,14 +7,27 @@ use crate::mdb::ffi; /// /// You must write the exact amount of bytes, no less, no more. pub struct ReservedSpace { + /// Total size of the space. size: usize, + + /// Pointer to the start of the reserved space. start_ptr: *mut u8, + + /// Number of bytes which have been written: all bytes in `0..written`. written: usize, + + /// Index of the byte which should be written next. + write_head: usize, } impl ReservedSpace { pub(crate) unsafe fn from_val(val: ffi::MDB_val) -> ReservedSpace { - ReservedSpace { size: val.mv_size, start_ptr: val.mv_data as *mut u8, written: 0 } + ReservedSpace { + size: val.mv_size, + start_ptr: val.mv_data as *mut u8, + written: 0, + write_head: 0, + } } /// The total number of bytes that this memory buffer has. @@ -24,7 +37,7 @@ impl ReservedSpace { /// The remaining number of bytes that this memory buffer has. pub fn remaining(&self) -> usize { - self.size - self.written + self.size - self.write_head } /// Get a slice of all the bytes that have previously been written. @@ -51,6 +64,7 @@ impl ReservedSpace { unsafe { self.start_ptr.add(i).write(0) }; } self.written = self.size; + self.write_head = self.size; } /// Get a slice of bytes corresponding to the *entire* reserved space. @@ -77,15 +91,17 @@ impl ReservedSpace { pub unsafe fn assume_written(&mut self, len: usize) { debug_assert!(len <= self.size); self.written = len; + self.write_head = len; } } impl io::Write for ReservedSpace { fn write(&mut self, buf: &[u8]) -> io::Result { if self.remaining() >= buf.len() { - let dest = unsafe { self.start_ptr.add(self.written) }; + let dest = unsafe { self.start_ptr.add(self.write_head) }; unsafe { buf.as_ptr().copy_to_nonoverlapping(dest, buf.len()) }; - self.written += buf.len(); + self.write_head += buf.len(); + self.written = usize::max(self.written, self.write_head); Ok(buf.len()) } else { Err(io::Error::from(io::ErrorKind::WriteZero)) @@ -97,6 +113,47 @@ impl io::Write for ReservedSpace { } } +/// ## Note +/// +/// May only seek within the already within the previously written space. +/// Attempts to do otherwise will result in an error. +impl io::Seek for ReservedSpace { + fn seek(&mut self, pos: io::SeekFrom) -> io::Result { + let (base, offset) = match pos { + io::SeekFrom::Start(start) => (start, 0), + io::SeekFrom::End(offset) => (self.written as u64, offset), + io::SeekFrom::Current(offset) => (self.write_head as u64, offset), + }; + + let Some(new_pos) = base.checked_add_signed(offset) else { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "cannot seek before start of reserved space", + )); + }; + + if new_pos > self.written as u64 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "cannot seek past end of reserved space", + )); + } + + self.write_head = new_pos as usize; + + Ok(new_pos) + } + + fn rewind(&mut self) -> io::Result<()> { + self.write_head = 0; + Ok(()) + } + + fn stream_position(&mut self) -> io::Result { + Ok(self.write_head as u64) + } +} + impl fmt::Debug for ReservedSpace { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("ReservedSpace").finish() From 4ed1f18d85f6875e45064913aaa5cca40bd511d5 Mon Sep 17 00:00:00 2001 From: Christofer Nolander Date: Mon, 29 Apr 2024 23:48:45 +0200 Subject: [PATCH 4/5] specilaize `write_all` --- heed/src/reserved_space.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/heed/src/reserved_space.rs b/heed/src/reserved_space.rs index aa99a92b..fb64866a 100644 --- a/heed/src/reserved_space.rs +++ b/heed/src/reserved_space.rs @@ -108,6 +108,12 @@ impl io::Write for ReservedSpace { } } + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + let count = self.write(buf)?; + debug_assert_eq!(count, buf.len()); + Ok(()) + } + fn flush(&mut self) -> io::Result<()> { Ok(()) } From b3f065ed3a1808b82684fe65dae55e3e7b5240d3 Mon Sep 17 00:00:00 2001 From: Christofer Nolander Date: Wed, 1 May 2024 10:33:17 +0200 Subject: [PATCH 5/5] adjust docs for `ReservedSpace` --- heed/src/reserved_space.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/heed/src/reserved_space.rs b/heed/src/reserved_space.rs index fb64866a..a51bdf90 100644 --- a/heed/src/reserved_space.rs +++ b/heed/src/reserved_space.rs @@ -9,14 +9,16 @@ use crate::mdb::ffi; pub struct ReservedSpace { /// Total size of the space. size: usize, - /// Pointer to the start of the reserved space. start_ptr: *mut u8, - /// Number of bytes which have been written: all bytes in `0..written`. written: usize, - /// Index of the byte which should be written next. + /// + /// # Safety + /// + /// To ensure there are no unwritten gaps in the buffer this should be kept in the range + /// `0..=written` at all times. write_head: usize, } @@ -60,7 +62,7 @@ impl ReservedSpace { /// After calling this function, the entire space is considered to be filled and any /// further attempt to [`write`](std::io::Write::write) anything else will fail. pub fn fill_zeroes(&mut self) { - for i in self.written..self.size { + for i in self.write_head..self.size { unsafe { self.start_ptr.add(i).write(0) }; } self.written = self.size; @@ -121,7 +123,7 @@ impl io::Write for ReservedSpace { /// ## Note /// -/// May only seek within the already within the previously written space. +/// May only seek within the previously written space. /// Attempts to do otherwise will result in an error. impl io::Seek for ReservedSpace { fn seek(&mut self, pos: io::SeekFrom) -> io::Result {