Skip to content

Commit

Permalink
simplifies blockstore Column::index implementations (#4464)
Browse files Browse the repository at this point in the history 
The commits adds a small macro to simplify

    pub trait Column {
        fn index(key: &[u8]) -> Self::Index;
        // ...
    }

implementations.
  • Loading branch information
@behzadnouri
behzadnouri authored Jan 18, 2025
1 parent b441da9 commit 0f56076
Show file tree
Hide file tree
Showing 5 changed files with 71 additions and 64 deletions.
1 change: 0 additions & 1 deletion Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion ledger/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ edition = { workspace = true }
assert_matches = { workspace = true }
bincode = { workspace = true }
bitflags = { workspace = true, features = ["serde"] }
byteorder = { workspace = true }
bzip2 = { workspace = true }
chrono = { workspace = true, features = ["default", "serde"] }
chrono-humanize = { workspace = true }
Expand Down
131 changes: 71 additions & 60 deletions ledger/src/blockstore_db.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ use {
blockstore_options::{AccessType, BlockstoreOptions, LedgerColumnOptions},
},
bincode::{deserialize, Options as BincodeOptions},
byteorder::{BigEndian, ByteOrder},
log::*,
prost::Message,
rocksdb::{
Expand Down Expand Up @@ -103,6 +102,21 @@ const OPTIMISTIC_SLOTS_CF: &str = "optimistic_slots";
/// Column family for merkle roots
const MERKLE_ROOT_META_CF: &str = "merkle_root_meta";

macro_rules! convert_column_index_to_key_bytes {
($key:ident, $($range:expr => $bytes:expr),* $(,)?) => {{
let mut key = [0u8; std::mem::size_of::<Self::$key>()];
debug_assert_eq!(0 $(+$bytes.len())*, key.len());
$(key[$range].copy_from_slice($bytes);)*
key
}};
}

macro_rules! convert_column_key_bytes_to_index {
($k:ident, $($a:literal..$b:literal => $f:expr),* $(,)?) => {{
($($f(<[u8; $b-$a]>::try_from(&$k[$a..$b]).unwrap())),*)
}};
}

#[derive(Error, Debug)]
pub enum BlockstoreError {
#[error("shred for index exists")]
Expand Down Expand Up @@ -837,8 +851,8 @@ impl<T: SlotColumn> Column for T {
}

/// Converts a RocksDB key to its u64 Index.
fn index(key: &[u8]) -> u64 {
BigEndian::read_u64(&key[..8])
fn index(key: &[u8]) -> Self::Index {
convert_column_key_bytes_to_index!(key, 0..8 => Slot::from_be_bytes)
}

fn slot(index: Self::Index) -> Slot {
Expand Down Expand Up @@ -881,22 +895,13 @@ pub trait ColumnIndexDeprecation: Column {
}
}

macro_rules! concat_key_bytes {
($key:ident, $($range:expr => $bytes:expr),* $(,)?) => {{
let mut key = [0u8; std::mem::size_of::<Self::$key>()];
debug_assert_eq!(0 $(+$bytes.len())*, key.len());
$(key[$range].copy_from_slice($bytes);)*
key
}};
}

impl Column for columns::TransactionStatus {
type Index = (Signature, Slot);
type Key = [u8; SIGNATURE_BYTES + std::mem::size_of::<Slot>()];

#[inline]
fn key((signature, slot): &Self::Index) -> Self::Key {
concat_key_bytes!(Key,
convert_column_index_to_key_bytes!(Key,
..64 => signature.as_ref(),
64.. => &slot.to_be_bytes(),
)
Expand Down Expand Up @@ -929,7 +934,7 @@ impl ColumnIndexDeprecation for columns::TransactionStatus {
type DeprecatedKey = [u8; 80];

fn deprecated_key((index, signature, slot): Self::DeprecatedIndex) -> Self::DeprecatedKey {
concat_key_bytes!(DeprecatedKey,
convert_column_index_to_key_bytes!(DeprecatedKey,
..8 => &index.to_be_bytes(),
8..72 => signature.as_ref(),
72.. => &slot.to_be_bytes(),
Expand All @@ -940,19 +945,21 @@ impl ColumnIndexDeprecation for columns::TransactionStatus {
if key.len() != std::mem::size_of::<Self::DeprecatedKey>() {
return Err(IndexError::UnpackError);
}
let primary_index = BigEndian::read_u64(&key[0..8]);
let signature = Signature::try_from(&key[8..72]).unwrap();
let slot = BigEndian::read_u64(&key[72..80]);
Ok((primary_index, signature, slot))
Ok(convert_column_key_bytes_to_index!(key,
0..8 => u64::from_be_bytes, // primary index
8..72 => Signature::from,
72..80 => Slot::from_be_bytes,
))
}

fn try_current_index(key: &[u8]) -> std::result::Result<Self::Index, IndexError> {
if key.len() != Self::CURRENT_INDEX_LEN {
return Err(IndexError::UnpackError);
}
let signature = Signature::try_from(&key[0..64]).unwrap();
let slot = BigEndian::read_u64(&key[64..72]);
Ok((signature, slot))
Ok(convert_column_key_bytes_to_index!(key,
0..64 => Signature::from,
64..72 => Slot::from_be_bytes,
))
}

fn convert_index(deprecated_index: Self::DeprecatedIndex) -> Self::Index {
Expand All @@ -970,7 +977,7 @@ impl Column for columns::AddressSignatures {

#[inline]
fn key((pubkey, slot, transaction_index, signature): &Self::Index) -> Self::Key {
concat_key_bytes!(Key,
convert_column_index_to_key_bytes!(Key,
..32 => pubkey.as_ref(),
32..40 => &slot.to_be_bytes(),
40..44 => &transaction_index.to_be_bytes(),
Expand Down Expand Up @@ -1004,7 +1011,7 @@ impl ColumnIndexDeprecation for columns::AddressSignatures {
fn deprecated_key(
(primary_index, pubkey, slot, signature): Self::DeprecatedIndex,
) -> Self::DeprecatedKey {
concat_key_bytes!(DeprecatedKey,
convert_column_index_to_key_bytes!(DeprecatedKey,
..8 => &primary_index.to_be_bytes(),
8..40 => pubkey.as_ref(),
40..48 => &slot.to_be_bytes(),
Expand All @@ -1016,22 +1023,24 @@ impl ColumnIndexDeprecation for columns::AddressSignatures {
if key.len() != std::mem::size_of::<Self::DeprecatedKey>() {
return Err(IndexError::UnpackError);
}
let primary_index = BigEndian::read_u64(&key[0..8]);
let pubkey = Pubkey::try_from(&key[8..40]).unwrap();
let slot = BigEndian::read_u64(&key[40..48]);
let signature = Signature::try_from(&key[48..112]).unwrap();
Ok((primary_index, pubkey, slot, signature))
Ok(convert_column_key_bytes_to_index!(key,
0..8 => u64::from_be_bytes, // primary index
8..40 => Pubkey::from,
40..48 => Slot::from_be_bytes,
48..112 => Signature::from,
))
}

fn try_current_index(key: &[u8]) -> std::result::Result<Self::Index, IndexError> {
if key.len() != Self::CURRENT_INDEX_LEN {
return Err(IndexError::UnpackError);
}
let pubkey = Pubkey::try_from(&key[0..32]).unwrap();
let slot = BigEndian::read_u64(&key[32..40]);
let transaction_index = BigEndian::read_u32(&key[40..44]);
let signature = Signature::try_from(&key[44..108]).unwrap();
Ok((pubkey, slot, transaction_index, signature))
Ok(convert_column_key_bytes_to_index!(key,
0..32 => Pubkey::from,
32..40 => Slot::from_be_bytes,
40..44 => u32::from_be_bytes, // transaction index
44..108 => Signature::from,
))
}

fn convert_index(deprecated_index: Self::DeprecatedIndex) -> Self::Index {
Expand All @@ -1046,7 +1055,7 @@ impl Column for columns::TransactionMemos {

#[inline]
fn key((signature, slot): &Self::Index) -> Self::Key {
concat_key_bytes!(Key,
convert_column_index_to_key_bytes!(Key,
..64 => signature.as_ref(),
64.. => &slot.to_be_bytes(),
)
Expand Down Expand Up @@ -1085,9 +1094,10 @@ impl ColumnIndexDeprecation for columns::TransactionMemos {
if key.len() != Self::CURRENT_INDEX_LEN {
return Err(IndexError::UnpackError);
}
let signature = Signature::try_from(&key[0..64]).unwrap();
let slot = BigEndian::read_u64(&key[64..72]);
Ok((signature, slot))
Ok(convert_column_key_bytes_to_index!(key,
0..64 => Signature::from,
64..72 => Slot::from_be_bytes,
))
}

fn convert_index(deprecated_index: Self::DeprecatedIndex) -> Self::Index {
Expand All @@ -1104,8 +1114,8 @@ impl Column for columns::TransactionStatusIndex {
index.to_be_bytes()
}

fn index(key: &[u8]) -> u64 {
BigEndian::read_u64(&key[..8])
fn index(key: &[u8]) -> Self::Index {
convert_column_key_bytes_to_index!(key, 0..8 => u64::from_be_bytes)
}

fn slot(_index: Self::Index) -> Slot {
Expand Down Expand Up @@ -1165,7 +1175,7 @@ impl Column for columns::ProgramCosts {
}

fn index(key: &[u8]) -> Self::Index {
Pubkey::try_from(&key[..32]).unwrap()
convert_column_key_bytes_to_index!(key, 0..32 => Pubkey::from)
}

fn slot(_index: Self::Index) -> Slot {
Expand All @@ -1178,7 +1188,7 @@ impl Column for columns::ProgramCosts {
}

impl Column for columns::ShredCode {
type Index = (Slot, u64);
type Index = (Slot, /*shred index:*/ u64);
type Key = <columns::ShredData as Column>::Key;

#[inline]
Expand All @@ -1187,7 +1197,7 @@ impl Column for columns::ShredCode {
<columns::ShredData as Column>::key(index)
}

fn index(key: &[u8]) -> (Slot, u64) {
fn index(key: &[u8]) -> Self::Index {
columns::ShredData::index(key)
}

Expand All @@ -1204,21 +1214,22 @@ impl ColumnName for columns::ShredCode {
}

impl Column for columns::ShredData {
type Index = (Slot, u64);
type Index = (Slot, /*shred index:*/ u64);
type Key = [u8; std::mem::size_of::<Slot>() + std::mem::size_of::<u64>()];

#[inline]
fn key((slot, index): &Self::Index) -> Self::Key {
concat_key_bytes!(Key,
convert_column_index_to_key_bytes!(Key,
..8 => &slot.to_be_bytes(),
8.. => &index.to_be_bytes(),
)
}

fn index(key: &[u8]) -> (Slot, u64) {
let slot = BigEndian::read_u64(&key[..8]);
let index = BigEndian::read_u64(&key[8..16]);
(slot, index)
fn index(key: &[u8]) -> Self::Index {
convert_column_key_bytes_to_index!(key,
0..8 => Slot::from_be_bytes,
8..16 => u64::from_be_bytes, // shred index
)
}

fn slot(index: Self::Index) -> Slot {
Expand Down Expand Up @@ -1312,22 +1323,22 @@ impl TypedColumn for columns::SlotMeta {
}

impl Column for columns::ErasureMeta {
type Index = (Slot, u64);
type Index = (Slot, /*fec_set_index:*/ u64);
type Key = [u8; std::mem::size_of::<Slot>() + std::mem::size_of::<u64>()];

#[inline]
fn key((slot, fec_set_index): &Self::Index) -> Self::Key {
concat_key_bytes!(Key,
convert_column_index_to_key_bytes!(Key,
..8 => &slot.to_be_bytes(),
8.. => &fec_set_index.to_be_bytes(),
)
}

fn index(key: &[u8]) -> (Slot, u64) {
let slot = BigEndian::read_u64(&key[..8]);
let set_index = BigEndian::read_u64(&key[8..]);

(slot, set_index)
fn index(key: &[u8]) -> Self::Index {
convert_column_key_bytes_to_index!(key,
0..8 => Slot::from_be_bytes,
8..16 => u64::from_be_bytes, // fec_set_index
)
}

fn slot(index: Self::Index) -> Slot {
Expand Down Expand Up @@ -1359,17 +1370,17 @@ impl Column for columns::MerkleRootMeta {

#[inline]
fn key((slot, fec_set_index): &Self::Index) -> Self::Key {
concat_key_bytes!(Key,
convert_column_index_to_key_bytes!(Key,
..8 => &slot.to_be_bytes(),
8.. => &fec_set_index.to_be_bytes(),
)
}

fn index(key: &[u8]) -> Self::Index {
let slot = BigEndian::read_u64(&key[..8]);
let fec_set_index = BigEndian::read_u32(&key[8..]);

(slot, fec_set_index)
convert_column_key_bytes_to_index!(key,
0..8 => Slot::from_be_bytes,
8..12 => u32::from_be_bytes, // fec_set_index
)
}

fn slot((slot, _fec_set_index): Self::Index) -> Slot {
Expand Down
1 change: 0 additions & 1 deletion programs/sbf/Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion svm/examples/Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 0f56076

Please sign in to comment.