From 0f56076a2cb999726421e40f5f6b6b3201c4e172 Mon Sep 17 00:00:00 2001 From: behzad nouri Date: Sat, 18 Jan 2025 13:35:08 +0000 Subject: [PATCH] simplifies blockstore Column::index implementations (#4464) The commits adds a small macro to simplify pub trait Column { fn index(key: &[u8]) -> Self::Index; // ... } implementations. --- Cargo.lock | 1 - ledger/Cargo.toml | 1 - ledger/src/blockstore_db.rs | 131 +++++++++++++++++++----------------- programs/sbf/Cargo.lock | 1 - svm/examples/Cargo.lock | 1 - 5 files changed, 71 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 906564f8316908..cbe3fb20815121 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7712,7 +7712,6 @@ dependencies = [ "bincode", "bitflags 2.8.0", "bs58", - "byteorder", "bzip2", "chrono", "chrono-humanize", diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index 6d4ce7d5ab5cad..72a2e9a43fe12c 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -13,7 +13,6 @@ edition = { workspace = true } assert_matches = { workspace = true } bincode = { workspace = true } bitflags = { workspace = true, features = ["serde"] } -byteorder = { workspace = true } bzip2 = { workspace = true } chrono = { workspace = true, features = ["default", "serde"] } chrono-humanize = { workspace = true } diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 5d9ac24d57e79b..83c213b3a1b4bc 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -11,7 +11,6 @@ use { blockstore_options::{AccessType, BlockstoreOptions, LedgerColumnOptions}, }, bincode::{deserialize, Options as BincodeOptions}, - byteorder::{BigEndian, ByteOrder}, log::*, prost::Message, rocksdb::{ @@ -103,6 +102,21 @@ const OPTIMISTIC_SLOTS_CF: &str = "optimistic_slots"; /// Column family for merkle roots const MERKLE_ROOT_META_CF: &str = "merkle_root_meta"; +macro_rules! convert_column_index_to_key_bytes { + ($key:ident, $($range:expr => $bytes:expr),* $(,)?) => {{ + let mut key = [0u8; std::mem::size_of::()]; + debug_assert_eq!(0 $(+$bytes.len())*, key.len()); + $(key[$range].copy_from_slice($bytes);)* + key + }}; +} + +macro_rules! convert_column_key_bytes_to_index { + ($k:ident, $($a:literal..$b:literal => $f:expr),* $(,)?) => {{ + ($($f(<[u8; $b-$a]>::try_from(&$k[$a..$b]).unwrap())),*) + }}; +} + #[derive(Error, Debug)] pub enum BlockstoreError { #[error("shred for index exists")] @@ -837,8 +851,8 @@ impl Column for T { } /// Converts a RocksDB key to its u64 Index. - fn index(key: &[u8]) -> u64 { - BigEndian::read_u64(&key[..8]) + fn index(key: &[u8]) -> Self::Index { + convert_column_key_bytes_to_index!(key, 0..8 => Slot::from_be_bytes) } fn slot(index: Self::Index) -> Slot { @@ -881,22 +895,13 @@ pub trait ColumnIndexDeprecation: Column { } } -macro_rules! concat_key_bytes { - ($key:ident, $($range:expr => $bytes:expr),* $(,)?) => {{ - let mut key = [0u8; std::mem::size_of::()]; - debug_assert_eq!(0 $(+$bytes.len())*, key.len()); - $(key[$range].copy_from_slice($bytes);)* - key - }}; -} - impl Column for columns::TransactionStatus { type Index = (Signature, Slot); type Key = [u8; SIGNATURE_BYTES + std::mem::size_of::()]; #[inline] fn key((signature, slot): &Self::Index) -> Self::Key { - concat_key_bytes!(Key, + convert_column_index_to_key_bytes!(Key, ..64 => signature.as_ref(), 64.. => &slot.to_be_bytes(), ) @@ -929,7 +934,7 @@ impl ColumnIndexDeprecation for columns::TransactionStatus { type DeprecatedKey = [u8; 80]; fn deprecated_key((index, signature, slot): Self::DeprecatedIndex) -> Self::DeprecatedKey { - concat_key_bytes!(DeprecatedKey, + convert_column_index_to_key_bytes!(DeprecatedKey, ..8 => &index.to_be_bytes(), 8..72 => signature.as_ref(), 72.. => &slot.to_be_bytes(), @@ -940,19 +945,21 @@ impl ColumnIndexDeprecation for columns::TransactionStatus { if key.len() != std::mem::size_of::() { return Err(IndexError::UnpackError); } - let primary_index = BigEndian::read_u64(&key[0..8]); - let signature = Signature::try_from(&key[8..72]).unwrap(); - let slot = BigEndian::read_u64(&key[72..80]); - Ok((primary_index, signature, slot)) + Ok(convert_column_key_bytes_to_index!(key, + 0..8 => u64::from_be_bytes, // primary index + 8..72 => Signature::from, + 72..80 => Slot::from_be_bytes, + )) } fn try_current_index(key: &[u8]) -> std::result::Result { if key.len() != Self::CURRENT_INDEX_LEN { return Err(IndexError::UnpackError); } - let signature = Signature::try_from(&key[0..64]).unwrap(); - let slot = BigEndian::read_u64(&key[64..72]); - Ok((signature, slot)) + Ok(convert_column_key_bytes_to_index!(key, + 0..64 => Signature::from, + 64..72 => Slot::from_be_bytes, + )) } fn convert_index(deprecated_index: Self::DeprecatedIndex) -> Self::Index { @@ -970,7 +977,7 @@ impl Column for columns::AddressSignatures { #[inline] fn key((pubkey, slot, transaction_index, signature): &Self::Index) -> Self::Key { - concat_key_bytes!(Key, + convert_column_index_to_key_bytes!(Key, ..32 => pubkey.as_ref(), 32..40 => &slot.to_be_bytes(), 40..44 => &transaction_index.to_be_bytes(), @@ -1004,7 +1011,7 @@ impl ColumnIndexDeprecation for columns::AddressSignatures { fn deprecated_key( (primary_index, pubkey, slot, signature): Self::DeprecatedIndex, ) -> Self::DeprecatedKey { - concat_key_bytes!(DeprecatedKey, + convert_column_index_to_key_bytes!(DeprecatedKey, ..8 => &primary_index.to_be_bytes(), 8..40 => pubkey.as_ref(), 40..48 => &slot.to_be_bytes(), @@ -1016,22 +1023,24 @@ impl ColumnIndexDeprecation for columns::AddressSignatures { if key.len() != std::mem::size_of::() { return Err(IndexError::UnpackError); } - let primary_index = BigEndian::read_u64(&key[0..8]); - let pubkey = Pubkey::try_from(&key[8..40]).unwrap(); - let slot = BigEndian::read_u64(&key[40..48]); - let signature = Signature::try_from(&key[48..112]).unwrap(); - Ok((primary_index, pubkey, slot, signature)) + Ok(convert_column_key_bytes_to_index!(key, + 0..8 => u64::from_be_bytes, // primary index + 8..40 => Pubkey::from, + 40..48 => Slot::from_be_bytes, + 48..112 => Signature::from, + )) } fn try_current_index(key: &[u8]) -> std::result::Result { if key.len() != Self::CURRENT_INDEX_LEN { return Err(IndexError::UnpackError); } - let pubkey = Pubkey::try_from(&key[0..32]).unwrap(); - let slot = BigEndian::read_u64(&key[32..40]); - let transaction_index = BigEndian::read_u32(&key[40..44]); - let signature = Signature::try_from(&key[44..108]).unwrap(); - Ok((pubkey, slot, transaction_index, signature)) + Ok(convert_column_key_bytes_to_index!(key, + 0..32 => Pubkey::from, + 32..40 => Slot::from_be_bytes, + 40..44 => u32::from_be_bytes, // transaction index + 44..108 => Signature::from, + )) } fn convert_index(deprecated_index: Self::DeprecatedIndex) -> Self::Index { @@ -1046,7 +1055,7 @@ impl Column for columns::TransactionMemos { #[inline] fn key((signature, slot): &Self::Index) -> Self::Key { - concat_key_bytes!(Key, + convert_column_index_to_key_bytes!(Key, ..64 => signature.as_ref(), 64.. => &slot.to_be_bytes(), ) @@ -1085,9 +1094,10 @@ impl ColumnIndexDeprecation for columns::TransactionMemos { if key.len() != Self::CURRENT_INDEX_LEN { return Err(IndexError::UnpackError); } - let signature = Signature::try_from(&key[0..64]).unwrap(); - let slot = BigEndian::read_u64(&key[64..72]); - Ok((signature, slot)) + Ok(convert_column_key_bytes_to_index!(key, + 0..64 => Signature::from, + 64..72 => Slot::from_be_bytes, + )) } fn convert_index(deprecated_index: Self::DeprecatedIndex) -> Self::Index { @@ -1104,8 +1114,8 @@ impl Column for columns::TransactionStatusIndex { index.to_be_bytes() } - fn index(key: &[u8]) -> u64 { - BigEndian::read_u64(&key[..8]) + fn index(key: &[u8]) -> Self::Index { + convert_column_key_bytes_to_index!(key, 0..8 => u64::from_be_bytes) } fn slot(_index: Self::Index) -> Slot { @@ -1165,7 +1175,7 @@ impl Column for columns::ProgramCosts { } fn index(key: &[u8]) -> Self::Index { - Pubkey::try_from(&key[..32]).unwrap() + convert_column_key_bytes_to_index!(key, 0..32 => Pubkey::from) } fn slot(_index: Self::Index) -> Slot { @@ -1178,7 +1188,7 @@ impl Column for columns::ProgramCosts { } impl Column for columns::ShredCode { - type Index = (Slot, u64); + type Index = (Slot, /*shred index:*/ u64); type Key = ::Key; #[inline] @@ -1187,7 +1197,7 @@ impl Column for columns::ShredCode { ::key(index) } - fn index(key: &[u8]) -> (Slot, u64) { + fn index(key: &[u8]) -> Self::Index { columns::ShredData::index(key) } @@ -1204,21 +1214,22 @@ impl ColumnName for columns::ShredCode { } impl Column for columns::ShredData { - type Index = (Slot, u64); + type Index = (Slot, /*shred index:*/ u64); type Key = [u8; std::mem::size_of::() + std::mem::size_of::()]; #[inline] fn key((slot, index): &Self::Index) -> Self::Key { - concat_key_bytes!(Key, + convert_column_index_to_key_bytes!(Key, ..8 => &slot.to_be_bytes(), 8.. => &index.to_be_bytes(), ) } - fn index(key: &[u8]) -> (Slot, u64) { - let slot = BigEndian::read_u64(&key[..8]); - let index = BigEndian::read_u64(&key[8..16]); - (slot, index) + fn index(key: &[u8]) -> Self::Index { + convert_column_key_bytes_to_index!(key, + 0..8 => Slot::from_be_bytes, + 8..16 => u64::from_be_bytes, // shred index + ) } fn slot(index: Self::Index) -> Slot { @@ -1312,22 +1323,22 @@ impl TypedColumn for columns::SlotMeta { } impl Column for columns::ErasureMeta { - type Index = (Slot, u64); + type Index = (Slot, /*fec_set_index:*/ u64); type Key = [u8; std::mem::size_of::() + std::mem::size_of::()]; #[inline] fn key((slot, fec_set_index): &Self::Index) -> Self::Key { - concat_key_bytes!(Key, + convert_column_index_to_key_bytes!(Key, ..8 => &slot.to_be_bytes(), 8.. => &fec_set_index.to_be_bytes(), ) } - fn index(key: &[u8]) -> (Slot, u64) { - let slot = BigEndian::read_u64(&key[..8]); - let set_index = BigEndian::read_u64(&key[8..]); - - (slot, set_index) + fn index(key: &[u8]) -> Self::Index { + convert_column_key_bytes_to_index!(key, + 0..8 => Slot::from_be_bytes, + 8..16 => u64::from_be_bytes, // fec_set_index + ) } fn slot(index: Self::Index) -> Slot { @@ -1359,17 +1370,17 @@ impl Column for columns::MerkleRootMeta { #[inline] fn key((slot, fec_set_index): &Self::Index) -> Self::Key { - concat_key_bytes!(Key, + convert_column_index_to_key_bytes!(Key, ..8 => &slot.to_be_bytes(), 8.. => &fec_set_index.to_be_bytes(), ) } fn index(key: &[u8]) -> Self::Index { - let slot = BigEndian::read_u64(&key[..8]); - let fec_set_index = BigEndian::read_u32(&key[8..]); - - (slot, fec_set_index) + convert_column_key_bytes_to_index!(key, + 0..8 => Slot::from_be_bytes, + 8..12 => u32::from_be_bytes, // fec_set_index + ) } fn slot((slot, _fec_set_index): Self::Index) -> Slot { diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index de95dd72e0d5d4..1e9583d225bc91 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -6153,7 +6153,6 @@ dependencies = [ "assert_matches", "bincode", "bitflags 2.8.0", - "byteorder 1.5.0", "bzip2", "chrono", "chrono-humanize", diff --git a/svm/examples/Cargo.lock b/svm/examples/Cargo.lock index 99d7d319e985cf..b452e8d3ccde20 100644 --- a/svm/examples/Cargo.lock +++ b/svm/examples/Cargo.lock @@ -5983,7 +5983,6 @@ dependencies = [ "assert_matches", "bincode", "bitflags 2.8.0", - "byteorder", "bzip2", "chrono", "chrono-humanize",