simplifies blockstore Column::index implementations (#4464)

The commits adds a small macro to simplify

    pub trait Column {
        fn index(key: &[u8]) -> Self::Index;
        // ...
    }

implementations.

Author: behzadnouri

Cargo.lock

@@ -13,7 +13,6 @@ edition = { workspace = true }
 assert_matches = { workspace = true }
 bincode = { workspace = true }
 bitflags = { workspace = true, features = ["serde"] }
-byteorder = { workspace = true }
 bzip2 = { workspace = true }
 chrono = { workspace = true, features = ["default", "serde"] }
 chrono-humanize = { workspace = true }

ledger/Cargo.toml

@@ -11,7 +11,6 @@ use {
         blockstore_options::{AccessType, BlockstoreOptions, LedgerColumnOptions},
     },
     bincode::{deserialize, Options as BincodeOptions},
-    byteorder::{BigEndian, ByteOrder},
     log::*,
     prost::Message,
     rocksdb::{
@@ -103,6 +102,21 @@ const OPTIMISTIC_SLOTS_CF: &str = "optimistic_slots";
 /// Column family for merkle roots
 const MERKLE_ROOT_META_CF: &str = "merkle_root_meta";
 
+macro_rules! convert_column_index_to_key_bytes {
+    ($key:ident, $($range:expr => $bytes:expr),* $(,)?) => {{
+        let mut key = [0u8; std::mem::size_of::<Self::$key>()];
+        debug_assert_eq!(0 $(+$bytes.len())*, key.len());
+        $(key[$range].copy_from_slice($bytes);)*
+        key
+    }};
+}
+
+macro_rules! convert_column_key_bytes_to_index {
+    ($k:ident, $($a:literal..$b:literal => $f:expr),* $(,)?) => {{
+        ($($f(<[u8; $b-$a]>::try_from(&$k[$a..$b]).unwrap())),*)
+    }};
+}
+
 #[derive(Error, Debug)]
 pub enum BlockstoreError {
     #[error("shred for index exists")]
@@ -837,8 +851,8 @@ impl<T: SlotColumn> Column for T {
     }
 
     /// Converts a RocksDB key to its u64 Index.
-    fn index(key: &[u8]) -> u64 {
-        BigEndian::read_u64(&key[..8])
+    fn index(key: &[u8]) -> Self::Index {
+        convert_column_key_bytes_to_index!(key, 0..8 => Slot::from_be_bytes)
     }
 
     fn slot(index: Self::Index) -> Slot {
@@ -881,22 +895,13 @@ pub trait ColumnIndexDeprecation: Column {
     }
 }
 
-macro_rules! concat_key_bytes {
-    ($key:ident, $($range:expr => $bytes:expr),* $(,)?) => {{
-        let mut key = [0u8; std::mem::size_of::<Self::$key>()];
-        debug_assert_eq!(0 $(+$bytes.len())*, key.len());
-        $(key[$range].copy_from_slice($bytes);)*
-        key
-    }};
-}
-
 impl Column for columns::TransactionStatus {
     type Index = (Signature, Slot);
     type Key = [u8; SIGNATURE_BYTES + std::mem::size_of::<Slot>()];
 
     #[inline]
     fn key((signature, slot): &Self::Index) -> Self::Key {
-        concat_key_bytes!(Key,
+        convert_column_index_to_key_bytes!(Key,
             ..64 => signature.as_ref(),
             64.. => &slot.to_be_bytes(),
         )
@@ -929,7 +934,7 @@ impl ColumnIndexDeprecation for columns::TransactionStatus {
     type DeprecatedKey = [u8; 80];
 
     fn deprecated_key((index, signature, slot): Self::DeprecatedIndex) -> Self::DeprecatedKey {
-        concat_key_bytes!(DeprecatedKey,
+        convert_column_index_to_key_bytes!(DeprecatedKey,
               ..8  => &index.to_be_bytes(),
              8..72 => signature.as_ref(),
             72..   => &slot.to_be_bytes(),
@@ -940,19 +945,21 @@ impl ColumnIndexDeprecation for columns::TransactionStatus {
         if key.len() != std::mem::size_of::<Self::DeprecatedKey>() {
             return Err(IndexError::UnpackError);
         }
-        let primary_index = BigEndian::read_u64(&key[0..8]);
-        let signature = Signature::try_from(&key[8..72]).unwrap();
-        let slot = BigEndian::read_u64(&key[72..80]);
-        Ok((primary_index, signature, slot))
+        Ok(convert_column_key_bytes_to_index!(key,
+             0..8  => u64::from_be_bytes,  // primary index
+             8..72 => Signature::from,
+            72..80 => Slot::from_be_bytes,
+        ))
     }
 
     fn try_current_index(key: &[u8]) -> std::result::Result<Self::Index, IndexError> {
         if key.len() != Self::CURRENT_INDEX_LEN {
             return Err(IndexError::UnpackError);
         }
-        let signature = Signature::try_from(&key[0..64]).unwrap();
-        let slot = BigEndian::read_u64(&key[64..72]);
-        Ok((signature, slot))
+        Ok(convert_column_key_bytes_to_index!(key,
+             0..64 => Signature::from,
+            64..72 => Slot::from_be_bytes,
+        ))
     }
 
     fn convert_index(deprecated_index: Self::DeprecatedIndex) -> Self::Index {
@@ -970,7 +977,7 @@ impl Column for columns::AddressSignatures {
 
     #[inline]
     fn key((pubkey, slot, transaction_index, signature): &Self::Index) -> Self::Key {
-        concat_key_bytes!(Key,
+        convert_column_index_to_key_bytes!(Key,
               ..32 => pubkey.as_ref(),
             32..40 => &slot.to_be_bytes(),
             40..44 => &transaction_index.to_be_bytes(),
@@ -1004,7 +1011,7 @@ impl ColumnIndexDeprecation for columns::AddressSignatures {
     fn deprecated_key(
         (primary_index, pubkey, slot, signature): Self::DeprecatedIndex,
     ) -> Self::DeprecatedKey {
-        concat_key_bytes!(DeprecatedKey,
+        convert_column_index_to_key_bytes!(DeprecatedKey,
               ..8  => &primary_index.to_be_bytes(),
              8..40 => pubkey.as_ref(),
             40..48 => &slot.to_be_bytes(),
@@ -1016,22 +1023,24 @@ impl ColumnIndexDeprecation for columns::AddressSignatures {
         if key.len() != std::mem::size_of::<Self::DeprecatedKey>() {
             return Err(IndexError::UnpackError);
         }
-        let primary_index = BigEndian::read_u64(&key[0..8]);
-        let pubkey = Pubkey::try_from(&key[8..40]).unwrap();
-        let slot = BigEndian::read_u64(&key[40..48]);
-        let signature = Signature::try_from(&key[48..112]).unwrap();
-        Ok((primary_index, pubkey, slot, signature))
+        Ok(convert_column_key_bytes_to_index!(key,
+             0..8   => u64::from_be_bytes,  // primary index
+             8..40  => Pubkey::from,
+            40..48  => Slot::from_be_bytes,
+            48..112 => Signature::from,
+        ))
     }
 
     fn try_current_index(key: &[u8]) -> std::result::Result<Self::Index, IndexError> {
         if key.len() != Self::CURRENT_INDEX_LEN {
             return Err(IndexError::UnpackError);
         }
-        let pubkey = Pubkey::try_from(&key[0..32]).unwrap();
-        let slot = BigEndian::read_u64(&key[32..40]);
-        let transaction_index = BigEndian::read_u32(&key[40..44]);
-        let signature = Signature::try_from(&key[44..108]).unwrap();
-        Ok((pubkey, slot, transaction_index, signature))
+        Ok(convert_column_key_bytes_to_index!(key,
+             0..32  => Pubkey::from,
+            32..40  => Slot::from_be_bytes,
+            40..44  => u32::from_be_bytes,  // transaction index
+            44..108 => Signature::from,
+        ))
     }
 
     fn convert_index(deprecated_index: Self::DeprecatedIndex) -> Self::Index {
@@ -1046,7 +1055,7 @@ impl Column for columns::TransactionMemos {
 
     #[inline]
     fn key((signature, slot): &Self::Index) -> Self::Key {
-        concat_key_bytes!(Key,
+        convert_column_index_to_key_bytes!(Key,
             ..64 => signature.as_ref(),
             64.. => &slot.to_be_bytes(),
         )
@@ -1085,9 +1094,10 @@ impl ColumnIndexDeprecation for columns::TransactionMemos {
         if key.len() != Self::CURRENT_INDEX_LEN {
             return Err(IndexError::UnpackError);
         }
-        let signature = Signature::try_from(&key[0..64]).unwrap();
-        let slot = BigEndian::read_u64(&key[64..72]);
-        Ok((signature, slot))
+        Ok(convert_column_key_bytes_to_index!(key,
+             0..64 => Signature::from,
+            64..72 => Slot::from_be_bytes,
+        ))
     }
 
     fn convert_index(deprecated_index: Self::DeprecatedIndex) -> Self::Index {
@@ -1104,8 +1114,8 @@ impl Column for columns::TransactionStatusIndex {
         index.to_be_bytes()
     }
 
-    fn index(key: &[u8]) -> u64 {
-        BigEndian::read_u64(&key[..8])
+    fn index(key: &[u8]) -> Self::Index {
+        convert_column_key_bytes_to_index!(key, 0..8 => u64::from_be_bytes)
     }
 
     fn slot(_index: Self::Index) -> Slot {
@@ -1165,7 +1175,7 @@ impl Column for columns::ProgramCosts {
     }
 
     fn index(key: &[u8]) -> Self::Index {
-        Pubkey::try_from(&key[..32]).unwrap()
+        convert_column_key_bytes_to_index!(key, 0..32 => Pubkey::from)
     }
 
     fn slot(_index: Self::Index) -> Slot {
@@ -1178,7 +1188,7 @@ impl Column for columns::ProgramCosts {
 }
 
 impl Column for columns::ShredCode {
-    type Index = (Slot, u64);
+    type Index = (Slot, /*shred index:*/ u64);
     type Key = <columns::ShredData as Column>::Key;
 
     #[inline]
@@ -1187,7 +1197,7 @@ impl Column for columns::ShredCode {
         <columns::ShredData as Column>::key(index)
     }
 
-    fn index(key: &[u8]) -> (Slot, u64) {
+    fn index(key: &[u8]) -> Self::Index {
         columns::ShredData::index(key)
     }
 
@@ -1204,21 +1214,22 @@ impl ColumnName for columns::ShredCode {
 }
 
 impl Column for columns::ShredData {
-    type Index = (Slot, u64);
+    type Index = (Slot, /*shred index:*/ u64);
     type Key = [u8; std::mem::size_of::<Slot>() + std::mem::size_of::<u64>()];
 
     #[inline]
     fn key((slot, index): &Self::Index) -> Self::Key {
-        concat_key_bytes!(Key,
+        convert_column_index_to_key_bytes!(Key,
             ..8 => &slot.to_be_bytes(),
             8.. => &index.to_be_bytes(),
         )
     }
 
-    fn index(key: &[u8]) -> (Slot, u64) {
-        let slot = BigEndian::read_u64(&key[..8]);
-        let index = BigEndian::read_u64(&key[8..16]);
-        (slot, index)
+    fn index(key: &[u8]) -> Self::Index {
+        convert_column_key_bytes_to_index!(key,
+            0..8  => Slot::from_be_bytes,
+            8..16 => u64::from_be_bytes,  // shred index
+        )
     }
 
     fn slot(index: Self::Index) -> Slot {
@@ -1312,22 +1323,22 @@ impl TypedColumn for columns::SlotMeta {
 }
 
 impl Column for columns::ErasureMeta {
-    type Index = (Slot, u64);
+    type Index = (Slot, /*fec_set_index:*/ u64);
     type Key = [u8; std::mem::size_of::<Slot>() + std::mem::size_of::<u64>()];
 
     #[inline]
     fn key((slot, fec_set_index): &Self::Index) -> Self::Key {
-        concat_key_bytes!(Key,
+        convert_column_index_to_key_bytes!(Key,
             ..8 => &slot.to_be_bytes(),
             8.. => &fec_set_index.to_be_bytes(),
         )
     }
 
-    fn index(key: &[u8]) -> (Slot, u64) {
-        let slot = BigEndian::read_u64(&key[..8]);
-        let set_index = BigEndian::read_u64(&key[8..]);
-
-        (slot, set_index)
+    fn index(key: &[u8]) -> Self::Index {
+        convert_column_key_bytes_to_index!(key,
+            0..8  => Slot::from_be_bytes,
+            8..16 => u64::from_be_bytes,  // fec_set_index
+        )
     }
 
     fn slot(index: Self::Index) -> Slot {
@@ -1359,17 +1370,17 @@ impl Column for columns::MerkleRootMeta {
 
     #[inline]
     fn key((slot, fec_set_index): &Self::Index) -> Self::Key {
-        concat_key_bytes!(Key,
+        convert_column_index_to_key_bytes!(Key,
             ..8 => &slot.to_be_bytes(),
             8.. => &fec_set_index.to_be_bytes(),
         )
     }
 
     fn index(key: &[u8]) -> Self::Index {
-        let slot = BigEndian::read_u64(&key[..8]);
-        let fec_set_index = BigEndian::read_u32(&key[8..]);
-
-        (slot, fec_set_index)
+        convert_column_key_bytes_to_index!(key,
+            0..8  => Slot::from_be_bytes,
+            8..12 => u32::from_be_bytes,  // fec_set_index
+        )
     }
 
     fn slot((slot, _fec_set_index): Self::Index) -> Slot {