feat: get BlockMeta table values from static file or database (#13844)

Author: joshieDo

crates/storage/provider/src/providers/database/chain.rs

@@ -23,6 +23,7 @@ where
     T: FullSignedTx,
     N: FullNodePrimitives<
         Block = reth_primitives::Block<T>,
+        BlockHeader = alloy_consensus::Header,
         BlockBody = reth_primitives::BlockBody<T>,
         SignedTx = T,
     >,

crates/storage/provider/src/providers/database/mod.rs

@@ -562,14 +562,29 @@ impl<N: ProviderNodeTypes> BlockBodyIndicesProvider for ProviderFactory<N> {
         &self,
         number: BlockNumber,
     ) -> ProviderResult<Option<StoredBlockBodyIndices>> {
-        self.provider()?.block_body_indices(number)
+        self.static_file_provider.get_with_static_file_or_database(
+            StaticFileSegment::BlockMeta,
+            number,
+            |static_file| static_file.block_body_indices(number),
+            || self.provider()?.block_body_indices(number),
+        )
     }
 
     fn block_body_indices_range(
         &self,
         range: RangeInclusive<BlockNumber>,
     ) -> ProviderResult<Vec<StoredBlockBodyIndices>> {
-        self.provider()?.block_body_indices_range(range)
+        self.static_file_provider.get_range_with_static_file_or_database(
+            StaticFileSegment::BlockMeta,
+            *range.start()..*range.end() + 1,
+            |static_file, range, _| {
+                static_file.block_body_indices_range(range.start..=range.end.saturating_sub(1))
+            },
+            |range, _| {
+                self.provider()?.block_body_indices_range(range.start..=range.end.saturating_sub(1))
+            },
+            |_| true,
+        )
     }
 }
 

crates/storage/provider/src/providers/database/provider.rs

@@ -1574,14 +1574,21 @@ impl<TX: DbTx + 'static, N: NodeTypes<ChainSpec: EthereumHardforks>> Withdrawals
     ) -> ProviderResult<Option<Withdrawals>> {
         if self.chain_spec.is_shanghai_active_at_timestamp(timestamp) {
             if let Some(number) = self.convert_hash_or_number(id)? {
-                // If we are past shanghai, then all blocks should have a withdrawal list, even if
-                // empty
-                let withdrawals = self
-                    .tx
-                    .get::<tables::BlockWithdrawals>(number)
-                    .map(|w| w.map(|w| w.withdrawals))?
-                    .unwrap_or_default();
-                return Ok(Some(withdrawals))
+                return self.static_file_provider.get_with_static_file_or_database(
+                    StaticFileSegment::BlockMeta,
+                    number,
+                    |static_file| static_file.withdrawals_by_block(number.into(), timestamp),
+                    || {
+                        // If we are past shanghai, then all blocks should have a withdrawal list,
+                        // even if empty
+                        let withdrawals = self
+                            .tx
+                            .get::<tables::BlockWithdrawals>(number)
+                            .map(|w| w.map(|w| w.withdrawals))?
+                            .unwrap_or_default();
+                        Ok(Some(withdrawals))
+                    },
+                )
             }
         }
         Ok(None)
@@ -1601,9 +1608,12 @@ impl<TX: DbTx + 'static, N: NodeTypesForProvider> OmmersProvider for DatabasePro
                 return Ok(Some(Vec::new()))
             }
 
-            let ommers =
-                self.tx.get::<tables::BlockOmmers<Self::Header>>(number)?.map(|o| o.ommers);
-            return Ok(ommers)
+            return self.static_file_provider.get_with_static_file_or_database(
+                StaticFileSegment::BlockMeta,
+                number,
+                |static_file| static_file.ommers(id),
+                || Ok(self.tx.get::<tables::BlockOmmers<Self::Header>>(number)?.map(|o| o.ommers)),
+            )
         }
 
         Ok(None)
@@ -1614,19 +1624,27 @@ impl<TX: DbTx + 'static, N: NodeTypesForProvider> BlockBodyIndicesProvider
     for DatabaseProvider<TX, N>
 {
     fn block_body_indices(&self, num: u64) -> ProviderResult<Option<StoredBlockBodyIndices>> {
-        Ok(self.tx.get::<tables::BlockBodyIndices>(num)?)
+        self.static_file_provider.get_with_static_file_or_database(
+            StaticFileSegment::BlockMeta,
+            num,
+            |static_file| static_file.block_body_indices(num),
+            || Ok(self.tx.get::<tables::BlockBodyIndices>(num)?),
+        )
     }
 
     fn block_body_indices_range(
         &self,
         range: RangeInclusive<BlockNumber>,
     ) -> ProviderResult<Vec<StoredBlockBodyIndices>> {
-        Ok(self
-            .tx_ref()
-            .cursor_read::<tables::BlockBodyIndices>()?
-            .walk_range(range)?
-            .map(|r| r.map(|(_, b)| b))
-            .collect::<Result<_, _>>()?)
+        self.static_file_provider.get_range_with_static_file_or_database(
+            StaticFileSegment::BlockMeta,
+            *range.start()..*range.end() + 1,
+            |static_file, range, _| {
+                static_file.block_body_indices_range(range.start..=range.end.saturating_sub(1))
+            },
+            |range, _| self.cursor_read_collect::<tables::BlockBodyIndices>(range),
+            |_| true,
+        )
     }
 }
 

crates/storage/provider/src/providers/static_file/manager.rs

@@ -17,8 +17,8 @@ use reth_chainspec::{ChainInfo, ChainSpecProvider};
 use reth_db::{
     lockfile::StorageLock,
     static_file::{
-        iter_static_files, BlockHashMask, HeaderMask, HeaderWithHashMask, ReceiptMask,
-        StaticFileCursor, TDWithHashMask, TransactionMask,
+        iter_static_files, BlockHashMask, BodyIndicesMask, HeaderMask, HeaderWithHashMask,
+        ReceiptMask, StaticFileCursor, TDWithHashMask, TransactionMask,
     },
     table::{Decompress, Value},
     tables,
@@ -1027,7 +1027,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
                                 "Could not find block or tx number on a range request"
                             );
 
-                            let err = if segment.is_headers() {
+                            let err = if segment.is_block_based() {
                                 ProviderError::MissingStaticFileBlock(segment, number)
                             } else {
                                 ProviderError::MissingStaticFileTx(segment, number)
@@ -1732,10 +1732,14 @@ impl<N: NodePrimitives> BlockBodyIndicesProvider for StaticFileProvider<N> {
 
     fn block_body_indices_range(
         &self,
-        _range: RangeInclusive<BlockNumber>,
+        range: RangeInclusive<BlockNumber>,
     ) -> ProviderResult<Vec<StoredBlockBodyIndices>> {
-        // Required data not present in static_files
-        Err(ProviderError::UnsupportedProvider)
+        self.fetch_range_with_predicate(
+            StaticFileSegment::BlockMeta,
+            *range.start()..*range.end() + 1,
+            |cursor, number| cursor.get_one::<BodyIndicesMask>(number.into()),
+            |_| true,
+        )
     }
 }
 

crates/storage/storage-api/src/chain.rs

@@ -1,4 +1,5 @@
-use crate::{DBProvider, StorageLocation};
+use crate::{DBProvider, OmmersProvider, StorageLocation};
+use alloy_consensus::Header;
 use alloy_primitives::BlockNumber;
 use reth_chainspec::{ChainSpecProvider, EthereumHardforks};
 use reth_db::{
@@ -138,7 +139,9 @@ where
 
 impl<Provider, T> BlockBodyReader<Provider> for EthStorage<T>
 where
-    Provider: DBProvider + ChainSpecProvider<ChainSpec: EthereumHardforks>,
+    Provider: DBProvider
+        + ChainSpecProvider<ChainSpec: EthereumHardforks>
+        + OmmersProvider<Header = Header>,
     T: SignedTransaction,
 {
     type Block = reth_primitives::Block<T>;
@@ -151,7 +154,6 @@ where
         // TODO: Ideally storage should hold its own copy of chain spec
         let chain_spec = provider.chain_spec();
 
-        let mut ommers_cursor = provider.tx_ref().cursor_read::<tables::BlockOmmers>()?;
         let mut withdrawals_cursor = provider.tx_ref().cursor_read::<tables::BlockWithdrawals>()?;
 
         let mut bodies = Vec::with_capacity(inputs.len());
@@ -171,7 +173,7 @@ where
             let ommers = if chain_spec.final_paris_total_difficulty(header.number).is_some() {
                 Vec::new()
             } else {
-                ommers_cursor.seek_exact(header.number)?.map(|(_, o)| o.ommers).unwrap_or_default()
+                provider.ommers(header.number.into())?.unwrap_or_default()
             };
 
             bodies.push(reth_primitives::BlockBody { transactions, ommers, withdrawals });