|
| 1 | +use candid::CandidType; |
| 2 | +use ciborium::from_reader; |
| 3 | +use ic_icrc1::{blocks::generic_block_to_encoded_block, Block}; |
| 4 | +use ic_icrc1_tokens_u64::U64; |
| 5 | +use ic_ledger_core::block::{BlockType, EncodedBlock}; |
| 6 | +use icrc_ledger_types::icrc3::blocks::{GenericBlock, GetBlocksRequest, ICRC3GenericBlock}; |
| 7 | +use serde::{Deserialize, Serialize}; |
| 8 | +use serde_bytes::ByteArray; |
| 9 | +use std::{collections::BTreeMap, path::PathBuf}; |
| 10 | + |
| 11 | +use super::{format_error, ledger::Icrc1Agent, nat_to_u64, to_cbor_bytes}; |
| 12 | + |
| 13 | +pub type BlockToken64 = Block<U64>; |
| 14 | +// 1~100 blocks |
| 15 | +pub type Blocks = BTreeMap<u64, BlockToken64>; |
| 16 | +const BATCH_LENGTH: u64 = 100; |
| 17 | + |
| 18 | +#[derive(CandidType, Clone, Debug, Deserialize, Serialize)] |
| 19 | +pub struct TipBlock { |
| 20 | + pub index: u64, |
| 21 | + pub hash: ByteArray<32>, |
| 22 | +} |
| 23 | + |
| 24 | +#[derive(Clone)] |
| 25 | +pub struct BlocksClient { |
| 26 | + agent: Icrc1Agent, |
| 27 | + store_dir: PathBuf, |
| 28 | +} |
| 29 | + |
| 30 | +impl BlocksClient { |
| 31 | + pub fn new(agent: Icrc1Agent, store_dir: &str) -> Self { |
| 32 | + Self { |
| 33 | + agent, |
| 34 | + store_dir: PathBuf::from(store_dir), |
| 35 | + } |
| 36 | + } |
| 37 | + |
| 38 | + pub async fn start_synching_blocks(&self) -> Result<TipBlock, String> { |
| 39 | + let prev_tip: Option<u64> = match std::fs::read(self.store_dir.join("tip.txt")) { |
| 40 | + Ok(tip) => Some( |
| 41 | + String::from_utf8_lossy(&tip) |
| 42 | + .trim() |
| 43 | + .parse() |
| 44 | + .map_err(format_error)?, |
| 45 | + ), |
| 46 | + Err(_) => None, |
| 47 | + }; |
| 48 | + println!("Previous tip: {:?}", prev_tip); |
| 49 | + let latest_tip = self.agent.get_certified_chain_tip().await?; |
| 50 | + let latest_tip = TipBlock { |
| 51 | + index: latest_tip.1, |
| 52 | + hash: latest_tip.0.into(), |
| 53 | + }; |
| 54 | + println!("latest_tip: {:?}", latest_tip); |
| 55 | + |
| 56 | + let (mut start_index, mut parent_hash): (u64, Option<[u8; 32]>) = |
| 57 | + if let Some(prev_tip) = prev_tip { |
| 58 | + let idx = prev_tip / BATCH_LENGTH; |
| 59 | + |
| 60 | + match std::fs::read(self.store_dir.join(format!("{}.cbor", idx))) { |
| 61 | + Ok(data) => { |
| 62 | + let blks: Blocks = from_reader(data.as_slice()).map_err(format_error)?; |
| 63 | + if prev_tip == (idx + 1) * BATCH_LENGTH - 1 { |
| 64 | + let blk = blks |
| 65 | + .get(&prev_tip) |
| 66 | + .ok_or_else(|| format!("Block {} not found", prev_tip))?; |
| 67 | + ( |
| 68 | + prev_tip + 1, |
| 69 | + Some(BlockToken64::block_hash(&blk.clone().encode()).into_bytes()), |
| 70 | + ) |
| 71 | + } else { |
| 72 | + let start_index = idx * BATCH_LENGTH; |
| 73 | + let blk = blks |
| 74 | + .get(&start_index) |
| 75 | + .ok_or_else(|| format!("Block {} not found", start_index))?; |
| 76 | + (start_index, blk.parent_hash.map(|h| h.into_bytes())) |
| 77 | + } |
| 78 | + } |
| 79 | + Err(err) => Err(format_error(err))?, |
| 80 | + } |
| 81 | + } else { |
| 82 | + (0, None) |
| 83 | + }; |
| 84 | + |
| 85 | + loop { |
| 86 | + let tip = self |
| 87 | + .batch_sync_blocks(&latest_tip, start_index, parent_hash) |
| 88 | + .await?; |
| 89 | + println!( |
| 90 | + "Synced up to block {}, hash: {}", |
| 91 | + tip.index, |
| 92 | + hex::encode(tip.hash.as_slice()) |
| 93 | + ); |
| 94 | + if tip.index == latest_tip.index { |
| 95 | + if tip.hash != latest_tip.hash { |
| 96 | + return Err(format!( |
| 97 | + "The latest tip hash does not match, expected: {}, got: {}", |
| 98 | + hex::encode(latest_tip.hash.as_slice()), |
| 99 | + hex::encode(tip.hash.as_slice()) |
| 100 | + )); |
| 101 | + } |
| 102 | + return Ok(tip); |
| 103 | + } |
| 104 | + |
| 105 | + start_index = tip.index + 1; |
| 106 | + parent_hash = Some(*tip.hash); |
| 107 | + } |
| 108 | + } |
| 109 | + |
| 110 | + pub async fn batch_sync_blocks( |
| 111 | + &self, |
| 112 | + latest_tip_block: &TipBlock, |
| 113 | + start_index: u64, |
| 114 | + mut parent_hash: Option<[u8; 32]>, |
| 115 | + ) -> Result<TipBlock, String> { |
| 116 | + if start_index % BATCH_LENGTH != 0 { |
| 117 | + return Err(format!( |
| 118 | + "The start index {} is not a multiple of the batch length {}", |
| 119 | + start_index, BATCH_LENGTH |
| 120 | + )); |
| 121 | + } |
| 122 | + |
| 123 | + let idx = start_index / BATCH_LENGTH; |
| 124 | + let end_index = latest_tip_block.index.min((idx + 1) * BATCH_LENGTH - 1); |
| 125 | + let length = end_index + 1 - start_index; |
| 126 | + let mut blocks: Blocks = BTreeMap::new(); |
| 127 | + |
| 128 | + let res = self |
| 129 | + .agent |
| 130 | + .icrc3_get_blocks(vec![GetBlocksRequest { |
| 131 | + start: start_index.into(), |
| 132 | + length: length.into(), |
| 133 | + }]) |
| 134 | + .await?; |
| 135 | + for block in res.blocks { |
| 136 | + let id = nat_to_u64(&block.id); |
| 137 | + if id >= start_index && id <= end_index { |
| 138 | + let encoded = icrc3_block_to_encoded_block(block.block)?; |
| 139 | + blocks.insert(id, Block::decode(encoded)?); |
| 140 | + } |
| 141 | + } |
| 142 | + for archive_query in res.archived_blocks { |
| 143 | + let res2 = self |
| 144 | + .agent |
| 145 | + .icrc3_get_blocks_in(&archive_query.callback.canister_id, archive_query.args) |
| 146 | + .await?; |
| 147 | + for block in res2.blocks { |
| 148 | + let id = nat_to_u64(&block.id); |
| 149 | + if id >= start_index && id <= end_index { |
| 150 | + let encoded = icrc3_block_to_encoded_block(block.block)?; |
| 151 | + let blk = BlockToken64::decode(encoded); |
| 152 | + blocks.insert(id, blk?); |
| 153 | + } |
| 154 | + } |
| 155 | + } |
| 156 | + |
| 157 | + for index in start_index..=end_index { |
| 158 | + let blk = blocks |
| 159 | + .get(&index) |
| 160 | + .ok_or_else(|| format!("Block {} not found in the response from the IC", index))?; |
| 161 | + |
| 162 | + if blk.parent_hash.map(|h| h.into_bytes()) != parent_hash { |
| 163 | + return Err(format!( |
| 164 | + "Block {}'s parent hash does not match the previous block's hash", |
| 165 | + index |
| 166 | + )); |
| 167 | + } |
| 168 | + parent_hash = Some(BlockToken64::block_hash(&blk.clone().encode()).into_bytes()); |
| 169 | + } |
| 170 | + |
| 171 | + let blk = blocks.get(&end_index).unwrap(); |
| 172 | + let tip = TipBlock { |
| 173 | + index: end_index, |
| 174 | + hash: BlockToken64::block_hash(&blk.clone().encode()) |
| 175 | + .into_bytes() |
| 176 | + .into(), |
| 177 | + }; |
| 178 | + std::fs::write( |
| 179 | + self.store_dir.join(format!("{}.cbor", idx)), |
| 180 | + to_cbor_bytes(&blocks), |
| 181 | + ) |
| 182 | + .map_err(format_error)?; |
| 183 | + std::fs::write(self.store_dir.join("tip.txt"), tip.index.to_string()) |
| 184 | + .map_err(format_error)?; |
| 185 | + Ok(tip) |
| 186 | + } |
| 187 | + |
| 188 | + pub fn iter(&self) -> Result<BlocksIter<'_>, String> { |
| 189 | + let tip: u64 = match std::fs::read(self.store_dir.join("tip.txt")) { |
| 190 | + Ok(tip) => String::from_utf8_lossy(&tip) |
| 191 | + .trim() |
| 192 | + .parse() |
| 193 | + .map_err(format_error)?, |
| 194 | + Err(err) => Err(format_error(err))?, |
| 195 | + }; |
| 196 | + Ok(BlocksIter { |
| 197 | + client: self, |
| 198 | + next_index: 0, |
| 199 | + end: tip + 1, |
| 200 | + blocks: BTreeMap::new(), |
| 201 | + }) |
| 202 | + } |
| 203 | +} |
| 204 | + |
| 205 | +pub struct BlocksIter<'a> { |
| 206 | + client: &'a BlocksClient, |
| 207 | + next_index: u64, |
| 208 | + end: u64, |
| 209 | + blocks: Blocks, |
| 210 | +} |
| 211 | + |
| 212 | +impl<'a> Iterator for BlocksIter<'a> { |
| 213 | + type Item = (u64, BlockToken64); |
| 214 | + |
| 215 | + fn next(&mut self) -> Option<Self::Item> { |
| 216 | + let index = self.next_index; |
| 217 | + self.next_index += 1; |
| 218 | + |
| 219 | + if index >= self.end { |
| 220 | + return None; |
| 221 | + } |
| 222 | + |
| 223 | + if !self.blocks.contains_key(&index) { |
| 224 | + let idx = index / BATCH_LENGTH; |
| 225 | + let blks: Option<Blocks> = |
| 226 | + match std::fs::read(self.client.store_dir.join(format!("{}.cbor", idx))) { |
| 227 | + Ok(data) => from_reader(data.as_slice()).ok(), |
| 228 | + Err(_) => None, |
| 229 | + }; |
| 230 | + if let Some(blks) = blks { |
| 231 | + self.blocks.extend(blks); |
| 232 | + } |
| 233 | + } |
| 234 | + |
| 235 | + if let Some(block) = self.blocks.remove(&index) { |
| 236 | + Some((index, block)) |
| 237 | + } else { |
| 238 | + None |
| 239 | + } |
| 240 | + } |
| 241 | +} |
| 242 | + |
| 243 | +fn icrc3_block_to_encoded_block(block: ICRC3GenericBlock) -> Result<EncodedBlock, String> { |
| 244 | + generic_block_to_encoded_block(GenericBlock::from(block)) |
| 245 | +} |
| 246 | + |
| 247 | +// #[cfg(test)] |
| 248 | +// mod test { |
| 249 | +// use super::*; |
| 250 | + |
| 251 | +// #[test] |
| 252 | +// fn self_tag() {} |
| 253 | +// } |
0 commit comments