From ad78669204c7631af4f00ad0cadcb617bbff54d8 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 1 Mar 2024 16:14:52 -0500 Subject: [PATCH 01/12] feat: detect http / rpc errors as early as possible --- .../chainhook-sdk/src/indexer/bitcoin/mod.rs | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs index 034eda6a8..44c999b86 100644 --- a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs +++ b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs @@ -8,6 +8,7 @@ use crate::observer::BitcoinConfig; use crate::utils::Context; use bitcoincore_rpc::bitcoin::hashes::Hash; use bitcoincore_rpc::bitcoin::{self, Address, Amount, BlockHash}; +use bitcoincore_rpc::jsonrpc::error::RpcError; use bitcoincore_rpc_json::GetRawTransactionResultVoutScriptPubKey; use chainhook_types::bitcoin::{OutPoint, TxIn, TxOut}; use chainhook_types::{ @@ -264,6 +265,11 @@ pub async fn try_download_block_bytes_with_retry( Ok(response) } +#[derive(Debug, Clone, Deserialize)] +pub struct RpcErrorResponse { + pub error: RpcError, +} + pub async fn download_block( http_client: &HttpClient, block_hash: &str, @@ -276,7 +282,7 @@ pub async fn download_block( "method": "getblock", "params": [block_hash, 3] }); - let block = http_client + let res = http_client .post(&bitcoin_config.rpc_url) .basic_auth(&bitcoin_config.username, Some(&bitcoin_config.password)) .header("Content-Type", "application/json") @@ -284,12 +290,31 @@ pub async fn download_block( .json(&body) .send() .await - .map_err(|e| format!("unable to send request ({})", e))? + .map_err(|e| format!("unable to send request ({})", e))?; + + // Check status code + if !res.status().is_success() { + return Err(format!( + "http request unsuccessful ({:?})", + res.error_for_status() + )); + } + + let rpc_response_bytes = res .bytes() .await .map_err(|e| format!("unable to get bytes ({})", e))? .to_vec(); - Ok(block) + + // Check rpc error presence + if let Ok(rpc_error) = serde_json::from_slice::(&rpc_response_bytes[..]) { + return Err(format!( + "rpc request unsuccessful ({})", + rpc_error.error.message + )); + } + + Ok(rpc_response_bytes) } pub fn parse_downloaded_block( From 8f728f7e3f82306154478eceeb5c9d0ef4931028 Mon Sep 17 00:00:00 2001 From: Micaiah Reid Date: Thu, 14 Mar 2024 16:29:07 -0400 Subject: [PATCH 02/12] fix(metrics): update latest ingested block on reorg (#515) Chainhook's `chainhook_stx_highest_block_ingested` and `chainhook_btc_highest_block_ingested` metrics were not being updated on a reorg. This was causing the Chainhook node to appear to be lagging behing the source chains, event though Chainhook's reorg handler was functioning properly. This PR sets the highest block ingested metrics on a reorg based on the highest index block being applied in the reorg. --- components/chainhook-sdk/src/observer/mod.rs | 25 +++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs index 9bfa00c91..b27e3b292 100644 --- a/components/chainhook-sdk/src/observer/mod.rs +++ b/components/chainhook-sdk/src/observer/mod.rs @@ -971,11 +971,16 @@ pub async fn start_observer_commands_handler( .iter() .max_by_key(|b| b.block_identifier.index) { - Some(highest_tip_block) => prometheus_monitoring.btc_metrics_set_reorg( - highest_tip_block.timestamp.into(), - blocks_to_apply.len() as u64, - blocks_to_rollback.len() as u64, - ), + Some(highest_tip_block) => { + prometheus_monitoring.btc_metrics_set_reorg( + highest_tip_block.timestamp.into(), + blocks_to_apply.len() as u64, + blocks_to_rollback.len() as u64, + ); + prometheus_monitoring.btc_metrics_ingest_block( + highest_tip_block.block_identifier.index, + ); + } None => {} } @@ -1183,12 +1188,16 @@ pub async fn start_observer_commands_handler( .iter() .max_by_key(|b| b.block.block_identifier.index) { - Some(highest_tip_update) => prometheus_monitoring - .stx_metrics_set_reorg( + Some(highest_tip_update) => { + prometheus_monitoring.stx_metrics_set_reorg( highest_tip_update.block.timestamp, update.blocks_to_apply.len() as u64, update.blocks_to_rollback.len() as u64, - ), + ); + prometheus_monitoring.stx_metrics_ingest_block( + highest_tip_update.block.block_identifier.index, + ) + } None => {} } } From 485394e9f3eb35089e0b0082ca0c23fbb0e9028f Mon Sep 17 00:00:00 2001 From: Micaiah Reid Date: Thu, 14 Mar 2024 16:42:30 -0400 Subject: [PATCH 03/12] fix: seed forking handler with unconfirmed blocks to improve startup stability (#505) ### Description If Chainhook restarts in the middle of a reorg taking place, it doesn't have any context for choosing the canonical fork. Then when block collisions take place, Chainhook fails to process the new blocks, causing gaps in the blocks Chainhook has available for evaluation. This PR seeds the stacks block indexer with unconfirmed blocks on startup, so that Chainhook has the necessary context to handle a reorg. Most of the PR is to add two tests: - I've added some new functionality to our very thorough indexer tests - In addition to providing the blocks to be mined and the order to mine them, we also now allow providing some "unconfirmed" blocks to seed the block pool with. - I've added some test cases that reproduce what caused outages on the Platform's Chainhook node - I've added a new service test that: - Verifies that unconfirmed blocks are stored on restart - Verifies that those blocks are used to seed the block pool, and that a reorg is handled correctly. I committed these tests _before_ adding the fix, so you can confirm the fix by checking out the commits with the new tests (96d8249b239e53a877a98ff493ef7ae3571aca37 and 9aad55e2a88bf5b75c3e49d079c5967b7a8cf0e3), seeing that the tests break, then pulling the last commit to see that the fix works. Fixes #487 --- components/chainhook-cli/src/cli/mod.rs | 2 +- components/chainhook-cli/src/service/mod.rs | 24 +- .../src/service/tests/helpers/mock_service.rs | 263 ++++++- .../service/tests/helpers/mock_stacks_node.rs | 43 +- .../src/service/tests/helpers/mod.rs | 12 +- .../chainhook-cli/src/service/tests/mod.rs | 696 ++++++++---------- .../src/service/tests/observer_tests.rs | 38 +- components/chainhook-cli/src/storage/mod.rs | 49 +- components/chainhook-sdk/src/indexer/mod.rs | 6 +- .../src/indexer/stacks/blocks_pool.rs | 26 + .../chainhook-sdk/src/indexer/stacks/tests.rs | 113 +-- .../indexer/tests/helpers/stacks_shapes.rs | 90 ++- .../chainhook-sdk/src/indexer/tests/mod.rs | 12 +- components/chainhook-sdk/src/observer/mod.rs | 10 +- 14 files changed, 851 insertions(+), 533 deletions(-) diff --git a/components/chainhook-cli/src/cli/mod.rs b/components/chainhook-cli/src/cli/mod.rs index ecb6dc96f..3fa1b1a88 100644 --- a/components/chainhook-cli/src/cli/mod.rs +++ b/components/chainhook-cli/src/cli/mod.rs @@ -310,7 +310,7 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> { info!(ctx.expect_logger(), "Starting service...",); let mut service = Service::new(config, ctx); - return service.run(predicates).await; + return service.run(predicates, None).await; } }, Command::Config(subcmd) => match subcmd { diff --git a/components/chainhook-cli/src/service/mod.rs b/components/chainhook-cli/src/service/mod.rs index 5c524b568..1d1df8a08 100644 --- a/components/chainhook-cli/src/service/mod.rs +++ b/components/chainhook-cli/src/service/mod.rs @@ -6,7 +6,8 @@ use crate::scan::stacks::consolidate_local_stacks_chainstate_using_csv; use crate::service::http_api::{load_predicates_from_redis, start_predicate_api_server}; use crate::service::runloops::{start_bitcoin_scan_runloop, start_stacks_scan_runloop}; use crate::storage::{ - confirm_entries_in_stacks_blocks, draft_entries_in_stacks_blocks, open_readwrite_stacks_db_conn, + confirm_entries_in_stacks_blocks, draft_entries_in_stacks_blocks, get_all_unconfirmed_blocks, + open_readonly_stacks_db_conn_with_retry, open_readwrite_stacks_db_conn, }; use chainhook_sdk::chainhooks::types::{ChainhookConfig, ChainhookFullSpecification}; @@ -20,7 +21,7 @@ use chainhook_sdk::types::{Chain, StacksChainEvent}; use chainhook_sdk::utils::Context; use redis::{Commands, Connection}; -use std::sync::mpsc::channel; +use std::sync::mpsc::{channel, Receiver, Sender}; use std::time::{SystemTime, UNIX_EPOCH}; use self::http_api::get_entry_from_predicates_db; @@ -38,6 +39,7 @@ impl Service { pub async fn run( &mut self, predicates_from_startup: Vec, + observer_commands_tx_rx: Option<(Sender, Receiver)>, ) -> Result<(), String> { let mut chainhook_config = ChainhookConfig::new(); @@ -149,7 +151,8 @@ impl Service { } } - let (observer_command_tx, observer_command_rx) = channel(); + let (observer_command_tx, observer_command_rx) = + observer_commands_tx_rx.unwrap_or(channel()); let (observer_event_tx, observer_event_rx) = crossbeam_channel::unbounded(); // let (ordinal_indexer_command_tx, ordinal_indexer_command_rx) = channel(); @@ -211,6 +214,20 @@ impl Service { }); } + let ctx = self.ctx.clone(); + let stacks_db = + open_readonly_stacks_db_conn_with_retry(&config.expected_cache_path(), 3, &ctx)?; + let unconfirmed_blocks = match get_all_unconfirmed_blocks(&stacks_db, &ctx) { + Ok(blocks) => Some(blocks), + Err(e) => { + info!( + self.ctx.expect_logger(), + "Failed to get stacks blocks from db to seed block pool: {}", e + ); + None + } + }; + let observer_event_tx_moved = observer_event_tx.clone(); let moved_observer_command_tx = observer_command_tx.clone(); let _ = start_event_observer( @@ -219,6 +236,7 @@ impl Service { observer_command_rx, Some(observer_event_tx_moved), None, + unconfirmed_blocks, self.ctx.clone(), ); diff --git a/components/chainhook-cli/src/service/tests/helpers/mock_service.rs b/components/chainhook-cli/src/service/tests/helpers/mock_service.rs index 890f2ec63..55ae3e859 100644 --- a/components/chainhook-cli/src/service/tests/helpers/mock_service.rs +++ b/components/chainhook-cli/src/service/tests/helpers/mock_service.rs @@ -1,24 +1,21 @@ -use crate::config::Config; -use crate::config::EventSourceConfig; -use crate::config::LimitsConfig; -use crate::config::MonitoringConfig; -use crate::config::PathConfig; -use crate::config::PredicatesApi; -use crate::config::PredicatesApiConfig; -use crate::config::StorageConfig; -use crate::config::DEFAULT_REDIS_URI; -use crate::service::http_api::start_predicate_api_server; -use crate::service::PredicateStatus; -use crate::service::Service; -use chainhook_sdk::chainhooks::types::ChainhookFullSpecification; -use chainhook_sdk::indexer::IndexerConfig; -use chainhook_sdk::observer::ObserverCommand; -use chainhook_sdk::types::BitcoinBlockSignaling; -use chainhook_sdk::types::BitcoinNetwork; -use chainhook_sdk::types::Chain; -use chainhook_sdk::types::StacksNetwork; -use chainhook_sdk::types::StacksNodeConfig; -use chainhook_sdk::utils::Context; +use crate::config::{ + Config, EventSourceConfig, LimitsConfig, MonitoringConfig, PathConfig, PredicatesApi, + PredicatesApiConfig, StorageConfig, DEFAULT_REDIS_URI, +}; +use crate::scan::stacks::consolidate_local_stacks_chainstate_using_csv; +use crate::service::{ + http_api::start_predicate_api_server, update_predicate_spec, update_predicate_status, + PredicateStatus, Service, +}; +use chainhook_sdk::{ + chainhooks::types::{ + ChainhookFullSpecification, ChainhookSpecification, StacksChainhookFullSpecification, + }, + indexer::IndexerConfig, + observer::ObserverCommand, + types::{BitcoinBlockSignaling, BitcoinNetwork, Chain, StacksNetwork, StacksNodeConfig}, + utils::Context, +}; use redis::Commands; use reqwest::Method; use rocket::serde::json::Value as JsonValue; @@ -26,8 +23,15 @@ use rocket::Shutdown; use std::path::PathBuf; use std::process::Stdio; use std::process::{Child, Command}; +use std::sync::mpsc; use std::sync::mpsc::channel; use std::sync::mpsc::Receiver; +use std::sync::mpsc::Sender; + +use super::get_free_port; +use super::mock_bitcoin_rpc::mock_bitcoin_rpc; +use super::mock_stacks_node::create_tmp_working_dir; +use super::mock_stacks_node::write_stacks_blocks_to_tsv; pub async fn get_predicate_status(uuid: &str, port: u16) -> Result { let mut attempts = 0; @@ -328,14 +332,19 @@ pub fn get_chainhook_config( pub async fn start_chainhook_service( config: Config, - chainhook_port: u16, + ping_startup_port: u16, startup_predicates: Option>, ctx: &Context, -) -> Result<(), String> { +) -> Result, String> { let mut service = Service::new(config, ctx.clone()); + let (observer_command_tx, observer_command_rx) = mpsc::channel(); + let moved_observer_command_tx = observer_command_tx.clone(); let _ = hiro_system_kit::thread_named("Chainhook service") .spawn(move || { - let future = service.run(startup_predicates.unwrap_or(vec![])); + let future = service.run( + startup_predicates.unwrap_or(vec![]), + Some((moved_observer_command_tx, observer_command_rx)), + ); let _ = hiro_system_kit::nestable_block_on(future); }) .map_err(|e| { @@ -354,14 +363,216 @@ pub async fn start_chainhook_service( } if let Ok(_client) = reqwest::Client::new() - .get(format!("http://localhost:{}/ping", chainhook_port)) + .get(format!("http://localhost:{}/ping", ping_startup_port)) .send() .await { - break Ok(()); // Server is ready + break Ok(observer_command_tx); // Server is ready } tokio::time::sleep(std::time::Duration::from_secs(1)).await; attempts += 1; } } + +pub struct TestSetupResult { + pub redis_process: Child, + pub working_dir: String, + pub chainhook_service_port: u16, + pub redis_port: u16, + pub stacks_ingestion_port: u16, + pub stacks_rpc_port: u16, + pub bitcoin_rpc_port: u16, + pub prometheus_port: u16, + pub observer_command_tx: Sender, +} + +pub async fn setup_stacks_chainhook_test( + starting_chain_tip: u64, + redis_seed: Option<(StacksChainhookFullSpecification, PredicateStatus)>, + startup_predicates: Option>, +) -> TestSetupResult { + let ( + redis_port, + chainhook_service_port, + stacks_rpc_port, + stacks_ingestion_port, + bitcoin_rpc_port, + prometheus_port, + ) = setup_chainhook_service_ports().unwrap_or_else(|e| panic!("test failed with error: {e}")); + + let mut redis_process = start_redis(redis_port) + .await + .unwrap_or_else(|e| panic!("test failed with error: {e}")); + flush_redis(redis_port); + + let logger = hiro_system_kit::log::setup_logger(); + let _guard = hiro_system_kit::log::setup_global_logger(logger.clone()); + let ctx = Context { + logger: Some(logger), + tracer: false, + }; + + if let Some((predicate, status)) = redis_seed { + let client = redis::Client::open(format!("redis://localhost:{redis_port}/")) + .unwrap_or_else(|e| { + flush_redis(redis_port); + redis_process.kill().unwrap(); + panic!("test failed with error: {e}"); + }); + let mut connection = client.get_connection().unwrap_or_else(|e| { + flush_redis(redis_port); + redis_process.kill().unwrap(); + panic!("test failed with error: {e}"); + }); + let stacks_spec = predicate + .into_selected_network_specification(&StacksNetwork::Devnet) + .unwrap_or_else(|e| { + flush_redis(redis_port); + redis_process.kill().unwrap(); + panic!("test failed with error: {e}"); + }); + + let spec = ChainhookSpecification::Stacks(stacks_spec); + update_predicate_spec(&spec.key(), &spec, &mut connection, &ctx); + update_predicate_status(&spec.key(), status, &mut connection, &ctx); + } + + let (working_dir, tsv_dir) = create_tmp_working_dir().unwrap_or_else(|e| { + flush_redis(redis_port); + redis_process.kill().unwrap(); + panic!("test failed with error: {e}"); + }); + + write_stacks_blocks_to_tsv(starting_chain_tip, &tsv_dir).unwrap_or_else(|e| { + std::fs::remove_dir_all(&working_dir).unwrap(); + flush_redis(redis_port); + redis_process.kill().unwrap(); + panic!("test failed with error: {e}"); + }); + + let mut config = get_chainhook_config( + redis_port, + chainhook_service_port, + stacks_rpc_port, + stacks_ingestion_port, + bitcoin_rpc_port, + &working_dir, + &tsv_dir, + Some(prometheus_port), + ); + + consolidate_local_stacks_chainstate_using_csv(&mut config, &ctx) + .await + .unwrap_or_else(|e| { + std::fs::remove_dir_all(&working_dir).unwrap(); + flush_redis(redis_port); + redis_process.kill().unwrap(); + panic!("test failed with error: {e}"); + }); + + let observer_command_tx = + start_chainhook_service(config, chainhook_service_port, startup_predicates, &ctx) + .await + .unwrap_or_else(|e| { + std::fs::remove_dir_all(&working_dir).unwrap(); + flush_redis(redis_port); + redis_process.kill().unwrap(); + panic!("test failed with error: {e}"); + }); + TestSetupResult { + redis_process, + working_dir, + chainhook_service_port, + redis_port, + stacks_ingestion_port, + stacks_rpc_port, + bitcoin_rpc_port, + prometheus_port, + observer_command_tx, + } +} + +pub async fn setup_bitcoin_chainhook_test(starting_chain_tip: u64) -> TestSetupResult { + let ( + redis_port, + chainhook_service_port, + stacks_rpc_port, + stacks_ingestion_port, + bitcoin_rpc_port, + prometheus_port, + ) = setup_chainhook_service_ports().unwrap_or_else(|e| panic!("test failed with error: {e}")); + + let mut redis_process = start_redis(redis_port) + .await + .unwrap_or_else(|e| panic!("test failed with error: {e}")); + + flush_redis(redis_port); + let (working_dir, tsv_dir) = create_tmp_working_dir().unwrap_or_else(|e| { + flush_redis(redis_port); + redis_process.kill().unwrap(); + panic!("test failed with error: {e}"); + }); + + let logger = hiro_system_kit::log::setup_logger(); + let _guard = hiro_system_kit::log::setup_global_logger(logger.clone()); + let ctx = Context { + logger: Some(logger), + tracer: false, + }; + + let _ = hiro_system_kit::thread_named("Bitcoin rpc service") + .spawn(move || { + let future = mock_bitcoin_rpc(bitcoin_rpc_port, starting_chain_tip); + let _ = hiro_system_kit::nestable_block_on(future); + }) + .expect("unable to spawn thread"); + + let config = get_chainhook_config( + redis_port, + chainhook_service_port, + stacks_rpc_port, + stacks_ingestion_port, + bitcoin_rpc_port, + &working_dir, + &tsv_dir, + Some(prometheus_port), + ); + + let terminator_tx = start_chainhook_service(config, chainhook_service_port, None, &ctx) + .await + .unwrap_or_else(|e| { + std::fs::remove_dir_all(&working_dir).unwrap(); + flush_redis(redis_port); + redis_process.kill().unwrap(); + panic!("test failed with error: {e}"); + }); + TestSetupResult { + redis_process, + working_dir, + chainhook_service_port, + redis_port, + stacks_ingestion_port, + stacks_rpc_port, + bitcoin_rpc_port, + prometheus_port, + observer_command_tx: terminator_tx, + } +} + +pub fn setup_chainhook_service_ports() -> Result<(u16, u16, u16, u16, u16, u16), String> { + let redis_port = get_free_port()?; + let chainhook_service_port = get_free_port()?; + let stacks_rpc_port = get_free_port()?; + let stacks_ingestion_port = get_free_port()?; + let bitcoin_rpc_port = get_free_port()?; + let prometheus_port = get_free_port()?; + Ok(( + redis_port, + chainhook_service_port, + stacks_rpc_port, + stacks_ingestion_port, + bitcoin_rpc_port, + prometheus_port, + )) +} diff --git a/components/chainhook-cli/src/service/tests/helpers/mock_stacks_node.rs b/components/chainhook-cli/src/service/tests/helpers/mock_stacks_node.rs index ef7c32a88..6d03534ef 100644 --- a/components/chainhook-cli/src/service/tests/helpers/mock_stacks_node.rs +++ b/components/chainhook-cli/src/service/tests/helpers/mock_stacks_node.rs @@ -8,7 +8,7 @@ use chainhook_sdk::types::{ STXTransferEventData, SmartContractEventData, StacksTransactionEventPayload, }; -use super::{branch_and_height_to_prefixed_hash, height_to_prefixed_hash}; +use super::{branch_and_height_to_prefixed_hash, make_block_hash}; pub const TEST_WORKING_DIR: &str = "src/service/tests/fixtures/tmp"; @@ -128,12 +128,19 @@ fn create_stacks_new_transaction(index: u64) -> NewTransaction { } } -pub fn create_stacks_new_block(height: u64, burn_block_height: u64) -> NewBlock { - let parent_height = if height == 0 { 0 } else { height - 1 }; - let parent_burn_block_height = if burn_block_height == 0 { - 0 - } else { - burn_block_height - 1 +pub fn create_stacks_new_block( + fork_id: u8, + height: u64, + parent_fork_id: u8, + burn_block_height: u64, +) -> NewBlock { + let parent_height = match height { + 0 => 0, + _ => height - 1, + }; + let parent_burn_block_height = match burn_block_height { + 0 => 0, + _ => burn_block_height - 1, }; let mut events = vec![]; @@ -238,16 +245,16 @@ pub fn create_stacks_new_block(height: u64, burn_block_height: u64) -> NewBlock )); NewBlock { block_height: height, - block_hash: height_to_prefixed_hash(height), - index_block_hash: height_to_prefixed_hash(height), + block_hash: make_block_hash(fork_id, height), + index_block_hash: make_block_hash(fork_id, height), burn_block_height: burn_block_height, - burn_block_hash: height_to_prefixed_hash(burn_block_height), - parent_block_hash: height_to_prefixed_hash(parent_height), - parent_index_block_hash: height_to_prefixed_hash(parent_height), + burn_block_hash: make_block_hash(0, burn_block_height), + parent_block_hash: make_block_hash(parent_fork_id, parent_height), + parent_index_block_hash: make_block_hash(parent_fork_id, parent_height), parent_microblock: "0x0000000000000000000000000000000000000000000000000000000000000000" .into(), parent_microblock_sequence: 0, - parent_burn_block_hash: height_to_prefixed_hash(parent_burn_block_height), + parent_burn_block_hash: make_block_hash(0, parent_burn_block_height), parent_burn_block_height: burn_block_height, parent_burn_block_timestamp: 0, transactions: (0..4).map(|i| create_stacks_new_transaction(i)).collect(), @@ -257,10 +264,12 @@ pub fn create_stacks_new_block(height: u64, burn_block_height: u64) -> NewBlock } fn create_stacks_block_received_record( + fork_id: u8, height: u64, + parent_fork_id: u8, burn_block_height: u64, ) -> Result { - let block = create_stacks_new_block(height, burn_block_height); + let block = create_stacks_new_block(fork_id, height, parent_fork_id, burn_block_height); let serialized_block = serde_json::to_string(&block) .map_err(|e| format!("failed to serialize stacks block: {}", e.to_string()))?; Ok(Record { @@ -281,7 +290,7 @@ pub fn write_stacks_blocks_to_tsv(block_count: u64, dir: &str) -> Result<(), Str .expect("unable to create csv writer"); for i in 1..block_count + 1 { writer - .serialize(create_stacks_block_received_record(i, i + 100)?) + .serialize(create_stacks_block_received_record(0, i, 0, i + 100)?) .map_err(|e| format!("failed to write tsv file: {}", e.to_string()))?; } Ok(()) @@ -289,10 +298,12 @@ pub fn write_stacks_blocks_to_tsv(block_count: u64, dir: &str) -> Result<(), Str pub async fn mine_stacks_block( port: u16, + fork_id: u8, height: u64, + parent_fork_id: u8, burn_block_height: u64, ) -> Result<(), String> { - let block = create_stacks_new_block(height, burn_block_height); + let block = create_stacks_new_block(fork_id, height, parent_fork_id, burn_block_height); let serialized_block = serde_json::to_string(&block).unwrap(); let client = reqwest::Client::new(); let _res = client diff --git a/components/chainhook-cli/src/service/tests/helpers/mod.rs b/components/chainhook-cli/src/service/tests/helpers/mod.rs index 76e186edc..0dd9b9c71 100644 --- a/components/chainhook-cli/src/service/tests/helpers/mod.rs +++ b/components/chainhook-cli/src/service/tests/helpers/mod.rs @@ -5,11 +5,13 @@ pub mod mock_bitcoin_rpc; pub mod mock_service; pub mod mock_stacks_node; -pub fn height_to_prefixed_hash(height: u64) -> String { - format!("0x{}", height_to_hash_str(height)) -} -fn height_to_hash_str(height: u64) -> String { - format!("{:0>64}", height.to_string()) +pub fn make_block_hash(fork_id: u8, block_height: u64) -> String { + #![cfg_attr(rustfmt, rustfmt_skip)] + let mut hash = vec![ + fork_id, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + hash.append(&mut block_height.to_be_bytes().to_vec()); + hex::encode(&hash[..]) } pub fn branch_and_height_to_prefixed_hash(branch: Option, height: u64) -> String { diff --git a/components/chainhook-cli/src/service/tests/mod.rs b/components/chainhook-cli/src/service/tests/mod.rs index 5a9853774..8d44c4f88 100644 --- a/components/chainhook-cli/src/service/tests/mod.rs +++ b/components/chainhook-cli/src/service/tests/mod.rs @@ -1,7 +1,5 @@ -use chainhook_sdk::chainhooks::types::{ - ChainhookFullSpecification, ChainhookSpecification, StacksChainhookFullSpecification, -}; -use chainhook_sdk::types::{Chain, StacksNetwork}; +use chainhook_sdk::chainhooks::types::ChainhookFullSpecification; +use chainhook_sdk::types::Chain; use chainhook_sdk::utils::Context; use rocket::serde::json::Value as JsonValue; use rocket::Shutdown; @@ -16,26 +14,23 @@ use test_case::test_case; use chainhook_sdk::observer::ObserverCommand; use self::helpers::build_predicates::{build_bitcoin_payload, build_stacks_payload, DEFAULT_UUID}; -use self::helpers::mock_bitcoin_rpc::mock_bitcoin_rpc; use self::helpers::mock_service::{ call_deregister_predicate, filter_predicate_status_from_all_predicates, flush_redis, - start_chainhook_service, start_redis, -}; -use self::helpers::mock_stacks_node::{ - create_tmp_working_dir, mine_burn_block, mine_stacks_block, write_stacks_blocks_to_tsv, + start_chainhook_service, }; -use crate::scan::stacks::consolidate_local_stacks_chainstate_using_csv; +use self::helpers::mock_stacks_node::{mine_burn_block, mine_stacks_block}; +use crate::config::PredicatesApi; use crate::service::tests::helpers::build_predicates::get_random_uuid; -use crate::service::tests::helpers::get_free_port; use crate::service::tests::helpers::mock_service::{ - build_predicate_api_server, call_get_predicate, call_register_predicate, get_chainhook_config, - get_predicate_status, + build_predicate_api_server, call_get_predicate, call_ping, call_register_predicate, + get_chainhook_config, get_predicate_status, setup_bitcoin_chainhook_test, + setup_stacks_chainhook_test, TestSetupResult, }; use crate::service::tests::helpers::mock_stacks_node::create_burn_fork_at; use crate::service::{PredicateStatus, PredicateStatus::*, ScanningData, StreamingData}; +use crate::storage::{get_all_unconfirmed_blocks, open_readonly_stacks_db_conn}; use super::http_api::document_predicate_api_server; -use super::{update_predicate_spec, update_predicate_status}; pub mod helpers; mod observer_tests; @@ -346,23 +341,6 @@ fn _assert_interrupted_status((status, _, _): (PredicateStatus, Option, Opt } } -fn setup_chainhook_service_ports() -> Result<(u16, u16, u16, u16, u16, u16), String> { - let redis_port = get_free_port()?; - let chainhook_service_port = get_free_port()?; - let stacks_rpc_port = get_free_port()?; - let stacks_ingestion_port = get_free_port()?; - let bitcoin_rpc_port = get_free_port()?; - let prometheus_port = get_free_port()?; - Ok(( - redis_port, - chainhook_service_port, - stacks_rpc_port, - stacks_ingestion_port, - bitcoin_rpc_port, - prometheus_port, - )) -} - async fn await_new_scanning_status_complete( uuid: &str, chainhook_service_port: u16, @@ -381,110 +359,6 @@ async fn await_new_scanning_status_complete( } } } - -async fn setup_stacks_chainhook_test( - starting_chain_tip: u64, - redis_seed: Option<(StacksChainhookFullSpecification, PredicateStatus)>, - startup_predicates: Option>, -) -> (Child, String, u16, u16, u16, u16, u16) { - let ( - redis_port, - chainhook_service_port, - stacks_rpc_port, - stacks_ingestion_port, - bitcoin_rpc_port, - prometheus_port, - ) = setup_chainhook_service_ports().unwrap_or_else(|e| panic!("test failed with error: {e}")); - - let mut redis_process = start_redis(redis_port) - .await - .unwrap_or_else(|e| panic!("test failed with error: {e}")); - flush_redis(redis_port); - - let logger = hiro_system_kit::log::setup_logger(); - let _guard = hiro_system_kit::log::setup_global_logger(logger.clone()); - let ctx = Context { - logger: Some(logger), - tracer: false, - }; - - if let Some((predicate, status)) = redis_seed { - let client = redis::Client::open(format!("redis://localhost:{redis_port}/")) - .unwrap_or_else(|e| { - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); - let mut connection = client.get_connection().unwrap_or_else(|e| { - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); - let stacks_spec = predicate - .into_selected_network_specification(&StacksNetwork::Devnet) - .unwrap_or_else(|e| { - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); - - let spec = ChainhookSpecification::Stacks(stacks_spec); - update_predicate_spec(&spec.key(), &spec, &mut connection, &ctx); - update_predicate_status(&spec.key(), status, &mut connection, &ctx); - } - - let (working_dir, tsv_dir) = create_tmp_working_dir().unwrap_or_else(|e| { - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); - - write_stacks_blocks_to_tsv(starting_chain_tip, &tsv_dir).unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); - - let mut config = get_chainhook_config( - redis_port, - chainhook_service_port, - stacks_rpc_port, - stacks_ingestion_port, - bitcoin_rpc_port, - &working_dir, - &tsv_dir, - Some(prometheus_port), - ); - - consolidate_local_stacks_chainstate_using_csv(&mut config, &ctx) - .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); - - start_chainhook_service(config, chainhook_service_port, startup_predicates, &ctx) - .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); - ( - redis_process, - working_dir, - chainhook_service_port, - redis_port, - stacks_ingestion_port, - bitcoin_rpc_port, - prometheus_port, - ) -} - #[test_case(5, 0, Some(1), Some(3), Some(3), Some(3) => using assert_confirmed_expiration_status; "predicate_end_block lower than starting_chain_tip ends with ConfirmedExpiration status")] #[test_case(5, 0, Some(1), None, Some(5), Some(5) => using assert_streaming_status; "no predicate_end_block ends with Streaming status")] #[test_case(3, 0, Some(1), Some(5), Some(3), Some(3) => using assert_streaming_status; "predicate_end_block greater than chain_tip ends with Streaming status")] @@ -502,15 +376,17 @@ async fn test_stacks_predicate_status_is_updated( expected_evaluations: Option, expected_occurrences: Option, ) -> (PredicateStatus, Option, Option) { - let ( + let TestSetupResult { mut redis_process, working_dir, chainhook_service_port, redis_port, stacks_ingestion_port, - _, - _, - ) = setup_stacks_chainhook_test(starting_chain_tip, None, None).await; + stacks_rpc_port: _, + bitcoin_rpc_port: _, + prometheus_port: _, + observer_command_tx: _, + } = setup_stacks_chainhook_test(starting_chain_tip, None, None).await; let uuid = &get_random_uuid(); let predicate = build_stacks_payload( @@ -522,130 +398,43 @@ async fn test_stacks_predicate_status_is_updated( ); let _ = call_register_predicate(&predicate, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process)) + .unwrap(); await_new_scanning_status_complete(uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process)) + .unwrap(); for i in 1..blocks_to_mine + 1 { mine_stacks_block( stacks_ingestion_port, + 0, i + starting_chain_tip, + 0, i + starting_chain_tip + 100, ) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process)) + .unwrap(); } sleep(Duration::new(2, 0)); let result = get_predicate_status(uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process)) + .unwrap(); let found_predicate_status = filter_predicate_status_from_all_predicates(uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); - assert_eq!(found_predicate_status, result); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process)) + .unwrap(); - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); + cleanup(&working_dir, redis_port, &mut redis_process); + assert_eq!(found_predicate_status, result); (result, expected_evaluations, expected_occurrences) } -async fn setup_bitcoin_chainhook_test( - starting_chain_tip: u64, -) -> (Child, String, u16, u16, u16, u16, u16) { - let ( - redis_port, - chainhook_service_port, - stacks_rpc_port, - stacks_ingestion_port, - bitcoin_rpc_port, - prometheus_port, - ) = setup_chainhook_service_ports().unwrap_or_else(|e| panic!("test failed with error: {e}")); - - let mut redis_process = start_redis(redis_port) - .await - .unwrap_or_else(|e| panic!("test failed with error: {e}")); - - flush_redis(redis_port); - let (working_dir, tsv_dir) = create_tmp_working_dir().unwrap_or_else(|e| { - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); - - let logger = hiro_system_kit::log::setup_logger(); - let _guard = hiro_system_kit::log::setup_global_logger(logger.clone()); - let ctx = Context { - logger: Some(logger), - tracer: false, - }; - - let _ = hiro_system_kit::thread_named("Bitcoin rpc service") - .spawn(move || { - let future = mock_bitcoin_rpc(bitcoin_rpc_port, starting_chain_tip); - let _ = hiro_system_kit::nestable_block_on(future); - }) - .expect("unable to spawn thread"); - - let config = get_chainhook_config( - redis_port, - chainhook_service_port, - stacks_rpc_port, - stacks_ingestion_port, - bitcoin_rpc_port, - &working_dir, - &tsv_dir, - Some(prometheus_port), - ); - - start_chainhook_service(config, chainhook_service_port, None, &ctx) - .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); - ( - redis_process, - working_dir, - chainhook_service_port, - redis_port, - stacks_ingestion_port, - bitcoin_rpc_port, - prometheus_port, - ) -} - #[test_case(5, 1, Some(1), Some(3), Some(3), Some(3) => using assert_unconfirmed_expiration_status; "predicate_end_block lower than starting_chain_tip with predicate_end_block confirmations < CONFIRMED_SEGMENT_MINIMUM_LENGTH ends with UnconfirmedExpiration status")] #[test_case(10, 1, Some(1), Some(3), Some(3), Some(3) => using assert_confirmed_expiration_status; "predicate_end_block lower than starting_chain_tip with predicate_end_block confirmations >= CONFIRMED_SEGMENT_MINIMUM_LENGTH ends with ConfirmedExpiration status")] #[test_case(1, 3, Some(1), Some(3), Some(4), Some(3) => using assert_unconfirmed_expiration_status; "predicate_end_block greater than starting_chain_tip and mining blocks so that predicate_end_block confirmations < CONFIRMED_SEGMENT_MINIMUM_LENGTH ends with UnconfirmedExpiration status")] @@ -661,15 +450,17 @@ async fn test_bitcoin_predicate_status_is_updated( expected_evaluations: Option, expected_occurrences: Option, ) -> (PredicateStatus, Option, Option) { - let ( + let TestSetupResult { mut redis_process, working_dir, chainhook_service_port, redis_port, stacks_ingestion_port, + stacks_rpc_port: _, bitcoin_rpc_port, - _, - ) = setup_bitcoin_chainhook_test(starting_chain_tip).await; + prometheus_port: _, + observer_command_tx: _, + } = setup_bitcoin_chainhook_test(starting_chain_tip).await; let uuid = &get_random_uuid(); let predicate = build_bitcoin_payload( @@ -684,21 +475,13 @@ async fn test_bitcoin_predicate_status_is_updated( let _ = call_register_predicate(&predicate, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process)) + .unwrap(); await_new_scanning_status_complete(uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process)) + .unwrap(); for i in 1..blocks_to_mine + 1 { mine_burn_block( @@ -708,36 +491,22 @@ async fn test_bitcoin_predicate_status_is_updated( i + starting_chain_tip, ) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process)) + .unwrap(); } sleep(Duration::new(2, 0)); let result = get_predicate_status(uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process)) + .unwrap(); let found_predicate_status = filter_predicate_status_from_all_predicates(uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); - assert_eq!(found_predicate_status, result); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process)) + .unwrap(); - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); + cleanup(&working_dir, redis_port, &mut redis_process); + assert_eq!(found_predicate_status, result); (result, expected_evaluations, expected_occurrences) } @@ -758,17 +527,19 @@ async fn test_bitcoin_predicate_status_is_updated_with_reorg( fork_blocks_to_mine: u64, predicate_start_block: Option, predicate_end_block: Option, -) { +) -> Result<(), String> { let starting_chain_tip = 0; - let ( + let TestSetupResult { mut redis_process, working_dir, chainhook_service_port, redis_port, stacks_ingestion_port, + stacks_rpc_port: _, bitcoin_rpc_port, - _, - ) = setup_bitcoin_chainhook_test(starting_chain_tip).await; + prometheus_port: _, + observer_command_tx: _, + } = setup_bitcoin_chainhook_test(starting_chain_tip).await; let uuid = &get_random_uuid(); let predicate = build_bitcoin_payload( @@ -783,12 +554,7 @@ async fn test_bitcoin_predicate_status_is_updated_with_reorg( let _ = call_register_predicate(&predicate, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; let genesis_branch_key = '0'; let first_block_mined_height = starting_chain_tip + 1; @@ -801,23 +567,13 @@ async fn test_bitcoin_predicate_status_is_updated_with_reorg( block_height, ) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; } sleep(Duration::new(2, 0)); let status = get_predicate_status(uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; assert_streaming_status((status, None, None)); let branch_key = '1'; @@ -831,12 +587,7 @@ async fn test_bitcoin_predicate_status_is_updated_with_reorg( fork_point, ) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; let reorg_point = last_block_mined_height + 1; let first_fork_block_mined_height = first_fork_block_mined_height + 1; @@ -850,22 +601,12 @@ async fn test_bitcoin_predicate_status_is_updated_with_reorg( block_height, ) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; if block_height == reorg_point { sleep(Duration::new(2, 0)); let status = get_predicate_status(uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; assert_streaming_status((status, None, None)); } } @@ -873,27 +614,29 @@ async fn test_bitcoin_predicate_status_is_updated_with_reorg( sleep(Duration::new(2, 0)); let status = get_predicate_status(uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; + cleanup(&working_dir, redis_port, &mut redis_process); assert_confirmed_expiration_status((status, None, None)); - - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); + Ok(()) } #[test_case(Chain::Stacks; "for stacks chain")] #[test_case(Chain::Bitcoin; "for bitcoin chain")] #[tokio::test] #[cfg_attr(not(feature = "redis_tests"), ignore)] -async fn test_deregister_predicate(chain: Chain) { - let (mut redis_process, working_dir, chainhook_service_port, redis_port, _, _, _) = match &chain - { +async fn test_deregister_predicate(chain: Chain) -> Result<(), String> { + let TestSetupResult { + mut redis_process, + working_dir, + chainhook_service_port, + redis_port, + stacks_ingestion_port: _, + stacks_rpc_port: _, + bitcoin_rpc_port: _, + prometheus_port: _, + observer_command_tx: _, + } = match &chain { Chain::Stacks => setup_stacks_chainhook_test(3, None, None).await, Chain::Bitcoin => setup_bitcoin_chainhook_test(3).await, }; @@ -919,49 +662,27 @@ async fn test_deregister_predicate(chain: Chain) { let _ = call_register_predicate(&predicate, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; let result = call_get_predicate(uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; assert_eq!(result.get("status"), Some(&json!(200))); let result = call_deregister_predicate(&chain, uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; assert_eq!(result.get("status"), Some(&json!(200))); let mut attempts = 0; loop { let result = call_get_predicate(uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; if result.get("status") == Some(&json!(404)) { break; } else if attempts == 3 { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); + cleanup(&working_dir, redis_port, &mut redis_process); panic!("predicate was not successfully derigistered"); } else { attempts += 1; @@ -969,9 +690,8 @@ async fn test_deregister_predicate(chain: Chain) { } } - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); + cleanup(&working_dir, redis_port, &mut redis_process); + Ok(()) } #[test_case(New, 6 => using assert_confirmed_expiration_status; "preloaded predicate with new status should get scanned until completion")] @@ -1013,38 +733,37 @@ async fn test_restarting_with_saved_predicates( let predicate = serde_json::from_value(predicate).expect("failed to set up stacks chanhook spec for test"); - let (mut redis_process, working_dir, chainhook_service_port, redis_port, _, _, _) = - setup_stacks_chainhook_test(starting_chain_tip, Some((predicate, starting_status)), None) - .await; + let TestSetupResult { + mut redis_process, + working_dir, + chainhook_service_port, + redis_port, + stacks_ingestion_port: _, + stacks_rpc_port: _, + bitcoin_rpc_port: _, + prometheus_port: _, + observer_command_tx: _, + } = setup_stacks_chainhook_test(starting_chain_tip, Some((predicate, starting_status)), None) + .await; await_new_scanning_status_complete(uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process)) + .unwrap(); sleep(Duration::new(2, 0)); let result = get_predicate_status(uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process)) + .unwrap(); - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); + cleanup(&working_dir, redis_port, &mut redis_process); (result, None, None) } #[tokio::test] #[cfg_attr(not(feature = "redis_tests"), ignore)] -async fn it_allows_specifying_startup_predicate() { +async fn it_allows_specifying_startup_predicate() -> Result<(), String> { let uuid = &get_random_uuid(); let predicate = build_stacks_payload( Some("devnet"), @@ -1056,37 +775,35 @@ async fn it_allows_specifying_startup_predicate() { let predicate = serde_json::from_value(predicate).expect("failed to set up stacks chanhook spec for test"); let startup_predicate = ChainhookFullSpecification::Stacks(predicate); - let (mut redis_process, working_dir, chainhook_service_port, redis_port, _, _, _) = - setup_stacks_chainhook_test(3, None, Some(vec![startup_predicate])).await; + let TestSetupResult { + mut redis_process, + working_dir, + chainhook_service_port, + redis_port, + stacks_ingestion_port: _, + stacks_rpc_port: _, + bitcoin_rpc_port: _, + prometheus_port: _, + observer_command_tx: _, + } = setup_stacks_chainhook_test(3, None, Some(vec![startup_predicate])).await; await_new_scanning_status_complete(uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; sleep(Duration::new(2, 0)); let result = get_predicate_status(uuid, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); + cleanup(&working_dir, redis_port, &mut redis_process); assert_confirmed_expiration_status((result, None, None)); + Ok(()) } #[tokio::test] #[cfg_attr(not(feature = "redis_tests"), ignore)] -async fn register_predicate_responds_409_if_uuid_in_use() { +async fn register_predicate_responds_409_if_uuid_in_use() -> Result<(), String> { let uuid = &get_random_uuid(); let predicate = build_stacks_payload( Some("devnet"), @@ -1099,22 +816,25 @@ async fn register_predicate_responds_409_if_uuid_in_use() { .expect("failed to set up stacks chanhook spec for test"); let startup_predicate = ChainhookFullSpecification::Stacks(stacks_spec); - let (mut redis_process, working_dir, chainhook_service_port, redis_port, _, _, _) = - setup_stacks_chainhook_test(3, None, Some(vec![startup_predicate])).await; + let TestSetupResult { + mut redis_process, + working_dir, + chainhook_service_port, + redis_port, + stacks_ingestion_port: _, + stacks_rpc_port: _, + bitcoin_rpc_port: _, + prometheus_port: _, + observer_command_tx: _, + } = setup_stacks_chainhook_test(3, None, Some(vec![startup_predicate])).await; let result = call_register_predicate(&predicate, chainhook_service_port) .await - .unwrap_or_else(|e| { - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); - panic!("test failed with error: {e}"); - }); - assert_eq!(result.get("status"), Some(&json!(409))); + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; - std::fs::remove_dir_all(&working_dir).unwrap(); - flush_redis(redis_port); - redis_process.kill().unwrap(); + cleanup(&working_dir, redis_port, &mut redis_process); + assert_eq!(result.get("status"), Some(&json!(409))); + Ok(()) } #[test] @@ -1130,3 +850,161 @@ fn it_generates_open_api_spec() { "breaking change detected: open api spec has been updated" ) } + +#[tokio::test] +#[cfg_attr(not(feature = "redis_tests"), ignore)] +async fn it_seeds_block_pool_on_startup() -> Result<(), String> { + let starting_chain_tip = 3; + let TestSetupResult { + mut redis_process, + working_dir, + chainhook_service_port, + redis_port, + stacks_ingestion_port, + stacks_rpc_port, + bitcoin_rpc_port, + prometheus_port: _, + observer_command_tx, + } = setup_stacks_chainhook_test(starting_chain_tip, None, None).await; + + let blocks_to_mine = 4; + for i in 1..blocks_to_mine + 1 { + mine_stacks_block( + stacks_ingestion_port, + 0, + i + starting_chain_tip, + 0, + i + starting_chain_tip + 100, + ) + .await + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; + } + // we need these blocks to propagate through new stacks block events and save to the db, so give it some time + sleep(Duration::new(1, 0)); + + let logger = hiro_system_kit::log::setup_logger(); + let _guard = hiro_system_kit::log::setup_global_logger(logger.clone()); + let ctx = Context { + logger: Some(logger), + tracer: false, + }; + let db_path = { + let mut destination_path = PathBuf::new(); + destination_path.push(&working_dir); + destination_path + }; + let stacks_db = open_readonly_stacks_db_conn(&db_path, &ctx).expect("unable to read stacks_db"); + // validate that all blocks we just mined are saved as unconfirmed blocks in the database + let unconfirmed_blocks = get_all_unconfirmed_blocks(&stacks_db, &ctx) + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; + let mut unconfirmed_height = starting_chain_tip + 1; + assert_eq!( + blocks_to_mine, + unconfirmed_blocks.len() as u64, + "Number of blocks left unconfirmed in db is not what expected. Expected: {}, Actual: {}", + blocks_to_mine, + unconfirmed_blocks.len() + ); + for block in unconfirmed_blocks.iter() { + assert_eq!( + unconfirmed_height, block.block_identifier.index, + "Unexpected unconfirmed block height. Expected: {}, Actual: {}", + unconfirmed_height, block.block_identifier.index + ); + unconfirmed_height += 1; + } + // terminate chainhook service + let _ = observer_command_tx.send(ObserverCommand::Terminate); + sleep(Duration::new(1, 0)); + let tsv_dir = format!("./{working_dir}/stacks_blocks.tsv"); + let mut config = get_chainhook_config( + redis_port, + chainhook_service_port, + stacks_rpc_port, + stacks_ingestion_port, + bitcoin_rpc_port, + &working_dir, + &tsv_dir, + None, + ); + // the API is still running, so don't restart it + config.http_api = PredicatesApi::Off; + let _ = start_chainhook_service(config, stacks_ingestion_port, None, &ctx).await; + // validate that all of the unconfirmed blocks we just saved are still available after a restart + let unconfirmed_blocks = get_all_unconfirmed_blocks(&stacks_db, &ctx).unwrap(); + let mut unconfirmed_height = starting_chain_tip + 1; + assert_eq!( + blocks_to_mine, + unconfirmed_blocks.len() as u64, + "Number of blocks left unconfirmed in db is not what expected. Expected: {}, Actual: {}", + blocks_to_mine, + unconfirmed_blocks.len() + ); + for block in unconfirmed_blocks.iter() { + assert_eq!( + unconfirmed_height, block.block_identifier.index, + "Unexpected unconfirmed block height. Expected: {}, Actual: {}", + unconfirmed_height, block.block_identifier.index + ); + unconfirmed_height += 1; + } + // mine a block on that same fork + let next_block_height = blocks_to_mine + starting_chain_tip + 1; + mine_stacks_block( + stacks_ingestion_port, + 0, + next_block_height, + 0, + next_block_height + 100, + ) + .await + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; + + // mine the same block number we just mined, but on a different fork + mine_stacks_block( + stacks_ingestion_port, + 1, + next_block_height, + 0, + next_block_height + 100, + ) + .await + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; + + sleep(Duration::new(1, 0)); + // confirm that there was a reorg + let metrics = call_ping(stacks_ingestion_port) + .await + .map_err(|e| cleanup_err(e, &working_dir, redis_port, &mut redis_process))?; + let stacks_last_reorg_data = metrics.get("stacks").unwrap().get("last_reorg").unwrap(); + let applied_blocks = stacks_last_reorg_data + .get("applied_blocks") + .unwrap() + .as_u64() + .unwrap(); + let rolled_back_blocks = stacks_last_reorg_data + .get("rolled_back_blocks") + .unwrap() + .as_u64() + .unwrap(); + cleanup(&working_dir, redis_port, &mut redis_process); + assert_eq!(applied_blocks, 1); + assert_eq!(rolled_back_blocks, 1); + Ok(()) +} + +fn cleanup_err( + error: String, + working_dir: &str, + redis_port: u16, + redis_process: &mut Child, +) -> String { + cleanup(working_dir, redis_port, redis_process); + format!("test failed with error: {error}") +} + +fn cleanup(working_dir: &str, redis_port: u16, redis_process: &mut Child) { + std::fs::remove_dir_all(&working_dir).unwrap(); + flush_redis(redis_port); + redis_process.kill().unwrap(); +} diff --git a/components/chainhook-cli/src/service/tests/observer_tests.rs b/components/chainhook-cli/src/service/tests/observer_tests.rs index f094febd1..ae00f3cc2 100644 --- a/components/chainhook-cli/src/service/tests/observer_tests.rs +++ b/components/chainhook-cli/src/service/tests/observer_tests.rs @@ -14,6 +14,7 @@ use crate::service::tests::{ build_predicates::build_stacks_payload, mock_service::{ call_observer_svc, call_ping, call_prometheus, call_register_predicate, flush_redis, + TestSetupResult, }, }, setup_bitcoin_chainhook_test, setup_stacks_chainhook_test, @@ -26,15 +27,17 @@ use super::helpers::{ #[tokio::test] #[cfg_attr(not(feature = "redis_tests"), ignore)] async fn ping_endpoint_returns_metrics() { - let ( + let TestSetupResult { mut redis_process, working_dir, chainhook_service_port, redis_port, stacks_ingestion_port, - _, - _, - ) = setup_stacks_chainhook_test(1, None, None).await; + stacks_rpc_port: _, + bitcoin_rpc_port: _, + prometheus_port: _, + observer_command_tx: _, + } = setup_stacks_chainhook_test(1, None, None).await; let uuid = &get_random_uuid(); let predicate = build_stacks_payload(Some("devnet"), None, None, None, Some(uuid)); @@ -68,8 +71,17 @@ async fn ping_endpoint_returns_metrics() { #[tokio::test] #[cfg_attr(not(feature = "redis_tests"), ignore)] async fn prometheus_endpoint_returns_encoded_metrics() { - let (mut redis_process, working_dir, chainhook_service_port, redis_port, _, _, prometheus_port) = - setup_stacks_chainhook_test(1, None, None).await; + let TestSetupResult { + mut redis_process, + working_dir, + chainhook_service_port, + redis_port, + stacks_ingestion_port: _, + stacks_rpc_port: _, + bitcoin_rpc_port: _, + prometheus_port, + observer_command_tx: _, + } = setup_stacks_chainhook_test(1, None, None).await; let uuid = &get_random_uuid(); let predicate = build_stacks_payload(Some("devnet"), None, None, None, Some(uuid)); @@ -128,8 +140,17 @@ async fn await_observer_started(port: u16) { #[tokio::test] #[cfg_attr(not(feature = "redis_tests"), ignore)] async fn bitcoin_rpc_requests_are_forwarded(endpoint: &str, body: Value) { - let (mut redis_process, working_dir, _, redis_port, stacks_ingestion_port, _, _) = - setup_bitcoin_chainhook_test(1).await; + let TestSetupResult { + mut redis_process, + working_dir, + chainhook_service_port: _, + redis_port, + stacks_ingestion_port, + stacks_rpc_port: _, + bitcoin_rpc_port: _, + prometheus_port: _, + observer_command_tx: _, + } = setup_bitcoin_chainhook_test(1).await; await_observer_started(stacks_ingestion_port).await; @@ -158,6 +179,7 @@ async fn start_and_ping_event_observer(config: EventObserverConfig, ingestion_po observer_commands_rx, None, None, + None, ctx, ) .unwrap(); diff --git a/components/chainhook-cli/src/storage/mod.rs b/components/chainhook-cli/src/storage/mod.rs index c403a83cb..dc6a945d3 100644 --- a/components/chainhook-cli/src/storage/mod.rs +++ b/components/chainhook-cli/src/storage/mod.rs @@ -2,7 +2,13 @@ use std::path::PathBuf; use chainhook_sdk::types::{BlockIdentifier, StacksBlockData, StacksBlockUpdate}; use chainhook_sdk::utils::Context; -use rocksdb::{Options, DB}; +use rocksdb::{Direction, IteratorMode, Options, DB}; + +const UNCONFIRMED_KEY_PREFIX: &[u8; 2] = b"~:"; +const CONFIRMED_KEY_PREFIX: &[u8; 2] = b"b:"; +const KEY_SUFFIX: &[u8; 2] = b":d"; +const LAST_UNCONFIRMED_KEY_PREFIX: &[u8; 3] = b"m:~"; +const LAST_CONFIRMED_KEY_PREFIX: &[u8; 3] = b"m:t"; fn get_db_default_options() -> Options { let mut opts = Options::default(); @@ -87,26 +93,26 @@ pub fn open_readwrite_stacks_db_conn(base_dir: &PathBuf, _ctx: &Context) -> Resu fn get_block_key(block_identifier: &BlockIdentifier) -> [u8; 12] { let mut key = [0u8; 12]; - key[..2].copy_from_slice(b"b:"); + key[..2].copy_from_slice(CONFIRMED_KEY_PREFIX); key[2..10].copy_from_slice(&block_identifier.index.to_be_bytes()); - key[10..].copy_from_slice(b":d"); + key[10..].copy_from_slice(KEY_SUFFIX); key } fn get_unconfirmed_block_key(block_identifier: &BlockIdentifier) -> [u8; 12] { let mut key = [0u8; 12]; - key[..2].copy_from_slice(b"~:"); + key[..2].copy_from_slice(UNCONFIRMED_KEY_PREFIX); key[2..10].copy_from_slice(&block_identifier.index.to_be_bytes()); - key[10..].copy_from_slice(b":d"); + key[10..].copy_from_slice(KEY_SUFFIX); key } fn get_last_confirmed_insert_key() -> [u8; 3] { - *b"m:t" + *LAST_CONFIRMED_KEY_PREFIX } fn get_last_unconfirmed_insert_key() -> [u8; 3] { - *b"m:~" + *LAST_UNCONFIRMED_KEY_PREFIX } pub fn insert_entry_in_stacks_blocks(block: &StacksBlockData, stacks_db_rw: &DB, _ctx: &Context) { @@ -164,6 +170,35 @@ pub fn get_last_unconfirmed_block_height_inserted(stacks_db: &DB, _ctx: &Context }) } +pub fn get_all_unconfirmed_blocks( + stacks_db: &DB, + _ctx: &Context, +) -> Result, String> { + let unconfirmed_key_prefix = UNCONFIRMED_KEY_PREFIX; + let mut blocks = vec![]; + let iter = stacks_db.iterator(IteratorMode::From( + unconfirmed_key_prefix, + Direction::Forward, + )); + for item in iter { + match item { + Ok((k, v)) => { + if k.starts_with(unconfirmed_key_prefix) { + let spec: StacksBlockData = serde_json::from_slice(&v[..]).map_err(|e| { + format!("unable to deserialize Stacks block {}", e.to_string()) + })?; + blocks.push(spec); + } else { + // we're past the set of keys we're looking for, so we've found all unconfirmed + return Ok(blocks); + } + } + Err(e) => return Err(format!("failed to get all unconfirmed blocks: {e}")), + }; + } + Ok(blocks) +} + pub fn get_last_block_height_inserted(stacks_db: &DB, _ctx: &Context) -> Option { stacks_db .get(get_last_confirmed_insert_key()) diff --git a/components/chainhook-sdk/src/indexer/mod.rs b/components/chainhook-sdk/src/indexer/mod.rs index fbf580b20..9b0c97e92 100644 --- a/components/chainhook-sdk/src/indexer/mod.rs +++ b/components/chainhook-sdk/src/indexer/mod.rs @@ -6,7 +6,7 @@ use crate::utils::{AbstractBlock, Context}; use chainhook_types::{ BitcoinBlockSignaling, BitcoinNetwork, BlockHeader, BlockIdentifier, BlockchainEvent, - StacksChainEvent, StacksNetwork, StacksNodeConfig, + StacksBlockData, StacksChainEvent, StacksNetwork, StacksNodeConfig, }; use hiro_system_kit::slog; use rocket::serde::json::Value as JsonValue; @@ -92,6 +92,10 @@ impl Indexer { } } + pub fn seed_stacks_block_pool(&mut self, blocks: Vec, ctx: &Context) { + self.stacks_blocks_pool.seed_block_pool(blocks, ctx); + } + pub fn handle_bitcoin_header( &mut self, header: BlockHeader, diff --git a/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs b/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs index 4e52c7bfb..1d4ee9001 100644 --- a/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs +++ b/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs @@ -43,6 +43,32 @@ impl StacksBlockPool { } } + pub fn seed_block_pool(&mut self, blocks: Vec, ctx: &Context) { + ctx.try_log(|logger| { + slog::info!(logger, "Seeding block pool with {} blocks", blocks.len()) + }); + for block in blocks { + let existing_entry = self.block_store.get(&block.block_identifier.clone()); + if existing_entry.is_some() { + ctx.try_log(|logger| { + slog::info!( + logger, + "Seeding block pool: Stacks {} has already been processed; skipping", + block.block_identifier + ) + }); + continue; + } + + match self.process_block(block, ctx) { + Ok(_) => {} + Err(e) => { + ctx.try_log(|logger| slog::info!(logger, "Error seeding block pool: {}", e)); + } + } + } + } + pub fn process_block( &mut self, block: StacksBlockData, diff --git a/components/chainhook-sdk/src/indexer/stacks/tests.rs b/components/chainhook-sdk/src/indexer/stacks/tests.rs index 94f7eb8d7..d4d78f227 100644 --- a/components/chainhook-sdk/src/indexer/stacks/tests.rs +++ b/components/chainhook-sdk/src/indexer/stacks/tests.rs @@ -15,262 +15,271 @@ use test_case::test_case; #[test] fn test_stacks_vector_001() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_001()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_001(), None)); } #[test] fn test_stacks_vector_002() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_002()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_002(), None)); } #[test] fn test_stacks_vector_003() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_003()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_003(), None)); } #[test] fn test_stacks_vector_004() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_004()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_004(), None)); } #[test] fn test_stacks_vector_005() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_005()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_005(), None)); } #[test] fn test_stacks_vector_006() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_006()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_006(), None)); } #[test] fn test_stacks_vector_007() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_007()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_007(), None)); } #[test] fn test_stacks_vector_008() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_008()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_008(), None)); } #[test] fn test_stacks_vector_009() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_009()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_009(), None)); } #[test] fn test_stacks_vector_010() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_010()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_010(), None)); } #[test] fn test_stacks_vector_011() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_011()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_011(), None)); } #[test] fn test_stacks_vector_012() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_012()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_012(), None)); } #[test] fn test_stacks_vector_013() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_013()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_013(), None)); } #[test] fn test_stacks_vector_014() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_014()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_014(), None)); } #[test] fn test_stacks_vector_015() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_015()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_015(), None)); } #[test] fn test_stacks_vector_016() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_016()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_016(), None)); } #[test] fn test_stacks_vector_017() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_017()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_017(), None)); } #[test] fn test_stacks_vector_018() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_018()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_018(), None)); } #[test] fn test_stacks_vector_019() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_019()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_019(), None)); } #[test] fn test_stacks_vector_020() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_020()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_020(), None)); } #[test] fn test_stacks_vector_021() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_021()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_021(), None)); } #[test] fn test_stacks_vector_022() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_022()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_022(), None)); } #[test] fn test_stacks_vector_023() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_023()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_023(), None)); } #[test] fn test_stacks_vector_024() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_024()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_024(), None)); } #[test] fn test_stacks_vector_025() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_025()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_025(), None)); } #[test] fn test_stacks_vector_026() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_026()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_026(), None)); } #[test] fn test_stacks_vector_027() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_027()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_027(), None)); } #[test] fn test_stacks_vector_028() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_028()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_028(), None)); } #[test] fn test_stacks_vector_029() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_029()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_029(), None)); } #[test] fn test_stacks_vector_030() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_030()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_030(), None)); } #[test] fn test_stacks_vector_031() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_031()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_031(), None)); } #[test] fn test_stacks_vector_032() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_032()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_032(), None)); } #[test] fn test_stacks_vector_033() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_033()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_033(), None)); } #[test] fn test_stacks_vector_034() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_034()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_034(), None)); } #[test] fn test_stacks_vector_035() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_035()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_035(), None)); } #[test] fn test_stacks_vector_036() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_036()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_036(), None)); } #[test] fn test_stacks_vector_037() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_037()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_037(), None)); } #[test] fn test_stacks_vector_038() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_038()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_038(), None)); } #[test] fn test_stacks_vector_039() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_039()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_039(), None)); } #[test] fn test_stacks_vector_040() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_040()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_040(), None)); } // #[test] // fn test_stacks_vector_041() { -// process_stacks_blocks_and_check_expectations(helpers::shapes::get_vector_041()); +// process_stacks_blocks_and_check_expectations((helpers::shapes::get_vector_041(), None)); // } #[test] fn test_stacks_vector_042() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_042()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_042(), None)); } #[test] fn test_stacks_vector_043() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_043()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_043(), None)); } #[test] fn test_stacks_vector_044() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_044()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_044(), None)); } #[test] fn test_stacks_vector_045() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_045()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_045(), None)); } #[test] fn test_stacks_vector_046() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_046()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_046(), None)); } #[test] fn test_stacks_vector_047() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_047()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_047(), None)); } #[test] fn test_stacks_vector_048() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_048()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_048(), None)); } #[test] fn test_stacks_vector_049() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_049()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_049(), None)); } #[test] fn test_stacks_vector_050() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_050()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_050(), None)); } #[test] fn test_stacks_vector_051() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_051()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_051(), None)); } #[test] fn test_stacks_vector_052() { - process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_052()); + process_stacks_blocks_and_check_expectations((helpers::stacks_shapes::get_vector_052(), None)); +} + +#[test] +fn test_stacks_vector_053() { + process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_053()); +} +#[test] +fn test_stacks_vector_054() { + process_stacks_blocks_and_check_expectations(helpers::stacks_shapes::get_vector_054()); } #[test_case(StacksTransactionEventPayload::STXTransferEvent(STXTransferEventData { diff --git a/components/chainhook-sdk/src/indexer/tests/helpers/stacks_shapes.rs b/components/chainhook-sdk/src/indexer/tests/helpers/stacks_shapes.rs index 316849128..1a7f60073 100644 --- a/components/chainhook-sdk/src/indexer/tests/helpers/stacks_shapes.rs +++ b/components/chainhook-sdk/src/indexer/tests/helpers/stacks_shapes.rs @@ -2,7 +2,7 @@ use crate::utils::Context; use super::{super::StacksChainEventExpectation, BlockEvent}; use super::{microblocks, stacks_blocks}; -use chainhook_types::StacksChainEvent; +use chainhook_types::{StacksBlockData, StacksChainEvent}; use hiro_system_kit::slog; pub fn expect_no_chain_update() -> StacksChainEventExpectation { @@ -3739,3 +3739,91 @@ pub fn get_vector_052() -> Vec<(BlockEvent, StacksChainEventExpectation)> { ), ] } + +/// Vector 053: Generate the following blocks +/// +/// A1(0) - B1(1) - C1(3) +/// \ B2(2) +/// +/// +pub fn get_vector_053() -> ( + Vec<(BlockEvent, StacksChainEventExpectation)>, + Option>, +) { + ( + vec![ + ( + stacks_blocks::B1(None), + expect_chain_updated_with_block(stacks_blocks::B1(None), vec![]), + ), + ( + stacks_blocks::B2(None), + expect_chain_updated_with_block_reorg( + vec![stacks_blocks::B1(None)], + vec![stacks_blocks::B2(None)], + vec![], + ), + ), + ( + stacks_blocks::C1(None), + expect_chain_updated_with_block_reorg( + vec![stacks_blocks::B2(None)], + vec![stacks_blocks::B1(None), stacks_blocks::C1(None)], + vec![], + ), + ), + ], + Some(vec![get_block_from_block_event(stacks_blocks::A1(None))]), + ) +} +/// Vector 054: Generate the following blocks +/// +/// A1(0) - B1(0) - C1(1) - D1(4) +/// \ B2(2) - C2(3) +/// +/// +pub fn get_vector_054() -> ( + Vec<(BlockEvent, StacksChainEventExpectation)>, + Option>, +) { + ( + vec![ + ( + stacks_blocks::C1(None), + expect_chain_updated_with_block(stacks_blocks::C1(None), vec![]), + ), + (stacks_blocks::B2(None), expect_no_chain_update()), + ( + stacks_blocks::C2(None), + expect_chain_updated_with_block_reorg( + vec![stacks_blocks::B1(None), stacks_blocks::C1(None)], + vec![stacks_blocks::B2(None), stacks_blocks::C2(None)], + vec![], + ), + ), + ( + stacks_blocks::D1(None), + expect_chain_updated_with_block_reorg( + vec![stacks_blocks::B2(None), stacks_blocks::C2(None)], + vec![ + stacks_blocks::B1(None), + stacks_blocks::C1(None), + stacks_blocks::D1(None), + ], + vec![], + ), + ), + ], + Some(vec![ + get_block_from_block_event(stacks_blocks::A1(None)), + get_block_from_block_event(stacks_blocks::B1(None)), + ]), + ) +} + +fn get_block_from_block_event(block_event: BlockEvent) -> StacksBlockData { + match block_event { + BlockEvent::Block(block) => block, + _ => unreachable!(), + } +} diff --git a/components/chainhook-sdk/src/indexer/tests/mod.rs b/components/chainhook-sdk/src/indexer/tests/mod.rs index 9781a4c14..4705aafc6 100644 --- a/components/chainhook-sdk/src/indexer/tests/mod.rs +++ b/components/chainhook-sdk/src/indexer/tests/mod.rs @@ -3,14 +3,22 @@ use crate::utils::{AbstractBlock, Context}; use self::helpers::BlockEvent; use super::{fork_scratch_pad::ForkScratchPad, StacksBlockPool}; -use chainhook_types::{BitcoinBlockData, BlockchainEvent, StacksChainEvent}; +use chainhook_types::{BitcoinBlockData, BlockchainEvent, StacksBlockData, StacksChainEvent}; pub type StacksChainEventExpectation = Box) -> ()>; pub fn process_stacks_blocks_and_check_expectations( - steps: Vec<(BlockEvent, StacksChainEventExpectation)>, + (steps, block_pool_seed): ( + Vec<(BlockEvent, StacksChainEventExpectation)>, + Option>, + ), ) { let mut blocks_processor = StacksBlockPool::new(); + + if let Some(block_pool_seed) = block_pool_seed { + blocks_processor.seed_block_pool(block_pool_seed, &Context::empty()); + } + for (block_event, check_chain_event_expectations) in steps.into_iter() { match block_event { BlockEvent::Block(block) => { diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs index b27e3b292..cc1108b68 100644 --- a/components/chainhook-sdk/src/observer/mod.rs +++ b/components/chainhook-sdk/src/observer/mod.rs @@ -27,7 +27,7 @@ use bitcoincore_rpc::{Auth, Client, RpcApi}; use chainhook_types::{ BitcoinBlockData, BitcoinBlockSignaling, BitcoinChainEvent, BitcoinChainUpdatedWithBlocksData, BitcoinChainUpdatedWithReorgData, BitcoinNetwork, BlockIdentifier, BlockchainEvent, - StacksChainEvent, StacksNetwork, StacksNodeConfig, TransactionIdentifier, + StacksBlockData, StacksChainEvent, StacksNetwork, StacksNodeConfig, TransactionIdentifier, }; use hiro_system_kit; use hiro_system_kit::slog; @@ -426,6 +426,7 @@ pub fn start_event_observer( observer_commands_rx: Receiver, observer_events_tx: Option>, observer_sidecar: Option, + stacks_block_pool_seed: Option>, ctx: Context, ) -> Result<(), Box> { match config.bitcoin_block_signaling { @@ -460,6 +461,7 @@ pub fn start_event_observer( observer_commands_rx, observer_events_tx, observer_sidecar, + stacks_block_pool_seed, context_cloned, ); let _ = hiro_system_kit::nestable_block_on(future); @@ -542,6 +544,7 @@ pub async fn start_stacks_event_observer( observer_commands_rx: Receiver, observer_events_tx: Option>, observer_sidecar: Option, + stacks_block_pool_seed: Option>, ctx: Context, ) -> Result<(), Box> { let indexer_config = IndexerConfig { @@ -553,7 +556,10 @@ pub async fn start_stacks_event_observer( bitcoin_block_signaling: config.bitcoin_block_signaling.clone(), }; - let indexer = Indexer::new(indexer_config.clone()); + let mut indexer = Indexer::new(indexer_config.clone()); + if let Some(stacks_block_pool_seed) = stacks_block_pool_seed { + indexer.seed_stacks_block_pool(stacks_block_pool_seed, &ctx); + } let log_level = if config.display_logs { if cfg!(feature = "cli") { From 9073f42285605ed7625039b3aae2316949dfc127 Mon Sep 17 00:00:00 2001 From: Micaiah Reid Date: Mon, 18 Mar 2024 11:14:31 -0400 Subject: [PATCH 04/12] fix: update scan status for non-triggering predicates (#511) ### Description Previously, we would only update the scanning status every 10 blocks _if_ the block we just scanned had a trigger. This leads to users not getting status updates on the predicates that don't trigger often. Now, we update status every 10 blocks that we scan. This could lead to some noisy logs, so it should possibly be shipped with #498 --- components/chainhook-cli/src/scan/bitcoin.rs | 28 ++++++++++---------- components/chainhook-cli/src/scan/stacks.rs | 28 ++++++++++---------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/components/chainhook-cli/src/scan/bitcoin.rs b/components/chainhook-cli/src/scan/bitcoin.rs index 84a92ced1..41ae6e365 100644 --- a/components/chainhook-cli/src/scan/bitcoin.rs +++ b/components/chainhook-cli/src/scan/bitcoin.rs @@ -98,6 +98,20 @@ pub async fn scan_bitcoin_chainstate_via_rpc_using_predicate( let http_client = build_http_client(); while let Some(current_block_height) = block_heights_to_scan.pop_front() { + if let Some(ref mut predicates_db_conn) = predicates_db_conn { + if number_of_blocks_scanned % 10 == 0 || number_of_blocks_scanned == 0 { + set_predicate_scanning_status( + &predicate_spec.key(), + number_of_blocks_to_scan, + number_of_blocks_scanned, + number_of_times_triggered, + current_block_height, + predicates_db_conn, + ctx, + ); + } + } + if current_block_height > chain_tip { let prev_chain_tip = chain_tip; // we've scanned up to the chain tip as of the start of this scan @@ -182,20 +196,6 @@ pub async fn scan_bitcoin_chainstate_via_rpc_using_predicate( return Err(format!("Scan aborted (consecutive action errors >= 3)")); } } - - if let Some(ref mut predicates_db_conn) = predicates_db_conn { - if number_of_blocks_scanned % 10 == 0 || number_of_blocks_scanned == 1 { - set_predicate_scanning_status( - &predicate_spec.key(), - number_of_blocks_to_scan, - number_of_blocks_scanned, - number_of_times_triggered, - current_block_height, - predicates_db_conn, - ctx, - ); - } - } } info!( diff --git a/components/chainhook-cli/src/scan/stacks.rs b/components/chainhook-cli/src/scan/stacks.rs index 583b89a32..eee8162a3 100644 --- a/components/chainhook-cli/src/scan/stacks.rs +++ b/components/chainhook-cli/src/scan/stacks.rs @@ -221,6 +221,19 @@ pub async fn scan_stacks_chainstate_via_rocksdb_using_predicate( }; while let Some(current_block_height) = block_heights_to_scan.pop_front() { + if let Some(ref mut predicates_db_conn) = predicates_db_conn { + if number_of_blocks_scanned % 10 == 0 || number_of_blocks_scanned == 0 { + set_predicate_scanning_status( + &predicate_spec.key(), + number_of_blocks_to_scan, + number_of_blocks_scanned, + number_of_times_triggered, + current_block_height, + predicates_db_conn, + ctx, + ); + } + } if current_block_height > chain_tip { let prev_chain_tip = chain_tip; // we've scanned up to the chain tip as of the start of this scan @@ -279,6 +292,7 @@ pub async fn scan_stacks_chainstate_via_rocksdb_using_predicate( let (hits_per_blocks, _predicates_expired) = evaluate_stacks_chainhook_on_blocks(blocks, &predicate_spec, ctx); + if hits_per_blocks.is_empty() { continue; } @@ -325,20 +339,6 @@ pub async fn scan_stacks_chainstate_via_rocksdb_using_predicate( return Err(format!("Scan aborted (consecutive action errors >= 3)")); } } - - if let Some(ref mut predicates_db_conn) = predicates_db_conn { - if number_of_blocks_scanned % 10 == 0 || number_of_blocks_scanned == 1 { - set_predicate_scanning_status( - &predicate_spec.key(), - number_of_blocks_to_scan, - number_of_blocks_scanned, - number_of_times_triggered, - current_block_height, - predicates_db_conn, - ctx, - ); - } - } } info!( ctx.expect_logger(), From 983a1658b52cb5b4a89ac46bb85f7355b346a1fb Mon Sep 17 00:00:00 2001 From: Micaiah Reid Date: Tue, 19 Mar 2024 10:57:24 -0400 Subject: [PATCH 05/12] fix: skip db consolidation if no new dataset was downloaded (#513) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description The database consolidation loop doesn't need to happen if there is no new data. @lgalabru I'm interested in your thoughts on this. Potentially someone could break their stacks.rocksdb, then Chainhook wouldn't automatically fix it unless they deleted their archive to force a redownload, or a new archive becomes available. I think the tradeoff is worth it because of the improvements to startup time if users are repeatedly running `chainhook predicates scan`, or if the service goes down and needs to restart. --------- Co-authored-by: Chris Guimarães --- components/chainhook-cli/src/scan/stacks.rs | 101 +++++++++++--------- 1 file changed, 54 insertions(+), 47 deletions(-) diff --git a/components/chainhook-cli/src/scan/stacks.rs b/components/chainhook-cli/src/scan/stacks.rs index eee8162a3..6d3f90e40 100644 --- a/components/chainhook-cli/src/scan/stacks.rs +++ b/components/chainhook-cli/src/scan/stacks.rs @@ -516,59 +516,66 @@ pub async fn consolidate_local_stacks_chainstate_using_csv( "Building local chainstate from Stacks archive file" ); - let _ = download_stacks_dataset_if_required(config, ctx).await; - - let stacks_db = open_readonly_stacks_db_conn_with_retry(&config.expected_cache_path(), 3, ctx)?; - let confirmed_tip = get_last_block_height_inserted(&stacks_db, &ctx); - let mut canonical_fork = get_canonical_fork_from_tsv(config, confirmed_tip, ctx).await?; - - let mut indexer = Indexer::new(config.network.clone()); - let mut blocks_inserted = 0; - let mut blocks_read = 0; - let blocks_to_insert = canonical_fork.len(); - let stacks_db_rw = open_readwrite_stacks_db_conn(&config.expected_cache_path(), ctx)?; - info!( - ctx.expect_logger(), - "Begining import of {} Stacks blocks into rocks db", blocks_to_insert - ); - for (block_identifier, _parent_block_identifier, blob) in canonical_fork.drain(..) { - blocks_read += 1; - - // If blocks already stored, move on - if is_stacks_block_present(&block_identifier, 3, &stacks_db_rw) { - continue; - } - blocks_inserted += 1; + let downloaded_new_dataset = download_stacks_dataset_if_required(config, ctx).await; + + if downloaded_new_dataset { + let stacks_db = + open_readonly_stacks_db_conn_with_retry(&config.expected_cache_path(), 3, ctx)?; + let confirmed_tip = get_last_block_height_inserted(&stacks_db, &ctx); + let mut canonical_fork = get_canonical_fork_from_tsv(config, confirmed_tip, ctx).await?; + + let mut indexer = Indexer::new(config.network.clone()); + let mut blocks_inserted = 0; + let mut blocks_read = 0; + let blocks_to_insert = canonical_fork.len(); + let stacks_db_rw = open_readwrite_stacks_db_conn(&config.expected_cache_path(), ctx)?; + info!( + ctx.expect_logger(), + "Beginning import of {} Stacks blocks into rocks db", blocks_to_insert + ); + for (block_identifier, _parent_block_identifier, blob) in canonical_fork.drain(..) { + blocks_read += 1; - let block_data = match indexer::stacks::standardize_stacks_serialized_block( - &indexer.config, - &blob, - &mut indexer.stacks_context, - ctx, - ) { - Ok(block) => block, - Err(e) => { - error!(&ctx.expect_logger(), "{e}"); + // If blocks already stored, move on + if is_stacks_block_present(&block_identifier, 3, &stacks_db_rw) { continue; } - }; + blocks_inserted += 1; - // TODO: return a result - insert_entry_in_stacks_blocks(&block_data, &stacks_db_rw, ctx); + let block_data = match indexer::stacks::standardize_stacks_serialized_block( + &indexer.config, + &blob, + &mut indexer.stacks_context, + ctx, + ) { + Ok(block) => block, + Err(e) => { + error!(&ctx.expect_logger(), "{e}"); + continue; + } + }; - if blocks_inserted % 2500 == 0 { - info!( - ctx.expect_logger(), - "Importing Stacks blocks into rocks db: {}/{}", blocks_read, blocks_to_insert - ); - let _ = stacks_db_rw.flush(); + // TODO: return a result + insert_entry_in_stacks_blocks(&block_data, &stacks_db_rw, ctx); + + if blocks_inserted % 2500 == 0 { + info!( + ctx.expect_logger(), + "Importing Stacks blocks into rocks db: {}/{}", blocks_read, blocks_to_insert + ); + let _ = stacks_db_rw.flush(); + } } + let _ = stacks_db_rw.flush(); + info!( + ctx.expect_logger(), + "{blocks_read} Stacks blocks read, {blocks_inserted} inserted" + ); + } else { + info!( + ctx.expect_logger(), + "Skipping database consolidation - no new archive found since last consolidation." + ); } - let _ = stacks_db_rw.flush(); - info!( - ctx.expect_logger(), - "{blocks_read} Stacks blocks read, {blocks_inserted} inserted" - ); - Ok(()) } From a4f16635dcd8cc6a7d4a3ce6608013007b78b0a5 Mon Sep 17 00:00:00 2001 From: Micaiah Reid Date: Tue, 19 Mar 2024 11:08:37 -0400 Subject: [PATCH 06/12] feat: use stacks.rocksdb for predicate scan (#514) ### Description Previously, when using the `chainhook predicates scan` command, Chainhook would download a very large archive file to use as a data source for the scan **every time the command was run**. The Chainhook service (`chainhook service start`), however, implements rocksdb to allow storing this dataset locally. This PR uses the rocksdb that a user may have created locally to improve performance of the `chainhook predicates scan` command. If a rocksdb doesn't exist or can't be created, Chainhook will fallback to the original method of downloading the archive. Combined with #513, this PR reduces a simple scan of a few blocks from ~20 minutes to .02 seconds. Fixes #485 --- components/chainhook-cli/src/cli/mod.rs | 38 +++++++++++++++++++------ 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/components/chainhook-cli/src/cli/mod.rs b/components/chainhook-cli/src/cli/mod.rs index 3fa1b1a88..ed51d0b1a 100644 --- a/components/chainhook-cli/src/cli/mod.rs +++ b/components/chainhook-cli/src/cli/mod.rs @@ -3,6 +3,7 @@ use crate::config::Config; use crate::scan::bitcoin::scan_bitcoin_chainstate_via_rpc_using_predicate; use crate::scan::stacks::{ consolidate_local_stacks_chainstate_using_csv, scan_stacks_chainstate_via_csv_using_predicate, + scan_stacks_chainstate_via_rocksdb_using_predicate, }; use crate::service::http_api::document_predicate_api_server; use crate::service::Service; @@ -485,14 +486,35 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> { )); } }; - // TODO: if a stacks.rocksdb is present, use it. - // TODO: update Stacks archive file if required. - scan_stacks_chainstate_via_csv_using_predicate( - &predicate_spec, - &mut config, - &ctx, - ) - .await?; + match open_readonly_stacks_db_conn(&config.expected_cache_path(), &ctx) { + Ok(db_conn) => { + let _ = consolidate_local_stacks_chainstate_using_csv( + &mut config, + &ctx, + ) + .await; + scan_stacks_chainstate_via_rocksdb_using_predicate( + &predicate_spec, + None, + &db_conn, + &config, + &ctx, + ) + .await?; + } + Err(e) => { + info!( + ctx.expect_logger(), + "Could not open db. This will greatly increase scan times. Error: {}", e + ); + scan_stacks_chainstate_via_csv_using_predicate( + &predicate_spec, + &mut config, + &ctx, + ) + .await?; + } + }; } } } From 0fc38cbce00a3a1cfde38e9d2b9d6eb984bdd8cd Mon Sep 17 00:00:00 2001 From: Micaiah Reid Date: Tue, 19 Mar 2024 11:14:09 -0400 Subject: [PATCH 07/12] fix: log errors on block download failure; implement max retries (#503) ### Description Before this PR, Chainhook would loop infinitely if there is a problem downloading a BTC block, and it wouldn't log much information as to what was causing the failure. This PR implements better logging, and breaks out of the loop at 10 failed attempts. --- .../chainhook-sdk/src/indexer/bitcoin/mod.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs index 44c999b86..997c2c0bd 100644 --- a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs +++ b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs @@ -153,18 +153,21 @@ pub async fn download_and_parse_block_with_retry( ctx: &Context, ) -> Result { let mut errors_count = 0; + let max_retries = 10; let block = loop { match download_and_parse_block(http_client, block_hash, bitcoin_config, ctx).await { Ok(result) => break result, - Err(_e) => { + Err(e) => { errors_count += 1; - if errors_count > 3 { + if errors_count > 3 && errors_count < max_retries { ctx.try_log(|logger| { slog::warn!( logger, - "unable to fetch and parse block #{block_hash}: will retry in a few seconds (attempt #{errors_count}).", + "unable to fetch and parse block #{block_hash}: will retry in a few seconds (attempt #{errors_count}). Error: {e}", ) }); + } else if errors_count == max_retries { + return Err(format!("unable to fetch and parse block #{block_hash} after {errors_count} attempts. Error: {e}")); } std::thread::sleep(std::time::Duration::from_secs(1)); } @@ -180,18 +183,21 @@ pub async fn retrieve_block_hash_with_retry( ctx: &Context, ) -> Result { let mut errors_count = 0; + let max_retries = 10; let block_hash = loop { match retrieve_block_hash(http_client, block_height, bitcoin_config, ctx).await { Ok(result) => break result, - Err(_e) => { + Err(e) => { errors_count += 1; - if errors_count > 3 { + if errors_count > 3 && errors_count < max_retries { ctx.try_log(|logger| { slog::warn!( logger, - "unable to retrieve block hash #{block_height}: will retry in a few seconds (attempt #{errors_count}).", + "unable to retrieve block hash #{block_height}: will retry in a few seconds (attempt #{errors_count}). Error: {e}", ) }); + } else if errors_count == max_retries { + return Err(format!("unable to retrieve block hash #{block_height} after {errors_count} attempts. Error: {e}")); } std::thread::sleep(std::time::Duration::from_secs(2)); } From a11bc1c0f9120f11fa0a27cbeb336fd1fa78d7b3 Mon Sep 17 00:00:00 2001 From: Micaiah Reid Date: Tue, 19 Mar 2024 16:40:35 -0400 Subject: [PATCH 08/12] fix: order and filter blocks used to seed forking block pool (#534) ### Description #### Breaking change? ### Example --- ### Checklist - [ ] All tests pass - [ ] Tests added in this PR (if applicable) --- components/chainhook-cli/src/service/mod.rs | 15 ++++++- components/chainhook-cli/src/storage/mod.rs | 39 ++++++++----------- .../src/indexer/stacks/blocks_pool.rs | 2 +- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/components/chainhook-cli/src/service/mod.rs b/components/chainhook-cli/src/service/mod.rs index 1d1df8a08..86286c356 100644 --- a/components/chainhook-cli/src/service/mod.rs +++ b/components/chainhook-cli/src/service/mod.rs @@ -7,7 +7,8 @@ use crate::service::http_api::{load_predicates_from_redis, start_predicate_api_s use crate::service::runloops::{start_bitcoin_scan_runloop, start_stacks_scan_runloop}; use crate::storage::{ confirm_entries_in_stacks_blocks, draft_entries_in_stacks_blocks, get_all_unconfirmed_blocks, - open_readonly_stacks_db_conn_with_retry, open_readwrite_stacks_db_conn, + get_last_block_height_inserted, open_readonly_stacks_db_conn_with_retry, + open_readwrite_stacks_db_conn, }; use chainhook_sdk::chainhooks::types::{ChainhookConfig, ChainhookFullSpecification}; @@ -218,7 +219,17 @@ impl Service { let stacks_db = open_readonly_stacks_db_conn_with_retry(&config.expected_cache_path(), 3, &ctx)?; let unconfirmed_blocks = match get_all_unconfirmed_blocks(&stacks_db, &ctx) { - Ok(blocks) => Some(blocks), + Ok(blocks) => { + let confirmed_tip = get_last_block_height_inserted(&stacks_db, &ctx).unwrap_or(0); + // any unconfirmed blocks that are earlier than confirmed blocks are invalid + Some( + blocks + .iter() + .filter(|&b| b.block_identifier.index > confirmed_tip) + .cloned() + .collect(), + ) + } Err(e) => { info!( self.ctx.expect_logger(), diff --git a/components/chainhook-cli/src/storage/mod.rs b/components/chainhook-cli/src/storage/mod.rs index dc6a945d3..2d18aa33b 100644 --- a/components/chainhook-cli/src/storage/mod.rs +++ b/components/chainhook-cli/src/storage/mod.rs @@ -1,8 +1,9 @@ +use std::collections::VecDeque; use std::path::PathBuf; use chainhook_sdk::types::{BlockIdentifier, StacksBlockData, StacksBlockUpdate}; use chainhook_sdk::utils::Context; -use rocksdb::{Direction, IteratorMode, Options, DB}; +use rocksdb::{Options, DB}; const UNCONFIRMED_KEY_PREFIX: &[u8; 2] = b"~:"; const CONFIRMED_KEY_PREFIX: &[u8; 2] = b"b:"; @@ -172,28 +173,22 @@ pub fn get_last_unconfirmed_block_height_inserted(stacks_db: &DB, _ctx: &Context pub fn get_all_unconfirmed_blocks( stacks_db: &DB, - _ctx: &Context, -) -> Result, String> { - let unconfirmed_key_prefix = UNCONFIRMED_KEY_PREFIX; - let mut blocks = vec![]; - let iter = stacks_db.iterator(IteratorMode::From( - unconfirmed_key_prefix, - Direction::Forward, - )); - for item in iter { - match item { - Ok((k, v)) => { - if k.starts_with(unconfirmed_key_prefix) { - let spec: StacksBlockData = serde_json::from_slice(&v[..]).map_err(|e| { - format!("unable to deserialize Stacks block {}", e.to_string()) - })?; - blocks.push(spec); - } else { - // we're past the set of keys we're looking for, so we've found all unconfirmed - return Ok(blocks); + ctx: &Context, +) -> Result, String> { + let mut blocks = VecDeque::new(); + let Some(mut cursor) = get_last_unconfirmed_block_height_inserted(stacks_db, ctx) else { + return Ok(blocks); + }; + loop { + match get_stacks_block_at_block_height(cursor, false, 3, stacks_db) { + Ok(block) => match block { + Some(block) => { + blocks.push_front(block.clone()); + cursor = block.parent_block_identifier.index; } - } - Err(e) => return Err(format!("failed to get all unconfirmed blocks: {e}")), + None => break, + }, + Err(e) => return Err(e), }; } Ok(blocks) diff --git a/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs b/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs index 1d4ee9001..1a90f65c7 100644 --- a/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs +++ b/components/chainhook-sdk/src/indexer/stacks/blocks_pool.rs @@ -47,7 +47,7 @@ impl StacksBlockPool { ctx.try_log(|logger| { slog::info!(logger, "Seeding block pool with {} blocks", blocks.len()) }); - for block in blocks { + for block in blocks.into_iter() { let existing_entry = self.block_store.get(&block.block_identifier.clone()); if existing_entry.is_some() { ctx.try_log(|logger| { From 86b5c7859c8395a470e1b7d3249901624dc3c682 Mon Sep 17 00:00:00 2001 From: Micaiah Reid Date: Wed, 27 Mar 2024 11:26:57 -0400 Subject: [PATCH 09/12] fix: improve error handling, and more! (#524) This PR introduces a few fixes in an effort to improve reliability and debugging problems when running Chainhook as a service: - Revisits log levels throughout the tool (fixes #498, fixes #521). The general approach for the logs were: - `crit` - fatal errors that will crash mission critical component of Chainhook. In these cases, Chainhook should automatically kill all main threads (not individual scanning threads, which is tracked by #404) to crash the service. - `erro` - something went wrong the could lead to a critical error, or that could impact all users - `warn` - something went wrong that could impact an end user (usually due to user error) - `info` - control flow logging and updates on the state of _all_ registered predicates - `debug` - updates on the state of _a_ predicate - Crash the service if a mission critical thread fails (see https://github.com/hirosystems/chainhook/issues/517#issuecomment-1992135101 for a list of these threads). Previously, if one of these threads failed, the remaining services would keep running. For example, if the event observer handler crashed, the event observer API would keep running. This means that the stacks node is successfully emitting blocks that Chainhook is acknowledging but not ingesting. This causes gaps in our database Fixes #517 - Removes an infinite loop with bitcoin ingestion, crashing the service instead: Fixes #506 - Fixes how we delete predicates from our db when one is deregistered. This should reduce the number of logs we have on startup. Fixes #510 - Warns on all reorgs. Fixes #519 --- components/chainhook-cli/src/archive/mod.rs | 44 ++-- .../chainhook-cli/src/archive/tests/mod.rs | 10 +- components/chainhook-cli/src/cli/mod.rs | 4 +- components/chainhook-cli/src/config/mod.rs | 22 +- .../chainhook-cli/src/config/tests/mod.rs | 28 ++- components/chainhook-cli/src/scan/bitcoin.rs | 13 +- components/chainhook-cli/src/scan/stacks.rs | 37 +-- .../chainhook-cli/src/service/http_api.rs | 15 +- components/chainhook-cli/src/service/mod.rs | 175 +++++++++---- .../chainhook-cli/src/service/runloops.rs | 10 +- components/chainhook-cli/src/storage/mod.rs | 36 ++- components/chainhook-sdk/Cargo.toml | 2 +- .../src/indexer/fork_scratch_pad.rs | 43 +++- components/chainhook-sdk/src/indexer/mod.rs | 2 +- .../chainhook-sdk/src/indexer/stacks/mod.rs | 6 +- components/chainhook-sdk/src/observer/http.rs | 6 +- components/chainhook-sdk/src/observer/mod.rs | 238 +++++++++++------- .../chainhook-sdk/src/observer/tests/mod.rs | 14 +- components/chainhook-sdk/src/utils/mod.rs | 2 +- 19 files changed, 461 insertions(+), 246 deletions(-) diff --git a/components/chainhook-cli/src/archive/mod.rs b/components/chainhook-cli/src/archive/mod.rs index 81eee7911..b41c62245 100644 --- a/components/chainhook-cli/src/archive/mod.rs +++ b/components/chainhook-cli/src/archive/mod.rs @@ -21,7 +21,7 @@ pub async fn download_tsv_file(config: &Config) -> Result<(), String> { println!("{}", e.to_string()); }); - let remote_sha_url = config.expected_remote_stacks_tsv_sha256(); + let remote_sha_url = config.expected_remote_stacks_tsv_sha256()?; let res = reqwest::get(&remote_sha_url) .await .or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))? @@ -34,7 +34,7 @@ pub async fn download_tsv_file(config: &Config) -> Result<(), String> { write_file_content_at_path(&local_sha_file_path, &res.to_vec())?; - let file_url = config.expected_remote_stacks_tsv_url(); + let file_url = config.expected_remote_stacks_tsv_url()?; let res = reqwest::get(&file_url) .await .or(Err(format!("Failed to GET from '{}'", &file_url)))?; @@ -55,14 +55,17 @@ pub async fn download_tsv_file(config: &Config) -> Result<(), String> { Ok(0) => break, Ok(n) => { if let Err(e) = file.write_all(&buffer[..n]) { - let err = - format!("unable to update compressed archive: {}", e.to_string()); - return Err(err); + return Err(format!( + "unable to update compressed archive: {}", + e.to_string() + )); } } Err(e) => { - let err = format!("unable to write compressed archive: {}", e.to_string()); - return Err(err); + return Err(format!( + "unable to write compressed archive: {}", + e.to_string() + )); } } } @@ -83,12 +86,11 @@ pub async fn download_tsv_file(config: &Config) -> Result<(), String> { .map_err(|e| format!("unable to download stacks archive: {}", e.to_string()))?; } drop(tx); - tokio::task::spawn_blocking(|| decoder_thread.join()) .await - .unwrap() - .unwrap() - .unwrap(); + .map_err(|e| format!("failed to spawn thread: {e}"))? + .map_err(|e| format!("decoder thread failed when downloading tsv: {:?}", e))? + .map_err(|e| format!("failed to download tsv: {}", e))?; } Ok(()) @@ -124,11 +126,14 @@ impl Read for ChannelRead { } } -pub async fn download_stacks_dataset_if_required(config: &mut Config, ctx: &Context) -> bool { +pub async fn download_stacks_dataset_if_required( + config: &mut Config, + ctx: &Context, +) -> Result { if config.is_initial_ingestion_required() { // Download default tsv. if config.rely_on_remote_stacks_tsv() && config.should_download_remote_stacks_tsv() { - let url = config.expected_remote_stacks_tsv_url(); + let url = config.expected_remote_stacks_tsv_url()?; let mut tsv_file_path = config.expected_cache_path(); tsv_file_path.push(default_tsv_file_path(&config.network.stacks_network)); let mut tsv_sha_file_path = config.expected_cache_path(); @@ -137,7 +142,7 @@ pub async fn download_stacks_dataset_if_required(config: &mut Config, ctx: &Cont // Download archive if not already present in cache // Load the local let local_sha_file = read_file_content_at_path(&tsv_sha_file_path); - let sha_url = config.expected_remote_stacks_tsv_sha256(); + let sha_url = config.expected_remote_stacks_tsv_sha256()?; let remote_sha_file = match reqwest::get(&sha_url).await { Ok(response) => response.bytes().await, @@ -164,28 +169,25 @@ pub async fn download_stacks_dataset_if_required(config: &mut Config, ctx: &Cont "Stacks archive file already up to date" ); config.add_local_stacks_tsv_source(&tsv_file_path); - return false; + return Ok(false); } info!(ctx.expect_logger(), "Downloading {}", url); match download_tsv_file(&config).await { Ok(_) => {} - Err(e) => { - error!(ctx.expect_logger(), "{}", e); - std::process::exit(1); - } + Err(e) => return Err(e), } info!(ctx.expect_logger(), "Successfully downloaded tsv file"); config.add_local_stacks_tsv_source(&tsv_file_path); } - true + Ok(true) } else { info!( ctx.expect_logger(), "Streaming blocks from stacks-node {}", config.network.get_stacks_node_config().rpc_url ); - false + Ok(false) } } diff --git a/components/chainhook-cli/src/archive/tests/mod.rs b/components/chainhook-cli/src/archive/tests/mod.rs index eb97bfff9..35250e30b 100644 --- a/components/chainhook-cli/src/archive/tests/mod.rs +++ b/components/chainhook-cli/src/archive/tests/mod.rs @@ -72,8 +72,14 @@ async fn it_downloads_stacks_dataset_if_required() { tracer: false, }; let mut config_clone = config.clone(); - assert!(download_stacks_dataset_if_required(&mut config, &ctx).await); - assert!(!download_stacks_dataset_if_required(&mut config_clone, &ctx).await); + assert!(download_stacks_dataset_if_required(&mut config, &ctx) + .await + .unwrap()); + assert!( + !download_stacks_dataset_if_required(&mut config_clone, &ctx) + .await + .unwrap() + ); let mut tsv_file_path = config.expected_cache_path(); tsv_file_path.push(default_tsv_file_path(&config.network.stacks_network)); diff --git a/components/chainhook-cli/src/cli/mod.rs b/components/chainhook-cli/src/cli/mod.rs index ed51d0b1a..d92f37bc7 100644 --- a/components/chainhook-cli/src/cli/mod.rs +++ b/components/chainhook-cli/src/cli/mod.rs @@ -277,14 +277,14 @@ pub fn main() { let opts: Opts = match Opts::try_parse() { Ok(opts) => opts, Err(e) => { - error!(ctx.expect_logger(), "{e}"); + crit!(ctx.expect_logger(), "{e}"); process::exit(1); } }; match hiro_system_kit::nestable_block_on(handle_command(opts, ctx.clone())) { Err(e) => { - error!(ctx.expect_logger(), "{e}"); + crit!(ctx.expect_logger(), "{e}"); process::exit(1); } Ok(_) => {} diff --git a/components/chainhook-cli/src/config/mod.rs b/components/chainhook-cli/src/config/mod.rs index fc291659f..812d76021 100644 --- a/components/chainhook-cli/src/config/mod.rs +++ b/components/chainhook-cli/src/config/mod.rs @@ -256,13 +256,13 @@ impl Config { } } - pub fn expected_local_stacks_tsv_file(&self) -> &PathBuf { + pub fn expected_local_stacks_tsv_file(&self) -> Result<&PathBuf, String> { for source in self.event_sources.iter() { if let EventSourceConfig::StacksTsvPath(config) = source { - return &config.file_path; + return Ok(&config.file_path); } } - panic!("expected local-tsv source") + Err("could not find expected local tsv source")? } pub fn expected_cache_path(&self) -> PathBuf { @@ -271,21 +271,23 @@ impl Config { destination_path } - fn expected_remote_stacks_tsv_base_url(&self) -> &String { + fn expected_remote_stacks_tsv_base_url(&self) -> Result<&String, String> { for source in self.event_sources.iter() { if let EventSourceConfig::StacksTsvUrl(config) = source { - return &config.file_url; + return Ok(&config.file_url); } } - panic!("expected remote-tsv source") + Err("could not find expected remote tsv source")? } - pub fn expected_remote_stacks_tsv_sha256(&self) -> String { - format!("{}.sha256", self.expected_remote_stacks_tsv_base_url()) + pub fn expected_remote_stacks_tsv_sha256(&self) -> Result { + self.expected_remote_stacks_tsv_base_url() + .map(|url| format!("{}.sha256", url)) } - pub fn expected_remote_stacks_tsv_url(&self) -> String { - format!("{}.gz", self.expected_remote_stacks_tsv_base_url()) + pub fn expected_remote_stacks_tsv_url(&self) -> Result { + self.expected_remote_stacks_tsv_base_url() + .map(|url| format!("{}.gz", url)) } pub fn rely_on_remote_stacks_tsv(&self) -> bool { diff --git a/components/chainhook-cli/src/config/tests/mod.rs b/components/chainhook-cli/src/config/tests/mod.rs index 4adf04a8a..ddae000c2 100644 --- a/components/chainhook-cli/src/config/tests/mod.rs +++ b/components/chainhook-cli/src/config/tests/mod.rs @@ -108,7 +108,6 @@ fn should_download_remote_stacks_tsv_handles_both_modes() { } #[test] -#[should_panic(expected = "expected remote-tsv source")] fn expected_remote_stacks_tsv_base_url_panics_if_missing() { let url_src = EventSourceConfig::StacksTsvUrl(super::UrlConfig { file_url: format!("test"), @@ -116,15 +115,22 @@ fn expected_remote_stacks_tsv_base_url_panics_if_missing() { let mut config = Config::default(true, false, false, &None).unwrap(); config.event_sources = vec![url_src.clone()]; - assert_eq!(config.expected_remote_stacks_tsv_base_url(), "test"); + match config.expected_remote_stacks_tsv_base_url() { + Ok(tsv_url) => assert_eq!(tsv_url, "test"), + Err(e) => { + panic!("expected tsv file: {e}") + } + } config.event_sources = vec![]; - config.expected_remote_stacks_tsv_base_url(); + match config.expected_remote_stacks_tsv_base_url() { + Ok(tsv_url) => panic!("expected no tsv file, found {}", tsv_url), + Err(e) => assert_eq!(e, "could not find expected remote tsv source".to_string()), + }; } #[test] -#[should_panic(expected = "expected local-tsv source")] -fn expected_local_stacks_tsv_base_url_panics_if_missing() { +fn expected_local_stacks_tsv_base_url_errors_if_missing() { let path = PathBuf::from("test"); let path_src = EventSourceConfig::StacksTsvPath(PathConfig { file_path: path.clone(), @@ -132,10 +138,18 @@ fn expected_local_stacks_tsv_base_url_panics_if_missing() { let mut config = Config::default(true, false, false, &None).unwrap(); config.event_sources = vec![path_src.clone()]; - assert_eq!(config.expected_local_stacks_tsv_file(), &path); + match config.expected_local_stacks_tsv_file() { + Ok(tsv_path) => assert_eq!(tsv_path, &path), + Err(e) => { + panic!("expected tsv file: {e}") + } + } config.event_sources = vec![]; - config.expected_local_stacks_tsv_file(); + match config.expected_local_stacks_tsv_file() { + Ok(tsv_path) => panic!("expected no tsv file, found {}", tsv_path.to_string_lossy()), + Err(e) => assert_eq!(e, "could not find expected local tsv source".to_string()), + }; } #[test] diff --git a/components/chainhook-cli/src/scan/bitcoin.rs b/components/chainhook-cli/src/scan/bitcoin.rs index 41ae6e365..886f60998 100644 --- a/components/chainhook-cli/src/scan/bitcoin.rs +++ b/components/chainhook-cli/src/scan/bitcoin.rs @@ -29,6 +29,7 @@ pub async fn scan_bitcoin_chainstate_via_rpc_using_predicate( config: &Config, ctx: &Context, ) -> Result { + let predicate_uuid = &predicate_spec.uuid; let auth = Auth::UserPass( config.network.bitcoind_rpc_username.clone(), config.network.bitcoind_rpc_password.clone(), @@ -71,9 +72,9 @@ pub async fn scan_bitcoin_chainstate_via_rpc_using_predicate( PredicatesApi::Off => None, }; - info!( + debug!( ctx.expect_logger(), - "Starting predicate evaluation on Bitcoin blocks", + "Starting predicate evaluation on Bitcoin blocks for predicate {predicate_uuid}", ); let mut last_block_scanned = BlockIdentifier::default(); @@ -200,7 +201,7 @@ pub async fn scan_bitcoin_chainstate_via_rpc_using_predicate( info!( ctx.expect_logger(), - "{number_of_blocks_scanned} blocks scanned, {actions_triggered} actions triggered" + "Predicate {predicate_uuid} scan completed. {number_of_blocks_scanned} blocks scanned, {actions_triggered} actions triggered." ); if let Some(ref mut predicates_db_conn) = predicates_db_conn { @@ -269,9 +270,13 @@ pub async fn execute_predicates_action<'a>( if trigger.chainhook.include_proof { gather_proofs(&trigger, &mut proofs, &config, &ctx); } + let predicate_uuid = &trigger.chainhook.uuid; match handle_bitcoin_hook_action(trigger, &proofs) { Err(e) => { - error!(ctx.expect_logger(), "unable to handle action {}", e); + warn!( + ctx.expect_logger(), + "unable to handle action for predicate {}: {}", predicate_uuid, e + ); } Ok(action) => { actions_triggered += 1; diff --git a/components/chainhook-cli/src/scan/stacks.rs b/components/chainhook-cli/src/scan/stacks.rs index 6d3f90e40..6b50ba419 100644 --- a/components/chainhook-cli/src/scan/stacks.rs +++ b/components/chainhook-cli/src/scan/stacks.rs @@ -66,7 +66,7 @@ pub async fn get_canonical_fork_from_tsv( start_block: Option, ctx: &Context, ) -> Result, String> { - let seed_tsv_path = config.expected_local_stacks_tsv_file().clone(); + let seed_tsv_path = config.expected_local_stacks_tsv_file()?.clone(); let (record_tx, record_rx) = std::sync::mpsc::channel(); @@ -98,7 +98,7 @@ pub async fn get_canonical_fork_from_tsv( } let _ = record_tx.send(None); }) - .expect("unable to spawn thread"); + .map_err(|e| format!("unable to spawn thread: {e}"))?; let stacks_db = open_readonly_stacks_db_conn_with_retry(&config.expected_cache_path(), 3, ctx)?; let canonical_fork = { @@ -111,7 +111,10 @@ pub async fn get_canonical_fork_from_tsv( match standardize_stacks_serialized_block_header(&blob) { Ok(data) => data, Err(e) => { - error!(ctx.expect_logger(), "{e}"); + error!( + ctx.expect_logger(), + "Failed to standardize stacks header: {e}" + ); continue; } } @@ -169,12 +172,13 @@ pub async fn scan_stacks_chainstate_via_rocksdb_using_predicate( config: &Config, ctx: &Context, ) -> Result<(Option, bool), String> { + let predicate_uuid = &predicate_spec.uuid; let mut chain_tip = match get_last_unconfirmed_block_height_inserted(stacks_db_conn, ctx) { Some(chain_tip) => chain_tip, None => match get_last_block_height_inserted(stacks_db_conn, ctx) { Some(chain_tip) => chain_tip, None => { - info!(ctx.expect_logger(), "No blocks inserted in db; cannot determing Stacks chain tip. Skipping scan of predicate {}", predicate_spec.uuid); + info!(ctx.expect_logger(), "No blocks inserted in db; cannot determing Stacks chain tip. Skipping scan of predicate {}", predicate_uuid); return Ok((None, false)); } }, @@ -201,9 +205,9 @@ pub async fn scan_stacks_chainstate_via_rocksdb_using_predicate( }; let proofs = HashMap::new(); - info!( + debug!( ctx.expect_logger(), - "Starting predicate evaluation on Stacks blocks" + "Starting predicate evaluation on Stacks blocks for predicate {}", predicate_uuid ); let mut last_block_scanned = BlockIdentifier::default(); let mut err_count = 0; @@ -243,7 +247,7 @@ pub async fn scan_stacks_chainstate_via_rocksdb_using_predicate( None => match get_last_block_height_inserted(stacks_db_conn, ctx) { Some(chain_tip) => chain_tip, None => { - info!(ctx.expect_logger(), "No blocks inserted in db; cannot determing Stacks chain tip. Skipping scan of predicate {}", predicate_spec.uuid); + warn!(ctx.expect_logger(), "No blocks inserted in db; cannot determine Stacks chain tip. Skipping scan of predicate {}", predicate_uuid); return Ok((None, false)); } }, @@ -304,7 +308,10 @@ pub async fn scan_stacks_chainstate_via_rocksdb_using_predicate( }; let res = match handle_stacks_hook_action(trigger, &proofs, &ctx) { Err(e) => { - error!(ctx.expect_logger(), "unable to handle action {}", e); + warn!( + ctx.expect_logger(), + "unable to handle action for predicate {}: {}", predicate_uuid, e + ); Ok(()) // todo: should this error increment our err_count? } Ok(action) => { @@ -342,7 +349,7 @@ pub async fn scan_stacks_chainstate_via_rocksdb_using_predicate( } info!( ctx.expect_logger(), - "{number_of_blocks_scanned} blocks scanned, {number_of_times_triggered} blocks triggering predicate" + "Predicate {predicate_uuid} scan completed. {number_of_blocks_scanned} blocks scanned, {number_of_times_triggered} blocks triggering predicate.", ); if let Some(ref mut predicates_db_conn) = predicates_db_conn { @@ -420,7 +427,7 @@ pub async fn scan_stacks_chainstate_via_csv_using_predicate( } } - let _ = download_stacks_dataset_if_required(config, ctx).await; + let _ = download_stacks_dataset_if_required(config, ctx).await?; let mut canonical_fork = get_canonical_fork_from_tsv(config, None, ctx).await?; @@ -516,7 +523,7 @@ pub async fn consolidate_local_stacks_chainstate_using_csv( "Building local chainstate from Stacks archive file" ); - let downloaded_new_dataset = download_stacks_dataset_if_required(config, ctx).await; + let downloaded_new_dataset = download_stacks_dataset_if_required(config, ctx).await?; if downloaded_new_dataset { let stacks_db = @@ -550,13 +557,15 @@ pub async fn consolidate_local_stacks_chainstate_using_csv( ) { Ok(block) => block, Err(e) => { - error!(&ctx.expect_logger(), "{e}"); + error!( + &ctx.expect_logger(), + "Failed to standardize stacks block: {e}" + ); continue; } }; - // TODO: return a result - insert_entry_in_stacks_blocks(&block_data, &stacks_db_rw, ctx); + insert_entry_in_stacks_blocks(&block_data, &stacks_db_rw, ctx)?; if blocks_inserted % 2500 == 0 { info!( diff --git a/components/chainhook-cli/src/service/http_api.rs b/components/chainhook-cli/src/service/http_api.rs index 191a33a7c..460b2f9a1 100644 --- a/components/chainhook-cli/src/service/http_api.rs +++ b/components/chainhook-cli/src/service/http_api.rs @@ -28,7 +28,7 @@ pub async fn start_predicate_api_server( api_config: PredicatesApiConfig, observer_commands_tx: Sender, ctx: Context, -) -> Result> { +) -> Result> { let log_level = LogLevel::Off; let mut shutdown_config = config::Shutdown::default(); @@ -62,12 +62,12 @@ pub async fn start_predicate_api_server( .ignite() .await?; - let ingestion_shutdown = ignite.shutdown(); + let predicate_api_shutdown = ignite.shutdown(); let _ = std::thread::spawn(move || { let _ = hiro_system_kit::nestable_block_on(ignite.launch()); }); - Ok(ingestion_shutdown) + Ok(predicate_api_shutdown) } #[openapi(tag = "Health Check")] @@ -298,9 +298,12 @@ pub fn get_entry_from_predicates_db( let spec = ChainhookSpecification::deserialize_specification(&encoded_spec)?; let encoded_status = match entry.get("status") { - None => unimplemented!(), - Some(payload) => payload, - }; + None => Err(format!( + "found predicate specification with no status for predicate {}", + predicate_key + )), + Some(payload) => Ok(payload), + }?; let status = serde_json::from_str(&encoded_status).map_err(|e| format!("{}", e.to_string()))?; diff --git a/components/chainhook-cli/src/service/mod.rs b/components/chainhook-cli/src/service/mod.rs index 86286c356..9bb7e95e5 100644 --- a/components/chainhook-cli/src/service/mod.rs +++ b/components/chainhook-cli/src/service/mod.rs @@ -89,15 +89,16 @@ impl Service { } match chainhook_config.register_specification(predicate) { Ok(_) => { - info!( + debug!( self.ctx.expect_logger(), - "Predicate {} retrieved from storage and loaded", predicate_uuid, + "Predicate {} retrieved from storage and registered", predicate_uuid, ); } Err(e) => { - error!( + warn!( self.ctx.expect_logger(), - "Failed loading predicate from storage: {}", + "Failed to register predicate {} after retrieving from storage: {}", + predicate_uuid, e.to_string() ); } @@ -117,7 +118,7 @@ impl Service { &self.ctx, ) { Ok(Some(_)) => { - error!( + warn!( self.ctx.expect_logger(), "Predicate uuid already in use: {uuid}", ); @@ -136,16 +137,16 @@ impl Service { ) { Ok(spec) => { newly_registered_predicates.push(spec.clone()); - info!( + debug!( self.ctx.expect_logger(), "Predicate {} retrieved from config and loaded", spec.uuid(), ); } Err(e) => { - error!( + warn!( self.ctx.expect_logger(), - "Failed loading predicate from config: {}", + "Failed to load predicate from config: {}", e.to_string() ); } @@ -163,7 +164,7 @@ impl Service { // Download and ingest a Stacks dump if self.config.rely_on_remote_stacks_tsv() { let _ = - consolidate_local_stacks_chainstate_using_csv(&mut self.config, &self.ctx).await; + consolidate_local_stacks_chainstate_using_csv(&mut self.config, &self.ctx).await?; } // Stacks scan operation threadpool @@ -176,9 +177,12 @@ impl Service { start_stacks_scan_runloop( &config, stacks_scan_op_rx, - observer_command_tx_moved, + observer_command_tx_moved.clone(), &ctx, ); + // the scan runloop should loop forever; if it finishes, something is wrong + crit!(ctx.expect_logger(), "Stacks scan runloop stopped.",); + let _ = observer_command_tx_moved.send(ObserverCommand::Terminate); }) .expect("unable to spawn thread"); @@ -192,15 +196,18 @@ impl Service { start_bitcoin_scan_runloop( &config, bitcoin_scan_op_rx, - observer_command_tx_moved, + observer_command_tx_moved.clone(), &ctx, ); + // the scan runloop should loop forever; if it finishes, something is wrong + crit!(ctx.expect_logger(), "Bitcoin scan runloop stopped.",); + let _ = observer_command_tx_moved.send(ObserverCommand::Terminate); }) .expect("unable to spawn thread"); // Enable HTTP Predicates API, if required let config = self.config.clone(); - if let PredicatesApi::On(ref api_config) = config.http_api { + let predicate_api_shutdown = if let PredicatesApi::On(ref api_config) = config.http_api { info!( self.ctx.expect_logger(), "Listening on port {} for chainhook predicate registrations", api_config.http_port @@ -209,11 +216,29 @@ impl Service { let api_config = api_config.clone(); let moved_observer_command_tx = observer_command_tx.clone(); // Test and initialize a database connection - let _ = hiro_system_kit::thread_named("HTTP Predicate API").spawn(move || { - let future = start_predicate_api_server(api_config, moved_observer_command_tx, ctx); - let _ = hiro_system_kit::nestable_block_on(future); - }); - } + let res = hiro_system_kit::thread_named("HTTP Predicate API") + .spawn(move || { + let future = start_predicate_api_server( + api_config, + moved_observer_command_tx.clone(), + ctx.clone(), + ); + hiro_system_kit::nestable_block_on(future) + }) + .expect("unable to spawn thread"); + let res = res.join().expect("unable to terminate thread"); + match res { + Ok(predicate_api_shutdown) => Some(predicate_api_shutdown), + Err(e) => { + return Err(format!( + "Predicate API Registration server failed to start: {}", + e + )); + } + } + } else { + None + }; let ctx = self.ctx.clone(); let stacks_db = @@ -281,7 +306,7 @@ impl Service { let event = match observer_event_rx.recv() { Ok(cmd) => cmd, Err(e) => { - error!( + crit!( self.ctx.expect_logger(), "Error: broken channel {}", e.to_string() @@ -343,20 +368,20 @@ impl Service { ); } } - ObserverEvent::PredicateDeregistered(spec) => { + ObserverEvent::PredicateDeregistered(uuid) => { if let PredicatesApi::On(ref config) = self.config.http_api { let Ok(mut predicates_db_conn) = open_readwrite_predicates_db_conn_verbose(&config, &ctx) else { continue; }; - let predicate_key = spec.key(); + let predicate_key = ChainhookSpecification::either_stx_or_btc_key(&uuid); let res: Result<(), redis::RedisError> = - predicates_db_conn.del(predicate_key); + predicates_db_conn.del(predicate_key.clone()); if let Err(e) = res { - error!( + warn!( self.ctx.expect_logger(), - "unable to delete predicate: {}", + "unable to delete predicate {predicate_key}: {}", e.to_string() ); } @@ -429,7 +454,7 @@ impl Service { } } } - update_stats_from_report( + update_status_from_report( Chain::Bitcoin, report, &mut predicates_db_conn, @@ -446,7 +471,7 @@ impl Service { Err(e) => { error!( self.ctx.expect_logger(), - "unable to store stacks block: {}", + "unable to open stacks db: {}", e.to_string() ); continue; @@ -455,28 +480,48 @@ impl Service { match &chain_event { StacksChainEvent::ChainUpdatedWithBlocks(data) => { - confirm_entries_in_stacks_blocks( + if let Err(e) = confirm_entries_in_stacks_blocks( &data.confirmed_blocks, &stacks_db_conn_rw, &self.ctx, - ); - draft_entries_in_stacks_blocks( + ) { + error!( + self.ctx.expect_logger(), + "unable add confirmed entries to stacks db: {}", e + ); + }; + if let Err(e) = draft_entries_in_stacks_blocks( &data.new_blocks, &stacks_db_conn_rw, &self.ctx, - ) + ) { + error!( + self.ctx.expect_logger(), + "unable add unconfirmed entries to stacks db: {}", e + ); + }; } StacksChainEvent::ChainUpdatedWithReorg(data) => { - confirm_entries_in_stacks_blocks( + if let Err(e) = confirm_entries_in_stacks_blocks( &data.confirmed_blocks, &stacks_db_conn_rw, &self.ctx, - ); - draft_entries_in_stacks_blocks( + ) { + error!( + self.ctx.expect_logger(), + "unable add confirmed entries to stacks db: {}", e + ); + }; + if let Err(e) = draft_entries_in_stacks_blocks( &data.blocks_to_apply, &stacks_db_conn_rw, &self.ctx, - ) + ) { + error!( + self.ctx.expect_logger(), + "unable add unconfirmed entries to stacks db: {}", e + ); + }; } StacksChainEvent::ChainUpdatedWithMicroblocks(_) | StacksChainEvent::ChainUpdatedWithMicroblocksReorg(_) => {} @@ -546,7 +591,7 @@ impl Service { StacksChainEvent::ChainUpdatedWithMicroblocks(_) | StacksChainEvent::ChainUpdatedWithMicroblocksReorg(_) => {} }; - update_stats_from_report( + update_status_from_report( Chain::Stacks, report, &mut predicates_db_conn, @@ -557,15 +602,36 @@ impl Service { // Every 32 blocks, we will check if there's a new Stacks file archive to ingest if stacks_event > 32 { stacks_event = 0; - let _ = consolidate_local_stacks_chainstate_using_csv( - &mut self.config, - &self.ctx, - ) - .await; + if self.config.rely_on_remote_stacks_tsv() { + match consolidate_local_stacks_chainstate_using_csv( + &mut self.config, + &self.ctx, + ) + .await + { + Err(e) => { + error!( + self.ctx.expect_logger(), + "Failed to update database from archive: {e}" + ) + } + Ok(()) => {} + }; + } } } ObserverEvent::Terminate => { - info!(self.ctx.expect_logger(), "Terminating runloop"); + info!( + self.ctx.expect_logger(), + "Terminating ObserverEvent runloop" + ); + if let Some(predicate_api_shutdown) = predicate_api_shutdown { + info!( + self.ctx.expect_logger(), + "Terminating Predicate Registration API" + ); + predicate_api_shutdown.notify(); + } break; } _ => {} @@ -615,7 +681,7 @@ pub struct ExpiredData { pub expired_at_block_height: u64, } -fn update_stats_from_report( +fn update_status_from_report( chain: Chain, report: PredicateEvaluationReport, predicates_db_conn: &mut Connection, @@ -623,7 +689,7 @@ fn update_stats_from_report( ) { for (predicate_uuid, blocks_ids) in report.predicates_triggered.iter() { if let Some(last_triggered_height) = blocks_ids.last().and_then(|b| Some(b.index)) { - let triggered_count = blocks_ids.len().try_into().unwrap(); + let triggered_count = blocks_ids.len().try_into().unwrap_or(0); set_predicate_streaming_status( StreamingDataType::Occurrence { last_triggered_height, @@ -652,7 +718,7 @@ fn update_stats_from_report( } } if let Some(last_evaluated_height) = blocks_ids.last().and_then(|b| Some(b.index)) { - let evaluated_count = blocks_ids.len().try_into().unwrap(); + let evaluated_count = blocks_ids.len().try_into().unwrap_or(0); set_predicate_streaming_status( StreamingDataType::Evaluation { last_evaluated_height, @@ -666,7 +732,7 @@ fn update_stats_from_report( } for (predicate_uuid, blocks_ids) in report.predicates_expired.iter() { if let Some(last_evaluated_height) = blocks_ids.last().and_then(|b| Some(b.index)) { - let evaluated_count = blocks_ids.len().try_into().unwrap(); + let evaluated_count = blocks_ids.len().try_into().unwrap_or(0); set_unconfirmed_expiration_status( &chain, evaluated_count, @@ -1035,13 +1101,13 @@ fn insert_predicate_expiration( if let Err(e) = predicates_db_conn.hset::<_, _, _, ()>(&key, "predicates", &serialized_expiring_predicates) { - error!( + warn!( ctx.expect_logger(), "Error updating expired predicates index: {}", e.to_string() ); } else { - info!( + debug!( ctx.expect_logger(), "Updating expired predicates at block height {expired_at_block_height} with predicate: {predicate_key}" ); @@ -1060,7 +1126,7 @@ fn get_predicates_expiring_at_block( Ok(data) => { if let Err(e) = predicates_db_conn.hdel::<_, _, u64>(key.to_string(), "predicates") { - error!( + warn!( ctx.expect_logger(), "Error removing expired predicates index: {}", e.to_string() @@ -1084,13 +1150,14 @@ pub fn update_predicate_status( if let Err(e) = predicates_db_conn.hset::<_, _, _, ()>(&predicate_key, "status", &serialized_status) { - error!( + warn!( ctx.expect_logger(), - "Error updating status: {}", + "Error updating status for {}: {}", + predicate_key, e.to_string() ); } else { - info!( + debug!( ctx.expect_logger(), "Updating predicate {predicate_key} status: {serialized_status}" ); @@ -1107,13 +1174,14 @@ fn update_predicate_spec( if let Err(e) = predicates_db_conn.hset::<_, _, _, ()>(&predicate_key, "specification", &serialized_spec) { - error!( + warn!( ctx.expect_logger(), - "Error updating status: {}", + "Error updating status for {}: {}", + predicate_key, e.to_string() ); } else { - info!( + debug!( ctx.expect_logger(), "Updating predicate {predicate_key} with spec: {serialized_spec}" ); @@ -1154,6 +1222,7 @@ pub fn open_readwrite_predicates_db_conn_verbose( res } +// todo: evaluate expects pub fn open_readwrite_predicates_db_conn_or_panic( config: &PredicatesApiConfig, ctx: &Context, diff --git a/components/chainhook-cli/src/service/runloops.rs b/components/chainhook-cli/src/service/runloops.rs index ee441008a..d1c4d32cc 100644 --- a/components/chainhook-cli/src/service/runloops.rs +++ b/components/chainhook-cli/src/service/runloops.rs @@ -43,9 +43,12 @@ pub fn start_stacks_scan_runloop( { Ok(db_conn) => db_conn, Err(e) => { + // todo: if we repeatedly can't connect to the database, we should restart the + // service to get to a healthy state. I don't know if this has been an issue, though + // so we can monitor and possibly remove this todo error!( moved_ctx.expect_logger(), - "unable to store stacks block: {}", + "unable to open stacks db: {}", e.to_string() ); unimplemented!() @@ -108,8 +111,7 @@ pub fn start_stacks_scan_runloop( } }); } - let res = stacks_scan_pool.join(); - res + let _ = stacks_scan_pool.join(); } pub fn start_bitcoin_scan_runloop( @@ -138,7 +140,7 @@ pub fn start_bitcoin_scan_runloop( let predicate_is_expired = match hiro_system_kit::nestable_block_on(op) { Ok(predicate_is_expired) => predicate_is_expired, Err(e) => { - error!( + warn!( moved_ctx.expect_logger(), "Unable to evaluate predicate on Bitcoin chainstate: {e}", ); diff --git a/components/chainhook-cli/src/storage/mod.rs b/components/chainhook-cli/src/storage/mod.rs index 2d18aa33b..6c2d04ca3 100644 --- a/components/chainhook-cli/src/storage/mod.rs +++ b/components/chainhook-cli/src/storage/mod.rs @@ -116,12 +116,16 @@ fn get_last_unconfirmed_insert_key() -> [u8; 3] { *LAST_UNCONFIRMED_KEY_PREFIX } -pub fn insert_entry_in_stacks_blocks(block: &StacksBlockData, stacks_db_rw: &DB, _ctx: &Context) { +pub fn insert_entry_in_stacks_blocks( + block: &StacksBlockData, + stacks_db_rw: &DB, + _ctx: &Context, +) -> Result<(), String> { let key = get_block_key(&block.block_identifier); let block_bytes = json!(block); stacks_db_rw .put(&key, &block_bytes.to_string().as_bytes()) - .expect("unable to insert blocks"); + .map_err(|e| format!("unable to insert blocks: {}", e))?; let previous_last_inserted = get_last_block_height_inserted(stacks_db_rw, _ctx).unwrap_or(0); if block.block_identifier.index > previous_last_inserted { stacks_db_rw @@ -129,35 +133,39 @@ pub fn insert_entry_in_stacks_blocks(block: &StacksBlockData, stacks_db_rw: &DB, get_last_confirmed_insert_key(), block.block_identifier.index.to_be_bytes(), ) - .expect("unable to insert metadata"); + .map_err(|e| format!("unable to insert metadata: {}", e))?; } + Ok(()) } pub fn insert_unconfirmed_entry_in_stacks_blocks( block: &StacksBlockData, stacks_db_rw: &DB, _ctx: &Context, -) { +) -> Result<(), String> { let key = get_unconfirmed_block_key(&block.block_identifier); let block_bytes = json!(block); stacks_db_rw .put(&key, &block_bytes.to_string().as_bytes()) - .expect("unable to insert blocks"); + .map_err(|e| format!("unable to insert blocks: {}", e))?; stacks_db_rw .put( get_last_unconfirmed_insert_key(), block.block_identifier.index.to_be_bytes(), ) - .expect("unable to insert metadata"); + .map_err(|e| format!("unable to insert metadata: {}", e))?; + Ok(()) } pub fn delete_unconfirmed_entry_from_stacks_blocks( block_identifier: &BlockIdentifier, stacks_db_rw: &DB, _ctx: &Context, -) { +) -> Result<(), String> { let key = get_unconfirmed_block_key(&block_identifier); - stacks_db_rw.delete(&key).expect("unable to delete blocks"); + stacks_db_rw + .delete(&key) + .map_err(|e| format!("unable to delete blocks: {}", e)) } pub fn get_last_unconfirmed_block_height_inserted(stacks_db: &DB, _ctx: &Context) -> Option { @@ -209,22 +217,24 @@ pub fn confirm_entries_in_stacks_blocks( blocks: &Vec, stacks_db_rw: &DB, ctx: &Context, -) { +) -> Result<(), String> { for block in blocks.iter() { - insert_entry_in_stacks_blocks(block, stacks_db_rw, ctx); - delete_unconfirmed_entry_from_stacks_blocks(&block.block_identifier, stacks_db_rw, ctx); + insert_entry_in_stacks_blocks(block, stacks_db_rw, ctx)?; + delete_unconfirmed_entry_from_stacks_blocks(&block.block_identifier, stacks_db_rw, ctx)?; } + Ok(()) } pub fn draft_entries_in_stacks_blocks( block_updates: &Vec, stacks_db_rw: &DB, ctx: &Context, -) { +) -> Result<(), String> { for update in block_updates.iter() { // TODO: Could be imperfect, from a microblock point of view - insert_unconfirmed_entry_in_stacks_blocks(&update.block, stacks_db_rw, ctx); + insert_unconfirmed_entry_in_stacks_blocks(&update.block, stacks_db_rw, ctx)?; } + Ok(()) } pub fn get_stacks_block_at_block_height( diff --git a/components/chainhook-sdk/Cargo.toml b/components/chainhook-sdk/Cargo.toml index d85a8dcd5..b8789ae2f 100644 --- a/components/chainhook-sdk/Cargo.toml +++ b/components/chainhook-sdk/Cargo.toml @@ -50,4 +50,4 @@ test-case = "3.1.0" default = ["hiro-system-kit/log"] zeromq = ["zmq"] debug = ["hiro-system-kit/debug"] -release = ["hiro-system-kit/release"] +release = ["hiro-system-kit/debug"] diff --git a/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs b/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs index 251913a06..4bd584e1f 100644 --- a/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs +++ b/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs @@ -293,7 +293,7 @@ impl ForkScratchPad { ctx.try_log(|logger| { slog::error!( logger, - "unable to retrive Bitcoin {} from block store", + "unable to retrieve Bitcoin block {} from block store", block_identifier ) }); @@ -317,7 +317,16 @@ impl ForkScratchPad { let block_identifier = &divergence.block_ids_to_apply[i]; let header = match self.headers_store.get(block_identifier) { Some(header) => header.clone(), - None => panic!("unable to retrive block from block store"), + None => { + ctx.try_log(|logger| { + slog::error!( + logger, + "unable to retrieve Bitcoin block {} from block store", + block_identifier + ) + }); + return Err(ChainSegmentIncompatibility::Unknown); + } }; new_headers.push(header) } @@ -336,22 +345,40 @@ impl ForkScratchPad { .map(|block_id| { let block = match self.headers_store.get(block_id) { Some(block) => block.clone(), - None => panic!("unable to retrive block from block store"), + None => { + ctx.try_log(|logger| { + slog::error!( + logger, + "unable to retrieve Bitcoin block {} from block store", + block_id + ) + }); + return Err(ChainSegmentIncompatibility::Unknown); + } }; - block + Ok(block) }) - .collect::>(), + .collect::, _>>()?, headers_to_apply: divergence .block_ids_to_apply .iter() .map(|block_id| { let block = match self.headers_store.get(block_id) { Some(block) => block.clone(), - None => panic!("unable to retrive block from block store"), + None => { + ctx.try_log(|logger| { + slog::error!( + logger, + "unable to retrieve Bitcoin block {} from block store", + block_id + ) + }); + return Err(ChainSegmentIncompatibility::Unknown); + } }; - block + Ok(block) }) - .collect::>(), + .collect::, _>>()?, confirmed_headers: vec![], }, )); diff --git a/components/chainhook-sdk/src/indexer/mod.rs b/components/chainhook-sdk/src/indexer/mod.rs index 9b0c97e92..a518aaaa6 100644 --- a/components/chainhook-sdk/src/indexer/mod.rs +++ b/components/chainhook-sdk/src/indexer/mod.rs @@ -367,7 +367,7 @@ impl ChainSegment { } Err(incompatibility) => { ctx.try_log(|logger| { - slog::info!(logger, "Will have to fork: {:?}", incompatibility) + slog::warn!(logger, "Will have to fork: {:?}", incompatibility) }); match incompatibility { ChainSegmentIncompatibility::BlockCollision => { diff --git a/components/chainhook-sdk/src/indexer/stacks/mod.rs b/components/chainhook-sdk/src/indexer/stacks/mod.rs index 4cff47b7f..0eeb70128 100644 --- a/components/chainhook-sdk/src/indexer/stacks/mod.rs +++ b/components/chainhook-sdk/src/indexer/stacks/mod.rs @@ -290,9 +290,11 @@ pub fn standardize_stacks_serialized_block_header( .parent_index_block_hash .take() .ok_or(format!("unable to retrieve parent_index_block_hash"))?; + + let parent_height = block_identifier.index.saturating_sub(1); let parent_block_identifier = BlockIdentifier { hash: parent_hash, - index: block_identifier.index - 1, + index: parent_height, }; Ok((block_identifier, parent_block_identifier)) } @@ -902,7 +904,7 @@ pub fn get_standardized_non_fungible_currency_from_asset_class_id( }), } } - +//todo: this function has a lot of expects/panics. should return result instead pub fn get_standardized_stacks_receipt( _txid: &str, events: Vec, diff --git a/components/chainhook-sdk/src/observer/http.rs b/components/chainhook-sdk/src/observer/http.rs index 25da770d9..f844718d5 100644 --- a/components/chainhook-sdk/src/observer/http.rs +++ b/components/chainhook-sdk/src/observer/http.rs @@ -132,7 +132,7 @@ pub async fn handle_new_bitcoin_block( }; } Ok(None) => { - ctx.try_log(|logger| slog::info!(logger, "unable to infer chain progress")); + ctx.try_log(|logger| slog::warn!(logger, "unable to infer chain progress")); } Err(e) => { ctx.try_log(|logger| slog::error!(logger, "unable to handle bitcoin block: {}", e)) @@ -202,7 +202,7 @@ pub fn handle_new_stacks_block( }; } Ok(None) => { - ctx.try_log(|logger| slog::info!(logger, "unable to infer chain progress")); + ctx.try_log(|logger| slog::warn!(logger, "unable to infer chain progress")); } Err(e) => ctx.try_log(|logger| slog::error!(logger, "{}", e)), } @@ -273,7 +273,7 @@ pub fn handle_new_microblocks( }; } Ok(None) => { - ctx.try_log(|logger| slog::info!(logger, "unable to infer chain progress")); + ctx.try_log(|logger| slog::warn!(logger, "unable to infer chain progress")); } Err(e) => { ctx.try_log(|logger| slog::error!(logger, "unable to handle stacks microblock: {}", e)); diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs index cc1108b68..4e04e62a7 100644 --- a/components/chainhook-sdk/src/observer/mod.rs +++ b/components/chainhook-sdk/src/observer/mod.rs @@ -304,7 +304,7 @@ pub enum ObserverEvent { StacksChainEvent((StacksChainEvent, PredicateEvaluationReport)), NotifyBitcoinTransactionProxied, PredicateRegistered(ChainhookSpecification), - PredicateDeregistered(ChainhookSpecification), + PredicateDeregistered(String), PredicateEnabled(ChainhookSpecification), BitcoinPredicateTriggered(BitcoinChainhookOccurrencePayload), StacksPredicateTriggered(StacksChainhookOccurrencePayload), @@ -437,35 +437,65 @@ pub fn start_event_observer( let context_cloned = ctx.clone(); let event_observer_config_moved = config.clone(); let observer_commands_tx_moved = observer_commands_tx.clone(); - let _ = hiro_system_kit::thread_named("Chainhook event observer").spawn(move || { - let future = start_bitcoin_event_observer( - event_observer_config_moved, - observer_commands_tx_moved, - observer_commands_rx, - observer_events_tx, - observer_sidecar, - context_cloned, - ); - let _ = hiro_system_kit::nestable_block_on(future); - }); + let _ = hiro_system_kit::thread_named("Chainhook event observer") + .spawn(move || { + let future = start_bitcoin_event_observer( + event_observer_config_moved, + observer_commands_tx_moved, + observer_commands_rx, + observer_events_tx.clone(), + observer_sidecar, + context_cloned.clone(), + ); + match hiro_system_kit::nestable_block_on(future) { + Ok(_) => {} + Err(e) => { + if let Some(tx) = observer_events_tx { + context_cloned.try_log(|logger| { + slog::crit!( + logger, + "Chainhook event observer thread failed with error: {e}", + ) + }); + let _ = tx.send(ObserverEvent::Terminate); + } + } + } + }) + .expect("unable to spawn thread"); } BitcoinBlockSignaling::Stacks(ref _url) => { // Start chainhook event observer let context_cloned = ctx.clone(); let event_observer_config_moved = config.clone(); let observer_commands_tx_moved = observer_commands_tx.clone(); - let _ = hiro_system_kit::thread_named("Chainhook event observer").spawn(move || { - let future = start_stacks_event_observer( - event_observer_config_moved, - observer_commands_tx_moved, - observer_commands_rx, - observer_events_tx, - observer_sidecar, - stacks_block_pool_seed, - context_cloned, - ); - let _ = hiro_system_kit::nestable_block_on(future); - }); + let _ = hiro_system_kit::thread_named("Chainhook event observer") + .spawn(move || { + let future = start_stacks_event_observer( + event_observer_config_moved, + observer_commands_tx_moved, + observer_commands_rx, + observer_events_tx.clone(), + observer_sidecar, + stacks_block_pool_seed, + context_cloned.clone(), + ); + match hiro_system_kit::nestable_block_on(future) { + Ok(_) => {} + Err(e) => { + if let Some(tx) = observer_events_tx { + context_cloned.try_log(|logger| { + slog::crit!( + logger, + "Chainhook event observer thread failed with error: {e}", + ) + }); + let _ = tx.send(ObserverEvent::Terminate); + } + } + } + }) + .expect("unable to spawn thread"); ctx.try_log(|logger| { slog::info!( @@ -707,7 +737,7 @@ pub fn gather_proofs<'a>( for transaction in transactions.iter() { if !proofs.contains_key(&transaction.transaction_identifier) { ctx.try_log(|logger| { - slog::info!( + slog::debug!( logger, "Collecting proof for transaction {}", transaction.transaction_identifier.hash @@ -722,7 +752,7 @@ pub fn gather_proofs<'a>( proofs.insert(&transaction.transaction_identifier, proof); } Err(e) => { - ctx.try_log(|logger| slog::error!(logger, "{e}")); + ctx.try_log(|logger| slog::warn!(logger, "{e}")); } } } @@ -758,38 +788,36 @@ pub async fn start_observer_commands_handler( let command = match observer_commands_rx.recv() { Ok(cmd) => cmd, Err(e) => { - if let Some(ref tx) = observer_events_tx { - let _ = tx.send(ObserverEvent::Error(format!("Channel error: {:?}", e))); - } - continue; + ctx.try_log(|logger| { + slog::crit!(logger, "Error: broken channel {}", e.to_string()) + }); + break; } }; match command { ObserverCommand::Terminate => { - ctx.try_log(|logger| slog::info!(logger, "Handling Termination command")); - if let Some(ingestion_shutdown) = ingestion_shutdown { - ingestion_shutdown.notify(); - } - if let Some(ref tx) = observer_events_tx { - let _ = tx.send(ObserverEvent::Info("Terminating event observer".into())); - let _ = tx.send(ObserverEvent::Terminate); - } break; } ObserverCommand::ProcessBitcoinBlock(mut block_data) => { let block_hash = block_data.hash.to_string(); + let mut attempts = 0; + let max_attempts = 10; let block = loop { match standardize_bitcoin_block( block_data.clone(), &config.bitcoin_network, &ctx, ) { - Ok(block) => break block, - Err((e, retry)) => { + Ok(block) => break Some(block), + Err((e, refetch_block)) => { + attempts += 1; + if attempts > max_attempts { + break None; + } ctx.try_log(|logger| { - slog::error!(logger, "Error standardizing block: {}", e) + slog::warn!(logger, "Error standardizing block: {}", e) }); - if retry { + if refetch_block { block_data = match download_and_parse_block_with_retry( &http_client, &block_hash, @@ -814,7 +842,16 @@ pub async fn start_observer_commands_handler( } }; }; - + let Some(block) = block else { + ctx.try_log(|logger| { + slog::crit!( + logger, + "Could not process bitcoin block after {} attempts.", + attempts + ) + }); + break; + }; prometheus_monitoring.btc_metrics_ingest_block(block.block_identifier.index); bitcoin_block_store.insert( @@ -1095,10 +1132,16 @@ pub async fn start_observer_commands_handler( )); } for chainhook_to_trigger in chainhooks_to_trigger.into_iter() { + let predicate_uuid = &chainhook_to_trigger.chainhook.uuid; match handle_bitcoin_hook_action(chainhook_to_trigger, &proofs) { Err(e) => { ctx.try_log(|logger| { - slog::error!(logger, "unable to handle action {}", e) + slog::warn!( + logger, + "unable to handle action for predicate {}: {}", + predicate_uuid, + e + ) }); } Ok(BitcoinChainhookOccurrence::Http(request, data)) => { @@ -1106,7 +1149,7 @@ pub async fn start_observer_commands_handler( } Ok(BitcoinChainhookOccurrence::File(_path, _bytes)) => { ctx.try_log(|logger| { - slog::info!(logger, "Writing to disk not supported in server mode") + slog::warn!(logger, "Writing to disk not supported in server mode") }) } Ok(BitcoinChainhookOccurrence::Data(payload)) => { @@ -1125,21 +1168,20 @@ pub async fn start_observer_commands_handler( }); for hook_uuid in hooks_ids_to_deregister.iter() { - if let Some(chainhook) = chainhook_store + if chainhook_store .predicates .deregister_bitcoin_hook(hook_uuid.clone()) + .is_some() { prometheus_monitoring.btc_metrics_deregister_predicate(); - - if let Some(ref tx) = observer_events_tx { - let _ = tx.send(ObserverEvent::PredicateDeregistered( - ChainhookSpecification::Bitcoin(chainhook), - )); - } + } + if let Some(ref tx) = observer_events_tx { + let _ = tx.send(ObserverEvent::PredicateDeregistered(hook_uuid.clone())); } } for (request, data) in requests.into_iter() { + // todo: need to handle failure case - we should be setting interrupted status: https://github.com/hirosystems/chainhook/issues/523 if send_request(request, 3, 1, &ctx).await.is_ok() { if let Some(ref tx) = observer_events_tx { let _ = tx.send(ObserverEvent::BitcoinPredicateTriggered(data)); @@ -1264,10 +1306,16 @@ pub async fn start_observer_commands_handler( } let proofs = HashMap::new(); for chainhook_to_trigger in chainhooks_to_trigger.into_iter() { + let predicate_uuid = &chainhook_to_trigger.chainhook.uuid; match handle_stacks_hook_action(chainhook_to_trigger, &proofs, &ctx) { Err(e) => { ctx.try_log(|logger| { - slog::error!(logger, "unable to handle action {}", e) + slog::warn!( + logger, + "unable to handle action for predicate {}: {}", + predicate_uuid, + e + ) }); } Ok(StacksChainhookOccurrence::Http(request)) => { @@ -1275,7 +1323,7 @@ pub async fn start_observer_commands_handler( } Ok(StacksChainhookOccurrence::File(_path, _bytes)) => { ctx.try_log(|logger| { - slog::info!(logger, "Writing to disk not supported in server mode") + slog::warn!(logger, "Writing to disk not supported in server mode") }) } Ok(StacksChainhookOccurrence::Data(payload)) => { @@ -1287,24 +1335,22 @@ pub async fn start_observer_commands_handler( } for hook_uuid in hooks_ids_to_deregister.iter() { - if let Some(chainhook) = chainhook_store + if chainhook_store .predicates .deregister_stacks_hook(hook_uuid.clone()) + .is_some() { prometheus_monitoring.stx_metrics_deregister_predicate(); - - if let Some(ref tx) = observer_events_tx { - let _ = tx.send(ObserverEvent::PredicateDeregistered( - ChainhookSpecification::Stacks(chainhook), - )); - } + } + if let Some(ref tx) = observer_events_tx { + let _ = tx.send(ObserverEvent::PredicateDeregistered(hook_uuid.clone())); } } for request in requests.into_iter() { // todo(lgalabru): collect responses for reporting ctx.try_log(|logger| { - slog::info!( + slog::debug!( logger, "Dispatching request from stacks chainhook {:?}", request @@ -1327,7 +1373,7 @@ pub async fn start_observer_commands_handler( } ObserverCommand::NotifyBitcoinTransactionProxied => { ctx.try_log(|logger| { - slog::info!(logger, "Handling NotifyBitcoinTransactionProxied command") + slog::debug!(logger, "Handling NotifyBitcoinTransactionProxied command") }); if let Some(ref tx) = observer_events_tx { let _ = tx.send(ObserverEvent::NotifyBitcoinTransactionProxied); @@ -1343,14 +1389,13 @@ pub async fn start_observer_commands_handler( Ok(spec) => spec, Err(e) => { ctx.try_log(|logger| { - slog::error!( + slog::warn!( logger, "Unable to register new chainhook spec: {}", e.to_string() ) }); - panic!("Unable to register new chainhook spec: {}", e.to_string()); - //continue; + continue; } }; @@ -1363,11 +1408,15 @@ pub async fn start_observer_commands_handler( } }; - ctx.try_log(|logger| slog::info!(logger, "Registering chainhook {}", spec.uuid(),)); + ctx.try_log( + |logger| slog::debug!(logger, "Registering chainhook {}", spec.uuid(),), + ); if let Some(ref tx) = observer_events_tx { let _ = tx.send(ObserverEvent::PredicateRegistered(spec.clone())); } else { - ctx.try_log(|logger| slog::info!(logger, "Enabling Predicate {}", spec.uuid())); + ctx.try_log(|logger| { + slog::debug!(logger, "Enabling Predicate {}", spec.uuid()) + }); chainhook_store.predicates.enable_specification(&mut spec); } } @@ -1382,15 +1431,19 @@ pub async fn start_observer_commands_handler( ctx.try_log(|logger| { slog::info!(logger, "Handling DeregisterStacksPredicate command") }); - let hook = chainhook_store.predicates.deregister_stacks_hook(hook_uuid); - - prometheus_monitoring.stx_metrics_deregister_predicate(); + let hook = chainhook_store + .predicates + .deregister_stacks_hook(hook_uuid.clone()); - if let (Some(tx), Some(hook)) = (&observer_events_tx, hook) { - let _ = tx.send(ObserverEvent::PredicateDeregistered( - ChainhookSpecification::Stacks(hook), - )); - } + if hook.is_some() { + // on startup, only the predicates in the `chainhook_store` are added to the monitoring count, + // so only those that we find in the store should be removed + prometheus_monitoring.stx_metrics_deregister_predicate(); + }; + // event if the predicate wasn't in the `chainhook_store`, propogate this event to delete from redis + if let Some(tx) = &observer_events_tx { + let _ = tx.send(ObserverEvent::PredicateDeregistered(hook_uuid)); + }; } ObserverCommand::DeregisterBitcoinPredicate(hook_uuid) => { ctx.try_log(|logger| { @@ -1398,15 +1451,17 @@ pub async fn start_observer_commands_handler( }); let hook = chainhook_store .predicates - .deregister_bitcoin_hook(hook_uuid); + .deregister_bitcoin_hook(hook_uuid.clone()); - prometheus_monitoring.btc_metrics_deregister_predicate(); - - if let (Some(tx), Some(hook)) = (&observer_events_tx, hook) { - let _ = tx.send(ObserverEvent::PredicateDeregistered( - ChainhookSpecification::Bitcoin(hook), - )); - } + if hook.is_some() { + // on startup, only the predicates in the `chainhook_store` are added to the monitoring count, + // so only those that we find in the store should be removed + prometheus_monitoring.btc_metrics_deregister_predicate(); + }; + // event if the predicate wasn't in the `chainhook_store`, propogate this event to delete from redis + if let Some(tx) = &observer_events_tx { + let _ = tx.send(ObserverEvent::PredicateDeregistered(hook_uuid)); + }; } ObserverCommand::ExpireStacksPredicate(HookExpirationData { hook_uuid, @@ -1430,8 +1485,23 @@ pub async fn start_observer_commands_handler( } } } + terminate(ingestion_shutdown, observer_events_tx, &ctx); Ok(()) } +fn terminate( + ingestion_shutdown: Option, + observer_events_tx: Option>, + ctx: &Context, +) { + ctx.try_log(|logger| slog::info!(logger, "Handling Termination command")); + if let Some(ingestion_shutdown) = ingestion_shutdown { + ingestion_shutdown.notify(); + } + if let Some(ref tx) = observer_events_tx { + let _ = tx.send(ObserverEvent::Info("Terminating event observer".into())); + let _ = tx.send(ObserverEvent::Terminate); + } +} #[cfg(test)] pub mod tests; diff --git a/components/chainhook-sdk/src/observer/tests/mod.rs b/components/chainhook-sdk/src/observer/tests/mod.rs index 7b678d2ce..053360e10 100644 --- a/components/chainhook-sdk/src/observer/tests/mod.rs +++ b/components/chainhook-sdk/src/observer/tests/mod.rs @@ -498,10 +498,7 @@ fn test_stacks_chainhook_register_deregister() { )); assert!(match observer_events_rx.recv() { Ok(ObserverEvent::PredicateDeregistered(deregistered_chainhook)) => { - assert_eq!( - ChainhookSpecification::Stacks(chainhook), - deregistered_chainhook - ); + assert_eq!(chainhook.uuid, deregistered_chainhook); true } _ => false, @@ -692,7 +689,7 @@ fn test_stacks_chainhook_auto_deregister() { // Should signal that a hook was deregistered assert!(match observer_events_rx.recv() { Ok(ObserverEvent::PredicateDeregistered(deregistered_hook)) => { - assert_eq!(deregistered_hook.uuid(), chainhook.uuid); + assert_eq!(deregistered_hook, chainhook.uuid); true } _ => false, @@ -858,10 +855,7 @@ fn test_bitcoin_chainhook_register_deregister() { )); assert!(match observer_events_rx.recv() { Ok(ObserverEvent::PredicateDeregistered(deregistered_chainhook)) => { - assert_eq!( - ChainhookSpecification::Bitcoin(chainhook), - deregistered_chainhook - ); + assert_eq!(chainhook.uuid, deregistered_chainhook); true } _ => false, @@ -1069,7 +1063,7 @@ fn test_bitcoin_chainhook_auto_deregister() { // Should signal that a hook was deregistered assert!(match observer_events_rx.recv() { Ok(ObserverEvent::PredicateDeregistered(deregistered_hook)) => { - assert_eq!(deregistered_hook.uuid(), chainhook.uuid); + assert_eq!(deregistered_hook, chainhook.uuid); true } _ => false, diff --git a/components/chainhook-sdk/src/utils/mod.rs b/components/chainhook-sdk/src/utils/mod.rs index 7f88e541c..541b6f352 100644 --- a/components/chainhook-sdk/src/utils/mod.rs +++ b/components/chainhook-sdk/src/utils/mod.rs @@ -164,7 +164,7 @@ pub async fn send_request( let err_msg = match request_builder.send().await { Ok(res) => { if res.status().is_success() { - ctx.try_log(|logger| slog::info!(logger, "Trigger {} successful", res.url())); + ctx.try_log(|logger| slog::debug!(logger, "Trigger {} successful", res.url())); return Ok(()); } else { retry += 1; From fb49e28d3621a0db8d725a66985be3f18d99abee Mon Sep 17 00:00:00 2001 From: Micaiah Reid Date: Wed, 27 Mar 2024 15:52:00 -0400 Subject: [PATCH 10/12] fix: enable debug logs in release mode (#537) --- Cargo.lock | 144 ++++++++++++++-------------- components/chainhook-cli/Cargo.toml | 4 +- components/chainhook-sdk/Cargo.toml | 6 +- 3 files changed, 75 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c23a4808b..f490053f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -138,6 +147,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base-x" version = "0.2.11" @@ -372,9 +396,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.83" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" dependencies = [ "jobserver", "libc", @@ -423,7 +447,7 @@ dependencies = [ "flume", "futures-util", "hex", - "hiro-system-kit 0.3.1", + "hiro-system-kit 0.3.2", "num_cpus", "rand 0.8.5", "redis", @@ -458,7 +482,7 @@ dependencies = [ "futures", "fxhash", "hex", - "hiro-system-kit 0.3.1", + "hiro-system-kit 0.3.2", "hyper", "lazy_static", "miniscript", @@ -1333,6 +1357,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + [[package]] name = "glob" version = "0.3.1" @@ -1457,9 +1487,9 @@ dependencies = [ [[package]] name = "hiro-system-kit" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a96596d2b33489f33f79b2df6f340dbbd2baba05a251715bb84661d33bf1c9" +checksum = "cc0029b908b021c2170ce55e1146fc85f347ebd2c3c21af7207c1d2d8c28982b" dependencies = [ "ansi_term", "atty", @@ -1575,7 +1605,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.10", "tokio", "tower-service", "tracing", @@ -2004,7 +2034,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", - "log", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -2103,6 +2132,15 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + [[package]] name = "okapi" version = "0.7.0-rc.1" @@ -2825,6 +2863,12 @@ dependencies = [ "smallvec 1.11.2", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -3381,6 +3425,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "spin" version = "0.9.8" @@ -3872,33 +3926,32 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.24.2" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ - "autocfg", + "backtrace", "bytes", "libc", - "memchr", "mio", "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.6", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.43", ] [[package]] @@ -4349,21 +4402,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -4412,12 +4450,6 @@ dependencies = [ "windows_x86_64_msvc 0.52.0", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -4430,12 +4462,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -4448,12 +4474,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -4466,12 +4486,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -4484,12 +4498,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -4502,12 +4510,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -4520,12 +4522,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" diff --git a/components/chainhook-cli/Cargo.toml b/components/chainhook-cli/Cargo.toml index f5ce0c451..391cc1263 100644 --- a/components/chainhook-cli/Cargo.toml +++ b/components/chainhook-cli/Cargo.toml @@ -18,7 +18,7 @@ rand = "0.8.5" chainhook-sdk = { version = "0.12.1", default-features = false, features = [ "zeromq", ], path = "../chainhook-sdk" } -hiro-system-kit = "0.3.1" +hiro-system-kit = "0.3.2" # clarinet-files = { path = "../../../clarinet/components/clarinet-files" } # hiro-system-kit = { path = "../../../clarinet/components/hiro-system-kit" } clap = { version = "3.2.23", features = ["derive"], optional = true } @@ -31,7 +31,7 @@ reqwest = { version = "0.11", default-features = false, features = [ "json", "rustls-tls", ] } -tokio = { version = "=1.24", features = ["full"] } +tokio = { version = "1.35.1", features = ["full"] } futures-util = "0.3.24" flate2 = "1.0.24" tar = "0.4.38" diff --git a/components/chainhook-sdk/Cargo.toml b/components/chainhook-sdk/Cargo.toml index b8789ae2f..44456add8 100644 --- a/components/chainhook-sdk/Cargo.toml +++ b/components/chainhook-sdk/Cargo.toml @@ -13,7 +13,7 @@ serde_json = { version = "1", features = ["arbitrary_precision"] } serde-hex = "0.1.0" serde_derive = "1" stacks-rpc-client = "2" -hiro-system-kit = { version = "0.3.1", optional = true } +hiro-system-kit = { version = "0.3.2", optional = true } # stacks-rpc-client = { version = "1", path = "../../../clarinet/components/stacks-rpc-client" } # hiro-system-kit = { version = "0.1.0", path = "../../../clarinet/components/hiro-system-kit" } chainhook-types = { version = "1.3.3", path = "../chainhook-types-rs" } @@ -26,7 +26,7 @@ reqwest = { version = "0.11", default-features = false, features = [ "json", "rustls-tls", ] } -tokio = { version = "1.24", features = ["full"] } +tokio = { version = "1.35.1", features = ["full"] } base58 = "0.2.0" schemars = { version = "0.8.10", git = "https://github.com/hirosystems/schemars.git", branch = "feat-chainhook-fixes" } crossbeam-channel = "0.5.6" @@ -50,4 +50,4 @@ test-case = "3.1.0" default = ["hiro-system-kit/log"] zeromq = ["zmq"] debug = ["hiro-system-kit/debug"] -release = ["hiro-system-kit/debug"] +release = ["hiro-system-kit/release_debug", "hiro-system-kit/full_log_level_prefix"] From c4ed30db957853a018e7ee59474bbd6b32306eba Mon Sep 17 00:00:00 2001 From: Hugo C <911307+hugocaillard@users.noreply.github.com> Date: Wed, 27 Mar 2024 21:39:06 +0100 Subject: [PATCH 11/12] chore: update clarinet and clarity-vm (#475) ### Description Upgrade clarinet dependency --------- Co-authored-by: MicaiahReid --- Cargo.lock | 1574 +++++++++++++---- components/chainhook-cli/Cargo.toml | 4 +- components/chainhook-sdk/Cargo.toml | 6 +- .../src/chainhooks/stacks/mod.rs | 2 +- .../chainhook-sdk/src/indexer/stacks/mod.rs | 14 +- components/chainhook-sdk/src/observer/mod.rs | 2 +- components/chainhook-types-js/src/index.ts | 18 +- components/chainhook-types-rs/src/rosetta.rs | 1 + .../typescript/src/schemas/stacks/tx_kind.ts | 6 + .../components/chainhook-node.dockerfile | 4 +- 10 files changed, 1297 insertions(+), 334 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f490053f1..e37932259 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,15 +17,51 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array 0.14.7", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle 2.5.0", +] + [[package]] name = "ahash" -version = "0.7.7" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ - "getrandom 0.2.11", + "cfg-if", "once_cell", "version_check", + "zerocopy", ] [[package]] @@ -37,6 +73,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -61,6 +103,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "anyhow" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" + +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" + [[package]] name = "arc-swap" version = "1.6.0" @@ -82,6 +136,12 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + [[package]] name = "async-stream" version = "0.3.5" @@ -162,12 +222,6 @@ dependencies = [ "rustc-demangle", ] -[[package]] -name = "base-x" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" - [[package]] name = "base58" version = "0.1.0" @@ -210,6 +264,15 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bindgen" version = "0.64.0" @@ -301,13 +364,25 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" dependencies = [ - "block-padding 0.1.5", + "block-padding", "byte-tools", "byteorder", "generic-array 0.12.4", @@ -319,7 +394,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "block-padding 0.2.1", "generic-array 0.14.7", ] @@ -342,10 +416,19 @@ dependencies = [ ] [[package]] -name = "block-padding" -version = "0.2.1" +name = "bs58" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" + +[[package]] +name = "bs58" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] [[package]] name = "bumpalo" @@ -353,6 +436,12 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +[[package]] +name = "byte-slice-cast" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" + [[package]] name = "byte-tools" version = "0.3.1" @@ -438,7 +527,6 @@ dependencies = [ "chainhook-sdk", "clap 3.2.25", "clap_generate", - "clarity-vm", "criterion", "crossbeam-channel", "csv", @@ -529,6 +617,16 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clang-sys" version = "1.6.1" @@ -610,59 +708,48 @@ dependencies = [ ] [[package]] -name = "clarity-repl" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb0935839b51a01d7436b31420713e0a9698d2920cfa0493482c9dbb4fe0a696" +name = "clarity" +version = "2.3.0" +source = "git+https://github.com/stacks-network/stacks-core.git?branch=feat/clarity-wasm-next#35cce11f4a7e49f29dd1cf06680abfbd31099669" dependencies = [ - "ansi_term", - "atty", - "chrono", - "clarity-vm", - "getrandom 0.2.11", - "hiro-system-kit 0.1.0", + "hashbrown 0.14.3", "integer-sqrt", "lazy_static", + "rand 0.8.5", + "rand_chacha 0.3.1", "regex", - "reqwest", + "rusqlite", "serde", "serde_derive", "serde_json", - "sha2 0.10.8", - "sha3 0.9.1", + "serde_stacker", + "sha2-asm", + "slog", + "stacks-common", + "wasmtime", ] [[package]] -name = "clarity-vm" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90cebf93044798d7729008e9a467b16578aead7f0b3568219a2b20b421b683db" +name = "clarity-repl" +version = "2.4.1" +source = "git+https://github.com/hirosystems/clarinet.git#175103ecfbeb9d592640650d25947eeb3a7e3196" dependencies = [ + "ansi_term", + "atty", + "chrono", + "clarity", + "getrandom 0.2.11", + "hiro-system-kit 0.1.0", "integer-sqrt", "lazy_static", - "rand 0.7.3", - "rand_chacha 0.2.2", + "pox-locking", "regex", - "rstest", - "rstest_reuse", - "rusqlite", + "reqwest", "serde", "serde_derive", "serde_json", - "serde_stacker", - "sha2-asm", - "slog", - "stacks-common", - "time 0.2.27", -] - -[[package]] -name = "clear_on_drop" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38508a63f4979f0048febc9966fadbd48e5dab31fd0ec6a3f151bbf4a74f7423" -dependencies = [ - "cc", + "sha2 0.10.8", + "wsts", ] [[package]] @@ -676,10 +763,10 @@ dependencies = [ ] [[package]] -name = "const_fn" -version = "0.4.9" +name = "const-oid" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "cookie" @@ -688,7 +775,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7efb37c3e1ccb1ff97164ad95ac1606e8ccd35b3fa0a7d99a304c7f4a428cc24" dependencies = [ "percent-encoding", - "time 0.3.31", + "time", "version_check", ] @@ -708,6 +795,15 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "cpp_demangle" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaa953eaad386a53111e47172c2fedba671e5684c8dd601a5f474f4f118710f" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpufeatures" version = "0.2.11" @@ -717,6 +813,115 @@ dependencies = [ "libc", ] +[[package]] +name = "cranelift-bforest" +version = "0.102.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e7e56668d2263f92b691cb9e4a2fcb186ca0384941fe420484322fa559c3329" +dependencies = [ + "cranelift-entity", +] + +[[package]] +name = "cranelift-codegen" +version = "0.102.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a9ff61938bf11615f55b80361288c68865318025632ea73c65c0b44fa16283c" +dependencies = [ + "bumpalo", + "cranelift-bforest", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-control", + "cranelift-entity", + "cranelift-isle", + "gimli", + "hashbrown 0.14.3", + "log", + "regalloc2", + "smallvec 1.11.2", + "target-lexicon", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.102.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50656bf19e3d4a153b404ff835b8b59e924cfa3682ebe0d3df408994f37983f6" +dependencies = [ + "cranelift-codegen-shared", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.102.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388041deeb26109f1ea73c1812ea26bfd406c94cbce0bb5230aa44277e43b209" + +[[package]] +name = "cranelift-control" +version = "0.102.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39b7c512ffac527e5b5df9beae3d67ab85d07dca6d88942c16195439fedd1d3" +dependencies = [ + "arbitrary", +] + +[[package]] +name = "cranelift-entity" +version = "0.102.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb25f573701284fe2bcf88209d405342125df00764b396c923e11eafc94d892" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "cranelift-frontend" +version = "0.102.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57374fd11d72cf9ffb85ff64506ed831440818318f58d09f45b4185e5e9c376" +dependencies = [ + "cranelift-codegen", + "log", + "smallvec 1.11.2", + "target-lexicon", +] + +[[package]] +name = "cranelift-isle" +version = "0.102.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae769b235f6ea2f86623a3ff157cc04a4ff131dc9fe782c2ebd35f272043581e" + +[[package]] +name = "cranelift-native" +version = "0.102.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dc7bfb8f13a0526fe20db338711d9354729b861c336978380bb10f7f17dd207" +dependencies = [ + "cranelift-codegen", + "libc", + "target-lexicon", +] + +[[package]] +name = "cranelift-wasm" +version = "0.102.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c5f41a4af931b756be05af0dd374ce200aae2d52cea16b0beb07e8b52732c35" +dependencies = [ + "cranelift-codegen", + "cranelift-entity", + "cranelift-frontend", + "itertools", + "log", + "smallvec 1.11.2", + "wasmparser", + "wasmtime-types", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -840,6 +1045,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array 0.14.7", + "rand_core 0.6.4", "typenum", ] @@ -884,13 +1090,22 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "ctrlc" version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b467862cc8610ca6fc9a1532d7777cee0804e678ab45410897b9396495994a0b" dependencies = [ - "nix", + "nix 0.27.1", "windows-sys 0.52.0", ] @@ -908,6 +1123,34 @@ dependencies = [ "zeroize", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "platforms", + "rustc_version", + "subtle 2.5.0", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.43", +] + [[package]] name = "darling" version = "0.13.4" @@ -956,6 +1199,25 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid", +] + +[[package]] +name = "der" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.10" @@ -1038,6 +1300,16 @@ dependencies = [ "walkdir", ] +[[package]] +name = "directories-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339ee130d97a610ea5a5872d2bbb130fdf68884ff09d3028b81bec8a1ac23bbc" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -1059,29 +1331,36 @@ dependencies = [ "winapi", ] -[[package]] -name = "discard" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" - [[package]] name = "dyn-clone" version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "serde", + "signature", +] + [[package]] name = "ed25519-dalek" -version = "1.0.0-pre.3" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978710b352437433c97b2bff193f2fb1dfd58a093f863dd95e225a19baa599a2" +checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" dependencies = [ - "clear_on_drop", - "curve25519-dalek", - "rand 0.7.3", + "curve25519-dalek 4.1.2", + "ed25519", + "rand_core 0.6.4", "serde", - "sha2 0.8.2", + "sha2 0.10.8", + "subtle 2.5.0", + "zeroize", ] [[package]] @@ -1128,9 +1407,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] @@ -1139,6 +1424,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "fiat-crypto" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c007b1ae3abe1cb6f85a16305acd418b7ca6343b953633fee2b76d8f108b830f" + [[package]] name = "figment" version = "0.10.12" @@ -1165,6 +1456,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fixed-hash" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" +dependencies = [ + "byteorder", + "rand 0.8.5", + "rustc-hex", + "static_assertions", +] + [[package]] name = "flate2" version = "1.0.28" @@ -1203,6 +1506,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.30" @@ -1301,6 +1610,19 @@ dependencies = [ "byteorder", ] +[[package]] +name = "fxprof-processed-profile" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27d12c0aed7f1e24276a241aadc4cb8ea9f83000f34bc062b7cc2d51e3b0fabd" +dependencies = [ + "bitflags 2.4.1", + "debugid", + "fxhash", + "serde", + "serde_json", +] + [[package]] name = "generator" version = "0.7.5" @@ -1357,11 +1679,26 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug 0.3.0", + "polyval", +] + [[package]] name = "gimli" version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +dependencies = [ + "fallible-iterator 0.3.0", + "indexmap 2.1.0", + "stable_deref_trait", +] [[package]] name = "glob" @@ -1396,32 +1733,37 @@ checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" [[package]] name = "hashbrown" -version = "0.11.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" -dependencies = [ - "ahash", -] +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.12.3" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash", +] [[package]] name = "hashbrown" version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash", + "allocator-api2", + "serde", +] [[package]] name = "hashlink" -version = "0.7.0" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.11.2", + "hashbrown 0.14.3", ] [[package]] @@ -1475,12 +1817,10 @@ checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" [[package]] name = "hiro-system-kit" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69a7ca4bddaacbc8180886d378ad8c2b9f74217503002346719b57adbd83124" +source = "git+https://github.com/hirosystems/clarinet.git#175103ecfbeb9d592640650d25947eeb3a7e3196" dependencies = [ "ansi_term", "atty", - "futures", "lazy_static", "tokio", ] @@ -1554,6 +1894,15 @@ dependencies = [ "hmac 0.8.1", ] +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "http" version = "0.2.11" @@ -1649,6 +1998,12 @@ dependencies = [ "cc", ] +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + [[package]] name = "ident_case" version = "1.0.1" @@ -1665,6 +2020,26 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "impl-codec" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -1684,6 +2059,7 @@ checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", "hashbrown 0.14.3", + "serde", ] [[package]] @@ -1692,6 +2068,15 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array 0.14.7", +] + [[package]] name = "integer-sqrt" version = "0.1.5" @@ -1733,6 +2118,26 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +[[package]] +name = "ittapi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b996fe614c41395cdaedf3cf408a9534851090959d90d54a535f675550b64b1" +dependencies = [ + "anyhow", + "ittapi-sys", + "log", +] + +[[package]] +name = "ittapi-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5385394064fa2c886205dba02598013ce83d3e92d33dbdc0c52fe0e7bf4fc" +dependencies = [ + "cc", +] + [[package]] name = "jobserver" version = "0.1.27" @@ -1793,6 +2198,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + [[package]] name = "libc" version = "0.2.151" @@ -1901,9 +2312,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.24.2" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14" +checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa" dependencies = [ "cc", "pkg-config", @@ -1968,6 +2379,15 @@ dependencies = [ "libc", ] +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + [[package]] name = "matchers" version = "0.1.0" @@ -1989,6 +2409,33 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +[[package]] +name = "memfd" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" +dependencies = [ + "rustix", +] + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "memzero" version = "0.1.0" @@ -2067,6 +2514,19 @@ dependencies = [ "getrandom 0.2.11", ] +[[package]] +name = "nix" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" +dependencies = [ + "bitflags 1.3.2", + "cc", + "cfg-if", + "libc", + "memoffset 0.6.5", +] + [[package]] name = "nix" version = "0.27.1" @@ -2104,6 +2564,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-traits" version = "0.2.17" @@ -2138,6 +2604,9 @@ version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ + "crc32fast", + "hashbrown 0.14.3", + "indexmap 2.1.0", "memchr", ] @@ -2188,6 +2657,54 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "p256k1" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a40a031a559eb38c35a14096f21c366254501a06d41c4b327d2a7515d713a5b7" +dependencies = [ + "bitvec", + "bs58 0.4.0", + "cc", + "hex", + "itertools", + "num-traits", + "primitive-types", + "proc-macro2", + "quote", + "rand_core 0.6.4", + "rustfmt-wrapper", + "serde", + "sha2 0.10.8", + "syn 2.0.43", +] + +[[package]] +name = "parity-scale-codec" +version = "3.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "881331e34fa842a2fb61cc2db9643a8fedc615e47cfcc52597d1af0db9a7e8fe" +dependencies = [ + "arrayvec", + "bitvec", + "byte-slice-cast", + "impl-trait-for-tuples", + "parity-scale-codec-derive", + "serde", +] + +[[package]] +name = "parity-scale-codec-derive" +version = "3.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be30eaf4b0a9fba5336683b38de57bb86d179a35862ba6bfcf57625d006bde5b" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -2222,6 +2739,12 @@ dependencies = [ "subtle 2.5.0", ] +[[package]] +name = "paste" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" + [[package]] name = "pbkdf2" version = "0.12.2" @@ -2269,17 +2792,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "pest" -version = "2.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" -dependencies = [ - "memchr", - "thiserror", - "ucd-trie", -] - [[package]] name = "pin-project" version = "1.1.3" @@ -2312,12 +2824,28 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" +[[package]] +name = "platforms" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db23d408679286588f4d4644f965003d056e3dd5abcaaa938116871d7ce2fee7" + [[package]] name = "plotters" version = "0.3.5" @@ -2346,18 +2874,70 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "polynomial" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27abb6e4638dcecc65a92b50d7f1d87dd6dea987ba71db987b6bf881f4877e9d" +dependencies = [ + "num-traits", + "serde", +] + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug 0.3.0", + "universal-hash", +] + [[package]] name = "powerfmt" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "pox-locking" +version = "2.4.0" +source = "git+https://github.com/stacks-network/stacks-core.git?branch=feat/clarity-wasm-next#35cce11f4a7e49f29dd1cf06680abfbd31099669" +dependencies = [ + "clarity", + "slog", + "stacks-common", +] + [[package]] name = "ppv-lite86" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "primitive-types" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2" +dependencies = [ + "fixed-hash", + "impl-codec", + "uint", +] + +[[package]] +name = "proc-macro-crate" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8" +dependencies = [ + "toml_edit 0.20.7", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -2382,12 +2962,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" version = "1.0.71" @@ -2449,6 +3023,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.7.3" @@ -2596,15 +3176,28 @@ dependencies = [ ] [[package]] -name = "regex" -version = "1.10.2" +name = "regalloc2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "ad156d539c879b7a24a363a2016d77961786e71f48f2e2fc8302a92abd2429a6" dependencies = [ - "aho-corasick", - "memchr", - "regex-automata 0.4.3", - "regex-syntax 0.8.2", + "hashbrown 0.13.2", + "log", + "rustc-hash", + "slice-group-by", + "smallvec 1.11.2", +] + +[[package]] +name = "regex" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.3", + "regex-syntax 0.8.2", ] [[package]] @@ -2734,7 +3327,7 @@ dependencies = [ "serde_json", "state", "tempfile", - "time 0.3.31", + "time", "tokio", "tokio-stream", "tokio-util", @@ -2781,7 +3374,7 @@ dependencies = [ "smallvec 1.11.2", "stable-pattern", "state", - "time 0.3.31", + "time", "tokio", "uncased", ] @@ -2823,42 +3416,17 @@ dependencies = [ "librocksdb-sys", ] -[[package]] -name = "rstest" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2288c66aeafe3b2ed227c981f364f9968fa952ef0b30e84ada4486e7ee24d00a" -dependencies = [ - "cfg-if", - "proc-macro2", - "quote", - "rustc_version 0.4.0", - "syn 1.0.109", -] - -[[package]] -name = "rstest_reuse" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32c6cfaae58c048728261723a72b80a0aa9f3768e9a7da3b302a24d262525219" -dependencies = [ - "quote", - "rustc_version 0.3.3", - "syn 1.0.109", -] - [[package]] name = "rusqlite" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85127183a999f7db96d1a976a309eebbfb6ea3b0b400ddd8340190129de6eb7a" +checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a" dependencies = [ "bitflags 1.3.2", - "fallible-iterator", + "fallible-iterator 0.2.0", "fallible-streaming-iterator", "hashlink", "libsqlite3-sys", - "memchr", "serde_json", "smallvec 1.11.2", ] @@ -2876,30 +3444,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] -name = "rustc_version" -version = "0.2.3" +name = "rustc-hex" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] +checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" [[package]] name = "rustc_version" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 0.11.0", + "semver", ] [[package]] -name = "rustc_version" -version = "0.4.0" +name = "rustfmt-wrapper" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "f1adc9dfed5cc999077978cc7163b9282c5751c8d39827c4ea8c8c220ca5a440" dependencies = [ - "semver 1.0.20", + "serde", + "tempfile", + "thiserror", + "toml 0.8.8", + "toolchain_find", ] [[package]] @@ -3053,45 +3622,12 @@ dependencies = [ "cc", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser 0.7.0", -] - -[[package]] -name = "semver" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" -dependencies = [ - "semver-parser 0.10.2", -] - [[package]] name = "semver" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - -[[package]] -name = "semver-parser" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" -dependencies = [ - "pest", -] - [[package]] name = "serde" version = "1.0.193" @@ -3281,18 +3817,6 @@ dependencies = [ "cc", ] -[[package]] -name = "sha3" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f81199417d4e5de3f04b1e871023acea7389672c4135918f05aa9cbf2f2fa809" -dependencies = [ - "block-buffer 0.9.0", - "digest 0.9.0", - "keccak", - "opaque-debug 0.3.0", -] - [[package]] name = "sha3" version = "0.10.8" @@ -3327,6 +3851,15 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "slab" version = "0.4.9" @@ -3336,6 +3869,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "slice-group-by" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" + [[package]] name = "slog" version = "2.7.0" @@ -3373,7 +3912,7 @@ dependencies = [ "serde", "serde_json", "slog", - "time 0.3.31", + "time", ] [[package]] @@ -3397,7 +3936,7 @@ dependencies = [ "slog", "term", "thread_local", - "time 0.3.31", + "time", ] [[package]] @@ -3444,6 +3983,22 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sptr" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" + [[package]] name = "stable-pattern" version = "0.1.0" @@ -3453,6 +4008,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "stacker" version = "0.1.15" @@ -3469,16 +4030,17 @@ dependencies = [ [[package]] name = "stacks-common" version = "0.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c887baefb047fb7cd1c82d5a62e1deaca4b9ea9c701f965407db6c39a3fc902f" +source = "git+https://github.com/stacks-network/stacks-core.git?branch=feat/clarity-wasm-next#35cce11f4a7e49f29dd1cf06680abfbd31099669" dependencies = [ "chrono", - "curve25519-dalek", + "curve25519-dalek 2.0.0", "ed25519-dalek", + "hashbrown 0.14.3", "lazy_static", "libc", + "nix 0.23.2", "percent-encoding", - "rand 0.7.3", + "rand 0.8.5", "ripemd", "rusqlite", "secp256k1 0.24.3", @@ -3486,18 +4048,19 @@ dependencies = [ "serde_derive", "serde_json", "sha2 0.10.8", - "sha3 0.10.8", + "sha3", "slog", "slog-json", "slog-term", - "time 0.2.27", + "time", + "winapi", + "wsts", ] [[package]] name = "stacks-rpc-client" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab91f2053d36c8caedfdb2b98ace2cf852aba6d27c40ae92e2b35d946504a6e" +version = "2.4.1" +source = "git+https://github.com/hirosystems/clarinet.git#175103ecfbeb9d592640650d25947eeb3a7e3196" dependencies = [ "clarity-repl", "hmac 0.12.1", @@ -3511,15 +4074,6 @@ dependencies = [ "tiny-hderive", ] -[[package]] -name = "standback" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" -dependencies = [ - "version_check", -] - [[package]] name = "state" version = "0.5.3" @@ -3530,53 +4084,10 @@ dependencies = [ ] [[package]] -name = "stdweb" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" -dependencies = [ - "discard", - "rustc_version 0.2.3", - "stdweb-derive", - "stdweb-internal-macros", - "stdweb-internal-runtime", - "wasm-bindgen", -] - -[[package]] -name = "stdweb-derive" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "serde_derive", - "syn 1.0.109", -] - -[[package]] -name = "stdweb-internal-macros" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" -dependencies = [ - "base-x", - "proc-macro2", - "quote", - "serde", - "serde_derive", - "serde_json", - "sha1", - "syn 1.0.109", -] - -[[package]] -name = "stdweb-internal-runtime" -version = "0.1.5" +name = "static_assertions" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strsim" @@ -3680,6 +4191,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tar" version = "0.4.40" @@ -3819,33 +4336,19 @@ dependencies = [ [[package]] name = "time" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" -dependencies = [ - "const_fn", - "libc", - "standback", - "stdweb", - "time-macros 0.1.1", - "version_check", - "winapi", -] - -[[package]] -name = "time" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", "itoa", "libc", + "num-conv", "num_threads", "powerfmt", "serde", "time-core", - "time-macros 0.2.16", + "time-macros", ] [[package]] @@ -3856,36 +4359,14 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" -dependencies = [ - "proc-macro-hack", - "time-macros-impl", -] - -[[package]] -name = "time-macros" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" dependencies = [ + "num-conv", "time-core", ] -[[package]] -name = "time-macros-impl" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" -dependencies = [ - "proc-macro-hack", - "proc-macro2", - "quote", - "standback", - "syn 1.0.109", -] - [[package]] name = "tiny-hderive" version = "0.3.0" @@ -4008,7 +4489,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_edit 0.21.0", ] [[package]] @@ -4020,6 +4501,17 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_edit" +version = "0.20.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" +dependencies = [ + "indexmap 2.1.0", + "toml_datetime", + "winnow", +] + [[package]] name = "toml_edit" version = "0.21.0" @@ -4033,6 +4525,19 @@ dependencies = [ "winnow", ] +[[package]] +name = "toolchain_find" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc8c9a7f0a2966e1acdaf0461023d0b01471eeead645370cf4c3f5cff153f2a" +dependencies = [ + "home", + "once_cell", + "regex", + "semver", + "walkdir", +] + [[package]] name = "tower-service" version = "0.3.2" @@ -4122,10 +4627,16 @@ dependencies = [ ] [[package]] -name = "ucd-trie" -version = "0.1.6" +name = "uint" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] [[package]] name = "uncased" @@ -4176,6 +4687,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle 2.5.0", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -4324,6 +4845,24 @@ version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +[[package]] +name = "wasm-encoder" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "822b645bf4f2446b949776ffca47e2af60b167209ffb70814ef8779d299cd421" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-encoder" +version = "0.202.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfd106365a7f5f7aa3c1916a98cbb3ad477f5ff96ddb130285a91c6e7429e67a" +dependencies = [ + "leb128", +] + [[package]] name = "wasm-streams" version = "0.3.0" @@ -4337,6 +4876,322 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.116.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a58e28b80dd8340cb07b8242ae654756161f6fc8d0038123d679b7b99964fa50" +dependencies = [ + "indexmap 2.1.0", + "semver", +] + +[[package]] +name = "wasmtime" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "642e12d108e800215263e3b95972977f473957923103029d7d617db701d67ba4" +dependencies = [ + "anyhow", + "async-trait", + "bincode", + "bumpalo", + "cfg-if", + "fxprof-processed-profile", + "indexmap 2.1.0", + "libc", + "log", + "object", + "once_cell", + "paste", + "psm", + "rayon", + "serde", + "serde_derive", + "serde_json", + "target-lexicon", + "wasm-encoder 0.36.2", + "wasmparser", + "wasmtime-cache", + "wasmtime-component-macro", + "wasmtime-cranelift", + "wasmtime-environ", + "wasmtime-fiber", + "wasmtime-jit", + "wasmtime-runtime", + "wat", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-asm-macros" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beada8bb15df52503de0a4c58de4357bfd2f96d9a44a6e547bad11efdd988b47" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "wasmtime-cache" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aba5bf44d044d25892c03fb3534373936ee204141ff92bac8297787ac7f22318" +dependencies = [ + "anyhow", + "base64 0.21.5", + "bincode", + "directories-next", + "log", + "rustix", + "serde", + "serde_derive", + "sha2 0.10.8", + "toml 0.5.11", + "windows-sys 0.48.0", + "zstd", +] + +[[package]] +name = "wasmtime-component-macro" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ccba556991465cca68d5a54769684bcf489fb532059da55105f851642d52c1" +dependencies = [ + "anyhow", + "proc-macro2", + "quote", + "syn 2.0.43", + "wasmtime-component-util", + "wasmtime-wit-bindgen", + "wit-parser", +] + +[[package]] +name = "wasmtime-component-util" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05492a177a6006cb73f034d6e9a6fad6da55b23c4398835cb0012b5fa51ecf67" + +[[package]] +name = "wasmtime-cranelift" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe2e7532f1d6adbcc57e69bb6a7c503f0859076d07a9b4b6aabe8021ff8a05fd" +dependencies = [ + "anyhow", + "cfg-if", + "cranelift-codegen", + "cranelift-control", + "cranelift-entity", + "cranelift-frontend", + "cranelift-native", + "cranelift-wasm", + "gimli", + "log", + "object", + "target-lexicon", + "thiserror", + "wasmparser", + "wasmtime-cranelift-shared", + "wasmtime-environ", + "wasmtime-versioned-export-macros", +] + +[[package]] +name = "wasmtime-cranelift-shared" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c98d5378a856cbf058d36278627dfabf0ed68a888142958c7ae8e6af507dafa" +dependencies = [ + "anyhow", + "cranelift-codegen", + "cranelift-control", + "cranelift-native", + "gimli", + "object", + "target-lexicon", + "wasmtime-environ", +] + +[[package]] +name = "wasmtime-environ" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6d33a9f421da810a070cd56add9bc51f852bd66afbb8b920489d6242f15b70e" +dependencies = [ + "anyhow", + "cranelift-entity", + "gimli", + "indexmap 2.1.0", + "log", + "object", + "serde", + "serde_derive", + "target-lexicon", + "thiserror", + "wasmparser", + "wasmtime-types", +] + +[[package]] +name = "wasmtime-fiber" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404741f4c6d7f4e043be2e8b466406a2aee289ccdba22bf9eba6399921121b97" +dependencies = [ + "anyhow", + "cc", + "cfg-if", + "rustix", + "wasmtime-asm-macros", + "wasmtime-versioned-export-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-jit" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d0994a86d6dca5f7d9740d7f2bd0568be06d2014a550361dc1c397d289d81ef" +dependencies = [ + "addr2line", + "anyhow", + "bincode", + "cfg-if", + "cpp_demangle", + "gimli", + "ittapi", + "log", + "object", + "rustc-demangle", + "rustix", + "serde", + "serde_derive", + "target-lexicon", + "wasmtime-environ", + "wasmtime-jit-debug", + "wasmtime-jit-icache-coherence", + "wasmtime-runtime", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-jit-debug" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e0c4b74e606d1462d648631d5bc328e3d5b14e7f9d3ff93bc6db062fb8c5cd8" +dependencies = [ + "object", + "once_cell", + "rustix", + "wasmtime-versioned-export-macros", +] + +[[package]] +name = "wasmtime-jit-icache-coherence" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3090a69ba1476979e090aa7ed4bc759178bafdb65b22f98b9ba24fc6e7e578d5" +dependencies = [ + "cfg-if", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-runtime" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b993ac8380385ed67bf71b51b9553edcf1ab0801b78a805a067de581b9a3e88a" +dependencies = [ + "anyhow", + "cc", + "cfg-if", + "indexmap 2.1.0", + "libc", + "log", + "mach", + "memfd", + "memoffset 0.9.1", + "paste", + "rand 0.8.5", + "rustix", + "sptr", + "wasm-encoder 0.36.2", + "wasmtime-asm-macros", + "wasmtime-environ", + "wasmtime-fiber", + "wasmtime-jit-debug", + "wasmtime-versioned-export-macros", + "wasmtime-wmemcheck", + "windows-sys 0.48.0", +] + +[[package]] +name = "wasmtime-types" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5778112fcab2dc3d4371f4203ab8facf0c453dd94312b0a88dd662955e64e0" +dependencies = [ + "cranelift-entity", + "serde", + "serde_derive", + "thiserror", + "wasmparser", +] + +[[package]] +name = "wasmtime-versioned-export-macros" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50f51f8d79bfd2aa8e9d9a0ae7c2d02b45fe412e62ff1b87c0c81b07c738231" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.43", +] + +[[package]] +name = "wasmtime-wit-bindgen" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b804dfd3d0c0d6d37aa21026fe7772ba1a769c89ee4f5c4f13b82d91d75216f" +dependencies = [ + "anyhow", + "heck 0.4.1", + "indexmap 2.1.0", + "wit-parser", +] + +[[package]] +name = "wasmtime-wmemcheck" +version = "15.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6060bc082cc32d9a45587c7640e29e3c7b89ada82677ac25d87850aaccb368" + +[[package]] +name = "wast" +version = "202.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbcb11204515c953c9b42ede0a46a1c5e17f82af05c4fae201a8efff1b0f4fe" +dependencies = [ + "bumpalo", + "leb128", + "memchr", + "unicode-width", + "wasm-encoder 0.202.0", +] + +[[package]] +name = "wat" +version = "1.202.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de4b15a47135c56a3573406e9977b9518787a6154459b4842a9b9d3d1684848" +dependencies = [ + "wast", +] + [[package]] name = "web-sys" version = "0.3.66" @@ -4553,6 +5408,54 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "wit-parser" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "316b36a9f0005f5aa4b03c39bc3728d045df136f8c13a73b7db4510dec725e08" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.1.0", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", +] + +[[package]] +name = "wsts" +version = "8.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "467aa8e40ed0277d19922fd0e7357c16552cb900e5138f61a48ac23c4b7878e0" +dependencies = [ + "aes-gcm", + "bs58 0.5.1", + "hashbrown 0.14.3", + "hex", + "num-traits", + "p256k1", + "polynomial", + "primitive-types", + "rand_core 0.6.4", + "serde", + "sha2 0.10.8", + "thiserror", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "xattr" version = "1.1.3" @@ -4576,6 +5479,26 @@ version = "1.0.0-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1367295b8f788d371ce2dbc842c7b709c73ee1364d30351dd300ec2203b12377" +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.43", +] + [[package]] name = "zeroize" version = "1.7.0" @@ -4613,3 +5536,32 @@ dependencies = [ "system-deps", "zeromq-src", ] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.10+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/components/chainhook-cli/Cargo.toml b/components/chainhook-cli/Cargo.toml index 391cc1263..d7305cdd5 100644 --- a/components/chainhook-cli/Cargo.toml +++ b/components/chainhook-cli/Cargo.toml @@ -18,8 +18,7 @@ rand = "0.8.5" chainhook-sdk = { version = "0.12.1", default-features = false, features = [ "zeromq", ], path = "../chainhook-sdk" } -hiro-system-kit = "0.3.2" -# clarinet-files = { path = "../../../clarinet/components/clarinet-files" } +hiro-system-kit = "0.3.1" # hiro-system-kit = { path = "../../../clarinet/components/hiro-system-kit" } clap = { version = "3.2.23", features = ["derive"], optional = true } clap_generate = { version = "3.0.3", optional = true } @@ -52,7 +51,6 @@ features = ["lz4", "snappy"] [dev-dependencies] criterion = "0.3" redis = "0.21.5" -clarity-vm = "=2.1.1" hex = "0.4.3" test-case = "3.1.0" serial_test = "2.0.0" diff --git a/components/chainhook-sdk/Cargo.toml b/components/chainhook-sdk/Cargo.toml index 44456add8..860f511d2 100644 --- a/components/chainhook-sdk/Cargo.toml +++ b/components/chainhook-sdk/Cargo.toml @@ -12,8 +12,8 @@ serde = { version = "1", features = ["rc"] } serde_json = { version = "1", features = ["arbitrary_precision"] } serde-hex = "0.1.0" serde_derive = "1" -stacks-rpc-client = "2" -hiro-system-kit = { version = "0.3.2", optional = true } +stacks-rpc-client = { version = "2", git = "https://github.com/hirosystems/clarinet.git" } +hiro-system-kit = { version = "0.3.1", optional = true } # stacks-rpc-client = { version = "1", path = "../../../clarinet/components/stacks-rpc-client" } # hiro-system-kit = { version = "0.1.0", path = "../../../clarinet/components/hiro-system-kit" } chainhook-types = { version = "1.3.3", path = "../chainhook-types-rs" } @@ -50,4 +50,4 @@ test-case = "3.1.0" default = ["hiro-system-kit/log"] zeromq = ["zmq"] debug = ["hiro-system-kit/debug"] -release = ["hiro-system-kit/release_debug", "hiro-system-kit/full_log_level_prefix"] +release = ["hiro-system-kit/release"] diff --git a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs index 21dfbb048..ca4069552 100644 --- a/components/chainhook-sdk/src/chainhooks/stacks/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/stacks/mod.rs @@ -12,7 +12,7 @@ use hiro_system_kit::slog; use regex::Regex; use reqwest::{Client, Method}; use serde_json::Value as JsonValue; -use stacks_rpc_client::clarity::stacks_common::codec::StacksMessageCodec; +use stacks_rpc_client::clarity::codec::StacksMessageCodec; use stacks_rpc_client::clarity::vm::types::{CharType, SequenceData, Value as ClarityValue}; use std::collections::{BTreeMap, HashMap}; use std::io::Cursor; diff --git a/components/chainhook-sdk/src/indexer/stacks/mod.rs b/components/chainhook-sdk/src/indexer/stacks/mod.rs index 0eeb70128..5764df40a 100644 --- a/components/chainhook-sdk/src/indexer/stacks/mod.rs +++ b/components/chainhook-sdk/src/indexer/stacks/mod.rs @@ -10,8 +10,8 @@ use chainhook_types::*; use hiro_system_kit::slog; use rocket::serde::json::Value as JsonValue; use rocket::serde::Deserialize; +use stacks_rpc_client::clarity::codec::StacksMessageCodec; use stacks_rpc_client::clarity::codec::{StacksTransaction, TransactionAuth, TransactionPayload}; -use stacks_rpc_client::clarity::stacks_common::codec::StacksMessageCodec; use stacks_rpc_client::clarity::vm::types::{SequenceData, Value as ClarityValue}; use std::collections::{BTreeMap, HashMap, HashSet}; use std::convert::TryInto; @@ -332,7 +332,7 @@ pub fn standardize_stacks_block( .into(); let current_len = u64::saturating_sub( block.burn_block_height, - 1 + chain_ctx.pox_info.first_burnchain_block_height, + 1 + (chain_ctx.pox_info.first_burnchain_block_height as u64), ); let pox_cycle_id: u32 = (current_len / pox_cycle_length).try_into().unwrap_or(0); let mut events: HashMap<&String, Vec<&NewEvent>> = HashMap::new(); @@ -818,10 +818,16 @@ pub fn get_tx_description( StacksTransactionKind::ContractDeployment(data), ) } - TransactionPayload::Coinbase(_, _) => { + TransactionPayload::Coinbase(_, _, _) => { (format!("coinbase"), StacksTransactionKind::Coinbase) } - _ => (format!("other"), StacksTransactionKind::Unsupported), + TransactionPayload::TenureChange(_) => ( + format!("tenure change"), + StacksTransactionKind::TenureChange, + ), + TransactionPayload::PoisonMicroblock(_, _) => { + unimplemented!() + } }; Ok((description, tx_type, fee, nonce, sender, sponsor)) } diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs index 4e04e62a7..a8a0c1699 100644 --- a/components/chainhook-sdk/src/observer/mod.rs +++ b/components/chainhook-sdk/src/observer/mod.rs @@ -639,7 +639,7 @@ pub async fn start_stacks_event_observer( let ingestion_config = Config { port: ingestion_port, - workers: 3, + workers: 1, address: IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), keep_alive: 5, temp_dir: std::env::temp_dir().into(), diff --git a/components/chainhook-types-js/src/index.ts b/components/chainhook-types-js/src/index.ts index eb45c7edc..2075a4138 100644 --- a/components/chainhook-types-js/src/index.ts +++ b/components/chainhook-types-js/src/index.ts @@ -323,7 +323,7 @@ export interface StacksTransactionMetadata { * @memberof StacksTransactionMetadata */ position: AnchorBlockPosition | MicroBlockPosition; - /** + /** * @type {string} * @memberof StacksTransactionMetadata */ @@ -335,9 +335,9 @@ export interface StacksTransactionMetadata { * @export * @interface MicroBlockPosition */ - export interface MicroBlockPosition { - micro_block_identifier: BlockIdentifier, - index: number +export interface MicroBlockPosition { + micro_block_identifier: BlockIdentifier; + index: number; } /** @@ -345,8 +345,8 @@ export interface StacksTransactionMetadata { * @export * @interface AnchorBlockPosition */ - export interface AnchorBlockPosition { - index: number +export interface AnchorBlockPosition { + index: number; } export interface StacksTransactionReceipt { @@ -428,11 +428,13 @@ export interface StacksTransactionExecutionCost { runtime: number; } +// todo: tenure change export enum StacksTransactionKind { ContractCall = "ContractCall", ContractDeployment = "ContractDeployment", NativeTokenTransfer = "NativeTokenTransfer", Coinbase = "Coinbase", + TenureChange = "TenureChange", Other = "Other", } @@ -1061,9 +1063,7 @@ export function checkBitcoinProof( blockHash: string, merkleProof: string[], index: number -): boolean { - -} +): boolean {} // I'll add some documentation, but this is a "standard" bitcoin proof, so it goes as follow: // So for instance, you'll receive something like this: diff --git a/components/chainhook-types-rs/src/rosetta.rs b/components/chainhook-types-rs/src/rosetta.rs index ef0ab908a..410421285 100644 --- a/components/chainhook-types-rs/src/rosetta.rs +++ b/components/chainhook-types-rs/src/rosetta.rs @@ -163,6 +163,7 @@ pub enum StacksTransactionKind { ContractDeployment(StacksContractDeploymentData), NativeTokenTransfer, Coinbase, + TenureChange, BitcoinOp(BitcoinOpData), Unsupported, } diff --git a/components/client/typescript/src/schemas/stacks/tx_kind.ts b/components/client/typescript/src/schemas/stacks/tx_kind.ts index dbe65fabe..a1bbe17c9 100644 --- a/components/client/typescript/src/schemas/stacks/tx_kind.ts +++ b/components/client/typescript/src/schemas/stacks/tx_kind.ts @@ -28,6 +28,11 @@ export const StacksTransactionCoinbaseKindSchema = Type.Object({ }); export type StacksTransactionCoinbaseKind = Static; +export const StacksTransactionTenureChangeKindSchema = Type.Object({ + type: Type.Literal('TenureChange'), +}); +export type StacksTransactionTenureChangeKind = Static; + export const StacksTransactionNativeTokenTransferKindSchema = Type.Object({ type: Type.Literal('NativeTokenTransfer'), }); @@ -70,6 +75,7 @@ export type StacksTransactionUnsupportedKind = Static< export const StacksTransactionKindSchema = Type.Union([ StacksTransactionCoinbaseKindSchema, + StacksTransactionTenureChangeKindSchema, StacksTransactionContractCallKindSchema, StacksTransactionContractDeploymentKindSchema, StacksTransactionNativeTokenTransferKindSchema, diff --git a/dockerfiles/components/chainhook-node.dockerfile b/dockerfiles/components/chainhook-node.dockerfile index 0f216864d..91b7836ed 100644 --- a/dockerfiles/components/chainhook-node.dockerfile +++ b/dockerfiles/components/chainhook-node.dockerfile @@ -4,7 +4,7 @@ WORKDIR /src RUN apt update && apt install -y ca-certificates pkg-config libssl-dev libclang-11-dev -RUN rustup update 1.70.0 && rustup default 1.70.0 +RUN rustup update 1.73.0 && rustup default 1.73.0 COPY ./Cargo.* /src/ @@ -30,4 +30,4 @@ COPY --from=build /out/ /bin/ WORKDIR /workspace -ENTRYPOINT ["chainhook"] \ No newline at end of file +ENTRYPOINT ["chainhook"] From 0a5a0a71ff24d920dbc08d16eb8cbed323a102cd Mon Sep 17 00:00:00 2001 From: MicaiahReid Date: Wed, 27 Mar 2024 14:45:39 -0600 Subject: [PATCH 12/12] chore: bump version --- Cargo.lock | 2 +- components/chainhook-cli/Cargo.toml | 2 +- docs/chainhook-openapi.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e37932259..8a48a7151 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -520,7 +520,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chainhook" -version = "1.3.1" +version = "1.4.0" dependencies = [ "ansi_term", "atty", diff --git a/components/chainhook-cli/Cargo.toml b/components/chainhook-cli/Cargo.toml index d7305cdd5..a83c9d372 100644 --- a/components/chainhook-cli/Cargo.toml +++ b/components/chainhook-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chainhook" -version = "1.3.1" +version = "1.4.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/docs/chainhook-openapi.json b/docs/chainhook-openapi.json index c7c4957d0..b3901a66b 100644 --- a/docs/chainhook-openapi.json +++ b/docs/chainhook-openapi.json @@ -2,7 +2,7 @@ "openapi": "3.0.0", "info": { "title": "chainhook", - "version": "1.3.1" + "version": "1.4.0" }, "paths": { "/ping": {