Merge pull request #538 from hirosystems/develop

chore(release): publish v1.4.0

components/chainhook-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "chainhook"
-version = "1.3.1"
+version = "1.4.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -18,8 +18,7 @@ rand = "0.8.5"
 chainhook-sdk = { version = "0.12.1", default-features = false, features = [
     "zeromq",
 ], path = "../chainhook-sdk" }
-hiro-system-kit = "0.3.1"
-# clarinet-files = { path = "../../../clarinet/components/clarinet-files" }
+hiro-system-kit = "0.3.1" 
 # hiro-system-kit = { path = "../../../clarinet/components/hiro-system-kit" }
 clap = { version = "3.2.23", features = ["derive"], optional = true }
 clap_generate = { version = "3.0.3", optional = true }
@@ -31,7 +30,7 @@ reqwest = { version = "0.11", default-features = false, features = [
     "json",
     "rustls-tls",
 ] }
-tokio = { version = "=1.24", features = ["full"] }
+tokio = { version = "1.35.1", features = ["full"] }
 futures-util = "0.3.24"
 flate2 = "1.0.24"
 tar = "0.4.38"
@@ -52,7 +51,6 @@ features = ["lz4", "snappy"]
 [dev-dependencies]
 criterion = "0.3"
 redis = "0.21.5"
-clarity-vm = "=2.1.1"
 hex = "0.4.3"
 test-case = "3.1.0"
 serial_test = "2.0.0"

components/chainhook-cli/src/archive/mod.rs

@@ -21,7 +21,7 @@ pub async fn download_tsv_file(config: &Config) -> Result<(), String> {
         println!("{}", e.to_string());
     });
 
-    let remote_sha_url = config.expected_remote_stacks_tsv_sha256();
+    let remote_sha_url = config.expected_remote_stacks_tsv_sha256()?;
     let res = reqwest::get(&remote_sha_url)
         .await
         .or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))?
@@ -34,7 +34,7 @@ pub async fn download_tsv_file(config: &Config) -> Result<(), String> {
 
     write_file_content_at_path(&local_sha_file_path, &res.to_vec())?;
 
-    let file_url = config.expected_remote_stacks_tsv_url();
+    let file_url = config.expected_remote_stacks_tsv_url()?;
     let res = reqwest::get(&file_url)
         .await
         .or(Err(format!("Failed to GET from '{}'", &file_url)))?;
@@ -55,14 +55,17 @@ pub async fn download_tsv_file(config: &Config) -> Result<(), String> {
                     Ok(0) => break,
                     Ok(n) => {
                         if let Err(e) = file.write_all(&buffer[..n]) {
-                            let err =
-                                format!("unable to update compressed archive: {}", e.to_string());
-                            return Err(err);
+                            return Err(format!(
+                                "unable to update compressed archive: {}",
+                                e.to_string()
+                            ));
                         }
                     }
                     Err(e) => {
-                        let err = format!("unable to write compressed archive: {}", e.to_string());
-                        return Err(err);
+                        return Err(format!(
+                            "unable to write compressed archive: {}",
+                            e.to_string()
+                        ));
                     }
                 }
             }
@@ -83,12 +86,11 @@ pub async fn download_tsv_file(config: &Config) -> Result<(), String> {
                 .map_err(|e| format!("unable to download stacks archive: {}", e.to_string()))?;
         }
         drop(tx);
-
         tokio::task::spawn_blocking(|| decoder_thread.join())
             .await
-            .unwrap()
-            .unwrap()
-            .unwrap();
+            .map_err(|e| format!("failed to spawn thread: {e}"))?
+            .map_err(|e| format!("decoder thread failed when downloading tsv: {:?}", e))?
+            .map_err(|e| format!("failed to download tsv: {}", e))?;
     }
 
     Ok(())
@@ -124,11 +126,14 @@ impl Read for ChannelRead {
     }
 }
 
-pub async fn download_stacks_dataset_if_required(config: &mut Config, ctx: &Context) -> bool {
+pub async fn download_stacks_dataset_if_required(
+    config: &mut Config,
+    ctx: &Context,
+) -> Result<bool, String> {
     if config.is_initial_ingestion_required() {
         // Download default tsv.
         if config.rely_on_remote_stacks_tsv() && config.should_download_remote_stacks_tsv() {
-            let url = config.expected_remote_stacks_tsv_url();
+            let url = config.expected_remote_stacks_tsv_url()?;
             let mut tsv_file_path = config.expected_cache_path();
             tsv_file_path.push(default_tsv_file_path(&config.network.stacks_network));
             let mut tsv_sha_file_path = config.expected_cache_path();
@@ -137,7 +142,7 @@ pub async fn download_stacks_dataset_if_required(config: &mut Config, ctx: &Cont
             // Download archive if not already present in cache
             // Load the local
             let local_sha_file = read_file_content_at_path(&tsv_sha_file_path);
-            let sha_url = config.expected_remote_stacks_tsv_sha256();
+            let sha_url = config.expected_remote_stacks_tsv_sha256()?;
 
             let remote_sha_file = match reqwest::get(&sha_url).await {
                 Ok(response) => response.bytes().await,
@@ -164,28 +169,25 @@ pub async fn download_stacks_dataset_if_required(config: &mut Config, ctx: &Cont
                     "Stacks archive file already up to date"
                 );
                 config.add_local_stacks_tsv_source(&tsv_file_path);
-                return false;
+                return Ok(false);
             }
 
             info!(ctx.expect_logger(), "Downloading {}", url);
             match download_tsv_file(&config).await {
                 Ok(_) => {}
-                Err(e) => {
-                    error!(ctx.expect_logger(), "{}", e);
-                    std::process::exit(1);
-                }
+                Err(e) => return Err(e),
             }
             info!(ctx.expect_logger(), "Successfully downloaded tsv file");
             config.add_local_stacks_tsv_source(&tsv_file_path);
         }
-        true
+        Ok(true)
     } else {
         info!(
             ctx.expect_logger(),
             "Streaming blocks from stacks-node {}",
             config.network.get_stacks_node_config().rpc_url
         );
-        false
+        Ok(false)
     }
 }
 

components/chainhook-cli/src/archive/tests/mod.rs

@@ -72,8 +72,14 @@ async fn it_downloads_stacks_dataset_if_required() {
         tracer: false,
     };
     let mut config_clone = config.clone();
-    assert!(download_stacks_dataset_if_required(&mut config, &ctx).await);
-    assert!(!download_stacks_dataset_if_required(&mut config_clone, &ctx).await);
+    assert!(download_stacks_dataset_if_required(&mut config, &ctx)
+        .await
+        .unwrap());
+    assert!(
+        !download_stacks_dataset_if_required(&mut config_clone, &ctx)
+            .await
+            .unwrap()
+    );
 
     let mut tsv_file_path = config.expected_cache_path();
     tsv_file_path.push(default_tsv_file_path(&config.network.stacks_network));

components/chainhook-cli/src/cli/mod.rs

@@ -3,6 +3,7 @@ use crate::config::Config;
 use crate::scan::bitcoin::scan_bitcoin_chainstate_via_rpc_using_predicate;
 use crate::scan::stacks::{
     consolidate_local_stacks_chainstate_using_csv, scan_stacks_chainstate_via_csv_using_predicate,
+    scan_stacks_chainstate_via_rocksdb_using_predicate,
 };
 use crate::service::http_api::document_predicate_api_server;
 use crate::service::Service;
@@ -276,14 +277,14 @@ pub fn main() {
     let opts: Opts = match Opts::try_parse() {
         Ok(opts) => opts,
         Err(e) => {
-            error!(ctx.expect_logger(), "{e}");
+            crit!(ctx.expect_logger(), "{e}");
             process::exit(1);
         }
     };
 
     match hiro_system_kit::nestable_block_on(handle_command(opts, ctx.clone())) {
         Err(e) => {
-            error!(ctx.expect_logger(), "{e}");
+            crit!(ctx.expect_logger(), "{e}");
             process::exit(1);
         }
         Ok(_) => {}
@@ -310,7 +311,7 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
                 info!(ctx.expect_logger(), "Starting service...",);
 
                 let mut service = Service::new(config, ctx);
-                return service.run(predicates).await;
+                return service.run(predicates, None).await;
             }
         },
         Command::Config(subcmd) => match subcmd {
@@ -485,14 +486,35 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
                                 ));
                             }
                         };
-                        // TODO: if a stacks.rocksdb is present, use it.
-                        // TODO: update Stacks archive file if required.
-                        scan_stacks_chainstate_via_csv_using_predicate(
-                            &predicate_spec,
-                            &mut config,
-                            &ctx,
-                        )
-                        .await?;
+                        match open_readonly_stacks_db_conn(&config.expected_cache_path(), &ctx) {
+                            Ok(db_conn) => {
+                                let _ = consolidate_local_stacks_chainstate_using_csv(
+                                    &mut config,
+                                    &ctx,
+                                )
+                                .await;
+                                scan_stacks_chainstate_via_rocksdb_using_predicate(
+                                    &predicate_spec,
+                                    None,
+                                    &db_conn,
+                                    &config,
+                                    &ctx,
+                                )
+                                .await?;
+                            }
+                            Err(e) => {
+                                info!(
+                                    ctx.expect_logger(),
+                                    "Could not open db. This will greatly increase scan times. Error: {}", e
+                                );
+                                scan_stacks_chainstate_via_csv_using_predicate(
+                                    &predicate_spec,
+                                    &mut config,
+                                    &ctx,
+                                )
+                                .await?;
+                            }
+                        };
                     }
                 }
             }

components/chainhook-cli/src/config/mod.rs

@@ -256,13 +256,13 @@ impl Config {
         }
     }
 
-    pub fn expected_local_stacks_tsv_file(&self) -> &PathBuf {
+    pub fn expected_local_stacks_tsv_file(&self) -> Result<&PathBuf, String> {
         for source in self.event_sources.iter() {
             if let EventSourceConfig::StacksTsvPath(config) = source {
-                return &config.file_path;
+                return Ok(&config.file_path);
             }
         }
-        panic!("expected local-tsv source")
+        Err("could not find expected local tsv source")?
     }
 
     pub fn expected_cache_path(&self) -> PathBuf {
@@ -271,21 +271,23 @@ impl Config {
         destination_path
     }
 
-    fn expected_remote_stacks_tsv_base_url(&self) -> &String {
+    fn expected_remote_stacks_tsv_base_url(&self) -> Result<&String, String> {
         for source in self.event_sources.iter() {
             if let EventSourceConfig::StacksTsvUrl(config) = source {
-                return &config.file_url;
+                return Ok(&config.file_url);
             }
         }
-        panic!("expected remote-tsv source")
+        Err("could not find expected remote tsv source")?
     }
 
-    pub fn expected_remote_stacks_tsv_sha256(&self) -> String {
-        format!("{}.sha256", self.expected_remote_stacks_tsv_base_url())
+    pub fn expected_remote_stacks_tsv_sha256(&self) -> Result<String, String> {
+        self.expected_remote_stacks_tsv_base_url()
+            .map(|url| format!("{}.sha256", url))
     }
 
-    pub fn expected_remote_stacks_tsv_url(&self) -> String {
-        format!("{}.gz", self.expected_remote_stacks_tsv_base_url())
+    pub fn expected_remote_stacks_tsv_url(&self) -> Result<String, String> {
+        self.expected_remote_stacks_tsv_base_url()
+            .map(|url| format!("{}.gz", url))
     }
 
     pub fn rely_on_remote_stacks_tsv(&self) -> bool {

components/chainhook-cli/src/config/tests/mod.rs

@@ -108,34 +108,48 @@ fn should_download_remote_stacks_tsv_handles_both_modes() {
 }
 
 #[test]
-#[should_panic(expected = "expected remote-tsv source")]
 fn expected_remote_stacks_tsv_base_url_panics_if_missing() {
     let url_src = EventSourceConfig::StacksTsvUrl(super::UrlConfig {
         file_url: format!("test"),
     });
     let mut config = Config::default(true, false, false, &None).unwrap();
 
     config.event_sources = vec![url_src.clone()];
-    assert_eq!(config.expected_remote_stacks_tsv_base_url(), "test");
+    match config.expected_remote_stacks_tsv_base_url() {
+        Ok(tsv_url) => assert_eq!(tsv_url, "test"),
+        Err(e) => {
+            panic!("expected tsv file: {e}")
+        }
+    }
 
     config.event_sources = vec![];
-    config.expected_remote_stacks_tsv_base_url();
+    match config.expected_remote_stacks_tsv_base_url() {
+        Ok(tsv_url) => panic!("expected no tsv file, found {}", tsv_url),
+        Err(e) => assert_eq!(e, "could not find expected remote tsv source".to_string()),
+    };
 }
 
 #[test]
-#[should_panic(expected = "expected local-tsv source")]
-fn expected_local_stacks_tsv_base_url_panics_if_missing() {
+fn expected_local_stacks_tsv_base_url_errors_if_missing() {
     let path = PathBuf::from("test");
     let path_src = EventSourceConfig::StacksTsvPath(PathConfig {
         file_path: path.clone(),
     });
     let mut config = Config::default(true, false, false, &None).unwrap();
 
     config.event_sources = vec![path_src.clone()];
-    assert_eq!(config.expected_local_stacks_tsv_file(), &path);
+    match config.expected_local_stacks_tsv_file() {
+        Ok(tsv_path) => assert_eq!(tsv_path, &path),
+        Err(e) => {
+            panic!("expected tsv file: {e}")
+        }
+    }
 
     config.event_sources = vec![];
-    config.expected_local_stacks_tsv_file();
+    match config.expected_local_stacks_tsv_file() {
+        Ok(tsv_path) => panic!("expected no tsv file, found {}", tsv_path.to_string_lossy()),
+        Err(e) => assert_eq!(e, "could not find expected local tsv source".to_string()),
+    };
 }
 
 #[test]