feat: optionally serve Prometheus metrics (#473)

### Description

To enable improved alerts on downtime for Hiro's hosted Chainhook
service, we need Chainhook to provide metrics that can be ingested by
Prometheus. This PR changes some how we track our metrics (that are
served over the `/ping` endpoint of the observer) to enable Prometheus
compatibility, and adds a flag to optionally start a server to supply
metrics to a Prometheus client.


### Example

Starting chainhook with the `--prometheus-port XXXX` flag now enables a
service that can supply Prometheus metrics at `localhost:XXXX/metrics`.

If using a config file, this option can be specified via:
```yaml
[monitoring]
prometheus_monitoring_port = XXXX
```

Chainhook will behave as usual with this flag ommitted - metrics can
still be retrieved via the observer's `/ping` endpoint, but they will
not be formatted for ingestion by a Prometheus client.

---

### Checklist

- [X] All tests pass
- [X] Tests added in this PR (if applicable)

Fixes #474, addresses #466

components/chainhook-cli/src/cli/mod.rs

@@ -193,6 +193,9 @@ struct StartCommand {
     /// Start REST API for managing predicates
     #[clap(long = "start-http-api")]
     pub start_http_api: bool,
+    /// If provided, serves Prometheus metrics at localhost:{port}/metrics. If not specified, does not start Prometheus server.
+    #[clap(long = "prometheus-port")]
+    pub prometheus_monitoring_port: Option<u16>,
 }
 
 #[derive(Subcommand, PartialEq, Clone, Debug)]
@@ -291,9 +294,13 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
     match opts.command {
         Command::Service(subcmd) => match subcmd {
             ServiceCommand::Start(cmd) => {
-                let config =
+                let mut config =
                     Config::default(cmd.devnet, cmd.testnet, cmd.mainnet, &cmd.config_path)?;
 
+                if cmd.prometheus_monitoring_port.is_some() {
+                    config.monitoring.prometheus_monitoring_port = cmd.prometheus_monitoring_port;
+                }
+
                 let predicates = cmd
                     .predicates_paths
                     .iter()

components/chainhook-cli/src/config/file.rs

@@ -7,6 +7,7 @@ pub struct ConfigFile {
     pub event_source: Option<Vec<EventSourceConfigFile>>,
     pub limits: LimitsConfigFile,
     pub network: NetworkConfigFile,
+    pub monitoring: Option<MonitoringConfigFile>,
 }
 
 #[derive(Deserialize, Debug, Clone)]
@@ -81,3 +82,8 @@ impl NetworkConfigMode {
         }
     }
 }
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct MonitoringConfigFile {
+    pub prometheus_monitoring_port: Option<u16>,
+}

components/chainhook-cli/src/config/generator.rs

@@ -45,6 +45,11 @@ max_caching_memory_size_mb = 32000
 # If this is not a requirement, you can comment out the `tsv_file_url` line.
 [[event_source]]
 tsv_file_url = "https://archive.hiro.so/{network}/stacks-blockchain-api/{network}-stacks-blockchain-api-latest"
+
+# Enables a server that provides metrics that can be scraped by Prometheus.
+# This is disabled by default.
+# [monitoring]
+# prometheus_monitoring_port = 20457
 "#,
         mode = mode.as_str(),
         network = network.to_lowercase(),

components/chainhook-cli/src/config/mod.rs

@@ -31,6 +31,7 @@ pub struct Config {
     pub event_sources: Vec<EventSourceConfig>,
     pub limits: LimitsConfig,
     pub network: IndexerConfig,
+    pub monitoring: MonitoringConfig,
 }
 
 #[derive(Clone, Debug, PartialEq)]
@@ -80,6 +81,10 @@ pub struct LimitsConfig {
     pub max_caching_memory_size_mb: usize,
 }
 
+#[derive(Clone, Debug, PartialEq)]
+pub struct MonitoringConfig {
+    pub prometheus_monitoring_port: Option<u16>,
+}
 impl Config {
     pub fn from_file_path(file_path: &str) -> Result<Config, String> {
         let file = File::open(file_path)
@@ -120,6 +125,7 @@ impl Config {
             bitcoin_network: self.network.bitcoin_network.clone(),
             stacks_network: self.network.stacks_network.clone(),
             data_handler_tx: None,
+            prometheus_monitoring_port: self.monitoring.prometheus_monitoring_port,
         }
     }
 
@@ -144,15 +150,19 @@ impl Config {
                 continue;
             }
         }
-
+        let prometheus_monitoring_port = if let Some(monitoring) = config_file.monitoring {
+            monitoring.prometheus_monitoring_port
+        } else {
+            None
+        };
         let config = Config {
             storage: StorageConfig {
                 working_dir: config_file.storage.working_dir.unwrap_or("cache".into()),
             },
             http_api: match config_file.http_api {
                 None => PredicatesApi::Off,
                 Some(http_api) => match http_api.disabled {
-                    Some(false) => PredicatesApi::Off,
+                    Some(true) => PredicatesApi::Off,
                     _ => PredicatesApi::On(PredicatesApiConfig {
                         http_port: http_api.http_port.unwrap_or(DEFAULT_CONTROL_PORT),
                         display_logs: http_api.display_logs.unwrap_or(true),
@@ -209,6 +219,9 @@ impl Config {
                 stacks_network,
                 bitcoin_network,
             },
+            monitoring: MonitoringConfig {
+                prometheus_monitoring_port,
+            },
         };
         Ok(config)
     }
@@ -340,6 +353,9 @@ impl Config {
                 stacks_network: StacksNetwork::Devnet,
                 bitcoin_network: BitcoinNetwork::Regtest,
             },
+            monitoring: MonitoringConfig {
+                prometheus_monitoring_port: None,
+            },
         }
     }
 
@@ -371,6 +387,9 @@ impl Config {
                 stacks_network: StacksNetwork::Testnet,
                 bitcoin_network: BitcoinNetwork::Testnet,
             },
+            monitoring: MonitoringConfig {
+                prometheus_monitoring_port: None,
+            },
         }
     }
 
@@ -402,6 +421,9 @@ impl Config {
                 stacks_network: StacksNetwork::Mainnet,
                 bitcoin_network: BitcoinNetwork::Mainnet,
             },
+            monitoring: MonitoringConfig {
+                prometheus_monitoring_port: None,
+            },
         }
     }
 }

components/chainhook-cli/src/config/tests/mod.rs

@@ -1,8 +1,14 @@
 use std::path::PathBuf;
 
-use crate::config::{file::NetworkConfigMode, PredicatesApi, PredicatesApiConfig};
-
-use super::{generator::generate_config, Config, ConfigFile, EventSourceConfig, PathConfig};
+use crate::config::{
+    file::{NetworkConfigMode, PredicatesApiConfigFile},
+    PredicatesApi, PredicatesApiConfig,
+};
+
+use super::{
+    file::MonitoringConfigFile, generator::generate_config, Config, ConfigFile, EventSourceConfig,
+    PathConfig,
+};
 use chainhook_sdk::types::{BitcoinNetwork, StacksNetwork};
 use test_case::test_case;
 
@@ -24,6 +30,28 @@ fn config_from_file_matches_generator_for_all_networks(network: BitcoinNetwork)
     assert_eq!(generated_config, from_path_config);
 }
 
+#[test]
+fn config_from_file_allows_setting_disabled_fields() {
+    let generated_config_str = generate_config(&BitcoinNetwork::Regtest);
+    let mut generated_config_file: ConfigFile = toml::from_str(&generated_config_str).unwrap();
+    // http_api and monitoring are optional, so they are disabled in generated config file
+    generated_config_file.http_api = Some(PredicatesApiConfigFile {
+        http_port: Some(0),
+        database_uri: Some(format!("")),
+        display_logs: Some(false),
+        disabled: Some(false),
+    });
+    generated_config_file.monitoring = Some(MonitoringConfigFile {
+        prometheus_monitoring_port: Some(20457),
+    });
+    let generated_config = Config::from_config_file(generated_config_file).unwrap();
+    assert!(generated_config.is_http_api_enabled());
+    assert_eq!(
+        generated_config.monitoring.prometheus_monitoring_port,
+        Some(20457)
+    );
+}
+
 #[test]
 fn config_from_file_allows_local_tsv_file() {
     let path = format!(

components/chainhook-cli/src/service/tests/helpers/mock_service.rs

@@ -1,6 +1,7 @@
 use crate::config::Config;
 use crate::config::EventSourceConfig;
 use crate::config::LimitsConfig;
+use crate::config::MonitoringConfig;
 use crate::config::PathConfig;
 use crate::config::PredicatesApi;
 use crate::config::PredicatesApiConfig;
@@ -12,7 +13,6 @@ use crate::service::Service;
 use chainhook_sdk::chainhooks::types::ChainhookFullSpecification;
 use chainhook_sdk::indexer::IndexerConfig;
 use chainhook_sdk::observer::ObserverCommand;
-use chainhook_sdk::observer::ObserverMetrics;
 use chainhook_sdk::types::BitcoinBlockSignaling;
 use chainhook_sdk::types::BitcoinNetwork;
 use chainhook_sdk::types::Chain;
@@ -172,7 +172,7 @@ pub async fn call_observer_svc(
         .map_err(|e| format!("Failed to deserialize response of {method} request to {url}: {e}",))
 }
 
-pub async fn call_ping(port: u16) -> Result<ObserverMetrics, String> {
+pub async fn call_ping(port: u16) -> Result<JsonValue, String> {
     let url = format!("http://localhost:{port}/ping");
     let res = call_observer_svc(&url, Method::GET, None).await?;
     match res.get("result") {
@@ -182,6 +182,19 @@ pub async fn call_ping(port: u16) -> Result<ObserverMetrics, String> {
     }
 }
 
+pub async fn call_prometheus(port: u16) -> Result<String, String> {
+    let url = format!("http://localhost:{port}/metrics");
+    let client = reqwest::Client::new();
+    client
+        .get(&url)
+        .send()
+        .await
+        .map_err(|e| format!("Failed to make GET request to {url}: {e}",))?
+        .text()
+        .await
+        .map_err(|e| format!("Failed to deserialize response of GET request to {url}: {e}",))
+}
+
 pub async fn build_predicate_api_server(port: u16) -> (Receiver<ObserverCommand>, Shutdown) {
     let ctx = Context {
         logger: None,
@@ -272,6 +285,7 @@ pub fn get_chainhook_config(
     bitcoin_rpc_port: u16,
     working_dir: &str,
     tsv_dir: &str,
+    prometheus_port: Option<u16>,
 ) -> Config {
     let api_config = PredicatesApiConfig {
         http_port: chainhook_port,
@@ -306,6 +320,9 @@ pub fn get_chainhook_config(
                 ingestion_port: stacks_ingestion_port,
             }),
         },
+        monitoring: MonitoringConfig {
+            prometheus_monitoring_port: prometheus_port,
+        },
     }
 }