Dz nym node stats (#5418)

* Remove blacklisted, inactive, reserve fields

* Remove gw.blacklisted

* Remove blacklisted and bonded count

* DB operations

* Improve logging

* Remove unused functions

* get_nym_nodes for scraping WIP

* Separate nym_nodes from mixnode stats
- fixes FOREIGN_KEY_CONSTRAINT error when storing
  stats for nym_nodes which aren't in mixnodes table

* Daily aggregation works

* mixnodes/stats exposes correct info

* Undo unnecessary tidbits

* Replace obsolete stats

* Add total_stake

* Bump cargo.toml version

* Rename MixingNodeKind for better clarity

Author: dynco-nym

Cargo.lock

@@ -768,7 +768,7 @@ where
     E: DeserializeOwned + Display,
 {
     let status = res.status();
-    tracing::debug!("Status: {} (success: {})", &status, status.is_success());
+    tracing::trace!("Status: {} (success: {})", &status, status.is_success());
 
     if !allow_empty {
         if let Some(0) = res.content_length() {

common/http-api-client/src/lib.rs

@@ -1,6 +1,7 @@
 nym-node-status-agent/nym-gateway-probe
 nym-node-status-agent/keys/
 data/
+settings.sql
 enter_db.sh
 nym-gateway-probe
 *.sqlite

nym-node-status-api/.gitignore

@@ -3,7 +3,7 @@
 
 [package]
 name = "nym-node-status-api"
-version = "1.0.0-rc.7"
+version = "1.0.0-rc.8"
 authors.workspace = true
 repository.workspace = true
 homepage.workspace = true
@@ -22,6 +22,7 @@ cosmwasm-std = { workspace = true }
 envy = { workspace = true }
 futures-util = { workspace = true }
 moka = { workspace = true, features = ["future"] }
+nym-contracts-common = { path = "../../common/cosmwasm-smart-contracts/contracts-common" }
 nym-bin-common = { path = "../../common/bin-common", features = ["models"] }
 nym-node-status-client = { path = "../nym-node-status-client" }
 nym-crypto = { path = "../../common/crypto", features = ["asymmetric", "serde"] }
@@ -50,10 +51,6 @@ tracing-log = { workspace = true }
 tower-http = { workspace = true, features = ["cors", "trace"] }
 utoipa = { workspace = true, features = ["axum_extras", "time"] }
 utoipa-swagger-ui = { workspace = true, features = ["axum"] }
-# TODO dz `cargo update async-trait`
-# for automatic schema detection, which was merged, but not released yet
-# https://github.com/ProbablyClem/utoipauto/pull/38
-# utoipauto = { git = "https://github.com/ProbablyClem/utoipauto", rev = "eb04cba" }
 utoipauto = { workspace = true }
 
 nym-node-metrics = { path = "../../nym-node/nym-node-metrics" }

nym-node-status-api/nym-node-status-api/Cargo.toml

@@ -36,10 +36,18 @@ fn read_env_var(var: &str) -> Result<String> {
 
 /// use `./enter_db.sh` to inspect DB
 async fn write_db_path_to_file(out_dir: &str, db_filename: &str) -> anyhow::Result<()> {
+    let mut file = File::create("settings.sql").await?;
+    let settings = ".mode columns
+.headers on";
+    file.write_all(settings.as_bytes()).await?;
+
     let mut file = File::create("enter_db.sh").await?;
-    let _ = file.write(b"#!/bin/bash\n").await?;
-    file.write_all(format!("sqlite3 {}/{}", out_dir, db_filename).as_bytes())
-        .await?;
+    let contents = format!(
+        "#!/bin/bash\n\
+        sqlite3 -init settings.sql {}/{}",
+        out_dir, db_filename,
+    );
+    file.write_all(contents.as_bytes()).await?;
 
     #[cfg(target_family = "unix")]
     file.set_permissions(Permissions::from_mode(0o755))

nym-node-status-api/nym-node-status-api/build.rs

@@ -3,16 +3,19 @@
 set -e
 
 user_rust_log_preference=$RUST_LOG
+export ENVIRONMENT=${ENVIRONMENT:-"sandbox"}
 export NYM_API_CLIENT_TIMEOUT=60
 export EXPLORER_CLIENT_TIMEOUT=60
 export NODE_STATUS_API_TESTRUN_REFRESH_INTERVAL=120
 
+if [ "$ENVIRONMENT" = "mainnet" ]; then
+    export NYM_NODE_STATUS_API_HM_URL="https://harbourmaster.nymtech.net"
+fi
+
 # public counterpart of the agent's private key.
 # For TESTING only. NOT used in any other environment
 export NODE_STATUS_API_AGENT_KEY_LIST="H4z8kx5Kkf5JMQHhxaW1MwYndjKCDHC7HsVhHTFfBZ4J"
 
-export ENVIRONMENT=${ENVIRONMENT:-"sandbox"}
-
 script_dir=$(dirname $(realpath "$0"))
 monorepo_root=$(realpath "${script_dir}/../..")
 

nym-node-status-api/nym-node-status-api/launch_node_status_api.sh

@@ -0,0 +1,54 @@
+ALTER TABLE mixnodes DROP COLUMN blacklisted;
+ALTER TABLE gateways DROP COLUMN blacklisted;
+
+CREATE TABLE nym_nodes (
+    node_id INTEGER PRIMARY KEY,
+    ed25519_identity_pubkey VARCHAR NOT NULL UNIQUE,
+    total_stake INTEGER NOT NULL,
+    ip_addresses TEXT NOT NULL,
+    mix_port INTEGER NOT NULL,
+    x25519_sphinx_pubkey VARCHAR NOT NULL UNIQUE,
+    node_role TEXT NOT NULL,
+    supported_roles TEXT NOT NULL,
+    performance VARCHAR NOT NULL,
+    entry TEXT,
+    last_updated_utc INTEGER NOT NULL
+);
+
+CREATE INDEX idx_nym_nodes_node_id ON nym_nodes (node_id);
+CREATE INDEX idx_nym_nodes_ed25519_identity_pubkey ON nym_nodes (ed25519_identity_pubkey);
+
+CREATE TABLE nym_node_descriptions (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    node_id INTEGER UNIQUE NOT NULL,
+    moniker VARCHAR,
+    website VARCHAR,
+    security_contact VARCHAR,
+    details VARCHAR,
+    last_updated_utc INTEGER NOT NULL,
+    FOREIGN KEY (node_id) REFERENCES nym_nodes (node_id)
+);
+
+CREATE TABLE nym_nodes_packet_stats_raw (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    node_id INTEGER NOT NULL,
+    timestamp_utc INTEGER NOT NULL,
+    packets_received INTEGER,
+    packets_sent INTEGER,
+    packets_dropped INTEGER,
+    FOREIGN KEY (node_id) REFERENCES nym_nodes (node_id)
+);
+
+CREATE INDEX idx_nym_nodes_packet_stats_raw_node_id_timestamp_utc ON nym_nodes_packet_stats_raw (node_id, timestamp_utc);
+
+CREATE TABLE nym_node_daily_mixing_stats (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    node_id INTEGER NOT NULL,
+    total_stake BIGINT NOT NULL,
+    date_utc VARCHAR NOT NULL,
+    packets_received INTEGER DEFAULT 0,
+    packets_sent INTEGER DEFAULT 0,
+    packets_dropped INTEGER DEFAULT 0,
+    FOREIGN KEY (node_id) REFERENCES nym_nodes (node_id),
+    UNIQUE (node_id, date_utc) -- This constraint automatically creates an index
+);

nym-node-status-api/nym-node-status-api/migrations/004_obsolete_fields.sql

@@ -83,6 +83,9 @@ pub(crate) struct Cli {
         env = "NYM_NODE_STATUS_API_MAX_AGENT_COUNT"
     )]
     pub(crate) max_agent_count: i64,
+
+    #[clap(long, default_value = "", env = "NYM_NODE_STATUS_API_HM_URL")]
+    pub(crate) hm_url: String,
 }
 
 fn parse_duration(arg: &str) -> Result<std::time::Duration, std::num::ParseIntError> {