diff --git a/stats/config/charts.json b/stats/config/charts.json index 66340dec4..827004a7e 100644 --- a/stats/config/charts.json +++ b/stats/config/charts.json @@ -22,6 +22,11 @@ "title": "Transactions (24h)", "description": "Number of new transactions within last 24 hours" }, + "new_operational_txns_24h": { + "enabled": false, + "title": "Operational transactions (24h)", + "description": "Number of new transactions within last 24 hours without block creation transactions" + }, "pending_txns_30m": { "title": "Pending transactions (30m)", "description": "Transactions in the past 30 minutes waiting to be processed by the network" diff --git a/stats/config/layout.json b/stats/config/layout.json index b9e115b87..a7cba95a8 100644 --- a/stats/config/layout.json +++ b/stats/config/layout.json @@ -15,6 +15,7 @@ "total_operational_txns", "total_verified_contracts", "new_txns_24h", + "new_operational_txns_24h", "pending_txns_30m", "txns_fee_24h", "average_txn_fee_24h" diff --git a/stats/stats-proto/proto/stats.proto b/stats/stats-proto/proto/stats.proto index 89be8260c..fcc55959d 100644 --- a/stats/stats-proto/proto/stats.proto +++ b/stats/stats-proto/proto/stats.proto @@ -98,6 +98,7 @@ message TransactionsPageStats { optional Counter transactions_fee_24h = 2; optional Counter average_transactions_fee_24h = 3; optional Counter transactions_24h = 4; + optional Counter operational_transactions_24h = 5; } message GetContractsPageStatsRequest {} diff --git a/stats/stats-proto/swagger/stats.swagger.yaml b/stats/stats-proto/swagger/stats.swagger.yaml index 63a197876..189612bb9 100644 --- a/stats/stats-proto/swagger/stats.swagger.yaml +++ b/stats/stats-proto/swagger/stats.swagger.yaml @@ -306,3 +306,5 @@ definitions: $ref: '#/definitions/v1Counter' transactions_24h: $ref: '#/definitions/v1Counter' + operational_transactions_24h: + $ref: '#/definitions/v1Counter' diff --git a/stats/stats-server/src/read_service.rs b/stats/stats-server/src/read_service.rs index 929cbadf7..13c5463e9 100644 --- a/stats/stats-server/src/read_service.rs +++ b/stats/stats-server/src/read_service.rs @@ -16,9 +16,10 @@ use proto_v1::stats_service_server::StatsService; use sea_orm::{DatabaseConnection, DbErr}; use stats::{ counters::{ - AverageBlockTime, AverageTxnFee24h, NewContracts24h, NewTxns24h, NewVerifiedContracts24h, - PendingTxns30m, TotalAddresses, TotalBlocks, TotalContracts, TotalOperationalTxns, - TotalTxns, TotalVerifiedContracts, TxnsFee24h, YesterdayOperationalTxns, YesterdayTxns, + AverageBlockTime, AverageTxnFee24h, NewContracts24h, NewOperationalTxns24h, NewTxns24h, + NewVerifiedContracts24h, PendingTxns30m, TotalAddresses, TotalBlocks, TotalContracts, + TotalOperationalTxns, TotalTxns, TotalVerifiedContracts, TxnsFee24h, + YesterdayOperationalTxns, YesterdayTxns, }, data_source::{types::BlockscoutMigrations, UpdateContext, UpdateParameters}, lines::{NewOperationalTxnsWindow, NewTxnsWindow, NEW_TXNS_WINDOW_RANGE}, @@ -189,12 +190,14 @@ impl ReadService { transactions_fee_24h: None, average_transactions_fee_24h: None, transactions_24h: None, + operational_transactions_24h: None, }; vec![ PendingTxns30m::name(), TxnsFee24h::name(), AverageTxnFee24h::name(), NewTxns24h::name(), + NewOperationalTxns24h::name(), ] } } @@ -480,17 +483,20 @@ impl StatsService for ReadService { transactions_fee_24h, average_transactions_fee_24h, transactions_24h, + operational_transactions_24h, ) = join!( self.query_counter(PendingTxns30m::name(), now), self.query_counter(TxnsFee24h::name(), now), self.query_counter(AverageTxnFee24h::name(), now), self.query_counter(NewTxns24h::name(), now), + self.query_counter(NewOperationalTxns24h::name(), now), ); Ok(Response::new(proto_v1::TransactionsPageStats { pending_transactions_30m, transactions_fee_24h, average_transactions_fee_24h, transactions_24h, + operational_transactions_24h, })) } diff --git a/stats/stats-server/src/settings.rs b/stats/stats-server/src/settings.rs index 88acb45c9..935e6953b 100644 --- a/stats/stats-server/src/settings.rs +++ b/stats/stats-server/src/settings.rs @@ -8,7 +8,7 @@ use cron::Schedule; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; use stats::{ - counters::{TotalOperationalTxns, YesterdayOperationalTxns}, + counters::{NewOperationalTxns24h, TotalOperationalTxns, YesterdayOperationalTxns}, lines::{NewOperationalTxns, NewOperationalTxnsWindow, OperationalTxnsGrowth}, ChartProperties, IndexingStatus, }; @@ -141,10 +141,11 @@ pub fn handle_enable_all_arbitrum( if enable_all_arbitrum { for enable_key in [ NewOperationalTxns::key().name(), - OperationalTxnsGrowth::key().name(), + NewOperationalTxnsWindow::key().name(), TotalOperationalTxns::key().name(), + NewOperationalTxns24h::key().name(), + OperationalTxnsGrowth::key().name(), YesterdayOperationalTxns::key().name(), - NewOperationalTxnsWindow::key().name(), ] { let settings = match ( charts.lines.get_mut(enable_key), diff --git a/stats/stats-server/tests/it/chart_endpoints/counters.rs b/stats/stats-server/tests/it/chart_endpoints/counters.rs index 70750fccd..991ab2dd0 100644 --- a/stats/stats-server/tests/it/chart_endpoints/counters.rs +++ b/stats/stats-server/tests/it/chart_endpoints/counters.rs @@ -33,6 +33,7 @@ pub async fn test_counters_ok(base: Url) { // on a different page // "yesterdayTxns", "newTxns24h", + "newOperationalTxns24h", "pendingTxns30m", "txnsFee24h", "averageTxnFee24h", diff --git a/stats/stats-server/tests/it/chart_endpoints/mod.rs b/stats/stats-server/tests/it/chart_endpoints/mod.rs index 062d28740..2aa899b31 100644 --- a/stats/stats-server/tests/it/chart_endpoints/mod.rs +++ b/stats/stats-server/tests/it/chart_endpoints/mod.rs @@ -52,7 +52,7 @@ async fn test_chart_endpoints_ok() { test_lines_ok(base.clone()).boxed(), test_counters_ok(base.clone()).boxed(), test_main_page_ok(base.clone(), true).boxed(), - test_transactions_page_ok(base.clone()).boxed(), + test_transactions_page_ok(base.clone(), true).boxed(), test_contracts_page_ok(base).boxed(), ] .into_iter() @@ -85,7 +85,7 @@ async fn test_chart_endpoints_work_with_not_indexed_blockscout() { let tests: JoinSet<_> = [ test_main_page_ok(base.clone(), true).boxed(), - test_transactions_page_ok(base.clone()).boxed(), + test_transactions_page_ok(base.clone(), true).boxed(), test_contracts_page_ok(base).boxed(), ] .into_iter() @@ -109,5 +109,11 @@ async fn test_chart_endpoints_work_with_disabled_arbitrum() { // Sleep until server will start and calculate all values tokio::time::sleep(std::time::Duration::from_secs(8)).await; - test_main_page_ok(base, false).await; + let tests: JoinSet<_> = [ + test_main_page_ok(base.clone(), false).boxed(), + test_transactions_page_ok(base, false).boxed(), + ] + .into_iter() + .collect(); + run_consolidated_tests(tests, test_name).await; } diff --git a/stats/stats-server/tests/it/chart_endpoints/transactions_page.rs b/stats/stats-server/tests/it/chart_endpoints/transactions_page.rs index 5347a2843..1e122e007 100644 --- a/stats/stats-server/tests/it/chart_endpoints/transactions_page.rs +++ b/stats/stats-server/tests/it/chart_endpoints/transactions_page.rs @@ -5,19 +5,26 @@ use url::Url; use crate::array_of_variables_with_names; -pub async fn test_transactions_page_ok(base: Url) { +pub async fn test_transactions_page_ok(base: Url, expect_arbitrum: bool) { let TransactionsPageStats { pending_transactions_30m, transactions_fee_24h, average_transactions_fee_24h, transactions_24h, + operational_transactions_24h, } = send_get_request(&base, "/api/v1/pages/transactions").await; - let counters = array_of_variables_with_names!([ + let mut counters = array_of_variables_with_names!([ pending_transactions_30m, transactions_fee_24h, average_transactions_fee_24h, transactions_24h, - ]); + ]) + .to_vec(); + if expect_arbitrum { + counters.extend(array_of_variables_with_names!([ + operational_transactions_24h + ])); + } for (name, counter) in counters { #[allow(clippy::expect_fun_call)] let counter = counter.expect(&format!("page counter {} must be available", name)); diff --git a/stats/stats/src/charts/counters/mod.rs b/stats/stats/src/charts/counters/mod.rs index ec1cb4883..cec864668 100644 --- a/stats/stats/src/charts/counters/mod.rs +++ b/stats/stats/src/charts/counters/mod.rs @@ -1,8 +1,13 @@ +#[cfg(test)] +mod mock; +mod yesterday_operational_txns; + mod average_block_time; mod completed_txns; mod last_new_contracts; mod last_new_verified_contracts; mod new_contracts_24h; +mod new_operational_txns_24h; mod new_verified_contracts_24h; mod pending_txns; mod total_accounts; @@ -18,15 +23,12 @@ mod total_verified_contracts; mod txns_stats_24h; mod yesterday_txns; -#[cfg(test)] -mod mock; -mod yesterday_operational_txns; - pub use average_block_time::AverageBlockTime; pub use completed_txns::CompletedTxns; pub use last_new_contracts::LastNewContracts; pub use last_new_verified_contracts::LastNewVerifiedContracts; pub use new_contracts_24h::NewContracts24h; +pub use new_operational_txns_24h::NewOperationalTxns24h; pub use new_verified_contracts_24h::NewVerifiedContracts24h; pub use pending_txns::PendingTxns30m; pub use total_accounts::TotalAccounts; @@ -39,7 +41,6 @@ pub use total_operational_txns::TotalOperationalTxns; pub use total_tokens::TotalTokens; pub use total_txns::TotalTxns; pub use total_verified_contracts::TotalVerifiedContracts; -pub(crate) use txns_stats_24h::TxnsStatsValue; pub use txns_stats_24h::{ average_txn_fee_24h::AverageTxnFee24h, new_txns_24h::NewTxns24h, txns_fee_24h::TxnsFee24h, }; @@ -49,6 +50,7 @@ pub use yesterday_txns::YesterdayTxns; pub(crate) use total_blocks::TotalBlocksInt; pub(crate) use total_operational_txns::CalculateOperationalTxns; pub(crate) use total_txns::TotalTxnsInt; +pub(crate) use txns_stats_24h::{new_txns_24h::NewTxns24hInt, TxnsStatsValue}; #[cfg(test)] pub use mock::MockCounter; diff --git a/stats/stats/src/charts/counters/new_operational_txns_24h.rs b/stats/stats/src/charts/counters/new_operational_txns_24h.rs new file mode 100644 index 000000000..7fff74bd4 --- /dev/null +++ b/stats/stats/src/charts/counters/new_operational_txns_24h.rs @@ -0,0 +1,99 @@ +use std::ops::Range; + +use crate::{ + data_source::{ + kinds::{ + data_manipulation::map::{Map, MapParseTo, StripWrapper}, + local_db::DirectPointLocalDbChartSource, + remote_db::{PullOne24hCached, RemoteDatabaseSource, StatementFromRange}, + }, + types::{BlockscoutMigrations, WrappedValue}, + }, + utils::sql_with_range_filter_opt, + ChartProperties, IndexingStatus, MissingDatePolicy, Named, +}; +use chrono::{DateTime, NaiveDate, Utc}; +use entity::sea_orm_active_enums::ChartType; +use sea_orm::{DbBackend, Statement}; + +use super::{CalculateOperationalTxns, NewTxns24hInt}; + +pub struct NewBlocks24hStatement; + +impl StatementFromRange for NewBlocks24hStatement { + fn get_statement( + range: Option>>, + _completed_migrations: &BlockscoutMigrations, + ) -> Statement { + sql_with_range_filter_opt!( + DbBackend::Postgres, + r#" + SELECT + COUNT(*)::TEXT as value + FROM public.blocks + WHERE + blocks.timestamp != to_timestamp(0) AND + consensus = true {filter}; + "#, + [], + "blocks.timestamp", + range + ) + } +} + +// caching is not needed but I don't want to make another type just for this +// +// btw the caching should solve the problem with not storing `NewBlocks24h` in local +// db while not introducing any new unnecessary entries to the db. so it should be safe +// to use this in other places as well (in terms of efficiency) +pub type NewBlocks24h = + RemoteDatabaseSource>>; + +pub type NewBlocks24hInt = MapParseTo, i64>; + +pub struct Properties; + +impl Named for Properties { + fn name() -> String { + "newOperationalTxns24h".into() + } +} + +impl ChartProperties for Properties { + type Resolution = NaiveDate; + + fn chart_type() -> ChartType { + ChartType::Counter + } + fn missing_date_policy() -> MissingDatePolicy { + MissingDatePolicy::FillPrevious + } + fn indexing_status_requirement() -> IndexingStatus { + IndexingStatus::NoneIndexed + } +} + +pub type NewOperationalTxns24h = DirectPointLocalDbChartSource< + Map<(NewBlocks24hInt, NewTxns24hInt), CalculateOperationalTxns>, + Properties, +>; + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{point_construction::dt, simple_test::simple_test_counter}; + + #[tokio::test] + #[ignore = "needs database to run"] + async fn update_new_operational_txns_24h() { + simple_test_counter::( + "update_new_operational_txns_24h", + // block at `2022-11-11T00:00:00` is not counted because + // the relation is 'less than' in query + "9", + Some(dt("2022-11-11T00:00:00")), + ) + .await; + } +} diff --git a/stats/stats/src/charts/counters/txns_stats_24h/new_txns_24h.rs b/stats/stats/src/charts/counters/txns_stats_24h/new_txns_24h.rs index 969f6c40e..dffbe63fe 100644 --- a/stats/stats/src/charts/counters/txns_stats_24h/new_txns_24h.rs +++ b/stats/stats/src/charts/counters/txns_stats_24h/new_txns_24h.rs @@ -1,6 +1,6 @@ use crate::{ data_source::kinds::{ - data_manipulation::map::{Map, MapFunction}, + data_manipulation::map::{Map, MapFunction, MapParseTo}, local_db::DirectPointLocalDbChartSource, }, types::TimespanValue, @@ -51,6 +51,8 @@ impl ChartProperties for Properties { pub type NewTxns24h = DirectPointLocalDbChartSource; +pub type NewTxns24hInt = MapParseTo; + #[cfg(test)] mod tests { use super::*; @@ -67,7 +69,8 @@ mod tests { async fn update_new_txns_24h_2() { simple_test_counter::( "update_new_txns_24h_2", - // block at `2022-11-11T00:00:00` is not counted because sql is not that precise :/ + // block at `2022-11-11T00:00:00` is not counted because + // the relation is 'less than' in query "12", Some(dt("2022-11-11T00:00:00")), ) diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs index feb178a3d..4ec24a10f 100644 --- a/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/mod.rs @@ -17,11 +17,13 @@ mod parse; mod strip_extension; mod to_string; mod unwrap_or; +mod wrapper; pub use parse::MapParseTo; pub use strip_extension::StripExt; pub use to_string::MapToString; pub use unwrap_or::UnwrapOr; +pub use wrapper::StripWrapper; /// Apply `F` to each value queried from data source `D` pub struct Map(PhantomData<(D, F)>) diff --git a/stats/stats/src/data_source/kinds/data_manipulation/map/wrapper.rs b/stats/stats/src/data_source/kinds/data_manipulation/map/wrapper.rs new file mode 100644 index 000000000..6108fcb1a --- /dev/null +++ b/stats/stats/src/data_source/kinds/data_manipulation/map/wrapper.rs @@ -0,0 +1,53 @@ +use sea_orm::TryGetable; + +use crate::{ + data_source::{kinds::data_manipulation::map::MapFunction, types::WrappedValue}, + types::TimespanValue, + ChartError, +}; + +use super::Map; + +pub struct StripWrapperFunction; + +impl MapFunction>>> + for StripWrapperFunction +where + Resolution: Send, + Value: TryGetable + Send, +{ + type Output = Vec>; + + fn function( + inner_data: Vec>>, + ) -> Result>, ChartError> { + Ok(inner_data + .into_iter() + .map(|p| TimespanValue { + timespan: p.timespan, + value: p.value.into_inner(), + }) + .collect::>()) + } +} + +impl MapFunction>> + for StripWrapperFunction +where + Resolution: Send, + Value: TryGetable + Send, +{ + type Output = TimespanValue; + + fn function( + inner_data: TimespanValue>, + ) -> Result { + Ok(TimespanValue { + timespan: inner_data.timespan, + value: inner_data.value.into_inner(), + }) + } +} + +/// Remove [`WrappedValue`] +pub type StripWrapper = Map; diff --git a/stats/stats/src/data_source/kinds/remote_db/query/one.rs b/stats/stats/src/data_source/kinds/remote_db/query/one.rs index b0cf95ae3..8e4c4d6ae 100644 --- a/stats/stats/src/data_source/kinds/remote_db/query/one.rs +++ b/stats/stats/src/data_source/kinds/remote_db/query/one.rs @@ -93,7 +93,7 @@ where } } -/// Will reuse result for the same produced query within one update +/// Will reuse result for the same produced query **within one update** /// (based on update context) pub struct PullOne24hCached(PhantomData<(S, Value)>) where diff --git a/stats/stats/src/data_source/types.rs b/stats/stats/src/data_source/types.rs index 3724d6b3b..ed36b9e2f 100644 --- a/stats/stats/src/data_source/types.rs +++ b/stats/stats/src/data_source/types.rs @@ -171,15 +171,32 @@ macro_rules! impl_cacheable { }; } +impl_cacheable!(TxnsStatsValue, ValueTxnsStats); +// for testing impl_cacheable!(String, ValueString); impl_cacheable!(Option, ValueOptionF64); -impl_cacheable!(TxnsStatsValue, ValueTxnsStats); +// To allow using the scalar(?) types in context requiring +// `FromQueryResult` #[derive(Debug, Clone, FromQueryResult, PartialEq, Eq, PartialOrd, Ord)] pub struct WrappedValue { pub value: V, } +impl From for WrappedValue { + fn from(value: V) -> Self { + WrappedValue { value } + } +} + +impl WrappedValue { + pub fn into_inner(self) -> V { + self.value + } +} + +impl Copy for WrappedValue {} + macro_rules! impl_cacheable_wrapped { ($type: ty, $cache_value_variant:ident) => { impl Cacheable for $type { @@ -284,11 +301,16 @@ mod tests { let stmt_a = Statement::from_string(DbBackend::Sqlite, "abcde"); let stmt_b = Statement::from_string(DbBackend::Sqlite, "edcba"); - let val_1 = Some(1.2); + let val_1 = Some(1.2).into(); let val_2 = "kekekek".to_string(); - cache.insert::>(&stmt_a, val_1).await; - assert_eq!(cache.get::>(&stmt_a).await, Some(val_1)); + cache + .insert::>>(&stmt_a, val_1) + .await; + assert_eq!( + cache.get::>>(&stmt_a).await, + Some(val_1) + ); assert_eq!(cache.get::(&stmt_a).await, None); cache.insert::>(&stmt_a, None).await; @@ -299,8 +321,13 @@ mod tests { assert_eq!(cache.get::>(&stmt_a).await, None); assert_eq!(cache.get::(&stmt_a).await, Some(val_2.clone())); - cache.insert::>(&stmt_b, val_1).await; - assert_eq!(cache.get::>(&stmt_b).await, Some(val_1)); + cache + .insert::>>(&stmt_b, val_1) + .await; + assert_eq!( + cache.get::>>(&stmt_b).await, + Some(val_1) + ); assert_eq!(cache.get::(&stmt_b).await, None); assert_eq!(cache.get::>(&stmt_a).await, None); assert_eq!(cache.get::(&stmt_a).await, Some(val_2)); diff --git a/stats/stats/src/update_groups.rs b/stats/stats/src/update_groups.rs index 59980c1a4..d730be53a 100644 --- a/stats/stats/src/update_groups.rs +++ b/stats/stats/src/update_groups.rs @@ -156,7 +156,12 @@ construct_update_group!(TxnsSuccessRateGroup { }); construct_update_group!(TxnsStats24hGroup { - charts: [AverageTxnFee24h, NewTxns24h, TxnsFee24h,] + charts: [ + AverageTxnFee24h, + NewTxns24h, + TxnsFee24h, + NewOperationalTxns24h, + ] }); construct_update_group!(NewAccountsGroup {