@@ -215,6 +215,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
215215 uint32_t _max_block_cpu_usage_threshold_us = 0 ;
216216 uint32_t _max_block_net_usage_threshold_bytes = 0 ;
217217 int32_t _max_scheduled_transaction_time_per_block_ms = 0 ;
218+ bool _disable_persist_until_expired = false ;
218219 fc::time_point _irreversible_block_time;
219220 fc::microseconds _keosd_provider_timeout_us;
220221
@@ -584,7 +585,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
584585 send_response ( e_ptr );
585586 }
586587 } else {
587- if ( persist_until_expired ) {
588+ if ( persist_until_expired && !_disable_persist_until_expired ) {
588589 // if this trx didnt fail/soft-fail and the persist flag is set, store its ID so that we can
589590 // ensure its applied to all future speculative blocks as well.
590591 _unapplied_transactions.add_persisted ( trx );
@@ -727,6 +728,8 @@ void producer_plugin::set_program_options(
727728 " ratio between incoming transactions and deferred transactions when both are queued for execution"
728729 (" incoming-transaction-queue-size-mb" uint16_t >()->default_value ( 1024 ),
729730 " Maximum size (in MiB) of the incoming transaction queue. Exceeding this value will subjectively drop transaction with resource exhaustion."
731+ (" disable-api-persisted-trx" bpo::bool_switch ()->default_value (false ),
732+ " Disable the re-apply of API transactions."
730733 (" producer-threads" uint16_t >()->default_value (config::default_controller_thread_pool_size),
731734 " Number of worker threads in producer thread pool"
732735 (" snapshots-dir" default_value (" snapshots"
@@ -910,6 +913,8 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
910913
911914 my->_incoming_defer_ratio = options.at (" incoming-defer-ratio" as <double >();
912915
916+ my->_disable_persist_until_expired = options.at (" disable-api-persisted-trx" as <bool >();
917+
913918 auto thread_pool_size = options.at ( " producer-threads" as <uint16_t >();
914919 EOS_ASSERT ( thread_pool_size > 0 , plugin_config_exception,
915920 " producer-threads ${num} must be greater than 0" " num"
@@ -1739,10 +1744,79 @@ bool producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point
17391744 return !exhausted;
17401745}
17411746
1747+ namespace {
1748+ // track multiple deadline / transaction cpu exceeded exceptions on unapplied transactions
1749+ class account_failures {
1750+ public:
1751+ constexpr static uint32_t max_failures_per_account = 3 ;
1752+
1753+ void add ( const account_name& n, int64_t exception_code ) {
1754+ if ( exception_code == deadline_exception::code_value || exception_code == tx_cpu_usage_exceeded::code_value ) {
1755+ auto & fa = failed_accounts[n];
1756+ ++fa.num_failures ;
1757+ fa.add ( exception_code );
1758+ }
1759+ }
1760+
1761+ // return true if exceeds max_failures_per_account and should be dropped
1762+ bool failure_limit ( const account_name& n ) {
1763+ auto fitr = failed_accounts.find ( n );
1764+ if ( fitr != failed_accounts.end () && fitr->second .num_failures >= max_failures_per_account ) {
1765+ ++fitr->second .num_failures ;
1766+ return true ;
1767+ }
1768+ return false ;
1769+ }
1770+
1771+ void report () const {
1772+ if ( _log.is_enabled ( fc::log_level::debug ) ) {
1773+ for ( const auto & e : failed_accounts ) {
1774+ std::string reason;
1775+ if ( e.second .num_failures > max_failures_per_account ) {
1776+ reason.clear ();
1777+ if ( e.second .is_deadline () ) reason += " deadline"
1778+ if ( e.second .is_tx_cpu_usage () ) {
1779+ if ( !reason.empty () ) reason += " , "
1780+ reason += " tx_cpu_usage"
1781+ }
1782+ fc_dlog ( _log, " Dropped ${n} trxs, account: ${a}, reason: ${r} exceeded"
1783+ (" n" second .num_failures - max_failures_per_account)(" a" first )(" r"
1784+ }
1785+ }
1786+ }
1787+ }
1788+
1789+ private:
1790+ struct account_failure {
1791+ enum class ex_fields : uint8_t {
1792+ ex_deadline_exception = 1 ,
1793+ ex_tx_cpu_usage_exceeded = 2
1794+ };
1795+
1796+ void add (int64_t exception_code = 0 ) {
1797+ if ( exception_code == tx_cpu_usage_exceeded::code_value ) {
1798+ ex_flags = set_field ( ex_flags, ex_fields::ex_tx_cpu_usage_exceeded );
1799+ } else if ( exception_code == deadline_exception::code_value ) {
1800+ ex_flags = set_field ( ex_flags, ex_fields::ex_deadline_exception );
1801+ }
1802+ }
1803+ bool is_deadline () const { return has_field ( ex_flags, ex_fields::ex_deadline_exception ); }
1804+ bool is_tx_cpu_usage () const { return has_field ( ex_flags, ex_fields::ex_tx_cpu_usage_exceeded ); }
1805+
1806+ uint32_t num_failures = 0 ;
1807+ uint8_t ex_flags = 0 ;
1808+ };
1809+
1810+ std::map<account_name, account_failure> failed_accounts;
1811+ };
1812+
1813+ } // anonymous namespace
1814+
17421815bool producer_plugin_impl::process_unapplied_trxs ( const fc::time_point& deadline )
17431816{
17441817 bool exhausted = false ;
17451818 if ( !_unapplied_transactions.empty () ) {
1819+ account_failures account_fails;
17461820 chain::controller& chain = chain_plug->chain ();
17471821 int num_applied = 0 , num_failed = 0 , num_processed = 0 ;
17481822 auto unapplied_trxs_size = _unapplied_transactions.size ();
@@ -1761,11 +1835,16 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
17611835 try {
17621836 auto start = fc::time_point::now ();
17631837 auto trx_deadline = start + fc::milliseconds ( _max_transaction_time_ms );
1838+ if ( account_fails.failure_limit ( trx->packed_trx ()->get_transaction ().first_authorizer () ) ) {
1839+ ++num_failed;
1840+ itr = _unapplied_transactions.erase ( itr );
1841+ continue ;
1842+ }
17641843
17651844 auto prev_billed_cpu_time_us = trx->billed_cpu_time_us ;
17661845 if ( prev_billed_cpu_time_us > 0 ) {
1767- auto prev_billed_plus50 = prev_billed_cpu_time_us + EOS_PERCENT ( prev_billed_cpu_time_us, 50 * config::percent_1 );
1768- auto trx_dl = start + fc::microseconds ( prev_billed_plus50 );
1846+ auto prev_billed_plus100 = prev_billed_cpu_time_us + EOS_PERCENT ( prev_billed_cpu_time_us, 100 * config::percent_1 );
1847+ auto trx_dl = start + fc::microseconds ( prev_billed_plus100 );
17691848 if ( trx_dl < trx_deadline ) trx_deadline = trx_dl;
17701849 }
17711850 bool deadline_is_subjective = false ;
@@ -1784,10 +1863,12 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
17841863 break ;
17851864 }
17861865 } else {
1866+ auto failure_code = trace->except ->code ();
17871867 // this failed our configured maximum transaction time, we don't want to replay it
17881868 fc_dlog ( _log, " Failed ${c} trx, prev billed: ${p}us, ran: ${r}us, id: ${id}"
17891869 (" c" except ->code ())(" p"
17901870 (" r" fc::time_point::now () - start)(" id" id ()) );
1871+ account_fails.add ( trx->packed_trx ()->get_transaction ().first_authorizer (), failure_code );
17911872 ++num_failed;
17921873 itr = _unapplied_transactions.erase ( itr );
17931874 continue ;
@@ -1803,6 +1884,7 @@ bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin
18031884
18041885 fc_dlog ( _log, " Processed ${m} of ${n} previously applied transactions, Applied ${applied}, Failed/Dropped ${failed}"
18051886 (" m" " n" " applied" " failed"
1887+ account_fails.report ();
18061888 }
18071889 return !exhausted;
18081890}
0 commit comments