poh, pack: adjust cost limits dynamically

Author: jherrera-jump

agave

@@ -251,6 +251,7 @@
 | pack_​cus_​scheduled | `histogram` | The number of cost units scheduled for each block pack produced.  This can be higher than the block limit because of returned CUs. |
 | pack_​cus_​rebated | `histogram` | The number of compute units rebated for each block pack produced.  Compute units are rebated when a transaction fails prior to execution or requests more compute units than it uses. |
 | pack_​cus_​net | `histogram` | The net number of cost units (scheduled - rebated) in each block pack produced. |
+| pack_​cus_​pct | `histogram` | The percent of the total block cost limit used for each block pack produced. |
 | pack_​delete_​missed | `counter` | Count of attempts to delete a transaction that wasn't found |
 | pack_​delete_​hit | `counter` | Count of attempts to delete a transaction that was found and deleted |
 

book/api/metrics-generated.md

@@ -1487,7 +1487,7 @@ dynamic_port_range = "8900-9000"
         affinity = "auto"
 
         # Solana has a hard-coded maximum CUs per block limit of
-        # 48,000,000 which works out to around 80,000 transfers a
+        # 50,000,000+ which works out to around 83,000 transfers a
         # second, since each consumes about 1500 CUs.  When
         # benchmarking, this can be the limiting bottleneck of the
         # system, so this option is provided to raise the limit.  If set

src/app/fdctl/config/default.toml

@@ -218,7 +218,7 @@ after_frag( fd_gui_ctx_t *      ctx,
   else if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_POH_PACK ) ) {
     FD_TEST( fd_disco_poh_sig_pkt_type( sig )==POH_PKT_TYPE_BECAME_LEADER );
     fd_became_leader_t * became_leader = (fd_became_leader_t *)ctx->buf;
-    fd_gui_became_leader( ctx->gui, fd_disco_poh_sig_slot( sig ), became_leader->slot_start_ns, became_leader->slot_end_ns, FD_PACK_MAX_COST_PER_BLOCK, became_leader->max_microblocks_in_slot );
+    fd_gui_became_leader( ctx->gui, fd_disco_poh_sig_slot( sig ), became_leader->slot_start_ns, became_leader->slot_end_ns, became_leader->limits.slot_max_cost, became_leader->max_microblocks_in_slot );
   } else if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_PACK_BANK ) ) {
     if( FD_LIKELY( fd_disco_poh_sig_pkt_type( sig )==POH_PKT_TYPE_MICROBLOCK ) ) {
       FD_TEST( sz<ULONG_MAX );

src/disco/gui/fd_gui_tile.c

@@ -153,7 +153,7 @@ extern const fd_metrics_meta_t FD_METRICS_ALL_LINK_IN[FD_METRICS_ALL_LINK_IN_TOT
 #define FD_METRICS_ALL_LINK_OUT_TOTAL (1UL)
 extern const fd_metrics_meta_t FD_METRICS_ALL_LINK_OUT[FD_METRICS_ALL_LINK_OUT_TOTAL];
 
-#define FD_METRICS_TOTAL_SZ (8UL*229UL)
+#define FD_METRICS_TOTAL_SZ (8UL*246UL)
 
 #define FD_METRICS_TILE_KIND_CNT 17
 extern const char * FD_METRICS_TILE_KIND_NAMES[FD_METRICS_TILE_KIND_CNT];

src/disco/metrics/generated/fd_metrics_all.h

@@ -69,6 +69,7 @@ const fd_metrics_meta_t FD_METRICS_PACK[FD_METRICS_PACK_TOTAL] = {
     DECLARE_METRIC_HISTOGRAM_NONE( PACK_CUS_SCHEDULED ),
     DECLARE_METRIC_HISTOGRAM_NONE( PACK_CUS_REBATED ),
     DECLARE_METRIC_HISTOGRAM_NONE( PACK_CUS_NET ),
+    DECLARE_METRIC_HISTOGRAM_NONE( PACK_CUS_PCT ),
     DECLARE_METRIC( PACK_DELETE_MISSED, COUNTER ),
     DECLARE_METRIC( PACK_DELETE_HIT, COUNTER ),
 };

src/disco/metrics/generated/fd_metrics_pack.c

@@ -226,19 +226,27 @@
 #define FD_METRICS_HISTOGRAM_PACK_CUS_NET_DESC "The net number of cost units (scheduled - rebated) in each block pack produced."
 #define FD_METRICS_HISTOGRAM_PACK_CUS_NET_CVT  (FD_METRICS_CONVERTER_NONE)
 #define FD_METRICS_HISTOGRAM_PACK_CUS_NET_MIN  (1000000UL)
-#define FD_METRICS_HISTOGRAM_PACK_CUS_NET_MAX  (48000000UL)
+#define FD_METRICS_HISTOGRAM_PACK_CUS_NET_MAX  (60000000UL)
 
-#define FD_METRICS_COUNTER_PACK_DELETE_MISSED_OFF  (227UL)
+#define FD_METRICS_HISTOGRAM_PACK_CUS_PCT_OFF  (227UL)
+#define FD_METRICS_HISTOGRAM_PACK_CUS_PCT_NAME "pack_cus_pct"
+#define FD_METRICS_HISTOGRAM_PACK_CUS_PCT_TYPE (FD_METRICS_TYPE_HISTOGRAM)
+#define FD_METRICS_HISTOGRAM_PACK_CUS_PCT_DESC "The percent of the total block cost limit used for each block pack produced."
+#define FD_METRICS_HISTOGRAM_PACK_CUS_PCT_CVT  (FD_METRICS_CONVERTER_NONE)
+#define FD_METRICS_HISTOGRAM_PACK_CUS_PCT_MIN  (0UL)
+#define FD_METRICS_HISTOGRAM_PACK_CUS_PCT_MAX  (100UL)
+
+#define FD_METRICS_COUNTER_PACK_DELETE_MISSED_OFF  (244UL)
 #define FD_METRICS_COUNTER_PACK_DELETE_MISSED_NAME "pack_delete_missed"
 #define FD_METRICS_COUNTER_PACK_DELETE_MISSED_TYPE (FD_METRICS_TYPE_COUNTER)
 #define FD_METRICS_COUNTER_PACK_DELETE_MISSED_DESC "Count of attempts to delete a transaction that wasn't found"
 #define FD_METRICS_COUNTER_PACK_DELETE_MISSED_CVT  (FD_METRICS_CONVERTER_NONE)
 
-#define FD_METRICS_COUNTER_PACK_DELETE_HIT_OFF  (228UL)
+#define FD_METRICS_COUNTER_PACK_DELETE_HIT_OFF  (245UL)
 #define FD_METRICS_COUNTER_PACK_DELETE_HIT_NAME "pack_delete_hit"
 #define FD_METRICS_COUNTER_PACK_DELETE_HIT_TYPE (FD_METRICS_TYPE_COUNTER)
 #define FD_METRICS_COUNTER_PACK_DELETE_HIT_DESC "Count of attempts to delete a transaction that was found and deleted"
 #define FD_METRICS_COUNTER_PACK_DELETE_HIT_CVT  (FD_METRICS_CONVERTER_NONE)
 
-#define FD_METRICS_PACK_TOTAL (69UL)
+#define FD_METRICS_PACK_TOTAL (70UL)
 extern const fd_metrics_meta_t FD_METRICS_PACK[FD_METRICS_PACK_TOTAL];

src/disco/metrics/generated/fd_metrics_pack.h

@@ -362,9 +362,12 @@ metric introduced.
         <!-- Note: The switch from cost units to compute units here is deliberate.  Of the various types of cost units, only compute units can be rebated. -->
         <summary>The number of compute units rebated for each block pack produced.  Compute units are rebated when a transaction fails prior to execution or requests more compute units than it uses.</summary>
     </histogram>
-    <histogram name="CusNet" min="1000000" max="48000000">
+    <histogram name="CusNet" min="1000000" max="60000000">
         <summary>The net number of cost units (scheduled - rebated) in each block pack produced.</summary>
     </histogram>
+    <histogram name="CusPct" min="0" max="100">
+        <summary>The percent of the total block cost limit used for each block pack produced.</summary>
+    </histogram>
 
     <counter name="DeleteMissed" summary="Count of attempts to delete a transaction that wasn't found" />
     <counter name="DeleteHit" summary="Count of attempts to delete a transaction that was found and deleted" />

src/disco/metrics/metrics.xml

@@ -544,6 +544,7 @@ struct fd_pack_private {
   fd_histf_t scheduled_cus_per_block[ 1 ];
   fd_histf_t rebated_cus_per_block  [ 1 ];
   fd_histf_t net_cus_per_block      [ 1 ];
+  fd_histf_t pct_cus_per_block      [ 1 ];
   ulong      cumulative_rebated_cus;
 
 
@@ -633,7 +634,7 @@ fd_pack_new( void                   * mem,
              ulong                    bundle_meta_sz,
              ulong                    bank_tile_cnt,
              fd_pack_limits_t const * limits,
-             fd_rng_t                * rng           ) {
+             fd_rng_t               * rng           ) {
 
   int enable_bundles = !!bundle_meta_sz;
   ulong extra_depth        = fd_ulong_if( enable_bundles, 1UL+2UL*FD_PACK_MAX_TXN_PER_BUNDLE, 1UL );
@@ -749,6 +750,8 @@ fd_pack_new( void                   * mem,
                                                FD_MHIST_MAX( PACK, CUS_REBATED   ) );
   fd_histf_new( pack->net_cus_per_block,       FD_MHIST_MIN( PACK, CUS_NET       ),
                                                FD_MHIST_MAX( PACK, CUS_NET       ) );
+  fd_histf_new( pack->pct_cus_per_block,       FD_MHIST_MIN( PACK, CUS_PCT       ),
+                                               FD_MHIST_MAX( PACK, CUS_PCT       ) );
 
   pack->compressed_slot_number = (ushort)(FD_PACK_SKIP_CNT+1);
 
@@ -2337,11 +2340,16 @@ ulong fd_pack_current_block_cost( fd_pack_t const * pack ) { return pack->cumula
 
 
 void
-fd_pack_set_block_limits( fd_pack_t * pack,
-                          ulong       max_microblocks_per_block,
-                          ulong       max_data_bytes_per_block ) {
-  pack->lim->max_microblocks_per_block = max_microblocks_per_block;
-  pack->lim->max_data_bytes_per_block  = max_data_bytes_per_block;
+fd_pack_set_block_limits( fd_pack_t * pack, fd_pack_limits_t const * limits ) {
+  FD_TEST( limits->max_cost_per_block      >= FD_PACK_MAX_COST_PER_BLOCK_LOWER_BOUND      );
+  FD_TEST( limits->max_vote_cost_per_block >= FD_PACK_MAX_VOTE_COST_PER_BLOCK_LOWER_BOUND );
+  FD_TEST( limits->max_write_cost_per_acct >= FD_PACK_MAX_WRITE_COST_PER_ACCT_LOWER_BOUND );
+
+  pack->lim->max_microblocks_per_block = limits->max_microblocks_per_block;
+  pack->lim->max_data_bytes_per_block  = limits->max_data_bytes_per_block;
+  pack->lim->max_cost_per_block        = limits->max_cost_per_block;
+  pack->lim->max_vote_cost_per_block   = limits->max_vote_cost_per_block;
+  pack->lim->max_write_cost_per_acct   = limits->max_write_cost_per_acct;
 }
 
 void
@@ -2391,6 +2399,9 @@ fd_pack_expire_before( fd_pack_t * pack,
 
 void
 fd_pack_end_block( fd_pack_t * pack ) {
+  /* rounded division */
+  ulong pct_cus_per_block = (pack->cumulative_block_cost*100UL + (pack->lim->max_cost_per_block>>1))/pack->lim->max_cost_per_block;
+  fd_histf_sample( pack->pct_cus_per_block,       pct_cus_per_block                                          );
   fd_histf_sample( pack->net_cus_per_block,       pack->cumulative_block_cost                                );
   fd_histf_sample( pack->rebated_cus_per_block,   pack->cumulative_rebated_cus                               );
   fd_histf_sample( pack->scheduled_cus_per_block, pack->cumulative_rebated_cus + pack->cumulative_block_cost );
@@ -2455,6 +2466,7 @@ fd_pack_end_block( fd_pack_t * pack ) {
   FD_MHIST_COPY( PACK, CUS_SCHEDULED,         pack->scheduled_cus_per_block );
   FD_MHIST_COPY( PACK, CUS_REBATED,           pack->rebated_cus_per_block   );
   FD_MHIST_COPY( PACK, CUS_NET,               pack->net_cus_per_block       );
+  FD_MHIST_COPY( PACK, CUS_PCT,               pack->pct_cus_per_block       );
 }
 
 static void

src/disco/pack/fd_pack.c

@@ -17,9 +17,28 @@
 
 /* NOTE: THE FOLLOWING CONSTANTS ARE CONSENSUS CRITICAL AND CANNOT BE
    CHANGED WITHOUT COORDINATING WITH ANZA. */
-#define FD_PACK_MAX_COST_PER_BLOCK      (48000000UL)
-#define FD_PACK_MAX_VOTE_COST_PER_BLOCK (36000000UL)
-#define FD_PACK_MAX_WRITE_COST_PER_ACCT (12000000UL)
+
+/* These are bounds on known limits. Upper bound values are used to
+   calculate memory footprints while lower bounds are used for
+   initializing consensus-dependent logic and invariant checking.  As a
+   leader, it is OK to produce blocks using limits smaller than the
+   active on-chain limits. Replay should always use the correct
+   chain-derived limits.
+
+   The actual limits used by pack may be updated dynamically to some
+   in-bounds value. If there is an anticipated feature activation that
+   changes these limits, the upper bound should be the largest
+   anticipated value while the lower bound should be the current active
+   limit. For Frankendancer, the actual value used for consensus will be
+   retreived from Agave. */
+#define FD_PACK_MAX_COST_PER_BLOCK_LOWER_BOUND      (48000000UL)
+#define FD_PACK_MAX_VOTE_COST_PER_BLOCK_LOWER_BOUND (36000000UL)
+#define FD_PACK_MAX_WRITE_COST_PER_ACCT_LOWER_BOUND (12000000UL)
+
+#define FD_PACK_MAX_COST_PER_BLOCK_UPPER_BOUND      (60000000UL) /* simd 0256 */
+#define FD_PACK_MAX_VOTE_COST_PER_BLOCK_UPPER_BOUND (36000000UL)
+#define FD_PACK_MAX_WRITE_COST_PER_ACCT_UPPER_BOUND (12000000UL)
+
 #define FD_PACK_FEE_PER_SIGNATURE           (5000UL) /* In lamports */
 
 /* Each block is limited to 32k parity shreds.  We don't want pack to
@@ -202,20 +221,26 @@ FD_FN_PURE ulong fd_pack_current_block_cost( fd_pack_t const * pack );
 FD_FN_PURE ulong fd_pack_bank_tile_cnt( fd_pack_t const * pack );
 
 /* fd_pack_set_block_limits: Updates the limits provided fd_pack_new to
-   the new values.  Any future microblocks produced by this pack object
-   will not cause a block to have more than max_microblocks_per_block
-   non-empty microblocks or more than max_data_bytes_per_block data
-   bytes (counting microblock headers as before).  Limits are inclusive,
-   as per usual (i.e. a block may have exactly
-   max_microblocks_per_block microblocks, but not more).  pack must be
-   a valid local join.
+   these new values.  Any future microblocks produced by this pack
+   object will not cause a block to have more than
+   limits->max_microblocks_per_block non-empty microblocks or more than
+   limits->max_data_bytes_per_block data bytes (counting microblock
+   headers as before).  future microblocks will also exclude those that
+   cause the total block cost to exceed limits->max_cost_per_block.
+   Similarly those that cause the total vote-only cost to exceed
+   limits->max_vote_cost_per_block. Also, those that cause the total
+   per-account, per block write cost to exceed
+   limits->max_write_cost_per_acct.  Note that
+   limits->max_txn_per_microblock is ignored. Limits are inclusive, as
+   per usual (i.e. a block may have exactly max_microblocks_per_block
+   microblocks, but not more).  pack must be a valid local join.
 
    The typical place to call this is immediately after
    fd_pack_end_block; if this is called after some microblocks have been
    produced for the current block, and the current block already exceeds
    the limits, all the remaining microblocks in the block will be empty,
    but the call is valid. */
-void fd_pack_set_block_limits( fd_pack_t * pack, ulong max_microblocks_per_block, ulong max_data_bytes_per_block );
+void fd_pack_set_block_limits( fd_pack_t * pack, fd_pack_limits_t const * limits );
 
 /* Return values for fd_pack_insert_txn_fini:  Non-negative values
    indicate the transaction was accepted and may be returned in a future