Skip to content

poh, pack: adjust cost limits dynamically #4702

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jherrera-jump merged 1 commit into from
Apr 14, 2025
Merged

poh, pack: adjust cost limits dynamically #4702

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion agave
Open in desktop
Submodule agave updated 4 files
+1 −0 Cargo.lock
+1 −1 core/src/banking_simulation.rs
+1 −0 poh/Cargo.toml
+12 −3 poh/src/firedancer_poh_recorder.rs
1 change: 1 addition & 0 deletions book/api/metrics-generated.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,7 @@
| pack_​cus_​scheduled | `histogram` | The number of cost units scheduled for each block pack produced. This can be higher than the block limit because of returned CUs. |
| pack_​cus_​rebated | `histogram` | The number of compute units rebated for each block pack produced. Compute units are rebated when a transaction fails prior to execution or requests more compute units than it uses. |
| pack_​cus_​net | `histogram` | The net number of cost units (scheduled - rebated) in each block pack produced. |
| pack_​cus_​pct | `histogram` | The percent of the total block cost limit used for each block pack produced. |
| pack_​delete_​missed | `counter` | Count of attempts to delete a transaction that wasn't found |
| pack_​delete_​hit | `counter` | Count of attempts to delete a transaction that was found and deleted |

Expand Down
2 changes: 1 addition & 1 deletion src/app/fdctl/config/default.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1487,7 +1487,7 @@ dynamic_port_range = "8900-9000"
affinity = "auto"

# Solana has a hard-coded maximum CUs per block limit of
# 48,000,000 which works out to around 80,000 transfers a
# 50,000,000+ which works out to around 83,000 transfers a
# second, since each consumes about 1500 CUs. When
# benchmarking, this can be the limiting bottleneck of the
# system, so this option is provided to raise the limit. If set
Expand Down
2 changes: 1 addition & 1 deletion src/disco/gui/fd_gui_tile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ after_frag( fd_gui_ctx_t * ctx,
else if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_POH_PACK ) ) {
FD_TEST( fd_disco_poh_sig_pkt_type( sig )==POH_PKT_TYPE_BECAME_LEADER );
fd_became_leader_t * became_leader = (fd_became_leader_t *)ctx->buf;
fd_gui_became_leader( ctx->gui, fd_disco_poh_sig_slot( sig ), became_leader->slot_start_ns, became_leader->slot_end_ns, FD_PACK_MAX_COST_PER_BLOCK, became_leader->max_microblocks_in_slot );
fd_gui_became_leader( ctx->gui, fd_disco_poh_sig_slot( sig ), became_leader->slot_start_ns, became_leader->slot_end_ns, became_leader->limits.slot_max_cost, became_leader->max_microblocks_in_slot );
} else if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_PACK_BANK ) ) {
if( FD_LIKELY( fd_disco_poh_sig_pkt_type( sig )==POH_PKT_TYPE_MICROBLOCK ) ) {
FD_TEST( sz<ULONG_MAX );
Expand Down
2 changes: 1 addition & 1 deletion src/disco/metrics/generated/fd_metrics_all.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ extern const fd_metrics_meta_t FD_METRICS_ALL_LINK_IN[FD_METRICS_ALL_LINK_IN_TOT
#define FD_METRICS_ALL_LINK_OUT_TOTAL (1UL)
extern const fd_metrics_meta_t FD_METRICS_ALL_LINK_OUT[FD_METRICS_ALL_LINK_OUT_TOTAL];

#define FD_METRICS_TOTAL_SZ (8UL*229UL)
#define FD_METRICS_TOTAL_SZ (8UL*246UL)

#define FD_METRICS_TILE_KIND_CNT 17
extern const char * FD_METRICS_TILE_KIND_NAMES[FD_METRICS_TILE_KIND_CNT];
Expand Down
1 change: 1 addition & 0 deletions src/disco/metrics/generated/fd_metrics_pack.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ const fd_metrics_meta_t FD_METRICS_PACK[FD_METRICS_PACK_TOTAL] = {
DECLARE_METRIC_HISTOGRAM_NONE( PACK_CUS_SCHEDULED ),
DECLARE_METRIC_HISTOGRAM_NONE( PACK_CUS_REBATED ),
DECLARE_METRIC_HISTOGRAM_NONE( PACK_CUS_NET ),
DECLARE_METRIC_HISTOGRAM_NONE( PACK_CUS_PCT ),
DECLARE_METRIC( PACK_DELETE_MISSED, COUNTER ),
DECLARE_METRIC( PACK_DELETE_HIT, COUNTER ),
};
16 changes: 12 additions & 4 deletions src/disco/metrics/generated/fd_metrics_pack.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,19 +226,27 @@
#define FD_METRICS_HISTOGRAM_PACK_CUS_NET_DESC "The net number of cost units (scheduled - rebated) in each block pack produced."
#define FD_METRICS_HISTOGRAM_PACK_CUS_NET_CVT (FD_METRICS_CONVERTER_NONE)
#define FD_METRICS_HISTOGRAM_PACK_CUS_NET_MIN (1000000UL)
#define FD_METRICS_HISTOGRAM_PACK_CUS_NET_MAX (48000000UL)
#define FD_METRICS_HISTOGRAM_PACK_CUS_NET_MAX (60000000UL)

#define FD_METRICS_COUNTER_PACK_DELETE_MISSED_OFF (227UL)
#define FD_METRICS_HISTOGRAM_PACK_CUS_PCT_OFF (227UL)
#define FD_METRICS_HISTOGRAM_PACK_CUS_PCT_NAME "pack_cus_pct"
#define FD_METRICS_HISTOGRAM_PACK_CUS_PCT_TYPE (FD_METRICS_TYPE_HISTOGRAM)
#define FD_METRICS_HISTOGRAM_PACK_CUS_PCT_DESC "The percent of the total block cost limit used for each block pack produced."
#define FD_METRICS_HISTOGRAM_PACK_CUS_PCT_CVT (FD_METRICS_CONVERTER_NONE)
#define FD_METRICS_HISTOGRAM_PACK_CUS_PCT_MIN (0UL)
#define FD_METRICS_HISTOGRAM_PACK_CUS_PCT_MAX (100UL)

#define FD_METRICS_COUNTER_PACK_DELETE_MISSED_OFF (244UL)
#define FD_METRICS_COUNTER_PACK_DELETE_MISSED_NAME "pack_delete_missed"
#define FD_METRICS_COUNTER_PACK_DELETE_MISSED_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_DELETE_MISSED_DESC "Count of attempts to delete a transaction that wasn't found"
#define FD_METRICS_COUNTER_PACK_DELETE_MISSED_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_COUNTER_PACK_DELETE_HIT_OFF (228UL)
#define FD_METRICS_COUNTER_PACK_DELETE_HIT_OFF (245UL)
#define FD_METRICS_COUNTER_PACK_DELETE_HIT_NAME "pack_delete_hit"
#define FD_METRICS_COUNTER_PACK_DELETE_HIT_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_DELETE_HIT_DESC "Count of attempts to delete a transaction that was found and deleted"
#define FD_METRICS_COUNTER_PACK_DELETE_HIT_CVT (FD_METRICS_CONVERTER_NONE)

#define FD_METRICS_PACK_TOTAL (69UL)
#define FD_METRICS_PACK_TOTAL (70UL)
extern const fd_metrics_meta_t FD_METRICS_PACK[FD_METRICS_PACK_TOTAL];
5 changes: 4 additions & 1 deletion src/disco/metrics/metrics.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,9 +362,12 @@ metric introduced.
<!-- Note: The switch from cost units to compute units here is deliberate. Of the various types of cost units, only compute units can be rebated. -->
<summary>The number of compute units rebated for each block pack produced. Compute units are rebated when a transaction fails prior to execution or requests more compute units than it uses.</summary>
</histogram>
<histogram name="CusNet" min="1000000" max="48000000">
<histogram name="CusNet" min="1000000" max="60000000">
Copy link
Contributor

@mmcgee-jump mmcgee-jump Apr 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm. This histogram doesn't really make much sense, if CUs changes dynamically. Maybe we should make this CUSPct which is % filled.

Copy link
Contributor Author

@jherrera-jump jherrera-jump Apr 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let me know if you think we should also remove the old metric, I've kept it in for now

<summary>The net number of cost units (scheduled - rebated) in each block pack produced.</summary>
</histogram>
<histogram name="CusPct" min="0" max="100">
<summary>The percent of the total block cost limit used for each block pack produced.</summary>
</histogram>

<counter name="DeleteMissed" summary="Count of attempts to delete a transaction that wasn't found" />
<counter name="DeleteHit" summary="Count of attempts to delete a transaction that was found and deleted" />
Expand Down
24 changes: 18 additions & 6 deletions src/disco/pack/fd_pack.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -544,6 +544,7 @@ struct fd_pack_private {
fd_histf_t scheduled_cus_per_block[ 1 ];
fd_histf_t rebated_cus_per_block [ 1 ];
fd_histf_t net_cus_per_block [ 1 ];
fd_histf_t pct_cus_per_block [ 1 ];
ulong cumulative_rebated_cus;


Expand Down Expand Up @@ -633,7 +634,7 @@ fd_pack_new( void * mem,
ulong bundle_meta_sz,
ulong bank_tile_cnt,
fd_pack_limits_t const * limits,
fd_rng_t * rng ) {
fd_rng_t * rng ) {

int enable_bundles = !!bundle_meta_sz;
ulong extra_depth = fd_ulong_if( enable_bundles, 1UL+2UL*FD_PACK_MAX_TXN_PER_BUNDLE, 1UL );
Expand Down Expand Up @@ -749,6 +750,8 @@ fd_pack_new( void * mem,
FD_MHIST_MAX( PACK, CUS_REBATED ) );
fd_histf_new( pack->net_cus_per_block, FD_MHIST_MIN( PACK, CUS_NET ),
FD_MHIST_MAX( PACK, CUS_NET ) );
fd_histf_new( pack->pct_cus_per_block, FD_MHIST_MIN( PACK, CUS_PCT ),
FD_MHIST_MAX( PACK, CUS_PCT ) );

pack->compressed_slot_number = (ushort)(FD_PACK_SKIP_CNT+1);

Expand Down Expand Up @@ -2337,11 +2340,16 @@ ulong fd_pack_current_block_cost( fd_pack_t const * pack ) { return pack->cumula


void
fd_pack_set_block_limits( fd_pack_t * pack,
ulong max_microblocks_per_block,
ulong max_data_bytes_per_block ) {
pack->lim->max_microblocks_per_block = max_microblocks_per_block;
pack->lim->max_data_bytes_per_block = max_data_bytes_per_block;
fd_pack_set_block_limits( fd_pack_t * pack, fd_pack_limits_t const * limits ) {
FD_TEST( limits->max_cost_per_block >= FD_PACK_MAX_COST_PER_BLOCK_LOWER_BOUND );
FD_TEST( limits->max_vote_cost_per_block >= FD_PACK_MAX_VOTE_COST_PER_BLOCK_LOWER_BOUND );
FD_TEST( limits->max_write_cost_per_acct >= FD_PACK_MAX_WRITE_COST_PER_ACCT_LOWER_BOUND );

pack->lim->max_microblocks_per_block = limits->max_microblocks_per_block;
pack->lim->max_data_bytes_per_block = limits->max_data_bytes_per_block;
pack->lim->max_cost_per_block = limits->max_cost_per_block;
pack->lim->max_vote_cost_per_block = limits->max_vote_cost_per_block;
pack->lim->max_write_cost_per_acct = limits->max_write_cost_per_acct;
}

void
Expand Down Expand Up @@ -2391,6 +2399,9 @@ fd_pack_expire_before( fd_pack_t * pack,

void
fd_pack_end_block( fd_pack_t * pack ) {
/* rounded division */
ulong pct_cus_per_block = (pack->cumulative_block_cost*100UL + (pack->lim->max_cost_per_block>>1))/pack->lim->max_cost_per_block;
fd_histf_sample( pack->pct_cus_per_block, pct_cus_per_block );
fd_histf_sample( pack->net_cus_per_block, pack->cumulative_block_cost );
fd_histf_sample( pack->rebated_cus_per_block, pack->cumulative_rebated_cus );
fd_histf_sample( pack->scheduled_cus_per_block, pack->cumulative_rebated_cus + pack->cumulative_block_cost );
Expand Down Expand Up @@ -2455,6 +2466,7 @@ fd_pack_end_block( fd_pack_t * pack ) {
FD_MHIST_COPY( PACK, CUS_SCHEDULED, pack->scheduled_cus_per_block );
FD_MHIST_COPY( PACK, CUS_REBATED, pack->rebated_cus_per_block );
FD_MHIST_COPY( PACK, CUS_NET, pack->net_cus_per_block );
FD_MHIST_COPY( PACK, CUS_PCT, pack->pct_cus_per_block );
}

static void
Expand Down
47 changes: 36 additions & 11 deletions src/disco/pack/fd_pack.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,28 @@

/* NOTE: THE FOLLOWING CONSTANTS ARE CONSENSUS CRITICAL AND CANNOT BE
CHANGED WITHOUT COORDINATING WITH ANZA. */
#define FD_PACK_MAX_COST_PER_BLOCK (48000000UL)
#define FD_PACK_MAX_VOTE_COST_PER_BLOCK (36000000UL)
#define FD_PACK_MAX_WRITE_COST_PER_ACCT (12000000UL)

/* These are bounds on known limits. Upper bound values are used to
calculate memory footprints while lower bounds are used for
initializing consensus-dependent logic and invariant checking. As a
leader, it is OK to produce blocks using limits smaller than the
active on-chain limits. Replay should always use the correct
chain-derived limits.

The actual limits used by pack may be updated dynamically to some
in-bounds value. If there is an anticipated feature activation that
changes these limits, the upper bound should be the largest
anticipated value while the lower bound should be the current active
limit. For Frankendancer, the actual value used for consensus will be
retreived from Agave. */
#define FD_PACK_MAX_COST_PER_BLOCK_LOWER_BOUND (48000000UL)
#define FD_PACK_MAX_VOTE_COST_PER_BLOCK_LOWER_BOUND (36000000UL)
#define FD_PACK_MAX_WRITE_COST_PER_ACCT_LOWER_BOUND (12000000UL)

#define FD_PACK_MAX_COST_PER_BLOCK_UPPER_BOUND (60000000UL) /* simd 0256 */
#define FD_PACK_MAX_VOTE_COST_PER_BLOCK_UPPER_BOUND (36000000UL)
#define FD_PACK_MAX_WRITE_COST_PER_ACCT_UPPER_BOUND (12000000UL)

#define FD_PACK_FEE_PER_SIGNATURE (5000UL) /* In lamports */

/* Each block is limited to 32k parity shreds. We don't want pack to
Expand Down Expand Up @@ -202,20 +221,26 @@ FD_FN_PURE ulong fd_pack_current_block_cost( fd_pack_t const * pack );
FD_FN_PURE ulong fd_pack_bank_tile_cnt( fd_pack_t const * pack );

/* fd_pack_set_block_limits: Updates the limits provided fd_pack_new to
the new values. Any future microblocks produced by this pack object
will not cause a block to have more than max_microblocks_per_block
non-empty microblocks or more than max_data_bytes_per_block data
bytes (counting microblock headers as before). Limits are inclusive,
as per usual (i.e. a block may have exactly
max_microblocks_per_block microblocks, but not more). pack must be
a valid local join.
these new values. Any future microblocks produced by this pack
object will not cause a block to have more than
limits->max_microblocks_per_block non-empty microblocks or more than
limits->max_data_bytes_per_block data bytes (counting microblock
headers as before). future microblocks will also exclude those that
cause the total block cost to exceed limits->max_cost_per_block.
Similarly those that cause the total vote-only cost to exceed
limits->max_vote_cost_per_block. Also, those that cause the total
per-account, per block write cost to exceed
limits->max_write_cost_per_acct. Note that
limits->max_txn_per_microblock is ignored. Limits are inclusive, as
per usual (i.e. a block may have exactly max_microblocks_per_block
microblocks, but not more). pack must be a valid local join.

The typical place to call this is immediately after
fd_pack_end_block; if this is called after some microblocks have been
produced for the current block, and the current block already exceeds
the limits, all the remaining microblocks in the block will be empty,
but the call is valid. */
void fd_pack_set_block_limits( fd_pack_t * pack, ulong max_microblocks_per_block, ulong max_data_bytes_per_block );
void fd_pack_set_block_limits( fd_pack_t * pack, fd_pack_limits_t const * limits );

/* Return values for fd_pack_insert_txn_fini: Non-negative values
indicate the transaction was accepted and may be returned in a future
Expand Down
Loading
Loading