Skip to content

Commit f8262f7

Browse files
flamenco, choreo, app: integrate new Funkier
1 parent 2124f0d commit f8262f7

File tree

89 files changed

+2658
-10639
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

89 files changed

+2658
-10639
lines changed

Diff for: config/extra/with-handholding.mk

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,5 @@ CPPFLAGS+=-DFD_SPAD_USE_HANDHOLDING=1
66
CPPFLAGS+=-DFD_TOWER_USE_HANDHOLDING=1
77
CPPFLAGS+=-DFD_TMPL_USE_HANDHOLDING=1
88
CPPFLAGS+=-DFD_TXN_HANDHOLDING=1
9-
CPPFLAGS+=-DFD_FUNKIER_HANDHOLDING=1
9+
CPPFLAGS+=-DFD_FUNK_HANDHOLDING=1
1010
CPPFLAGS+=-DFD_RUNTIME_ERR_HANDHOLDING=1

Diff for: src/app/fdctl/config.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ fdctl_obj_footprint( fd_topo_t const * topo,
115115
} else if( FD_UNLIKELY( !strcmp( obj->name, "dbl_buf" ) ) ) {
116116
return fd_dbl_buf_footprint( VAL("mtu") );
117117
} else if( FD_UNLIKELY( !strcmp( obj->name, "funk" ) ) ) {
118-
return fd_funk_footprint();
118+
return fd_funk_footprint( VAL("txn_max"), VAL("rec_max") );
119119
} else if( FD_UNLIKELY( !strcmp( obj->name, "neigh4_hmap" ) ) ) {
120120
return fd_neigh4_hmap_footprint( VAL("ele_max"), VAL("lock_cnt"), VAL("probe_max") );
121121
} else if( FD_UNLIKELY( !strcmp( obj->name, "fib4" ) ) ) {

Diff for: src/app/ledger/main.c

+22-35
Original file line numberDiff line numberDiff line change
@@ -664,7 +664,6 @@ allocator_setup( fd_wksp_t * wksp ) {
664664
void
665665
fd_ledger_main_setup( fd_ledger_args_t * args ) {
666666
fd_flamenco_boot( NULL, NULL );
667-
fd_funk_t * funk = args->funk;
668667

669668
/* Setup capture context */
670669
int has_solcap = args->capture_fpath && args->capture_fpath[0] != '\0';
@@ -720,9 +719,7 @@ fd_ledger_main_setup( fd_ledger_args_t * args ) {
720719
fd_runtime_update_leaders( args->slot_ctx, args->slot_ctx->slot_bank.slot, args->runtime_spad );
721720
fd_calculate_epoch_accounts_hash_values( args->slot_ctx );
722721

723-
fd_funk_start_write( funk );
724722
fd_bpf_scan_and_create_bpf_program_cache_entry_tpool( args->slot_ctx, args->slot_ctx->funk_txn, args->tpool, args->runtime_spad );
725-
fd_funk_end_write( funk );
726723

727724
/* First, load in the sysvars into the sysvar cache. This is required to
728725
make the StakeHistory sysvar available to the rewards calculation. */
@@ -910,8 +907,7 @@ init_funk( fd_ledger_args_t * args ) {
910907
}
911908
args->funk = funk;
912909
args->funk_wksp = fd_funk_wksp( funk );
913-
FD_LOG_NOTICE(( "funky at global address 0x%016lx with %lu records", fd_wksp_gaddr_fast( args->funk_wksp, funk ),
914-
fd_funk_rec_cnt( fd_funk_rec_map( funk, args->funk_wksp ) ) ));
910+
FD_LOG_NOTICE(( "funky at global address 0x%016lx", fd_wksp_gaddr_fast( args->funk_wksp, funk ) ));
915911
}
916912

917913
void
@@ -958,13 +954,7 @@ checkpt( fd_ledger_args_t * args ) {
958954
}
959955
FD_LOG_NOTICE(( "writing funk checkpt %s", args->checkpt_funk ));
960956
unlink( args->checkpt_funk );
961-
#ifdef FD_FUNK_WKSP_PROTECT
962-
fd_wksp_mprotect( args->funk_wksp, 0 );
963-
#endif
964957
int err = fd_wksp_checkpt( args->funk_wksp, args->checkpt_funk, 0666, 0, NULL );
965-
#ifdef FD_FUNK_WKSP_PROTECT
966-
fd_wksp_mprotect( args->funk_wksp, 1 );
967-
#endif
968958
if( err ) {
969959
FD_LOG_ERR(( "funk checkpt failed: error %d", err ));
970960
}
@@ -1129,7 +1119,7 @@ ingest( fd_ledger_args_t * args ) {
11291119
args->exec_spads,
11301120
args->exec_spad_cnt,
11311121
args->runtime_spad );
1132-
FD_LOG_NOTICE(( "imported %lu records from snapshot", fd_funk_rec_cnt( fd_funk_rec_map( funk, fd_funk_wksp( funk ) ) ) ));
1122+
FD_LOG_NOTICE(( "imported records from snapshot" ));
11331123
}
11341124
if( args->incremental ) {
11351125
fd_snapshot_load_all( args->incremental,
@@ -1142,7 +1132,7 @@ ingest( fd_ledger_args_t * args ) {
11421132
args->exec_spads,
11431133
args->exec_spad_cnt,
11441134
args->runtime_spad );
1145-
FD_LOG_NOTICE(( "imported %lu records from incremental snapshot", fd_funk_rec_cnt( fd_funk_rec_map( funk, fd_funk_wksp( funk ) ) ) ));
1135+
FD_LOG_NOTICE(( "imported records from incremental snapshot" ));
11461136
}
11471137

11481138
if( args->genesis ) {
@@ -1185,12 +1175,14 @@ ingest( fd_ledger_args_t * args ) {
11851175
}
11861176
}
11871177

1178+
#ifdef FD_FUNK_HANDHOLDING
11881179
if( args->verify_funk ) {
11891180
FD_LOG_NOTICE(( "verifying funky" ));
11901181
if( fd_funk_verify( funk ) ) {
11911182
FD_LOG_ERR(( "verification failed" ));
11921183
}
11931184
}
1185+
#endif
11941186

11951187
checkpt( args );
11961188

@@ -1304,22 +1296,19 @@ replay( fd_ledger_args_t * args ) {
13041296
/* Check number of records in funk. If rec_cnt == 0, then it can be assumed
13051297
that you need to load in snapshot(s). */
13061298

1307-
ulong rec_cnt = fd_funk_rec_cnt( fd_funk_rec_map( funk, fd_funk_wksp( funk ) ) );
1308-
if( !rec_cnt ) {
1309-
/* Load in snapshot(s) */
1310-
if( args->snapshot ) {
1311-
fd_snapshot_load_all( args->snapshot,
1312-
args->slot_ctx,
1313-
NULL,
1314-
args->tpool,
1315-
args->verify_acc_hash,
1316-
args->check_acc_hash,
1317-
FD_SNAPSHOT_TYPE_FULL,
1318-
args->exec_spads,
1319-
args->exec_spad_cnt,
1320-
args->runtime_spad );
1321-
FD_LOG_NOTICE(( "imported %lu records from snapshot", fd_funk_rec_cnt( fd_funk_rec_map( funk, fd_funk_wksp( funk ) ) ) ));
1322-
}
1299+
/* Load in snapshot(s) */
1300+
if( args->snapshot ) {
1301+
fd_snapshot_load_all( args->snapshot,
1302+
args->slot_ctx,
1303+
NULL,
1304+
args->tpool,
1305+
args->verify_acc_hash,
1306+
args->check_acc_hash,
1307+
FD_SNAPSHOT_TYPE_FULL,
1308+
args->exec_spads,
1309+
args->exec_spad_cnt,
1310+
args->runtime_spad );
1311+
FD_LOG_NOTICE(( "imported from snapshot" ));
13231312
if( args->incremental ) {
13241313
fd_snapshot_load_all( args->incremental,
13251314
args->slot_ctx,
@@ -1331,13 +1320,11 @@ replay( fd_ledger_args_t * args ) {
13311320
args->exec_spads,
13321321
args->exec_spad_cnt,
13331322
args->runtime_spad );
1334-
FD_LOG_NOTICE(( "imported %lu records from snapshot", fd_funk_rec_cnt( fd_funk_rec_map( funk, fd_funk_wksp( funk ) ) ) ));
1335-
}
1336-
if( args->genesis ) {
1337-
fd_runtime_read_genesis( args->slot_ctx, args->genesis, args->snapshot != NULL, NULL, args->tpool, args->runtime_spad );
1323+
FD_LOG_NOTICE(( "imported from snapshot" ));
13381324
}
1339-
} else {
1340-
FD_LOG_NOTICE(( "found funk with %lu records", rec_cnt ));
1325+
}
1326+
if( args->genesis ) {
1327+
fd_runtime_read_genesis( args->slot_ctx, args->genesis, args->snapshot != NULL, NULL, args->tpool, args->runtime_spad );
13411328
}
13421329

13431330
FD_LOG_NOTICE(( "Used memory in spad after loading in snapshot %lu", args->runtime_spad->mem_used ));

Diff for: src/app/rpcserver/main.c

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
#define _DEFAULT_SOURCE
22

3-
#include "../../discof/rpcserver/fd_rpc_service.h"
4-
#include "../../funk/fd_funk_filemap.h"
5-
63
#include <fcntl.h>
74
#include <stdio.h>
85
#include <stdlib.h>
@@ -13,6 +10,8 @@
1310
#include <sys/socket.h>
1411
#include <netinet/in.h>
1512
#include <arpa/inet.h>
13+
#include "../../discof/rpcserver/fd_rpc_service.h"
14+
#include "../../funk/fd_funk_filemap.h"
1615

1716
#define SHAM_LINK_CONTEXT fd_rpc_ctx_t
1817
#define SHAM_LINK_STATE fd_replay_notif_msg_t

Diff for: src/choreo/epoch/fd_epoch.c

+2-2
Original file line numberDiff line numberDiff line change
@@ -115,9 +115,9 @@ fd_epoch_init( fd_epoch_t * epoch, fd_epoch_bank_t const * epoch_bank ) {
115115
vote_accounts->vote_accounts_root );
116116
curr;
117117
curr = fd_vote_accounts_pair_t_map_successor( vote_accounts->vote_accounts_pool, curr ) ) {
118-
118+
119119
if( FD_UNLIKELY( curr->elem.stake > 0UL ) ) {
120-
120+
121121
#if FD_EPOCH_USE_HANDHOLDING
122122
FD_TEST( !fd_epoch_voters_query( epoch_voters, curr->elem.key, NULL ) );
123123
FD_TEST( fd_epoch_voters_key_cnt( epoch_voters ) < fd_epoch_voters_key_max( epoch_voters ) );

Diff for: src/choreo/forks/fd_forks.c

+74-44
Original file line numberDiff line numberDiff line change
@@ -170,14 +170,14 @@ fd_forks_query_const( fd_forks_t const * forks, ulong slot ) {
170170
// fork->slot_ctx->slot_bank.prev_slot = fork->slot_ctx->slot_bank.slot;
171171
// fork->slot_ctx->slot_bank.slot = curr_slot;
172172

173-
// fork->slot_ctx->status_cache = status_cache;
173+
// fork->slot_ctx.status_cache = status_cache;
174174
// fd_funk_txn_xid_t xid;
175175

176176
// fd_memcpy( xid.uc, blockhash.uc, sizeof( fd_funk_txn_xid_t));
177-
// xid.ul[0] = fork->slot_ctx->slot_bank.slot;
177+
// xid.ul[0] = fork->slot_ctx.slot_bank.slot;
178178
// /* push a new transaction on the stack */
179179
// fd_funk_start_write( funk );
180-
// fork->slot_ctx->funk_txn = fd_funk_txn_prepare( funk, fork->slot_ctx->funk_txn, &xid, 1 );
180+
// fork->slot_ctx.funk_txn = fd_funk_txn_prepare( funk, fork->slot_ctx.funk_txn, &xid, 1 );
181181
// fd_funk_end_write( funk );
182182

183183
// int res = fd_runtime_publish_old_txns( &fork->slot_ctx, capture_ctx );
@@ -191,64 +191,80 @@ slot_ctx_restore( ulong slot,
191191
fd_acc_mgr_t * acc_mgr,
192192
fd_blockstore_t * blockstore,
193193
fd_exec_epoch_ctx_t * epoch_ctx,
194-
fd_funk_t * funk,
194+
fd_funk_t * funk,
195195
fd_spad_t * runtime_spad,
196196
fd_exec_slot_ctx_t * slot_ctx_out ) {
197-
fd_funk_txn_t * txn_map = fd_funk_txn_map( funk, fd_funk_wksp( funk ) );
197+
fd_funk_txn_map_t txn_map = fd_funk_txn_map( funk, fd_funk_wksp( funk ) );
198198
bool block_exists = fd_blockstore_shreds_complete( blockstore, slot );
199199

200200
FD_LOG_DEBUG( ( "Current slot %lu", slot ) );
201201
if( !block_exists )
202202
FD_LOG_ERR( ( "missing block at slot we're trying to restore" ) );
203203

204-
fd_funk_txn_xid_t xid = { .ul = { slot, slot } };
204+
fd_funk_txn_xid_t xid = { .ul = { slot, slot } };
205205
fd_funk_rec_key_t id = fd_runtime_slot_bank_key();
206-
fd_funk_txn_t * txn = fd_funk_txn_query( &xid, txn_map );
207-
if( !txn ) {
208-
memset( xid.uc, 0, sizeof( fd_funk_txn_xid_t ) );
209-
xid.ul[0] = slot;
210-
txn = fd_funk_txn_query( &xid, txn_map );
206+
for( ; ; ) {
207+
fd_funk_txn_start_read( funk );
208+
fd_funk_txn_t * txn = fd_funk_txn_query( &xid, &txn_map );
211209
if( !txn ) {
212-
FD_LOG_ERR( ( "missing txn, parent slot %lu", slot ) );
210+
memset( xid.uc, 0, sizeof( fd_funk_txn_xid_t ) );
211+
xid.ul[0] = slot;
212+
txn = fd_funk_txn_query( &xid, &txn_map );
213+
if( !txn ) {
214+
FD_LOG_ERR( ( "missing txn, parent slot %lu", slot ) );
215+
}
213216
}
214-
}
215-
fd_funk_rec_t const * rec = fd_funk_rec_query_global( funk, txn, &id, NULL );
216-
if( rec == NULL ) FD_LOG_ERR( ( "failed to read banks record" ) );
217-
void * val = fd_funk_val( rec, fd_funk_wksp( funk ) );
217+
fd_funk_txn_end_read( funk );
218218

219-
uint magic = *(uint *)val;
219+
fd_funk_rec_query_t query[1];
220+
fd_funk_rec_t const * rec = fd_funk_rec_query_try_global( funk, txn, &id, NULL, query );
221+
if( rec == NULL ) FD_LOG_ERR( ( "failed to read banks record" ) );
222+
void * val = fd_funk_val( rec, fd_funk_wksp( funk ) );
220223

221-
fd_bincode_decode_ctx_t decode_ctx = {
222-
.data = (uchar *)val + sizeof(uint),
223-
.dataend = (uchar *)val + fd_funk_val_sz( rec )
224-
};
224+
uint magic = *(uint *)val;
225225

226-
FD_TEST( slot_ctx_out->magic == FD_EXEC_SLOT_CTX_MAGIC );
226+
fd_bincode_decode_ctx_t decode_ctx = {
227+
.data = (uchar *)val + sizeof(uint),
228+
.dataend = (uchar *)val + fd_funk_val_sz( rec )
229+
};
227230

228-
slot_ctx_out->funk_txn = txn;
229-
slot_ctx_out->acc_mgr = acc_mgr;
230-
slot_ctx_out->blockstore = blockstore;
231-
slot_ctx_out->epoch_ctx = epoch_ctx;
232-
233-
if( FD_LIKELY( magic==FD_RUNTIME_ENC_BINCODE ) ) {
234-
ulong total_sz = 0UL;
235-
int err = fd_slot_bank_decode_footprint( &decode_ctx, &total_sz );
236-
if( FD_UNLIKELY( err != FD_BINCODE_SUCCESS ) ) {
237-
FD_LOG_ERR( ( "failed to decode banks record" ) );
231+
if( slot_ctx_out == NULL || slot_ctx_out->magic != FD_EXEC_SLOT_CTX_MAGIC ) {
232+
FD_LOG_WARNING(( "bad slot context" ));
233+
continue;
238234
}
239235

240-
uchar * mem = fd_spad_alloc( runtime_spad, fd_slot_bank_align(), total_sz );
241-
if( FD_UNLIKELY( !mem ) ) {
242-
FD_LOG_ERR( ( "failed to allocate memory for slot bank" ) );
243-
}
236+
FD_TEST( slot_ctx_out->magic == FD_EXEC_SLOT_CTX_MAGIC );
237+
238+
slot_ctx_out->funk_txn = txn;
239+
slot_ctx_out->acc_mgr = acc_mgr;
240+
slot_ctx_out->blockstore = blockstore;
241+
slot_ctx_out->epoch_ctx = epoch_ctx;
242+
243+
if( FD_LIKELY( magic==FD_RUNTIME_ENC_BINCODE ) ) {
244+
ulong total_sz = 0UL;
245+
int err = fd_slot_bank_decode_footprint( &decode_ctx, &total_sz );
246+
if( FD_UNLIKELY( err != FD_BINCODE_SUCCESS ) ) {
247+
FD_LOG_WARNING( ( "failed to decode banks record" ) );
248+
continue;
249+
}
250+
251+
uchar * mem = fd_spad_alloc( runtime_spad, fd_slot_bank_align(), total_sz );
252+
if( FD_UNLIKELY( !mem ) ) {
253+
FD_LOG_ERR( ( "failed to allocate memory for slot bank" ) );
254+
}
255+
256+
fd_slot_bank_t * slot_bank = fd_slot_bank_decode( mem, &decode_ctx );
244257

245-
fd_slot_bank_t * slot_bank = fd_slot_bank_decode( mem, &decode_ctx );
258+
fd_memcpy( &slot_ctx_out->slot_bank, slot_bank, sizeof(fd_slot_bank_t) );
259+
} else {
260+
FD_LOG_ERR( ( "failed to read banks record: invalid magic number" ) );
261+
}
262+
FD_TEST( !fd_runtime_sysvar_cache_load( slot_ctx_out, runtime_spad ) );
246263

247-
fd_memcpy( &slot_ctx_out->slot_bank, slot_bank, sizeof(fd_slot_bank_t) );
248-
} else {
249-
FD_LOG_ERR( ( "failed to read banks record: invalid magic number" ) );
264+
if( FD_LIKELY( fd_funk_rec_query_test( query ) == FD_FUNK_SUCCESS ) ) {
265+
break;
266+
}
250267
}
251-
FD_TEST( !fd_runtime_sysvar_cache_load( slot_ctx_out, runtime_spad ) );
252268

253269
// TODO how do i get this info, ignoring rewards for now
254270
// slot_ctx_out->epoch_reward_status = ???
@@ -347,12 +363,27 @@ fd_forks_update( fd_forks_t * forks,
347363
rec_query_global traverses all the way back to the root. */
348364

349365
fd_voter_t * voter = &epoch_voters[i];
350-
fd_voter_state_t const * state = fd_voter_state( funk, txn, &voter->rec );
366+
367+
/* Fetch the vote account's vote slot and root slot from the vote account, re-trying if there is
368+
a Funk conflict.
369+
370+
TODO: factor this out into a convenience function. */
371+
ulong vote = 0UL;
372+
ulong root = 0UL;
373+
for( ; ; ) {
374+
fd_funk_rec_query_t query[1];
375+
fd_voter_state_t const * state = fd_voter_state( funk, query, txn, &voter->rec );
376+
vote = fd_voter_state_vote( state );
377+
root = fd_voter_state_root( state );
378+
379+
if( FD_LIKELY( fd_funk_rec_query_test( query ) == FD_FUNK_SUCCESS ) ) {
380+
break;
381+
}
382+
}
351383

352384
/* Only process votes for slots >= root. Ghost requires vote slot
353385
to already exist in the ghost tree. */
354386

355-
ulong vote = fd_voter_state_vote( state );
356387
if( FD_LIKELY( vote != FD_SLOT_NULL && vote >= fd_ghost_root( ghost )->slot ) ) {
357388
fd_ghost_replay_vote( ghost, voter, vote );
358389

@@ -376,7 +407,6 @@ fd_forks_update( fd_forks_t * forks,
376407
/* Check if this voter's root >= ghost root. We can't process
377408
other voters' roots that precede the ghost root. */
378409

379-
ulong root = fd_voter_state_root( state );
380410
if( FD_LIKELY( root != FD_SLOT_NULL && root >= fd_ghost_root( ghost )->slot ) ) {
381411
fd_ghost_node_t const * node = fd_ghost_query( ghost, root );
382412

0 commit comments

Comments
 (0)