discoh: turbine fanout experiments

Author: jvarela-jump

agave

@@ -78,7 +78,7 @@ fd_topo_initialize( config_t * config ) {
   FOR(bank_tile_cnt)   fd_topob_link( topo, "bank_poh",     "bank_poh",     16384UL,                                  USHORT_MAX,             1UL );
   FOR(bank_tile_cnt)   fd_topob_link( topo, "bank_pack",    "bank_pack",    16384UL,                                  USHORT_MAX,             3UL );
   /**/                 fd_topob_link( topo, "poh_pack",     "bank_poh",     128UL,                                    sizeof(fd_became_leader_t), 1UL );
-  /**/                 fd_topob_link( topo, "poh_shred",    "poh_shred",    16384UL,                                  USHORT_MAX,             1UL );
+  /**/                 fd_topob_link( topo, "poh_shred",    "poh_shred",    16384UL,                                  USHORT_MAX,             2UL );
   /**/                 fd_topob_link( topo, "crds_shred",   "poh_shred",    128UL,                                    8UL  + 40200UL * 38UL,  1UL );
   /**/                 fd_topob_link( topo, "replay_resol", "bank_poh",     128UL,                                    sizeof(fd_completed_bank_t), 1UL );
   /* See long comment in fd_shred.c for an explanation about the size of this dcache. */

src/app/fdctl/topology.c

@@ -17,6 +17,7 @@
 #define POH_PKT_TYPE_MICROBLOCK    (0UL)
 #define POH_PKT_TYPE_BECAME_LEADER (1UL)
 #define POH_PKT_TYPE_DONE_PACKING  (2UL)
+#define POH_PKT_TYPE_FEAT_ACT_SLOT (3UL)
 
 #define REPLAY_FLAG_FINISHED_BLOCK      (0x01UL)
 #define REPLAY_FLAG_PACKED_MICROBLOCK   (0x02UL)

src/disco/fd_disco_base.h

@@ -286,8 +286,8 @@ fd_shred_dest_compute_first( fd_shred_dest_t          * sdest,
   for( ulong i=0UL; i<shred_cnt; i++ ) {
     fd_wsample_seed_rng( fd_wsample_get_rng( sdest->staked ), dest_hash_outputs[ i ] );
     /* Map FD_WSAMPLE_INDETERMINATE to FD_SHRED_DEST_NO_DEST */
-    if( FD_LIKELY( any_staked_candidates ) ) out[i] = (ushort)fd_ulong_min( fd_wsample_sample( sdest->staked ), FD_SHRED_DEST_NO_DEST );
-    else                                     out[i] = (ushort)sample_unstaked_noprepare( sdest, sdest->source_validator_orig_idx );
+    if( FD_LIKELY( any_staked_candidates ) ) out[i] = (fd_shred_dest_idx_t)fd_ulong_min( fd_wsample_sample( sdest->staked ), FD_SHRED_DEST_NO_DEST );
+    else                                     out[i] = (fd_shred_dest_idx_t)sample_unstaked_noprepare( sdest, sdest->source_validator_orig_idx );
   }
   fd_wsample_restore_all( sdest->staked );
 

src/disco/shred/fd_shred_dest.c

@@ -10,18 +10,20 @@
    to compute the destination of a specific shred for the leader and
    non-leader.  This is where the Turbine tree logic is implemented. */
 
-/* For a given FEC, we might need to produce 200 destinations for each
-   of 134 shreds, which is a lot of destinations!  Full destination
+/* For a given FEC, we might need to produce "fanout" destinations for
+   each of 134 shreds, which is a lot of destinations!  Full destination
    information (ip, port, mac) is 12 B. A pointer is 8 B, but an index
-   can be as small as 2 B, since currently Turbine doesn't work with
-   more than fanout^2 nodes which is less than USHORT_MAX.  Thus, we go
-   with the index, which can cheaply be mapped to the full information
-   using fd_shred_dest_idx_to_dest below. */
-typedef ushort fd_shred_dest_idx_t;
+   can be as small as 4 B, since currently Turbine doesn't work with
+   more than fanout^2 nodes, which is less than UINT_MAX for a maximum
+   fanout of 1536 (the fanout is dependent on feature activation). Thus,
+   we go with the index, which can cheaply be mapped to the full
+   information using fd_shred_dest_idx_to_dest below. */
+typedef uint fd_shred_dest_idx_t;
 
 
 #define FD_SHRED_DEST_MAX_SHRED_CNT (134UL) /* DATA_SHREDS_MAX+PARITY_SHREDS_MAX */
-#define FD_SHRED_DEST_NO_DEST       (USHORT_MAX)
+#define FD_SHRED_DEST_NO_DEST       (UINT_MAX)
+#define FD_SHRED_DEST_MAX_FANOUT    (1536UL)
 
 /* fd_shred_dest_weighted_t specifies a destination to which a shred might be
    sent.  The information comes from Gossip typically. */

src/disco/shred/fd_shred_dest.h

@@ -26,6 +26,22 @@
 typedef void (fd_shredder_sign_fn)( void * ctx, uchar * sig, uchar const * merkle_root );
 
 
+#define FD_SHRED_FEATURES_ACTIVATION_SLOT_CNT      (4UL)
+#define FD_SHRED_FEATURES_ACTIVATION_SLOT_SZ       (8UL)
+#define FD_SHRED_FEATURES_ACTIVATION_SLOT_DISABLED (ULONG_MAX)
+
+union fd_shred_features_activation_private {
+   /* slots for features of interest - update cnt as needed in the future. */
+   ulong slots[ FD_SHRED_FEATURES_ACTIVATION_SLOT_CNT ];
+   struct {
+      /* 0 */ ulong disable_turbine_fanout_experiments;
+      /* 1 */ ulong enable_turbine_extended_fanout_experiments;
+      /* 2 */ ulong enable_chained_merkle_shreds;
+      /* 3 */ ulong drop_unchained_merkle_shreds;
+   };
+};
+typedef union fd_shred_features_activation_private fd_shred_features_activation_t;
+
 
 static ulong const fd_shredder_data_to_parity_cnt[ 33UL ] = {
    0UL, 17UL, 18UL, 19UL, 19UL, 20UL, 21UL, 21UL,

src/disco/shred/fd_shred_tile.c

@@ -156,7 +156,7 @@ test_distribution_is_tree( fd_shred_dest_weighted_t const * info, ulong cnt, fd_
     }
 
     for( ulong i=0; i<dest_cnt; i++ ) {
-      ushort child = out[i];
+      fd_shred_dest_idx_t child = out[i];
       if( FD_LIKELY( child != FD_SHRED_DEST_NO_DEST ) ) {
         FD_TEST( !hit[child] );
         hit[child] = 1;

src/disco/shred/fd_shredder.h

@@ -542,6 +542,9 @@ typedef struct {
   fd_histf_t first_microblock_delay[ 1 ];
   fd_histf_t slot_done_delay[ 1 ];
   fd_histf_t bundle_init_delay[ 1 ];
+
+  ulong features_activation_avail;
+  fd_shred_features_activation_t features_activation[1];
 } fd_poh_ctx_t;
 
 /* The PoH recorder is implemented in Firedancer but for now needs to
@@ -1224,7 +1227,10 @@ no_longer_leader( fd_poh_ctx_t * ctx ) {
 CALLED_FROM_RUST void
 fd_ext_poh_reset( ulong         completed_bank_slot, /* The slot that successfully produced a block */
                   uchar const * reset_blockhash,     /* The hash of the last tick in the produced block */
-                  ulong         hashcnt_per_tick     /* The hashcnt per tick of the bank that completed */ ) {
+                  ulong         hashcnt_per_tick,    /* The hashcnt per tick of the bank that completed */
+                  uchar const * parent_block_id,     /* The block id of the parent block */
+                  ulong const * features_activation  /* The activation slot of shred-tile features */ ) {
+  (void)parent_block_id;
   fd_poh_ctx_t * ctx = fd_ext_poh_write_lock();
 
   ulong slot_before_reset = ctx->slot;
@@ -1316,6 +1322,16 @@ fd_ext_poh_reset( ulong         completed_bank_slot, /* The slot that successful
     if( FD_UNLIKELY( leader_before_reset ) ) publish_plugin_slot_end( ctx, slot_before_reset, ctx->cus_used );
   }
 
+  /* There is a subset of FD_SHRED_FEATURES_ACTIVATION_... slots that
+      the shred tile needs to be aware of.  Since their computation
+      requires the bank, we are forced (so far) to receive them here
+      from the Rust side, before forwarding them to the shred tile as
+      POH_PKT_TYPE_FEAT_ACT_SLOT.  This is not elegant, and it should
+      be revised in the future (TODO), but it provides a "temporary"
+      working solution to handle features activation. */
+  fd_memcpy( ctx->features_activation->slots, features_activation, sizeof(fd_shred_features_activation_t) );
+  ctx->features_activation_avail = 1UL;
+
   fd_ext_poh_write_unlock();
 }
 
@@ -1403,6 +1419,21 @@ publish_tick( fd_poh_ctx_t *      ctx,
   }
 }
 
+static inline void
+publish_features_activation(  fd_poh_ctx_t *      ctx,
+                              fd_stem_context_t * stem ) {
+  uchar * dst = (uchar *)fd_chunk_to_laddr( ctx->shred_out->mem, ctx->shred_out->chunk );
+  fd_shred_features_activation_t * act_data = (fd_shred_features_activation_t *)dst;
+  fd_memcpy( act_data, ctx->features_activation, sizeof(fd_shred_features_activation_t) );
+
+  ulong tspub = (ulong)fd_frag_meta_ts_comp( fd_tickcount() );
+  ulong sz = sizeof(fd_shred_features_activation_t);
+  ulong sig = fd_disco_poh_sig( ctx->slot, POH_PKT_TYPE_FEAT_ACT_SLOT, 0UL );
+  fd_stem_publish( stem, ctx->shred_out->idx, sig, ctx->shred_out->chunk, sz, 0UL, 0UL, tspub );
+  ctx->shred_seq = stem->seqs[ ctx->shred_out->idx ];
+  ctx->shred_out->chunk = fd_dcache_compact_next( ctx->shred_out->chunk, sz, ctx->shred_out->chunk0, ctx->shred_out->wmark );
+}
+
 static inline void
 after_credit( fd_poh_ctx_t *      ctx,
               fd_stem_context_t * stem,
@@ -1426,6 +1457,14 @@ after_credit( fd_poh_ctx_t *      ctx,
   }
   FD_COMPILER_MFENCE();
 
+  if( FD_UNLIKELY( ctx->features_activation_avail ) ) {
+    /* If we have received an update on features_activation, then
+        forward them to the shred tile.  In principle, this should
+        happen at most once per slot. */
+    publish_features_activation( ctx, stem );
+    ctx->features_activation_avail = 0UL;
+  }
+
   int is_leader = ctx->next_leader_slot!=ULONG_MAX && ctx->slot>=ctx->next_leader_slot;
   if( FD_UNLIKELY( is_leader && !ctx->current_leader_bank ) ) {
     /* If we are the leader, but we didn't yet learn what the leader
@@ -2272,14 +2311,18 @@ unprivileged_init( fd_topo_t *      topo,
     *ctx->plugin_out = out1( topo, tile, "poh_plugin" );
   }
 
+  ctx->features_activation_avail = 0UL;
+  for( ulong i=0UL; i<FD_SHRED_FEATURES_ACTIVATION_SLOT_CNT; i++ )
+    ctx->features_activation->slots[i] = FD_SHRED_FEATURES_ACTIVATION_SLOT_DISABLED;
+
   ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, 1UL );
   if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
     FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
 }
 
 /* One tick, one microblock, one plugin slot end, one plugin slot start,
-   and one leader update. */
-#define STEM_BURST (5UL)
+   one leader update, and one features activation. */
+#define STEM_BURST (6UL)
 
 /* See explanation in fd_pack */
 #define STEM_LAZY  (128L*3000L)