fix(tower): stem burst and pr comments

Author: lidatong

book/api/metrics-generated.md

@@ -1159,6 +1159,9 @@
 
 | Metric | Type | Description |
 |--------|------|-------------|
+| <span class="metrics-name">tower_&#8203;vote_&#8203;txn_&#8203;invalid</span> | counter | Number of times we dropped a vote txn because it was invalid (malformed, bad signature, etc.) |
+| <span class="metrics-name">tower_&#8203;vote_&#8203;txn_&#8203;ignored</span> | counter | Number of times we ignored all or part of a vote txn because we didn't recognize a slot (eg. our replay was behind) |
+| <span class="metrics-name">tower_&#8203;vote_&#8203;txn_&#8203;mismatch</span> | counter | Number of times a vote txn mismatched our own block id |
 | <span class="metrics-name">tower_&#8203;ancestor_&#8203;rollback</span> | counter | Rollback to an ancestor of our prev vote (can't vote) |
 | <span class="metrics-name">tower_&#8203;sibling_&#8203;confirmed</span> | counter | Duplicate sibling got confirmed (can't vote) |
 | <span class="metrics-name">tower_&#8203;same_&#8203;fork</span> | counter | Same fork as prev vote (can vote) |

src/app/firedancer/config/default.toml

@@ -472,15 +472,6 @@ user = ""
     # abort with an error.
     expected_genesis_hash = ""
 
-    # Firedancer can process at most this many slots without rooting in
-    # the consensus rules before it must begin evicting.
-    #
-    # This is an estimate and should be set as generously as possible to
-    # allow for brief anomalies such as the validator disconnecting from
-    # part of the cluster due to data center issues.  Roughly, at 400 ms
-    # per slot, the default allows for 30 minutes without rooting.
-    max_unrooted_slots = 4096
-
 # This section configures the "funk" account database.  Currently, funk
 # stores all Solana accounts.  In future versions of Firedancer, most
 # accounts will be offloaded to the "groove" database.
@@ -1378,6 +1369,27 @@ user = ""
         # TODO: What is this ... needs to be deleted
         cluster_version =  "1.18.0"
 
+    # The tower tile runs the fork choice and tower rules to determine
+    # both what block to vote on and what block to build our own leader
+    # blocks on top of.
+    [tiles.tower]
+        # Firedancer can process at most this many slots without rooting
+        # in the consensus rules before it must begin evicting.
+        #
+        # This is an estimate and should be set as generously as
+        # possible to allow for temporary outages such as network
+        # partitions.  For example, the validator might get disconnected
+        # from part of the cluster due to data center issues.  Roughly,
+        # the default of 4096 allows for 30 minutes without rooting.
+        #
+        # Specifically, tower will ignore gossip votes that exceed max
+        # unrooted slots ahead of the current root.  Additionally, both
+        # fork choice and tower structures will OOM and cause Firedancer
+        # to exit if it needs to maintain more than max unrooted slots
+        # tower forks (TODO in the future Firedancer will instead
+        # gracefully degrade by evicting forks).
+        max_unrooted_slots = 4096
+
     [tiles.send]
         # The port the send tile uses for QUIC, to send votes and other
         # transactions. It also uses this as the UDP src port.

src/app/firedancer/topology.c

@@ -409,7 +409,7 @@ fd_topo_initialize( config_t * config ) {
 
   /* TODO: Explain this .... USHORT_MAX is not dcache max */
   ulong pending_fec_shreds_depth = fd_ulong_min( fd_ulong_pow2_up( config->tiles.shred.max_pending_shred_sets * FD_REEDSOL_DATA_SHREDS_MAX ), USHORT_MAX + 1 /* dcache max */ );
-  ulong max_unrooted_slots       = config->firedancer.consensus.max_unrooted_slots;
+  ulong max_unrooted_slots       = config->tiles.tower.max_unrooted_slots;
 
   /*                                  topo, link_name,      wksp_name,      depth,                                    mtu,                           burst */
   /**/                 fd_topob_link( topo, "gossip_net",   "net_gossip",   32768UL,                                  FD_NET_MTU,                    1UL );
@@ -1210,7 +1210,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
 
   } else if( FD_UNLIKELY( !strcmp( tile->name, "tower" ) ) ) {
 
-    tile->tower.slot_max = config->firedancer.consensus.max_unrooted_slots;
+    tile->tower.slot_max = config->tiles.tower.max_unrooted_slots;
     strncpy( tile->tower.identity_key, config->paths.identity_key, sizeof(tile->tower.identity_key) );
     strncpy( tile->tower.vote_account, config->paths.vote_account, sizeof(tile->tower.vote_account) );
     strncpy( tile->tower.base_path, config->paths.base, sizeof(tile->tower.base_path) );

src/app/shared/fd_config.h

@@ -133,10 +133,6 @@ struct fd_configf {
     char host[ 256 ];
   } gossip;
 
-  struct {
-    ulong max_unrooted_slots;
-  } consensus;
-
   struct {
     struct {
       uint max_local_full_effective_age;
@@ -486,6 +482,10 @@ struct fd_config {
       ulong write_buffer_size;
     } shredcap;
 
+    struct {
+      ulong max_unrooted_slots;
+    } tower;
+
   } tiles;
   struct {
     ulong capture_start_slot;

src/app/shared/fd_config_parse.c

@@ -80,8 +80,6 @@ fd_config_extract_podf( uchar *        pod,
                         fd_configf_t * config ) {
   CFG_POP      ( cstr,   gossip.host                                         );
 
-  CFG_POP      ( ulong,  consensus.max_unrooted_slots                        );
-
   CFG_POP      ( uint,   layout.exec_tile_count                              );
   CFG_POP      ( uint,   layout.sign_tile_count                              );
   CFG_POP      ( uint,   layout.gossvf_tile_count                            );
@@ -256,13 +254,10 @@ fd_config_extract_pod( uchar *       pod,
   CFG_POP      ( cstr,   tiles.replay.cluster_version                     );
   CFG_POP_ARRAY( cstr,   tiles.replay.enable_features                     );
 
-  CFG_POP      ( cstr,   tiles.store_int.slots_pending                    );
-  CFG_POP      ( cstr,   tiles.store_int.shred_cap_archive                );
-  CFG_POP      ( cstr,   tiles.store_int.shred_cap_replay                 );
-  CFG_POP      ( ulong,  tiles.store_int.shred_cap_end_slot               );
-
   CFG_POP      ( ushort, tiles.send.send_src_port                         );
 
+  CFG_POP      ( ulong,  tiles.tower.max_unrooted_slots                   );
+
   CFG_POP      ( bool,   tiles.archiver.enabled                           );
   CFG_POP      ( ulong,  tiles.archiver.end_slot                          );
   CFG_POP      ( cstr,   tiles.archiver.rocksdb_path                      );

src/choreo/epoch/Local.mk

@@ -101,8 +101,6 @@ fd_notar_count_vote( fd_notar_t *        notar,
                      ulong               vote_slot,
                      fd_hash_t const *   vote_block_id ) {
 
-  if( FD_UNLIKELY( !notar ) ) { FD_LOG_WARNING(( "NULL notar" )); return NULL; }
-
   /* Ignore if this vote slot isn't in range. */
 
   if( FD_UNLIKELY( vote_slot < notar->lo_wmark || vote_slot > notar->hi_wmark ) ) return NULL;
@@ -138,10 +136,14 @@ fd_notar_count_vote( fd_notar_t *        notar,
 
   fd_notar_blk_t * notar_blk = fd_notar_blk_query( notar->blk_map, *vote_block_id, NULL );
   if( FD_UNLIKELY( !notar_blk ) ) {
-    notar_blk        = fd_notar_blk_insert( notar->blk_map, *vote_block_id );
-    notar_blk->slot  = vote_slot;
-    notar_blk->stake = 0;
     FD_TEST( notar_slot->block_ids_cnt < FD_VOTER_MAX ); /* at most one unique block id per voter in a slot */
+    notar_blk            = fd_notar_blk_insert( notar->blk_map, *vote_block_id );
+    notar_blk->slot      = vote_slot;
+    notar_blk->stake     = 0;
+    notar_blk->dup_conf  = 0;
+    notar_blk->opt_conf  = 0;
+    notar_blk->dup_notif = 0;
+    notar_blk->opt_notif = 0;
     notar_slot->block_ids[notar_slot->block_ids_cnt++] = *vote_block_id;
   }
   notar_blk->stake   += vtr->stake;
@@ -185,7 +187,7 @@ fd_notar_advance_wmark( fd_notar_t * notar,
       for( ulong i=0; i<notar_slot->block_ids_cnt; i++ ) {
         fd_hash_t const * block_id = &notar_slot->block_ids[i];
         fd_notar_blk_t * notar_blk = fd_notar_blk_query( notar->blk_map, *block_id, NULL );
-        if( FD_UNLIKELY( !notar_blk ) ) FD_LOG_CRIT(( "missing %lu %s %lu %lu", slot, FD_BASE58_ENC_32_ALLOCA( block_id ), i, notar_slot->block_ids_cnt ));
+        if( FD_UNLIKELY( !notar_blk ) ) { FD_BASE58_ENCODE_32_BYTES( block_id->uc, crit ); FD_LOG_CRIT(( "missing %lu %s %lu %lu", slot, crit, i, notar_slot->block_ids_cnt )); };
         fd_notar_blk_remove( notar->blk_map, notar_blk );
       }
       fd_notar_slot_remove( notar->slot_map, notar_slot );

src/choreo/notar/fd_notar.c

@@ -110,12 +110,14 @@ struct fd_notar_slot {
 typedef struct fd_notar_slot fd_notar_slot_t;
 
 struct fd_notar_blk {
-  fd_hash_t block_id; /* map key */
-  uint      hash;     /* reserved for fd_map_dynamic */
-  ulong     slot;     /* slot associated with this block */
-  ulong     stake;    /* sum of stake that has voted for this block_id */
-  int       dup_conf; /* whether this block has reached 52% of stake */
-  int       opt_conf; /* whether this block has reached 2/3 of stake */
+  fd_hash_t block_id;  /* map key */
+  uint      hash;      /* reserved for fd_map_dynamic */
+  ulong     slot;      /* slot associated with this block */
+  ulong     stake;     /* sum of stake that has voted for this block_id */
+  int       dup_conf;  /* whether this block has reached 52% of stake */
+  int       opt_conf;  /* whether this block has reached 2/3 of stake */
+  int       dup_notif; /* set by caller, whether we've notified consumers this is duplicate confirmed */
+  int       opt_notif; /* set by caller, whether we've notified consumers this is optimistically confirmed */
 };
 typedef struct fd_notar_blk fd_notar_blk_t;
 

src/choreo/notar/fd_notar.h

@@ -5,8 +5,6 @@ $(call add-objs,fd_tower_forks,fd_choreo)
 $(call add-objs,fd_tower_serde,fd_choreo)
 ifdef FD_HAS_HOSTED
 $(call make-unit-test,test_tower,test_tower,fd_choreo fd_flamenco fd_tango fd_ballet fd_util,$(SECP256K1_LIBS))
-$(call make-unit-test,test_tower_serde,test_tower_serde,fd_choreo fd_flamenco fd_tango fd_ballet fd_util,$(SECP256K1_LIBS))
 $(call run-unit-test,test_tower)
-$(call run-unit-test,test_tower_serde)
 endif
 endif

src/choreo/tower/Local.mk

@@ -42,12 +42,8 @@ fd_tower_forks_lowest_common_ancestor( fd_tower_forks_t * forks,
 }
 
 fd_hash_t const *
-fd_tower_forks_canonical_block_id( fd_tower_forks_t * forks,
-                                   ulong              slot ) {
-  fd_tower_forks_t * fork = fd_tower_forks_query( forks, slot, NULL );
-  if( FD_UNLIKELY( !fork ) ) return NULL;
+fd_tower_forks_canonical_block_id( fd_tower_forks_t * fork ) {
   if     ( FD_LIKELY( fork->confirmed ) ) return &fork->confirmed_block_id;
   else if( FD_LIKELY( fork->voted     ) ) return &fork->voted_block_id;
-  else if( FD_LIKELY( fork->replayed  ) ) return &fork->replayed_block_id;
-  else return NULL;
+  else                                    return &fork->replayed_block_id;
 }