gui: overview network stats

Author: jherrera-jump

book/api/metrics-generated.md

@@ -194,6 +194,7 @@
 |--------|------|-------------|
 | <span class="metrics-name">bundle_&#8203;transaction_&#8203;received</span> | counter | Total count of transactions received, including transactions within bundles |
 | <span class="metrics-name">bundle_&#8203;packet_&#8203;received</span> | counter | Total count of packets received |
+| <span class="metrics-name">bundle_&#8203;proto_&#8203;received_&#8203;bytes</span> | counter | Total count of bytes from received grpc protobuf payloads |
 | <span class="metrics-name">bundle_&#8203;bundle_&#8203;received</span> | counter | Total count of bundles received |
 | <span class="metrics-name">bundle_&#8203;errors</span><br/>{bundle_&#8203;error="<span class="metrics-enum">protobuf</span>"} | counter | Number of gRPC errors encountered (Protobuf decode/encode error) |
 | <span class="metrics-name">bundle_&#8203;errors</span><br/>{bundle_&#8203;error="<span class="metrics-enum">transport</span>"} | counter | Number of gRPC errors encountered (Transport error) |
@@ -450,7 +451,9 @@
 | <span class="metrics-name">shred_&#8203;force_&#8203;complete_&#8203;failure</span> | counter | The number of times we failed to force complete a FEC set on request |
 | <span class="metrics-name">shred_&#8203;force_&#8203;complete_&#8203;success</span> | counter | The number of times we successfully forced completed a FEC set on request |
 | <span class="metrics-name">shred_&#8203;shred_&#8203;out_&#8203;rcv</span> | counter | The number of times we received a repair shred |
+| <span class="metrics-name">shred_&#8203;shred_&#8203;out_&#8203;rcv_&#8203;bytes</span> | counter | The number bytes received from network packets with repair shreds. Bytes include network headers. |
 | <span class="metrics-name">shred_&#8203;shred_&#8203;turbine_&#8203;rcv</span> | counter | The number of times we received a turbine shred |
+| <span class="metrics-name">shred_&#8203;shred_&#8203;turbine_&#8203;rcv_&#8203;bytes</span> | counter | The number bytes received from network packets with turbine shreds. Bytes include network headers. |
 | <span class="metrics-name">shred_&#8203;store_&#8203;insert_&#8203;wait</span> | histogram | Time in seconds spent waiting for the store to insert a new FEC set |
 | <span class="metrics-name">shred_&#8203;store_&#8203;insert_&#8203;work</span> | histogram | Time in seconds spent on inserting a new FEC set |
 
@@ -483,6 +486,9 @@
 | Metric | Type | Description |
 |--------|------|-------------|
 | <span class="metrics-name">metric_&#8203;boot_&#8203;timestamp_&#8203;nanos</span> | gauge | Timestamp when validator was started (nanoseconds since epoch) |
+| <span class="metrics-name">metric_&#8203;connection_&#8203;count</span> | gauge | The number of active http connections to the Prometheus endpoint |
+| <span class="metrics-name">metric_&#8203;bytes_&#8203;written</span> | counter | The total number of bytes written to all responses on the Prometheus endpoint |
+| <span class="metrics-name">metric_&#8203;bytes_&#8203;read</span> | counter | The total number of bytes read from all requests to the Prometheus endpoint |
 
 </div>
 
@@ -492,7 +498,7 @@
 
 | Metric | Type | Description |
 |--------|------|-------------|
-| <span class="metrics-name">gui_&#8203;connection_&#8203;count</span> | gauge | The number of active connections to the GUI service |
+| <span class="metrics-name">gui_&#8203;connection_&#8203;count</span> | gauge | The number of active http connections to the GUI service, excluding connections that have been upgraded to a WebSocket connection |
 | <span class="metrics-name">gui_&#8203;websocket_&#8203;connection_&#8203;count</span> | gauge | The number of active websocket connections to the GUI service |
 | <span class="metrics-name">gui_&#8203;websocket_&#8203;frames_&#8203;sent</span> | counter | The total number of websocket frames sent to all connections to the GUI service |
 | <span class="metrics-name">gui_&#8203;websocket_&#8203;frames_&#8203;received</span> | counter | The total number of websocket frames received from all connections to the GUI service |

book/api/websocket.md

@@ -748,6 +748,56 @@ transactions per second.
 }
 ```
 
+#### `summary.live_network_metrics`
+| frequency        | type             | example |
+|------------------|------------------|---------|
+| *Once* + *100ms* | `NetworkMetrics` | below   |
+
+Live network metrics provides a live view of network bandwidth
+utilization across the various protocols used in the client.
+
+The `protocols` list contains various different protocols the client
+uses to communicate with the internet.
+
+```json
+[
+    "turbine",
+    "gossip",
+    "tpu",
+    "repair",
+    "metrics"
+]
+```
+
+- turbine: the protocol used to disseminate blockchain data, which
+contains primarily executable transactions.
+- gossip: the protocol used to disseminate node metadata, including node
+IP addresses used to help nodes find each other on the network
+- tpu: "transaction processing unit", refers to the various subsystems
+in a client used to consume and forward incoming Solana transactions for
+their next leader slot.
+- repair: a client subsystem which requests any missing block data
+needed by the replay pipeline which may have been lost over the network
+- metrics: refers to the Firedancer metrics tile, which serves an http
+Prometheus metrics endpoint
+
+```json
+{
+    "topic": "summary",
+    "key": "live_network_metrics",
+    "value": {
+        "ingress": [12345432, 5431234, 92345, ...],
+        "egress": [12345432, 5431234, 92345, ...],
+    }
+}
+```
+
+**`NetworkMetrics`**
+| Field   | Type       | Description |
+|---------|------------|-------------|
+| ingress | `number[]` | `ingress[i]` is the total number of ingress network bytes for `protocols[i]` |
+| egress  | `number[]` | `egress[i]` is the total number of egress network bytes for `protocols[i]` |
+
 #### `summary.live_txn_waterfall`
 | frequency        | type               | example |
 |------------------|--------------------|---------|

src/disco/bundle/fd_bundle_client.c

@@ -799,6 +799,7 @@ fd_bundle_client_grpc_rx_msg(
     ulong        request_ctx
 ) {
   fd_bundle_tile_t * ctx = app_ctx;
+  ctx->metrics.proto_received_bytes += protobuf_sz;
   pb_istream_t istream = pb_istream_from_buffer( protobuf, protobuf_sz );
   switch( request_ctx ) {
   case FD_BUNDLE_CLIENT_REQ_Auth_GenerateAuthChallenge:

src/disco/bundle/fd_bundle_tile.c

@@ -49,6 +49,7 @@ metrics_write( fd_bundle_tile_t * ctx ) {
   FD_MCNT_SET( BUNDLE, TRANSACTION_RECEIVED,   ctx->metrics.txn_received_cnt          );
   FD_MCNT_SET( BUNDLE, BUNDLE_RECEIVED,        ctx->metrics.bundle_received_cnt       );
   FD_MCNT_SET( BUNDLE, PACKET_RECEIVED,        ctx->metrics.packet_received_cnt       );
+  FD_MCNT_SET( BUNDLE, PROTO_RECEIVED_BYTES,   ctx->metrics.proto_received_bytes      );
   FD_MCNT_SET( BUNDLE, SHREDSTREAM_HEARTBEATS, ctx->metrics.shredstream_heartbeat_cnt );
   FD_MCNT_SET( BUNDLE, KEEPALIVES,             ctx->metrics.ping_ack_cnt              );
   FD_MCNT_SET( BUNDLE, ERRORS_PROTOBUF,        ctx->metrics.decode_fail_cnt           );

src/disco/bundle/fd_bundle_tile_private.h

@@ -33,6 +33,7 @@ struct fd_bundle_metrics {
   ulong txn_received_cnt;
   ulong bundle_received_cnt;
   ulong packet_received_cnt;
+  ulong proto_received_bytes;
   ulong shredstream_heartbeat_cnt;
   ulong ping_ack_cnt;
 

src/disco/gui/Local.mk

@@ -1,5 +1,5 @@
 ifdef FD_HAS_INT128
-$(call add-hdrs,fd_gui.h fd_gui_printf.h fd_gui_peers.h fd_gui_config_parse.h)
+$(call add-hdrs,fd_gui.h fd_gui_printf.h fd_gui_peers.h fd_gui_config_parse.h fd_gui_metrics.h)
 $(call add-objs,fd_gui fd_gui_printf fd_gui_peers fd_gui_config_parse fd_gui_tile generated/http_import_dist,fd_disco)
 $(OBJDIR)/obj/disco/gui/fd_gui_tile.o: book/public/fire.svg
 $(call make-unit-test,test_live_table,test_live_table,fd_disco fd_util)

src/disco/gui/fd_gui.c

@@ -1,6 +1,7 @@
 #include "fd_gui.h"
 #include "fd_gui_peers.h"
 #include "fd_gui_printf.h"
+#include "fd_gui_metrics.h"
 
 #include "../metrics/fd_metrics.h"
 #include "../plugin/fd_plugin.h"
@@ -397,6 +398,82 @@ fd_gui_estimated_tps_snap( fd_gui_t * gui ) {
   gui->summary.estimated_tps_history_idx = (gui->summary.estimated_tps_history_idx+1UL) % FD_GUI_TPS_HISTORY_SAMPLE_CNT;
 }
 
+static void
+fd_gui_network_stats_snap( fd_gui_t *               gui,
+                           fd_gui_network_stats_t * cur ) {
+  fd_topo_t * topo = gui->topo;
+  ulong gossvf_tile_cnt = fd_topo_tile_name_cnt( topo, "gossvf" );
+  ulong gossip_tile_cnt = fd_topo_tile_name_cnt( topo, "gossip" );
+  ulong shred_tile_cnt  = fd_topo_tile_name_cnt( topo, "shred" );
+  ulong net_tile_cnt    = fd_topo_tile_name_cnt( topo, "net" );
+  ulong quic_tile_cnt   = fd_topo_tile_name_cnt( topo, "quic" );
+
+  cur->in.gossip   = fd_gui_metrics_gossip_total_ingress_bytes( topo, gossvf_tile_cnt );
+  cur->out.gossip  = fd_gui_metrics_gosip_total_egress_bytes( topo, gossip_tile_cnt );
+  cur->in.turbine  = fd_gui_metrics_sum_tiles_counter( topo, "shred", shred_tile_cnt, MIDX( COUNTER, SHRED, SHRED_TURBINE_RCV_BYTES ) );
+
+  cur->out.turbine = 0UL;
+  cur->out.repair  = 0UL;
+  cur->out.tpu     = 0UL;
+  for( ulong i=0UL; i<net_tile_cnt; i++ ) {
+    ulong net_tile_idx = fd_topo_find_tile( topo, "net", i );
+    if( FD_UNLIKELY( net_tile_idx==ULONG_MAX ) ) continue;
+    fd_topo_tile_t const * net = &topo->tiles[ net_tile_idx ];
+    for( ulong j=0UL; j<net->in_cnt; j++ ) {
+      if( FD_UNLIKELY( !strcmp( topo->links[ net->in_link_id[ j ] ].name, "shred_net" ) ) ) {
+          cur->out.turbine += fd_metrics_link_in( net->metrics, j )[ FD_METRICS_COUNTER_LINK_CONSUMED_SIZE_BYTES_OFF ];
+      }
+
+      if( FD_UNLIKELY( !strcmp( topo->links[ net->in_link_id[ j ] ].name, "repair_net" ) ) ) {
+          cur->out.repair += fd_metrics_link_in( net->metrics, j )[ FD_METRICS_COUNTER_LINK_CONSUMED_SIZE_BYTES_OFF ];
+      }
+
+      if( FD_UNLIKELY( !strcmp( topo->links[ net->in_link_id[ j ] ].name, "send_net" ) ) ) {
+          cur->out.tpu += fd_metrics_link_in( net->metrics, j )[ FD_METRICS_COUNTER_LINK_CONSUMED_SIZE_BYTES_OFF ];
+      }
+    }
+  }
+
+  cur->in.repair = fd_gui_metrics_sum_tiles_counter( topo, "shred", shred_tile_cnt, MIDX( COUNTER, SHRED, SHRED_OUT_RCV_BYTES ) );
+  ulong repair_tile_idx = fd_topo_find_tile( topo, "repair", 0UL );
+  if( FD_LIKELY( repair_tile_idx!=ULONG_MAX ) ) {
+    fd_topo_tile_t const * repair = &topo->tiles[ repair_tile_idx ];
+
+    for( ulong i=0UL; i<repair->in_cnt; i++ ) {
+      if( FD_UNLIKELY( !strcmp( topo->links[ repair->in_link_id[ i ] ].name, "net_repair" ) ) ) {
+          cur->in.repair += fd_metrics_link_in( repair->metrics, i )[ FD_METRICS_COUNTER_LINK_CONSUMED_SIZE_BYTES_OFF ];
+      }
+    }
+  }
+
+  cur->in.tpu = 0UL;
+  for( ulong i=0UL; i<quic_tile_cnt; i++ ) {
+    ulong quic_tile_idx = fd_topo_find_tile( topo, "quic", i );
+    if( FD_UNLIKELY( quic_tile_idx==ULONG_MAX ) ) continue;
+    fd_topo_tile_t const * quic = &topo->tiles[ quic_tile_idx ];
+    volatile ulong * quic_metrics = fd_metrics_tile( quic->metrics );
+    cur->in.tpu += quic_metrics[ MIDX( COUNTER, QUIC, RECEIVED_BYTES ) ];
+  }
+
+  ulong bundle_tile_idx = fd_topo_find_tile( topo, "bundle", 0UL );
+  if( FD_LIKELY( bundle_tile_idx!=ULONG_MAX ) ) {
+    fd_topo_tile_t const * bundle = &topo->tiles[ bundle_tile_idx ];
+    volatile ulong * bundle_metrics = fd_metrics_tile( bundle->metrics );
+    cur->in.tpu += bundle_metrics[ MIDX( COUNTER, BUNDLE, PROTO_RECEIVED_BYTES ) ];
+  }
+
+  ulong metric_tile_idx = fd_topo_find_tile( topo, "metric", 0UL );
+  if( FD_LIKELY( metric_tile_idx!=ULONG_MAX ) ) {
+    fd_topo_tile_t const * metric = &topo->tiles[ metric_tile_idx ];
+    volatile ulong * metric_metrics = fd_metrics_tile( metric->metrics );
+    cur->in.metric  = metric_metrics[ MIDX( COUNTER, METRIC, BYTES_READ ) ];
+    cur->out.metric = metric_metrics[ MIDX( COUNTER, METRIC, BYTES_WRITTEN ) ];
+  } else {
+    cur->in.metric  = 0UL;
+    cur->out.metric = 0UL;
+  }
+}
+
 /* Snapshot all of the data from metrics to construct a view of the
    transaction waterfall.
 
@@ -828,6 +905,10 @@ fd_gui_poll( fd_gui_t * gui, long now ) {
     fd_gui_printf_live_txn_waterfall( gui, gui->summary.txn_waterfall_reference, gui->summary.txn_waterfall_current, 0UL /* TODO: REAL NEXT LEADER SLOT */ );
     fd_http_server_ws_broadcast( gui->http );
 
+    fd_gui_network_stats_snap( gui, gui->summary.network_stats_current );
+    fd_gui_printf_live_network_metrics( gui, gui->summary.network_stats_current );
+    fd_http_server_ws_broadcast( gui->http );
+
     *gui->summary.tile_stats_reference = *gui->summary.tile_stats_current;
     fd_gui_tile_stats_snap( gui, gui->summary.txn_waterfall_current, gui->summary.tile_stats_current, now );
     fd_gui_printf_live_tile_stats( gui, gui->summary.tile_stats_reference, gui->summary.tile_stats_current );

src/disco/gui/fd_gui.h

@@ -257,6 +257,19 @@ struct fd_gui_scheduler_counts {
 
 typedef struct fd_gui_scheduler_counts fd_gui_scheduler_counts_t;
 
+struct fd_gui_network_stats {
+  /* total bytes accumulated */
+  struct {
+    ulong turbine;
+    ulong gossip;
+    ulong tpu;
+    ulong repair;
+    ulong metric;
+  } in, out;
+};
+
+typedef struct fd_gui_network_stats fd_gui_network_stats_t;
+
 struct fd_gui_leader_slot {
   ulong slot;
   long  leader_start_time; /* UNIX timestamp of when we first became leader in this slot */
@@ -633,6 +646,8 @@ struct fd_gui {
     ulong estimated_tps_history_idx;
     ulong estimated_tps_history[ FD_GUI_TPS_HISTORY_SAMPLE_CNT ][ 3UL ];
 
+    fd_gui_network_stats_t network_stats_current[ 1 ];
+
     fd_gui_txn_waterfall_t txn_waterfall_reference[ 1 ];
     fd_gui_txn_waterfall_t txn_waterfall_current[ 1 ];
 

src/disco/gui/fd_gui_metrics.h

@@ -0,0 +1,59 @@
+#ifndef HEADER_fd_src_disco_gui_fd_gui_metrics_h
+#define HEADER_fd_src_disco_gui_fd_gui_metrics_h
+
+#include "../../util/fd_util_base.h"
+#include "../topo/fd_topo.h"
+#include "../../disco/metrics/fd_metrics.h"
+
+static inline ulong
+fd_gui_metrics_sum_tiles_counter( fd_topo_t const *    topo,
+                                  char const *         name,
+                                  ulong                tile_cnt,
+                                  ulong                metric_idx ) {
+  ulong total = 0UL;
+  for( ulong i = 0UL; i < topo->tile_cnt; i++ ) {
+    if( FD_UNLIKELY( !strcmp( topo->tiles[ i ].name, name ) ) ) {
+      FD_TEST( topo->tiles[ i ].kind_id < tile_cnt );
+      fd_topo_tile_t const * tile = &topo->tiles[ i ];
+      volatile ulong const * tile_metrics = fd_metrics_tile( tile->metrics );
+      total += tile_metrics[ metric_idx ];
+    }
+  }
+  return total;
+}
+
+static inline ulong
+fd_gui_metrics_gossip_total_ingress_bytes( fd_topo_t const * topo, ulong gossvf_tile_cnt ) {
+  return fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_SUCCESS_PULL_REQUEST) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_SUCCESS_PULL_RESPONSE) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_SUCCESS_PUSH) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_SUCCESS_PRUNE) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_SUCCESS_PING) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_SUCCESS_PONG) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_DROPPED_UNPARSEABLE) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_DROPPED_PULL_REQUEST_NOT_CONTACT_INFO) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_DROPPED_PULL_REQUEST_LOOPBACK) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_DROPPED_PULL_REQUEST_INACTIVE) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_DROPPED_PULL_REQUEST_WALLCLOCK) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_DROPPED_PULL_REQUEST_SIGNATURE) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_DROPPED_PULL_REQUEST_SHRED_VERSION) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_DROPPED_PRUNE_DESTINATION) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_DROPPED_PRUNE_WALLCLOCK) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_DROPPED_PRUNE_SIGNATURE) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_DROPPED_PUSH_NO_VALID_CRDS) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_DROPPED_PULL_RESPONSE_NO_VALID_CRDS) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_DROPPED_PING_SIGNATURE) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossvf", gossvf_tile_cnt, MIDX( COUNTER, GOSSVF, MESSAGE_RX_BYTES_DROPPED_PONG_SIGNATURE) );
+}
+
+static inline ulong
+fd_gui_metrics_gosip_total_egress_bytes( fd_topo_t const * topo, ulong gossip_tile_cnt ) {
+  return fd_gui_metrics_sum_tiles_counter( topo, "gossip", gossip_tile_cnt, MIDX( COUNTER, GOSSIP, MESSAGE_TX_BYTES_PING ) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossip", gossip_tile_cnt, MIDX( COUNTER, GOSSIP, MESSAGE_TX_BYTES_PONG ) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossip", gossip_tile_cnt, MIDX( COUNTER, GOSSIP, MESSAGE_TX_BYTES_PRUNE ) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossip", gossip_tile_cnt, MIDX( COUNTER, GOSSIP, MESSAGE_TX_BYTES_PULL_REQUEST ) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossip", gossip_tile_cnt, MIDX( COUNTER, GOSSIP, MESSAGE_TX_BYTES_PULL_RESPONSE ) )
+       + fd_gui_metrics_sum_tiles_counter( topo, "gossip", gossip_tile_cnt, MIDX( COUNTER, GOSSIP, MESSAGE_TX_BYTES_PUSH ) );
+}
+
+#endif /* HEADER_fd_src_disco_gui_fd_gui_metrics_h */