gui: overview network stats

Author: jherrera-jump

book/api/metrics-generated.md

@@ -194,6 +194,7 @@
 |--------|------|-------------|
 | <span class="metrics-name">bundle_&#8203;transaction_&#8203;received</span> | counter | Total count of transactions received, including transactions within bundles |
 | <span class="metrics-name">bundle_&#8203;packet_&#8203;received</span> | counter | Total count of packets received |
+| <span class="metrics-name">bundle_&#8203;proto_&#8203;received_&#8203;bytes</span> | counter | Total count of bytes from received grpc protobuf payloads |
 | <span class="metrics-name">bundle_&#8203;bundle_&#8203;received</span> | counter | Total count of bundles received |
 | <span class="metrics-name">bundle_&#8203;errors</span><br/>{bundle_&#8203;error="<span class="metrics-enum">protobuf</span>"} | counter | Number of gRPC errors encountered (Protobuf decode/encode error) |
 | <span class="metrics-name">bundle_&#8203;errors</span><br/>{bundle_&#8203;error="<span class="metrics-enum">transport</span>"} | counter | Number of gRPC errors encountered (Transport error) |
@@ -450,7 +451,9 @@
 | <span class="metrics-name">shred_&#8203;force_&#8203;complete_&#8203;failure</span> | counter | The number of times we failed to force complete a FEC set on request |
 | <span class="metrics-name">shred_&#8203;force_&#8203;complete_&#8203;success</span> | counter | The number of times we successfully forced completed a FEC set on request |
 | <span class="metrics-name">shred_&#8203;shred_&#8203;out_&#8203;rcv</span> | counter | The number of times we received a repair shred |
-| <span class="metrics-name">shred_&#8203;shred_&#8203;turbine_&#8203;rcv</span> | counter | The number of times we received a turbine shred |
+| <span class="metrics-name">shred_&#8203;shred_&#8203;out_&#8203;rcv_&#8203;bytes</span> | counter | The number bytes received from network packets with repair shreds. Bytes include network headers. |
+| <span class="metrics-name">shred_&#8203;shred_&#8203;turbine_&#8203;rcv</span> | counter | The number bytes received from network packets with turbine shreds. Bytes include network headers. |
+| <span class="metrics-name">shred_&#8203;shred_&#8203;turbine_&#8203;rcv_&#8203;bytes</span> | counter | The number of bytes received from turbine shreds |
 | <span class="metrics-name">shred_&#8203;store_&#8203;insert_&#8203;wait</span> | histogram | Time in seconds spent waiting for the store to insert a new FEC set |
 | <span class="metrics-name">shred_&#8203;store_&#8203;insert_&#8203;work</span> | histogram | Time in seconds spent on inserting a new FEC set |
 
@@ -483,6 +486,9 @@
 | Metric | Type | Description |
 |--------|------|-------------|
 | <span class="metrics-name">metric_&#8203;boot_&#8203;timestamp_&#8203;nanos</span> | gauge | Timestamp when validator was started (nanoseconds since epoch) |
+| <span class="metrics-name">metric_&#8203;connection_&#8203;count</span> | gauge | The number of active http connections to the Prometheus endpoint |
+| <span class="metrics-name">metric_&#8203;bytes_&#8203;written</span> | counter | The total number of bytes written to all responses on the Prometheus endpoint |
+| <span class="metrics-name">metric_&#8203;bytes_&#8203;read</span> | counter | The total number of bytes read from all requests to the Prometheus endpoint |
 
 </div>
 
@@ -492,7 +498,7 @@
 
 | Metric | Type | Description |
 |--------|------|-------------|
-| <span class="metrics-name">gui_&#8203;connection_&#8203;count</span> | gauge | The number of active connections to the GUI service |
+| <span class="metrics-name">gui_&#8203;connection_&#8203;count</span> | gauge | The number of active http connections to the GUI service, excluding connections that have been upgraded to a WebSocket connection |
 | <span class="metrics-name">gui_&#8203;websocket_&#8203;connection_&#8203;count</span> | gauge | The number of active websocket connections to the GUI service |
 | <span class="metrics-name">gui_&#8203;websocket_&#8203;frames_&#8203;sent</span> | counter | The total number of websocket frames sent to all connections to the GUI service |
 | <span class="metrics-name">gui_&#8203;websocket_&#8203;frames_&#8203;received</span> | counter | The total number of websocket frames received from all connections to the GUI service |

book/api/websocket.md

@@ -748,6 +748,56 @@ transactions per second.
 }
 ```
 
+#### `summary.live_network_metrics`
+| frequency       | type             | example |
+|-----------------|------------------|---------|
+| *Once* + *10ms* | `NetworkMetrics` | below   |
+
+Live network metrics provides a live view of network bandwidth
+utilization across the various protocols used in the client.
+
+The `protocols` list contains various different protocols the client
+uses to communicate with the internet.
+
+```json
+[
+    "turbine",
+    "gossip",
+    "tpu",
+    "repair",
+    "metrics"
+]
+```
+
+- turbine: the protocol used to disseminate blockchain data, which
+contains primarily executable transactions.
+- gossip: the protocol used to disseminate node metadata, including node
+IP addresses used to help nodes find each other on the network
+- tpu: "transaction processing unit", refers to the various subsystems
+in a client used to consume and forward incoming Solana transactions for
+their next leader slot.
+- repair: a client subsystem which requests any missing block data
+needed by the replay pipeline which may have been lost over the network
+- metrics: refers to the Firedancer metrics tile, which serves an http
+Prometheus metrics endpoint
+
+```json
+{
+    "topic": "summary",
+    "key": "live_network_metrics",
+    "value": {
+        "ingress": [12345432, 5431234, 92345, ...],
+        "egress": [12345432, 5431234, 92345, ...],
+    }
+}
+```
+
+**`NetworkMetrics`**
+| Field   | Type       | Description |
+|---------|------------|-------------|
+| ingress | `number[]` | `ingress[i]` is the total number of ingress network bytes for `protocols[i]` |
+| egress  | `number[]` | `egress[i]` is the total number of egress network bytes for `protocols[i]` |
+
 #### `summary.live_txn_waterfall`
 | frequency        | type               | example |
 |------------------|--------------------|---------|

src/disco/bundle/fd_bundle_client.c

@@ -799,6 +799,7 @@ fd_bundle_client_grpc_rx_msg(
     ulong        request_ctx
 ) {
   fd_bundle_tile_t * ctx = app_ctx;
+  ctx->metrics.proto_received_bytes += protobuf_sz;
   pb_istream_t istream = pb_istream_from_buffer( protobuf, protobuf_sz );
   switch( request_ctx ) {
   case FD_BUNDLE_CLIENT_REQ_Auth_GenerateAuthChallenge:

src/disco/bundle/fd_bundle_tile.c

@@ -49,6 +49,7 @@ metrics_write( fd_bundle_tile_t * ctx ) {
   FD_MCNT_SET( BUNDLE, TRANSACTION_RECEIVED,   ctx->metrics.txn_received_cnt          );
   FD_MCNT_SET( BUNDLE, BUNDLE_RECEIVED,        ctx->metrics.bundle_received_cnt       );
   FD_MCNT_SET( BUNDLE, PACKET_RECEIVED,        ctx->metrics.packet_received_cnt       );
+  FD_MCNT_SET( BUNDLE, PROTO_RECEIVED_BYTES,   ctx->metrics.proto_received_bytes      );
   FD_MCNT_SET( BUNDLE, SHREDSTREAM_HEARTBEATS, ctx->metrics.shredstream_heartbeat_cnt );
   FD_MCNT_SET( BUNDLE, KEEPALIVES,             ctx->metrics.ping_ack_cnt              );
   FD_MCNT_SET( BUNDLE, ERRORS_PROTOBUF,        ctx->metrics.decode_fail_cnt           );

src/disco/bundle/fd_bundle_tile_private.h

@@ -33,6 +33,7 @@ struct fd_bundle_metrics {
   ulong txn_received_cnt;
   ulong bundle_received_cnt;
   ulong packet_received_cnt;
+  ulong proto_received_bytes;
   ulong shredstream_heartbeat_cnt;
   ulong ping_ack_cnt;
 

src/disco/gui/Local.mk

@@ -1,5 +1,5 @@
 ifdef FD_HAS_INT128
-$(call add-hdrs,fd_gui.h fd_gui_printf.h fd_gui_peers.h fd_gui_config_parse.h)
+$(call add-hdrs,fd_gui.h fd_gui_printf.h fd_gui_peers.h fd_gui_config_parse.h fd_gui_metrics.h)
 $(call add-objs,fd_gui fd_gui_printf fd_gui_peers fd_gui_config_parse fd_gui_tile generated/http_import_dist,fd_disco)
 $(OBJDIR)/obj/disco/gui/fd_gui_tile.o: book/public/fire.svg
 $(call make-unit-test,test_live_table,test_live_table,fd_disco fd_util)

src/disco/gui/fd_gui.c

@@ -1,6 +1,7 @@
 #include "fd_gui.h"
 #include "fd_gui_peers.h"
 #include "fd_gui_printf.h"
+#include "fd_gui_metrics.h"
 
 #include "../metrics/fd_metrics.h"
 #include "../plugin/fd_plugin.h"
@@ -397,6 +398,28 @@ fd_gui_estimated_tps_snap( fd_gui_t * gui ) {
   gui->summary.estimated_tps_history_idx = (gui->summary.estimated_tps_history_idx+1UL) % FD_GUI_TPS_HISTORY_SAMPLE_CNT;
 }
 
+static void
+fd_gui_network_stats_snap( fd_gui_t *               gui,
+                           fd_gui_network_stats_t * cur ) {
+  fd_topo_t * topo = gui->topo;
+  ulong gossvf_tile_cnt = fd_topo_tile_name_cnt( topo, "gossvf" );
+  ulong gossip_tile_cnt = fd_topo_tile_name_cnt( topo, "gossip" );
+  ulong shred_tile_cnt  = fd_topo_tile_name_cnt( topo, "shred" );
+  ulong net_tile_cnt    = fd_topo_tile_name_cnt( topo, "net" );
+  ulong quic_tile_cnt   = fd_topo_tile_name_cnt( topo, "quic" );
+
+  cur->in.turbine  = fd_gui_metrics_turbine_total_ingress_bytes( topo, shred_tile_cnt );
+  cur->out.turbine = fd_gui_metrics_turbine_total_egress_bytes( topo, net_tile_cnt );
+  cur->in.repair   = fd_gui_metrics_repair_total_ingress_bytes( topo, shred_tile_cnt );
+  cur->out.repair  = fd_gui_metrics_repair_total_egress_bytes( topo, net_tile_cnt );
+  cur->in.gossip   = fd_gui_metrics_gossip_total_ingress_bytes( topo, gossvf_tile_cnt );
+  cur->out.gossip  = fd_gui_metrics_gosip_total_egress_bytes( topo, gossip_tile_cnt );
+  cur->in.tpu      = fd_gui_metrics_tpu_total_ingress_bytes( topo, quic_tile_cnt );
+  cur->out.tpu     = fd_gui_metrics_tpu_total_egress_bytes( topo, net_tile_cnt );
+  cur->in.metric   = fd_gui_metrics_metric_total_ingress_bytes( topo );
+  cur->out.metric  = fd_gui_metrics_metric_total_egress_bytes( topo );
+}
+
 /* Snapshot all of the data from metrics to construct a view of the
    transaction waterfall.
 
@@ -828,6 +851,10 @@ fd_gui_poll( fd_gui_t * gui, long now ) {
     fd_gui_printf_live_txn_waterfall( gui, gui->summary.txn_waterfall_reference, gui->summary.txn_waterfall_current, 0UL /* TODO: REAL NEXT LEADER SLOT */ );
     fd_http_server_ws_broadcast( gui->http );
 
+    fd_gui_network_stats_snap( gui, gui->summary.network_stats_current );
+    fd_gui_printf_live_network_metrics( gui, gui->summary.network_stats_current );
+    fd_http_server_ws_broadcast( gui->http );
+
     *gui->summary.tile_stats_reference = *gui->summary.tile_stats_current;
     fd_gui_tile_stats_snap( gui, gui->summary.txn_waterfall_current, gui->summary.tile_stats_current, now );
     fd_gui_printf_live_tile_stats( gui, gui->summary.tile_stats_reference, gui->summary.tile_stats_current );

src/disco/gui/fd_gui.h

@@ -257,6 +257,19 @@ struct fd_gui_scheduler_counts {
 
 typedef struct fd_gui_scheduler_counts fd_gui_scheduler_counts_t;
 
+struct fd_gui_network_stats {
+  /* total bytes accumulated */
+  struct {
+    ulong turbine;
+    ulong gossip;
+    ulong tpu;
+    ulong repair;
+    ulong metric;
+  } in, out;
+};
+
+typedef struct fd_gui_network_stats fd_gui_network_stats_t;
+
 struct fd_gui_leader_slot {
   ulong slot;
   long  leader_start_time; /* UNIX timestamp of when we first became leader in this slot */
@@ -633,6 +646,8 @@ struct fd_gui {
     ulong estimated_tps_history_idx;
     ulong estimated_tps_history[ FD_GUI_TPS_HISTORY_SAMPLE_CNT ][ 3UL ];
 
+    fd_gui_network_stats_t network_stats_current[ 1 ];
+
     fd_gui_txn_waterfall_t txn_waterfall_reference[ 1 ];
     fd_gui_txn_waterfall_t txn_waterfall_current[ 1 ];