Add more instance-level metrics

src/downloads_counter.rs

@@ -199,6 +199,10 @@ impl DownloadsCounter {
     pub fn shards_count(&self) -> usize {
         self.inner.shards().len()
     }
+
+    pub(crate) fn pending_count(&self) -> i64 {
+        self.pending_count.load(Ordering::SeqCst)
+    }
 }
 
 #[derive(Debug, Default, Copy, Clone, Eq, PartialEq)]

src/metrics/instance.rs

@@ -19,7 +19,9 @@
 
 use crate::util::errors::AppResult;
 use crate::{app::App, db::DieselPool};
-use prometheus::{proto::MetricFamily, IntCounter, IntGauge, IntGaugeVec};
+use prometheus::{
+    proto::MetricFamily, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec,
+};
 
 metrics! {
     pub struct InstanceMetrics {
@@ -33,10 +35,17 @@ metrics! {
         /// Number of requests currently being processed
         pub requests_in_flight: IntGauge,
 
+        /// Response times of our endpoints
+        pub response_times: HistogramVec["endpoint"],
+        /// Nmber of responses per status code
+        pub responses_by_status_code_total: IntCounterVec["status"],
+
         /// Number of download requests that were served with an unconditional redirect.
         pub downloads_unconditional_redirects_total: IntCounter,
         /// Number of download requests with a non-canonical crate name.
         pub downloads_non_canonical_crate_name_total: IntCounter,
+        /// Number of download requests that are not counted yet.
+        downloads_not_counted_total: IntGauge,
     }
 
     // All instance metrics will be prefixed with this namespace.
@@ -51,6 +60,9 @@ impl InstanceMetrics {
             self.refresh_pool_stats("follower", follower)?;
         }
 
+        self.downloads_not_counted_total
+            .set(app.downloads_counter.pending_count());
+
         Ok(self.registry.gather())
     }
 

src/metrics/log_encoder.rs

@@ -58,6 +58,27 @@ fn families_to_json_events(families: &[MetricFamily]) -> Vec<VectorEvent<'_>> {
                 MetricType::GAUGE => VectorMetricData::Gauge {
                     value: metric.get_gauge().get_value(),
                 },
+                MetricType::HISTOGRAM => {
+                    let histogram = metric.get_histogram();
+
+                    // We need to convert from cumulative counts (used by the Prometheus library)
+                    // to plain counts (used by Vector).
+                    let mut buckets = Vec::new();
+                    let mut last_cumulative_count = 0;
+                    for bucket in histogram.get_bucket() {
+                        buckets.push(VectorHistogramBucket {
+                            upper_limit: bucket.get_upper_bound(),
+                            count: bucket.get_cumulative_count() - last_cumulative_count,
+                        });
+                        last_cumulative_count = bucket.get_cumulative_count();
+                    }
+
+                    VectorMetricData::AggregatedHistogram {
+                        count: histogram.get_sample_count(),
+                        sum: histogram.get_sample_sum(),
+                        buckets,
+                    }
+                }
                 other => {
                     panic!("unsupported metric type: {:?}", other)
                 }
@@ -96,15 +117,30 @@ struct VectorMetric<'a> {
 #[derive(Serialize, Debug, PartialEq)]
 #[serde(rename_all = "snake_case")]
 enum VectorMetricData {
-    Counter { value: f64 },
-    Gauge { value: f64 },
+    AggregatedHistogram {
+        buckets: Vec<VectorHistogramBucket>,
+        count: u64,
+        sum: f64,
+    },
+    Counter {
+        value: f64,
+    },
+    Gauge {
+        value: f64,
+    },
+}
+
+#[derive(Serialize, Debug, PartialEq)]
+struct VectorHistogramBucket {
+    upper_limit: f64,
+    count: u64,
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use anyhow::Error;
-    use prometheus::{IntCounter, IntGauge, IntGaugeVec, Opts, Registry};
+    use prometheus::{Histogram, HistogramOpts, IntCounter, IntGauge, IntGaugeVec, Opts, Registry};
 
     #[test]
     fn test_counter_to_json() -> Result<(), Error> {
@@ -175,6 +211,85 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_histogram_to_json() -> Result<(), Error> {
+        let histogram = Histogram::with_opts(HistogramOpts::new(
+            "sample_histogram",
+            "sample_histogram help message",
+        ))?;
+        let registry = Registry::new();
+        registry.register(Box::new(histogram.clone()))?;
+
+        let mut value = 0.0;
+        while value < 11.0 {
+            histogram.observe(value);
+            value += 0.001;
+        }
+
+        assert_eq!(
+            vec![VectorEvent {
+                metric: VectorMetric {
+                    data: VectorMetricData::AggregatedHistogram {
+                        buckets: vec![
+                            VectorHistogramBucket {
+                                upper_limit: 0.005,
+                                count: 6,
+                            },
+                            VectorHistogramBucket {
+                                upper_limit: 0.01,
+                                count: 4,
+                            },
+                            VectorHistogramBucket {
+                                upper_limit: 0.025,
+                                count: 15,
+                            },
+                            VectorHistogramBucket {
+                                upper_limit: 0.05,
+                                count: 25,
+                            },
+                            VectorHistogramBucket {
+                                upper_limit: 0.1,
+                                count: 50,
+                            },
+                            VectorHistogramBucket {
+                                upper_limit: 0.25,
+                                count: 150,
+                            },
+                            VectorHistogramBucket {
+                                upper_limit: 0.5,
+                                count: 250,
+                            },
+                            VectorHistogramBucket {
+                                upper_limit: 1.0,
+                                count: 500,
+                            },
+                            VectorHistogramBucket {
+                                upper_limit: 2.5,
+                                count: 1501,
+                            },
+                            VectorHistogramBucket {
+                                upper_limit: 5.0,
+                                count: 2499,
+                            },
+                            VectorHistogramBucket {
+                                upper_limit: 10.0,
+                                count: 5001,
+                            },
+                        ],
+                        count: 11001,
+                        sum: 60505.50000000138,
+                    },
+                    kind: "absolute",
+                    name: "sample_histogram",
+                    tags: IndexMap::new(),
+                }
+            }],
+            families_to_json_events(&registry.gather())
+        );
+
+        Ok(())
+    }
+
     #[test]
     fn test_metric_with_tags_to_json() -> Result<(), Error> {
         let gauge_vec = IntGaugeVec::new(

src/metrics/macros.rs

@@ -59,7 +59,7 @@ macro_rules! load_metric_type {
         use prometheus::$name;
         impl crate::metrics::macros::MetricFromOpts for $name {
             fn from_opts(opts: prometheus::Opts) -> Result<Self, prometheus::Error> {
-                $name::with_opts(opts)
+                $name::with_opts(opts.into())
             }
         }
     };

src/metrics/mod.rs

@@ -11,4 +11,6 @@ mod service;
 
 load_metric_type!(IntGauge as single);
 load_metric_type!(IntCounter as single);
+load_metric_type!(IntCounterVec as vec);
 load_metric_type!(IntGaugeVec as vec);
+load_metric_type!(HistogramVec as vec);

src/middleware/update_metrics.rs

@@ -1,5 +1,6 @@
 use super::app::RequestApp;
 use super::prelude::*;
+use conduit_router::RoutePattern;
 
 #[derive(Debug, Default)]
 pub(super) struct UpdateMetrics;
@@ -19,6 +20,25 @@ impl Middleware for UpdateMetrics {
         metrics.requests_in_flight.dec();
         metrics.requests_total.inc();
 
+        let endpoint = req
+            .extensions()
+            .find::<RoutePattern>()
+            .map(|p| p.pattern())
+            .unwrap_or("<unknown>");
+        metrics
+            .response_times
+            .with_label_values(&[endpoint])
+            .observe(req.elapsed().as_millis() as f64 / 1000.0);
+
+        let status = match &res {
+            Ok(res) => res.status().as_u16(),
+            Err(_) => 500,
+        };
+        metrics
+            .responses_by_status_code_total
+            .with_label_values(&[&status.to_string()])
+            .inc();
+
         res
     }
 }