From 63eef4078038bad67291c1877c6ed8158fac24f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Rakic?= Date: Thu, 29 Oct 2020 03:08:38 +0100 Subject: [PATCH] Wrap `analyzeme` 0.7.0 and 9.0.0 under a single API --- Cargo.lock | 32 ++- site/Cargo.toml | 3 +- site/src/self_profile.rs | 51 +---- site/src/self_profile/crox.rs | 37 +-- site/src/self_profile/flamegraph.rs | 10 +- site/src/self_profile/versioning.rs | 334 ++++++++++++++++++++++++++++ 6 files changed, 373 insertions(+), 94 deletions(-) create mode 100644 site/src/self_profile/versioning.rs diff --git a/Cargo.lock b/Cargo.lock index 81df58abe..a357130d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,10 +21,23 @@ dependencies = [ [[package]] name = "analyzeme" version = "0.7.1" -source = "git+https://github.com/rust-lang/measureme#de3edc5cfb6961580dc04b720de963510c209fc4" +source = "git+https://github.com/rust-lang/measureme?rev=de3edc5cfb6961580dc04b720de963510c209fc4#de3edc5cfb6961580dc04b720de963510c209fc4" dependencies = [ "byteorder", - "measureme", + "measureme 0.7.1", + "memchr", + "rustc-hash", + "serde", + "serde_json", +] + +[[package]] +name = "analyzeme" +version = "9.0.0" +source = "git+https://github.com/rust-lang/measureme?rev=2c45b109a98ff143be39bc0c14ae2153fa0ca8ca#2c45b109a98ff143be39bc0c14ae2153fa0ca8ca" +dependencies = [ + "byteorder", + "measureme 9.0.0", "memchr", "rustc-hash", "serde", @@ -951,7 +964,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "measureme" version = "0.7.1" -source = "git+https://github.com/rust-lang/measureme#de3edc5cfb6961580dc04b720de963510c209fc4" +source = "git+https://github.com/rust-lang/measureme?rev=de3edc5cfb6961580dc04b720de963510c209fc4#de3edc5cfb6961580dc04b720de963510c209fc4" dependencies = [ "byteorder", "memmap", @@ -959,6 +972,16 @@ dependencies = [ "rustc-hash", ] +[[package]] +name = "measureme" +version = "9.0.0" +source = "git+https://github.com/rust-lang/measureme?rev=2c45b109a98ff143be39bc0c14ae2153fa0ca8ca#2c45b109a98ff143be39bc0c14ae2153fa0ca8ca" +dependencies = [ + "parking_lot 0.11.0", + "rustc-hash", + "smallvec", +] + [[package]] name = "memchr" version = "2.3.3" @@ -1738,7 +1761,8 @@ checksum = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7" name = "site" version = "0.1.0" dependencies = [ - "analyzeme", + "analyzeme 0.7.1", + "analyzeme 9.0.0", "anyhow", "arc-swap", "async-trait", diff --git a/site/Cargo.toml b/site/Cargo.toml index 4a934baaa..4cc37313c 100644 --- a/site/Cargo.toml +++ b/site/Cargo.toml @@ -40,10 +40,11 @@ async-trait = "0.1" database = { path = "../database" } bytes = "0.5.6" url = "2" -analyzeme = { git = "https://github.com/rust-lang/measureme" } tar = "0.4" inferno = { version="0.10", default-features = false } mime = "0.3" +analyzeme-7 = { package = "analyzeme", git = "https://github.com/rust-lang/measureme", rev = "de3edc5cfb6961580dc04b720de963510c209fc4" } +analyzeme-9 = { package = "analyzeme", git = "https://github.com/rust-lang/measureme", rev = "2c45b109a98ff143be39bc0c14ae2153fa0ca8ca" } [dependencies.collector] path = "../collector" diff --git a/site/src/self_profile.rs b/site/src/self_profile.rs index f139b98fe..7a04b55de 100644 --- a/site/src/self_profile.rs +++ b/site/src/self_profile.rs @@ -6,14 +6,15 @@ use anyhow::Context; use bytes::buf::BufExt; use hyper::StatusCode; use std::collections::HashMap; -use std::fmt; -use std::io::Read; type Response = http::Response; +mod versioning; pub mod crox; pub mod flamegraph; +use versioning::Pieces; + pub struct Output { pub data: Vec, pub filename: &'static str, @@ -49,22 +50,6 @@ pub fn generate( } } -pub struct Pieces { - pub string_data: Vec, - pub string_index: Vec, - pub events: Vec, -} - -impl fmt::Debug for Pieces { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Pieces") - .field("string_data", &self.string_data.len()) - .field("string_index", &self.string_index.len()) - .field("events", &self.events.len()) - .finish() - } -} - pub async fn get_pieces( body: crate::api::self_profile_raw::Request, data: &InputData, @@ -118,33 +103,3 @@ pub async fn get_pieces( }; Ok(pieces) } - -impl Pieces { - fn from_tarball(mut tarball: tar::Archive) -> anyhow::Result { - let mut pieces = Pieces { - string_data: Vec::new(), - string_index: Vec::new(), - events: Vec::new(), - }; - - for entry in tarball.entries().context("entries")? { - let mut entry = entry.context("tarball entry")?; - let path = entry.path_bytes(); - if *path == *b"self-profile.string_index" { - entry - .read_to_end(&mut pieces.string_index) - .context("reading string index")?; - } else if *path == *b"self-profile.string_data" { - entry - .read_to_end(&mut pieces.string_data) - .context("reading string data")?; - } else if *path == *b"self-profile.events" { - entry - .read_to_end(&mut pieces.events) - .context("reading events")?; - } - } - - Ok(pieces) - } -} diff --git a/site/src/self_profile/crox.rs b/site/src/self_profile/crox.rs index 297792695..316ae038e 100644 --- a/site/src/self_profile/crox.rs +++ b/site/src/self_profile/crox.rs @@ -4,7 +4,7 @@ use hashbrown::HashMap; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use analyzeme::{ProfilingData, Timestamp}; +use super::versioning::ProfilingData; use serde::ser::SerializeSeq; use serde::{Serialize, Serializer}; @@ -65,11 +65,11 @@ fn generate_thread_to_collapsed_thread_mapping( thread_start_and_end .entry(event.thread_id) .and_modify(|(thread_start, thread_end)| { - let (event_min, event_max) = timestamp_to_min_max(event.timestamp); + let (event_min, event_max) = event.timestamp.to_min_max(); *thread_start = cmp::min(*thread_start, event_min); *thread_end = cmp::max(*thread_end, event_max); }) - .or_insert_with(|| timestamp_to_min_max(event.timestamp)); + .or_insert_with(|| event.timestamp.to_min_max()); } // collect the the threads in order of the end time let mut end_and_thread = thread_start_and_end @@ -109,29 +109,13 @@ fn generate_thread_to_collapsed_thread_mapping( thread_to_collapsed_thread } -fn get_args(full_event: &analyzeme::Event) -> Option> { - if !full_event.additional_data.is_empty() { - Some( - full_event - .additional_data - .iter() - .enumerate() - .map(|(i, arg)| (format!("arg{}", i).to_string(), arg.to_string())) - .collect(), - ) - } else { - None - } -} - /// Returns JSON blob fit, `chrome_profiler.json`. pub fn generate(pieces: super::Pieces, opt: Opt) -> anyhow::Result> { let mut serializer = serde_json::Serializer::new(Vec::new()); let mut seq = serializer.serialize_seq(None)?; - let data = ProfilingData::from_buffers(pieces.string_data, pieces.string_index, pieces.events) - .map_err(|e| anyhow::format_err!("{:?}", e))?; + let data = pieces.into_profiling_data()?; let thread_to_collapsed_thread = generate_thread_to_collapsed_thread_mapping(opt.collapse_threads, &data); @@ -156,7 +140,7 @@ pub fn generate(pieces: super::Pieces, opt: Opt) -> anyhow::Result> { thread_id: *thread_to_collapsed_thread .get(&event.thread_id) .unwrap_or(&event.thread_id), - args: get_args(&full_event), + args: full_event.get_args(), }; seq.serialize_element(&crox_event)?; } @@ -201,14 +185,3 @@ pub fn generate(pieces: super::Pieces, opt: Opt) -> anyhow::Result> { Ok(serializer.into_inner()) } - -fn timestamp_to_min_max(timestamp: Timestamp) -> (SystemTime, SystemTime) { - match timestamp { - Timestamp::Instant(t) => (t, t), - Timestamp::Interval { start, end } => { - // Usually start should always be greater than end, but let's not - // choke on invalid data here. - (cmp::min(start, end), cmp::max(start, end)) - } - } -} diff --git a/site/src/self_profile/flamegraph.rs b/site/src/self_profile/flamegraph.rs index 02722d585..b6b537b03 100644 --- a/site/src/self_profile/flamegraph.rs +++ b/site/src/self_profile/flamegraph.rs @@ -1,4 +1,3 @@ -use analyzeme::{collapse_stacks, ProfilingData}; use anyhow::Context; use inferno::flamegraph::{from_lines, Options as FlamegraphOptions}; @@ -6,14 +5,7 @@ use inferno::flamegraph::{from_lines, Options as FlamegraphOptions}; pub struct Opt {} pub fn generate(title: &str, pieces: super::Pieces, _: Opt) -> anyhow::Result> { - let profiling_data = - ProfilingData::from_buffers(pieces.string_data, pieces.string_index, pieces.events) - .map_err(|e| anyhow::format_err!("{:?}", e))?; - - let recorded_stacks = collapse_stacks(&profiling_data) - .iter() - .map(|(unique_stack, count)| format!("{} {}", unique_stack, count)) - .collect::>(); + let recorded_stacks = pieces.into_collapsed_stacks()?; let mut file = Vec::new(); let mut flamegraph_options = FlamegraphOptions::default(); diff --git a/site/src/self_profile/versioning.rs b/site/src/self_profile/versioning.rs new file mode 100644 index 000000000..11d98748f --- /dev/null +++ b/site/src/self_profile/versioning.rs @@ -0,0 +1,334 @@ +//! This module wraps `analyzeme` APIs for versions 0.7.* (v7) and 9.0.* (v9): +//! - `analyzeme` structs, with data referenced in the `crox` and `flamegraph` modules +//! are kept as-is, with an additional inner field versioning the data when needed. +//! - the API referenced in other modules is kept very similar to `analyzeme` +//! with additions where owning the impls allowed using more methods. The +//! implementations are delegated to the real `analyzeme` impls, regardless of how trivial +//! they are, via versioned enums wrapping the original structs. + +use anyhow::Context; +use hashbrown::HashMap; +use std::borrow::Cow; +use std::cmp; +use std::fmt; +use std::io::Read; +use std::time::{Duration, SystemTime}; + +pub enum Pieces { + V7(PiecesV7), + V9(PiecesV9), +} + +pub struct PiecesV7 { + pub string_data: Vec, + pub string_index: Vec, + pub events: Vec, +} + +pub struct PiecesV9 { + pub data: Vec, +} + +pub struct ProfilingData { + pub metadata: Metadata, + inner: VersionedProfilingData, +} + +#[derive(Debug)] +pub struct Metadata { + pub start_time: SystemTime, + pub process_id: u32, + pub cmd: String, +} + +enum VersionedProfilingData { + V7(analyzeme_7::ProfilingData), + V9(analyzeme_9::ProfilingData), +} + +#[derive(Clone, Debug)] +pub struct LightweightEvent<'a> { + pub event_index: usize, + pub thread_id: u32, + pub timestamp: Timestamp, + inner: VersionedLightweightEvent<'a>, +} + +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +pub enum Timestamp { + V7(analyzeme_7::Timestamp), + V9(analyzeme_9::Timestamp), +} + +#[derive(Clone, Debug)] +enum VersionedLightweightEvent<'a> { + V7(analyzeme_7::LightweightEvent<'a>), + V9(analyzeme_9::LightweightEvent<'a>), +} + +#[derive(Clone, Eq, PartialEq, Hash, Debug)] +pub struct Event<'a> { + pub event_kind: Cow<'a, str>, + pub label: Cow<'a, str>, + pub additional_data: Vec>, + pub timestamp: Timestamp, + pub thread_id: u32, +} + +impl Pieces { + /// Tries to extract data for `analyzeme` versions 7 or 9 from a tarball: + /// - if the expected v9 `.mm_profdata` file is found in the tarball, v9 will be the chosen version + /// - otherwise, we fall back to the v7 3-file format + pub fn from_tarball(mut tarball: tar::Archive) -> anyhow::Result { + let mut pieces_v7 = PiecesV7 { + string_data: Vec::new(), + string_index: Vec::new(), + events: Vec::new(), + }; + + for entry in tarball.entries().context("entries")? { + let mut entry = entry.context("tarball entry")?; + let path = entry.path_bytes(); + if *path == *b"self-profile.string_index" { + entry + .read_to_end(&mut pieces_v7.string_index) + .context("reading v7 string index")?; + } else if *path == *b"self-profile.string_data" { + entry + .read_to_end(&mut pieces_v7.string_data) + .context("reading v7 string data")?; + } else if *path == *b"self-profile.events" { + entry + .read_to_end(&mut pieces_v7.events) + .context("reading v7 events")?; + } else if *path == *b"self-profile.mm_profdata" { + let mut pieces_v9 = PiecesV9 { data: Vec::new() }; + entry + .read_to_end(&mut pieces_v9.data) + .context("reading v9 data")?; + return Ok(Pieces::V9(pieces_v9)); + } + } + + Ok(Pieces::V7(pieces_v7)) + } + + pub fn into_collapsed_stacks(self) -> anyhow::Result> { + match self { + Pieces::V7(pieces) => { + let profiling_data = analyzeme_7::ProfilingData::from_buffers( + pieces.string_data, + pieces.string_index, + pieces.events, + ) + .map_err(|e| anyhow::format_err!("{:?}", e))?; + + let recorded_stacks = analyzeme_7::collapse_stacks(&profiling_data) + .iter() + .map(|(unique_stack, count)| format!("{} {}", unique_stack, count)) + .collect::>(); + Ok(recorded_stacks) + } + Pieces::V9(pieces) => { + let profiling_data = analyzeme_9::ProfilingData::from_paged_buffer(pieces.data) + .map_err(|e| anyhow::format_err!("{:?}", e))?; + + let recorded_stacks = analyzeme_9::collapse_stacks(&profiling_data) + .iter() + .map(|(unique_stack, count)| format!("{} {}", unique_stack, count)) + .collect::>(); + Ok(recorded_stacks) + } + } + } + + pub fn into_profiling_data(self) -> anyhow::Result { + match self { + Pieces::V7(pieces) => { + let profiling_data = analyzeme_7::ProfilingData::from_buffers( + pieces.string_data, + pieces.string_index, + pieces.events, + ) + .map_err(|e| anyhow::format_err!("{:?}", e))?; + Ok(profiling_data.into()) + } + Pieces::V9(pieces) => { + let profiling_data = analyzeme_9::ProfilingData::from_paged_buffer(pieces.data) + .map_err(|e| anyhow::format_err!("{:?}", e))?; + Ok(profiling_data.into()) + } + } + } +} + +impl fmt::Debug for Pieces { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Pieces::V7(pieces) => f + .debug_struct("Pieces (V7)") + .field("string_data", &pieces.string_data.len()) + .field("string_index", &pieces.string_index.len()) + .field("events", &pieces.events.len()) + .finish(), + Pieces::V9(pieces) => f + .debug_struct("Pieces (V9)") + .field("data", &pieces.data.len()) + .finish(), + } + } +} + +impl ProfilingData { + pub fn iter<'a>(&'a self) -> Box> + 'a> { + match &self.inner { + VersionedProfilingData::V7(data) => Box::new(data.iter().map(|e| e.into())), + VersionedProfilingData::V9(data) => Box::new(data.iter().map(|e| e.into())), + } + } +} + +impl From for ProfilingData { + fn from(data: analyzeme_7::ProfilingData) -> Self { + ProfilingData { + metadata: Metadata { + start_time: data.metadata.start_time, + process_id: data.metadata.process_id, + cmd: data.metadata.cmd.clone(), + }, + inner: VersionedProfilingData::V7(data), + } + } +} + +impl From for ProfilingData { + fn from(data: analyzeme_9::ProfilingData) -> Self { + ProfilingData { + metadata: Metadata { + start_time: data.metadata.start_time, + process_id: data.metadata.process_id, + cmd: data.metadata.cmd.clone(), + }, + inner: VersionedProfilingData::V9(data), + } + } +} + +impl<'a> LightweightEvent<'a> { + pub fn duration(&self) -> Option { + match &self.inner { + VersionedLightweightEvent::V7(lightweight_event) => lightweight_event.duration(), + VersionedLightweightEvent::V9(lightweight_event) => lightweight_event.duration(), + } + } + + pub fn to_event(&self) -> Event<'a> { + match &self.inner { + VersionedLightweightEvent::V7(lightweight_event) => lightweight_event.to_event().into(), + VersionedLightweightEvent::V9(lightweight_event) => lightweight_event.to_event().into(), + } + } +} + +impl<'a> Event<'a> { + pub fn get_args(&self) -> Option> { + if !self.additional_data.is_empty() { + Some( + self.additional_data + .iter() + .enumerate() + .map(|(i, arg)| (format!("arg{}", i).to_string(), arg.to_string())) + .collect(), + ) + } else { + None + } + } +} + +impl Timestamp { + pub fn is_instant(&self) -> bool { + match *self { + Timestamp::V7(timestamp) => timestamp.is_instant(), + Timestamp::V9(timestamp) => timestamp.is_instant(), + } + } + + pub fn start(&self) -> SystemTime { + match *self { + Timestamp::V7(timestamp) => timestamp.start(), + Timestamp::V9(timestamp) => timestamp.start(), + } + } + + pub fn to_min_max(self) -> (SystemTime, SystemTime) { + match self { + Timestamp::V7(analyzeme_7::Timestamp::Instant(t)) + | Timestamp::V9(analyzeme_9::Timestamp::Instant(t)) => (t, t), + Timestamp::V7(analyzeme_7::Timestamp::Interval { start, end }) + | Timestamp::V9(analyzeme_9::Timestamp::Interval { start, end }) => { + // Usually start should always be greater than end, but let's not + // choke on invalid data here. + (cmp::min(start, end), cmp::max(start, end)) + } + } + } +} + +impl<'a> From> for LightweightEvent<'a> { + fn from(lightweight_event: analyzeme_7::LightweightEvent<'a>) -> Self { + LightweightEvent { + event_index: lightweight_event.event_index, + thread_id: lightweight_event.thread_id, + timestamp: lightweight_event.timestamp.into(), + inner: VersionedLightweightEvent::V7(lightweight_event), + } + } +} + +impl<'a> From> for LightweightEvent<'a> { + fn from(lightweight_event: analyzeme_9::LightweightEvent<'a>) -> Self { + LightweightEvent { + event_index: lightweight_event.event_index, + thread_id: lightweight_event.thread_id, + timestamp: lightweight_event.timestamp.into(), + inner: VersionedLightweightEvent::V9(lightweight_event), + } + } +} + +impl<'a> From> for Event<'a> { + fn from(event: analyzeme_7::Event<'a>) -> Self { + Event { + additional_data: event.additional_data, + event_kind: event.event_kind, + label: event.label, + timestamp: event.timestamp.into(), + thread_id: event.thread_id, + } + } +} + +impl<'a> From> for Event<'a> { + fn from(event: analyzeme_9::Event<'a>) -> Self { + Event { + additional_data: event.additional_data, + event_kind: event.event_kind, + label: event.label, + timestamp: event.timestamp.into(), + thread_id: event.thread_id, + } + } +} + +impl From for Timestamp { + fn from(timestamp: analyzeme_7::Timestamp) -> Self { + Timestamp::V7(timestamp) + } +} + +impl From for Timestamp { + fn from(timestamp: analyzeme_9::Timestamp) -> Self { + Timestamp::V9(timestamp) + } +}