Turbopack: next build --analyze (#85197)

This implements a portion of the first iteration of a bundle analyzer for Turbopack in Next.js by writing data files to disk. A separate visualization and explorer tool will follow.

This data is written when builds are run with `next build --experimental-analyze`. An upcoming PR will implement a dedicated `next analyze` command that will write the data files without writing a complete build.

The data is derived from Turbopack’s module and chunk graphs, and precise module sizes are reversed from the source map data of each chunk. Importantly, this requires that source maps are generated within Turbopack and the project is recomputed with them -- however, no source maps are written as part of the build artifacts if the user did not configure their Next.js config that way.

The binary format written in these files is subject to change, so the feature is experimental for the time being. Until it stabilizes, no guarantees will be provided between it and the visualizer tool.

Test Plan: 

In a test app:
- do not configure `productionBrowserSourceMaps`
- run `next build` and confirm **no files are written** to `.next/diagnostics/analyze`
- run `next build --analyze` and confirm files **are** written to `.next/diagnostics/analyze`
- Open the data with the yet-to-be-published visualization tool and confirm data is visible

Author: sokra

Cargo.lock

@@ -317,6 +317,7 @@ turbopack-cli-utils = { path = "turbopack/crates/turbopack-cli-utils" }
 turbopack-core = { path = "turbopack/crates/turbopack-core" }
 turbopack-create-test-app = { path = "turbopack/crates/turbopack-create-test-app" }
 turbopack-css = { path = "turbopack/crates/turbopack-css" }
+turbopack-analyze = { path = "turbopack/crates/turbopack-analyze" }
 turbopack-browser = { path = "turbopack/crates/turbopack-browser" }
 turbopack-dev-server = { path = "turbopack/crates/turbopack-dev-server" }
 turbopack-ecmascript = { path = "turbopack/crates/turbopack-ecmascript" }
@@ -377,6 +378,7 @@ async-compression = { version = "0.3.13", default-features = false, features = [
 ] }
 async-trait = "0.1.64"
 bitfield = "0.18.0"
+byteorder = "1.5.0"
 bytes = "1.1.0"
 bytes-str = "0.2.7"
 chrono = "0.4.23"

Cargo.toml

@@ -0,0 +1,108 @@
+use std::{iter::once, sync::Arc};
+
+use anyhow::Result;
+use next_api::{
+    analyze::{AnalyzeDataOutputAsset, ModulesDataOutputAsset},
+    project::ProjectContainer,
+};
+use turbo_tasks::{Effects, ReadRef, ResolvedVc, TryJoinIterExt, Vc};
+use turbopack_core::{diagnostics::PlainDiagnostic, issue::PlainIssue, output::OutputAssets};
+
+use crate::next_api::utils::strongly_consistent_catch_collectables;
+
+#[turbo_tasks::value(serialization = "none")]
+pub struct WriteAnalyzeResult {
+    pub issues: Arc<Vec<ReadRef<PlainIssue>>>,
+    pub diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>,
+    pub effects: Arc<Effects>,
+}
+
+#[turbo_tasks::function(operation)]
+pub async fn write_analyze_data_with_issues_operation(
+    project: ResolvedVc<ProjectContainer>,
+    app_dir_only: bool,
+) -> Result<Vc<WriteAnalyzeResult>> {
+    let analyze_data_op = write_analyze_data_with_issues_operation_inner(project, app_dir_only);
+
+    let (_analyze_data, issues, diagnostics, effects) =
+        strongly_consistent_catch_collectables(analyze_data_op).await?;
+
+    Ok(WriteAnalyzeResult {
+        issues,
+        diagnostics,
+        effects,
+    }
+    .cell())
+}
+
+#[turbo_tasks::function(operation)]
+async fn write_analyze_data_with_issues_operation_inner(
+    project: ResolvedVc<ProjectContainer>,
+    app_dir_only: bool,
+) -> Result<()> {
+    let analyze_data_op = get_analyze_data_operation(project, app_dir_only);
+
+    project
+        .project()
+        .emit_all_output_assets(analyze_data_op)
+        .as_side_effect()
+        .await?;
+
+    Ok(())
+}
+
+#[turbo_tasks::function(operation)]
+async fn get_analyze_data_operation(
+    container: ResolvedVc<ProjectContainer>,
+    app_dir_only: bool,
+) -> Result<Vc<OutputAssets>> {
+    let project = container.project();
+    let project =
+        project.with_next_config(project.next_config().with_production_browser_source_maps());
+
+    let analyze_output_root = project
+        .node_root()
+        .owned()
+        .await?
+        .join("diagnostics/analyze")?;
+    let whole_app_module_graphs = project.whole_app_module_graphs();
+    let analyze_output_root = &analyze_output_root;
+    let analyze_data = project
+        .get_all_endpoint_groups(app_dir_only)
+        .await?
+        .iter()
+        .map(|(key, endpoint_group)| async move {
+            let output_assets = endpoint_group.output_assets();
+            let analyze_data = AnalyzeDataOutputAsset::new(
+                analyze_output_root
+                    .join(&key.to_string())?
+                    .join("analyze.data")?,
+                output_assets,
+            )
+            .to_resolved()
+            .await?;
+
+            Ok(ResolvedVc::upcast(analyze_data))
+        })
+        .try_join()
+        .await?;
+
+    whole_app_module_graphs.as_side_effect().await?;
+
+    let modules_data = ResolvedVc::upcast(
+        ModulesDataOutputAsset::new(
+            analyze_output_root.join("modules.data")?,
+            Vc::cell(vec![whole_app_module_graphs.await?.full]),
+        )
+        .to_resolved()
+        .await?,
+    );
+
+    Ok(Vc::cell(
+        analyze_data
+            .iter()
+            .cloned()
+            .chain(once(modules_data))
+            .collect(),
+    ))
+}

crates/napi/src/next_api/analyze.rs

@@ -1,3 +1,4 @@
+pub mod analyze;
 pub mod endpoint;
 pub mod project;
 pub mod turbopack_ctx;

crates/napi/src/next_api/mod.rs

@@ -64,6 +64,7 @@ use url::Url;
 
 use crate::{
     next_api::{
+        analyze::{WriteAnalyzeResult, write_analyze_data_with_issues_operation},
         endpoint::ExternalEndpoint,
         turbopack_ctx::{
             NapiNextTurbopackCallbacks, NapiNextTurbopackCallbacksJsObject, NextTurboTasks,
@@ -915,6 +916,13 @@ fn project_container_entrypoints_operation(
     container.entrypoints()
 }
 
+#[turbo_tasks::value(serialization = "none")]
+struct OperationResult {
+    issues: Arc<Vec<ReadRef<PlainIssue>>>,
+    diagnostics: Arc<Vec<ReadRef<PlainDiagnostic>>>,
+    effects: Arc<Effects>,
+}
+
 #[turbo_tasks::value(serialization = "none")]
 struct AllWrittenEntrypointsWithIssues {
     entrypoints: Option<ReadRef<EntrypointsOperation>>,
@@ -995,9 +1003,10 @@ pub async fn all_entrypoints_write_to_disk_operation(
     project: ResolvedVc<ProjectContainer>,
     app_dir_only: bool,
 ) -> Result<Vc<Entrypoints>> {
+    let output_assets_operation = output_assets_operation(project, app_dir_only);
     project
         .project()
-        .emit_all_output_assets(output_assets_operation(project, app_dir_only))
+        .emit_all_output_assets(output_assets_operation)
         .as_side_effect()
         .await?;
 
@@ -1037,6 +1046,49 @@ async fn output_assets_operation(
 }
 
 #[tracing::instrument(level = "info", name = "get entrypoints", skip_all)]
+#[napi]
+pub async fn project_entrypoints(
+    #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
+) -> napi::Result<TurbopackResult<Option<NapiEntrypoints>>> {
+    let container = project.container;
+
+    let (entrypoints, issues, diags) = project
+        .turbopack_ctx
+        .turbo_tasks()
+        .run_once(async move {
+            let entrypoints_with_issues_op = get_entrypoints_with_issues_operation(container);
+
+            // Read and compile the files
+            let EntrypointsWithIssues {
+                entrypoints,
+                issues,
+                diagnostics,
+                effects: _,
+            } = &*entrypoints_with_issues_op
+                .read_strongly_consistent()
+                .await?;
+
+            Ok((entrypoints.clone(), issues.clone(), diagnostics.clone()))
+        })
+        .await
+        .map_err(|e| napi::Error::from_reason(PrettyPrintError(&e).to_string()))?;
+
+    let result = match entrypoints {
+        Some(entrypoints) => Some(NapiEntrypoints::from_entrypoints_op(
+            &entrypoints,
+            &project.turbopack_ctx,
+        )?),
+        None => None,
+    };
+
+    Ok(TurbopackResult {
+        result,
+        issues: issues.iter().map(|i| NapiIssue::from(&**i)).collect(),
+        diagnostics: diags.iter().map(|d| NapiDiagnostic::from(d)).collect(),
+    })
+}
+
+#[tracing::instrument(level = "info", name = "subscribe to entrypoints", skip_all)]
 #[napi(ts_return_type = "{ __napiType: \"RootTask\" }")]
 pub fn project_entrypoints_subscribe(
     #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
@@ -1718,3 +1770,37 @@ pub fn project_get_source_map_sync(
         tokio::runtime::Handle::current().block_on(project_get_source_map(project, file_path))
     })
 }
+
+#[napi]
+pub async fn project_write_analyze_data(
+    #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
+    app_dir_only: bool,
+) -> napi::Result<TurbopackResult<()>> {
+    let container = project.container;
+    let (issues, diagnostics) = project
+        .turbopack_ctx
+        .turbo_tasks()
+        .run_once(async move {
+            let analyze_data_op = write_analyze_data_with_issues_operation(container, app_dir_only);
+            let WriteAnalyzeResult {
+                issues,
+                diagnostics,
+                effects,
+            } = &*analyze_data_op.read_strongly_consistent().await?;
+
+            // Write the files to disk
+            effects.apply().await?;
+            Ok((issues.clone(), diagnostics.clone()))
+        })
+        .await
+        .map_err(|e| napi::Error::from_reason(PrettyPrintError(&e).to_string()))?;
+
+    Ok(TurbopackResult {
+        result: (),
+        issues: issues.iter().map(|i| NapiIssue::from(&**i)).collect(),
+        diagnostics: diagnostics
+            .iter()
+            .map(|d| NapiDiagnostic::from(d))
+            .collect(),
+    })
+}

crates/napi/src/next_api/project.rs

@@ -14,6 +14,7 @@ workspace = true
 
 [dependencies]
 anyhow = { workspace = true, features = ["backtrace"] }
+byteorder = { workspace = true }
 either = { workspace = true }
 futures = { workspace = true }
 indexmap = { workspace = true }
@@ -31,6 +32,7 @@ turbo-tasks-env = { workspace = true }
 turbo-tasks-fs = { workspace = true }
 turbo-unix-path = { workspace = true }
 turbopack = { workspace = true }
+turbopack-analyze = { workspace = true }
 turbopack-browser = { workspace = true }
 turbopack-core = { workspace = true }
 turbopack-ecmascript = { workspace = true }