diff --git a/src/bootstrap/src/core/builder/cargo.rs b/src/bootstrap/src/core/builder/cargo.rs index a878291a33c0d..deb7106f185c4 100644 --- a/src/bootstrap/src/core/builder/cargo.rs +++ b/src/bootstrap/src/core/builder/cargo.rs @@ -131,7 +131,10 @@ impl Cargo { } pub fn into_cmd(self) -> BootstrapCommand { - self.into() + let mut cmd: BootstrapCommand = self.into(); + // Disable caching for commands originating from Cargo-related operations. + cmd.do_not_cache(); + cmd } /// Same as [`Cargo::new`] except this one doesn't configure the linker with diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index 7cb7866953a8e..8e9e8b496de7c 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -22,8 +22,7 @@ use crate::core::build_steps::{ use crate::core::config::flags::Subcommand; use crate::core::config::{DryRun, TargetSelection}; use crate::utils::cache::Cache; -use crate::utils::exec::{BootstrapCommand, command}; -use crate::utils::execution_context::ExecutionContext; +use crate::utils::exec::{BootstrapCommand, ExecutionContext, command}; use crate::utils::helpers::{self, LldThreads, add_dylib_path, exe, libdir, linker_args, t}; use crate::{Build, Crate, trace}; diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index b2754a5ad32c2..d1ffdf24acd0e 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -47,8 +47,7 @@ use crate::core::config::{ }; use crate::core::download::is_download_ci_available; use crate::utils::channel; -use crate::utils::exec::command; -use crate::utils::execution_context::ExecutionContext; +use crate::utils::exec::{ExecutionContext, command}; use crate::utils::helpers::{exe, get_host_target}; use crate::{GitInfo, OnceLock, TargetSelection, check_ci_llvm, helpers, t}; diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index 26fac6b3b603f..ef5c28272b8e1 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -31,12 +31,12 @@ use cc::Tool; use termcolor::{ColorChoice, StandardStream, WriteColor}; use utils::build_stamp::BuildStamp; use utils::channel::GitInfo; -use utils::execution_context::ExecutionContext; +use utils::exec::ExecutionContext; use crate::core::builder; use crate::core::builder::Kind; use crate::core::config::{DryRun, LldMode, LlvmLibunwind, TargetSelection, flags}; -use crate::utils::exec::{BehaviorOnFailure, BootstrapCommand, CommandOutput, OutputMode, command}; +use crate::utils::exec::{BootstrapCommand, command}; use crate::utils::helpers::{ self, dir_is_empty, exe, libdir, set_file_times, split_debuginfo, symlink_dir, }; diff --git a/src/bootstrap/src/utils/channel.rs b/src/bootstrap/src/utils/channel.rs index 16aa9ba0585b2..21b4257e54d0b 100644 --- a/src/bootstrap/src/utils/channel.rs +++ b/src/bootstrap/src/utils/channel.rs @@ -8,7 +8,7 @@ use std::fs; use std::path::Path; -use super::execution_context::ExecutionContext; +use super::exec::ExecutionContext; use super::helpers; use crate::Build; use crate::utils::helpers::t; diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs index a7b92441d747e..d092765ef762f 100644 --- a/src/bootstrap/src/utils/exec.rs +++ b/src/bootstrap/src/utils/exec.rs @@ -1,16 +1,29 @@ //! Command Execution Module //! -//! This module provides a structured way to execute and manage commands efficiently, -//! ensuring controlled failure handling and output management. -use std::ffi::OsStr; +//! Provides a structured interface for executing and managing commands during bootstrap, +//! with support for controlled failure handling and output management. +//! +//! This module defines the [`ExecutionContext`] type, which encapsulates global configuration +//! relevant to command execution in the bootstrap process. This includes settings such as +//! dry-run mode, verbosity level, and failure behavior. + +use std::collections::HashMap; +use std::ffi::{OsStr, OsString}; use std::fmt::{Debug, Formatter}; +use std::hash::Hash; +use std::panic::Location; use std::path::Path; -use std::process::{Command, CommandArgs, CommandEnvs, ExitStatus, Output, Stdio}; +use std::process::{Child, Command, CommandArgs, CommandEnvs, ExitStatus, Output, Stdio}; +use std::sync::{Arc, Mutex}; use build_helper::ci::CiEnv; use build_helper::drop_bomb::DropBomb; +use build_helper::exit; -use super::execution_context::{DeferredCommand, ExecutionContext}; +use crate::PathBuf; +use crate::core::config::DryRun; +#[cfg(feature = "tracing")] +use crate::trace_cmd; /// What should be done when the command fails. #[derive(Debug, Copy, Clone)] @@ -49,6 +62,14 @@ impl OutputMode { } } +#[derive(Clone, Debug, PartialEq, Eq, Hash, Default)] +pub struct CommandCacheKey { + program: OsString, + args: Vec, + envs: Vec<(OsString, Option)>, + cwd: Option, +} + /// Wrapper around `std::process::Command`. /// /// By default, the command will exit bootstrap if it fails. @@ -60,6 +81,9 @@ impl OutputMode { /// /// Bootstrap will print a debug log to stdout if the command fails and failure is not allowed. /// +/// By default, command executions are cached based on their workdir, program, arguments, and environment variables. +/// This avoids re-running identical commands unnecessarily, unless caching is explicitly disabled. +/// /// [allow_failure]: BootstrapCommand::allow_failure /// [delay_failure]: BootstrapCommand::delay_failure pub struct BootstrapCommand { @@ -70,6 +94,7 @@ pub struct BootstrapCommand { // This field makes sure that each command is executed (or disarmed) before it is dropped, // to avoid forgetting to execute a command. drop_bomb: DropBomb, + should_cache: bool, } impl<'a> BootstrapCommand { @@ -77,12 +102,16 @@ impl<'a> BootstrapCommand { pub fn new>(program: S) -> Self { Command::new(program).into() } - pub fn arg>(&mut self, arg: S) -> &mut Self { self.command.arg(arg.as_ref()); self } + pub fn do_not_cache(&mut self) -> &mut Self { + self.should_cache = false; + self + } + pub fn args(&mut self, args: I) -> &mut Self where I: IntoIterator, @@ -183,9 +212,11 @@ impl<'a> BootstrapCommand { /// Provides access to the stdlib Command inside. /// FIXME: This function should be eventually removed from bootstrap. pub fn as_command_mut(&mut self) -> &mut Command { - // We don't know what will happen with the returned command, so we need to mark this - // command as executed proactively. + // We proactively mark this command as executed since we can't be certain how the returned + // command will be handled. Caching must also be avoided here, as the inner command could be + // modified externally without us being aware. self.mark_as_executed(); + self.do_not_cache(); &mut self.command } @@ -211,6 +242,22 @@ impl<'a> BootstrapCommand { self.env("TERM", "xterm").args(["--color", "always"]); } } + + pub fn cache_key(&self) -> Option { + if !self.should_cache { + return None; + } + let command = &self.command; + Some(CommandCacheKey { + program: command.get_program().into(), + args: command.get_args().map(OsStr::to_os_string).collect(), + envs: command + .get_envs() + .map(|(k, v)| (k.to_os_string(), v.map(|val| val.to_os_string()))) + .collect(), + cwd: command.get_current_dir().map(Path::to_path_buf), + }) + } } impl Debug for BootstrapCommand { @@ -224,8 +271,8 @@ impl From for BootstrapCommand { #[track_caller] fn from(command: Command) -> Self { let program = command.get_program().to_owned(); - Self { + should_cache: true, command, failure_behavior: BehaviorOnFailure::Exit, run_in_dry_run: false, @@ -235,6 +282,7 @@ impl From for BootstrapCommand { } /// Represents the current status of `BootstrapCommand`. +#[derive(Clone, PartialEq)] enum CommandStatus { /// The command has started and finished with some status. Finished(ExitStatus), @@ -251,6 +299,7 @@ pub fn command>(program: S) -> BootstrapCommand { } /// Represents the output of an executed process. +#[derive(Clone, PartialEq)] pub struct CommandOutput { status: CommandStatus, stdout: Option>, @@ -373,3 +422,329 @@ impl FormatShortCmd for Command { line.join(" ") } } + +#[derive(Clone, Default)] +pub struct ExecutionContext { + dry_run: DryRun, + verbose: u8, + pub fail_fast: bool, + delayed_failures: Arc>>, + command_cache: Arc, +} + +#[derive(Default)] +pub struct CommandCache { + cache: Mutex>, +} + +enum CommandState<'a> { + Cached(CommandOutput), + Deferred { + process: Option>, + command: &'a mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + executed_at: &'a Location<'a>, + cache_key: Option, + }, +} + +#[must_use] +pub struct DeferredCommand<'a> { + state: CommandState<'a>, +} + +impl CommandCache { + pub fn get(&self, key: &CommandCacheKey) -> Option { + self.cache.lock().unwrap().get(key).cloned() + } + + pub fn insert(&self, key: CommandCacheKey, output: CommandOutput) { + self.cache.lock().unwrap().insert(key, output); + } +} + +impl ExecutionContext { + pub fn new() -> Self { + ExecutionContext::default() + } + + pub fn dry_run(&self) -> bool { + match self.dry_run { + DryRun::Disabled => false, + DryRun::SelfCheck | DryRun::UserSelected => true, + } + } + + pub fn get_dry_run(&self) -> &DryRun { + &self.dry_run + } + + pub fn verbose(&self, f: impl Fn()) { + if self.is_verbose() { + f() + } + } + + pub fn is_verbose(&self) -> bool { + self.verbose > 0 + } + + pub fn fail_fast(&self) -> bool { + self.fail_fast + } + + pub fn set_dry_run(&mut self, value: DryRun) { + self.dry_run = value; + } + + pub fn set_verbose(&mut self, value: u8) { + self.verbose = value; + } + + pub fn set_fail_fast(&mut self, value: bool) { + self.fail_fast = value; + } + + pub fn add_to_delay_failure(&self, message: String) { + self.delayed_failures.lock().unwrap().push(message); + } + + pub fn report_failures_and_exit(&self) { + let failures = self.delayed_failures.lock().unwrap(); + if failures.is_empty() { + return; + } + eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); + for failure in &*failures { + eprintln!(" - {failure}"); + } + exit!(1); + } + + /// Execute a command and return its output. + /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to + /// execute commands. They internally call this method. + #[track_caller] + pub fn start<'a>( + &self, + command: &'a mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + ) -> DeferredCommand<'a> { + let cache_key = command.cache_key(); + + if let Some(cached_output) = cache_key.as_ref().and_then(|key| self.command_cache.get(key)) + { + command.mark_as_executed(); + self.verbose(|| println!("Cache hit: {command:?}")); + return DeferredCommand { state: CommandState::Cached(cached_output) }; + } + + let created_at = command.get_created_location(); + let executed_at = std::panic::Location::caller(); + + if self.dry_run() && !command.run_in_dry_run { + return DeferredCommand { + state: CommandState::Deferred { + process: None, + command, + stdout, + stderr, + executed_at, + cache_key, + }, + }; + } + + #[cfg(feature = "tracing")] + let _run_span = trace_cmd!(command); + + self.verbose(|| { + println!("running: {command:?} (created at {created_at}, executed at {executed_at})") + }); + + let cmd = &mut command.command; + cmd.stdout(stdout.stdio()); + cmd.stderr(stderr.stdio()); + + let child = cmd.spawn(); + + DeferredCommand { + state: CommandState::Deferred { + process: Some(child), + command, + stdout, + stderr, + executed_at, + cache_key, + }, + } + } + + /// Execute a command and return its output. + /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to + /// execute commands. They internally call this method. + #[track_caller] + pub fn run( + &self, + command: &mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + ) -> CommandOutput { + self.start(command, stdout, stderr).wait_for_output(self) + } + + fn fail(&self, message: &str, output: CommandOutput) -> ! { + if self.is_verbose() { + println!("{message}"); + } else { + let (stdout, stderr) = (output.stdout_if_present(), output.stderr_if_present()); + // If the command captures output, the user would not see any indication that + // it has failed. In this case, print a more verbose error, since to provide more + // context. + if stdout.is_some() || stderr.is_some() { + if let Some(stdout) = output.stdout_if_present().take_if(|s| !s.trim().is_empty()) { + println!("STDOUT:\n{stdout}\n"); + } + if let Some(stderr) = output.stderr_if_present().take_if(|s| !s.trim().is_empty()) { + println!("STDERR:\n{stderr}\n"); + } + println!("Command has failed. Rerun with -v to see more details."); + } else { + println!("Command has failed. Rerun with -v to see more details."); + } + } + exit!(1); + } +} + +impl AsRef for ExecutionContext { + fn as_ref(&self) -> &ExecutionContext { + self + } +} + +impl<'a> DeferredCommand<'a> { + pub fn wait_for_output(self, exec_ctx: impl AsRef) -> CommandOutput { + match self.state { + CommandState::Cached(output) => output, + CommandState::Deferred { process, command, stdout, stderr, executed_at, cache_key } => { + let exec_ctx = exec_ctx.as_ref(); + + let output = + Self::finish_process(process, command, stdout, stderr, executed_at, exec_ctx); + + if (!exec_ctx.dry_run() || command.run_in_dry_run) + && let (Some(cache_key), Some(_)) = (&cache_key, output.status()) + { + exec_ctx.command_cache.insert(cache_key.clone(), output.clone()); + } + + output + } + } + } + + pub fn finish_process( + mut process: Option>, + command: &mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + executed_at: &'a std::panic::Location<'a>, + exec_ctx: &ExecutionContext, + ) -> CommandOutput { + command.mark_as_executed(); + + let process = match process.take() { + Some(p) => p, + None => return CommandOutput::default(), + }; + + let created_at = command.get_created_location(); + + let mut message = String::new(); + + let output = match process { + Ok(child) => match child.wait_with_output() { + Ok(result) if result.status.success() => { + // Successful execution + CommandOutput::from_output(result, stdout, stderr) + } + Ok(result) => { + // Command ran but failed + use std::fmt::Write; + + writeln!( + message, + r#" +Command {command:?} did not execute successfully. +Expected success, got {} +Created at: {created_at} +Executed at: {executed_at}"#, + result.status, + ) + .unwrap(); + + let output = CommandOutput::from_output(result, stdout, stderr); + + if stdout.captures() { + writeln!(message, "\nSTDOUT ----\n{}", output.stdout().trim()).unwrap(); + } + if stderr.captures() { + writeln!(message, "\nSTDERR ----\n{}", output.stderr().trim()).unwrap(); + } + + output + } + Err(e) => { + // Failed to wait for output + use std::fmt::Write; + + writeln!( + message, + "\n\nCommand {command:?} did not execute successfully.\ + \nIt was not possible to execute the command: {e:?}" + ) + .unwrap(); + + CommandOutput::did_not_start(stdout, stderr) + } + }, + Err(e) => { + // Failed to spawn the command + use std::fmt::Write; + + writeln!( + message, + "\n\nCommand {command:?} did not execute successfully.\ + \nIt was not possible to execute the command: {e:?}" + ) + .unwrap(); + + CommandOutput::did_not_start(stdout, stderr) + } + }; + + if !output.is_success() { + match command.failure_behavior { + BehaviorOnFailure::DelayFail => { + if exec_ctx.fail_fast { + exec_ctx.fail(&message, output); + } + exec_ctx.add_to_delay_failure(message); + } + BehaviorOnFailure::Exit => { + exec_ctx.fail(&message, output); + } + BehaviorOnFailure::Ignore => { + // If failures are allowed, either the error has been printed already + // (OutputMode::Print) or the user used a capture output mode and wants to + // handle the error output on their own. + } + } + } + + output + } +} diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs deleted file mode 100644 index 66a4be9252e4e..0000000000000 --- a/src/bootstrap/src/utils/execution_context.rs +++ /dev/null @@ -1,265 +0,0 @@ -//! Shared execution context for running bootstrap commands. -//! -//! This module provides the [`ExecutionContext`] type, which holds global configuration -//! relevant during the execution of commands in bootstrap. This includes dry-run -//! mode, verbosity level, and behavior on failure. -use std::panic::Location; -use std::process::Child; -use std::sync::{Arc, Mutex}; - -use crate::core::config::DryRun; -#[cfg(feature = "tracing")] -use crate::trace_cmd; -use crate::{BehaviorOnFailure, BootstrapCommand, CommandOutput, OutputMode, exit}; - -#[derive(Clone, Default)] -pub struct ExecutionContext { - dry_run: DryRun, - verbose: u8, - pub fail_fast: bool, - delayed_failures: Arc>>, -} - -impl ExecutionContext { - pub fn new() -> Self { - ExecutionContext::default() - } - - pub fn dry_run(&self) -> bool { - match self.dry_run { - DryRun::Disabled => false, - DryRun::SelfCheck | DryRun::UserSelected => true, - } - } - - pub fn get_dry_run(&self) -> &DryRun { - &self.dry_run - } - - pub fn verbose(&self, f: impl Fn()) { - if self.is_verbose() { - f() - } - } - - pub fn is_verbose(&self) -> bool { - self.verbose > 0 - } - - pub fn fail_fast(&self) -> bool { - self.fail_fast - } - - pub fn set_dry_run(&mut self, value: DryRun) { - self.dry_run = value; - } - - pub fn set_verbose(&mut self, value: u8) { - self.verbose = value; - } - - pub fn set_fail_fast(&mut self, value: bool) { - self.fail_fast = value; - } - - pub fn add_to_delay_failure(&self, message: String) { - self.delayed_failures.lock().unwrap().push(message); - } - - pub fn report_failures_and_exit(&self) { - let failures = self.delayed_failures.lock().unwrap(); - if failures.is_empty() { - return; - } - eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); - for failure in &*failures { - eprintln!(" - {failure}"); - } - exit!(1); - } - - /// Execute a command and return its output. - /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to - /// execute commands. They internally call this method. - #[track_caller] - pub fn start<'a>( - &self, - command: &'a mut BootstrapCommand, - stdout: OutputMode, - stderr: OutputMode, - ) -> DeferredCommand<'a> { - command.mark_as_executed(); - - let created_at = command.get_created_location(); - let executed_at = std::panic::Location::caller(); - - if self.dry_run() && !command.run_in_dry_run { - return DeferredCommand { process: None, stdout, stderr, command, executed_at }; - } - - #[cfg(feature = "tracing")] - let _run_span = trace_cmd!(command); - - self.verbose(|| { - println!("running: {command:?} (created at {created_at}, executed at {executed_at})") - }); - - let cmd = command.as_command_mut(); - cmd.stdout(stdout.stdio()); - cmd.stderr(stderr.stdio()); - - let child = cmd.spawn(); - - DeferredCommand { process: Some(child), stdout, stderr, command, executed_at } - } - - /// Execute a command and return its output. - /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to - /// execute commands. They internally call this method. - #[track_caller] - pub fn run( - &self, - command: &mut BootstrapCommand, - stdout: OutputMode, - stderr: OutputMode, - ) -> CommandOutput { - self.start(command, stdout, stderr).wait_for_output(self) - } - - fn fail(&self, message: &str, output: CommandOutput) -> ! { - if self.is_verbose() { - println!("{message}"); - } else { - let (stdout, stderr) = (output.stdout_if_present(), output.stderr_if_present()); - // If the command captures output, the user would not see any indication that - // it has failed. In this case, print a more verbose error, since to provide more - // context. - if stdout.is_some() || stderr.is_some() { - if let Some(stdout) = output.stdout_if_present().take_if(|s| !s.trim().is_empty()) { - println!("STDOUT:\n{stdout}\n"); - } - if let Some(stderr) = output.stderr_if_present().take_if(|s| !s.trim().is_empty()) { - println!("STDERR:\n{stderr}\n"); - } - println!("Command has failed. Rerun with -v to see more details."); - } else { - println!("Command has failed. Rerun with -v to see more details."); - } - } - exit!(1); - } -} - -impl AsRef for ExecutionContext { - fn as_ref(&self) -> &ExecutionContext { - self - } -} - -pub struct DeferredCommand<'a> { - process: Option>, - command: &'a mut BootstrapCommand, - stdout: OutputMode, - stderr: OutputMode, - executed_at: &'a Location<'a>, -} - -impl<'a> DeferredCommand<'a> { - pub fn wait_for_output(mut self, exec_ctx: impl AsRef) -> CommandOutput { - let exec_ctx = exec_ctx.as_ref(); - - let process = match self.process.take() { - Some(p) => p, - None => return CommandOutput::default(), - }; - - let created_at = self.command.get_created_location(); - let executed_at = self.executed_at; - - let mut message = String::new(); - - let output = match process { - Ok(child) => match child.wait_with_output() { - Ok(result) if result.status.success() => { - // Successful execution - CommandOutput::from_output(result, self.stdout, self.stderr) - } - Ok(result) => { - // Command ran but failed - use std::fmt::Write; - - writeln!( - message, - r#" -Command {:?} did not execute successfully. -Expected success, got {} -Created at: {created_at} -Executed at: {executed_at}"#, - self.command, result.status, - ) - .unwrap(); - - let output = CommandOutput::from_output(result, self.stdout, self.stderr); - - if self.stdout.captures() { - writeln!(message, "\nSTDOUT ----\n{}", output.stdout().trim()).unwrap(); - } - if self.stderr.captures() { - writeln!(message, "\nSTDERR ----\n{}", output.stderr().trim()).unwrap(); - } - - output - } - Err(e) => { - // Failed to wait for output - use std::fmt::Write; - - writeln!( - message, - "\n\nCommand {:?} did not execute successfully.\ - \nIt was not possible to execute the command: {e:?}", - self.command - ) - .unwrap(); - - CommandOutput::did_not_start(self.stdout, self.stderr) - } - }, - Err(e) => { - // Failed to spawn the command - use std::fmt::Write; - - writeln!( - message, - "\n\nCommand {:?} did not execute successfully.\ - \nIt was not possible to execute the command: {e:?}", - self.command - ) - .unwrap(); - - CommandOutput::did_not_start(self.stdout, self.stderr) - } - }; - - if !output.is_success() { - match self.command.failure_behavior { - BehaviorOnFailure::DelayFail => { - if exec_ctx.fail_fast { - exec_ctx.fail(&message, output); - } - exec_ctx.add_to_delay_failure(message); - } - BehaviorOnFailure::Exit => { - exec_ctx.fail(&message, output); - } - BehaviorOnFailure::Ignore => { - // If failures are allowed, either the error has been printed already - // (OutputMode::Print) or the user used a capture output mode and wants to - // handle the error output on their own. - } - } - } - - output - } -} diff --git a/src/bootstrap/src/utils/mod.rs b/src/bootstrap/src/utils/mod.rs index 5a0b90801e73a..169fcec303e90 100644 --- a/src/bootstrap/src/utils/mod.rs +++ b/src/bootstrap/src/utils/mod.rs @@ -8,7 +8,6 @@ pub(crate) mod cc_detect; pub(crate) mod change_tracker; pub(crate) mod channel; pub(crate) mod exec; -pub(crate) mod execution_context; pub(crate) mod helpers; pub(crate) mod job; pub(crate) mod render_tests;