Auto merge of rust-lang#139244 - jieyouxu:exp/auto-cross-run-make, r=<try>

[WIP] [WIP] Enable automatic cross-compilation in run-make tests

WIP for the WIP rust-lang#138066.

Based on rust-lang#138066 with rust-lang#139242 + rust-lang#139239 cherry-picked in, plus `rustdoc()` cross-compile changes.

r? `@ghost`

try-job: armhf-gnu
try-job: test-various
try-job: x86_64-msvc-1
try-job: i686-msvc-1
try-job: x86_64-mingw-1
try-job: aarch64-apple
try-job: x86_64-apple-1

Author: bors

src/tools/run-make-support/src/external_deps/rustc.rs

@@ -6,7 +6,7 @@ use crate::command::Command;
 use crate::env::env_var;
 use crate::path_helpers::cwd;
 use crate::util::set_host_compiler_dylib_path;
-use crate::{is_aix, is_darwin, is_msvc, is_windows, uname};
+use crate::{is_aix, is_darwin, is_msvc, is_windows, target, uname};
 
 /// Construct a new `rustc` invocation. This will automatically set the library
 /// search path as `-L cwd()`. Use [`bare_rustc`] to avoid this.
@@ -27,9 +27,15 @@ pub fn bare_rustc() -> Rustc {
 #[must_use]
 pub struct Rustc {
     cmd: Command,
+    target: Option<String>,
 }
 
-crate::macros::impl_common_helpers!(Rustc);
+// Only fill in the target just before execution, so that it can be overridden.
+crate::macros::impl_common_helpers!(Rustc, |rustc: &mut Rustc| {
+    if let Some(target) = &rustc.target {
+        rustc.cmd.arg(&format!("--target={target}"));
+    }
+});
 
 pub fn rustc_path() -> String {
     env_var("RUSTC")
@@ -46,19 +52,22 @@ impl Rustc {
     // `rustc` invocation constructor methods
 
     /// Construct a new `rustc` invocation. This will automatically set the library
-    /// search path as `-L cwd()`. Use [`bare_rustc`] to avoid this.
+    /// search path as `-L cwd()` and also the compilation target.
+    /// Use [`bare_rustc`] to avoid this.
     #[track_caller]
     pub fn new() -> Self {
         let mut cmd = setup_common();
         cmd.arg("-L").arg(cwd());
-        Self { cmd }
+
+        // Automatically default to cross-compilation
+        Self { cmd, target: Some(target()) }
     }
 
     /// Construct a bare `rustc` invocation with no flags set.
     #[track_caller]
     pub fn bare() -> Self {
         let cmd = setup_common();
-        Self { cmd }
+        Self { cmd, target: None }
     }
 
     // Argument provider methods
@@ -234,8 +243,9 @@ impl Rustc {
 
     /// Specify the target triple, or a path to a custom target json spec file.
     pub fn target<S: AsRef<str>>(&mut self, target: S) -> &mut Self {
-        let target = target.as_ref();
-        self.cmd.arg(format!("--target={target}"));
+        // We store the target as a separate field, so that it can be specified multiple times.
+        // This is in particular useful to override the default target set in Rustc::new().
+        self.target = Some(target.as_ref().to_string());
         self
     }
 

src/tools/run-make-support/src/external_deps/rustdoc.rs

@@ -3,21 +3,35 @@ use std::path::Path;
 
 use crate::command::Command;
 use crate::env::env_var;
+use crate::target;
 use crate::util::set_host_compiler_dylib_path;
 
-/// Construct a new `rustdoc` invocation.
+/// Construct a new `rustdoc` invocation with target automatically set to cross-compile target. Use
+/// [`bare_rustdoc`] to avoid this.
 #[track_caller]
 pub fn rustdoc() -> Rustdoc {
     Rustdoc::new()
 }
 
+/// Bare `rustdoc` invocation, no args set.
+#[track_caller]
+pub fn bare_rustdoc() -> Rustdoc {
+    Rustdoc::bare()
+}
+
 #[derive(Debug)]
 #[must_use]
 pub struct Rustdoc {
     cmd: Command,
+    target: Option<String>,
 }
 
-crate::macros::impl_common_helpers!(Rustdoc);
+// Only fill in the target just before execution, so that it can be overridden.
+crate::macros::impl_common_helpers!(Rustdoc, |rustdoc: &mut Rustdoc| {
+    if let Some(target) = &rustdoc.target {
+        rustdoc.cmd.arg(&format!("--target={target}"));
+    }
+});
 
 #[track_caller]
 fn setup_common() -> Command {
@@ -28,11 +42,19 @@ fn setup_common() -> Command {
 }
 
 impl Rustdoc {
-    /// Construct a bare `rustdoc` invocation.
+    /// Construct a new `rustdoc` invocation with target automatically set to cross-compile target.
+    /// Use [`bare_rustdoc`] to avoid this.
     #[track_caller]
     pub fn new() -> Self {
         let cmd = setup_common();
-        Self { cmd }
+        Self { cmd, target: Some(target()) }
+    }
+
+    /// Bare `rustdoc` invocation, no args set.
+    #[track_caller]
+    pub fn bare() -> Self {
+        let cmd = setup_common();
+        Self { cmd, target: None }
     }
 
     /// Specify where an external library is located.
@@ -85,8 +107,9 @@ impl Rustdoc {
 
     /// Specify the target triple, or a path to a custom target json spec file.
     pub fn target<S: AsRef<str>>(&mut self, target: S) -> &mut Self {
-        let target = target.as_ref();
-        self.cmd.arg(format!("--target={target}"));
+        // We store the target as a separate field, so that it can be specified multiple times.
+        // This is in particular useful to override the default target set in `Rustdoc::new()`.
+        self.target = Some(target.as_ref().to_string());
         self
     }
 

src/tools/run-make-support/src/lib.rs

@@ -68,7 +68,7 @@ pub use llvm::{
 };
 pub use python::python_command;
 pub use rustc::{bare_rustc, rustc, rustc_path, Rustc};
-pub use rustdoc::{rustdoc, Rustdoc};
+pub use rustdoc::{bare_rustdoc, rustdoc, Rustdoc};
 
 /// [`diff`][mod@diff] is implemented in terms of the [similar] library.
 ///

src/tools/run-make-support/src/macros.rs

@@ -23,10 +23,16 @@
 /// }
 /// ```
 ///
+/// You can pass an optional second parameter which should be a function that is passed
+/// `&mut self` just before the command is executed.
+///
 /// [`Command`]: crate::command::Command
 /// [`CompletedProcess`]: crate::command::CompletedProcess
 macro_rules! impl_common_helpers {
     ($wrapper: ident) => {
+        $crate::macros::impl_common_helpers!($wrapper, |_| {});
+    };
+    ($wrapper: ident, $before_exec: expr) => {
         impl $wrapper {
             /// Specify an environment variable.
             pub fn env<K, V>(&mut self, key: K, value: V) -> &mut Self
@@ -118,12 +124,14 @@ macro_rules! impl_common_helpers {
             /// Run the constructed command and assert that it is successfully run.
             #[track_caller]
             pub fn run(&mut self) -> crate::command::CompletedProcess {
+                $before_exec(&mut *self);
                 self.cmd.run()
             }
 
             /// Run the constructed command and assert that it does not successfully run.
             #[track_caller]
             pub fn run_fail(&mut self) -> crate::command::CompletedProcess {
+                $before_exec(&mut *self);
                 self.cmd.run_fail()
             }
 
@@ -133,6 +141,7 @@ macro_rules! impl_common_helpers {
             /// whenever possible.
             #[track_caller]
             pub fn run_unchecked(&mut self) -> crate::command::CompletedProcess {
+                $before_exec(&mut *self);
                 self.cmd.run_unchecked()
             }
 

tests/run-make/apple-deployment-target/rmake.rs

@@ -41,7 +41,6 @@ fn main() {
 
     // Remove env vars to get `rustc`'s default
     let output = rustc()
-        .target(target())
         .env_remove("MACOSX_DEPLOYMENT_TARGET")
         .env_remove("IPHONEOS_DEPLOYMENT_TARGET")
         .env_remove("WATCHOS_DEPLOYMENT_TARGET")
@@ -58,7 +57,6 @@ fn main() {
     run_in_tmpdir(|| {
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.crate_type("lib");
             rustc.emit("obj");
             rustc.input("foo.rs");
@@ -82,7 +80,6 @@ fn main() {
 
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.crate_type("dylib");
             rustc.input("foo.rs");
             rustc.output("libfoo.dylib");
@@ -108,7 +105,6 @@ fn main() {
     run_in_tmpdir(|| {
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.crate_type("bin");
             rustc.input("foo.rs");
             rustc.output("foo");
@@ -147,7 +143,6 @@ fn main() {
     run_in_tmpdir(|| {
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.incremental("incremental");
             rustc.crate_type("lib");
             rustc.emit("obj");

tests/run-make/apple-sdk-version/rmake.rs

@@ -24,8 +24,7 @@ fn has_sdk_version(file: &str, version: &str) {
 
 fn main() {
     // Fetch rustc's inferred deployment target.
-    let current_deployment_target =
-        rustc().target(target()).print("deployment-target").run().stdout_utf8();
+    let current_deployment_target = rustc().print("deployment-target").run().stdout_utf8();
     let current_deployment_target = current_deployment_target.split('=').last().unwrap().trim();
 
     // Fetch current SDK version via. xcrun.
@@ -45,15 +44,15 @@ fn main() {
     let current_sdk_version = current_sdk_version.trim();
 
     // Check the SDK version in the object file produced by the codegen backend.
-    rustc().target(target()).crate_type("lib").emit("obj").input("foo.rs").output("foo.o").run();
+    rustc().crate_type("lib").emit("obj").input("foo.rs").output("foo.o").run();
     // Set to 0, which means not set or "n/a".
     has_sdk_version("foo.o", "n/a");
 
     // Check the SDK version in the .rmeta file, as set in `create_object_file`.
     //
     // This is just to ensure that we don't set some odd version in `create_object_file`,
     // if the rmeta file is packed in a different way in the future, this can safely be removed.
-    rustc().target(target()).crate_type("rlib").input("foo.rs").output("libfoo.rlib").run();
+    rustc().crate_type("rlib").input("foo.rs").output("libfoo.rlib").run();
     // Extra .rmeta file (which is encoded as an object file).
     cmd("ar").arg("-x").arg("libfoo.rlib").arg("lib.rmeta").run();
     has_sdk_version("lib.rmeta", "n/a");
@@ -69,7 +68,6 @@ fn main() {
         // Test with clang
         let file_name = format!("foo_cc{file_ext}");
         rustc()
-            .target(target())
             .crate_type("bin")
             .arg("-Clinker-flavor=gcc")
             .input("foo.rs")
@@ -80,7 +78,6 @@ fn main() {
         // Test with ld64
         let file_name = format!("foo_ld{file_ext}");
         rustc()
-            .target(target())
             .crate_type("bin")
             .arg("-Clinker-flavor=ld")
             .input("foo.rs")

tests/run-make/doctests-merge/rmake.rs

@@ -1,3 +1,5 @@
+//@ ignore-cross-compile (needs to run doctests)
+
 use std::path::Path;
 
 use run_make_support::{cwd, diff, rustc, rustdoc};

tests/run-make/doctests-runtool/rmake.rs

@@ -1,5 +1,7 @@
 // Tests behavior of rustdoc `--runtool`.
 
+//@ ignore-cross-compile (attempts to run the doctests)
+
 use std::path::PathBuf;
 
 use run_make_support::rfs::{create_dir, remove_dir_all};

tests/run-make/embed-metadata/rmake.rs

@@ -1,8 +1,13 @@
 // Tests the -Zembed-metadata compiler flag.
 // Tracking issue: https://github.com/rust-lang/rust/issues/139165
 
+//@ needs-dynamic-linking
+//@ needs-crate-type: dylib
+
 use run_make_support::rfs::{create_dir, remove_file, rename};
-use run_make_support::{Rustc, dynamic_lib_name, path, run_in_tmpdir, rust_lib_name, rustc};
+use run_make_support::{
+    Rustc, dynamic_lib_name, path, run_in_tmpdir, rust_lib_name, rustc, target,
+};
 
 #[derive(Debug, Copy, Clone)]
 enum LibraryKind {
@@ -42,7 +47,7 @@ fn main() {
 fn lookup_rmeta_in_lib_dir(kind: LibraryKind) {
     run_in_tmpdir(|| {
         build_dep_rustc(kind).run();
-        rustc().input("foo.rs").run();
+        rustc().target(target()).input("foo.rs").run();
     });
 }
 
@@ -54,6 +59,7 @@ fn lookup_rmeta_through_extern(kind: LibraryKind) {
         build_dep_rustc(kind).out_dir("deps").run();
 
         let mut rustc = rustc();
+        rustc.target(target());
         kind.add_extern(&mut rustc, "dep1", "deps");
         rustc.extern_("dep1", path("deps").join("libdep1.rmeta"));
         rustc.input("foo.rs").run();
@@ -67,13 +73,15 @@ fn lookup_rmeta_missing(kind: LibraryKind) {
         build_dep_rustc(kind).out_dir("deps").run();
 
         let mut rustc = rustc();
+        rustc.target(target());
         kind.add_extern(&mut rustc, "dep1", "deps");
         rustc.input("foo.rs").run_fail().assert_stderr_contains("only metadata stub found");
     });
 }
 
 fn build_dep_rustc(kind: LibraryKind) -> Rustc {
     let mut dep_rustc = rustc();
+    dep_rustc.target(target());
     dep_rustc
         .arg("-Zembed-metadata=no")
         .crate_type(kind.crate_type())

tests/run-make/embed-source-dwarf/rmake.rs

@@ -1,5 +1,6 @@
-//@ ignore-windows
-//@ ignore-apple
+//@ ignore-apple (needs DWARF)
+//@ ignore-windows-msvc (needs DWARF)
+//@ ignore-cross-compile (cross-compiled target can produce artifacts `object` can't yet parse)
 
 // This test should be replaced with one in tests/debuginfo once we can easily
 // tell via GDB or LLDB if debuginfo contains source code. Cheap tricks in LLDB

tests/run-make/emit-stack-sizes/rmake.rs

@@ -6,15 +6,18 @@
 // this diagnostics information should be located.
 // See https://github.com/rust-lang/rust/pull/51946
 
-//@ ignore-windows
-//@ ignore-apple
-// Reason: this feature only works when the output object format is ELF.
-// This won't be the case on Windows/OSX - for example, OSX produces a Mach-O binary.
+//@ only-elf
 
-use run_make_support::{llvm_readobj, rustc};
+use run_make_support::{llvm_readobj, rustc, target};
 
 fn main() {
-    rustc().opt_level("3").arg("-Zemit-stack-sizes").emit("obj").input("foo.rs").run();
+    rustc()
+        .target(target())
+        .opt_level("3")
+        .arg("-Zemit-stack-sizes")
+        .emit("obj")
+        .input("foo.rs")
+        .run();
     llvm_readobj()
         .arg("--section-headers")
         .input("foo.o")

tests/run-make/env-dep-info/rmake.rs

@@ -4,6 +4,8 @@
 // appear in the output dep-info files.
 // See https://github.com/rust-lang/rust/issues/40364
 
+//@ ignore-cross-compile (cross-compile targets can introduce complicated env differences)
+
 use run_make_support::{diff, rustc};
 
 fn main() {

tests/run-make/invalid-so/rmake.rs

@@ -4,11 +4,14 @@
 // explains that the file exists, but that its metadata is incorrect.
 // See https://github.com/rust-lang/rust/pull/88368
 
-use run_make_support::{dynamic_lib_name, rfs, rustc};
+//@ needs-crate-type: dylib
+
+use run_make_support::{dynamic_lib_name, rfs, rustc, target};
 
 fn main() {
     rfs::create_file(dynamic_lib_name("foo"));
     rustc()
+        .target(target())
         .crate_type("lib")
         .extern_("foo", dynamic_lib_name("foo"))
         .input("bar.rs")

tests/run-make/link-args-order/rmake.rs

@@ -3,6 +3,8 @@
 // checks that linker arguments remain intact and in the order they were originally passed in.
 // See https://github.com/rust-lang/rust/pull/70665
 
+//@ ignore-cross-compile (exercises host linker)
+
 use run_make_support::{is_msvc, rustc};
 
 fn main() {