fix(build-std): determine root crates by target spec std:bool

In #14183 Cargo starts bailing out if the `metadata.std`
field in a target spec JSON is set to `false`.
This is problematic because std for some targets are actually buildable
even they've declared as std-unsupported.

This patch removes the hard error, and instead determines the required
root crates by the `metadata.std` field. That is to say, if a target
is explicitly declared as `std: false`, `-Zbuild-std` will build `core`
and `compiler-builtins` only, no `std` will be built.

This patch doesn't change the behavior of `-Zbuild-std` with explicit
crates set. For example `-Zbuild-std=std` will force building `std`.

See Zulip discussion:
https://rust-lang.zulipchat.com/#narrow/channel/246057-t-cargo/topic/help.20debugging.20a.20docs.2Ers.20issue.20with.20a.20new.20cargo.20error

Author: weihanglo

.github/workflows/main.yml

@@ -161,6 +161,8 @@ jobs:
     - run: rustup update --no-self-update stable
     - run: rustup update --no-self-update ${{ matrix.rust }} && rustup default ${{ matrix.rust }}
     - run: rustup target add ${{ matrix.other }}
+    - run: rustup target add aarch64-unknown-none # need this for build-std mock tests
+      if: startsWith(matrix.rust, 'nightly')
     - run: rustup component add rustc-dev llvm-tools-preview rust-docs
       if: startsWith(matrix.rust, 'nightly')
     - run: sudo apt update -y && sudo apt install lldb gcc-multilib libsecret-1-0 libsecret-1-dev -y

src/cargo/core/compiler/standard_lib.rs

@@ -15,12 +15,12 @@ use std::path::PathBuf;
 
 use super::BuildConfig;
 
-fn std_crates<'a>(crates: &'a [String], units: Option<&[Unit]>) -> HashSet<&'a str> {
+fn std_crates<'a>(crates: &'a [String], default: &'static str, units: &[Unit]) -> HashSet<&'a str> {
     let mut crates = HashSet::from_iter(crates.iter().map(|s| s.as_str()));
     // This is a temporary hack until there is a more principled way to
     // declare dependencies in Cargo.toml.
     if crates.is_empty() {
-        crates.insert("std");
+        crates.insert(default);
     }
     if crates.contains("std") {
         crates.insert("core");
@@ -30,13 +30,11 @@ fn std_crates<'a>(crates: &'a [String], units: Option<&[Unit]>) -> HashSet<&'a s
         crates.insert("compiler_builtins");
         // Only build libtest if it looks like it is needed (libtest depends on libstd)
         // If we know what units we're building, we can filter for libtest depending on the jobs.
-        if let Some(units) = units {
-            if units
-                .iter()
-                .any(|unit| unit.mode.is_rustc_test() && unit.target.harness())
-            {
-                crates.insert("test");
-            }
+        if units
+            .iter()
+            .any(|unit| unit.mode.is_rustc_test() && unit.target.harness())
+        {
+            crates.insert("test");
         }
     } else if crates.contains("core") {
         crates.insert("compiler_builtins");
@@ -115,14 +113,52 @@ pub fn generate_std_roots(
     profiles: &Profiles,
     target_data: &RustcTargetData<'_>,
 ) -> CargoResult<HashMap<CompileKind, Vec<Unit>>> {
-    let std_ids = std_crates(crates, Some(units))
+    // Generate a map of Units for each kind requested.
+    let mut ret = HashMap::new();
+    let (core_only, maybe_std): (Vec<&CompileKind>, Vec<_>) = kinds.iter().partition(|kind|
+        // Only include targets that explicitly don't support std
+        target_data.info(**kind).supports_std == Some(false));
+    for (default_crate, kinds) in [("core", core_only), ("std", maybe_std)] {
+        if kinds.is_empty() {
+            continue;
+        }
+        generate_roots(
+            &mut ret,
+            default_crate,
+            crates,
+            units,
+            std_resolve,
+            std_features,
+            &kinds,
+            package_set,
+            interner,
+            profiles,
+            target_data,
+        )?;
+    }
+
+    Ok(ret)
+}
+
+fn generate_roots(
+    ret: &mut HashMap<CompileKind, Vec<Unit>>,
+    default: &'static str,
+    crates: &[String],
+    units: &[Unit],
+    std_resolve: &Resolve,
+    std_features: &ResolvedFeatures,
+    kinds: &[&CompileKind],
+    package_set: &PackageSet<'_>,
+    interner: &UnitInterner,
+    profiles: &Profiles,
+    target_data: &RustcTargetData<'_>,
+) -> CargoResult<()> {
+    let std_ids = std_crates(crates, default, units)
         .iter()
         .map(|crate_name| std_resolve.query(crate_name))
         .collect::<CargoResult<Vec<PackageId>>>()?;
-    // Convert PackageId to Package.
     let std_pkgs = package_set.get_many(std_ids)?;
-    // Generate a map of Units for each kind requested.
-    let mut ret = HashMap::new();
+
     for pkg in std_pkgs {
         let lib = pkg
             .targets()
@@ -135,33 +171,34 @@ pub fn generate_std_roots(
         let mode = CompileMode::Build;
         let features = std_features.activated_features(pkg.package_id(), FeaturesFor::NormalOrDev);
         for kind in kinds {
-            let list = ret.entry(*kind).or_insert_with(Vec::new);
-            let unit_for = UnitFor::new_normal(*kind);
+            let kind = **kind;
+            let list = ret.entry(kind).or_insert_with(Vec::new);
+            let unit_for = UnitFor::new_normal(kind);
             let profile = profiles.get_profile(
                 pkg.package_id(),
                 /*is_member*/ false,
                 /*is_local*/ false,
                 unit_for,
-                *kind,
+                kind,
             );
             list.push(interner.intern(
                 pkg,
                 lib,
                 profile,
-                *kind,
+                kind,
                 mode,
                 features.clone(),
-                target_data.info(*kind).rustflags.clone(),
-                target_data.info(*kind).rustdocflags.clone(),
-                target_data.target_config(*kind).links_overrides.clone(),
+                target_data.info(kind).rustflags.clone(),
+                target_data.info(kind).rustdocflags.clone(),
+                target_data.target_config(kind).links_overrides.clone(),
                 /*is_std*/ true,
                 /*dep_hash*/ 0,
                 IsArtifact::No,
                 None,
             ));
         }
     }
-    Ok(ret)
+    Ok(())
 }
 
 fn detect_sysroot_src_path(target_data: &RustcTargetData<'_>) -> CargoResult<PathBuf> {

tests/testsuite/standard_lib.rs

@@ -429,12 +429,72 @@ fn build_std_with_no_arg_for_core_only_target() {
     p.cargo("build -v")
         .arg("--target=aarch64-unknown-none")
         .build_std(&setup)
-        .with_status(101)
-        .with_stderr_data(str![[r#"
-...
-error[E0463]: can't find crate for `std`
-...
-"#]])
+        .with_stderr_data(
+            str![[r#"
+[UPDATING] `dummy-registry` index
+[DOWNLOADING] crates ...
+[DOWNLOADED] registry-dep-using-std v1.0.0 (registry `dummy-registry`)
+[DOWNLOADED] registry-dep-using-core v1.0.0 (registry `dummy-registry`)
+[DOWNLOADED] registry-dep-using-alloc v1.0.0 (registry `dummy-registry`)
+[COMPILING] compiler_builtins v0.1.0 ([..]/library/compiler_builtins)
+[COMPILING] core v0.1.0 ([..]/library/core)
+[COMPILING] foo v0.0.1 ([ROOT]/foo)
+[RUNNING] `[..] rustc --crate-name compiler_builtins [..]--target aarch64-unknown-none[..]`
+[RUNNING] `[..] rustc --crate-name core [..]--target aarch64-unknown-none[..]`
+[RUNNING] `[..] rustc --crate-name foo [..]--target aarch64-unknown-none[..]`
+[FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
+
+"#]]
+            .unordered(),
+        )
+        .run();
+
+    p.cargo("clean").run();
+
+    // Also work for a mix of std and core-only targets,
+    // though not sure how common it is...
+    //
+    // Note that we don't  download std dependencies for the second call
+    // because `-Zbuild-std` downloads them all also when building for core only.
+    p.cargo("build -v")
+        .arg("--target=aarch64-unknown-none")
+        .target_host()
+        .build_std(&setup)
+        .with_stderr_data(
+            str![[r#"
+[UPDATING] `dummy-registry` index
+[COMPILING] core v0.1.0 ([..]/library/core)
+[COMPILING] dep_test v0.1.0 ([..]/dep_test)
+[COMPILING] compiler_builtins v0.1.0 ([..]/library/compiler_builtins)
+[COMPILING] proc_macro v0.1.0 ([..]/library/proc_macro)
+[COMPILING] panic_unwind v0.1.0 ([..]/library/panic_unwind)
+[COMPILING] rustc-std-workspace-core v1.9.0 ([..]/library/rustc-std-workspace-core)
+[COMPILING] foo v0.0.1 ([ROOT]/foo)
+[COMPILING] registry-dep-using-core v1.0.0
+[COMPILING] alloc v0.1.0 ([..]/library/alloc)
+[COMPILING] rustc-std-workspace-alloc v1.9.0 ([..]/library/rustc-std-workspace-alloc)
+[COMPILING] registry-dep-using-alloc v1.0.0
+[COMPILING] std v0.1.0 ([..]/library/std)
+[RUNNING] `[..]rustc --crate-name compiler_builtins [..]--target aarch64-unknown-none[..]`
+[RUNNING] `[..]rustc --crate-name core [..]--target aarch64-unknown-none[..]`
+[RUNNING] `[..]rustc --crate-name foo [..]--target aarch64-unknown-none[..]`
+[RUNNING] `[..]rustc --crate-name core [..]--target [HOST_TARGET][..]`
+[RUNNING] `[..]rustc --crate-name dep_test [..]--target [HOST_TARGET][..]`
+[RUNNING] `[..]rustc --crate-name proc_macro [..]--target [HOST_TARGET][..]`
+[RUNNING] `[..]rustc --crate-name panic_unwind [..]--target [HOST_TARGET][..]`
+[RUNNING] `[..]rustc --crate-name compiler_builtins [..]--target [HOST_TARGET][..]`
+[RUNNING] `[..]rustc --crate-name rustc_std_workspace_core [..]--target [HOST_TARGET][..]`
+[RUNNING] `[..]rustc --crate-name registry_dep_using_core [..]--target [HOST_TARGET][..]`
+[RUNNING] `[..]rustc --crate-name alloc [..]--target [HOST_TARGET][..]`
+[RUNNING] `[..]rustc --crate-name rustc_std_workspace_alloc [..]--target [HOST_TARGET][..]`
+[RUNNING] `[..]rustc --crate-name registry_dep_using_alloc [..]--target [HOST_TARGET][..]`
+[RUNNING] `[..]rustc --crate-name std [..]--target [HOST_TARGET][..]`
+[RUNNING] `[..]rustc --crate-name foo [..]--target [HOST_TARGET][..]`
+[FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
+
+"#]]
+            .unordered(),
+        )
         .run();
 }