Fix extended bootstrap issues with OpenSSL on NetBSD build hosts

src/Cargo.toml

@@ -38,6 +38,8 @@ members = [
   "tools/rls/test_data/infer_custom_bin",
   "tools/rls/test_data/infer_lib",
   "tools/rls/test_data/omit_init_build",
+  "tools/rls/test_data/unicødë",
+  "tools/rls/test_data/workspace_symbol",
 ]
 
 # Curiously, compiletest will segfault if compiled with opt-level=3 on 64-bit
@@ -60,10 +62,5 @@ debug-assertions = false
 [patch."https://github.com/rust-lang/cargo"]
 cargo = { path = "tools/cargo" }
 
-# Override rustfmt dependencies both on the repo and the crate (the RLS
-# sometimes uses either).
-# FIXME should only need the crates.io patch, long term.
-[patch."https://github.com/rust-lang-nursery/rustfmt"]
-rustfmt-nightly = { path = "tools/rustfmt" }
 [patch.crates-io]
 rustfmt-nightly = { path = "tools/rustfmt" }

src/bootstrap/Cargo.toml

@@ -34,7 +34,7 @@ cmake = "0.1.23"
 filetime = "0.1"
 num_cpus = "1.0"
 getopts = "0.2"
-gcc = "0.3.54"
+cc = "1.0"
 libc = "0.2"
 serde = "1.0.8"
 serde_derive = "1.0.8"

src/bootstrap/bin/sccache-plus-cl.rs

@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-extern crate gcc;
+extern crate cc;
 
 use std::env;
 use std::process::{self, Command};
@@ -18,7 +18,7 @@ fn main() {
     // Locate the actual compiler that we're invoking
     env::remove_var("CC");
     env::remove_var("CXX");
-    let mut cfg = gcc::Build::new();
+    let mut cfg = cc::Build::new();
     cfg.cargo_metadata(false)
        .out_dir("/")
        .target(&target)

src/bootstrap/builder.rs

@@ -306,7 +306,7 @@ impl<'a> Builder<'a> {
             Subcommand::Bench { ref paths, .. } => (Kind::Bench, &paths[..]),
             Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]),
             Subcommand::Install { ref paths } => (Kind::Install, &paths[..]),
-            Subcommand::Clean => panic!(),
+            Subcommand::Clean { .. } => panic!(),
         };
 
         let builder = Builder {

src/bootstrap/cc.rs → src/bootstrap/cc_detect.rs

@@ -23,7 +23,7 @@
 //! 6. "cc"
 //!
 //! Some of this logic is implemented here, but much of it is farmed out to the
-//! `gcc` crate itself, so we end up having the same fallbacks as there.
+//! `cc` crate itself, so we end up having the same fallbacks as there.
 //! Similar logic is then used to find a C++ compiler, just some s/cc/c++/ is
 //! used.
 //!
@@ -35,7 +35,7 @@ use std::process::Command;
 use std::iter;
 
 use build_helper::{cc2ar, output};
-use gcc;
+use cc;
 
 use Build;
 use config::Target;
@@ -45,7 +45,7 @@ pub fn find(build: &mut Build) {
     // For all targets we're going to need a C compiler for building some shims
     // and such as well as for being a linker for Rust code.
     for target in build.targets.iter().chain(&build.hosts).cloned().chain(iter::once(build.build)) {
-        let mut cfg = gcc::Build::new();
+        let mut cfg = cc::Build::new();
         cfg.cargo_metadata(false).opt_level(0).warnings(false).debug(false)
            .target(&target).host(&build.build);
 
@@ -67,7 +67,7 @@ pub fn find(build: &mut Build) {
 
     // For all host triples we need to find a C++ compiler as well
     for host in build.hosts.iter().cloned().chain(iter::once(build.build)) {
-        let mut cfg = gcc::Build::new();
+        let mut cfg = cc::Build::new();
         cfg.cargo_metadata(false).opt_level(0).warnings(false).debug(false).cpp(true)
            .target(&host).host(&build.build);
         let config = build.config.target_config.get(&host);
@@ -82,7 +82,7 @@ pub fn find(build: &mut Build) {
     }
 }
 
-fn set_compiler(cfg: &mut gcc::Build,
+fn set_compiler(cfg: &mut cc::Build,
                 gnu_compiler: &str,
                 target: Interned<String>,
                 config: Option<&Target>,

src/bootstrap/check.rs

@@ -254,7 +254,11 @@ impl Step for Rls {
 
         builder.add_rustc_lib_path(compiler, &mut cargo);
 
-        try_run(build, &mut cargo);
+        try_run_expecting(
+            build,
+            &mut cargo,
+            builder.build.config.toolstate.rls.passes(ToolState::Testing),
+        );
     }
 }
 
@@ -295,7 +299,11 @@ impl Step for Rustfmt {
 
         builder.add_rustc_lib_path(compiler, &mut cargo);
 
-        try_run(build, &mut cargo);
+        try_run_expecting(
+            build,
+            &mut cargo,
+            builder.build.config.toolstate.rustfmt.passes(ToolState::Testing),
+        );
     }
 }
 

src/bootstrap/clean.rs

@@ -13,32 +13,37 @@
 //! Responsible for cleaning out a build directory of all old and stale
 //! artifacts to prepare for a fresh build. Currently doesn't remove the
 //! `build/cache` directory (download cache) or the `build/$target/llvm`
-//! directory as we want that cached between builds.
+//! directory unless the --all flag is present.
 
 use std::fs;
 use std::io::{self, ErrorKind};
 use std::path::Path;
 
 use Build;
 
-pub fn clean(build: &Build) {
+pub fn clean(build: &Build, all: bool) {
     rm_rf("tmp".as_ref());
-    rm_rf(&build.out.join("tmp"));
-    rm_rf(&build.out.join("dist"));
 
-    for host in &build.hosts {
-        let entries = match build.out.join(host).read_dir() {
-            Ok(iter) => iter,
-            Err(_) => continue,
-        };
+    if all {
+        rm_rf(&build.out);
+    } else {
+        rm_rf(&build.out.join("tmp"));
+        rm_rf(&build.out.join("dist"));
 
-        for entry in entries {
-            let entry = t!(entry);
-            if entry.file_name().to_str() == Some("llvm") {
-                continue
+        for host in &build.hosts {
+            let entries = match build.out.join(host).read_dir() {
+                Ok(iter) => iter,
+                Err(_) => continue,
+            };
+
+            for entry in entries {
+                let entry = t!(entry);
+                if entry.file_name().to_str() == Some("llvm") {
+                    continue
+                }
+                let path = t!(entry.path().canonicalize());
+                rm_rf(&path);
             }
-            let path = t!(entry.path().canonicalize());
-            rm_rf(&path);
         }
     }
 }

src/bootstrap/configure.py

@@ -109,6 +109,8 @@ def v(*args):
   "arm-unknown-linux-musleabihf install directory")
 v("musl-root-armv7", "target.armv7-unknown-linux-musleabihf.musl-root",
   "armv7-unknown-linux-musleabihf install directory")
+v("musl-root-aarch64", "target.aarch64-unknown-linux-musl.musl-root",
+  "aarch64-unknown-linux-musl install directory")
 v("qemu-armhf-rootfs", "target.arm-unknown-linux-gnueabihf.qemu-rootfs",
   "rootfs in qemu testing, you probably don't want to use this")
 v("qemu-aarch64-rootfs", "target.aarch64-unknown-linux-gnu.qemu-rootfs",

src/bootstrap/dist.rs

@@ -1098,13 +1098,8 @@ impl Step for Rls {
            .arg("--output-dir").arg(&distdir(build))
            .arg("--non-installed-overlay").arg(&overlay)
            .arg(format!("--package-name={}-{}", name, target))
-           .arg("--legacy-manifest-dirs=rustlib,cargo");
-
-        if build.config.channel == "nightly" {
-            cmd.arg("--component-name=rls");
-        } else {
-            cmd.arg("--component-name=rls-preview");
-        }
+           .arg("--legacy-manifest-dirs=rustlib,cargo")
+           .arg("--component-name=rls-preview");
 
         build.run(&mut cmd);
         distdir(build).join(format!("{}-{}.tar.gz", name, target))
@@ -1333,12 +1328,8 @@ impl Step for Extended {
             cp_r(&work.join(&format!("{}-{}", pkgname(build, "rust-std"), target))
                         .join(format!("rust-std-{}", target)),
                     &exe.join("rust-std"));
-            let rls_path = if build.config.channel == "nightly" {
-                work.join(&format!("{}-{}", pkgname(build, "rls"), target)).join("rls")
-            } else {
-                work.join(&format!("{}-{}", pkgname(build, "rls"), target)).join("rls-preview")
-            };
-            cp_r(&rls_path, &exe.join("rls"));
+            cp_r(&work.join(&format!("{}-{}", pkgname(build, "rls"), target)).join("rls-preview"),
+                 &exe.join("rls"));
             cp_r(&work.join(&format!("{}-{}", pkgname(build, "rust-analysis"), target))
                         .join(format!("rust-analysis-{}", target)),
                     &exe.join("rust-analysis"));

src/bootstrap/flags.rs

@@ -60,7 +60,9 @@ pub enum Subcommand {
         paths: Vec<PathBuf>,
         test_args: Vec<String>,
     },
-    Clean,
+    Clean {
+        all: bool,
+    },
     Dist {
         paths: Vec<PathBuf>,
     },
@@ -147,6 +149,7 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`");
                 opts.optmulti("", "test-args", "extra arguments", "ARGS");
             },
             "bench" => { opts.optmulti("", "test-args", "extra arguments", "ARGS"); },
+            "clean" => { opts.optflag("", "all", "clean all build artifacts"); },
             _ => { },
         };
 
@@ -250,17 +253,18 @@ Arguments:
             }
         });
 
-        // All subcommands can have an optional "Available paths" section
+        // All subcommands except `clean` can have an optional "Available paths" section
         if matches.opt_present("verbose") {
             let config = Config::parse(&["build".to_string()]);
             let mut build = Build::new(config);
             metadata::build(&mut build);
 
             let maybe_rules_help = Builder::get_help(&build, subcommand.as_str());
             extra_help.push_str(maybe_rules_help.unwrap_or_default().as_str());
-        } else {
-            extra_help.push_str(format!("Run `./x.py {} -h -v` to see a list of available paths.",
-                     subcommand).as_str());
+        } else if subcommand.as_str() != "clean" {
+            extra_help.push_str(format!(
+                "Run `./x.py {} -h -v` to see a list of available paths.",
+                subcommand).as_str());
         }
 
         // User passed in -h/--help?
@@ -290,10 +294,13 @@ Arguments:
             }
             "clean" => {
                 if paths.len() > 0 {
-                    println!("\nclean takes no arguments\n");
+                    println!("\nclean does not take a path argument\n");
                     usage(1, &opts, &subcommand_help, &extra_help);
                 }
-                Subcommand::Clean
+
+                Subcommand::Clean {
+                    all: matches.opt_present("all"),
+                }
             }
             "dist" => {
                 Subcommand::Dist {

src/bootstrap/lib.rs

@@ -126,7 +126,7 @@ extern crate lazy_static;
 extern crate serde_json;
 extern crate cmake;
 extern crate filetime;
-extern crate gcc;
+extern crate cc;
 extern crate getopts;
 extern crate num_cpus;
 extern crate toml;
@@ -148,7 +148,7 @@ use build_helper::{run_silent, run_suppressed, try_run_silent, try_run_suppresse
 
 use util::{exe, libdir, OutputFolder, CiEnv};
 
-mod cc;
+mod cc_detect;
 mod channel;
 mod check;
 mod clean;
@@ -241,9 +241,9 @@ pub struct Build {
 
     // Runtime state filled in later on
     // target -> (cc, ar)
-    cc: HashMap<Interned<String>, (gcc::Tool, Option<PathBuf>)>,
+    cc: HashMap<Interned<String>, (cc::Tool, Option<PathBuf>)>,
     // host -> (cc, ar)
-    cxx: HashMap<Interned<String>, gcc::Tool>,
+    cxx: HashMap<Interned<String>, cc::Tool>,
     crates: HashMap<Interned<String>, Crate>,
     is_sudo: bool,
     ci_env: CiEnv,
@@ -345,12 +345,12 @@ impl Build {
             job::setup(self);
         }
 
-        if let Subcommand::Clean = self.config.cmd {
-            return clean::clean(self);
+        if let Subcommand::Clean { all } = self.config.cmd {
+            return clean::clean(self, all);
         }
 
         self.verbose("finding compilers");
-        cc::find(self);
+        cc_detect::find(self);
         self.verbose("running sanity check");
         sanity::check(self);
         // If local-rust is the same major.minor as the current version, then force a local-rebuild
@@ -619,7 +619,7 @@ impl Build {
     /// specified.
     fn cflags(&self, target: Interned<String>) -> Vec<String> {
         // Filter out -O and /O (the optimization flags) that we picked up from
-        // gcc-rs because the build scripts will determine that for themselves.
+        // cc-rs because the build scripts will determine that for themselves.
         let mut base = self.cc[&target].0.args().iter()
                            .map(|s| s.to_string_lossy().into_owned())
                            .filter(|s| !s.starts_with("-O") && !s.starts_with("/O"))

src/bootstrap/native.rs

@@ -27,7 +27,7 @@ use std::process::Command;
 
 use build_helper::output;
 use cmake;
-use gcc;
+use cc;
 
 use Build;
 use util;
@@ -289,7 +289,7 @@ impl Step for TestHelpers {
         let _folder = build.fold_output(|| "build_test_helpers");
         println!("Building test helpers");
         t!(fs::create_dir_all(&dst));
-        let mut cfg = gcc::Build::new();
+        let mut cfg = cc::Build::new();
 
         // We may have found various cross-compilers a little differently due to our
         // extra configuration, so inform gcc of these compilers. Note, though, that
@@ -367,7 +367,7 @@ impl Step for Openssl {
             if !ok {
                 panic!("failed to download openssl source")
             }
-            let mut shasum = if target.contains("apple") {
+            let mut shasum = if target.contains("apple") || build.build.contains("netbsd") {
                 let mut cmd = Command::new("shasum");
                 cmd.arg("-a").arg("256");
                 cmd
@@ -387,7 +387,7 @@ impl Step for Openssl {
         let dst = build.openssl_install_dir(target).unwrap();
         drop(fs::remove_dir_all(&obj));
         drop(fs::remove_dir_all(&dst));
-        build.run(Command::new("tar").arg("xf").arg(&tarball).current_dir(&out));
+        build.run(Command::new("tar").arg("zxf").arg(&tarball).current_dir(&out));
 
         let mut configure = Command::new("perl");
         configure.arg(obj.join("Configure"));
@@ -399,6 +399,7 @@ impl Step for Openssl {
         let os = match &*target {
             "aarch64-linux-android" => "linux-aarch64",
             "aarch64-unknown-linux-gnu" => "linux-aarch64",
+            "aarch64-unknown-linux-musl" => "linux-aarch64",
             "arm-linux-androideabi" => "android",
             "arm-unknown-linux-gnueabi" => "linux-armv4",
             "arm-unknown-linux-gnueabihf" => "linux-armv4",