Auto merge of #39353 - alexcrichton:rollup, r=alexcrichton

Rollup of 21 pull requests

- Successful merges: #38617, #39284, #39285, #39290, #39302, #39305, #39306, #39307, #39311, #39313, #39314, #39321, #39325, #39332, #39335, #39344, #39345, #39346, #39348, #39350, #39351
- Failed merges:

Author: bors

.mailmap

@@ -43,8 +43,8 @@ Brian Anderson <[email protected]> <[email protected]>
 Brian Dawn <[email protected]>
 Brian Leibig <[email protected]> Brian Leibig <[email protected]>
 Carl-Anton Ingmarsson <[email protected]> <[email protected]>
-Carol (Nichols || Goulding) <[email protected]> Carol Nichols <[email protected]>
-Carol (Nichols || Goulding) <[email protected]> Carol Nichols <[email protected]>
+Carol (Nichols || Goulding) <[email protected]>
+Carol (Nichols || Goulding) <[email protected]>
 Carol Willing <[email protected]>
 Chris C Cerami <[email protected]> Chris C Cerami <[email protected]>
 Chris Pressey <[email protected]>
@@ -53,6 +53,7 @@ Clark Gaebel <[email protected]> <[email protected]>
 Clinton Ryan <[email protected]>
 Corey Farwell <[email protected]> Corey Farwell <[email protected]>
 Corey Richardson <[email protected]> Elaine "See More" Nemo <[email protected]>
+Cyryl Płotnicki <[email protected]>
 Damien Schoof <[email protected]>
 Daniel Ramos <[email protected]>
 David Klein <[email protected]>
@@ -102,6 +103,7 @@ Jason Toffaletti <[email protected]> Jason Toffaletti <[email protected]>
 Jauhien Piatlicki <[email protected]> Jauhien Piatlicki <[email protected]>
 Jay True <[email protected]>
 Jeremy Letang <[email protected]>
+Jethro Beekman <[email protected]>
 Jihyun Yu <[email protected]> <[email protected]>
 Jihyun Yu <[email protected]> jihyun <[email protected]>
 Jihyun Yu <[email protected]> Jihyun Yu <[email protected]>

.travis.yml

@@ -110,9 +110,9 @@ before_deploy:
   - mkdir -p deploy/$TRAVIS_COMMIT
   - >
       if [ "$TRAVIS_OS_NAME" == "osx" ]; then
-          cp build/dist/*.tar.gz deploy/$TRAVIS_COMMIT;
+          cp -r build/dist deploy/$TRAVIS_COMMIT;
       else
-          cp obj/build/dist/*.tar.gz deploy/$TRAVIS_COMMIT;
+          cp -r obj/build/dist deploy/$TRAVIS_COMMIT;
       fi
 
 deploy:

appveyor.yml

@@ -137,7 +137,7 @@ branches:
 before_deploy:
   - ps: |
         New-Item -Path deploy -ItemType directory
-        Get-ChildItem -Path build\dist -Filter '*.tar.gz' | Move-Item -Destination deploy
+        Get-ChildItem -Path build\dist | Move-Item -Destination deploy
         Get-ChildItem -Path deploy | Foreach-Object {
           Push-AppveyorArtifact $_.FullName -FileName ${env:APPVEYOR_REPO_COMMIT}/$_
         }
@@ -151,7 +151,7 @@ deploy:
     bucket: rust-lang-ci
     set_public: true
     region: us-east-1
-    artifact: /.*\.tar.gz/
+    artifact: /.*/
     folder: rustc-builds
     on:
       branch: auto

src/Cargo.lock

@@ -10,6 +10,7 @@ members = [
   "tools/linkchecker",
   "tools/rustbook",
   "tools/tidy",
+  "tools/build-manifest",
 ]
 
 # Curiously, compiletest will segfault if compiled with opt-level=3 on 64-bit

src/Cargo.toml

@@ -78,6 +78,11 @@ pub struct Config {
     pub cargo: Option<PathBuf>,
     pub local_rebuild: bool,
 
+    // dist misc
+    pub dist_sign_folder: Option<PathBuf>,
+    pub dist_upload_addr: Option<String>,
+    pub dist_gpg_password_file: Option<PathBuf>,
+
     // libstd features
     pub debug_jemalloc: bool,
     pub use_jemalloc: bool,
@@ -123,6 +128,7 @@ struct TomlConfig {
     llvm: Option<Llvm>,
     rust: Option<Rust>,
     target: Option<HashMap<String, TomlTarget>>,
+    dist: Option<Dist>,
 }
 
 /// TOML representation of various global build decisions.
@@ -166,6 +172,13 @@ struct Llvm {
     targets: Option<String>,
 }
 
+#[derive(RustcDecodable, Default, Clone)]
+struct Dist {
+    sign_folder: Option<String>,
+    gpg_password_file: Option<String>,
+    upload_addr: Option<String>,
+}
+
 #[derive(RustcDecodable)]
 enum StringOrBool {
     String(String),
@@ -352,6 +365,12 @@ impl Config {
             }
         }
 
+        if let Some(ref t) = toml.dist {
+            config.dist_sign_folder = t.sign_folder.clone().map(PathBuf::from);
+            config.dist_gpg_password_file = t.gpg_password_file.clone().map(PathBuf::from);
+            config.dist_upload_addr = t.upload_addr.clone();
+        }
+
         return config
     }
 

src/bootstrap/config.rs

@@ -242,3 +242,33 @@
 # that this option only makes sense for MUSL targets that produce statically
 # linked binaries
 #musl-root = "..."
+
+# =============================================================================
+# Distribution options
+#
+# These options are related to distribution, mostly for the Rust project itself.
+# You probably won't need to concern yourself with any of these options
+# =============================================================================
+[dist]
+
+# This is the folder of artifacts that the build system will sign. All files in
+# this directory will be signed with the default gpg key using the system `gpg`
+# binary. The `asc` and `sha256` files will all be output into the standard dist
+# output folder (currently `build/dist`)
+#
+# This folder should be populated ahead of time before the build system is
+# invoked.
+#sign-folder = "path/to/folder/to/sign"
+
+# This is a file which contains the password of the default gpg key. This will
+# be passed to `gpg` down the road when signing all files in `sign-folder`
+# above. This should be stored in plaintext.
+#gpg-password-file = "path/to/gpg/password"
+
+# The remote address that all artifacts will eventually be uploaded to. The
+# build system generates manifests which will point to these urls, and for the
+# manifests to be correct they'll have to have the right URLs encoded.
+#
+# Note that this address should not contain a trailing slash as file names will
+# be appended to it.
+#upload-addr = "https://example.com/folder"

src/bootstrap/config.toml.example

@@ -22,7 +22,7 @@ use std::env;
 use std::fs::{self, File};
 use std::io::{Read, Write};
 use std::path::{PathBuf, Path};
-use std::process::Command;
+use std::process::{Command, Stdio};
 
 use build_helper::output;
 
@@ -876,3 +876,34 @@ fn add_env(build: &Build, cmd: &mut Command, target: &str) {
        cmd.env("CFG_PLATFORM", "x86");
     }
 }
+
+pub fn hash_and_sign(build: &Build) {
+    let compiler = Compiler::new(0, &build.config.build);
+    let mut cmd = build.tool_cmd(&compiler, "build-manifest");
+    let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
+        panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
+    });
+    let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
+        panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
+    });
+    let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| {
+        panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n")
+    });
+    let mut pass = String::new();
+    t!(t!(File::open(&file)).read_to_string(&mut pass));
+
+    let today = output(Command::new("date").arg("+%Y-%m-%d"));
+
+    cmd.arg(sign);
+    cmd.arg(distdir(build));
+    cmd.arg(today.trim());
+    cmd.arg(package_vers(build));
+    cmd.arg(addr);
+
+    t!(fs::create_dir_all(distdir(build)));
+
+    let mut child = t!(cmd.stdin(Stdio::piped()).spawn());
+    t!(child.stdin.take().unwrap().write_all(pass.as_bytes()));
+    let status = t!(child.wait());
+    assert!(status.success());
+}

src/bootstrap/dist.rs

@@ -513,6 +513,9 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     rules.build("tool-compiletest", "src/tools/compiletest")
          .dep(|s| s.name("libtest"))
          .run(move |s| compile::tool(build, s.stage, s.target, "compiletest"));
+    rules.build("tool-build-manifest", "src/tools/build-manifest")
+         .dep(|s| s.name("libstd"))
+         .run(move |s| compile::tool(build, s.stage, s.target, "build-manifest"));
 
     // ========================================================================
     // Documentation targets
@@ -633,6 +636,13 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .dep(|d| d.name("dist-cargo"))
          .run(move |s| dist::extended(build, s.stage, s.target));
 
+    rules.dist("dist-sign", "hash-and-sign")
+         .host(true)
+         .only_build(true)
+         .only_host_build(true)
+         .dep(move |s| s.name("tool-build-manifest").target(&build.config.build).stage(0))
+         .run(move |_| dist::hash_and_sign(build));
+
     rules.verify();
     return rules;
 }

src/bootstrap/step.rs

@@ -710,7 +710,7 @@ Please note that [`catch_unwind()`] will only catch unwinding panics, not
 those who abort the process. See the documentation of [`catch_unwind()`]
 for more information.
 
-[`catch_unwind()`]: https://doc.rust-lang.org/std/panic/fn.catch_unwind.html
+[`catch_unwind()`]: ../std/panic/fn.catch_unwind.html
 
 # Representing opaque structs