Auto merge of #80435 - pietroalbini:compression-formats, r=Mark-Simulacrum

Only produce .xz tarballs on CI

This PR adds a `./configure` option to choose which tarball compression formats to produce, and changes our CI configuration to only produce `.xz` tarballs. The release process will then recompress everything into `.gz` when producing a release.

This will drastically reduce our storage costs for CI artifacts, as we'd stop storing the same data twice. **Stable, beta and nightly releases will not be affected by this at all.**

Before landing this we'll need to increase the VM size of our release process, to recompress everything in a reasonable amount of time.
r? `@Mark-Simulacrum`

Author: bors

config.toml.example

@@ -669,3 +669,7 @@ changelog-seen = 2
 
 # Whether to allow failures when building tools
 #missing-tools = false
+
+# List of compression formats to use when generating dist tarballs. The list of
+# formats is provided to rust-installer, which must support all of them.
+#compression-formats = ["gz", "xz"]

src/bootstrap/config.rs

@@ -148,6 +148,7 @@ pub struct Config {
     pub dist_sign_folder: Option<PathBuf>,
     pub dist_upload_addr: Option<String>,
     pub dist_gpg_password_file: Option<PathBuf>,
+    pub dist_compression_formats: Option<Vec<String>>,
 
     // libstd features
     pub backtrace: bool, // support for RUST_BACKTRACE
@@ -438,6 +439,7 @@ struct Dist {
     upload_addr: Option<String>,
     src_tarball: Option<bool>,
     missing_tools: Option<bool>,
+    compression_formats: Option<Vec<String>>,
 }
 
 #[derive(Deserialize)]
@@ -936,6 +938,7 @@ impl Config {
             config.dist_sign_folder = t.sign_folder.map(PathBuf::from);
             config.dist_gpg_password_file = t.gpg_password_file.map(PathBuf::from);
             config.dist_upload_addr = t.upload_addr;
+            config.dist_compression_formats = t.compression_formats;
             set(&mut config.rust_dist_src, t.src_tarball);
             set(&mut config.missing_tools, t.missing_tools);
         }

src/bootstrap/configure.py

@@ -147,6 +147,8 @@ def v(*args):
   "experimental LLVM targets to build")
 v("release-channel", "rust.channel", "the name of the release channel to build")
 v("release-description", "rust.description", "optional descriptive string for version output")
+v("dist-compression-formats", None,
+  "comma-separated list of compression formats to use")
 
 # Used on systems where "cc" is unavailable
 v("default-linker", "rust.default-linker", "the default linker")
@@ -349,6 +351,8 @@ def set(key, value):
     elif option.name == 'option-checking':
         # this was handled above
         pass
+    elif option.name == 'dist-compression-formats':
+        set('dist.compression-formats', value.split(','))
     else:
         raise RuntimeError("unhandled option {}".format(option.name))
 

src/bootstrap/tarball.rs

@@ -294,11 +294,25 @@ impl<'a> Tarball<'a> {
 
         build_cli(&self, &mut cmd);
         cmd.arg("--work-dir").arg(&self.temp_dir);
+        if let Some(formats) = &self.builder.config.dist_compression_formats {
+            assert!(!formats.is_empty(), "dist.compression-formats can't be empty");
+            cmd.arg("--compression-formats").arg(formats.join(","));
+        }
         self.builder.run(&mut cmd);
         if self.delete_temp_dir {
             t!(std::fs::remove_dir_all(&self.temp_dir));
         }
 
-        crate::dist::distdir(self.builder).join(format!("{}.tar.gz", package_name))
+        // Use either the first compression format defined, or "gz" as the default.
+        let ext = self
+            .builder
+            .config
+            .dist_compression_formats
+            .as_ref()
+            .and_then(|formats| formats.get(0))
+            .map(|s| s.as_str())
+            .unwrap_or("gz");
+
+        crate::dist::distdir(self.builder).join(format!("{}.tar.{}", package_name, ext))
     }
 }

src/bootstrap/test.rs

@@ -1962,7 +1962,7 @@ impl Step for Distcheck {
         builder.ensure(dist::Src);
 
         let mut cmd = Command::new("tar");
-        cmd.arg("-xzf")
+        cmd.arg("-xf")
             .arg(builder.ensure(dist::PlainSourceTarball))
             .arg("--strip-components=1")
             .current_dir(&dir);
@@ -1986,10 +1986,7 @@ impl Step for Distcheck {
         t!(fs::create_dir_all(&dir));
 
         let mut cmd = Command::new("tar");
-        cmd.arg("-xzf")
-            .arg(builder.ensure(dist::Src))
-            .arg("--strip-components=1")
-            .current_dir(&dir);
+        cmd.arg("-xf").arg(builder.ensure(dist::Src)).arg("--strip-components=1").current_dir(&dir);
         builder.run(&mut cmd);
 
         let toml = dir.join("rust-src/lib/rustlib/src/rust/library/std/Cargo.toml");

src/ci/run.sh

@@ -53,6 +53,11 @@ RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-locked-deps"
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-cargo-native-static"
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-units-std=1"
 
+# Only produce xz tarballs on CI. gz tarballs will be generated by the release
+# process by recompressing the existing xz ones. This decreases the storage
+# space required for CI artifacts.
+RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --dist-compression-formats=xz"
+
 if [ "$DIST_SRC" = "" ]; then
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-dist-src"
 fi