From a2cff48048b5090557fbd53931431c73681d0cad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 12 Dec 2025 16:13:58 +0100 Subject: [PATCH 1/2] Refactor and document `PlainSourceTarball` --- src/bootstrap/src/core/build_steps/dist.rs | 193 ++++++++++----------- 1 file changed, 96 insertions(+), 97 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 40149ee09427c..24781083bfabf 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1210,6 +1210,8 @@ impl Step for Src { } } +/// Tarball for people who want to build rustc and other components from the source. +/// Does not contain GPL code for licensing reasons. #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct PlainSourceTarball; @@ -1232,51 +1234,18 @@ impl Step for PlainSourceTarball { /// Creates the plain source tarball fn run(self, builder: &Builder<'_>) -> GeneratedTarball { - // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which - // means neither rustup nor rustup-toolchain-install-master know how to download it. - // It also contains symbolic links, unlike other any other dist tarball. - // It's used for distros building rustc from source in a pre-vendored environment. - let mut tarball = Tarball::new(builder, "rustc", "src"); - tarball.permit_symlinks(true); - let plain_dst_src = tarball.image_dir(); - - // This is the set of root paths which will become part of the source package - let src_files = [ - // tidy-alphabetical-start - ".gitmodules", - "CONTRIBUTING.md", - "COPYRIGHT", - "Cargo.lock", - "Cargo.toml", - "LICENSE-APACHE", - "LICENSE-MIT", - "README.md", - "RELEASES.md", - "REUSE.toml", - "bootstrap.example.toml", - "configure", - "license-metadata.json", - "package.json", - "x", - "x.ps1", - "x.py", - "yarn.lock", - // tidy-alphabetical-end - ]; - let src_dirs = ["src", "compiler", "library", "tests", "LICENSES"]; - - copy_src_dirs( + let tarball = prepare_source_tarball( builder, - &builder.src, - &src_dirs, + "src", &[ // We don't currently use the GCC source code for building any official components, // it is very big, and has unclear licensing implications due to being GPL licensed. // We thus exclude it from the source tarball from now. "src/gcc", ], - plain_dst_src, ); + + let plain_dst_src = tarball.image_dir(); // We keep something in src/gcc because it is a registered submodule, // and if it misses completely it can cause issues elsewhere // (see https://github.com/rust-lang/rust/issues/137332). @@ -1288,74 +1257,104 @@ impl Step for PlainSourceTarball { "The GCC source code is not included due to unclear licensing implications\n" )); } + tarball.bare() + } +} - // Copy the files normally - for item in &src_files { - builder.copy_link( - &builder.src.join(item), - &plain_dst_src.join(item), - FileType::Regular, - ); - } +fn prepare_source_tarball(builder: &Builder<'_>, name: &str, exclude_dirs: &[&str]) -> Tarball { + // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which + // means neither rustup nor rustup-toolchain-install-master know how to download it. + // It also contains symbolic links, unlike other any other dist tarball. + // It's used for distros building rustc from source in a pre-vendored environment. + let mut tarball = Tarball::new(builder, "rustc", name); + tarball.permit_symlinks(true); + let plain_dst_src = tarball.image_dir(); + + // This is the set of root paths which will become part of the source package + let src_files = [ + // tidy-alphabetical-start + ".gitmodules", + "CONTRIBUTING.md", + "COPYRIGHT", + "Cargo.lock", + "Cargo.toml", + "LICENSE-APACHE", + "LICENSE-MIT", + "README.md", + "RELEASES.md", + "REUSE.toml", + "bootstrap.example.toml", + "configure", + "license-metadata.json", + "package.json", + "x", + "x.ps1", + "x.py", + "yarn.lock", + // tidy-alphabetical-end + ]; + let src_dirs = ["src", "compiler", "library", "tests", "LICENSES"]; - // Create the version file - builder.create(&plain_dst_src.join("version"), &builder.rust_version()); + copy_src_dirs(builder, &builder.src, &src_dirs, exclude_dirs, plain_dst_src); - // Create the files containing git info, to ensure --version outputs the same. - let write_git_info = |info: Option<&Info>, path: &Path| { - if let Some(info) = info { - t!(std::fs::create_dir_all(path)); - channel::write_commit_hash_file(path, &info.sha); - channel::write_commit_info_file(path, info); - } - }; - write_git_info(builder.rust_info().info(), plain_dst_src); - write_git_info(builder.cargo_info.info(), &plain_dst_src.join("./src/tools/cargo")); - - if builder.config.dist_vendor { - builder.require_and_update_all_submodules(); - - // Vendor packages that are required by opt-dist to collect PGO profiles. - let pkgs_for_pgo_training = build_helper::LLVM_PGO_CRATES - .iter() - .chain(build_helper::RUSTC_PGO_CRATES) - .map(|pkg| { - let mut manifest_path = - builder.src.join("./src/tools/rustc-perf/collector/compile-benchmarks"); - manifest_path.push(pkg); - manifest_path.push("Cargo.toml"); - manifest_path - }); - - // Vendor all Cargo dependencies - let vendor = builder.ensure(Vendor { - sync_args: pkgs_for_pgo_training.collect(), - versioned_dirs: true, - root_dir: plain_dst_src.into(), - output_dir: VENDOR_DIR.into(), - }); + // Copy the files normally + for item in &src_files { + builder.copy_link(&builder.src.join(item), &plain_dst_src.join(item), FileType::Regular); + } - let cargo_config_dir = plain_dst_src.join(".cargo"); - builder.create_dir(&cargo_config_dir); - builder.create(&cargo_config_dir.join("config.toml"), &vendor.config); - } + // Create the version file + builder.create(&plain_dst_src.join("version"), &builder.rust_version()); - // Delete extraneous directories - // FIXME: if we're managed by git, we should probably instead ask git if the given path - // is managed by it? - for entry in walkdir::WalkDir::new(tarball.image_dir()) - .follow_links(true) - .into_iter() - .filter_map(|e| e.ok()) - { - if entry.path().is_dir() && entry.path().file_name() == Some(OsStr::new("__pycache__")) - { - t!(fs::remove_dir_all(entry.path())); - } + // Create the files containing git info, to ensure --version outputs the same. + let write_git_info = |info: Option<&Info>, path: &Path| { + if let Some(info) = info { + t!(std::fs::create_dir_all(path)); + channel::write_commit_hash_file(path, &info.sha); + channel::write_commit_info_file(path, info); } + }; + write_git_info(builder.rust_info().info(), plain_dst_src); + write_git_info(builder.cargo_info.info(), &plain_dst_src.join("./src/tools/cargo")); + + if builder.config.dist_vendor { + builder.require_and_update_all_submodules(); + + // Vendor packages that are required by opt-dist to collect PGO profiles. + let pkgs_for_pgo_training = + build_helper::LLVM_PGO_CRATES.iter().chain(build_helper::RUSTC_PGO_CRATES).map(|pkg| { + let mut manifest_path = + builder.src.join("./src/tools/rustc-perf/collector/compile-benchmarks"); + manifest_path.push(pkg); + manifest_path.push("Cargo.toml"); + manifest_path + }); - tarball.bare() + // Vendor all Cargo dependencies + let vendor = builder.ensure(Vendor { + sync_args: pkgs_for_pgo_training.collect(), + versioned_dirs: true, + root_dir: plain_dst_src.into(), + output_dir: VENDOR_DIR.into(), + }); + + let cargo_config_dir = plain_dst_src.join(".cargo"); + builder.create_dir(&cargo_config_dir); + builder.create(&cargo_config_dir.join("config.toml"), &vendor.config); + } + + // Delete extraneous directories + // FIXME: if we're managed by git, we should probably instead ask git if the given path + // is managed by it? + for entry in walkdir::WalkDir::new(tarball.image_dir()) + .follow_links(true) + .into_iter() + .filter_map(|e| e.ok()) + { + if entry.path().is_dir() && entry.path().file_name() == Some(OsStr::new("__pycache__")) { + t!(fs::remove_dir_all(entry.path())); + } } + tarball } #[derive(Debug, Clone, Hash, PartialEq, Eq)] From 0d2a418798a030ebf72115e6c4624609f690b249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 12 Dec 2025 16:35:41 +0100 Subject: [PATCH 2/2] Add `PlainSourceTarballGpl` component that includes also GPL code --- src/bootstrap/src/core/build_steps/dist.rs | 37 ++++++++++++++++++- .../builder/cli_paths/snapshots/x_dist.snap | 3 ++ src/bootstrap/src/core/builder/mod.rs | 1 + 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 24781083bfabf..b9bf5ba61eed6 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1211,7 +1211,8 @@ impl Step for Src { } /// Tarball for people who want to build rustc and other components from the source. -/// Does not contain GPL code for licensing reasons. +/// Does not contain GPL code, which is separated into `PlainSourceTarballGpl` +/// for licensing reasons. #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct PlainSourceTarball; @@ -1261,7 +1262,39 @@ impl Step for PlainSourceTarball { } } -fn prepare_source_tarball(builder: &Builder<'_>, name: &str, exclude_dirs: &[&str]) -> Tarball { +/// Tarball with *all* source code for source builds, including GPL-licensed code. +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct PlainSourceTarballGpl; + +impl Step for PlainSourceTarballGpl { + /// Produces the location of the tarball generated + type Output = GeneratedTarball; + const IS_HOST: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rustc-src-gpl") + } + + fn is_default_step(builder: &Builder<'_>) -> bool { + builder.config.rust_dist_src + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(PlainSourceTarballGpl); + } + + /// Creates the plain source tarball + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + let tarball = prepare_source_tarball(builder, "src-gpl", &[]); + tarball.bare() + } +} + +fn prepare_source_tarball<'a>( + builder: &'a Builder<'a>, + name: &str, + exclude_dirs: &[&str], +) -> Tarball<'a> { // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which // means neither rustup nor rustup-toolchain-install-master know how to download it. // It also contains symbolic links, unlike other any other dist tarball. diff --git a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_dist.snap b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_dist.snap index 2fc8ca143dd0d..7fae0b24df94d 100644 --- a/src/bootstrap/src/core/builder/cli_paths/snapshots/x_dist.snap +++ b/src/bootstrap/src/core/builder/cli_paths/snapshots/x_dist.snap @@ -29,6 +29,9 @@ expression: dist [Dist] dist::PlainSourceTarball targets: [x86_64-unknown-linux-gnu] - Set({dist::rustc-src}) +[Dist] dist::PlainSourceTarballGpl + targets: [x86_64-unknown-linux-gnu] + - Set({dist::rustc-src-gpl}) [Dist] dist::ReproducibleArtifacts targets: [x86_64-unknown-linux-gnu] - Set({dist::reproducible-artifacts}) diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index 961d0cd855ae3..a5d3395112673 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -986,6 +986,7 @@ impl<'a> Builder<'a> { // and force us to rebuild tools after vendoring dependencies. // To work around this, create the Tarball after building all the tools. dist::PlainSourceTarball, + dist::PlainSourceTarballGpl, dist::BuildManifest, dist::ReproducibleArtifacts, dist::Gcc