Skip to content

Commit a200391

Browse files
committed
Auto merge of #17025 - lnicola:josh, r=lnicola
internal: Use josh for subtree syncs
2 parents 8ea8c74 + 8f21381 commit a200391

File tree

8 files changed

+248
-30
lines changed

8 files changed

+248
-30
lines changed

src/tools/rust-analyzer/Cargo.lock

+49
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/tidy.rs

+1
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,7 @@ MIT OR Apache-2.0
144144
MIT OR Apache-2.0 OR Zlib
145145
MIT OR Zlib OR Apache-2.0
146146
MIT/Apache-2.0
147+
MPL-2.0
147148
Unlicense OR MIT
148149
Unlicense/MIT
149150
Zlib OR Apache-2.0 OR MIT

src/tools/rust-analyzer/docs/dev/README.md

+15-5
Original file line numberDiff line numberDiff line change
@@ -229,12 +229,22 @@ Release steps:
229229
* publishes the VS Code extension to the marketplace
230230
* call the GitHub API for PR details
231231
* create a new changelog in `rust-analyzer.github.io`
232-
3. While the release is in progress, fill in the changelog
233-
4. Commit & push the changelog
232+
3. While the release is in progress, fill in the changelog.
233+
4. Commit & push the changelog.
234234
5. Run `cargo xtask publish-release-notes <CHANGELOG>` -- this will convert the changelog entry in AsciiDoc to Markdown and update the body of GitHub Releases entry.
235-
6. Tweet
236-
7. Inside `rust-analyzer`, run `cargo xtask promote` -- this will create a PR to rust-lang/rust updating rust-analyzer's subtree.
237-
Self-approve the PR.
235+
6. Tweet.
236+
7. Make a new branch and run `cargo xtask rustc-pull`, open a PR, and merge it.
237+
This will pull any changes from `rust-lang/rust` into `rust-analyzer`.
238+
8. Switch to `master`, pull, then run `cargo xtask rustc-push --rust-path ../rust-rust-analyzer --rust-fork matklad/rust`.
239+
Replace `matklad/rust` with your own fork of `rust-lang/rust`.
240+
You can use the token to authenticate when you get prompted for a password, since `josh` will push over HTTPS, not SSH.
241+
This will push the `rust-analyzer` changes to your fork.
242+
You can then open a PR against `rust-lang/rust`.
243+
244+
Note: besides the `rust-rust-analyzer` clone, the Josh cache (stored under `~/.cache/rust-analyzer-josh`) will contain a bare clone of `rust-lang/rust`.
245+
This currently takes about 3.5 GB.
246+
247+
This [HackMD](https://hackmd.io/7pOuxnkdQDaL1Y1FQr65xg) has details about how `josh` syncs work.
238248

239249
If the GitHub Actions release fails because of a transient problem like a timeout, you can re-run the job from the Actions console.
240250
If it fails because of something that needs to be fixed, remove the release tag (if needed), fix the problem, then start over.

src/tools/rust-analyzer/rust-version

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
688c30dc9f8434d63bddb65bd6a4d2258d19717c

src/tools/rust-analyzer/xtask/Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ rust-version.workspace = true
88

99
[dependencies]
1010
anyhow.workspace = true
11+
directories = "5.0"
1112
flate2 = "1.0.24"
1213
write-json = "0.1.2"
1314
xshell.workspace = true

src/tools/rust-analyzer/xtask/src/flags.rs

+29-9
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,16 @@ xflags::xflags! {
1414
cmd install {
1515
/// Install only VS Code plugin.
1616
optional --client
17-
/// One of 'code', 'code-exploration', 'code-insiders', 'codium', or 'code-oss'.
17+
/// One of `code`, `code-exploration`, `code-insiders`, `codium`, or `code-oss`.
1818
optional --code-bin name: String
1919

2020
/// Install only the language server.
2121
optional --server
22-
/// Use mimalloc allocator for server
22+
/// Use mimalloc allocator for server.
2323
optional --mimalloc
24-
/// Use jemalloc allocator for server
24+
/// Use jemalloc allocator for server.
2525
optional --jemalloc
26-
/// build in release with debug info set to 2
26+
/// build in release with debug info set to 2.
2727
optional --dev-rel
2828
}
2929

@@ -32,9 +32,21 @@ xflags::xflags! {
3232
cmd release {
3333
optional --dry-run
3434
}
35-
cmd promote {
36-
optional --dry-run
35+
36+
cmd rustc-pull {
37+
/// rustc commit to pull.
38+
optional --commit refspec: String
39+
}
40+
41+
cmd rustc-push {
42+
/// rust local path, e.g. `../rust-rust-analyzer`.
43+
required --rust-path rust_path: String
44+
/// rust fork name, e.g. `matklad/rust`.
45+
required --rust-fork rust_fork: String
46+
/// branch name.
47+
optional --branch branch: String
3748
}
49+
3850
cmd dist {
3951
/// Use mimalloc allocator for server
4052
optional --mimalloc
@@ -77,7 +89,8 @@ pub enum XtaskCmd {
7789
Install(Install),
7890
FuzzTests(FuzzTests),
7991
Release(Release),
80-
Promote(Promote),
92+
RustcPull(RustcPull),
93+
RustcPush(RustcPush),
8194
Dist(Dist),
8295
PublishReleaseNotes(PublishReleaseNotes),
8396
Metrics(Metrics),
@@ -104,8 +117,15 @@ pub struct Release {
104117
}
105118

106119
#[derive(Debug)]
107-
pub struct Promote {
108-
pub dry_run: bool,
120+
pub struct RustcPull {
121+
pub commit: Option<String>,
122+
}
123+
124+
#[derive(Debug)]
125+
pub struct RustcPush {
126+
pub rust_path: String,
127+
pub rust_fork: String,
128+
pub branch: Option<String>,
109129
}
110130

111131
#[derive(Debug)]

src/tools/rust-analyzer/xtask/src/main.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ fn main() -> anyhow::Result<()> {
3434
flags::XtaskCmd::Install(cmd) => cmd.run(sh),
3535
flags::XtaskCmd::FuzzTests(_) => run_fuzzer(sh),
3636
flags::XtaskCmd::Release(cmd) => cmd.run(sh),
37-
flags::XtaskCmd::Promote(cmd) => cmd.run(sh),
37+
flags::XtaskCmd::RustcPull(cmd) => cmd.run(sh),
38+
flags::XtaskCmd::RustcPush(cmd) => cmd.run(sh),
3839
flags::XtaskCmd::Dist(cmd) => cmd.run(sh),
3940
flags::XtaskCmd::PublishReleaseNotes(cmd) => cmd.run(sh),
4041
flags::XtaskCmd::Metrics(cmd) => cmd.run(sh),

src/tools/rust-analyzer/xtask/src/release.rs

+150-15
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
mod changelog;
22

3+
use std::process::{Command, Stdio};
4+
use std::thread;
5+
use std::time::Duration;
6+
7+
use anyhow::{bail, Context as _};
8+
use directories::ProjectDirs;
9+
use stdx::JodChild;
310
use xshell::{cmd, Shell};
411

512
use crate::{codegen, date_iso, flags, is_release_tag, project_root};
@@ -71,26 +78,154 @@ impl flags::Release {
7178
}
7279
}
7380

74-
impl flags::Promote {
81+
// git sync implementation adapted from https://github.com/rust-lang/miri/blob/62039ac/miri-script/src/commands.rs
82+
impl flags::RustcPull {
7583
pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> {
76-
let _dir = sh.push_dir("../rust-rust-analyzer");
77-
cmd!(sh, "git switch master").run()?;
78-
cmd!(sh, "git fetch upstream").run()?;
79-
cmd!(sh, "git reset --hard upstream/master").run()?;
84+
sh.change_dir(project_root());
85+
let commit = self.commit.map(Result::Ok).unwrap_or_else(|| {
86+
let rust_repo_head =
87+
cmd!(sh, "git ls-remote https://github.com/rust-lang/rust/ HEAD").read()?;
88+
rust_repo_head
89+
.split_whitespace()
90+
.next()
91+
.map(|front| front.trim().to_owned())
92+
.ok_or_else(|| anyhow::format_err!("Could not obtain Rust repo HEAD from remote."))
93+
})?;
94+
// Make sure the repo is clean.
95+
if !cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty() {
96+
bail!("working directory must be clean before running `cargo xtask pull`");
97+
}
98+
// Make sure josh is running.
99+
let josh = start_josh()?;
80100

81-
let date = date_iso(sh)?;
82-
let branch = format!("rust-analyzer-{date}");
83-
cmd!(sh, "git switch -c {branch}").run()?;
84-
cmd!(sh, "git subtree pull -m ':arrow_up: rust-analyzer' -P src/tools/rust-analyzer rust-analyzer release").run()?;
101+
// Update rust-version file. As a separate commit, since making it part of
102+
// the merge has confused the heck out of josh in the past.
103+
// We pass `--no-verify` to avoid running any git hooks that might exist,
104+
// in case they dirty the repository.
105+
sh.write_file("rust-version", format!("{commit}\n"))?;
106+
const PREPARING_COMMIT_MESSAGE: &str = "Preparing for merge from rust-lang/rust";
107+
cmd!(sh, "git commit rust-version --no-verify -m {PREPARING_COMMIT_MESSAGE}")
108+
.run()
109+
.context("FAILED to commit rust-version file, something went wrong")?;
85110

86-
if !self.dry_run {
87-
cmd!(sh, "git push -u origin {branch}").run()?;
88-
cmd!(
89-
sh,
90-
"xdg-open https://github.com/matklad/rust/pull/new/{branch}?body=r%3F%20%40ghost"
91-
)
111+
// Fetch given rustc commit.
112+
cmd!(sh, "git fetch http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git")
113+
.run()
114+
.map_err(|e| {
115+
// Try to un-do the previous `git commit`, to leave the repo in the state we found it it.
116+
cmd!(sh, "git reset --hard HEAD^")
117+
.run()
118+
.expect("FAILED to clean up again after failed `git fetch`, sorry for that");
119+
e
120+
})
121+
.context("FAILED to fetch new commits, something went wrong (committing the rust-version file has been undone)")?;
122+
123+
// Merge the fetched commit.
124+
const MERGE_COMMIT_MESSAGE: &str = "Merge from rust-lang/rust";
125+
cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}")
126+
.run()
127+
.context("FAILED to merge new commits, something went wrong")?;
128+
129+
drop(josh);
130+
Ok(())
131+
}
132+
}
133+
134+
impl flags::RustcPush {
135+
pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> {
136+
let branch = self.branch.as_deref().unwrap_or("sync-from-ra");
137+
let rust_path = self.rust_path;
138+
let rust_fork = self.rust_fork;
139+
140+
sh.change_dir(project_root());
141+
let base = sh.read_file("rust-version")?.trim().to_owned();
142+
// Make sure the repo is clean.
143+
if !cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty() {
144+
bail!("working directory must be clean before running `cargo xtask push`");
145+
}
146+
// Make sure josh is running.
147+
let josh = start_josh()?;
148+
149+
// Find a repo we can do our preparation in.
150+
sh.change_dir(rust_path);
151+
152+
// Prepare the branch. Pushing works much better if we use as base exactly
153+
// the commit that we pulled from last time, so we use the `rust-version`
154+
// file to find out which commit that would be.
155+
println!("Preparing {rust_fork} (base: {base})...");
156+
if cmd!(sh, "git fetch https://github.com/{rust_fork} {branch}")
157+
.ignore_stderr()
158+
.read()
159+
.is_ok()
160+
{
161+
bail!(
162+
"The branch `{branch}` seems to already exist in `https://github.com/{rust_fork}`. Please delete it and try again."
163+
);
164+
}
165+
cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}").run()?;
166+
cmd!(sh, "git push https://github.com/{rust_fork} {base}:refs/heads/{branch}")
167+
.ignore_stdout()
168+
.ignore_stderr() // silence the "create GitHub PR" message
92169
.run()?;
170+
println!();
171+
172+
// Do the actual push.
173+
sh.change_dir(project_root());
174+
println!("Pushing rust-analyzer changes...");
175+
cmd!(
176+
sh,
177+
"git push http://localhost:{JOSH_PORT}/{rust_fork}.git{JOSH_FILTER}.git HEAD:{branch}"
178+
)
179+
.run()?;
180+
println!();
181+
182+
// Do a round-trip check to make sure the push worked as expected.
183+
cmd!(
184+
sh,
185+
"git fetch http://localhost:{JOSH_PORT}/{rust_fork}.git{JOSH_FILTER}.git {branch}"
186+
)
187+
.ignore_stderr()
188+
.read()?;
189+
let head = cmd!(sh, "git rev-parse HEAD").read()?;
190+
let fetch_head = cmd!(sh, "git rev-parse FETCH_HEAD").read()?;
191+
if head != fetch_head {
192+
bail!("Josh created a non-roundtrip push! Do NOT merge this into rustc!");
93193
}
194+
println!("Confirmed that the push round-trips back to rust-analyzer properly. Please create a rustc PR:");
195+
// https://github.com/github-linguist/linguist/compare/master...octocat:linguist:master
196+
let fork_path = rust_fork.replace('/', ":");
197+
println!(
198+
" https://github.com/rust-lang/rust/compare/{fork_path}:{branch}?quick_pull=1&title=Subtree+update+of+rust-analyzer&body=r?+@ghost"
199+
);
200+
201+
drop(josh);
94202
Ok(())
95203
}
96204
}
205+
206+
/// Used for rustc syncs.
207+
const JOSH_FILTER: &str =
208+
":rev(55d9a533b309119c8acd13061581b43ae8840823:prefix=src/tools/rust-analyzer):/src/tools/rust-analyzer";
209+
const JOSH_PORT: &str = "42042";
210+
211+
fn start_josh() -> anyhow::Result<impl Drop> {
212+
// Determine cache directory.
213+
let local_dir = {
214+
let user_dirs = ProjectDirs::from("org", "rust-lang", "rust-analyzer-josh").unwrap();
215+
user_dirs.cache_dir().to_owned()
216+
};
217+
218+
// Start josh, silencing its output.
219+
let mut cmd = Command::new("josh-proxy");
220+
cmd.arg("--local").arg(local_dir);
221+
cmd.arg("--remote").arg("https://github.com");
222+
cmd.arg("--port").arg(JOSH_PORT);
223+
cmd.arg("--no-background");
224+
cmd.stdout(Stdio::null());
225+
cmd.stderr(Stdio::null());
226+
let josh = cmd.spawn().context("failed to start josh-proxy, make sure it is installed")?;
227+
// Give it some time so hopefully the port is open. (100ms was not enough.)
228+
thread::sleep(Duration::from_millis(200));
229+
230+
Ok(JodChild(josh))
231+
}

0 commit comments

Comments
 (0)