[cargo-miri] support nextest

Add the ability to list and run nextest commands.

Running tests out of archives is currently broken, as the comment in
run-test.py explains. But the list and run commands work fine.

Author: sunshowers

.github/workflows/ci.yml

@@ -77,6 +77,9 @@ jobs:
             ./rustup-toolchain "" --host ${{ matrix.host_target }}
           fi
 
+      - name: Install nextest
+        uses: taiki-e/install-action@nextest
+
       - name: Show Rust version
         run: |
           rustup show

cargo-miri/bin.rs

@@ -25,6 +25,7 @@ Usage:
 Subcommands:
     run, r                   Run binaries
     test, t                  Run tests
+    nextest                  Run tests with nextest (requires cargo-nextest installed)
     setup                    Only perform automatic setup, but without asking questions (for getting a proper libstd)
 
 The cargo options are exactly the same as for `cargo run` and `cargo test`, respectively.
@@ -34,10 +35,11 @@ Examples:
     cargo miri test -- test-suite-filter
 "#;
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 enum MiriCommand {
     Run,
     Test,
+    Nextest(String),
     Setup,
 }
 
@@ -575,18 +577,29 @@ fn phase_cargo_miri(mut args: env::Args) {
     // so we cannot detect subcommands later.
     let subcommand = match args.next().as_deref() {
         Some("test" | "t") => MiriCommand::Test,
+        Some("nextest") => {
+            // The next argument is the nextest subcommand.
+            let nextest_command = match args.next() {
+                Some(nextest_command) => nextest_command,
+                None =>
+                    show_error(format!(
+                        "`cargo miri nextest` requires a nextest subcommand as its next argument."
+                    )),
+            };
+            MiriCommand::Nextest(nextest_command)
+        }
         Some("run" | "r") => MiriCommand::Run,
         Some("setup") => MiriCommand::Setup,
         // Invalid command.
         _ =>
             show_error(format!(
-                "`cargo miri` supports the following subcommands: `run`, `test`, and `setup`."
+                "`cargo miri` supports the following subcommands: `run`, `test`, `nextest`, and `setup`."
             )),
     };
     let verbose = has_arg_flag("-v");
 
     // We always setup.
-    setup(subcommand);
+    setup(subcommand.clone());
 
     // Invoke actual cargo for the job, but with different flags.
     // We re-use `cargo test` and `cargo run`, which makes target and binary handling very easy but
@@ -595,13 +608,14 @@ fn phase_cargo_miri(mut args: env::Args) {
     // approach that uses `cargo check`, making that part easier but target and binary handling
     // harder.
     let cargo_miri_path = std::env::current_exe().expect("current executable path invalid");
-    let cargo_cmd = match subcommand {
-        MiriCommand::Test => "test",
-        MiriCommand::Run => "run",
+    let cargo_cmd: Vec<&str> = match &subcommand {
+        MiriCommand::Test => vec!["test"],
+        MiriCommand::Nextest(nextest_command) => vec!["nextest", nextest_command],
+        MiriCommand::Run => vec!["run"],
         MiriCommand::Setup => return, // `cargo miri setup` stops here.
     };
     let mut cmd = cargo();
-    cmd.arg(cargo_cmd);
+    cmd.args(cargo_cmd);
 
     // Make sure we know the build target, and cargo does, too.
     // This is needed to make the `CARGO_TARGET_*_RUNNER` env var do something,

test-cargo-miri/run-test.py

@@ -5,7 +5,7 @@
 and the working directory to contain the cargo-miri-test project.
 '''
 
-import sys, subprocess, os, re
+import shutil, sys, subprocess, os, re, typing
 
 CGREEN  = '\33[32m'
 CBOLD   = '\33[1m'
@@ -23,6 +23,12 @@ def cargo_miri(cmd, quiet = True):
         args += ["--target", os.environ['MIRI_TEST_TARGET']]
     return args
 
+def cargo_miri_nextest(cmd, quiet = True):
+    args = ["cargo", "miri", "nextest", cmd]
+    if 'MIRI_TEST_TARGET' in os.environ:
+        args += ["--target", os.environ['MIRI_TEST_TARGET']]
+    return args
+
 def normalize_stdout(str):
     str = str.replace("src\\", "src/") # normalize paths across platforms
     return re.sub("finished in \d+\.\d\ds", "finished in $TIME", str)
@@ -55,6 +61,35 @@ def test(name, cmd, stdout_ref, stderr_ref, stdin=b'', env={}):
     print("--- END test stderr ---")
     fail("exit code was {}".format(p.returncode))
 
+def test_nextest(name, cmd: typing.List[str], stdin=b'', env={}) -> typing.Tuple[str, str]:
+    print("Testing {}...".format(name))
+
+    ## Call `cargo miri`, capture all output
+    p_env = os.environ.copy()
+    p_env.update(env)
+    p = subprocess.Popen(
+        cmd,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env=p_env,
+    )
+    (stdout, stderr) = p.communicate(input=stdin)
+    stdout = stdout.decode("UTF-8")
+    stderr = stderr.decode("UTF-8")
+
+    if p.returncode == 0:
+        return (stdout, stderr)
+    # Show output
+    print("Nextest did not exit with 0!")
+    print("--- BEGIN test stdout ---")
+    print(stdout, end="")
+    print("--- END test stdout ---")
+    print("--- BEGIN test stderr ---")
+    print(stderr, end="")
+    print("--- END test stderr ---")
+    fail("exit code was {}".format(p.returncode))
+
 def test_no_rebuild(name, cmd, env={}):
     print("Testing {}...".format(name))
     p_env = os.environ.copy()
@@ -159,6 +194,26 @@ def test_cargo_miri_test():
         env={'MIRIFLAGS': "-Zmiri-permissive-provenance"},
     )
 
+def test_cargo_miri_nextest():
+    if shutil.which("cargo-nextest") is None:
+        print("Skipping `cargo miri nextest` (no cargo-nextest)")
+        return
+    # main() in src/main.rs is a custom test harness that doesn't implement the API required by
+    # nextest -- this means that we can't test it. However, all the other tests are regular ones.
+    nextest_filter = "!(package(cargo-miri-test) & binary(main))"
+    # These tests just check that the exit code is 0.
+    # TODO: maybe inspect stdout/stderr, especially for list output?
+    test_nextest("`cargo miri nextest list`",
+        cargo_miri_nextest("list") + ["-E", nextest_filter]
+    )
+    test_nextest("`cargo miri nextest run`",
+        cargo_miri_nextest("run") + ["-E", nextest_filter],
+    )
+    # Running nextest archives is currently not supported.
+    # See https://github.com/nextest-rs/nextest/issues/370 for one issue -- also note that cargo
+    # miri passes in cargo-related options to nextest, which nextest rejects when running tests
+    # from an archive.
+
 os.chdir(os.path.dirname(os.path.realpath(__file__)))
 os.environ["CARGO_TARGET_DIR"] = "target" # this affects the location of the target directory that we need to check
 os.environ["RUST_TEST_NOCAPTURE"] = "0" # this affects test output, so make sure it is not set
@@ -173,9 +228,13 @@ def test_cargo_miri_test():
     subprocess.run(cargo_miri("setup"), check=True)
 test_cargo_miri_run()
 test_cargo_miri_test()
+test_cargo_miri_nextest()
 # Ensure we did not create anything outside the expected target dir.
 for target_dir in ["target", "custom-run", "custom-test", "config-cli"]:
-    if os.listdir(target_dir) != ["miri"]:
+    listdir = os.listdir(target_dir)
+    if "nextest" in listdir:
+        listdir.remove("nextest")
+    if listdir != ["miri"]:
         fail(f"`{target_dir}` contains unexpected files")
     # Ensure something exists inside that target dir.
     os.access(os.path.join(target_dir, "miri", "debug", "deps"), os.F_OK)