Auto merge of #17105 - kpreid:nocapture, r=Veykril

Make test harness arguments configurable and not `--nocapture`.

* Added config `runnables.extraTestBinaryArgs` to control the args.
* The default is `--show-output` rather than `--nocapture` to prevent unreadable output when 2 or more tests fail or print output at once.
* Renamed variables in `CargoTargetSpec::runnable_args()` for clarity.

Fixes #12737.

Suggested changelog info:

> New Features
>
> * add `rust-analyzer.runnables.extraTestBinaryArgs` to configure test harness options when running tests; replaces a previously hard-coded `--nocapture` option.

I'm not sure I made the right choices in vocabulary, between “binary”, “executable”, and “runnable”, and “launch” vs. “execute”, so let me know if I missed something to be consistent with in the naming and documentation.

Author: bors

crates/rust-analyzer/src/cargo_target_spec.rs

@@ -33,53 +33,55 @@ impl CargoTargetSpec {
         kind: &RunnableKind,
         cfg: &Option<CfgExpr>,
     ) -> (Vec<String>, Vec<String>) {
-        let mut args = Vec::new();
-        let mut extra_args = Vec::new();
+        let extra_test_binary_args = snap.config.runnables().extra_test_binary_args;
+
+        let mut cargo_args = Vec::new();
+        let mut executable_args = Vec::new();
 
         match kind {
             RunnableKind::Test { test_id, attr } => {
-                args.push("test".to_owned());
-                extra_args.push(test_id.to_string());
+                cargo_args.push("test".to_owned());
+                executable_args.push(test_id.to_string());
                 if let TestId::Path(_) = test_id {
-                    extra_args.push("--exact".to_owned());
+                    executable_args.push("--exact".to_owned());
                 }
-                extra_args.push("--nocapture".to_owned());
+                executable_args.extend(extra_test_binary_args);
                 if attr.ignore {
-                    extra_args.push("--ignored".to_owned());
+                    executable_args.push("--ignored".to_owned());
                 }
             }
             RunnableKind::TestMod { path } => {
-                args.push("test".to_owned());
-                extra_args.push(path.clone());
-                extra_args.push("--nocapture".to_owned());
+                cargo_args.push("test".to_owned());
+                executable_args.push(path.clone());
+                executable_args.extend(extra_test_binary_args);
             }
             RunnableKind::Bench { test_id } => {
-                args.push("bench".to_owned());
-                extra_args.push(test_id.to_string());
+                cargo_args.push("bench".to_owned());
+                executable_args.push(test_id.to_string());
                 if let TestId::Path(_) = test_id {
-                    extra_args.push("--exact".to_owned());
+                    executable_args.push("--exact".to_owned());
                 }
-                extra_args.push("--nocapture".to_owned());
+                executable_args.extend(extra_test_binary_args);
             }
             RunnableKind::DocTest { test_id } => {
-                args.push("test".to_owned());
-                args.push("--doc".to_owned());
-                extra_args.push(test_id.to_string());
-                extra_args.push("--nocapture".to_owned());
+                cargo_args.push("test".to_owned());
+                cargo_args.push("--doc".to_owned());
+                executable_args.push(test_id.to_string());
+                executable_args.extend(extra_test_binary_args);
             }
             RunnableKind::Bin => {
                 let subcommand = match spec {
                     Some(CargoTargetSpec { target_kind: TargetKind::Test, .. }) => "test",
                     _ => "run",
                 };
-                args.push(subcommand.to_owned());
+                cargo_args.push(subcommand.to_owned());
             }
         }
 
         let (allowed_features, target_required_features) = if let Some(mut spec) = spec {
             let allowed_features = mem::take(&mut spec.features);
             let required_features = mem::take(&mut spec.required_features);
-            spec.push_to(&mut args, kind);
+            spec.push_to(&mut cargo_args, kind);
             (allowed_features, required_features)
         } else {
             (Default::default(), Default::default())
@@ -89,10 +91,10 @@ impl CargoTargetSpec {
 
         match &cargo_config.features {
             CargoFeatures::All => {
-                args.push("--all-features".to_owned());
+                cargo_args.push("--all-features".to_owned());
                 for feature in target_required_features {
-                    args.push("--features".to_owned());
-                    args.push(feature);
+                    cargo_args.push("--features".to_owned());
+                    cargo_args.push(feature);
                 }
             }
             CargoFeatures::Selected { features, no_default_features } => {
@@ -108,16 +110,16 @@ impl CargoTargetSpec {
 
                 feats.dedup();
                 for feature in feats {
-                    args.push("--features".to_owned());
-                    args.push(feature);
+                    cargo_args.push("--features".to_owned());
+                    cargo_args.push(feature);
                 }
 
                 if *no_default_features {
-                    args.push("--no-default-features".to_owned());
+                    cargo_args.push("--no-default-features".to_owned());
                 }
             }
         }
-        (args, extra_args)
+        (cargo_args, executable_args)
     }
 
     pub(crate) fn for_file(

crates/rust-analyzer/src/config.rs

@@ -392,6 +392,14 @@ config_data! {
         /// Additional arguments to be passed to cargo for runnables such as
         /// tests or binaries. For example, it may be `--release`.
         runnables_extraArgs: Vec<String>   = vec![],
+        /// Additional arguments to be passed through Cargo to launched tests, benchmarks, or
+        /// doc-tests.
+        ///
+        /// Unless the launched target uses a
+        /// [custom test harness](https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-harness-field),
+        /// they will end up being interpreted as options to
+        /// [`rustc`’s built-in test harness (“libtest”)](https://doc.rust-lang.org/rustc/tests/index.html#cli-arguments).
+        runnables_extraTestBinaryArgs: Vec<String> = vec!["--show-output".to_owned()],
 
         /// Path to the Cargo.toml of the rust compiler workspace, for usage in rustc_private
         /// projects, or "discover" to try to automatically find it if the `rustc-dev` component
@@ -823,6 +831,8 @@ pub struct RunnablesConfig {
     pub override_cargo: Option<String>,
     /// Additional arguments for the `cargo`, e.g. `--release`.
     pub cargo_extra_args: Vec<String>,
+    /// Additional arguments for the binary being run, if it is a test or benchmark.
+    pub extra_test_binary_args: Vec<String>,
 }
 
 /// Configuration for workspace symbol search requests.
@@ -1749,6 +1759,7 @@ impl Config {
         RunnablesConfig {
             override_cargo: self.runnables_command().clone(),
             cargo_extra_args: self.runnables_extraArgs().clone(),
+            extra_test_binary_args: self.runnables_extraTestBinaryArgs().clone(),
         }
     }
 

crates/rust-analyzer/tests/slow-tests/main.rs

@@ -157,7 +157,7 @@ fn main() {}
           {
             "args": {
               "cargoArgs": ["test", "--package", "foo", "--test", "spam"],
-              "executableArgs": ["test_eggs", "--exact", "--nocapture"],
+              "executableArgs": ["test_eggs", "--exact", "--show-output"],
               "cargoExtraArgs": [],
               "overrideCargo": null,
               "workspaceRoot": server.path().join("foo")
@@ -190,7 +190,7 @@ fn main() {}
               "cargoExtraArgs": [],
               "executableArgs": [
                 "",
-                "--nocapture"
+                "--show-output"
               ]
             },
             "kind": "cargo",

docs/user/generated_config.adoc

@@ -851,6 +851,24 @@ Command to be executed instead of 'cargo' for runnables.
 --
 Additional arguments to be passed to cargo for runnables such as
 tests or binaries. For example, it may be `--release`.
+--
+[[rust-analyzer.runnables.extraTestBinaryArgs]]rust-analyzer.runnables.extraTestBinaryArgs::
++
+--
+Default:
+----
+[
+  "--show-output"
+]
+----
+Additional arguments to be passed through Cargo to launched tests, benchmarks, or
+doc-tests.
+
+Unless the launched target uses a
+[custom test harness](https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-harness-field),
+they will end up being interpreted as options to
+[`rustc`’s built-in test harness (“libtest”)](https://doc.rust-lang.org/rustc/tests/index.html#cli-arguments).
+
 --
 [[rust-analyzer.rustc.source]]rust-analyzer.rustc.source (default: `null`)::
 +

editors/code/package.json

@@ -1594,6 +1594,16 @@
                         "type": "string"
                     }
                 },
+                "rust-analyzer.runnables.extraTestBinaryArgs": {
+                    "markdownDescription": "Additional arguments to be passed through Cargo to launched tests, benchmarks, or\ndoc-tests.\n\nUnless the launched target uses a\n[custom test harness](https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-harness-field),\nthey will end up being interpreted as options to\n[`rustc`’s built-in test harness (“libtest”)](https://doc.rust-lang.org/rustc/tests/index.html#cli-arguments).",
+                    "default": [
+                        "--show-output"
+                    ],
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
                 "rust-analyzer.rustc.source": {
                     "markdownDescription": "Path to the Cargo.toml of the rust compiler workspace, for usage in rustc_private\nprojects, or \"discover\" to try to automatically find it if the `rustc-dev` component\nis installed.\n\nAny project which uses rust-analyzer with the rustcPrivate\ncrates must set `[package.metadata.rust-analyzer] rustc_private=true` to use it.\n\nThis option does not take effect until rust-analyzer is restarted.",
                     "default": null,