Auto merge of #32007 - nikomatsakis:compiletest-incremental, r=alexcrichton

This PR extends compiletest to support **test revisions** and with a preliminary **incremental testing harness**. run-pass, compile-fail, and run-fail tests may be tagged with

```
// revisions: a b c d
```

This will cause the test to be re-run four times with `--cfg {a,b,c,d}` in turn. This means you can write very closely related things using `cfg`. You can also configure the headers/expected-errors by writing `//[foo] header: value` or `//[foo]~ ERROR bar`, where `foo` is the name of your revision. See the changes to `coherence-cow.rs` as a proof of concept.

The main point of this work is to support the incremental testing harness. This PR contains an initial, unused version. The code that uses it will land later. The incremental testing harness compiles each revision in turn, and requires that the revisions have particular names (e.g., `rpass2`, `cfail3`), which tell it whether a particular revision is expected to compile or not.

Two questions:

- Is there compiletest documentation anywhere I can update?
- Should I hold off on landing the incremental testing harness until I have the code to exercise it? (That will come in a separate PR, still fixing a few details)

r? @alexcrichton
cc @rust-lang/compiler <-- new testing capabilities

Author: bors

COMPILER_TESTS.md

@@ -42,3 +42,43 @@ whole, instead of just a few lines inside the test.
 * `ignore-test` always ignores the test
 * `ignore-lldb` and `ignore-gdb` will skip the debuginfo tests
 * `min-{gdb,lldb}-version`
+* `should-fail` indicates that the test should fail; used for "meta testing",
+  where we test the compiletest program itself to check that it will generate
+  errors in appropriate scenarios. This header is ignored for pretty-printer tests.
+
+## Revisions
+
+Certain classes of tests support "revisions" (as of the time of this
+writing, this includes run-pass, compile-fail, run-fail, and
+incremental, though incremental tests are somewhat
+different). Revisions allow a single test file to be used for multiple
+tests. This is done by adding a special header at the top of the file:
+
+```
+// revisions: foo bar baz
+```
+
+This will result in the test being compiled (and tested) three times,
+once with `--cfg foo`, once with `--cfg bar`, and once with `--cfg
+baz`. You can therefore use `#[cfg(foo)]` etc within the test to tweak
+each of these results.
+
+You can also customize headers and expected error messages to a particular
+revision. To do this, add `[foo]` (or `bar`, `baz`, etc) after the `//`
+comment, like so:
+
+```
+// A flag to pass in only for cfg `foo`:
+//[foo]compile-flags: -Z verbose
+
+#[cfg(foo)]
+fn test_foo() {
+    let x: usize = 32_u32; //[foo]~ ERROR mismatched types
+}
+```
+
+Note that not all headers have meaning when customized too a revision.
+For example, the `ignore-test` header (and all "ignore" headers)
+currently only apply to the test as a whole, not to particular
+revisions. The only headers that are intended to really work when
+customized to a revision are error patterns and compiler flags.

src/compiletest/compiletest.rs

@@ -354,11 +354,25 @@ pub fn is_test(config: &Config, testfile: &Path) -> bool {
 }
 
 pub fn make_test(config: &Config, testpaths: &TestPaths) -> test::TestDescAndFn {
+    let early_props = header::early_props(config, &testpaths.file);
+
+    // The `should-fail` annotation doesn't apply to pretty tests,
+    // since we run the pretty printer across all tests by default.
+    // If desired, we could add a `should-fail-pretty` annotation.
+    let should_panic = match config.mode {
+        Pretty => test::ShouldPanic::No,
+        _ => if early_props.should_fail {
+            test::ShouldPanic::Yes
+        } else {
+            test::ShouldPanic::No
+        }
+    };
+
     test::TestDescAndFn {
         desc: test::TestDesc {
             name: make_test_name(config, testpaths),
-            ignore: header::is_test_ignored(config, &testpaths.file),
-            should_panic: test::ShouldPanic::No,
+            ignore: early_props.ignore,
+            should_panic: should_panic,
         },
         testfn: make_test_closure(config, testpaths),
     }

src/compiletest/errors.rs

@@ -30,8 +30,10 @@ enum WhichLine { ThisLine, FollowPrevious(usize), AdjustBackward(usize) }
 /// Goal is to enable tests both like: //~^^^ ERROR go up three
 /// and also //~^ ERROR message one for the preceding line, and
 ///          //~| ERROR message two for that same line.
-// Load any test directives embedded in the file
-pub fn load_errors(testfile: &Path) -> Vec<ExpectedError> {
+///
+/// If cfg is not None (i.e., in an incremental test), then we look
+/// for `//[X]~` instead, where `X` is the current `cfg`.
+pub fn load_errors(testfile: &Path, cfg: Option<&str>) -> Vec<ExpectedError> {
     let rdr = BufReader::new(File::open(testfile).unwrap());
 
     // `last_nonfollow_error` tracks the most recently seen
@@ -44,30 +46,41 @@ pub fn load_errors(testfile: &Path) -> Vec<ExpectedError> {
     // updating it in the map callback below.)
     let mut last_nonfollow_error = None;
 
-    rdr.lines().enumerate().filter_map(|(line_no, ln)| {
-        parse_expected(last_nonfollow_error,
-                       line_no + 1,
-                       &ln.unwrap())
-            .map(|(which, error)| {
-                match which {
-                    FollowPrevious(_) => {}
-                    _ => last_nonfollow_error = Some(error.line),
-                }
-                error
-            })
-    }).collect()
+    let tag = match cfg {
+        Some(rev) => format!("//[{}]~", rev),
+        None => format!("//~")
+    };
+
+    rdr.lines()
+       .enumerate()
+       .filter_map(|(line_no, ln)| {
+           parse_expected(last_nonfollow_error,
+                          line_no + 1,
+                          &ln.unwrap(),
+                          &tag)
+               .map(|(which, error)| {
+                   match which {
+                       FollowPrevious(_) => {}
+                       _ => last_nonfollow_error = Some(error.line),
+                   }
+                   error
+               })
+       })
+       .collect()
 }
 
 fn parse_expected(last_nonfollow_error: Option<usize>,
                   line_num: usize,
-                  line: &str) -> Option<(WhichLine, ExpectedError)> {
-    let start = match line.find("//~") { Some(i) => i, None => return None };
-    let (follow, adjusts) = if line.char_at(start + 3) == '|' {
+                  line: &str,
+                  tag: &str)
+                  -> Option<(WhichLine, ExpectedError)> {
+    let start = match line.find(tag) { Some(i) => i, None => return None };
+    let (follow, adjusts) = if line.char_at(start + tag.len()) == '|' {
         (true, 0)
     } else {
-        (false, line[start + 3..].chars().take_while(|c| *c == '^').count())
+        (false, line[start + tag.len()..].chars().take_while(|c| *c == '^').count())
     };
-    let kind_start = start + 3 + adjusts + (follow as usize);
+    let kind_start = start + tag.len() + adjusts + (follow as usize);
     let letters = line[kind_start..].chars();
     let kind = letters.skip_while(|c| c.is_whitespace())
                       .take_while(|c| !c.is_whitespace())
@@ -91,7 +104,9 @@ fn parse_expected(last_nonfollow_error: Option<usize>,
         (which, line)
     };
 
-    debug!("line={} which={:?} kind={:?} msg={:?}", line_num, which, kind, msg);
+    debug!("line={} tag={:?} which={:?} kind={:?} msg={:?}",
+           line_num, tag, which, kind, msg);
+
     Some((which, ExpectedError { line: line,
                                  kind: kind,
                                  msg: msg, }))