-
Notifications
You must be signed in to change notification settings - Fork 13.7k
Description
From what I can tell, ui tests that don't have any //~ ERROR annotations and have no stderr file are implicitly assumed to require successful compilation. We ought to reject such tests, and instead require an explicit // must-compile-succesfully comment.
Here are some mentoring instructions. The file that controls the test suite is runtest.rs. UI tests in particular are controlled by run_ui_test():
rust/src/tools/compiletest/src/runtest.rs
Lines 2381 to 2448 in ab79caa
| fn run_ui_test(&self) { | |
| // if the user specified a format in the ui test | |
| // print the output to the stderr file, otherwise extract | |
| // the rendered error messages from json and print them | |
| let explicit = self.props | |
| .compile_flags | |
| .iter() | |
| .any(|s| s.contains("--error-format")); | |
| let proc_res = self.compile_test(); | |
| let expected_stderr_path = self.expected_output_path(UI_STDERR); | |
| let expected_stderr = self.load_expected_output(&expected_stderr_path); | |
| let expected_stdout_path = self.expected_output_path(UI_STDOUT); | |
| let expected_stdout = self.load_expected_output(&expected_stdout_path); | |
| let normalized_stdout = | |
| self.normalize_output(&proc_res.stdout, &self.props.normalize_stdout); | |
| let stderr = if explicit { | |
| proc_res.stderr.clone() | |
| } else { | |
| json::extract_rendered(&proc_res.stderr, &proc_res) | |
| }; | |
| let normalized_stderr = self.normalize_output(&stderr, &self.props.normalize_stderr); | |
| let mut errors = 0; | |
| errors += self.compare_output("stdout", &normalized_stdout, &expected_stdout); | |
| errors += self.compare_output("stderr", &normalized_stderr, &expected_stderr); | |
| if errors > 0 { | |
| println!("To update references, run this command from build directory:"); | |
| let relative_path_to_file = self.testpaths | |
| .relative_dir | |
| .join(self.testpaths.file.file_name().unwrap()); | |
| println!( | |
| "{}/update-references.sh '{}' '{}'", | |
| self.config.src_base.display(), | |
| self.config.build_base.display(), | |
| relative_path_to_file.display() | |
| ); | |
| self.fatal_proc_rec( | |
| &format!("{} errors occurred comparing output.", errors), | |
| &proc_res, | |
| ); | |
| } | |
| let expected_errors = errors::load_errors(&self.testpaths.file, self.revision); | |
| if self.props.run_pass { | |
| let proc_res = self.exec_compiled_test(); | |
| if !proc_res.status.success() { | |
| self.fatal_proc_rec("test run failed!", &proc_res); | |
| } | |
| } | |
| if !explicit { | |
| if !expected_errors.is_empty() || !proc_res.status.success() { | |
| // "// error-pattern" comments | |
| self.check_expected_errors(expected_errors, &proc_res); | |
| } else if !self.props.error_patterns.is_empty() || !proc_res.status.success() { | |
| // "//~ERROR comments" | |
| self.check_error_patterns(&proc_res.stderr, &proc_res); | |
| } | |
| } | |
| } |
You can see that it begins with a call to compile_test, but it does not inspect the return value from that function. In contrast, compile-fail tests check and -- if compilation is successful -- they require that a must_compile_successfully comment is present:
rust/src/tools/compiletest/src/runtest.rs
Lines 153 to 163 in ab79caa
| if self.props.must_compile_successfully { | |
| if !proc_res.status.success() { | |
| self.fatal_proc_rec("test compilation failed although it shouldn't!", &proc_res); | |
| } | |
| } else { | |
| if proc_res.status.success() { | |
| self.fatal_proc_rec( | |
| &format!("{} test compiled successfully!", self.config.mode)[..], | |
| &proc_res, | |
| ); | |
| } |
We want to do something similar, but in the run_ui_test function. This will likely reveal existing tests that need a // must-compile-successfully comment.