Skip to content

Add support for run-pass-valgrind tests #17868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Oct 23, 2014
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 2 additions & 9 deletions configure
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,7 @@ VAL_OPTIONS=""

opt valgrind 0 "run tests with valgrind (memcheck by default)"
opt helgrind 0 "run tests with helgrind instead of memcheck"
opt valgrind-rpass 1 "run rpass-valgrind tests with valgrind"
opt docs 1 "build documentation"
opt optimize 1 "build optimized rust code"
opt optimize-cxx 1 "build optimized C++ code"
Expand Down Expand Up @@ -906,6 +907,7 @@ do
done

make_dir $h/test/run-pass
make_dir $h/test/run-pass-valgrind
make_dir $h/test/run-pass-fulldeps
make_dir $h/test/run-fail
make_dir $h/test/compile-fail
Expand Down Expand Up @@ -1235,15 +1237,6 @@ then
putvar CFG_PANDOC
fi

# Valgrind is only reliable on Linux. On Windows it doesn't work at all, and
# on the Mac the dynamic linker causes Valgrind to emit a huge stream of
# errors.
if [ $CFG_OSTYPE != unknown-linux-gnu ] && [ $CFG_OSTYPE != apple-darwin ]
then
CFG_BAD_VALGRIND=1
putvar CFG_BAD_VALGRIND
fi

putvar CFG_LLVM_ROOT
putvar CFG_LLVM_SRC_DIR

Expand Down
14 changes: 9 additions & 5 deletions mk/main.mk
Original file line number Diff line number Diff line change
Expand Up @@ -174,16 +174,20 @@ else
CFG_VALGRIND_COMPILE :=
endif


ifndef CFG_DISABLE_VALGRIND_RPASS
$(info cfg: enabling valgrind run-pass tests (CFG_ENABLE_VALGRIND_RPASS))
CFG_VALGRIND_RPASS :=$(CFG_VALGRIND)
else
CFG_VALGRIND_RPASS :=
endif


ifdef CFG_ENABLE_VALGRIND
$(info cfg: enabling valgrind (CFG_ENABLE_VALGRIND))
else
CFG_VALGRIND :=
endif
ifdef CFG_BAD_VALGRIND
$(info cfg: disabling valgrind due to its unreliability on this platform)
CFG_VALGRIND :=
endif


######################################################################
# Target-and-rule "utility variables"
Expand Down
14 changes: 14 additions & 0 deletions mk/platform.mk
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,20 @@ ifdef CFG_VALGRIND
endif
endif

# If we actually want to run Valgrind on a given platform, set this variable
define DEF_GOOD_VALGRIND
ifeq ($(OSTYPE_$(1)),unknown-linux-gnu)
GOOD_VALGRIND_$(1) = 1
endif
ifneq (,$(filter $(OSTYPE_$(1)),darwin freebsd))
ifeq (HOST_$(1),x86_64)
GOOD_VALGRIND_$(1) = 1
endif
endif
endef
$(foreach t,$(CFG_TARGET),$(eval $(call DEF_GOOD_VALGRIND,$(t))))
$(foreach t,$(CFG_TARGET),$(info cfg: good valgrind for $(t) is $(GOOD_VALGRIND_$(t))))

ifneq ($(findstring linux,$(CFG_OSTYPE)),)
ifdef CFG_PERF
ifneq ($(CFG_PERF_WITH_LOGFD),)
Expand Down
57 changes: 40 additions & 17 deletions mk/tests.mk
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2012 The Rust Project Developers. See the COPYRIGHT
# Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
Expand Down Expand Up @@ -184,12 +184,12 @@ check-notidy: cleantmptestlogs cleantestlibs all check-stage2
# A slightly smaller set of tests for smoke testing.
check-lite: cleantestlibs cleantmptestlogs \
$(foreach crate,$(TEST_TARGET_CRATES),check-stage2-$(crate)) \
check-stage2-rpass \
check-stage2-rpass check-stage2-rpass-valgrind \
check-stage2-rfail check-stage2-cfail check-stage2-rmake
$(Q)$(CFG_PYTHON) $(S)src/etc/check-summary.py tmp/*.log

# Only check the 'reference' tests: rpass/cfail/rfail/rmake.
check-ref: cleantestlibs cleantmptestlogs check-stage2-rpass \
check-ref: cleantestlibs cleantmptestlogs check-stage2-rpass check-stage2-rpass-valgrind \
check-stage2-rfail check-stage2-cfail check-stage2-rmake
$(Q)$(CFG_PYTHON) $(S)src/etc/check-summary.py tmp/*.log

Expand Down Expand Up @@ -326,7 +326,8 @@ check-stage$(1)-T-$(2)-H-$(3)-exec: \
check-stage$(1)-T-$(2)-H-$(3)-rpass-exec \
check-stage$(1)-T-$(2)-H-$(3)-rfail-exec \
check-stage$(1)-T-$(2)-H-$(3)-cfail-exec \
check-stage$(1)-T-$(2)-H-$(3)-rpass-full-exec \
check-stage$(1)-T-$(2)-H-$(3)-rpass-valgrind-exec \
check-stage$(1)-T-$(2)-H-$(3)-rpass-full-exec \
check-stage$(1)-T-$(2)-H-$(3)-cfail-full-exec \
check-stage$(1)-T-$(2)-H-$(3)-rmake-exec \
check-stage$(1)-T-$(2)-H-$(3)-crates-exec \
Expand Down Expand Up @@ -364,6 +365,7 @@ check-stage$(1)-T-$(2)-H-$(3)-doc-exec: \

check-stage$(1)-T-$(2)-H-$(3)-pretty-exec: \
check-stage$(1)-T-$(2)-H-$(3)-pretty-rpass-exec \
check-stage$(1)-T-$(2)-H-$(3)-pretty-rpass-valgrind-exec \
check-stage$(1)-T-$(2)-H-$(3)-pretty-rpass-full-exec \
check-stage$(1)-T-$(2)-H-$(3)-pretty-rfail-exec \
check-stage$(1)-T-$(2)-H-$(3)-pretty-bench-exec \
Expand Down Expand Up @@ -489,15 +491,11 @@ $(foreach host,$(CFG_HOST), \
# Rules for the compiletest tests (rpass, rfail, etc.)
######################################################################

RPASS_RC := $(wildcard $(S)src/test/run-pass/*.rc)
RPASS_RS := $(wildcard $(S)src/test/run-pass/*.rs)
RPASS_FULL_RC := $(wildcard $(S)src/test/run-pass-fulldeps/*.rc)
RPASS_VALGRIND_RS := $(wildcard $(S)src/test/run-pass-valgrind/*.rs)
RPASS_FULL_RS := $(wildcard $(S)src/test/run-pass-fulldeps/*.rs)
CFAIL_FULL_RC := $(wildcard $(S)src/test/compile-fail-fulldeps/*.rc)
CFAIL_FULL_RS := $(wildcard $(S)src/test/compile-fail-fulldeps/*.rs)
RFAIL_RC := $(wildcard $(S)src/test/run-fail/*.rc)
RFAIL_RS := $(wildcard $(S)src/test/run-fail/*.rs)
CFAIL_RC := $(wildcard $(S)src/test/compile-fail/*.rc)
CFAIL_RS := $(wildcard $(S)src/test/compile-fail/*.rs)
BENCH_RS := $(wildcard $(S)src/test/bench/*.rs)
PRETTY_RS := $(wildcard $(S)src/test/pretty/*.rs)
Expand All @@ -510,11 +508,12 @@ CODEGEN_CC := $(wildcard $(S)src/test/codegen/*.cc)
# a performance monitor.
PERF_RS := $(wildcard $(S)src/test/bench/*.rs)

RPASS_TESTS := $(RPASS_RC) $(RPASS_RS)
RPASS_FULL_TESTS := $(RPASS_FULL_RC) $(RPASS_FULL_RS)
CFAIL_FULL_TESTS := $(CFAIL_FULL_RC) $(CFAIL_FULL_RS)
RFAIL_TESTS := $(RFAIL_RC) $(RFAIL_RS)
CFAIL_TESTS := $(CFAIL_RC) $(CFAIL_RS)
RPASS_TESTS := $(RPASS_RS)
RPASS_VALGRIND_TESTS := $(RPASS_VALGRIND_RS)
RPASS_FULL_TESTS := $(RPASS_FULL_RS)
CFAIL_FULL_TESTS := $(CFAIL_FULL_RS)
RFAIL_TESTS := $(RFAIL_RS)
CFAIL_TESTS := $(CFAIL_RS)
BENCH_TESTS := $(BENCH_RS)
PERF_TESTS := $(PERF_RS)
PRETTY_TESTS := $(PRETTY_RS)
Expand All @@ -527,6 +526,11 @@ CTEST_BUILD_BASE_rpass = run-pass
CTEST_MODE_rpass = run-pass
CTEST_RUNTOOL_rpass = $(CTEST_RUNTOOL)

CTEST_SRC_BASE_rpass-valgrind = run-pass-valgrind
CTEST_BUILD_BASE_rpass-valgrind = run-pass-valgrind
CTEST_MODE_rpass-valgrind = run-pass-valgrind
CTEST_RUNTOOL_rpass-valgrind = $(CTEST_RUNTOOL)

CTEST_SRC_BASE_rpass-full = run-pass-fulldeps
CTEST_BUILD_BASE_rpass-full = run-pass-fulldeps
CTEST_MODE_rpass-full = run-pass
Expand Down Expand Up @@ -622,7 +626,7 @@ TEST_SREQ$(1)_T_$(2)_H_$(3) = \
# remove directive, if present, from CFG_RUSTC_FLAGS (issue #7898).
CTEST_RUSTC_FLAGS := $$(subst --cfg ndebug,,$$(CFG_RUSTC_FLAGS))

# The tests can not be optimized while the rest of the compiler is optimized, so
# The tests cannot be optimized while the rest of the compiler is optimized, so
# filter out the optimization (if any) from rustc and then figure out if we need
# to be optimized
CTEST_RUSTC_FLAGS := $$(subst -O,,$$(CTEST_RUSTC_FLAGS))
Expand All @@ -634,6 +638,7 @@ endif
# slow things down.
CTEST_RUSTC_FLAGS += -C codegen-units=1


CTEST_COMMON_ARGS$(1)-T-$(2)-H-$(3) := \
--compile-lib-path $$(HLIB$(1)_H_$(3)) \
--run-lib-path $$(TLIB$(1)_T_$(2)_H_$(3)) \
Expand All @@ -654,7 +659,21 @@ CTEST_COMMON_ARGS$(1)-T-$(2)-H-$(3) := \
--target-rustcflags "$(RUSTC_FLAGS_$(2)) $$(CTEST_RUSTC_FLAGS) -L $$(RT_OUTPUT_DIR_$(2))" \
$$(CTEST_TESTARGS)

ifdef CFG_VALGRIND_RPASS
ifdef GOOD_VALGRIND_$(2)
$(info cfg: valgrind-path set to $(CFG_VALGRIND_RPASS))
CTEST_COMMON_ARGS$(1)-T-$(2)-H-$(3) += --valgrind-path "$(CFG_VALGRIND_RPASS)"
endif
endif

ifndef CFG_DISABLE_VALGRIND_RPASS
ifdef GOOD_VALGRIND_$(2)
CTEST_COMMON_ARGS$(1)-T-$(2)-H-$(3) += --force-valgrind
endif
endif

CTEST_DEPS_rpass_$(1)-T-$(2)-H-$(3) = $$(RPASS_TESTS)
CTEST_DEPS_rpass-valgrind_$(1)-T-$(2)-H-$(3) = $$(RPASS_VALGRIND_TESTS)
CTEST_DEPS_rpass-full_$(1)-T-$(2)-H-$(3) = $$(RPASS_FULL_TESTS) $$(CSREQ$(1)_T_$(3)_H_$(3)) $$(SREQ$(1)_T_$(2)_H_$(3))
CTEST_DEPS_cfail-full_$(1)-T-$(2)-H-$(3) = $$(CFAIL_FULL_TESTS) $$(CSREQ$(1)_T_$(3)_H_$(3)) $$(SREQ$(1)_T_$(2)_H_$(3))
CTEST_DEPS_rfail_$(1)-T-$(2)-H-$(3) = $$(RFAIL_TESTS)
Expand Down Expand Up @@ -726,16 +745,17 @@ endif

endef

CTEST_NAMES = rpass rpass-full cfail-full rfail cfail bench perf debuginfo-gdb debuginfo-lldb codegen
CTEST_NAMES = rpass rpass-valgrind rpass-full cfail-full rfail cfail bench perf debuginfo-gdb debuginfo-lldb codegen

$(foreach host,$(CFG_HOST), \
$(eval $(foreach target,$(CFG_TARGET), \
$(eval $(foreach stage,$(STAGES), \
$(eval $(foreach name,$(CTEST_NAMES), \
$(eval $(call DEF_RUN_COMPILETEST,$(stage),$(target),$(host),$(name))))))))))

PRETTY_NAMES = pretty-rpass pretty-rpass-full pretty-rfail pretty-bench pretty-pretty
PRETTY_NAMES = pretty-rpass pretty-rpass-valgrind pretty-rpass-full pretty-rfail pretty-bench pretty-pretty
PRETTY_DEPS_pretty-rpass = $(RPASS_TESTS)
PRETTY_DEPS_pretty-rpass-valgrind = $(RPASS_VALGRIND_TESTS)
PRETTY_DEPS_pretty-rpass-full = $(RPASS_FULL_TESTS)
PRETTY_DEPS_pretty-rfail = $(RFAIL_TESTS)
PRETTY_DEPS_pretty-bench = $(BENCH_TESTS)
Expand All @@ -748,6 +768,7 @@ PRETTY_DEPS$(1)_H_$(3)_pretty-rfail =
PRETTY_DEPS$(1)_H_$(3)_pretty-bench =
PRETTY_DEPS$(1)_H_$(3)_pretty-pretty =
PRETTY_DIRNAME_pretty-rpass = run-pass
PRETTY_DIRNAME_pretty-rpass-valgrind = run-pass-valgrind
PRETTY_DIRNAME_pretty-rpass-full = run-pass-fulldeps
PRETTY_DIRNAME_pretty-rfail = run-fail
PRETTY_DIRNAME_pretty-bench = bench
Expand Down Expand Up @@ -895,6 +916,7 @@ TEST_GROUPS = \
$(foreach crate,$(TEST_CRATES),$(crate)) \
$(foreach crate,$(TEST_DOC_CRATES),doc-crate-$(crate)) \
rpass \
rpass-valgrind \
rpass-full \
cfail-full \
rfail \
Expand All @@ -909,6 +931,7 @@ TEST_GROUPS = \
$(foreach docname,$(DOCS),doc-$(docname)) \
pretty \
pretty-rpass \
pretty-rpass-valgrind \
pretty-rpass-full \
pretty-rfail \
pretty-bench \
Expand Down
10 changes: 10 additions & 0 deletions src/compiletest/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ pub enum Mode {
CompileFail,
RunFail,
RunPass,
RunPassValgrind,
Pretty,
DebugInfoGdb,
DebugInfoLldb,
Expand All @@ -29,6 +30,7 @@ impl FromStr for Mode {
"compile-fail" => Some(CompileFail),
"run-fail" => Some(RunFail),
"run-pass" => Some(RunPass),
"run-pass-valgrind" => Some(RunPassValgrind),
"pretty" => Some(Pretty),
"debuginfo-lldb" => Some(DebugInfoLldb),
"debuginfo-gdb" => Some(DebugInfoGdb),
Expand All @@ -44,6 +46,7 @@ impl fmt::Show for Mode {
CompileFail => "compile-fail",
RunFail => "run-fail",
RunPass => "run-pass",
RunPassValgrind => "run-pass-valgrind",
Pretty => "pretty",
DebugInfoGdb => "debuginfo-gdb",
DebugInfoLldb => "debuginfo-lldb",
Expand All @@ -70,6 +73,13 @@ pub struct Config {
// The llvm binaries path
pub llvm_bin_path: Option<Path>,

// The valgrind path
pub valgrind_path: Option<String>,

// Whether to fail if we can't run run-pass-valgrind tests under valgrind
// (or, alternatively, to silently run them like regular run-pass tests).
pub force_valgrind: bool,

// The directory containing the tests to run
pub src_base: Path,

Expand Down
13 changes: 11 additions & 2 deletions src/compiletest/compiletest.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@ pub mod errors;
pub fn main() {
let args = os::args();
let config = parse_config(args);

if config.valgrind_path.is_none() && config.force_valgrind {
fail!("Can't find Valgrind to run Valgrind tests");
}

log_config(&config);
run_tests(&config);
}
Expand All @@ -50,13 +55,15 @@ pub fn parse_config(args: Vec<String> ) -> Config {
reqopt("", "run-lib-path", "path to target shared libraries", "PATH"),
reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH"),
optopt("", "clang-path", "path to executable for codegen tests", "PATH"),
optopt("", "valgrind-path", "path to Valgrind executable for Valgrind tests", "PROGRAM"),
optflag("", "force-valgrind", "fail if Valgrind tests cannot be run under Valgrind"),
optopt("", "llvm-bin-path", "path to directory holding llvm binaries", "DIR"),
reqopt("", "src-base", "directory to scan for test files", "PATH"),
reqopt("", "build-base", "directory to deposit test outputs", "PATH"),
reqopt("", "aux-base", "directory to find auxiliary test files", "PATH"),
reqopt("", "stage-id", "the target-stage identifier", "stageN-TARGET"),
reqopt("", "mode", "which sort of compile tests to run",
"(compile-fail|run-fail|run-pass|pretty|debug-info)"),
"(compile-fail|run-fail|run-pass|run-pass-valgrind|pretty|debug-info)"),
optflag("", "ignored", "run tests marked as ignored"),
optopt("", "runtool", "supervisor program to run tests under \
(eg. emulator, valgrind)", "PROGRAM"),
Expand Down Expand Up @@ -125,6 +132,8 @@ pub fn parse_config(args: Vec<String> ) -> Config {
run_lib_path: matches.opt_str("run-lib-path").unwrap(),
rustc_path: opt_path(matches, "rustc-path"),
clang_path: matches.opt_str("clang-path").map(|s| Path::new(s)),
valgrind_path: matches.opt_str("valgrind-path"),
force_valgrind: matches.opt_present("force-valgrind"),
llvm_bin_path: matches.opt_str("llvm-bin-path").map(|s| Path::new(s)),
src_base: opt_path(matches, "src-base"),
build_base: opt_path(matches, "build-base"),
Expand Down Expand Up @@ -162,7 +171,7 @@ pub fn parse_config(args: Vec<String> ) -> Config {
!opt_str2(matches.opt_str("adb-test-dir")).is_empty(),
lldb_python_dir: matches.opt_str("lldb-python-dir"),
test_shard: test::opt_shard(matches.opt_str("test-shard")),
verbose: matches.opt_present("verbose")
verbose: matches.opt_present("verbose"),
}
}

Expand Down
25 changes: 23 additions & 2 deletions src/compiletest/runtest.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
// except according to those terms.

use common::Config;
use common::{CompileFail, Pretty, RunFail, RunPass, DebugInfoGdb};
use common::{CompileFail, Pretty, RunFail, RunPass, RunPassValgrind, DebugInfoGdb};
use common::{Codegen, DebugInfoLldb};
use errors;
use header::TestProps;
Expand All @@ -35,7 +35,6 @@ use std::time::Duration;
use test::MetricMap;

pub fn run(config: Config, testfile: String) {

match config.target.as_slice() {

"arm-linux-androideabi" => {
Expand Down Expand Up @@ -64,6 +63,7 @@ pub fn run_metrics(config: Config, testfile: String, mm: &mut MetricMap) {
CompileFail => run_cfail_test(&config, &props, &testfile),
RunFail => run_rfail_test(&config, &props, &testfile),
RunPass => run_rpass_test(&config, &props, &testfile),
RunPassValgrind => run_valgrind_test(&config, &props, &testfile),
Pretty => run_pretty_test(&config, &props, &testfile),
DebugInfoGdb => run_debuginfo_gdb_test(&config, &props, &testfile),
DebugInfoLldb => run_debuginfo_lldb_test(&config, &props, &testfile),
Expand Down Expand Up @@ -164,6 +164,27 @@ fn run_rpass_test(config: &Config, props: &TestProps, testfile: &Path) {
}
}

fn run_valgrind_test(config: &Config, props: &TestProps, testfile: &Path) {
if config.valgrind_path.is_none() {
assert!(!config.force_valgrind);
return run_rpass_test(config, props, testfile);
}

let mut proc_res = compile_test(config, props, testfile);

if !proc_res.status.success() {
fatal_proc_rec("compilation failed!", &proc_res);
}

let mut new_config = config.clone();
new_config.runtool = new_config.valgrind_path.clone();
proc_res = exec_compiled_test(&new_config, props, testfile);

if !proc_res.status.success() {
fatal_proc_rec("test run failed!", &proc_res);
}
}

fn run_pretty_test(config: &Config, props: &TestProps, testfile: &Path) {
if props.pp_exact.is_some() {
logv(config, "testing for exact pretty-printing".to_string());
Expand Down
2 changes: 1 addition & 1 deletion src/libgetopts/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -804,7 +804,7 @@ pub fn short_usage(program_name: &str, opts: &[OptGroup]) -> String {
/// whitespace removed, and are only cut at whitespace boundaries.
///
/// Note: Function was moved here from `std::str` because this module is the only place that
/// uses it, and because it was to specific for a general string function.
/// uses it, and because it was too specific for a general string function.
///
/// #Failure:
///
Expand Down