Skip to content

Commit 172d561

Browse files
drodriguezdrodriguez
committed
[test] Split TestFoundation in smaller tests for CTest reporting.
At the moment, only one test was created for CTest, which would have run all the test in TestFoundation, and would have reported the final result as the test result, but without details of which test have failed. Following the similar case of GTest and gtest_discover_tests, XCTest can be used to list all the tests in the suite, and execute them as invidual CTest. They will report as different tests in the output, and individual failures will be shown. It will be also a little bit more resilient against crashes, which will only affect one test, and not the full suite. At first I was thinking in doing the parsing in some scripting language, but in order to be platform independent, and because the parsing is easy enough, the parsing of the XCTest output is done in CMake.
1 parent f62c026 commit 172d561

File tree

3 files changed

+200
-15
lines changed

3 files changed

+200
-15
lines changed

‎CMakeLists.txt

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ find_package(LibXml2 REQUIRED)
3838

3939
include(SwiftSupport)
4040
include(GNUInstallDirs)
41+
include(XCTest)
4142

4243
string(TOLOWER ${CMAKE_SYSTEM_NAME} swift_os)
4344
get_swift_host_arch(swift_arch)
@@ -635,30 +636,24 @@ if(ENABLE_TESTING)
635636
COMMAND
636637
${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_BINARY_DIR}/xdgTestHelper${CMAKE_EXECUTABLE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}/TestFoundation/xdgTestHelper${CMAKE_EXECUTABLE_SUFFIX}
637638
DEPENDS
638-
TestFoundation
639639
xdgTestHelper)
640-
add_test(NAME
641-
TestFoundation
642-
COMMAND
643-
${CMAKE_CURRENT_BINARY_DIR}/TestFoundation/TestFoundation
644-
WORKING_DIRECTORY
645-
${CMAKE_CURRENT_BINARY_DIR}/TestFoundation)
646-
set_tests_properties(TestFoundation
647-
PROPERTIES
648-
ENVIRONMENT
649-
LD_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR}:${FOUNDATION_PATH_TO_XCTEST_BUILD}:${FOUNDATION_PATH_TO_LIBDISPATCH_BUILD}:${FOUNDATION_PATH_TO_LIBDISPATCH_BUILD}/src
650-
DEPENDS
651-
${CMAKE_CURRENT_BINARY_DIR}/TestFoundation/xdgTestHelper${CMAKE_EXECUTABLE_SUFFIX})
652-
653640
add_custom_command(TARGET TestFoundation
654641
POST_BUILD
655642
BYPRODUCTS
656643
${CMAKE_CURRENT_BINARY_DIR}/TestFoundation/plutil${CMAKE_EXECUTABLE_SUFFIX}
657644
COMMAND
658645
${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_BINARY_DIR}/plutil${CMAKE_EXECUTABLE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}/TestFoundation/plutil${CMAKE_EXECUTABLE_SUFFIX}
659646
DEPENDS
660-
TestFoundation
661647
plutil)
648+
649+
xctest_discover_tests(TestFoundation
650+
COMMAND
651+
${CMAKE_CURRENT_BINARY_DIR}/TestFoundation/TestFoundation${CMAKE_EXECUTABLE_PREFIX}
652+
WORKING_DIRECTORY
653+
${CMAKE_CURRENT_BINARY_DIR}/TestFoundation
654+
PROPERTIES
655+
ENVIRONMENT
656+
LD_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR}:${FOUNDATION_PATH_TO_XCTEST_BUILD}:${FOUNDATION_PATH_TO_LIBDISPATCH_BUILD}:${FOUNDATION_PATH_TO_LIBDISPATCH_BUILD}/src)
662657
endif()
663658

664659
if(BUILD_SHARED_LIBS)

‎cmake/modules/XCTest.cmake

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
cmake_policy(PUSH)
2+
cmake_policy(SET CMP0057 NEW)
3+
4+
# Automatically add tests with CTest by querying the compiled test executable
5+
# for available tests.
6+
#
7+
# xctest_discover_tests(target
8+
# [COMMAND command]
9+
# [WORKING_DIRECTORY dir]
10+
# [PROPERTIES name1 value1...]
11+
# [DISCOVERY_TIMEOUT seconds]
12+
# )
13+
#
14+
# `xctest_discover_tests` sets up a post-build command on the test executable
15+
# that generates the list of tests by parsing the output from running the test
16+
# with the `--list-tests` argument.
17+
#
18+
# The options are:
19+
#
20+
# `target`
21+
# Specifies the XCTest executable, which must be a known CMake target. CMake
22+
# will substitute the location of the built executable when running the test.
23+
#
24+
# `COMMAND command`
25+
# Override the command used for the test executable. If you executable is not
26+
# created with CMake add_executable, you will have to provide a command path.
27+
# If this option is not provided, the target file of the target is used.
28+
#
29+
# `WORKING_DIRECTORY dir`
30+
# Specifies the directory in which to run the discovered test cases. If this
31+
# option is not provided, the current binary directory is used.
32+
#
33+
# `PROPERTIES name1 value1...`
34+
# Specifies additional properties to be set on all tests discovered by this
35+
# invocation of `xctest_discover_tests`.
36+
#
37+
# `DISCOVERY_TIMEOUT seconds`
38+
# Specifies how long (in seconds) CMake will wait for the test to enumerate
39+
# available tests. If the test takes longer than this, discovery (and your
40+
# build) will fail. The default is 5 seconds.
41+
#
42+
# The inspiration for this is CMake `gtest_discover_tests`. The official
43+
# documentation might be useful for using this function. Many details of that
44+
# function has been dropped in the name of simplicity, and others have been
45+
# improved.
46+
function(xctest_discover_tests TARGET)
47+
cmake_parse_arguments(
48+
""
49+
""
50+
"COMMAND;WORKING_DIRECTORY;DISCOVERY_TIMEOUT"
51+
"PROPERTIES"
52+
${ARGN}
53+
)
54+
55+
if(NOT _COMMAND)
56+
set(_COMMAND "$<TARGET_FILE:${TARGET}>")
57+
endif()
58+
if(NOT _WORKING_DIRECTORY)
59+
set(_WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
60+
endif()
61+
if(NOT _DISCOVERY_TIMEOUT)
62+
set(_DISCOVERY_TIMEOUT 5)
63+
endif()
64+
65+
set(ctest_file_base ${CMAKE_CURRENT_BINARY_DIR}/${TARGET})
66+
set(ctest_include_file "${ctest_file_base}_include.cmake")
67+
set(ctest_tests_file "${ctest_file_base}_tests.cmake")
68+
69+
add_custom_command(
70+
TARGET ${TARGET} POST_BUILD
71+
BYPRODUCTS "${ctest_tests_file}"
72+
COMMAND "${CMAKE_COMMAND}"
73+
-D "TEST_TARGET=${TARGET}"
74+
-D "TEST_EXECUTABLE=${_COMMAND}"
75+
-D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}"
76+
-D "TEST_PROPERTIES=${_PROPERTIES}"
77+
-D "CTEST_FILE=${ctest_tests_file}"
78+
-D "TEST_DISCOVERY_TIMEOUT=${_DISCOVERY_TIMEOUT}"
79+
-P "${_XCTEST_DISCOVER_TESTS_SCRIPT}"
80+
VERBATIM
81+
)
82+
83+
file(WRITE "${ctest_include_file}"
84+
"if(EXISTS \"${ctest_tests_file}\")\n"
85+
" include(\"${ctest_tests_file}\")\n"
86+
"else()\n"
87+
" add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)\n"
88+
"endif()\n"
89+
)
90+
91+
set_property(DIRECTORY
92+
APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
93+
)
94+
endfunction()
95+
96+
set(_XCTEST_DISCOVER_TESTS_SCRIPT
97+
${CMAKE_CURRENT_LIST_DIR}/XCTestAddTests.cmake
98+
)
99+
100+
cmake_policy(POP)

‎cmake/modules/XCTestAddTests.cmake

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
set(properties ${TEST_PROPERTIES})
2+
set(script)
3+
set(tests)
4+
5+
function(add_command NAME)
6+
set(_args "")
7+
foreach(_arg ${ARGN})
8+
if(_arg MATCHES "[^-./:a-zA-Z0-9_]")
9+
set(_args "${_args} [==[${_arg}]==]")
10+
else()
11+
set(_args "${_args} ${_arg}")
12+
endif()
13+
endforeach()
14+
set(script "${script}${NAME}(${_args})\n" PARENT_SCOPE)
15+
endfunction()
16+
17+
if(NOT EXISTS "${TEST_EXECUTABLE}")
18+
message(FATAL_ERROR
19+
"Specified test executable does not exist.\n"
20+
" Path: '${TEST_EXECUTABLE}'"
21+
)
22+
endif()
23+
# We need to figure out if some environment is needed to run the test listing.
24+
cmake_parse_arguments("_properties" "" "ENVIRONMENT" "" ${properties})
25+
if(_properties_ENVIRONMENT)
26+
foreach(_env ${_properties_ENVIRONMENT})
27+
string(REGEX REPLACE "([a-zA-Z0-9_]+)=(.*)" "\\1" _key "${_env}")
28+
string(REGEX REPLACE "([a-zA-Z0-9_]+)=(.*)" "\\2" _value "${_env}")
29+
if(NOT "${_key}" STREQUAL "")
30+
set(ENV{${_key}} "${_value}")
31+
endif()
32+
endforeach()
33+
endif()
34+
execute_process(
35+
COMMAND "${TEST_EXECUTABLE}" --list-tests
36+
WORKING_DIRECTORY "${TEST_WORKING_DIR}"
37+
TIMEOUT ${TEST_DISCOVERY_TIMEOUT}
38+
OUTPUT_VARIABLE output
39+
ERROR_VARIABLE error_output
40+
RESULT_VARIABLE result
41+
)
42+
if(NOT ${result} EQUAL 0)
43+
string(REPLACE "\n" "\n " output "${output}")
44+
string(REPLACE "\n" "\n " error_output "${error_output}")
45+
message(FATAL_ERROR
46+
"Error running test executable.\n"
47+
" Path: '${TEST_EXECUTABLE}'\n"
48+
" Result: ${result}\n"
49+
" Output:\n"
50+
" ${output}\n"
51+
" Error:\n"
52+
" ${error_output}\n"
53+
)
54+
endif()
55+
56+
string(REPLACE "\n" ";" output "${output}")
57+
58+
foreach(line ${output})
59+
if(line MATCHES "^[ \t]*$")
60+
continue()
61+
elseif(line MATCHES "^Listing [0-9]+ tests? in .+:$")
62+
continue()
63+
elseif(line MATCHES "^.+\\..+/.+$")
64+
# TODO: remove non-ASCII characters from module, class and method names
65+
set(pretty_target "${line}")
66+
string(REGEX REPLACE "/" "-" pretty_target "${pretty_target}")
67+
add_command(add_test
68+
"${pretty_target}"
69+
"${TEST_EXECUTABLE}"
70+
"${line}"
71+
)
72+
add_command(set_tests_properties
73+
"${pretty_target}"
74+
PROPERTIES
75+
WORKING_DIRECTORY "${TEST_WORKING_DIR}"
76+
${properties}
77+
)
78+
list(APPEND tests "${pretty_target}")
79+
else()
80+
message(FATAL_ERROR
81+
"Error parsing test executable output.\n"
82+
" Path: '${TEST_EXECUTABLE}'\n"
83+
" Line: '${line}'"
84+
)
85+
endif()
86+
endforeach()
87+
88+
add_command(set "${TARGET}_TESTS" ${tests})
89+
90+
file(WRITE "${CTEST_FILE}" "${script}")

0 commit comments

Comments
 (0)