Skip to content

[test] Split TestFoundation in smaller tests for CTest reporting. #2521

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 15, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ find_package(dispatch CONFIG REQUIRED)

include(SwiftSupport)
include(GNUInstallDirs)
include(XCTest)

set(CF_DEPLOYMENT_SWIFT YES CACHE BOOL "Build for Swift" FORCE)

Expand Down
12 changes: 7 additions & 5 deletions TestFoundation/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -168,9 +168,11 @@ add_custom_command(TARGET TestFoundation POST_BUILD
COMMAND
${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:FoundationXML> ${CMAKE_BINARY_DIR}/TestFoundation.app)

add_test(NAME TestFoundation
COMMAND ${CMAKE_BINARY_DIR}/TestFoundation.app/TestFoundation
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/TestFoundation.app)
set_tests_properties(TestFoundation PROPERTIES
ENVIRONMENT LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/TestFoundation.app:$<TARGET_LINKER_FILE_DIR:XCTest>:$<TARGET_LINKER_FILE_DIR:dispatch>:$<TARGET_LINKER_FILE_DIR:swiftDispatch>:$<TARGET_LINKER_FILE_DIR:BlocksRuntime>)
xctest_discover_tests(TestFoundation
COMMAND
${CMAKE_BINARY_DIR}/TestFoundation.app/TestFoundation${CMAKE_EXECUTABLE_SUFFIX}
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/TestFoundation.app
PROPERTIES
ENVIRONMENT
LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/TestFoundation.app:$<TARGET_LINKER_FILE_DIR:XCTest>:$<TARGET_LINKER_FILE_DIR:dispatch>:$<TARGET_LINKER_FILE_DIR:swiftDispatch>:$<TARGET_LINKER_FILE_DIR:BlocksRuntime>)

100 changes: 100 additions & 0 deletions cmake/modules/XCTest.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
cmake_policy(PUSH)
cmake_policy(SET CMP0057 NEW)

# Automatically add tests with CTest by querying the compiled test executable
# for available tests.
#
# xctest_discover_tests(target
# [COMMAND command]
# [WORKING_DIRECTORY dir]
# [PROPERTIES name1 value1...]
# [DISCOVERY_TIMEOUT seconds]
# )
#
# `xctest_discover_tests` sets up a post-build command on the test executable
# that generates the list of tests by parsing the output from running the test
# with the `--list-tests` argument.
#
# The options are:
#
# `target`
# Specifies the XCTest executable, which must be a known CMake target. CMake
# will substitute the location of the built executable when running the test.
#
# `COMMAND command`
# Override the command used for the test executable. If you executable is not
# created with CMake add_executable, you will have to provide a command path.
# If this option is not provided, the target file of the target is used.
#
# `WORKING_DIRECTORY dir`
# Specifies the directory in which to run the discovered test cases. If this
# option is not provided, the current binary directory is used.
#
# `PROPERTIES name1 value1...`
# Specifies additional properties to be set on all tests discovered by this
# invocation of `xctest_discover_tests`.
#
# `DISCOVERY_TIMEOUT seconds`
# Specifies how long (in seconds) CMake will wait for the test to enumerate
# available tests. If the test takes longer than this, discovery (and your
# build) will fail. The default is 5 seconds.
#
# The inspiration for this is CMake `gtest_discover_tests`. The official
# documentation might be useful for using this function. Many details of that
# function has been dropped in the name of simplicity, and others have been
# improved.
function(xctest_discover_tests TARGET)
cmake_parse_arguments(
""
""
"COMMAND;WORKING_DIRECTORY;DISCOVERY_TIMEOUT"
"PROPERTIES"
${ARGN}
)

if(NOT _COMMAND)
set(_COMMAND "$<TARGET_FILE:${TARGET}>")
endif()
if(NOT _WORKING_DIRECTORY)
set(_WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
endif()
if(NOT _DISCOVERY_TIMEOUT)
set(_DISCOVERY_TIMEOUT 5)
endif()

set(ctest_file_base ${CMAKE_CURRENT_BINARY_DIR}/${TARGET})
set(ctest_include_file "${ctest_file_base}_include.cmake")
set(ctest_tests_file "${ctest_file_base}_tests.cmake")

add_custom_command(
TARGET ${TARGET} POST_BUILD
BYPRODUCTS "${ctest_tests_file}"
COMMAND "${CMAKE_COMMAND}"
-D "TEST_TARGET=${TARGET}"
-D "TEST_EXECUTABLE=${_COMMAND}"
-D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}"
-D "TEST_PROPERTIES=${_PROPERTIES}"
-D "CTEST_FILE=${ctest_tests_file}"
-D "TEST_DISCOVERY_TIMEOUT=${_DISCOVERY_TIMEOUT}"
-P "${_XCTEST_DISCOVER_TESTS_SCRIPT}"
VERBATIM
)

file(WRITE "${ctest_include_file}"
"if(EXISTS \"${ctest_tests_file}\")\n"
" include(\"${ctest_tests_file}\")\n"
"else()\n"
" add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)\n"
"endif()\n"
)

set_property(DIRECTORY
APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
)
endfunction()

set(_XCTEST_DISCOVER_TESTS_SCRIPT
${CMAKE_CURRENT_LIST_DIR}/XCTestAddTests.cmake
)

cmake_policy(POP)
90 changes: 90 additions & 0 deletions cmake/modules/XCTestAddTests.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
set(properties ${TEST_PROPERTIES})
set(script)
set(tests)

function(add_command NAME)
set(_args "")
foreach(_arg ${ARGN})
if(_arg MATCHES "[^-./:a-zA-Z0-9_]")
set(_args "${_args} [==[${_arg}]==]")
else()
set(_args "${_args} ${_arg}")
endif()
endforeach()
set(script "${script}${NAME}(${_args})\n" PARENT_SCOPE)
endfunction()

if(NOT EXISTS "${TEST_EXECUTABLE}")
message(FATAL_ERROR
"Specified test executable does not exist.\n"
" Path: '${TEST_EXECUTABLE}'"
)
endif()
# We need to figure out if some environment is needed to run the test listing.
cmake_parse_arguments("_properties" "" "ENVIRONMENT" "" ${properties})
if(_properties_ENVIRONMENT)
foreach(_env ${_properties_ENVIRONMENT})
string(REGEX REPLACE "([a-zA-Z0-9_]+)=(.*)" "\\1" _key "${_env}")
string(REGEX REPLACE "([a-zA-Z0-9_]+)=(.*)" "\\2" _value "${_env}")
if(NOT "${_key}" STREQUAL "")
set(ENV{${_key}} "${_value}")
endif()
endforeach()
endif()
execute_process(
COMMAND "${TEST_EXECUTABLE}" --list-tests
WORKING_DIRECTORY "${TEST_WORKING_DIR}"
TIMEOUT ${TEST_DISCOVERY_TIMEOUT}
OUTPUT_VARIABLE output
ERROR_VARIABLE error_output
RESULT_VARIABLE result
)
if(NOT ${result} EQUAL 0)
string(REPLACE "\n" "\n " output "${output}")
string(REPLACE "\n" "\n " error_output "${error_output}")
message(FATAL_ERROR
"Error running test executable.\n"
" Path: '${TEST_EXECUTABLE}'\n"
" Result: ${result}\n"
" Output:\n"
" ${output}\n"
" Error:\n"
" ${error_output}\n"
)
endif()

string(REPLACE "\n" ";" output "${output}")

foreach(line ${output})
if(line MATCHES "^[ \t]*$")
continue()
elseif(line MATCHES "^Listing [0-9]+ tests? in .+:$")
continue()
elseif(line MATCHES "^.+\\..+/.+$")
# TODO: remove non-ASCII characters from module, class and method names
set(pretty_target "${line}")
string(REGEX REPLACE "/" "-" pretty_target "${pretty_target}")
add_command(add_test
"${pretty_target}"
"${TEST_EXECUTABLE}"
"${line}"
)
add_command(set_tests_properties
"${pretty_target}"
PROPERTIES
WORKING_DIRECTORY "${TEST_WORKING_DIR}"
${properties}
)
list(APPEND tests "${pretty_target}")
else()
message(FATAL_ERROR
"Error parsing test executable output.\n"
" Path: '${TEST_EXECUTABLE}'\n"
" Line: '${line}'"
)
endif()
endforeach()

add_command(set "${TARGET}_TESTS" ${tests})

file(WRITE "${CTEST_FILE}" "${script}")