[test] Split TestFoundation in smaller tests for CTest reporting.

CMakeLists.txt

@@ -30,6 +30,7 @@ find_package(dispatch CONFIG REQUIRED)
 
 include(SwiftSupport)
 include(GNUInstallDirs)
+include(XCTest)
 
 set(CF_DEPLOYMENT_SWIFT YES CACHE BOOL "Build for Swift" FORCE)
 

TestFoundation/CMakeLists.txt

@@ -168,9 +168,11 @@ add_custom_command(TARGET TestFoundation POST_BUILD
   COMMAND
     ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:FoundationXML> ${CMAKE_BINARY_DIR}/TestFoundation.app)
 
-add_test(NAME TestFoundation
-  COMMAND ${CMAKE_BINARY_DIR}/TestFoundation.app/TestFoundation
-  WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/TestFoundation.app)
-set_tests_properties(TestFoundation PROPERTIES
-  ENVIRONMENT LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/TestFoundation.app:$<TARGET_LINKER_FILE_DIR:XCTest>:$<TARGET_LINKER_FILE_DIR:dispatch>:$<TARGET_LINKER_FILE_DIR:swiftDispatch>:$<TARGET_LINKER_FILE_DIR:BlocksRuntime>)
+xctest_discover_tests(TestFoundation
+                      COMMAND
+                      ${CMAKE_BINARY_DIR}/TestFoundation.app/TestFoundation${CMAKE_EXECUTABLE_SUFFIX}
+                      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/TestFoundation.app
+                      PROPERTIES
+                        ENVIRONMENT
+                          LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/TestFoundation.app:$<TARGET_LINKER_FILE_DIR:XCTest>:$<TARGET_LINKER_FILE_DIR:dispatch>:$<TARGET_LINKER_FILE_DIR:swiftDispatch>:$<TARGET_LINKER_FILE_DIR:BlocksRuntime>)
 

cmake/modules/XCTest.cmake

@@ -0,0 +1,100 @@
+cmake_policy(PUSH)
+cmake_policy(SET CMP0057 NEW)
+
+# Automatically add tests with CTest by querying the compiled test executable
+# for available tests.
+#
+#   xctest_discover_tests(target
+#                         [COMMAND command]
+#                         [WORKING_DIRECTORY dir]
+#                         [PROPERTIES name1 value1...]
+#                         [DISCOVERY_TIMEOUT seconds]
+#   )
+#
+# `xctest_discover_tests` sets up a post-build command on the test executable
+# that generates the list of tests by parsing the output from running the test
+# with the `--list-tests` argument.
+#
+# The options are:
+#
+# `target`
+#   Specifies the XCTest executable, which must be a known CMake target. CMake
+#   will substitute the location of the built executable when running the test.
+#
+# `COMMAND command`
+#   Override the command used for the test executable. If you executable is not
+#   created with CMake add_executable, you will have to provide a command path.
+#   If this option is not provided, the target file of the target is used.
+#
+# `WORKING_DIRECTORY dir`
+#   Specifies the directory in which to run the discovered test cases. If this
+#   option is not provided, the current binary directory is used.
+#
+# `PROPERTIES name1 value1...`
+#   Specifies additional properties to be set on all tests discovered by this
+#   invocation of `xctest_discover_tests`.
+#
+# `DISCOVERY_TIMEOUT seconds`
+#   Specifies how long (in seconds) CMake will wait for the test to enumerate
+#   available tests. If the test takes longer than this, discovery (and your
+#   build) will fail. The default is 5 seconds.
+#
+# The inspiration for this is CMake `gtest_discover_tests`. The official
+# documentation might be useful for using this function. Many details of that
+# function has been dropped in the name of simplicity, and others have been
+# improved.
+function(xctest_discover_tests TARGET)
+  cmake_parse_arguments(
+    ""
+    ""
+    "COMMAND;WORKING_DIRECTORY;DISCOVERY_TIMEOUT"
+    "PROPERTIES"
+    ${ARGN}
+  )
+
+  if(NOT _COMMAND)
+    set(_COMMAND "$<TARGET_FILE:${TARGET}>")
+  endif()
+  if(NOT _WORKING_DIRECTORY)
+    set(_WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+  endif()
+  if(NOT _DISCOVERY_TIMEOUT)
+    set(_DISCOVERY_TIMEOUT 5)
+  endif()
+
+  set(ctest_file_base ${CMAKE_CURRENT_BINARY_DIR}/${TARGET})
+  set(ctest_include_file "${ctest_file_base}_include.cmake")
+  set(ctest_tests_file "${ctest_file_base}_tests.cmake")
+
+  add_custom_command(
+    TARGET ${TARGET} POST_BUILD
+    BYPRODUCTS "${ctest_tests_file}"
+    COMMAND "${CMAKE_COMMAND}"
+            -D "TEST_TARGET=${TARGET}"
+            -D "TEST_EXECUTABLE=${_COMMAND}"
+            -D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}"
+            -D "TEST_PROPERTIES=${_PROPERTIES}"
+            -D "CTEST_FILE=${ctest_tests_file}"
+            -D "TEST_DISCOVERY_TIMEOUT=${_DISCOVERY_TIMEOUT}"
+            -P "${_XCTEST_DISCOVER_TESTS_SCRIPT}"
+    VERBATIM
+  )
+
+  file(WRITE "${ctest_include_file}"
+    "if(EXISTS \"${ctest_tests_file}\")\n"
+    "  include(\"${ctest_tests_file}\")\n"
+    "else()\n"
+    "  add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)\n"
+    "endif()\n"
+  )
+
+  set_property(DIRECTORY
+    APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
+  )
+endfunction()
+
+set(_XCTEST_DISCOVER_TESTS_SCRIPT
+  ${CMAKE_CURRENT_LIST_DIR}/XCTestAddTests.cmake
+)
+
+cmake_policy(POP)

cmake/modules/XCTestAddTests.cmake

@@ -0,0 +1,90 @@
+set(properties ${TEST_PROPERTIES})
+set(script)
+set(tests)
+
+function(add_command NAME)
+  set(_args "")
+  foreach(_arg ${ARGN})
+    if(_arg MATCHES "[^-./:a-zA-Z0-9_]")
+      set(_args "${_args} [==[${_arg}]==]")
+    else()
+      set(_args "${_args} ${_arg}")
+    endif()
+  endforeach()
+  set(script "${script}${NAME}(${_args})\n" PARENT_SCOPE)
+endfunction()
+
+if(NOT EXISTS "${TEST_EXECUTABLE}")
+  message(FATAL_ERROR
+    "Specified test executable does not exist.\n"
+    "  Path: '${TEST_EXECUTABLE}'"
+  )
+endif()
+# We need to figure out if some environment is needed to run the test listing.
+cmake_parse_arguments("_properties" "" "ENVIRONMENT" "" ${properties})
+if(_properties_ENVIRONMENT)
+  foreach(_env ${_properties_ENVIRONMENT})
+    string(REGEX REPLACE "([a-zA-Z0-9_]+)=(.*)" "\\1" _key "${_env}")
+    string(REGEX REPLACE "([a-zA-Z0-9_]+)=(.*)" "\\2" _value "${_env}")
+    if(NOT "${_key}" STREQUAL "")
+      set(ENV{${_key}} "${_value}")
+    endif()
+  endforeach()
+endif()
+execute_process(
+  COMMAND "${TEST_EXECUTABLE}" --list-tests
+  WORKING_DIRECTORY "${TEST_WORKING_DIR}"
+  TIMEOUT ${TEST_DISCOVERY_TIMEOUT}
+  OUTPUT_VARIABLE output
+  ERROR_VARIABLE error_output
+  RESULT_VARIABLE result
+)
+if(NOT ${result} EQUAL 0)
+  string(REPLACE "\n" "\n    " output "${output}")
+  string(REPLACE "\n" "\n    " error_output "${error_output}")
+  message(FATAL_ERROR
+    "Error running test executable.\n"
+    "  Path: '${TEST_EXECUTABLE}'\n"
+    "  Result: ${result}\n"
+    "  Output:\n"
+    "    ${output}\n"
+    "  Error:\n"
+    "    ${error_output}\n"
+  )
+endif()
+
+string(REPLACE "\n" ";" output "${output}")
+
+foreach(line ${output})
+  if(line MATCHES "^[ \t]*$")
+    continue()
+  elseif(line MATCHES "^Listing [0-9]+ tests? in .+:$")
+    continue()
+  elseif(line MATCHES "^.+\\..+/.+$")
+    # TODO: remove non-ASCII characters from module, class and method names
+    set(pretty_target "${line}")
+    string(REGEX REPLACE "/" "-" pretty_target "${pretty_target}")
+    add_command(add_test
+      "${pretty_target}"
+      "${TEST_EXECUTABLE}"
+      "${line}"
+    )
+    add_command(set_tests_properties
+      "${pretty_target}"
+      PROPERTIES
+      WORKING_DIRECTORY "${TEST_WORKING_DIR}"
+      ${properties}
+    )
+    list(APPEND tests "${pretty_target}")
+  else()
+    message(FATAL_ERROR
+      "Error parsing test executable output.\n"
+      "  Path: '${TEST_EXECUTABLE}'\n"
+      "  Line: '${line}'"
+    )
+  endif()
+endforeach()
+
+add_command(set "${TARGET}_TESTS" ${tests})
+
+file(WRITE "${CTEST_FILE}" "${script}")