Merge pull request #997 from lplarson/code-coverage

[CMake] Support code coverage analysis

Author: lplarson

cmake/modules/AddSwift.cmake

@@ -47,9 +47,9 @@ function(compute_library_subdir result_var_name sdk arch)
   set("${result_var_name}" "${SWIFT_SDK_${sdk}_LIB_SUBDIR}/${arch}" PARENT_SCOPE)
 endfunction()
 
-
 function(_add_variant_c_compile_link_flags
-    sdk arch build_type enable_assertions result_var_name)
+    sdk arch build_type enable_assertions analyze_code_coverage
+    result_var_name)
   set(result
     ${${result_var_name}}
     "-target" "${SWIFT_SDK_${sdk}_ARCH_${arch}_TRIPLE}")
@@ -62,20 +62,27 @@ function(_add_variant_c_compile_link_flags
         "-arch" "${arch}"
         "-F" "${SWIFT_SDK_${sdk}_PATH}/../../../Developer/Library/Frameworks"
         "-m${SWIFT_SDK_${sdk}_VERSION_MIN_NAME}-version-min=${SWIFT_SDK_${sdk}_DEPLOYMENT_VERSION}")
+
+    if(analyze_code_coverage)
+      list(APPEND result "-fprofile-instr-generate=swift-%p.profraw"
+                         "-fcoverage-mapping")
+    endif()
   endif()
 
   set("${result_var_name}" "${result}" PARENT_SCOPE)
 endfunction()
 
 function(_add_variant_c_compile_flags
-    sdk arch build_type enable_assertions result_var_name)
+    sdk arch build_type enable_assertions analyze_code_coverage
+    result_var_name)
   set(result ${${result_var_name}})
 
   _add_variant_c_compile_link_flags(
       "${sdk}"
       "${arch}"
       "${build_type}"
       "${enable_assertions}"
+      FALSE
       result)
 
   is_build_type_optimized("${build_type}" optimized)
@@ -103,6 +110,11 @@ function(_add_variant_c_compile_flags
     list(APPEND result "-DNDEBUG")
   endif()
 
+  if(analyze_code_coverage)
+    list(APPEND result "-fprofile-instr-generate=swift-%p.profraw"
+                       "-fcoverage-mapping")
+  endif()
+
   set("${result_var_name}" "${result}" PARENT_SCOPE)
 endfunction()
 
@@ -139,7 +151,8 @@ function(_add_variant_swift_compile_flags
 endfunction()
 
 function(_add_variant_link_flags
-    sdk arch build_type enable_assertions result_var_name)
+    sdk arch build_type enable_assertions analyze_code_coverage
+    result_var_name)
 
   if("${sdk}" STREQUAL "")
     message(FATAL_ERROR "Should specify an SDK")
@@ -156,6 +169,7 @@ function(_add_variant_link_flags
       "${arch}"
       "${build_type}"
       "${enable_assertions}"
+      "${analyze_code_coverage}"
       result)
 
   if("${sdk}" STREQUAL "LINUX")
@@ -1041,18 +1055,21 @@ function(_add_swift_library_single target name)
   else()
     set(build_type "${CMAKE_BUILD_TYPE}")
     set(enable_assertions "${LLVM_ENABLE_ASSERTIONS}")
+    set(analyze_code_coverage "${SWIFT_ANALYZE_CODE_COVERAGE}")
   endif()
   _add_variant_c_compile_flags(
       "${SWIFTLIB_SINGLE_SDK}"
       "${SWIFTLIB_SINGLE_ARCHITECTURE}"
       "${build_type}"
       "${enable_assertions}"
+      "${analyze_code_coverage}"
       c_compile_flags)
   _add_variant_link_flags(
       "${SWIFTLIB_SINGLE_SDK}"
       "${SWIFTLIB_SINGLE_ARCHITECTURE}"
       "${build_type}"
       "${enable_assertions}"
+      "${analyze_code_coverage}"
       link_flags)
 
   # Handle gold linker flags for shared libraries.
@@ -1618,12 +1635,14 @@ function(_add_swift_executable_single name)
       "${SWIFTEXE_SINGLE_ARCHITECTURE}"
       "${CMAKE_BUILD_TYPE}"
       "${LLVM_ENABLE_ASSERTIONS}"
+      "${SWIFT_ANALYZE_CODE_COVERAGE}"
       c_compile_flags)
   _add_variant_link_flags(
       "${SWIFTEXE_SINGLE_SDK}"
       "${SWIFTEXE_SINGLE_ARCHITECTURE}"
       "${CMAKE_BUILD_TYPE}"
       "${LLVM_ENABLE_ASSERTIONS}"
+      "${SWIFT_ANALYZE_CODE_COVERAGE}"
       link_flags)
 
   list(APPEND link_flags

test/lit.site.cfg.in

@@ -42,6 +42,11 @@ if "@SWIFT_OPTIMIZED@" == "TRUE":
 if "@SWIFT_HAVE_WORKING_STD_REGEX@" == "FALSE":
     config.available_features.add('broken_std_regex')
 
+if "@SWIFT_ANALYZE_CODE_COVERAGE@" == "TRUE":
+    lit_config.useValgrind = True
+    lit_config.valgrindArgs = [os.path.join(config.swift_src_root,
+                                            "utils/use_profdir.py")]
+
 # Let the main config do the real work.
 if config.test_exec_root is None:
     config.test_exec_root = os.path.dirname(os.path.realpath(__file__))

tools/SourceKit/CMakeLists.txt

@@ -46,6 +46,7 @@ function(add_sourcekit_symbol_exports target_name export_file)
 
     set_property(TARGET ${target_name} APPEND_STRING PROPERTY
                  LINK_FLAGS " -Wl,-exported_symbols_list,${CMAKE_CURRENT_BINARY_DIR}/symbol.exports")
+
     add_dependencies(${target_name} ${target_name}_exports)
   endif()
 endfunction()
@@ -63,17 +64,20 @@ function(add_sourcekit_default_compiler_flags target)
   # Add variant-specific flags.
   set(build_type "${CMAKE_BUILD_TYPE}")
   set(enable_assertions "${LLVM_ENABLE_ASSERTIONS}")
+  set(analyze_code_coverage "${SWIFT_ANALYZE_CODE_COVERAGE}")
   _add_variant_c_compile_flags(
       "${sdk}"
       "${arch}"
       "${build_type}"
       "${enable_assertions}"
+      "${analyze_code_coverage}"
       c_compile_flags)
   _add_variant_link_flags(
       "${sdk}"
       "${arch}"
       "${build_type}"
       "${enable_assertions}"
+      "${analyze_code_coverage}"
       link_flags)
 
   # Convert variables to space-separated strings.
@@ -234,6 +238,11 @@ macro(add_sourcekit_executable name)
       set_target_properties(${name}
         PROPERTIES
         LINK_FLAGS "-Wl,-exported_symbol,_main")
+
+      if(SWIFT_ANALYZE_CODE_COVERAGE)
+        set_property(TARGET "${name}" APPEND_STRING PROPERTY
+            LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
+      endif()
     endif()
   endif()
   add_sourcekit_default_compiler_flags("${name}")

tools/SourceKit/tools/complete-test/CMakeLists.txt

@@ -14,6 +14,11 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
   set_target_properties(complete-test
     PROPERTIES
     LINK_FLAGS "-Wl,-rpath -Wl,@executable_path/../lib")
+
+  if(SWIFT_ANALYZE_CODE_COVERAGE)
+    set_property(TARGET complete-test APPEND_STRING PROPERTY
+        LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
+  endif()
 endif()
 
 swift_install_in_component(tools

tools/SourceKit/tools/sourcekitd-repl/CMakeLists.txt

@@ -8,6 +8,11 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
   set_target_properties(sourcekitd-repl
     PROPERTIES
     LINK_FLAGS "-Wl,-rpath -Wl,@executable_path/../lib")
+
+  if(SWIFT_ANALYZE_CODE_COVERAGE)
+    set_property(TARGET sourcekitd-repl APPEND_STRING PROPERTY
+        LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
+  endif()
 endif()
 
 swift_install_in_component(tools

tools/SourceKit/tools/sourcekitd-test/CMakeLists.txt

@@ -22,6 +22,11 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
   set_target_properties(sourcekitd-test
     PROPERTIES
     LINK_FLAGS "-Wl,-rpath -Wl,@executable_path/../lib")
+
+  if(SWIFT_ANALYZE_CODE_COVERAGE)
+    set_property(TARGET sourcekitd-test APPEND_STRING PROPERTY
+        LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
+  endif()
 endif()
 
 swift_install_in_component(tools

unittests/CMakeLists.txt

@@ -26,6 +26,11 @@ function(add_swift_unittest test_dirname)
   if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
     set_property(TARGET "${test_dirname}" APPEND_STRING PROPERTY
       LINK_FLAGS " -Xlinker -rpath -Xlinker ${SWIFT_LIBRARY_OUTPUT_INTDIR}/swift/macosx")
+
+    if(SWIFT_ANALYZE_CODE_COVERAGE)
+      set_property(TARGET "${test_dirname}" APPEND_STRING PROPERTY
+          LINK_FLAGS " -fprofile-instr-generate=swift-%p.profraw -fcoverage-mapping")
+    endif()
   endif()
 endfunction()
 

utils/build-script

@@ -491,6 +491,12 @@ also build for tvOS, but disallow tests that require an tvos device""",
 also build for Apple watchos, but disallow tests that require an watchOS device""",
         action="store_true")
 
+    parser.add_argument("--swift-analyze-code-coverage",
+        help="enable code coverage analysis in Swift",
+        action="store_const",
+        const=True,
+        dest="swift_analyze_code_coverage")
+
     parser.add_argument("--build-subdir",
         help="""
 name of the directory under $SWIFT_BUILD_ROOT where the build products will be
@@ -526,6 +532,10 @@ the number of parallel build jobs to use""",
         '--cmake',
     ]))
 
+    # Code coverage analysis disabled by default.
+    if args.swift_analyze_code_coverage is None:
+        args.swift_analyze_code_coverage = False
+
     # Build cmark if any cmark-related options were specified.
     if (args.cmark_build_variant is not None):
         args.build_cmark = True
@@ -685,6 +695,8 @@ the number of parallel build jobs to use""",
         swift_build_dir_label = args.swift_build_variant
         if args.swift_assertions:
             swift_build_dir_label += "Assert"
+        if args.swift_analyze_code_coverage:
+            swift_build_dir_label += "Coverage"
 
         swift_stdlib_build_dir_label = args.swift_stdlib_build_variant
         if args.swift_stdlib_assertions:
@@ -756,6 +768,7 @@ the number of parallel build jobs to use""",
         "--llvm-enable-assertions", str(args.llvm_assertions).lower(),
         "--swift-enable-assertions", str(args.swift_assertions).lower(),
         "--swift-stdlib-enable-assertions", str(args.swift_stdlib_assertions).lower(),
+        "--swift-analyze-code-coverage", str(args.swift_analyze_code_coverage).lower(),
         "--cmake-generator", args.cmake_generator,
         "--build-jobs", str(args.build_jobs),
         "--workspace", SWIFT_SOURCE_ROOT

utils/build-script-impl

@@ -71,6 +71,7 @@ KNOWN_SETTINGS=(
     llvm-enable-assertions      "1"              "enable assertions in LLVM and Clang"
     swift-build-type            "Debug"          "the CMake build variant for Swift"
     swift-enable-assertions     "1"              "enable assertions in Swift"
+    swift-analyze-code-coverage "0"              "enable code coverage analysis in Swift"
     swift-stdlib-build-type     "Debug"          "the CMake build variant for Swift"
     swift-stdlib-enable-assertions "1"           "enable assertions in Swift"
     lldb-build-type             "Debug"          "the CMake build variant for LLDB"
@@ -1619,6 +1620,7 @@ for deployment_target in "${NATIVE_TOOLS_DEPLOYMENT_TARGETS[@]}" "${CROSS_TOOLS_
                     -DCMAKE_CXX_FLAGS="$(swift_c_flags ${deployment_target})"
                     -DCMAKE_BUILD_TYPE:STRING="${SWIFT_BUILD_TYPE}"
                     -DLLVM_ENABLE_ASSERTIONS:BOOL=$(true_false "${SWIFT_ENABLE_ASSERTIONS}")
+                    -DSWIFT_ANALYZE_CODE_COVERAGE:BOOL=$(true_false "${SWIFT_ANALYZE_CODE_COVERAGE}")
                     -DSWIFT_STDLIB_BUILD_TYPE:STRING="${SWIFT_STDLIB_BUILD_TYPE}"
                     -DSWIFT_STDLIB_ASSERTIONS:BOOL=$(true_false "${SWIFT_STDLIB_ENABLE_ASSERTIONS}")
                     -DSWIFT_NATIVE_LLVM_TOOLS_PATH:STRING="${native_llvm_tools_path}"

utils/use_profdir.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+
+# utils/use_profdir.py
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+
+# This script is used to help prevent profile data clobbering during code
+# coverage profiling.
+
+import sys
+import subprocess
+import os
+import string
+import random
+
+def random_string(N):
+    """Return a random ascii_uppercase + digits string of length `N`"""
+    return ''.join(random.choice(string.ascii_uppercase + string.digits)
+                   for _ in range(N))
+
+def main():
+    # Grab passed in bash command
+    cmd = sys.argv[1:]
+
+    # Search arguments for test identifiers
+    script_files = [f for f in cmd if f.endswith('.script')]
+    gtests = [f.split('=')[1] for f in cmd if f.startswith('--gtest_filter')]
+
+    # Generate directory name using first test identifier, defaulting to
+    # random characters if no test identifier can be found
+    if script_files:
+        profdir = script_files[0] + '.profdir'
+    elif gtests:
+        profdir = os.path.join(os.path.dirname(cmd[0]), gtests[0] + '.profdir')
+    else:
+        profdir = random_string(12) + '.profdir'
+
+    # Create the directory using the generated name
+    try:
+        os.makedirs(profdir)
+    except OSError as e:
+        ERROR_FILE_EXISTS = 17
+        if e.errno != ERROR_FILE_EXISTS:
+            raise
+
+    # cd into the new directory and execute the passed in command
+    previous_cwd = os.getcwd()
+    os.chdir(profdir)
+    try:
+        return_code = subprocess.call(cmd)
+    finally:
+        os.chdir(previous_cwd)
+
+    return return_code
+
+if __name__ == '__main__':
+    exit(main())