Revert "Revert "[Build] Make cmark build a build-script product (swiftlang#37102)""

This reverts commit 3150086.

(cherry picked from commit 7cfabf5)

Author: drexin

utils/build-script

@@ -856,11 +856,12 @@ class BuildScriptInvocation(object):
 
         """
 
+        if self.args.build_cmark:
+            before_impl_product_classes.append(products.CMark)
+
         # FIXME: This is a weird division (returning a list of class objects),
         # but it matches the existing structure of the `build-script-impl`.
         impl_product_classes = []
-        if self.args.build_cmark:
-            impl_product_classes.append(products.CMark)
 
         # If --skip-build-llvm is passed in, LLVM cannot be completely disabled, as
         # Swift still needs a few LLVM targets like tblgen to be built for it to be
@@ -989,6 +990,17 @@ class BuildScriptInvocation(object):
 
         # Execute each "pass".
 
+        # Pre-build-script-impl products...
+        # Note: currently only supports building for the host.
+        for host_target in all_host_names:
+            for product_class in before_impl_product_classes:
+                if product_class.is_build_script_impl_product():
+                    continue
+                if not product_class.is_before_build_script_impl_product():
+                    continue
+                # Execute clean, build, test, install
+                self.execute_product_build_steps(product_class, host_target)
+
         # Build...
         for host_target in all_hosts:
             # FIXME: We should only compute these once.

utils/build_swift/build_swift/driver_arguments.py

@@ -282,6 +282,9 @@ def create_argument_parser():
            help='instead of building, write JSON to stdout containing '
                 'various values used to build in this configuration')
 
+    option(['--reconfigure'], store_true,
+           help="Reconfigure all projects as we build")
+
     option('--legacy-impl', store_true('legacy_impl'),
            help='use legacy implementation')
 

utils/build_swift/tests/expected_options.py

@@ -195,6 +195,7 @@
     'native_llvm_tools_path': None,
     'native_swift_tools_path': None,
     'dump_config': False,
+    'reconfigure': False,
     'relocate_xdg_cache_home_under_build_subdir': False,
     'show_sdks': False,
     'skip_build': False,
@@ -500,6 +501,7 @@ class BuildScriptImplOption(_BaseOption):
 
     SetTrueOption('--legacy-impl', dest='legacy_impl'),
     SetTrueOption('--infer', dest='infer_dependencies'),
+    SetTrueOption('--reconfigure'),
 
     EnableOption('--android'),
     EnableOption('--build-external-benchmarks'),

utils/swift_build_support/swift_build_support/products/cmake_product.py

@@ -0,0 +1,116 @@
+# swift_build_support/products/product.py -----------------------*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2021 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+
+import os
+
+from . import product
+from .. import cmake
+from .. import shell
+
+
+class CMakeProduct(product.Product):
+    def build_with_cmake(self, build_targets, build_type, build_args):
+        assert self.toolchain.cmake is not None
+        cmake_build = []
+        _cmake = cmake.CMake(self.args, self.toolchain)
+
+        if self.toolchain.distcc_pump:
+            cmake_build.append(self.toolchain.distcc_pump)
+        cmake_build.extend([self.toolchain.cmake, "--build"])
+
+        generator_output_path = ""
+        if self.args.cmake_generator == "Ninja":
+            generator_output_path = os.path.join(self.build_dir, "build.ninja")
+
+        cmake_cache_path = os.path.join(self.build_dir, "CMakeCache.txt")
+        if self.args.reconfigure or not os.path.isfile(cmake_cache_path) or \
+                (generator_output_path and not os.path.isfile(generator_output_path)):
+            if not os.path.exists(self.build_dir):
+                os.makedirs(self.build_dir)
+
+            # Use `cmake-file-api` in case it is available.
+            query_dir = os.path.join(self.build_dir, ".cmake", "api", "v1", "query")
+            if not os.path.exists(query_dir):
+                os.makedirs(query_dir)
+            open(os.path.join(query_dir, "codemodel-v2"), 'a').close()
+            open(os.path.join(query_dir, "cache-v2"), 'a').close()
+
+            env = None
+            if self.toolchain.distcc:
+                env = {
+                    "DISTCC_HOSTS": "localhost,lzo,cpp"
+                }
+
+            with shell.pushd(self.build_dir):
+                shell.call([self.toolchain.cmake] + list(self.cmake_options) +
+                           list(_cmake.common_options()) +
+                           self.args.extra_cmake_options + [self.source_dir],
+                           env=env)
+
+        if not self.args.skip_build or self.product_name() == "llvm":
+            if self.args.cmake_generator == "Xcode":
+                # Xcode generator uses "ALL_BUILD" instead of "all".
+                # Also, xcodebuild uses -target instead of bare names.
+                build_targets = build_targets.copy()
+                build_targets = [val for target in build_targets
+                                 for val in ["-target",
+                                             target if target != "all"
+                                             else "ALL_BUILD"]]
+
+                # Xcode can't restart itself if it turns out we need to reconfigure.
+                # Do an advance build to handle that.
+                shell.call(cmake_build + [self.build_dir, build_type])
+
+            shell.call(cmake_build + [self.build_dir, "--config", build_type, "--"]
+                       + build_args + build_targets)
+
+    def test_with_cmake(self, executable_target, results_targets,
+                        build_type, build_args):
+        assert self.toolchain.cmake is not None
+        cmake_build = []
+
+        if self.toolchain.distcc_pump:
+            cmake_build.append(self.toolchain.distcc_pump)
+        cmake_args = [self.toolchain.cmake, "--build", self.build_dir,
+                      "--config", build_type, "--"]
+        cmake_build.extend(cmake_args + build_args)
+
+        def target_flag(target):
+            if self.args.cmake_generator == "Xcode":
+                return ["-target", target]
+            return [target]
+
+        if executable_target:
+            shell.call(cmake_build + target_flag(executable_target))
+
+        for target in results_targets:
+            if target:
+                test_target = target
+                print("--- %s ---" % target)
+                if test_target.startswith("check-swift") and self.args.test_paths:
+                    test_target = test_target + "-custom"
+
+                shell.call(cmake_build + target_flag(test_target))
+
+                print("--- %s finished ---" % target)
+
+    def install_with_cmake(self, install_targets, install_destdir):
+        assert self.toolchain.cmake is not None
+        cmake_build = []
+
+        if self.toolchain.distcc_pump:
+            cmake_build.append(self.toolchain.distcc_pump)
+        cmake_args = [self.toolchain.cmake, "--build", self.build_dir, "--"]
+        cmake_build.extend(cmake_args + install_targets)
+
+        environment = {'DESTDIR': install_destdir}
+        shell.call(cmake_build, env=environment)

utils/swift_build_support/swift_build_support/products/cmark.py

@@ -10,19 +10,110 @@
 #
 # ----------------------------------------------------------------------------
 
-from . import product
+from build_swift.build_swift.wrappers import xcrun
 
+from . import cmake_product
 
-class CMark(product.Product):
+
+class CMark(cmake_product.CMakeProduct):
     @classmethod
     def is_build_script_impl_product(cls):
         """is_build_script_impl_product -> bool
 
         Whether this product is produced by build-script-impl.
         """
+        return False
+
+    @classmethod
+    def is_before_build_script_impl_product(cls):
+        """is_before_build_script_impl_product -> bool
+
+        Whether this product is build before any build-script-impl products.
+        """
         return True
 
     # This is the root of the build-graph, so it doesn't have any dependencies.
     @classmethod
     def get_dependencies(cls):
         return []
+
+    def should_build(self, host_target):
+        """should_build() -> Bool
+
+        Whether or not this product should be built with the given arguments.
+        """
+        return self.args.build_cmark
+
+    def build(self, host_target):
+        """build() -> void
+
+        Perform the build, for a non-build-script-impl product.
+        """
+        self.cmake_options.define('CMAKE_BUILD_TYPE:STRING',
+                                  self.args.cmark_build_variant)
+
+        (platform, arch) = host_target.split('-')
+
+        if host_target.startswith("macosx") or \
+           host_target.startswith("iphone") or \
+           host_target.startswith("appletv") or \
+           host_target.startswith("watch"):
+
+            cmake_os_sysroot = xcrun.sdk_path(platform)
+
+            cmake_osx_deployment_target = ''
+            if platform == "macosx":
+                cmake_osx_deployment_target = self.args.darwin_deployment_version_osx
+
+            common_c_flags = ' '.join(self.common_cross_c_flags(platform, arch))
+
+            self.cmake_options.define('CMAKE_C_FLAGS', common_c_flags)
+            self.cmake_options.define('CMAKE_CXX_FLAGS', common_c_flags)
+            self.cmake_options.define('CMAKE_OSX_SYSROOT:PATH', cmake_os_sysroot)
+            self.cmake_options.define('CMAKE_OSX_DEPLOYMENT_TARGET',
+                                      cmake_osx_deployment_target)
+            self.cmake_options.define('CMAKE_OSX_ARCHITECTURES', arch)
+
+        self.build_with_cmake(["all"], self.args.cmark_build_variant, [])
+
+    def should_test(self, host_target):
+        """should_test() -> Bool
+
+        Whether or not this product should be tested with the given arguments.
+        """
+        if self.is_cross_compile_target(host_target):
+            return False
+
+        return self.args.test
+
+    def test(self, host_target):
+        """
+        Perform the test phase for the product.
+
+        This phase might build and execute the product tests.
+        """
+        executable_target = 'api_test'
+        results_targets = ['test']
+        if self.args.cmake_generator == 'Xcode':
+            # Xcode generator uses "RUN_TESTS" instead of "test".
+            results_targets = ['RUN_TESTS']
+
+        self.test_with_cmake(executable_target, results_targets,
+                             self.args.cmark_build_variant, [])
+
+    def should_install(self, host_target):
+        """should_install() -> Bool
+
+        Whether or not this product should be installed with the given
+        arguments.
+        """
+        return self.args.install_all
+
+    def install(self, host_target):
+        """
+        Perform the install phase for the product.
+
+        This phase might copy the artifacts from the previous phases into a
+        destination directory.
+        """
+        self.install_with_cmake(["install"], self.host_install_destdir(host_target))

utils/swift_build_support/swift_build_support/products/product.py

@@ -155,6 +155,8 @@ def __init__(self, args, toolchain, source_dir, build_dir):
         self.source_dir = source_dir
         self.build_dir = build_dir
         self.cmake_options = cmake.CMakeOptions()
+        self.common_c_flags = ['-Wno-unknown-warning-option',
+                               '-Werror=unguarded-availability-new']
 
     def is_release(self):
         """is_release() -> Bool
@@ -179,6 +181,73 @@ def install_toolchain_path(self, host_target):
         return targets.toolchain_path(install_destdir,
                                       self.args.install_prefix)
 
+    def should_include_host_in_lipo(self, host_target):
+        if self.args.cross_compile_hosts:
+            if host_target.startswith("macosx") or \
+               host_target.startswith("iphone") or \
+               host_target.startswith("appletv") or \
+               host_target.startswith("watch"):
+                return True
+        return False
+
+    def host_install_destdir(self, host_target):
+        if self.args.cross_compile_hosts:
+            # If cross compiling tools, install into a host-specific subdirectory.
+            if self.should_include_host_in_lipo(host_target):
+                # If this is one of the hosts we should lipo,
+                # install in to a temporary subdirectory.
+                return '%s/intermediate-install/%s' % \
+                    (self.args.install_destdir, host_target)
+            elif host_target == "merged-hosts":
+                # This assumes that all hosts are merged to the lipo.
+                return self.args.install_destdir
+            else:
+                return '%s/%s' % (self.args.install_destdir, host_target)
+        else:
+            return self.args.install_destdir
+
+    def is_cross_compile_target(self, host_target):
+        return self.args.cross_compile_hosts and \
+            host_target in self.args.cross_compile_hosts
+
+    # TODO: Remove once we've moved over to cmake toolchains
+    def common_cross_c_flags(self, platform, arch):
+        cross_flags = []
+
+        if platform == 'macosx':
+            target = '{}-apple-macosx{}'.format(
+                arch, self.args.darwin_deployment_version_osx)
+            cross_flags.extend(['-arch', arch, '-target', target])
+        elif platform == 'iphonesimulator':
+            target = '{}-apple-ios{}'.format(
+                arch, self.args.darwin_deployment_version_ios)
+            cross_flags.extend(['-arch', arch, '-target', target])
+        elif platform == 'iphoneos':
+            target = '{}-apple-ios{}'.format(
+                arch, self.args.darwin_deployment_version_ios)
+            cross_flags.extend(['-arch', arch, '-target', target])
+        elif platform == 'appletvsimulator':
+            target = '{}-apple-tvos{}'.format(
+                arch, self.args.darwin_deployment_version_tvos)
+            cross_flags.extend(['-arch', arch, '-target', target])
+        elif platform == 'appletvos':
+            target = '{}-apple-tvos{}'.format(
+                arch, self.args.darwin_deployment_version_tvos)
+            cross_flags.extend(['-arch', arch, '-target', target])
+        elif platform == 'watchsimulator':
+            target = '{}-apple-watchos{}'.format(
+                arch, self.args.darwin_deployment_version_watchos)
+            cross_flags.extend(['-arch', arch, '-target', target])
+        elif platform == 'watchos':
+            target = '{}-apple-watchos{}'.format(
+                arch, self.args.darwin_deployment_version_watchos)
+            cross_flags.extend(['-arch', arch, '-target', target])
+
+        if self.is_release():
+            cross_flags.append('-fno-stack-protector')
+
+        return self.common_c_flags + cross_flags
+
 
 class ProductBuilder(object):
     """