diff --git a/build-script-helper.py b/build-script-helper.py index db0782b3d..9f06ee87c 100755 --- a/build-script-helper.py +++ b/build-script-helper.py @@ -18,208 +18,328 @@ import sys import os, platform import subprocess +from pathlib import Path +from typing import List, Union, Optional +import json -def printerr(message): - print(message, file=sys.stderr) +# ----------------------------------------------------------------------------- +# General utilities -def main(argv_prefix = []): - args = parse_args(argv_prefix + sys.argv[1:]) - run(args) -def parse_args(args): - parser = argparse.ArgumentParser(prog='build-script-helper.py') +def fatal_error(message: str) -> None: + print(message, file=sys.stderr) + raise SystemExit(1) - parser.add_argument('--package-path', default='') - parser.add_argument('-v', '--verbose', action='store_true', help='log executed commands') - parser.add_argument('--prefix', dest='install_prefixes', nargs='*', metavar='PATHS', help='install path') - parser.add_argument('--configuration', default='debug') - parser.add_argument('--build-path', default=None) - parser.add_argument('--multiroot-data-file', help='Path to an Xcode workspace to create a unified build of SwiftSyntax with other projects.') - parser.add_argument('--toolchain', required=True, help='the toolchain to use when building this package') - parser.add_argument('--update', action='store_true', help='update all SwiftPM dependencies') - parser.add_argument('--no-local-deps', action='store_true', help='use normal remote dependencies when building') - parser.add_argument('build_actions', help="Extra actions to perform. Can be any number of the following", choices=['all', 'build', 'test', 'generate-xcodeproj', 'install'], nargs="*", default=['build']) - parsed = parser.parse_args(args) +def printerr(message: str) -> None: + print(message, file=sys.stderr) - parsed.swift_exec = os.path.join(parsed.toolchain, 'bin', 'swift') - # Convert package_path to absolute path, relative to root of repo. - repo_path = os.path.dirname(__file__) - parsed.package_path = os.path.realpath( - os.path.join(repo_path, parsed.package_path)) +def check_call( + cmd: List[Union[str, Path]], verbose: bool, env=os.environ, **kwargs +) -> None: + if verbose: + print(" ".join([escape_cmd_arg(arg) for arg in cmd])) + subprocess.check_call(cmd, env=env, stderr=subprocess.STDOUT, **kwargs) - if not parsed.build_path: - parsed.build_path = os.path.join(parsed.package_path, '.build') - return parsed +def check_output( + cmd: List[Union[str, Path]], verbose, env=os.environ, capture_stderr=True, **kwargs +) -> str: + if verbose: + print(" ".join([escape_cmd_arg(arg) for arg in cmd])) + if capture_stderr: + stderr = subprocess.STDOUT + else: + stderr = subprocess.DEVNULL + return subprocess.check_output( + cmd, env=env, stderr=stderr, encoding="utf-8", **kwargs + ) -def run(args): - package_name = os.path.basename(args.package_path) - env = get_swiftpm_environment_variables(no_local_deps=args.no_local_deps) - # Use local dependencies (i.e. checked out next swift-format). - - if args.update: - print("** Updating dependencies of %s **" % package_name) - try: - update_swiftpm_dependencies(package_path=args.package_path, - swift_exec=args.swift_exec, - build_path=args.build_path, - env=env, - verbose=args.verbose) - except subprocess.CalledProcessError as e: - printerr('FAIL: Updating dependencies of %s failed' % package_name) - printerr('Executing: %s' % ' '.join(e.cmd)) - sys.exit(1) - - # The test action creates its own build. No need to build if we are just testing. - if should_run_action('build', args.build_actions) or should_run_action('install', args.build_actions): - print("** Building %s **" % package_name) - try: - invoke_swift(package_path=args.package_path, +def escape_cmd_arg(arg: Union[str, Path]) -> str: + arg = str(arg) + if '"' in arg or " " in arg: + return '"%s"' % arg.replace('"', '\\"') + else: + return arg + + +# ----------------------------------------------------------------------------- +# SwiftPM wrappers + + +def get_build_target(swift_exec: Path, cross_compile_config: Optional[Path]) -> str: + """Returns the target-triple of the current machine or for cross-compilation.""" + command = [swift_exec, "-print-target-info"] + if cross_compile_config: + cross_compile_json = json.load(open(cross_compile_config)) + command += ["-target", cross_compile_json["target"]] + target_info_json = subprocess.check_output( + command, stderr=subprocess.PIPE, universal_newlines=True + ).strip() + target_info = json.loads(target_info_json) + if "-apple-macosx" in target_info["target"]["unversionedTriple"]: + return target_info["target"]["unversionedTriple"] + return target_info["target"]["triple"] + + +def get_swiftpm_options( + swift_exec: Path, + package_path: Path, + build_path: Path, + multiroot_data_file: Optional[Path], + configuration: str, + cross_compile_host: Optional[str], + cross_compile_config: Optional[Path], + verbose: bool, +) -> List[Union[str, Path]]: + args: List[Union[str, Path]] = [ + "--package-path", + package_path, + "--configuration", + configuration, + "--scratch-path", + build_path, + ] + if multiroot_data_file: + args += ["--multiroot-data-file", multiroot_data_file] + if verbose: + args += ["--verbose"] + build_target = get_build_target( + swift_exec, cross_compile_config=cross_compile_config + ) + build_os = build_target.split("-")[2] + if build_os.startswith("macosx"): + args += [ + "-Xlinker", + "-rpath", + "-Xlinker", + "/usr/lib/swift", + ] + args += [ + "-Xlinker", + "-rpath", + "-Xlinker", + "@executable_path/../lib/swift/macosx", + ] + args += [ + "-Xlinker", + "-rpath", + "-Xlinker", + "@executable_path/../lib/swift-5.5/macosx", + ] + else: + # Library rpath for swift, dispatch, Foundation, etc. when installing + args += [ + "-Xlinker", + "-rpath", + "-Xlinker", + "$ORIGIN/../lib/swift/" + build_os, + ] + + if cross_compile_host: + if build_os.startswith("macosx") and cross_compile_host.startswith("macosx-"): + args += ["--arch", "x86_64", "--arch", "arm64"] + else: + fatal_error("cannot cross-compile for %s" % cross_compile_host) + + return args + + +def get_swiftpm_environment_variables(): + env = dict(os.environ) + env["SWIFTCI_USE_LOCAL_DEPS"] = "1" + return env + + +def invoke_swiftpm( + package_path: Path, + swift_exec: Path, + action: str, + product: str, + build_path: Path, + multiroot_data_file: Optional[Path], + configuration: str, + cross_compile_host: Optional[str], + cross_compile_config: Optional[Path], + env, + verbose: bool, +): + """ + Build or test a single SwiftPM product. + """ + args = [swift_exec, action] + args += get_swiftpm_options( + swift_exec=swift_exec, + package_path=package_path, + build_path=build_path, + multiroot_data_file=multiroot_data_file, + configuration=configuration, + cross_compile_host=cross_compile_host, + cross_compile_config=cross_compile_config, + verbose=verbose, + ) + if action == "test": + args += ["--test-product", product, "--disable-testable-imports"] + else: + args += ["--product", product] + + check_call(args, env=env, verbose=verbose) + + +# ----------------------------------------------------------------------------- +# Actions + + +def build(args: argparse.Namespace) -> None: + print("** Building swift-format **") + env = get_swiftpm_environment_variables() + invoke_swiftpm( + package_path=args.package_path, swift_exec=args.swift_exec, - action='build', - products=['swift-format'], + action="build", + product="swift-format", build_path=args.build_path, multiroot_data_file=args.multiroot_data_file, configuration=args.configuration, + cross_compile_host=args.cross_compile_host, + cross_compile_config=args.cross_compile_config, env=env, - verbose=args.verbose) - except subprocess.CalledProcessError as e: - printerr('FAIL: Building %s failed' % package_name) - printerr('Executing: %s' % ' '.join(e.cmd)) - sys.exit(1) - - output_dir = os.path.realpath(os.path.join(args.build_path, args.configuration)) - - if should_run_action("generate-xcodeproj", args.build_actions): - print("** Generating Xcode project for %s **" % package_name) - try: - generate_xcodeproj(args.package_path, - swift_exec=args.swift_exec, - env=env, - verbose=args.verbose) - except subprocess.CalledProcessError as e: - printerr('FAIL: Generating the Xcode project failed') - printerr('Executing: %s' % ' '.join(e.cmd)) - sys.exit(1) - - if should_run_action("test", args.build_actions): - print("** Testing %s **" % package_name) - try: - invoke_swift(package_path=args.package_path, + verbose=args.verbose, + ) + + +def test(args: argparse.Namespace) -> None: + print("** Testing swift-format **") + env = get_swiftpm_environment_variables() + invoke_swiftpm( + package_path=args.package_path, swift_exec=args.swift_exec, - action='test', - products=['%sPackageTests' % package_name], + action="test", + product="swift-formatPackageTests", build_path=args.build_path, multiroot_data_file=args.multiroot_data_file, configuration=args.configuration, + cross_compile_host=args.cross_compile_host, + cross_compile_config=args.cross_compile_config, env=env, - verbose=args.verbose) - except subprocess.CalledProcessError as e: - printerr('FAIL: Testing %s failed' % package_name) - printerr('Executing: %s' % ' '.join(e.cmd)) - sys.exit(1) + verbose=args.verbose, + ) + - if should_run_action("install", args.build_actions): - print("** Installing %s **" % package_name) +def install(args: argparse.Namespace) -> None: + build(args) + print("** Installing swift-format **") + + env = get_swiftpm_environment_variables() swiftpm_args = get_swiftpm_options( - package_path=args.package_path, - build_path=args.build_path, - multiroot_data_file=args.multiroot_data_file, - configuration=args.configuration, - verbose=args.verbose + swift_exec=args.swift_exec, + package_path=args.package_path, + build_path=args.build_path, + multiroot_data_file=args.multiroot_data_file, + configuration=args.configuration, + cross_compile_host=args.cross_compile_host, + cross_compile_config=args.cross_compile_config, + verbose=args.verbose, ) - cmd = [args.swift_exec, 'build', '--show-bin-path'] + swiftpm_args - bin_path = check_output(cmd, env=env, capture_stderr=False, verbose=args.verbose).strip() - + cmd = [args.swift_exec, "build", "--show-bin-path"] + swiftpm_args + bin_path = check_output( + cmd, env=env, capture_stderr=False, verbose=args.verbose + ).strip() + for prefix in args.install_prefixes: - cmd = ['rsync', '-a', os.path.join(bin_path, 'swift-format'), os.path.join(prefix, 'bin')] + cmd = [ + "rsync", + "-a", + Path(bin_path) / "swift-format", + prefix / "bin", + ] check_call(cmd, verbose=args.verbose) -def should_run_action(action_name, selected_actions): - if action_name in selected_actions: - return True - elif "all" in selected_actions: - return True - else: - return False - -def update_swiftpm_dependencies(package_path, swift_exec, build_path, env, verbose): - args = [swift_exec, 'package', '--package-path', package_path, '--scratch-path', build_path, 'update'] - check_call(args, env=env, verbose=verbose) - -def invoke_swift(package_path, swift_exec, action, products, build_path, multiroot_data_file, configuration, env, verbose): - # Until rdar://53881101 is implemented, we cannot request a build of multiple - # targets simultaneously. For now, just build one product after the other. - for product in products: - invoke_swift_single_product(package_path, swift_exec, action, product, build_path, multiroot_data_file, configuration, env, verbose) - -def get_swiftpm_options(package_path, build_path, multiroot_data_file, configuration, verbose): - args = [ - '--package-path', package_path, - '--configuration', configuration, - '--scratch-path', build_path - ] - if multiroot_data_file: - args += ['--multiroot-data-file', multiroot_data_file] - if verbose: - args += ['--verbose'] - if platform.system() == 'Darwin': - args += [ - '-Xlinker', '-rpath', '-Xlinker', '/usr/lib/swift', - '-Xlinker', '-rpath', '-Xlinker', '@executable_path/../lib/swift/macosx', - '-Xlinker', '-rpath', '-Xlinker', '@executable_path/../lib/swift-5.5/macosx', - ] - return args - -def get_swiftpm_environment_variables(no_local_deps): - env = dict(os.environ) - if not no_local_deps: - env['SWIFTCI_USE_LOCAL_DEPS'] = "1" - return env - - -def invoke_swift_single_product(package_path, swift_exec, action, product, build_path, multiroot_data_file, configuration, env, verbose): - args = [swift_exec, action] - args += get_swiftpm_options(package_path, build_path, multiroot_data_file, configuration, verbose) - if action == 'test': - args += [ - '--test-product', product, - '--disable-testable-imports' - ] - else: - args += ['--product', product] - - check_call(args, env=env, verbose=verbose) - -def generate_xcodeproj(package_path, swift_exec, env, verbose): - package_name = os.path.basename(package_path) - xcodeproj_path = os.path.join(package_path, '%s.xcodeproj' % package_name) - args = [swift_exec, 'package', '--package-path', package_path, 'generate-xcodeproj', '--output', xcodeproj_path] - check_call(args, env=env, verbose=verbose) - -def check_call(cmd, verbose, env=os.environ, **kwargs): - if verbose: - print(' '.join([escape_cmd_arg(arg) for arg in cmd])) - return subprocess.check_call(cmd, env=env, stderr=subprocess.STDOUT, **kwargs) - -def check_output(cmd, verbose, env=os.environ, capture_stderr=True, **kwargs): - if verbose: - print(' '.join([escape_cmd_arg(arg) for arg in cmd])) - if capture_stderr: - stderr = subprocess.STDOUT - else: - stderr = subprocess.DEVNULL - return subprocess.check_output(cmd, env=env, stderr=stderr, encoding='utf-8', **kwargs) - -def escape_cmd_arg(arg): - if '"' in arg or ' ' in arg: - return '"%s"' % arg.replace('"', '\\"') - else: - return arg - -if __name__ == '__main__': - main() + +# ----------------------------------------------------------------------------- +# Argument parsing + + +def add_common_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--package-path", default="") + parser.add_argument( + "-v", "--verbose", action="store_true", help="log executed commands" + ) + parser.add_argument("--configuration", default="debug") + parser.add_argument("--build-path", type=Path, default=None) + parser.add_argument( + "--multiroot-data-file", + type=Path, + help="Path to an Xcode workspace to create a unified build of SwiftSyntax with other projects.", + ) + parser.add_argument( + "--toolchain", + required=True, + type=Path, + help="the toolchain to use when building this package", + ) + parser.add_argument( + "--cross-compile-host", help="cross-compile for another host instead" + ) + parser.add_argument( + "--cross-compile-config", + help="an SPM JSON destination file containing Swift cross-compilation flags", + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(prog="build-script-helper.py") + subparsers = parser.add_subparsers( + title="subcommands", dest="action", required=True, metavar="action" + ) + + build_parser = subparsers.add_parser("build", help="build the package") + add_common_args(build_parser) + + test_parser = subparsers.add_parser("test", help="test the package") + add_common_args(test_parser) + + install_parser = subparsers.add_parser("install", help="install the package") + add_common_args(install_parser) + install_parser.add_argument( + "--prefix", + dest="install_prefixes", + nargs="*", + type=Path, + metavar="PATHS", + help="install path", + ) + + parsed = parser.parse_args(sys.argv[1:]) + + parsed.swift_exec = parsed.toolchain / "bin" / "swift" + + # Convert package_path to absolute path, relative to root of repo. + repo_path = Path(__file__).parent + parsed.package_path = (repo_path / parsed.package_path).resolve() + + if not parsed.build_path: + parsed.build_path = parsed.package_path / ".build" + + return parsed + + +def main(): + args = parse_args() + + # The test action creates its own build. No need to build if we are just testing. + if args.action == "build": + build(args) + elif args.action == "test": + test(args) + elif args.action == "install": + install(args) + else: + fatal_error(f"unknown action '{args.action}'") + + +if __name__ == "__main__": + main()