From cedbd82993f977f401f16526ff23cea713a9a6a2 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 20 Jan 2024 19:37:49 -0800 Subject: [PATCH 1/4] Group functions in build-script-helper.py --- build-script-helper.py | 182 ++++++++++++++++++++++------------------- 1 file changed, 96 insertions(+), 86 deletions(-) diff --git a/build-script-helper.py b/build-script-helper.py index db0782b3d..38b7e300e 100755 --- a/build-script-helper.py +++ b/build-script-helper.py @@ -19,12 +19,92 @@ import os, platform import subprocess -def printerr(message): +# ----------------------------------------------------------------------------- +# General utilities + +def printerr(message: str): print(message, file=sys.stderr) -def main(argv_prefix = []): - args = parse_args(argv_prefix + sys.argv[1:]) - run(args) +def check_call(cmd, verbose, env=os.environ, **kwargs): + if verbose: + print(' '.join([escape_cmd_arg(arg) for arg in cmd])) + return subprocess.check_call(cmd, env=env, stderr=subprocess.STDOUT, **kwargs) + +def check_output(cmd, verbose, env=os.environ, capture_stderr=True, **kwargs): + if verbose: + print(' '.join([escape_cmd_arg(arg) for arg in cmd])) + if capture_stderr: + stderr = subprocess.STDOUT + else: + stderr = subprocess.DEVNULL + return subprocess.check_output(cmd, env=env, stderr=stderr, encoding='utf-8', **kwargs) + +def escape_cmd_arg(arg): + if '"' in arg or ' ' in arg: + return '"%s"' % arg.replace('"', '\\"') + else: + return arg + +# ----------------------------------------------------------------------------- +# SwiftPM wrappers + +def update_swiftpm_dependencies(package_path, swift_exec, build_path, env, verbose): + args = [swift_exec, 'package', '--package-path', package_path, '--scratch-path', build_path, 'update'] + check_call(args, env=env, verbose=verbose) + +def invoke_swift(package_path, swift_exec, action, products, build_path, multiroot_data_file, configuration, env, verbose): + # Until rdar://53881101 is implemented, we cannot request a build of multiple + # targets simultaneously. For now, just build one product after the other. + for product in products: + invoke_swift_single_product(package_path, swift_exec, action, product, build_path, multiroot_data_file, configuration, env, verbose) + +def get_swiftpm_options(package_path, build_path, multiroot_data_file, configuration, verbose): + args = [ + '--package-path', package_path, + '--configuration', configuration, + '--scratch-path', build_path + ] + if multiroot_data_file: + args += ['--multiroot-data-file', multiroot_data_file] + if verbose: + args += ['--verbose'] + if platform.system() == 'Darwin': + args += [ + '-Xlinker', '-rpath', '-Xlinker', '/usr/lib/swift', + '-Xlinker', '-rpath', '-Xlinker', '@executable_path/../lib/swift/macosx', + '-Xlinker', '-rpath', '-Xlinker', '@executable_path/../lib/swift-5.5/macosx', + ] + return args + +def get_swiftpm_environment_variables(no_local_deps): + env = dict(os.environ) + if not no_local_deps: + env['SWIFTCI_USE_LOCAL_DEPS'] = "1" + return env + + +def invoke_swift_single_product(package_path, swift_exec, action, product, build_path, multiroot_data_file, configuration, env, verbose): + args = [swift_exec, action] + args += get_swiftpm_options(package_path, build_path, multiroot_data_file, configuration, verbose) + if action == 'test': + args += [ + '--test-product', product, + '--disable-testable-imports' + ] + else: + args += ['--product', product] + + check_call(args, env=env, verbose=verbose) + +def generate_xcodeproj(package_path, swift_exec, env, verbose): + package_name = os.path.basename(package_path) + xcodeproj_path = os.path.join(package_path, '%s.xcodeproj' % package_name) + args = [swift_exec, 'package', '--package-path', package_path, 'generate-xcodeproj', '--output', xcodeproj_path] + check_call(args, env=env, verbose=verbose) + +# ----------------------------------------------------------------------------- +# Argument parsing + def parse_args(args): parser = argparse.ArgumentParser(prog='build-script-helper.py') @@ -54,6 +134,14 @@ def parse_args(args): return parsed +def should_run_action(action_name, selected_actions): + if action_name in selected_actions: + return True + elif "all" in selected_actions: + return True + else: + return False + def run(args): package_name = os.path.basename(args.package_path) @@ -139,87 +227,9 @@ def run(args): cmd = ['rsync', '-a', os.path.join(bin_path, 'swift-format'), os.path.join(prefix, 'bin')] check_call(cmd, verbose=args.verbose) -def should_run_action(action_name, selected_actions): - if action_name in selected_actions: - return True - elif "all" in selected_actions: - return True - else: - return False - -def update_swiftpm_dependencies(package_path, swift_exec, build_path, env, verbose): - args = [swift_exec, 'package', '--package-path', package_path, '--scratch-path', build_path, 'update'] - check_call(args, env=env, verbose=verbose) - -def invoke_swift(package_path, swift_exec, action, products, build_path, multiroot_data_file, configuration, env, verbose): - # Until rdar://53881101 is implemented, we cannot request a build of multiple - # targets simultaneously. For now, just build one product after the other. - for product in products: - invoke_swift_single_product(package_path, swift_exec, action, product, build_path, multiroot_data_file, configuration, env, verbose) - -def get_swiftpm_options(package_path, build_path, multiroot_data_file, configuration, verbose): - args = [ - '--package-path', package_path, - '--configuration', configuration, - '--scratch-path', build_path - ] - if multiroot_data_file: - args += ['--multiroot-data-file', multiroot_data_file] - if verbose: - args += ['--verbose'] - if platform.system() == 'Darwin': - args += [ - '-Xlinker', '-rpath', '-Xlinker', '/usr/lib/swift', - '-Xlinker', '-rpath', '-Xlinker', '@executable_path/../lib/swift/macosx', - '-Xlinker', '-rpath', '-Xlinker', '@executable_path/../lib/swift-5.5/macosx', - ] - return args - -def get_swiftpm_environment_variables(no_local_deps): - env = dict(os.environ) - if not no_local_deps: - env['SWIFTCI_USE_LOCAL_DEPS'] = "1" - return env - - -def invoke_swift_single_product(package_path, swift_exec, action, product, build_path, multiroot_data_file, configuration, env, verbose): - args = [swift_exec, action] - args += get_swiftpm_options(package_path, build_path, multiroot_data_file, configuration, verbose) - if action == 'test': - args += [ - '--test-product', product, - '--disable-testable-imports' - ] - else: - args += ['--product', product] - - check_call(args, env=env, verbose=verbose) - -def generate_xcodeproj(package_path, swift_exec, env, verbose): - package_name = os.path.basename(package_path) - xcodeproj_path = os.path.join(package_path, '%s.xcodeproj' % package_name) - args = [swift_exec, 'package', '--package-path', package_path, 'generate-xcodeproj', '--output', xcodeproj_path] - check_call(args, env=env, verbose=verbose) - -def check_call(cmd, verbose, env=os.environ, **kwargs): - if verbose: - print(' '.join([escape_cmd_arg(arg) for arg in cmd])) - return subprocess.check_call(cmd, env=env, stderr=subprocess.STDOUT, **kwargs) - -def check_output(cmd, verbose, env=os.environ, capture_stderr=True, **kwargs): - if verbose: - print(' '.join([escape_cmd_arg(arg) for arg in cmd])) - if capture_stderr: - stderr = subprocess.STDOUT - else: - stderr = subprocess.DEVNULL - return subprocess.check_output(cmd, env=env, stderr=stderr, encoding='utf-8', **kwargs) - -def escape_cmd_arg(arg): - if '"' in arg or ' ' in arg: - return '"%s"' % arg.replace('"', '\\"') - else: - return arg +def main(argv_prefix = []): + args = parse_args(argv_prefix + sys.argv[1:]) + run(args) if __name__ == '__main__': - main() + main() \ No newline at end of file From 9a6e304be1f2bf22865cccdafcad6a37537c03e2 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 20 Jan 2024 19:38:40 -0800 Subject: [PATCH 2/4] Format build-script-helper.py with `black` --- build-script-helper.py | 473 ++++++++++++++++++++++++++--------------- 1 file changed, 296 insertions(+), 177 deletions(-) diff --git a/build-script-helper.py b/build-script-helper.py index 38b7e300e..b54845307 100755 --- a/build-script-helper.py +++ b/build-script-helper.py @@ -22,214 +22,333 @@ # ----------------------------------------------------------------------------- # General utilities + def printerr(message: str): - print(message, file=sys.stderr) + print(message, file=sys.stderr) + def check_call(cmd, verbose, env=os.environ, **kwargs): - if verbose: - print(' '.join([escape_cmd_arg(arg) for arg in cmd])) - return subprocess.check_call(cmd, env=env, stderr=subprocess.STDOUT, **kwargs) + if verbose: + print(" ".join([escape_cmd_arg(arg) for arg in cmd])) + return subprocess.check_call(cmd, env=env, stderr=subprocess.STDOUT, **kwargs) + def check_output(cmd, verbose, env=os.environ, capture_stderr=True, **kwargs): - if verbose: - print(' '.join([escape_cmd_arg(arg) for arg in cmd])) - if capture_stderr: - stderr = subprocess.STDOUT - else: - stderr = subprocess.DEVNULL - return subprocess.check_output(cmd, env=env, stderr=stderr, encoding='utf-8', **kwargs) + if verbose: + print(" ".join([escape_cmd_arg(arg) for arg in cmd])) + if capture_stderr: + stderr = subprocess.STDOUT + else: + stderr = subprocess.DEVNULL + return subprocess.check_output( + cmd, env=env, stderr=stderr, encoding="utf-8", **kwargs + ) + def escape_cmd_arg(arg): - if '"' in arg or ' ' in arg: - return '"%s"' % arg.replace('"', '\\"') - else: - return arg + if '"' in arg or " " in arg: + return '"%s"' % arg.replace('"', '\\"') + else: + return arg + # ----------------------------------------------------------------------------- # SwiftPM wrappers + def update_swiftpm_dependencies(package_path, swift_exec, build_path, env, verbose): - args = [swift_exec, 'package', '--package-path', package_path, '--scratch-path', build_path, 'update'] - check_call(args, env=env, verbose=verbose) - -def invoke_swift(package_path, swift_exec, action, products, build_path, multiroot_data_file, configuration, env, verbose): - # Until rdar://53881101 is implemented, we cannot request a build of multiple - # targets simultaneously. For now, just build one product after the other. - for product in products: - invoke_swift_single_product(package_path, swift_exec, action, product, build_path, multiroot_data_file, configuration, env, verbose) - -def get_swiftpm_options(package_path, build_path, multiroot_data_file, configuration, verbose): - args = [ - '--package-path', package_path, - '--configuration', configuration, - '--scratch-path', build_path - ] - if multiroot_data_file: - args += ['--multiroot-data-file', multiroot_data_file] - if verbose: - args += ['--verbose'] - if platform.system() == 'Darwin': - args += [ - '-Xlinker', '-rpath', '-Xlinker', '/usr/lib/swift', - '-Xlinker', '-rpath', '-Xlinker', '@executable_path/../lib/swift/macosx', - '-Xlinker', '-rpath', '-Xlinker', '@executable_path/../lib/swift-5.5/macosx', + args = [ + swift_exec, + "package", + "--package-path", + package_path, + "--scratch-path", + build_path, + "update", + ] + check_call(args, env=env, verbose=verbose) + + +def invoke_swift( + package_path, + swift_exec, + action, + products, + build_path, + multiroot_data_file, + configuration, + env, + verbose, +): + # Until rdar://53881101 is implemented, we cannot request a build of multiple + # targets simultaneously. For now, just build one product after the other. + for product in products: + invoke_swift_single_product( + package_path, + swift_exec, + action, + product, + build_path, + multiroot_data_file, + configuration, + env, + verbose, + ) + + +def get_swiftpm_options( + package_path, build_path, multiroot_data_file, configuration, verbose +): + args = [ + "--package-path", + package_path, + "--configuration", + configuration, + "--scratch-path", + build_path, ] - return args + if multiroot_data_file: + args += ["--multiroot-data-file", multiroot_data_file] + if verbose: + args += ["--verbose"] + if platform.system() == "Darwin": + args += [ + "-Xlinker", + "-rpath", + "-Xlinker", + "/usr/lib/swift", + "-Xlinker", + "-rpath", + "-Xlinker", + "@executable_path/../lib/swift/macosx", + "-Xlinker", + "-rpath", + "-Xlinker", + "@executable_path/../lib/swift-5.5/macosx", + ] + return args + def get_swiftpm_environment_variables(no_local_deps): - env = dict(os.environ) - if not no_local_deps: - env['SWIFTCI_USE_LOCAL_DEPS'] = "1" - return env - - -def invoke_swift_single_product(package_path, swift_exec, action, product, build_path, multiroot_data_file, configuration, env, verbose): - args = [swift_exec, action] - args += get_swiftpm_options(package_path, build_path, multiroot_data_file, configuration, verbose) - if action == 'test': - args += [ - '--test-product', product, - '--disable-testable-imports' - ] - else: - args += ['--product', product] + env = dict(os.environ) + if not no_local_deps: + env["SWIFTCI_USE_LOCAL_DEPS"] = "1" + return env + + +def invoke_swift_single_product( + package_path, + swift_exec, + action, + product, + build_path, + multiroot_data_file, + configuration, + env, + verbose, +): + args = [swift_exec, action] + args += get_swiftpm_options( + package_path, build_path, multiroot_data_file, configuration, verbose + ) + if action == "test": + args += ["--test-product", product, "--disable-testable-imports"] + else: + args += ["--product", product] + + check_call(args, env=env, verbose=verbose) - check_call(args, env=env, verbose=verbose) def generate_xcodeproj(package_path, swift_exec, env, verbose): - package_name = os.path.basename(package_path) - xcodeproj_path = os.path.join(package_path, '%s.xcodeproj' % package_name) - args = [swift_exec, 'package', '--package-path', package_path, 'generate-xcodeproj', '--output', xcodeproj_path] - check_call(args, env=env, verbose=verbose) + package_name = os.path.basename(package_path) + xcodeproj_path = os.path.join(package_path, "%s.xcodeproj" % package_name) + args = [ + swift_exec, + "package", + "--package-path", + package_path, + "generate-xcodeproj", + "--output", + xcodeproj_path, + ] + check_call(args, env=env, verbose=verbose) + # ----------------------------------------------------------------------------- # Argument parsing def parse_args(args): - parser = argparse.ArgumentParser(prog='build-script-helper.py') + parser = argparse.ArgumentParser(prog="build-script-helper.py") + + parser.add_argument("--package-path", default="") + parser.add_argument( + "-v", "--verbose", action="store_true", help="log executed commands" + ) + parser.add_argument( + "--prefix", + dest="install_prefixes", + nargs="*", + metavar="PATHS", + help="install path", + ) + parser.add_argument("--configuration", default="debug") + parser.add_argument("--build-path", default=None) + parser.add_argument( + "--multiroot-data-file", + help="Path to an Xcode workspace to create a unified build of SwiftSyntax with other projects.", + ) + parser.add_argument( + "--toolchain", + required=True, + help="the toolchain to use when building this package", + ) + parser.add_argument( + "--update", action="store_true", help="update all SwiftPM dependencies" + ) + parser.add_argument( + "--no-local-deps", + action="store_true", + help="use normal remote dependencies when building", + ) + parser.add_argument( + "build_actions", + help="Extra actions to perform. Can be any number of the following", + choices=["all", "build", "test", "generate-xcodeproj", "install"], + nargs="*", + default=["build"], + ) - parser.add_argument('--package-path', default='') - parser.add_argument('-v', '--verbose', action='store_true', help='log executed commands') - parser.add_argument('--prefix', dest='install_prefixes', nargs='*', metavar='PATHS', help='install path') - parser.add_argument('--configuration', default='debug') - parser.add_argument('--build-path', default=None) - parser.add_argument('--multiroot-data-file', help='Path to an Xcode workspace to create a unified build of SwiftSyntax with other projects.') - parser.add_argument('--toolchain', required=True, help='the toolchain to use when building this package') - parser.add_argument('--update', action='store_true', help='update all SwiftPM dependencies') - parser.add_argument('--no-local-deps', action='store_true', help='use normal remote dependencies when building') - parser.add_argument('build_actions', help="Extra actions to perform. Can be any number of the following", choices=['all', 'build', 'test', 'generate-xcodeproj', 'install'], nargs="*", default=['build']) + parsed = parser.parse_args(args) - parsed = parser.parse_args(args) + parsed.swift_exec = os.path.join(parsed.toolchain, "bin", "swift") - parsed.swift_exec = os.path.join(parsed.toolchain, 'bin', 'swift') + # Convert package_path to absolute path, relative to root of repo. + repo_path = os.path.dirname(__file__) + parsed.package_path = os.path.realpath(os.path.join(repo_path, parsed.package_path)) - # Convert package_path to absolute path, relative to root of repo. - repo_path = os.path.dirname(__file__) - parsed.package_path = os.path.realpath( - os.path.join(repo_path, parsed.package_path)) + if not parsed.build_path: + parsed.build_path = os.path.join(parsed.package_path, ".build") - if not parsed.build_path: - parsed.build_path = os.path.join(parsed.package_path, '.build') + return parsed - return parsed def should_run_action(action_name, selected_actions): - if action_name in selected_actions: - return True - elif "all" in selected_actions: - return True - else: - return False + if action_name in selected_actions: + return True + elif "all" in selected_actions: + return True + else: + return False + def run(args): - package_name = os.path.basename(args.package_path) - - env = get_swiftpm_environment_variables(no_local_deps=args.no_local_deps) - # Use local dependencies (i.e. checked out next swift-format). - - if args.update: - print("** Updating dependencies of %s **" % package_name) - try: - update_swiftpm_dependencies(package_path=args.package_path, - swift_exec=args.swift_exec, - build_path=args.build_path, - env=env, - verbose=args.verbose) - except subprocess.CalledProcessError as e: - printerr('FAIL: Updating dependencies of %s failed' % package_name) - printerr('Executing: %s' % ' '.join(e.cmd)) - sys.exit(1) - - # The test action creates its own build. No need to build if we are just testing. - if should_run_action('build', args.build_actions) or should_run_action('install', args.build_actions): - print("** Building %s **" % package_name) - try: - invoke_swift(package_path=args.package_path, - swift_exec=args.swift_exec, - action='build', - products=['swift-format'], - build_path=args.build_path, - multiroot_data_file=args.multiroot_data_file, - configuration=args.configuration, - env=env, - verbose=args.verbose) - except subprocess.CalledProcessError as e: - printerr('FAIL: Building %s failed' % package_name) - printerr('Executing: %s' % ' '.join(e.cmd)) - sys.exit(1) - - output_dir = os.path.realpath(os.path.join(args.build_path, args.configuration)) - - if should_run_action("generate-xcodeproj", args.build_actions): - print("** Generating Xcode project for %s **" % package_name) - try: - generate_xcodeproj(args.package_path, - swift_exec=args.swift_exec, - env=env, - verbose=args.verbose) - except subprocess.CalledProcessError as e: - printerr('FAIL: Generating the Xcode project failed') - printerr('Executing: %s' % ' '.join(e.cmd)) - sys.exit(1) - - if should_run_action("test", args.build_actions): - print("** Testing %s **" % package_name) - try: - invoke_swift(package_path=args.package_path, - swift_exec=args.swift_exec, - action='test', - products=['%sPackageTests' % package_name], - build_path=args.build_path, - multiroot_data_file=args.multiroot_data_file, - configuration=args.configuration, - env=env, - verbose=args.verbose) - except subprocess.CalledProcessError as e: - printerr('FAIL: Testing %s failed' % package_name) - printerr('Executing: %s' % ' '.join(e.cmd)) - sys.exit(1) - - if should_run_action("install", args.build_actions): - print("** Installing %s **" % package_name) - - swiftpm_args = get_swiftpm_options( - package_path=args.package_path, - build_path=args.build_path, - multiroot_data_file=args.multiroot_data_file, - configuration=args.configuration, - verbose=args.verbose - ) - cmd = [args.swift_exec, 'build', '--show-bin-path'] + swiftpm_args - bin_path = check_output(cmd, env=env, capture_stderr=False, verbose=args.verbose).strip() - - for prefix in args.install_prefixes: - cmd = ['rsync', '-a', os.path.join(bin_path, 'swift-format'), os.path.join(prefix, 'bin')] - check_call(cmd, verbose=args.verbose) - -def main(argv_prefix = []): - args = parse_args(argv_prefix + sys.argv[1:]) - run(args) - -if __name__ == '__main__': - main() \ No newline at end of file + package_name = os.path.basename(args.package_path) + + env = get_swiftpm_environment_variables(no_local_deps=args.no_local_deps) + # Use local dependencies (i.e. checked out next swift-format). + + if args.update: + print("** Updating dependencies of %s **" % package_name) + try: + update_swiftpm_dependencies( + package_path=args.package_path, + swift_exec=args.swift_exec, + build_path=args.build_path, + env=env, + verbose=args.verbose, + ) + except subprocess.CalledProcessError as e: + printerr("FAIL: Updating dependencies of %s failed" % package_name) + printerr("Executing: %s" % " ".join(e.cmd)) + sys.exit(1) + + # The test action creates its own build. No need to build if we are just testing. + if should_run_action("build", args.build_actions) or should_run_action( + "install", args.build_actions + ): + print("** Building %s **" % package_name) + try: + invoke_swift( + package_path=args.package_path, + swift_exec=args.swift_exec, + action="build", + products=["swift-format"], + build_path=args.build_path, + multiroot_data_file=args.multiroot_data_file, + configuration=args.configuration, + env=env, + verbose=args.verbose, + ) + except subprocess.CalledProcessError as e: + printerr("FAIL: Building %s failed" % package_name) + printerr("Executing: %s" % " ".join(e.cmd)) + sys.exit(1) + + output_dir = os.path.realpath(os.path.join(args.build_path, args.configuration)) + + if should_run_action("generate-xcodeproj", args.build_actions): + print("** Generating Xcode project for %s **" % package_name) + try: + generate_xcodeproj( + args.package_path, + swift_exec=args.swift_exec, + env=env, + verbose=args.verbose, + ) + except subprocess.CalledProcessError as e: + printerr("FAIL: Generating the Xcode project failed") + printerr("Executing: %s" % " ".join(e.cmd)) + sys.exit(1) + + if should_run_action("test", args.build_actions): + print("** Testing %s **" % package_name) + try: + invoke_swift( + package_path=args.package_path, + swift_exec=args.swift_exec, + action="test", + products=["%sPackageTests" % package_name], + build_path=args.build_path, + multiroot_data_file=args.multiroot_data_file, + configuration=args.configuration, + env=env, + verbose=args.verbose, + ) + except subprocess.CalledProcessError as e: + printerr("FAIL: Testing %s failed" % package_name) + printerr("Executing: %s" % " ".join(e.cmd)) + sys.exit(1) + + if should_run_action("install", args.build_actions): + print("** Installing %s **" % package_name) + + swiftpm_args = get_swiftpm_options( + package_path=args.package_path, + build_path=args.build_path, + multiroot_data_file=args.multiroot_data_file, + configuration=args.configuration, + verbose=args.verbose, + ) + cmd = [args.swift_exec, "build", "--show-bin-path"] + swiftpm_args + bin_path = check_output( + cmd, env=env, capture_stderr=False, verbose=args.verbose + ).strip() + + for prefix in args.install_prefixes: + cmd = [ + "rsync", + "-a", + os.path.join(bin_path, "swift-format"), + os.path.join(prefix, "bin"), + ] + check_call(cmd, verbose=args.verbose) + + +def main(argv_prefix=[]): + args = parse_args(argv_prefix + sys.argv[1:]) + run(args) + + +if __name__ == "__main__": + main() From 4e397f4c9430a22558241a5a988388809bbddf35 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 20 Jan 2024 20:16:31 -0800 Subject: [PATCH 3/4] Clean up build-script-helper.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert the file to more modern Python, remove actions that weren’t used anymore and make it more readable in general. --- build-script-helper.py | 361 ++++++++++++++++------------------------- 1 file changed, 143 insertions(+), 218 deletions(-) diff --git a/build-script-helper.py b/build-script-helper.py index b54845307..cfb70d11a 100755 --- a/build-script-helper.py +++ b/build-script-helper.py @@ -18,22 +18,33 @@ import sys import os, platform import subprocess +from pathlib import Path +from typing import List, Union, Optional # ----------------------------------------------------------------------------- # General utilities -def printerr(message: str): +def fatal_error(message: str) -> None: print(message, file=sys.stderr) + raise SystemExit(1) -def check_call(cmd, verbose, env=os.environ, **kwargs): +def printerr(message: str) -> None: + print(message, file=sys.stderr) + + +def check_call( + cmd: List[Union[str, Path]], verbose: bool, env=os.environ, **kwargs +) -> None: if verbose: print(" ".join([escape_cmd_arg(arg) for arg in cmd])) - return subprocess.check_call(cmd, env=env, stderr=subprocess.STDOUT, **kwargs) + subprocess.check_call(cmd, env=env, stderr=subprocess.STDOUT, **kwargs) -def check_output(cmd, verbose, env=os.environ, capture_stderr=True, **kwargs): +def check_output( + cmd: List[Union[str, Path]], verbose, env=os.environ, capture_stderr=True, **kwargs +) -> str: if verbose: print(" ".join([escape_cmd_arg(arg) for arg in cmd])) if capture_stderr: @@ -45,7 +56,8 @@ def check_output(cmd, verbose, env=os.environ, capture_stderr=True, **kwargs): ) -def escape_cmd_arg(arg): +def escape_cmd_arg(arg: Union[str, Path]) -> str: + arg = str(arg) if '"' in arg or " " in arg: return '"%s"' % arg.replace('"', '\\"') else: @@ -56,50 +68,14 @@ def escape_cmd_arg(arg): # SwiftPM wrappers -def update_swiftpm_dependencies(package_path, swift_exec, build_path, env, verbose): - args = [ - swift_exec, - "package", - "--package-path", - package_path, - "--scratch-path", - build_path, - "update", - ] - check_call(args, env=env, verbose=verbose) - - -def invoke_swift( - package_path, - swift_exec, - action, - products, - build_path, - multiroot_data_file, - configuration, - env, - verbose, -): - # Until rdar://53881101 is implemented, we cannot request a build of multiple - # targets simultaneously. For now, just build one product after the other. - for product in products: - invoke_swift_single_product( - package_path, - swift_exec, - action, - product, - build_path, - multiroot_data_file, - configuration, - env, - verbose, - ) - - def get_swiftpm_options( - package_path, build_path, multiroot_data_file, configuration, verbose -): - args = [ + package_path: Path, + build_path: Path, + multiroot_data_file: Optional[Path], + configuration: str, + verbose: bool, +) -> List[Union[str, Path]]: + args: List[Union[str, Path]] = [ "--package-path", package_path, "--configuration", @@ -112,15 +88,14 @@ def get_swiftpm_options( if verbose: args += ["--verbose"] if platform.system() == "Darwin": + args += ["-Xlinker", "-rpath", "-Xlinker", "/usr/lib/swift"] args += [ - "-Xlinker", - "-rpath", - "-Xlinker", - "/usr/lib/swift", "-Xlinker", "-rpath", "-Xlinker", "@executable_path/../lib/swift/macosx", + ] + args += [ "-Xlinker", "-rpath", "-Xlinker", @@ -129,24 +104,26 @@ def get_swiftpm_options( return args -def get_swiftpm_environment_variables(no_local_deps): +def get_swiftpm_environment_variables(): env = dict(os.environ) - if not no_local_deps: - env["SWIFTCI_USE_LOCAL_DEPS"] = "1" + env["SWIFTCI_USE_LOCAL_DEPS"] = "1" return env -def invoke_swift_single_product( - package_path, - swift_exec, - action, - product, - build_path, - multiroot_data_file, - configuration, +def invoke_swiftpm( + package_path: Path, + swift_exec: Path, + action: str, + product: str, + build_path: Path, + multiroot_data_file: Optional[Path], + configuration: str, env, - verbose, + verbose: bool, ): + """ + Build or test a single SwiftPM product. + """ args = [swift_exec, action] args += get_swiftpm_options( package_path, build_path, multiroot_data_file, configuration, verbose @@ -159,195 +136,143 @@ def invoke_swift_single_product( check_call(args, env=env, verbose=verbose) -def generate_xcodeproj(package_path, swift_exec, env, verbose): - package_name = os.path.basename(package_path) - xcodeproj_path = os.path.join(package_path, "%s.xcodeproj" % package_name) - args = [ - swift_exec, - "package", - "--package-path", - package_path, - "generate-xcodeproj", - "--output", - xcodeproj_path, - ] - check_call(args, env=env, verbose=verbose) +# ----------------------------------------------------------------------------- +# Actions + + +def build(args: argparse.Namespace) -> None: + print("** Building swift-format **") + env = get_swiftpm_environment_variables() + invoke_swiftpm( + package_path=args.package_path, + swift_exec=args.swift_exec, + action="build", + product="swift-format", + build_path=args.build_path, + multiroot_data_file=args.multiroot_data_file, + configuration=args.configuration, + env=env, + verbose=args.verbose, + ) + + +def test(args: argparse.Namespace) -> None: + print("** Testing swift-format **") + env = get_swiftpm_environment_variables() + invoke_swiftpm( + package_path=args.package_path, + swift_exec=args.swift_exec, + action="test", + product="swift-formatPackageTests", + build_path=args.build_path, + multiroot_data_file=args.multiroot_data_file, + configuration=args.configuration, + env=env, + verbose=args.verbose, + ) + + +def install(args: argparse.Namespace) -> None: + build(args) + + print("** Installing swift-format **") + + env = get_swiftpm_environment_variables() + swiftpm_args = get_swiftpm_options( + package_path=args.package_path, + build_path=args.build_path, + multiroot_data_file=args.multiroot_data_file, + configuration=args.configuration, + verbose=args.verbose, + ) + cmd = [args.swift_exec, "build", "--show-bin-path"] + swiftpm_args + bin_path = check_output( + cmd, env=env, capture_stderr=False, verbose=args.verbose + ).strip() + + for prefix in args.install_prefixes: + cmd = [ + "rsync", + "-a", + Path(bin_path) / "swift-format", + prefix / "bin", + ] + check_call(cmd, verbose=args.verbose) # ----------------------------------------------------------------------------- # Argument parsing -def parse_args(args): - parser = argparse.ArgumentParser(prog="build-script-helper.py") - +def add_common_args(parser: argparse.ArgumentParser) -> None: parser.add_argument("--package-path", default="") parser.add_argument( "-v", "--verbose", action="store_true", help="log executed commands" ) - parser.add_argument( - "--prefix", - dest="install_prefixes", - nargs="*", - metavar="PATHS", - help="install path", - ) parser.add_argument("--configuration", default="debug") - parser.add_argument("--build-path", default=None) + parser.add_argument("--build-path", type=Path, default=None) parser.add_argument( "--multiroot-data-file", + type=Path, help="Path to an Xcode workspace to create a unified build of SwiftSyntax with other projects.", ) parser.add_argument( "--toolchain", required=True, + type=Path, help="the toolchain to use when building this package", ) - parser.add_argument( - "--update", action="store_true", help="update all SwiftPM dependencies" - ) - parser.add_argument( - "--no-local-deps", - action="store_true", - help="use normal remote dependencies when building", + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(prog="build-script-helper.py") + subparsers = parser.add_subparsers( + title="subcommands", dest="action", required=True, metavar="action" ) - parser.add_argument( - "build_actions", - help="Extra actions to perform. Can be any number of the following", - choices=["all", "build", "test", "generate-xcodeproj", "install"], + + build_parser = subparsers.add_parser("build", help="build the package") + add_common_args(build_parser) + + test_parser = subparsers.add_parser("test", help="test the package") + add_common_args(test_parser) + + install_parser = subparsers.add_parser("install", help="install the package") + add_common_args(install_parser) + install_parser.add_argument( + "--prefix", + dest="install_prefixes", nargs="*", - default=["build"], + type=Path, + metavar="PATHS", + help="install path", ) - parsed = parser.parse_args(args) + parsed = parser.parse_args(sys.argv[1:]) - parsed.swift_exec = os.path.join(parsed.toolchain, "bin", "swift") + parsed.swift_exec = parsed.toolchain / "bin" / "swift" # Convert package_path to absolute path, relative to root of repo. - repo_path = os.path.dirname(__file__) - parsed.package_path = os.path.realpath(os.path.join(repo_path, parsed.package_path)) + repo_path = Path(__file__).parent + parsed.package_path = (repo_path / parsed.package_path).resolve() if not parsed.build_path: - parsed.build_path = os.path.join(parsed.package_path, ".build") + parsed.build_path = parsed.package_path / ".build" return parsed -def should_run_action(action_name, selected_actions): - if action_name in selected_actions: - return True - elif "all" in selected_actions: - return True - else: - return False - - -def run(args): - package_name = os.path.basename(args.package_path) - - env = get_swiftpm_environment_variables(no_local_deps=args.no_local_deps) - # Use local dependencies (i.e. checked out next swift-format). - - if args.update: - print("** Updating dependencies of %s **" % package_name) - try: - update_swiftpm_dependencies( - package_path=args.package_path, - swift_exec=args.swift_exec, - build_path=args.build_path, - env=env, - verbose=args.verbose, - ) - except subprocess.CalledProcessError as e: - printerr("FAIL: Updating dependencies of %s failed" % package_name) - printerr("Executing: %s" % " ".join(e.cmd)) - sys.exit(1) +def main(): + args = parse_args() # The test action creates its own build. No need to build if we are just testing. - if should_run_action("build", args.build_actions) or should_run_action( - "install", args.build_actions - ): - print("** Building %s **" % package_name) - try: - invoke_swift( - package_path=args.package_path, - swift_exec=args.swift_exec, - action="build", - products=["swift-format"], - build_path=args.build_path, - multiroot_data_file=args.multiroot_data_file, - configuration=args.configuration, - env=env, - verbose=args.verbose, - ) - except subprocess.CalledProcessError as e: - printerr("FAIL: Building %s failed" % package_name) - printerr("Executing: %s" % " ".join(e.cmd)) - sys.exit(1) - - output_dir = os.path.realpath(os.path.join(args.build_path, args.configuration)) - - if should_run_action("generate-xcodeproj", args.build_actions): - print("** Generating Xcode project for %s **" % package_name) - try: - generate_xcodeproj( - args.package_path, - swift_exec=args.swift_exec, - env=env, - verbose=args.verbose, - ) - except subprocess.CalledProcessError as e: - printerr("FAIL: Generating the Xcode project failed") - printerr("Executing: %s" % " ".join(e.cmd)) - sys.exit(1) - - if should_run_action("test", args.build_actions): - print("** Testing %s **" % package_name) - try: - invoke_swift( - package_path=args.package_path, - swift_exec=args.swift_exec, - action="test", - products=["%sPackageTests" % package_name], - build_path=args.build_path, - multiroot_data_file=args.multiroot_data_file, - configuration=args.configuration, - env=env, - verbose=args.verbose, - ) - except subprocess.CalledProcessError as e: - printerr("FAIL: Testing %s failed" % package_name) - printerr("Executing: %s" % " ".join(e.cmd)) - sys.exit(1) - - if should_run_action("install", args.build_actions): - print("** Installing %s **" % package_name) - - swiftpm_args = get_swiftpm_options( - package_path=args.package_path, - build_path=args.build_path, - multiroot_data_file=args.multiroot_data_file, - configuration=args.configuration, - verbose=args.verbose, - ) - cmd = [args.swift_exec, "build", "--show-bin-path"] + swiftpm_args - bin_path = check_output( - cmd, env=env, capture_stderr=False, verbose=args.verbose - ).strip() - - for prefix in args.install_prefixes: - cmd = [ - "rsync", - "-a", - os.path.join(bin_path, "swift-format"), - os.path.join(prefix, "bin"), - ] - check_call(cmd, verbose=args.verbose) - - -def main(argv_prefix=[]): - args = parse_args(argv_prefix + sys.argv[1:]) - run(args) + if args.action == "build": + build(args) + elif args.action == "test": + test(args) + elif args.action == "install": + install(args) + else: + fatal_error(f"unknown action '{args.action}'") if __name__ == "__main__": From 33158505f0ab7cde9244f407983774be136568b3 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Sat, 20 Jan 2024 20:41:21 -0800 Subject: [PATCH 4/4] Allow cross-compiling swift-format This should build swift-format as a fat binary containing both an x86_64 and an arm64 slice in the open source toolchains. --- build-script-helper.py | 72 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/build-script-helper.py b/build-script-helper.py index cfb70d11a..9f06ee87c 100755 --- a/build-script-helper.py +++ b/build-script-helper.py @@ -20,6 +20,7 @@ import subprocess from pathlib import Path from typing import List, Union, Optional +import json # ----------------------------------------------------------------------------- # General utilities @@ -68,11 +69,29 @@ def escape_cmd_arg(arg: Union[str, Path]) -> str: # SwiftPM wrappers +def get_build_target(swift_exec: Path, cross_compile_config: Optional[Path]) -> str: + """Returns the target-triple of the current machine or for cross-compilation.""" + command = [swift_exec, "-print-target-info"] + if cross_compile_config: + cross_compile_json = json.load(open(cross_compile_config)) + command += ["-target", cross_compile_json["target"]] + target_info_json = subprocess.check_output( + command, stderr=subprocess.PIPE, universal_newlines=True + ).strip() + target_info = json.loads(target_info_json) + if "-apple-macosx" in target_info["target"]["unversionedTriple"]: + return target_info["target"]["unversionedTriple"] + return target_info["target"]["triple"] + + def get_swiftpm_options( + swift_exec: Path, package_path: Path, build_path: Path, multiroot_data_file: Optional[Path], configuration: str, + cross_compile_host: Optional[str], + cross_compile_config: Optional[Path], verbose: bool, ) -> List[Union[str, Path]]: args: List[Union[str, Path]] = [ @@ -87,8 +106,17 @@ def get_swiftpm_options( args += ["--multiroot-data-file", multiroot_data_file] if verbose: args += ["--verbose"] - if platform.system() == "Darwin": - args += ["-Xlinker", "-rpath", "-Xlinker", "/usr/lib/swift"] + build_target = get_build_target( + swift_exec, cross_compile_config=cross_compile_config + ) + build_os = build_target.split("-")[2] + if build_os.startswith("macosx"): + args += [ + "-Xlinker", + "-rpath", + "-Xlinker", + "/usr/lib/swift", + ] args += [ "-Xlinker", "-rpath", @@ -101,6 +129,21 @@ def get_swiftpm_options( "-Xlinker", "@executable_path/../lib/swift-5.5/macosx", ] + else: + # Library rpath for swift, dispatch, Foundation, etc. when installing + args += [ + "-Xlinker", + "-rpath", + "-Xlinker", + "$ORIGIN/../lib/swift/" + build_os, + ] + + if cross_compile_host: + if build_os.startswith("macosx") and cross_compile_host.startswith("macosx-"): + args += ["--arch", "x86_64", "--arch", "arm64"] + else: + fatal_error("cannot cross-compile for %s" % cross_compile_host) + return args @@ -118,6 +161,8 @@ def invoke_swiftpm( build_path: Path, multiroot_data_file: Optional[Path], configuration: str, + cross_compile_host: Optional[str], + cross_compile_config: Optional[Path], env, verbose: bool, ): @@ -126,7 +171,14 @@ def invoke_swiftpm( """ args = [swift_exec, action] args += get_swiftpm_options( - package_path, build_path, multiroot_data_file, configuration, verbose + swift_exec=swift_exec, + package_path=package_path, + build_path=build_path, + multiroot_data_file=multiroot_data_file, + configuration=configuration, + cross_compile_host=cross_compile_host, + cross_compile_config=cross_compile_config, + verbose=verbose, ) if action == "test": args += ["--test-product", product, "--disable-testable-imports"] @@ -151,6 +203,8 @@ def build(args: argparse.Namespace) -> None: build_path=args.build_path, multiroot_data_file=args.multiroot_data_file, configuration=args.configuration, + cross_compile_host=args.cross_compile_host, + cross_compile_config=args.cross_compile_config, env=env, verbose=args.verbose, ) @@ -167,6 +221,8 @@ def test(args: argparse.Namespace) -> None: build_path=args.build_path, multiroot_data_file=args.multiroot_data_file, configuration=args.configuration, + cross_compile_host=args.cross_compile_host, + cross_compile_config=args.cross_compile_config, env=env, verbose=args.verbose, ) @@ -179,10 +235,13 @@ def install(args: argparse.Namespace) -> None: env = get_swiftpm_environment_variables() swiftpm_args = get_swiftpm_options( + swift_exec=args.swift_exec, package_path=args.package_path, build_path=args.build_path, multiroot_data_file=args.multiroot_data_file, configuration=args.configuration, + cross_compile_host=args.cross_compile_host, + cross_compile_config=args.cross_compile_config, verbose=args.verbose, ) cmd = [args.swift_exec, "build", "--show-bin-path"] + swiftpm_args @@ -222,6 +281,13 @@ def add_common_args(parser: argparse.ArgumentParser) -> None: type=Path, help="the toolchain to use when building this package", ) + parser.add_argument( + "--cross-compile-host", help="cross-compile for another host instead" + ) + parser.add_argument( + "--cross-compile-config", + help="an SPM JSON destination file containing Swift cross-compilation flags", + ) def parse_args() -> argparse.Namespace: