From cb3cf90fb3319060064f4bff30b66729f6a7e4dd Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Sat, 7 May 2022 10:56:49 -0700 Subject: [PATCH 1/9] Move install_x86_support_libraries to utils. --- scripts/gha/build_desktop.py | 2 +- scripts/gha/utils.py | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/scripts/gha/build_desktop.py b/scripts/gha/build_desktop.py index 1978ff0ba..0a0090f81 100644 --- a/scripts/gha/build_desktop.py +++ b/scripts/gha/build_desktop.py @@ -281,7 +281,7 @@ def main(): # To build x86 on x86_64 linux hosts, we also need x86 support libraries if args.arch == 'x86' and utils.is_linux_os(): - install_x86_support_libraries(args.gha_build) + utils.install_x86_support_libraries(args.gha_build) # Install C++ dependencies using vcpkg if not args.disable_vcpkg: diff --git a/scripts/gha/utils.py b/scripts/gha/utils.py index 9201fed01..7f964e414 100644 --- a/scripts/gha/utils.py +++ b/scripts/gha/utils.py @@ -218,3 +218,45 @@ def clean_vcpkg_temp_data(): for directory_to_remove in directories_to_remove: abspath = os.path.join(vcpkg_root_dir_path, directory_to_remove) delete_directory(abspath) + + +def install_x86_support_libraries(gha_build=False): + """Install support libraries needed to build x86 on x86_64 hosts. + + Args: + gha_build: Pass in True if running on a GitHub runner; this will activate + workarounds that might be undesirable on a personal system (e.g. + downgrading Ubuntu packages). + """ + if is_linux_os(): + packages = ['gcc-multilib', 'g++-multilib', 'libglib2.0-dev:i386', + 'libsecret-1-dev:i386', 'libpthread-stubs0-dev:i386', + 'libssl-dev:i386'] + if gha_build: + # Workaround for GitHub runners, which have an incompatibility between the + # 64-bit and 32-bit versions of the Ubuntu package libpcre2-8-0. Downgrade + # the installed 64-bit version of the library to get around this issue. + # This will presumably be fixed in a future Ubuntu update. (If you remove + # it, remove the workaround further down this function as well.) + packages = ['--allow-downgrades'] + packages + ['libpcre2-8-0=10.34-7'] + + # First check if these packages exist on the machine already + devnull = open(os.devnull, "w") + process = subprocess.run(["dpkg", "-s"] + packages, stdout=devnull, stderr=subprocess.STDOUT) + devnull.close() + if process.returncode != 0: + # This implies not all of the required packages are already installed on user's machine + # Install them. + run_command(['dpkg', '--add-architecture', 'i386'], as_root=True, check=True) + run_command(['apt', 'update'], as_root=True, check=True) + run_command(['apt', 'install', '-V', '-y'] + packages, as_root=True, check=True) + + if gha_build: + # One more workaround: downgrading libpcre2-8-0 above may have uninstalled + # libsecret, which is required for the Linux build. Force it to be + # reinstalled, but do it as a separate command to ensure that held + # packages aren't modified. (Once the workaround above is removed, this can + # be removed as well.) + # Note: "-f" = "fix" - let apt do what it needs to do to fix dependencies. + run_command(['apt', 'install', '-f', '-V', '-y', 'libsecret-1-dev'], + as_root=True, check=True) From a6da2e647cf947f4a0811af76babcd91945504e8 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 11:49:37 -0700 Subject: [PATCH 2/9] Update packaging x86 to use new prereqs script option --- .github/workflows/cpp-packaging.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index 165145a57..de845ba02 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -408,9 +408,13 @@ jobs: # binutils, used by older version of homebrew for hosting packages. brew update - - name: Install prerequisites - run: | - python scripts/gha/install_prereqs_desktop.py + - name: Install Desktop SDK prerequisites + uses: nick-invision/retry@v2 + with: + timeout_minutes: 15 + max_attempts: 3 + command: | + python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.architecture }}' - name: Export verbose flag shell: bash From bdf2d61ed8dee4ae1d3010aeb95816c622bac6a5 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 11:57:11 -0700 Subject: [PATCH 3/9] Fix x86 CI builds to retry prereqs as well --- .github/workflows/desktop.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 9891e3196..11176ae44 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -176,9 +176,13 @@ jobs: # binutils, used by older version of homebrew for hosting packages. brew update - - name: Install prerequisites - run: | - python scripts/gha/install_prereqs_desktop.py + - name: Install Desktop SDK prerequisites + uses: nick-invision/retry@v2 + with: + timeout_minutes: 15 + max_attempts: 3 + command: | + python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.architecture }}' - name: Build SDK shell: bash From 381088ff415901addb8f82c440c60de40750b67e Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 12:05:04 -0700 Subject: [PATCH 4/9] Add call to install_x86_support_libraries to prereqs script if needed. --- scripts/gha/install_prereqs_desktop.py | 118 ++++++++++++++----------- 1 file changed, 66 insertions(+), 52 deletions(-) diff --git a/scripts/gha/install_prereqs_desktop.py b/scripts/gha/install_prereqs_desktop.py index a8511c493..0cf4f54a5 100644 --- a/scripts/gha/install_prereqs_desktop.py +++ b/scripts/gha/install_prereqs_desktop.py @@ -29,62 +29,76 @@ """ +import argparse import utils def main(): - # Install protobuf on linux/mac if its not installed already - if not utils.is_command_installed('protoc'): - if utils.is_linux_os(): - # sudo apt install protobuf-compiler - utils.run_command(['apt', 'install', '-y','protobuf-compiler'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'protobuf']) - - # Install go on linux/mac if its not installed already - if not utils.is_command_installed('go'): - if utils.is_linux_os(): - # sudo apt install -y golang - utils.run_command(['apt', 'install', '-y','golang'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'go']) - - # Install openssl on linux/mac if its not installed already - if not utils.is_command_installed('go'): - if utils.is_linux_os(): - # sudo apt install -y openssl - utils.run_command(['apt', 'install', '-y','openssl'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'openssl']) - - # Install ccache on linux/mac if its not installed already - if not utils.is_command_installed('ccache'): - if utils.is_linux_os(): - # sudo apt install ccache - utils.run_command(['apt', 'install', '-y', 'ccache'], as_root=True) - elif utils.is_mac_os(): - # brew install ccache - utils.run_command(['brew', 'install', 'ccache']) - - # Install clang-format on linux/mac if its not installed already - if not utils.is_command_installed('clang-format'): - if utils.is_linux_os(): - # sudo apt install clang-format - utils.run_command(['apt', 'install', '-y','clang-format'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'clang-format']) - - # Install required python dependencies. - # On Catalina, python2 in installed as default python. - # Example command: - # python3 -m pip install -r external/pip_requirements.txt --user - utils.run_command( - ['python3' if utils.is_command_installed('python3') else 'python', '-m', - 'pip', 'install', '-r', 'external/pip_requirements.txt', '--user'] ) + args = parse_cmdline_args() + if not args.running_only: + # Install protobuf on linux/mac if its not installed already + if not utils.is_command_installed('protoc'): + if utils.is_linux_os(): + # sudo apt install protobuf-compiler + utils.run_command(['apt', 'install', '-y','protobuf-compiler'], as_root=True) + elif utils.is_mac_os(): + # brew install protobuf + utils.run_command(['brew', 'install', 'protobuf']) + + # Install go on linux/mac if its not installed already + if not utils.is_command_installed('go'): + if utils.is_linux_os(): + # sudo apt install -y golang + utils.run_command(['apt', 'install', '-y','golang'], as_root=True) + elif utils.is_mac_os(): + # brew install protobuf + utils.run_command(['brew', 'install', 'go']) + + # Install openssl on linux/mac if its not installed already + if not utils.is_command_installed('go'): + if utils.is_linux_os(): + # sudo apt install -y openssl + utils.run_command(['apt', 'install', '-y','openssl'], as_root=True) + elif utils.is_mac_os(): + # brew install protobuf + utils.run_command(['brew', 'install', 'openssl']) + + # Install ccache on linux/mac if its not installed already + if not utils.is_command_installed('ccache'): + if utils.is_linux_os(): + # sudo apt install ccache + utils.run_command(['apt', 'install', '-y', 'ccache'], as_root=True) + elif utils.is_mac_os(): + # brew install ccache + utils.run_command(['brew', 'install', 'ccache']) + + # Install clang-format on linux/mac if its not installed already + if not utils.is_command_installed('clang-format'): + if utils.is_linux_os(): + # sudo apt install clang-format + utils.run_command(['apt', 'install', '-y','clang-format'], as_root=True) + elif utils.is_mac_os(): + # brew install protobuf + utils.run_command(['brew', 'install', 'clang-format']) + + # Install required python dependencies. + # On Catalina, python2 in installed as default python. + # Example command: + # python3 -m pip install -r external/pip_requirements.txt --user + utils.run_command( + ['python3' if utils.is_command_installed('python3') else 'python', '-m', + 'pip', 'install', '-r', 'external/pip_requirements.txt', '--user'] ) + + if args.arch == 'x86': + utils.install_x86_support_libraries(args.gha_build) + +def parse_cmdline_args(): + parser = argparse.ArgumentParser(description='Install prerequisites for building cpp sdk') + parser.add_argument('--arch', default=None, help='Install support libraries to build a specific architecture (currently supported: x86)') + parser.add_argument('--running_only', action='store_true', help='Only install prerequisites for running, not for building') + parser.add_argument('--gha_build', action='store_true', default=None, help='Set to true when building on GitHub, changing some prerequisite installation behavior.') + args = parser.parse_args() + return args if __name__ == '__main__': main() From 34b87f9219799945d1d71579a891d87e3dda60ed Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 12:09:42 -0700 Subject: [PATCH 5/9] Fix installation of openssl --- scripts/gha/install_prereqs_desktop.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gha/install_prereqs_desktop.py b/scripts/gha/install_prereqs_desktop.py index 0cf4f54a5..c6bd8283f 100644 --- a/scripts/gha/install_prereqs_desktop.py +++ b/scripts/gha/install_prereqs_desktop.py @@ -51,16 +51,16 @@ def main(): # sudo apt install -y golang utils.run_command(['apt', 'install', '-y','golang'], as_root=True) elif utils.is_mac_os(): - # brew install protobuf + # brew install go utils.run_command(['brew', 'install', 'go']) # Install openssl on linux/mac if its not installed already - if not utils.is_command_installed('go'): + if not utils.is_command_installed('openssl'): if utils.is_linux_os(): # sudo apt install -y openssl utils.run_command(['apt', 'install', '-y','openssl'], as_root=True) elif utils.is_mac_os(): - # brew install protobuf + # brew install openssl utils.run_command(['brew', 'install', 'openssl']) # Install ccache on linux/mac if its not installed already From 23f9771f3ec3a270ef49d15200e50792f70367c2 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 12:22:51 -0700 Subject: [PATCH 6/9] Add option to prereqs script to skip openssl, for boringssl builds. --- .github/workflows/cpp-packaging.yml | 4 ++-- .github/workflows/integration_tests.yml | 2 +- scripts/gha/install_prereqs_desktop.py | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index de845ba02..9cc495e8a 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -414,7 +414,7 @@ jobs: timeout_minutes: 15 max_attempts: 3 command: | - python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.architecture }}' + python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.architecture }}' --ssl boringssl - name: Export verbose flag shell: bash @@ -542,7 +542,7 @@ jobs: - name: Install prerequisites run: | cd sdk-src - python scripts/gha/install_prereqs_desktop.py + python scripts/gha/install_prereqs_desktop.py --ssl boringssl cd .. - name: postprocess and package built SDK diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 787031e30..3c0923e6f 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -277,7 +277,7 @@ jobs: timeout_minutes: 2 max_attempts: 3 command: | - python scripts/gha/install_prereqs_desktop.py + python scripts/gha/install_prereqs_desktop.py --ssl '${{ matrix.ssl_variant }}' pip install -r scripts/gha/requirements.txt python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" - name: Install OpenSSL (Windows) diff --git a/scripts/gha/install_prereqs_desktop.py b/scripts/gha/install_prereqs_desktop.py index c6bd8283f..b32c82c44 100644 --- a/scripts/gha/install_prereqs_desktop.py +++ b/scripts/gha/install_prereqs_desktop.py @@ -55,7 +55,7 @@ def main(): utils.run_command(['brew', 'install', 'go']) # Install openssl on linux/mac if its not installed already - if not utils.is_command_installed('openssl'): + if not utils.is_command_installed('openssl') and args.ssl == 'openssl': if utils.is_linux_os(): # sudo apt install -y openssl utils.run_command(['apt', 'install', '-y','openssl'], as_root=True) @@ -96,7 +96,8 @@ def parse_cmdline_args(): parser = argparse.ArgumentParser(description='Install prerequisites for building cpp sdk') parser.add_argument('--arch', default=None, help='Install support libraries to build a specific architecture (currently supported: x86)') parser.add_argument('--running_only', action='store_true', help='Only install prerequisites for running, not for building') - parser.add_argument('--gha_build', action='store_true', default=None, help='Set to true when building on GitHub, changing some prerequisite installation behavior.') + parser.add_argument('--gha_build', action='store_true', default=None, help='Set this option when building on GitHub, changing some prerequisite installation behavior') + parser.add_argument('--ssl', default='openssl', help='Which SSL is this build using (supported: openssl, boringssl)') args = parser.parse_args() return args From b114ed239fb159539591ca2dad02a50fe4b41aae Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 12:25:06 -0700 Subject: [PATCH 7/9] Fix conditional --- scripts/gha/install_prereqs_desktop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gha/install_prereqs_desktop.py b/scripts/gha/install_prereqs_desktop.py index b32c82c44..49452be9b 100644 --- a/scripts/gha/install_prereqs_desktop.py +++ b/scripts/gha/install_prereqs_desktop.py @@ -55,7 +55,7 @@ def main(): utils.run_command(['brew', 'install', 'go']) # Install openssl on linux/mac if its not installed already - if not utils.is_command_installed('openssl') and args.ssl == 'openssl': + if args.ssl == 'openssl' and not utils.is_command_installed('openssl'): if utils.is_linux_os(): # sudo apt install -y openssl utils.run_command(['apt', 'install', '-y','openssl'], as_root=True) From 21aba056a576c8d906f34e480349bf6c583d5f37 Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 12:39:40 -0700 Subject: [PATCH 8/9] Use 'with' syntax for file i/o --- scripts/gha/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gha/utils.py b/scripts/gha/utils.py index 7f964e414..3e4b8c18b 100644 --- a/scripts/gha/utils.py +++ b/scripts/gha/utils.py @@ -241,9 +241,9 @@ def install_x86_support_libraries(gha_build=False): packages = ['--allow-downgrades'] + packages + ['libpcre2-8-0=10.34-7'] # First check if these packages exist on the machine already - devnull = open(os.devnull, "w") - process = subprocess.run(["dpkg", "-s"] + packages, stdout=devnull, stderr=subprocess.STDOUT) - devnull.close() + with open(os.devnull, "w") as devnull: + process = subprocess.run(["dpkg", "-s"] + packages, stdout=devnull, stderr=subprocess.STDOUT) + if process.returncode != 0: # This implies not all of the required packages are already installed on user's machine # Install them. From 9613ee175be069ff9df27e2641c4622d43b9ae7a Mon Sep 17 00:00:00 2001 From: Jon Simantov Date: Mon, 9 May 2022 12:54:24 -0700 Subject: [PATCH 9/9] Fix spacing --- scripts/gha/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gha/utils.py b/scripts/gha/utils.py index 3e4b8c18b..b1ef778bc 100644 --- a/scripts/gha/utils.py +++ b/scripts/gha/utils.py @@ -259,4 +259,4 @@ def install_x86_support_libraries(gha_build=False): # be removed as well.) # Note: "-f" = "fix" - let apt do what it needs to do to fix dependencies. run_command(['apt', 'install', '-f', '-V', '-y', 'libsecret-1-dev'], - as_root=True, check=True) + as_root=True, check=True)