diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index 165145a57..9cc495e8a 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -408,9 +408,13 @@ jobs: # binutils, used by older version of homebrew for hosting packages. brew update - - name: Install prerequisites - run: | - python scripts/gha/install_prereqs_desktop.py + - name: Install Desktop SDK prerequisites + uses: nick-invision/retry@v2 + with: + timeout_minutes: 15 + max_attempts: 3 + command: | + python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.architecture }}' --ssl boringssl - name: Export verbose flag shell: bash @@ -538,7 +542,7 @@ jobs: - name: Install prerequisites run: | cd sdk-src - python scripts/gha/install_prereqs_desktop.py + python scripts/gha/install_prereqs_desktop.py --ssl boringssl cd .. - name: postprocess and package built SDK diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 9891e3196..11176ae44 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -176,9 +176,13 @@ jobs: # binutils, used by older version of homebrew for hosting packages. brew update - - name: Install prerequisites - run: | - python scripts/gha/install_prereqs_desktop.py + - name: Install Desktop SDK prerequisites + uses: nick-invision/retry@v2 + with: + timeout_minutes: 15 + max_attempts: 3 + command: | + python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.architecture }}' - name: Build SDK shell: bash diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index e304ef38e..5b8e9a6b2 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -278,7 +278,7 @@ jobs: timeout_minutes: 2 max_attempts: 3 command: | - python scripts/gha/install_prereqs_desktop.py + python scripts/gha/install_prereqs_desktop.py --ssl '${{ matrix.ssl_variant }}' pip install -r scripts/gha/requirements.txt python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" - name: Install OpenSSL (Windows) diff --git a/scripts/gha/build_desktop.py b/scripts/gha/build_desktop.py index 1978ff0ba..0a0090f81 100644 --- a/scripts/gha/build_desktop.py +++ b/scripts/gha/build_desktop.py @@ -281,7 +281,7 @@ def main(): # To build x86 on x86_64 linux hosts, we also need x86 support libraries if args.arch == 'x86' and utils.is_linux_os(): - install_x86_support_libraries(args.gha_build) + utils.install_x86_support_libraries(args.gha_build) # Install C++ dependencies using vcpkg if not args.disable_vcpkg: diff --git a/scripts/gha/install_prereqs_desktop.py b/scripts/gha/install_prereqs_desktop.py index a8511c493..49452be9b 100644 --- a/scripts/gha/install_prereqs_desktop.py +++ b/scripts/gha/install_prereqs_desktop.py @@ -29,62 +29,77 @@ """ +import argparse import utils def main(): - # Install protobuf on linux/mac if its not installed already - if not utils.is_command_installed('protoc'): - if utils.is_linux_os(): - # sudo apt install protobuf-compiler - utils.run_command(['apt', 'install', '-y','protobuf-compiler'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'protobuf']) - - # Install go on linux/mac if its not installed already - if not utils.is_command_installed('go'): - if utils.is_linux_os(): - # sudo apt install -y golang - utils.run_command(['apt', 'install', '-y','golang'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'go']) - - # Install openssl on linux/mac if its not installed already - if not utils.is_command_installed('go'): - if utils.is_linux_os(): - # sudo apt install -y openssl - utils.run_command(['apt', 'install', '-y','openssl'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'openssl']) - - # Install ccache on linux/mac if its not installed already - if not utils.is_command_installed('ccache'): - if utils.is_linux_os(): - # sudo apt install ccache - utils.run_command(['apt', 'install', '-y', 'ccache'], as_root=True) - elif utils.is_mac_os(): - # brew install ccache - utils.run_command(['brew', 'install', 'ccache']) - - # Install clang-format on linux/mac if its not installed already - if not utils.is_command_installed('clang-format'): - if utils.is_linux_os(): - # sudo apt install clang-format - utils.run_command(['apt', 'install', '-y','clang-format'], as_root=True) - elif utils.is_mac_os(): - # brew install protobuf - utils.run_command(['brew', 'install', 'clang-format']) - - # Install required python dependencies. - # On Catalina, python2 in installed as default python. - # Example command: - # python3 -m pip install -r external/pip_requirements.txt --user - utils.run_command( - ['python3' if utils.is_command_installed('python3') else 'python', '-m', - 'pip', 'install', '-r', 'external/pip_requirements.txt', '--user'] ) + args = parse_cmdline_args() + if not args.running_only: + # Install protobuf on linux/mac if its not installed already + if not utils.is_command_installed('protoc'): + if utils.is_linux_os(): + # sudo apt install protobuf-compiler + utils.run_command(['apt', 'install', '-y','protobuf-compiler'], as_root=True) + elif utils.is_mac_os(): + # brew install protobuf + utils.run_command(['brew', 'install', 'protobuf']) + + # Install go on linux/mac if its not installed already + if not utils.is_command_installed('go'): + if utils.is_linux_os(): + # sudo apt install -y golang + utils.run_command(['apt', 'install', '-y','golang'], as_root=True) + elif utils.is_mac_os(): + # brew install go + utils.run_command(['brew', 'install', 'go']) + + # Install openssl on linux/mac if its not installed already + if args.ssl == 'openssl' and not utils.is_command_installed('openssl'): + if utils.is_linux_os(): + # sudo apt install -y openssl + utils.run_command(['apt', 'install', '-y','openssl'], as_root=True) + elif utils.is_mac_os(): + # brew install openssl + utils.run_command(['brew', 'install', 'openssl']) + + # Install ccache on linux/mac if its not installed already + if not utils.is_command_installed('ccache'): + if utils.is_linux_os(): + # sudo apt install ccache + utils.run_command(['apt', 'install', '-y', 'ccache'], as_root=True) + elif utils.is_mac_os(): + # brew install ccache + utils.run_command(['brew', 'install', 'ccache']) + + # Install clang-format on linux/mac if its not installed already + if not utils.is_command_installed('clang-format'): + if utils.is_linux_os(): + # sudo apt install clang-format + utils.run_command(['apt', 'install', '-y','clang-format'], as_root=True) + elif utils.is_mac_os(): + # brew install protobuf + utils.run_command(['brew', 'install', 'clang-format']) + + # Install required python dependencies. + # On Catalina, python2 in installed as default python. + # Example command: + # python3 -m pip install -r external/pip_requirements.txt --user + utils.run_command( + ['python3' if utils.is_command_installed('python3') else 'python', '-m', + 'pip', 'install', '-r', 'external/pip_requirements.txt', '--user'] ) + + if args.arch == 'x86': + utils.install_x86_support_libraries(args.gha_build) + +def parse_cmdline_args(): + parser = argparse.ArgumentParser(description='Install prerequisites for building cpp sdk') + parser.add_argument('--arch', default=None, help='Install support libraries to build a specific architecture (currently supported: x86)') + parser.add_argument('--running_only', action='store_true', help='Only install prerequisites for running, not for building') + parser.add_argument('--gha_build', action='store_true', default=None, help='Set this option when building on GitHub, changing some prerequisite installation behavior') + parser.add_argument('--ssl', default='openssl', help='Which SSL is this build using (supported: openssl, boringssl)') + args = parser.parse_args() + return args if __name__ == '__main__': main() diff --git a/scripts/gha/utils.py b/scripts/gha/utils.py index 9201fed01..b1ef778bc 100644 --- a/scripts/gha/utils.py +++ b/scripts/gha/utils.py @@ -218,3 +218,45 @@ def clean_vcpkg_temp_data(): for directory_to_remove in directories_to_remove: abspath = os.path.join(vcpkg_root_dir_path, directory_to_remove) delete_directory(abspath) + + +def install_x86_support_libraries(gha_build=False): + """Install support libraries needed to build x86 on x86_64 hosts. + + Args: + gha_build: Pass in True if running on a GitHub runner; this will activate + workarounds that might be undesirable on a personal system (e.g. + downgrading Ubuntu packages). + """ + if is_linux_os(): + packages = ['gcc-multilib', 'g++-multilib', 'libglib2.0-dev:i386', + 'libsecret-1-dev:i386', 'libpthread-stubs0-dev:i386', + 'libssl-dev:i386'] + if gha_build: + # Workaround for GitHub runners, which have an incompatibility between the + # 64-bit and 32-bit versions of the Ubuntu package libpcre2-8-0. Downgrade + # the installed 64-bit version of the library to get around this issue. + # This will presumably be fixed in a future Ubuntu update. (If you remove + # it, remove the workaround further down this function as well.) + packages = ['--allow-downgrades'] + packages + ['libpcre2-8-0=10.34-7'] + + # First check if these packages exist on the machine already + with open(os.devnull, "w") as devnull: + process = subprocess.run(["dpkg", "-s"] + packages, stdout=devnull, stderr=subprocess.STDOUT) + + if process.returncode != 0: + # This implies not all of the required packages are already installed on user's machine + # Install them. + run_command(['dpkg', '--add-architecture', 'i386'], as_root=True, check=True) + run_command(['apt', 'update'], as_root=True, check=True) + run_command(['apt', 'install', '-V', '-y'] + packages, as_root=True, check=True) + + if gha_build: + # One more workaround: downgrading libpcre2-8-0 above may have uninstalled + # libsecret, which is required for the Linux build. Force it to be + # reinstalled, but do it as a separate command to ensure that held + # packages aren't modified. (Once the workaround above is removed, this can + # be removed as well.) + # Note: "-f" = "fix" - let apt do what it needs to do to fix dependencies. + run_command(['apt', 'install', '-f', '-V', '-y', 'libsecret-1-dev'], + as_root=True, check=True)